clang 24.0.0git
ARM.cpp
Go to the documentation of this file.
1//===---------- ARM.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfo.h"
14#include "CGBuiltin.h"
15#include "CGDebugInfo.h"
16#include "TargetInfo.h"
17#include "clang/Basic/AArch64CodeGenUtils.h"
18#include "clang/Basic/TargetBuiltins.h"
19#include "llvm/IR/InlineAsm.h"
20#include "llvm/IR/IntrinsicsAArch64.h"
21#include "llvm/IR/IntrinsicsARM.h"
22#include "llvm/IR/IntrinsicsBPF.h"
23#include "llvm/Support/AArch64MemoryHints.h"
24#include "llvm/TargetParser/AArch64TargetParser.h"
25
26#include <numeric>
27
28using namespace clang;
29using namespace CodeGen;
30using namespace llvm;
31using namespace clang::aarch64;
32
33static std::optional<CodeGenFunction::MSVCIntrin>
34translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
35 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
36 switch (BuiltinID) {
37 default:
38 return std::nullopt;
39 case clang::AArch64::BI_BitScanForward:
40 case clang::AArch64::BI_BitScanForward64:
41 return MSVCIntrin::_BitScanForward;
42 case clang::AArch64::BI_BitScanReverse:
43 case clang::AArch64::BI_BitScanReverse64:
44 return MSVCIntrin::_BitScanReverse;
45 case clang::AArch64::BI_InterlockedAnd64:
46 return MSVCIntrin::_InterlockedAnd;
47 case clang::AArch64::BI_InterlockedExchange64:
48 return MSVCIntrin::_InterlockedExchange;
49 case clang::AArch64::BI_InterlockedExchangeAdd64:
50 return MSVCIntrin::_InterlockedExchangeAdd;
51 case clang::AArch64::BI_InterlockedExchangeSub64:
52 return MSVCIntrin::_InterlockedExchangeSub;
53 case clang::AArch64::BI_InterlockedOr64:
54 return MSVCIntrin::_InterlockedOr;
55 case clang::AArch64::BI_InterlockedXor64:
56 return MSVCIntrin::_InterlockedXor;
57 case clang::AArch64::BI_InterlockedDecrement64:
58 return MSVCIntrin::_InterlockedDecrement;
59 case clang::AArch64::BI_InterlockedIncrement64:
60 return MSVCIntrin::_InterlockedIncrement;
61 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
62 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
63 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
64 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
65 return MSVCIntrin::_InterlockedExchangeAdd_acq;
66 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
67 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
68 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
69 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
70 return MSVCIntrin::_InterlockedExchangeAdd_rel;
71 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
72 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
73 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
74 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
75 return MSVCIntrin::_InterlockedExchangeAdd_nf;
76 case clang::AArch64::BI_InterlockedExchange8_acq:
77 case clang::AArch64::BI_InterlockedExchange16_acq:
78 case clang::AArch64::BI_InterlockedExchange_acq:
79 case clang::AArch64::BI_InterlockedExchange64_acq:
80 case clang::AArch64::BI_InterlockedExchangePointer_acq:
81 return MSVCIntrin::_InterlockedExchange_acq;
82 case clang::AArch64::BI_InterlockedExchange8_rel:
83 case clang::AArch64::BI_InterlockedExchange16_rel:
84 case clang::AArch64::BI_InterlockedExchange_rel:
85 case clang::AArch64::BI_InterlockedExchange64_rel:
86 case clang::AArch64::BI_InterlockedExchangePointer_rel:
87 return MSVCIntrin::_InterlockedExchange_rel;
88 case clang::AArch64::BI_InterlockedExchange8_nf:
89 case clang::AArch64::BI_InterlockedExchange16_nf:
90 case clang::AArch64::BI_InterlockedExchange_nf:
91 case clang::AArch64::BI_InterlockedExchange64_nf:
92 case clang::AArch64::BI_InterlockedExchangePointer_nf:
93 return MSVCIntrin::_InterlockedExchange_nf;
94 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
95 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
96 case clang::AArch64::BI_InterlockedCompareExchange_acq:
97 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
98 case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
99 return MSVCIntrin::_InterlockedCompareExchange_acq;
100 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
101 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
102 case clang::AArch64::BI_InterlockedCompareExchange_rel:
103 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
104 case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
105 return MSVCIntrin::_InterlockedCompareExchange_rel;
106 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
107 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
108 case clang::AArch64::BI_InterlockedCompareExchange_nf:
109 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
110 return MSVCIntrin::_InterlockedCompareExchange_nf;
111 case clang::AArch64::BI_InterlockedCompareExchange128:
112 return MSVCIntrin::_InterlockedCompareExchange128;
113 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
114 return MSVCIntrin::_InterlockedCompareExchange128_acq;
115 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
116 return MSVCIntrin::_InterlockedCompareExchange128_nf;
117 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
118 return MSVCIntrin::_InterlockedCompareExchange128_rel;
119 case clang::AArch64::BI_InterlockedOr8_acq:
120 case clang::AArch64::BI_InterlockedOr16_acq:
121 case clang::AArch64::BI_InterlockedOr_acq:
122 case clang::AArch64::BI_InterlockedOr64_acq:
123 return MSVCIntrin::_InterlockedOr_acq;
124 case clang::AArch64::BI_InterlockedOr8_rel:
125 case clang::AArch64::BI_InterlockedOr16_rel:
126 case clang::AArch64::BI_InterlockedOr_rel:
127 case clang::AArch64::BI_InterlockedOr64_rel:
128 return MSVCIntrin::_InterlockedOr_rel;
129 case clang::AArch64::BI_InterlockedOr8_nf:
130 case clang::AArch64::BI_InterlockedOr16_nf:
131 case clang::AArch64::BI_InterlockedOr_nf:
132 case clang::AArch64::BI_InterlockedOr64_nf:
133 return MSVCIntrin::_InterlockedOr_nf;
134 case clang::AArch64::BI_InterlockedXor8_acq:
135 case clang::AArch64::BI_InterlockedXor16_acq:
136 case clang::AArch64::BI_InterlockedXor_acq:
137 case clang::AArch64::BI_InterlockedXor64_acq:
138 return MSVCIntrin::_InterlockedXor_acq;
139 case clang::AArch64::BI_InterlockedXor8_rel:
140 case clang::AArch64::BI_InterlockedXor16_rel:
141 case clang::AArch64::BI_InterlockedXor_rel:
142 case clang::AArch64::BI_InterlockedXor64_rel:
143 return MSVCIntrin::_InterlockedXor_rel;
144 case clang::AArch64::BI_InterlockedXor8_nf:
145 case clang::AArch64::BI_InterlockedXor16_nf:
146 case clang::AArch64::BI_InterlockedXor_nf:
147 case clang::AArch64::BI_InterlockedXor64_nf:
148 return MSVCIntrin::_InterlockedXor_nf;
149 case clang::AArch64::BI_InterlockedAnd8_acq:
150 case clang::AArch64::BI_InterlockedAnd16_acq:
151 case clang::AArch64::BI_InterlockedAnd_acq:
152 case clang::AArch64::BI_InterlockedAnd64_acq:
153 return MSVCIntrin::_InterlockedAnd_acq;
154 case clang::AArch64::BI_InterlockedAnd8_rel:
155 case clang::AArch64::BI_InterlockedAnd16_rel:
156 case clang::AArch64::BI_InterlockedAnd_rel:
157 case clang::AArch64::BI_InterlockedAnd64_rel:
158 return MSVCIntrin::_InterlockedAnd_rel;
159 case clang::AArch64::BI_InterlockedAnd8_nf:
160 case clang::AArch64::BI_InterlockedAnd16_nf:
161 case clang::AArch64::BI_InterlockedAnd_nf:
162 case clang::AArch64::BI_InterlockedAnd64_nf:
163 return MSVCIntrin::_InterlockedAnd_nf;
164 case clang::AArch64::BI_InterlockedIncrement16_acq:
165 case clang::AArch64::BI_InterlockedIncrement_acq:
166 case clang::AArch64::BI_InterlockedIncrement64_acq:
167 return MSVCIntrin::_InterlockedIncrement_acq;
168 case clang::AArch64::BI_InterlockedIncrement16_rel:
169 case clang::AArch64::BI_InterlockedIncrement_rel:
170 case clang::AArch64::BI_InterlockedIncrement64_rel:
171 return MSVCIntrin::_InterlockedIncrement_rel;
172 case clang::AArch64::BI_InterlockedIncrement16_nf:
173 case clang::AArch64::BI_InterlockedIncrement_nf:
174 case clang::AArch64::BI_InterlockedIncrement64_nf:
175 return MSVCIntrin::_InterlockedIncrement_nf;
176 case clang::AArch64::BI_InterlockedDecrement16_acq:
177 case clang::AArch64::BI_InterlockedDecrement_acq:
178 case clang::AArch64::BI_InterlockedDecrement64_acq:
179 return MSVCIntrin::_InterlockedDecrement_acq;
180 case clang::AArch64::BI_InterlockedDecrement16_rel:
181 case clang::AArch64::BI_InterlockedDecrement_rel:
182 case clang::AArch64::BI_InterlockedDecrement64_rel:
183 return MSVCIntrin::_InterlockedDecrement_rel;
184 case clang::AArch64::BI_InterlockedDecrement16_nf:
185 case clang::AArch64::BI_InterlockedDecrement_nf:
186 case clang::AArch64::BI_InterlockedDecrement64_nf:
187 return MSVCIntrin::_InterlockedDecrement_nf;
188 }
189 llvm_unreachable("must return from switch");
190}
191
192static std::optional<CodeGenFunction::MSVCIntrin>
193translateArmToMsvcIntrin(unsigned BuiltinID) {
194 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
195 switch (BuiltinID) {
196 default:
197 return std::nullopt;
198 case clang::ARM::BI_BitScanForward:
199 case clang::ARM::BI_BitScanForward64:
200 return MSVCIntrin::_BitScanForward;
201 case clang::ARM::BI_BitScanReverse:
202 case clang::ARM::BI_BitScanReverse64:
203 return MSVCIntrin::_BitScanReverse;
204 case clang::ARM::BI_InterlockedAnd64:
205 return MSVCIntrin::_InterlockedAnd;
206 case clang::ARM::BI_InterlockedExchange64:
207 return MSVCIntrin::_InterlockedExchange;
208 case clang::ARM::BI_InterlockedExchangeAdd64:
209 return MSVCIntrin::_InterlockedExchangeAdd;
210 case clang::ARM::BI_InterlockedExchangeSub64:
211 return MSVCIntrin::_InterlockedExchangeSub;
212 case clang::ARM::BI_InterlockedOr64:
213 return MSVCIntrin::_InterlockedOr;
214 case clang::ARM::BI_InterlockedXor64:
215 return MSVCIntrin::_InterlockedXor;
216 case clang::ARM::BI_InterlockedDecrement64:
217 return MSVCIntrin::_InterlockedDecrement;
218 case clang::ARM::BI_InterlockedIncrement64:
219 return MSVCIntrin::_InterlockedIncrement;
220 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
221 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
222 case clang::ARM::BI_InterlockedExchangeAdd_acq:
223 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
224 return MSVCIntrin::_InterlockedExchangeAdd_acq;
225 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
226 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
227 case clang::ARM::BI_InterlockedExchangeAdd_rel:
228 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
229 return MSVCIntrin::_InterlockedExchangeAdd_rel;
230 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
231 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
232 case clang::ARM::BI_InterlockedExchangeAdd_nf:
233 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
234 return MSVCIntrin::_InterlockedExchangeAdd_nf;
235 case clang::ARM::BI_InterlockedExchange8_acq:
236 case clang::ARM::BI_InterlockedExchange16_acq:
237 case clang::ARM::BI_InterlockedExchange_acq:
238 case clang::ARM::BI_InterlockedExchange64_acq:
239 case clang::ARM::BI_InterlockedExchangePointer_acq:
240 return MSVCIntrin::_InterlockedExchange_acq;
241 case clang::ARM::BI_InterlockedExchange8_rel:
242 case clang::ARM::BI_InterlockedExchange16_rel:
243 case clang::ARM::BI_InterlockedExchange_rel:
244 case clang::ARM::BI_InterlockedExchange64_rel:
245 case clang::ARM::BI_InterlockedExchangePointer_rel:
246 return MSVCIntrin::_InterlockedExchange_rel;
247 case clang::ARM::BI_InterlockedExchange8_nf:
248 case clang::ARM::BI_InterlockedExchange16_nf:
249 case clang::ARM::BI_InterlockedExchange_nf:
250 case clang::ARM::BI_InterlockedExchange64_nf:
251 case clang::ARM::BI_InterlockedExchangePointer_nf:
252 return MSVCIntrin::_InterlockedExchange_nf;
253 case clang::ARM::BI_InterlockedCompareExchange8_acq:
254 case clang::ARM::BI_InterlockedCompareExchange16_acq:
255 case clang::ARM::BI_InterlockedCompareExchange_acq:
256 case clang::ARM::BI_InterlockedCompareExchange64_acq:
257 case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
258 return MSVCIntrin::_InterlockedCompareExchange_acq;
259 case clang::ARM::BI_InterlockedCompareExchange8_rel:
260 case clang::ARM::BI_InterlockedCompareExchange16_rel:
261 case clang::ARM::BI_InterlockedCompareExchange_rel:
262 case clang::ARM::BI_InterlockedCompareExchange64_rel:
263 case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
264 return MSVCIntrin::_InterlockedCompareExchange_rel;
265 case clang::ARM::BI_InterlockedCompareExchange8_nf:
266 case clang::ARM::BI_InterlockedCompareExchange16_nf:
267 case clang::ARM::BI_InterlockedCompareExchange_nf:
268 case clang::ARM::BI_InterlockedCompareExchange64_nf:
269 return MSVCIntrin::_InterlockedCompareExchange_nf;
270 case clang::ARM::BI_InterlockedOr8_acq:
271 case clang::ARM::BI_InterlockedOr16_acq:
272 case clang::ARM::BI_InterlockedOr_acq:
273 case clang::ARM::BI_InterlockedOr64_acq:
274 return MSVCIntrin::_InterlockedOr_acq;
275 case clang::ARM::BI_InterlockedOr8_rel:
276 case clang::ARM::BI_InterlockedOr16_rel:
277 case clang::ARM::BI_InterlockedOr_rel:
278 case clang::ARM::BI_InterlockedOr64_rel:
279 return MSVCIntrin::_InterlockedOr_rel;
280 case clang::ARM::BI_InterlockedOr8_nf:
281 case clang::ARM::BI_InterlockedOr16_nf:
282 case clang::ARM::BI_InterlockedOr_nf:
283 case clang::ARM::BI_InterlockedOr64_nf:
284 return MSVCIntrin::_InterlockedOr_nf;
285 case clang::ARM::BI_InterlockedXor8_acq:
286 case clang::ARM::BI_InterlockedXor16_acq:
287 case clang::ARM::BI_InterlockedXor_acq:
288 case clang::ARM::BI_InterlockedXor64_acq:
289 return MSVCIntrin::_InterlockedXor_acq;
290 case clang::ARM::BI_InterlockedXor8_rel:
291 case clang::ARM::BI_InterlockedXor16_rel:
292 case clang::ARM::BI_InterlockedXor_rel:
293 case clang::ARM::BI_InterlockedXor64_rel:
294 return MSVCIntrin::_InterlockedXor_rel;
295 case clang::ARM::BI_InterlockedXor8_nf:
296 case clang::ARM::BI_InterlockedXor16_nf:
297 case clang::ARM::BI_InterlockedXor_nf:
298 case clang::ARM::BI_InterlockedXor64_nf:
299 return MSVCIntrin::_InterlockedXor_nf;
300 case clang::ARM::BI_InterlockedAnd8_acq:
301 case clang::ARM::BI_InterlockedAnd16_acq:
302 case clang::ARM::BI_InterlockedAnd_acq:
303 case clang::ARM::BI_InterlockedAnd64_acq:
304 return MSVCIntrin::_InterlockedAnd_acq;
305 case clang::ARM::BI_InterlockedAnd8_rel:
306 case clang::ARM::BI_InterlockedAnd16_rel:
307 case clang::ARM::BI_InterlockedAnd_rel:
308 case clang::ARM::BI_InterlockedAnd64_rel:
309 return MSVCIntrin::_InterlockedAnd_rel;
310 case clang::ARM::BI_InterlockedAnd8_nf:
311 case clang::ARM::BI_InterlockedAnd16_nf:
312 case clang::ARM::BI_InterlockedAnd_nf:
313 case clang::ARM::BI_InterlockedAnd64_nf:
314 return MSVCIntrin::_InterlockedAnd_nf;
315 case clang::ARM::BI_InterlockedIncrement16_acq:
316 case clang::ARM::BI_InterlockedIncrement_acq:
317 case clang::ARM::BI_InterlockedIncrement64_acq:
318 return MSVCIntrin::_InterlockedIncrement_acq;
319 case clang::ARM::BI_InterlockedIncrement16_rel:
320 case clang::ARM::BI_InterlockedIncrement_rel:
321 case clang::ARM::BI_InterlockedIncrement64_rel:
322 return MSVCIntrin::_InterlockedIncrement_rel;
323 case clang::ARM::BI_InterlockedIncrement16_nf:
324 case clang::ARM::BI_InterlockedIncrement_nf:
325 case clang::ARM::BI_InterlockedIncrement64_nf:
326 return MSVCIntrin::_InterlockedIncrement_nf;
327 case clang::ARM::BI_InterlockedDecrement16_acq:
328 case clang::ARM::BI_InterlockedDecrement_acq:
329 case clang::ARM::BI_InterlockedDecrement64_acq:
330 return MSVCIntrin::_InterlockedDecrement_acq;
331 case clang::ARM::BI_InterlockedDecrement16_rel:
332 case clang::ARM::BI_InterlockedDecrement_rel:
333 case clang::ARM::BI_InterlockedDecrement64_rel:
334 return MSVCIntrin::_InterlockedDecrement_rel;
335 case clang::ARM::BI_InterlockedDecrement16_nf:
336 case clang::ARM::BI_InterlockedDecrement_nf:
337 case clang::ARM::BI_InterlockedDecrement64_nf:
338 return MSVCIntrin::_InterlockedDecrement_nf;
339 }
340 llvm_unreachable("must return from switch");
341}
342
343// Emit an intrinsic where all operands are of the same type as the result.
344// Depending on mode, this may be a constrained floating-point intrinsic.
345static Value *emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
346 unsigned IntrinsicID,
347 unsigned ConstrainedIntrinsicID,
348 llvm::Type *Ty,
349 ArrayRef<Value *> Args) {
350 Function *F;
351 if (CGF.Builder.getIsFPConstrained())
352 F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Ty);
353 else
354 F = CGF.CGM.getIntrinsic(IntrinsicID, Ty);
355
356 if (CGF.Builder.getIsFPConstrained())
357 return CGF.Builder.CreateConstrainedFPCall(F, Args);
358
359 return CGF.Builder.CreateCall(F, Args);
360}
361
362static llvm::FixedVectorType *GetNeonType(CodeGenFunction *CGF,
363 NeonTypeFlags TypeFlags,
364 bool HasFastHalfType = true,
365 bool V1Ty = false,
366 bool AllowBFloatArgsAndRet = true) {
367 int IsQuad = TypeFlags.isQuad();
368 switch (TypeFlags.getEltType()) {
369 case NeonTypeFlags::Int8:
370 case NeonTypeFlags::Poly8:
371 case NeonTypeFlags::MFloat8:
372 return llvm::FixedVectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad));
373 case NeonTypeFlags::Int16:
374 case NeonTypeFlags::Poly16:
375 return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
376 case NeonTypeFlags::BFloat16:
377 if (AllowBFloatArgsAndRet)
378 return llvm::FixedVectorType::get(CGF->BFloatTy, V1Ty ? 1 : (4 << IsQuad));
379 return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
380 case NeonTypeFlags::Float16:
381 if (HasFastHalfType)
382 return llvm::FixedVectorType::get(CGF->HalfTy, V1Ty ? 1 : (4 << IsQuad));
383 return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
384 case NeonTypeFlags::Int32:
385 return llvm::FixedVectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad));
386 case NeonTypeFlags::Int64:
387 case NeonTypeFlags::Poly64:
388 return llvm::FixedVectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad));
389 case NeonTypeFlags::Poly128:
390 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
391 // There is a lot of i128 and f128 API missing.
392 // so we use v16i8 to represent poly128 and get pattern matched.
393 return llvm::FixedVectorType::get(CGF->Int8Ty, 16);
394 case NeonTypeFlags::Float32:
395 return llvm::FixedVectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad));
396 case NeonTypeFlags::Float64:
397 return llvm::FixedVectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad));
398 }
399 llvm_unreachable("Unknown vector element type!");
400}
401
402static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF,
403 NeonTypeFlags IntTypeFlags) {
404 int IsQuad = IntTypeFlags.isQuad();
405 switch (IntTypeFlags.getEltType()) {
406 case NeonTypeFlags::Int16:
407 return llvm::FixedVectorType::get(CGF->HalfTy, (4 << IsQuad));
408 case NeonTypeFlags::Int32:
409 return llvm::FixedVectorType::get(CGF->FloatTy, (2 << IsQuad));
410 case NeonTypeFlags::Int64:
411 return llvm::FixedVectorType::get(CGF->DoubleTy, (1 << IsQuad));
412 default:
413 llvm_unreachable("Type can't be converted to floating-point!");
414 }
415}
416
417Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C,
418 const ElementCount &Count) {
419 Value *SV = llvm::ConstantVector::getSplat(Count, C);
420 return Builder.CreateShuffleVector(V, V, SV, "lane");
421}
422
423Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
424 ElementCount EC = cast<llvm::VectorType>(V->getType())->getElementCount();
425 return EmitNeonSplat(V, C, EC);
426}
427
428Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
429 const char *name,
430 unsigned shift, bool rightshift) {
431 unsigned j = 0;
432 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
433 ai != ae; ++ai, ++j) {
434 if (F->isConstrainedFPIntrinsic())
435 if (ai->getType()->isMetadataTy())
436 continue;
437 if (shift > 0 && shift == j)
438 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
439 else
440 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
441 }
442
443 if (F->isConstrainedFPIntrinsic())
444 return Builder.CreateConstrainedFPCall(F, Ops, name);
445 return Builder.CreateCall(F, Ops, name);
446}
447
448Value *CodeGenFunction::EmitFP8NeonCall(unsigned IID,
449 ArrayRef<llvm::Type *> Tys,
450 SmallVectorImpl<Value *> &Ops,
451 const CallExpr *E, const char *name) {
452 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_set_fpmr),
453 Ops.pop_back_val());
454 return EmitNeonCall(CGM.getIntrinsic(IID, Tys), Ops, name);
455}
456
457llvm::Value *CodeGenFunction::EmitFP8NeonFDOTCall(
458 unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy,
459 SmallVectorImpl<llvm::Value *> &Ops, const CallExpr *E, const char *name) {
460
461 const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
462 RetTy->getPrimitiveSizeInBits();
463 llvm::Type *Tys[] = {llvm::FixedVectorType::get(RetTy, ElemCount),
464 Ops[1]->getType()};
465 if (ExtendLaneArg) {
466 auto *VT = llvm::FixedVectorType::get(Int8Ty, 16);
467 Ops[2] = Builder.CreateInsertVector(VT, PoisonValue::get(VT), Ops[2],
468 uint64_t(0));
469 }
470 return EmitFP8NeonCall(IID, Tys, Ops, E, name);
471}
472
473llvm::Value *CodeGenFunction::EmitFP8NeonFMLACall(
474 unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy,
475 SmallVectorImpl<llvm::Value *> &Ops, const CallExpr *E, const char *name) {
476
477 if (ExtendLaneArg) {
478 auto *VT = llvm::FixedVectorType::get(Int8Ty, 16);
479 Ops[2] = Builder.CreateInsertVector(VT, PoisonValue::get(VT), Ops[2],
480 uint64_t(0));
481 }
482 const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
483 RetTy->getPrimitiveSizeInBits();
484 return EmitFP8NeonCall(IID, {llvm::FixedVectorType::get(RetTy, ElemCount)},
485 Ops, E, name);
486}
487
488Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
489 bool neg) {
490 int SV = cast<ConstantInt>(V)->getSExtValue();
491 return ConstantInt::getSigned(Ty, neg ? -SV : SV);
492}
493
494Value *CodeGenFunction::EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
495 llvm::Type *Ty1, bool Extract,
496 SmallVectorImpl<llvm::Value *> &Ops,
497 const CallExpr *E,
498 const char *name) {
499 llvm::Type *Tys[] = {Ty0, Ty1};
500 if (Extract) {
501 // Op[0] is mfloat8x16_t, but the intrinsic converts only the lower part of
502 // the vector.
503 Tys[1] = llvm::FixedVectorType::get(Int8Ty, 8);
504 Ops[0] = Builder.CreateExtractVector(Tys[1], Ops[0], uint64_t(0));
505 }
506 return EmitFP8NeonCall(IID, Tys, Ops, E, name);
507}
508
509// Right-shift a vector by a constant.
510Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift,
511 llvm::Type *Ty, bool usgn,
512 const char *name) {
513 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
514
515 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
516 int EltSize = VTy->getScalarSizeInBits();
517
518 Vec = Builder.CreateBitCast(Vec, Ty);
519
520 // lshr/ashr are undefined when the shift amount is equal to the vector
521 // element size.
522 if (ShiftAmt == EltSize) {
523 if (usgn) {
524 // Right-shifting an unsigned value by its size yields 0.
525 return llvm::ConstantAggregateZero::get(VTy);
526 } else {
527 // Right-shifting a signed value by its size is equivalent
528 // to a shift of size-1.
529 --ShiftAmt;
530 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
531 }
532 }
533
534 Shift = EmitNeonShiftVector(Shift, Ty, false);
535 if (usgn)
536 return Builder.CreateLShr(Vec, Shift, name);
537 return Builder.CreateAShr(Vec, Shift, name);
538}
539
540// clang-format off
541static const ARMNeonVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
542 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
543 NEONMAP0(splat_lane_v),
544 NEONMAP0(splat_laneq_v),
545 NEONMAP0(splatq_lane_v),
546 NEONMAP0(splatq_laneq_v),
547 NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
548 NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
549 NEONMAP1(vabs_v, arm_neon_vabs, 0),
550 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
551 NEONMAP0(vadd_v),
552 NEONMAP0(vaddhn_v),
553 NEONMAP0(vaddq_v),
554 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
555 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
556 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
557 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
558 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
559 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
560 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
561 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
562 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
563 NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType),
564 NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType),
565 NEONMAP1(vcadd_rot270_f16, arm_neon_vcadd_rot270, Add1ArgType),
566 NEONMAP1(vcadd_rot270_f32, arm_neon_vcadd_rot270, Add1ArgType),
567 NEONMAP1(vcadd_rot90_f16, arm_neon_vcadd_rot90, Add1ArgType),
568 NEONMAP1(vcadd_rot90_f32, arm_neon_vcadd_rot90, Add1ArgType),
569 NEONMAP1(vcaddq_rot270_f16, arm_neon_vcadd_rot270, Add1ArgType),
570 NEONMAP1(vcaddq_rot270_f32, arm_neon_vcadd_rot270, Add1ArgType),
571 NEONMAP1(vcaddq_rot270_f64, arm_neon_vcadd_rot270, Add1ArgType),
572 NEONMAP1(vcaddq_rot90_f16, arm_neon_vcadd_rot90, Add1ArgType),
573 NEONMAP1(vcaddq_rot90_f32, arm_neon_vcadd_rot90, Add1ArgType),
574 NEONMAP1(vcaddq_rot90_f64, arm_neon_vcadd_rot90, Add1ArgType),
575 NEONMAP1(vcage_v, arm_neon_vacge, 0),
576 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
577 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
578 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
579 NEONMAP1(vcale_v, arm_neon_vacge, 0),
580 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
581 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
582 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
583 NEONMAP0(vceqz_v),
584 NEONMAP0(vceqzq_v),
585 NEONMAP0(vcgez_v),
586 NEONMAP0(vcgezq_v),
587 NEONMAP0(vcgtz_v),
588 NEONMAP0(vcgtzq_v),
589 NEONMAP0(vclez_v),
590 NEONMAP0(vclezq_v),
591 NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType),
592 NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType),
593 NEONMAP0(vcltz_v),
594 NEONMAP0(vcltzq_v),
595 NEONMAP1(vclz_v, ctlz, Add1ArgType),
596 NEONMAP1(vclzq_v, ctlz, Add1ArgType),
597 NEONMAP1(vcnt_v, ctpop, Add1ArgType),
598 NEONMAP1(vcntq_v, ctpop, Add1ArgType),
599 NEONMAP0(vcvt_f16_s16),
600 NEONMAP0(vcvt_f16_u16),
601 NEONMAP0(vcvt_f32_v),
602 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
603 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
604 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
605 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
606 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
607 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
608 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
609 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
610 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
611 NEONMAP0(vcvt_s16_f16),
612 NEONMAP0(vcvt_s32_v),
613 NEONMAP0(vcvt_s64_v),
614 NEONMAP0(vcvt_u16_f16),
615 NEONMAP0(vcvt_u32_v),
616 NEONMAP0(vcvt_u64_v),
617 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
618 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
619 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
620 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
621 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
622 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
623 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
624 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
625 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
626 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
627 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
628 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
629 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
630 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
631 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
632 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
633 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
634 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
635 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
636 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
637 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
638 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
639 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
640 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
641 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
642 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
643 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
644 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
645 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
646 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
647 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
648 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
649 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
650 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
651 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
652 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
653 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
654 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
655 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
656 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
657 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
658 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
659 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
660 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
661 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
662 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
663 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
664 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
665 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
666 NEONMAP0(vcvtq_f16_s16),
667 NEONMAP0(vcvtq_f16_u16),
668 NEONMAP0(vcvtq_f32_v),
669 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
670 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
671 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
672 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
673 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
674 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
675 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
676 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
677 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
678 NEONMAP0(vcvtq_s16_f16),
679 NEONMAP0(vcvtq_s32_v),
680 NEONMAP0(vcvtq_s64_v),
681 NEONMAP0(vcvtq_u16_f16),
682 NEONMAP0(vcvtq_u32_v),
683 NEONMAP0(vcvtq_u64_v),
684 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
685 NEONMAP1(vdot_u32, arm_neon_udot, 0),
686 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
687 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
688 NEONMAP0(vext_v),
689 NEONMAP0(vextq_v),
690 NEONMAP0(vfma_v),
691 NEONMAP0(vfmaq_v),
692 NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
693 NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
694 NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
695 NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
696 NEONMAP0(vld1_dup_v),
697 NEONMAP1(vld1_v, arm_neon_vld1, 0),
698 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
699 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
700 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
701 NEONMAP0(vld1q_dup_v),
702 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
703 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
704 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
705 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
706 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
707 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
708 NEONMAP1(vld2_v, arm_neon_vld2, 0),
709 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
710 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
711 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
712 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
713 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
714 NEONMAP1(vld3_v, arm_neon_vld3, 0),
715 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
716 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
717 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
718 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
719 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
720 NEONMAP1(vld4_v, arm_neon_vld4, 0),
721 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
722 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
723 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
724 NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
725 NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType),
726 NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType),
727 NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
728 NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
729 NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType),
730 NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType),
731 NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
732 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
733 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
734 NEONMAP0(vmovl_v),
735 NEONMAP0(vmovn_v),
736 NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType),
737 NEONMAP0(vmull_v),
738 NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType),
739 NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
740 NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
741 NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType),
742 NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
743 NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
744 NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType),
745 NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts),
746 NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts),
747 NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType),
748 NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType),
749 NEONMAP2(vqadd_v, uadd_sat, sadd_sat, Add1ArgType | UnsignedAlts),
750 NEONMAP2(vqaddq_v, uadd_sat, sadd_sat, Add1ArgType | UnsignedAlts),
751 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
752 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
753 NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType),
754 NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType),
755 NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType),
756 NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts),
757 NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType),
758 NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType),
759 NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType),
760 NEONMAP1(vqrdmlah_s16, arm_neon_vqrdmlah, Add1ArgType),
761 NEONMAP1(vqrdmlah_s32, arm_neon_vqrdmlah, Add1ArgType),
762 NEONMAP1(vqrdmlahq_s16, arm_neon_vqrdmlah, Add1ArgType),
763 NEONMAP1(vqrdmlahq_s32, arm_neon_vqrdmlah, Add1ArgType),
764 NEONMAP1(vqrdmlsh_s16, arm_neon_vqrdmlsh, Add1ArgType),
765 NEONMAP1(vqrdmlsh_s32, arm_neon_vqrdmlsh, Add1ArgType),
766 NEONMAP1(vqrdmlshq_s16, arm_neon_vqrdmlsh, Add1ArgType),
767 NEONMAP1(vqrdmlshq_s32, arm_neon_vqrdmlsh, Add1ArgType),
768 NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType),
769 NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType),
770 NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
771 NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
772 NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
773 NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
774 NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
775 NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
776 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
777 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
778 NEONMAP2(vqsub_v, usub_sat, ssub_sat, Add1ArgType | UnsignedAlts),
779 NEONMAP2(vqsubq_v, usub_sat, ssub_sat, Add1ArgType | UnsignedAlts),
780 NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType),
781 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
782 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
783 NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType),
784 NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType),
785 NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
786 NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
787 NEONMAP1(vrnd_v, trunc, Add1ArgType),
788 NEONMAP1(vrnda_v, round, Add1ArgType),
789 NEONMAP1(vrndaq_v, round, Add1ArgType),
790 NEONMAP0(vrndi_v),
791 NEONMAP0(vrndiq_v),
792 NEONMAP1(vrndm_v, floor, Add1ArgType),
793 NEONMAP1(vrndmq_v, floor, Add1ArgType),
794 NEONMAP1(vrndn_v, roundeven, Add1ArgType),
795 NEONMAP1(vrndnq_v, roundeven, Add1ArgType),
796 NEONMAP1(vrndp_v, ceil, Add1ArgType),
797 NEONMAP1(vrndpq_v, ceil, Add1ArgType),
798 NEONMAP1(vrndq_v, trunc, Add1ArgType),
799 NEONMAP1(vrndx_v, rint, Add1ArgType),
800 NEONMAP1(vrndxq_v, rint, Add1ArgType),
801 NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
802 NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
803 NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
804 NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
805 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
806 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
807 NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType),
808 NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType),
809 NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType),
810 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
811 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
812 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
813 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
814 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
815 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
816 NEONMAP0(vshl_n_v),
817 NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
818 NEONMAP0(vshll_n_v),
819 NEONMAP0(vshlq_n_v),
820 NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
821 NEONMAP0(vshr_n_v),
822 NEONMAP0(vshrn_n_v),
823 NEONMAP0(vshrq_n_v),
824 NEONMAP1(vst1_v, arm_neon_vst1, 0),
825 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
826 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
827 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
828 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
829 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
830 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
831 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
832 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
833 NEONMAP1(vst2_v, arm_neon_vst2, 0),
834 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
835 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
836 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
837 NEONMAP1(vst3_v, arm_neon_vst3, 0),
838 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
839 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
840 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
841 NEONMAP1(vst4_v, arm_neon_vst4, 0),
842 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
843 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
844 NEONMAP0(vsubhn_v),
845 NEONMAP0(vtrn_v),
846 NEONMAP0(vtrnq_v),
847 NEONMAP0(vtst_v),
848 NEONMAP0(vtstq_v),
849 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
850 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
851 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
852 NEONMAP0(vuzp_v),
853 NEONMAP0(vuzpq_v),
854 NEONMAP0(vzip_v),
855 NEONMAP0(vzipq_v)
856};
857
858// clang-format on
859
860// Some intrinsics are equivalent for codegen.
861static const std::pair<unsigned, unsigned> NEONEquivalentIntrinsicMap[] = {
862 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
863 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
864 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
865 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
866 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
867 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
868 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
869 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
870 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
871 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
872 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
873 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
874 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
875 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
876 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
877 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
878 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
879 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
880 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
881 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
882 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
883 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
884 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
885 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
886 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
887 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
888 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
889 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
890 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
891 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
892 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
893 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
894 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
895 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
896 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
897 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
898 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
899 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
900 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
901 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
902 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
903 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
904 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
905 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
906 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
907 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
908 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
909 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
910 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
911 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
912 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
913 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
914 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
915 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
916 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
917 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
918 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
919 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
920 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
921 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
922 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
923 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
924 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
925 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
926 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
927 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
928 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
929 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
930 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
931 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
932 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
933 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
934 // The mangling rules cause us to have one ID for each type for vldap1(q)_lane
935 // and vstl1(q)_lane, but codegen is equivalent for all of them. Choose an
936 // arbitrary one to be handled as tha canonical variation.
937 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
938 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
939 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
940 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
941 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
942 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
943 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
944 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
945 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
946 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
947 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
948 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
949};
950
951#undef NEONMAP0
952#undef NEONMAP1
953#undef NEONMAP2
954
955#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
956 {SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, TypeModifier}
957
958#define SVEMAP2(NameBase, TypeModifier) \
959 {SVE::BI__builtin_sve_##NameBase, 0, TypeModifier}
960static const AArch64SVEAndSMEVectorIntrinsicInfo AArch64SVEIntrinsicMap[] = {
961#define GET_SVE_LLVM_INTRINSIC_MAP
962#include "clang/Basic/arm_sve_builtin_cg.inc"
963#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
964#undef GET_SVE_LLVM_INTRINSIC_MAP
965};
966
967#undef SVEMAP1
968#undef SVEMAP2
969
970#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
971 {SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, TypeModifier}
972
973#define SMEMAP2(NameBase, TypeModifier) \
974 {SME::BI__builtin_sme_##NameBase, 0, TypeModifier}
975static const AArch64SVEAndSMEVectorIntrinsicInfo AArch64SMEIntrinsicMap[] = {
976#define GET_SME_LLVM_INTRINSIC_MAP
977#include "clang/Basic/arm_sme_builtin_cg.inc"
978#undef GET_SME_LLVM_INTRINSIC_MAP
979};
980
981#undef SMEMAP1
982#undef SMEMAP2
983
984static bool NEONSIMDIntrinsicsProvenSorted = false;
985
986static bool AArch64SIMDIntrinsicsProvenSorted = false;
987static bool AArch64SISDIntrinsicsProvenSorted = false;
988static bool AArch64SVEIntrinsicsProvenSorted = false;
989static bool AArch64SMEIntrinsicsProvenSorted = false;
990
991// Check if Builtin `BuiltinId` is present in `IntrinsicMap`. If yes, returns
992// the corresponding info struct.
993template <typename IntrinsicInfo>
994static const IntrinsicInfo *
995findARMVectorIntrinsicInMap(ArrayRef<IntrinsicInfo> IntrinsicMap,
996 unsigned BuiltinID, bool &MapProvenSorted) {
997
998#ifndef NDEBUG
999 if (!MapProvenSorted) {
1000 assert(llvm::is_sorted(IntrinsicMap));
1001 MapProvenSorted = true;
1002 }
1003#endif
1004
1005 const IntrinsicInfo *Builtin = llvm::lower_bound(IntrinsicMap, BuiltinID);
1006
1007 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
1008 return Builtin;
1009
1010 return nullptr;
1011}
1012
1013Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
1014 unsigned Modifier,
1015 llvm::Type *ArgType,
1016 const CallExpr *E) {
1017 int VectorSize = 0;
1018 if (Modifier & Use64BitVectors)
1019 VectorSize = 64;
1020 else if (Modifier & Use128BitVectors)
1021 VectorSize = 128;
1022
1023 // Return type.
1024 SmallVector<llvm::Type *, 3> Tys;
1025 if (Modifier & AddRetType) {
1026 llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
1027 if (Modifier & VectorizeRetType)
1028 Ty = llvm::FixedVectorType::get(
1029 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
1030
1031 Tys.push_back(Ty);
1032 }
1033
1034 // Arguments.
1035 if (Modifier & VectorizeArgTypes) {
1036 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
1037 ArgType = llvm::FixedVectorType::get(ArgType, Elts);
1038 }
1039
1040 if (Modifier & (Add1ArgType | Add2ArgTypes))
1041 Tys.push_back(ArgType);
1042
1043 if (Modifier & Add2ArgTypes)
1044 Tys.push_back(ArgType);
1045
1046 if (Modifier & InventFloatType)
1047 Tys.push_back(FloatTy);
1048
1049 return CGM.getIntrinsic(IntrinsicID, Tys);
1050}
1051
1052//===----------------------------------------------------------------------===//
1053// Emit-helpers
1054//===----------------------------------------------------------------------===//
1055static Value *EmitCommonNeonSISDBuiltinExpr(
1056 CodeGenFunction &CGF, const ARMNeonVectorIntrinsicInfo &SISDInfo,
1057 SmallVectorImpl<Value *> &Ops, const CallExpr *E) {
1058 assert(SISDInfo.LLVMIntrinsic && "Generic code assumes a valid intrinsic");
1059
1060 switch (SISDInfo.BuiltinID) {
1061 case NEON::BI__builtin_neon_vcled_s64:
1062 case NEON::BI__builtin_neon_vcled_u64:
1063 case NEON::BI__builtin_neon_vcles_f32:
1064 case NEON::BI__builtin_neon_vcled_f64:
1065 case NEON::BI__builtin_neon_vcltd_s64:
1066 case NEON::BI__builtin_neon_vcltd_u64:
1067 case NEON::BI__builtin_neon_vclts_f32:
1068 case NEON::BI__builtin_neon_vcltd_f64:
1069 case NEON::BI__builtin_neon_vcales_f32:
1070 case NEON::BI__builtin_neon_vcaled_f64:
1071 case NEON::BI__builtin_neon_vcalts_f32:
1072 case NEON::BI__builtin_neon_vcaltd_f64:
1073 // Only one direction of comparisons actually exist, cmle is actually a cmge
1074 // with swapped operands. The table gives us the right intrinsic but we
1075 // still need to do the swap.
1076 std::swap(Ops[0], Ops[1]);
1077 break;
1078 }
1079
1080 // Use fptosi.sat/fptoui.sat unless under strict FP.
1081 unsigned LLVMIntrinsic = SISDInfo.LLVMIntrinsic;
1082 if (!CGF.Builder.getIsFPConstrained()) {
1083 if (LLVMIntrinsic == Intrinsic::aarch64_neon_fcvtzs)
1084 LLVMIntrinsic = Intrinsic::fptosi_sat;
1085 else if (LLVMIntrinsic == Intrinsic::aarch64_neon_fcvtzu)
1086 LLVMIntrinsic = Intrinsic::fptoui_sat;
1087 }
1088 llvm::Type *ArgTy = CGF.ConvertType(E->getArg(0)->getType());
1089 Function *F = CGF.LookupNeonLLVMIntrinsic(LLVMIntrinsic,
1090 SISDInfo.TypeModifier, ArgTy, E);
1091
1092 int j = 0;
1093 ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0);
1094 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
1095 ai != ae; ++ai, ++j) {
1096 llvm::Type *ArgTy = ai->getType();
1097 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
1098 ArgTy->getPrimitiveSizeInBits())
1099 continue;
1100 assert(
1101 ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy() &&
1102 "Expecting vector LLVM intrinsic type and scalar Clang builtin type!");
1103
1104 // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate
1105 // it before inserting.
1106 Ops[j] = CGF.Builder.CreateTruncOrBitCast(
1107 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType());
1108 Ops[j] =
1109 CGF.Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0);
1110 }
1111
1112 Value *Result = CGF.EmitNeonCall(F, Ops, SISDInfo.NameHint);
1113 llvm::Type *ResultType = CGF.ConvertType(E->getType());
1114 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
1115 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
1116 return CGF.Builder.CreateExtractElement(Result, C0);
1117
1118 return CGF.Builder.CreateBitCast(Result, ResultType, SISDInfo.NameHint);
1119}
1120
1121Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
1122 unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
1123 const char *NameHint, unsigned Modifier, const CallExpr *E,
1124 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1,
1125 llvm::Triple::ArchType Arch) {
1126
1127 // Extract the trailing immediate argument that encodes the type discriminator
1128 // for this overloaded intrinsic.
1129 // TODO: Move to the parent code that takes care of argument processing.
1130 const Expr *Arg = E->getArg(E->getNumArgs() - 1);
1131 std::optional<llvm::APSInt> NeonTypeConst =
1132 Arg->getIntegerConstantExpr(getContext());
1133 if (!NeonTypeConst)
1134 return nullptr;
1135
1136 // Determine the type of this overloaded NEON intrinsic.
1137 NeonTypeFlags Type(NeonTypeConst->getZExtValue());
1138 const bool Usgn = Type.isUnsigned();
1139 const bool Quad = Type.isQuad();
1140 const bool Floating = Type.isFloatingPoint();
1141 const bool HasFastHalfType = getTarget().hasFastHalfType();
1142 const bool AllowBFloatArgsAndRet =
1143 getTargetHooks().getABIInfo().allowBFloatArgsAndRet();
1144
1145 llvm::FixedVectorType *VTy =
1146 GetNeonType(this, Type, HasFastHalfType, false, AllowBFloatArgsAndRet);
1147 llvm::Type *Ty = VTy;
1148 if (!Ty)
1149 return nullptr;
1150
1151 auto getAlignmentValue32 = [&](Address addr) -> Value* {
1152 return Builder.getInt32(addr.getAlignment().getQuantity());
1153 };
1154
1155 unsigned Int = LLVMIntrinsic;
1156 if ((Modifier & UnsignedAlts) && !Usgn)
1157 Int = AltLLVMIntrinsic;
1158
1159 switch (BuiltinID) {
1160 default: break;
1161 case NEON::BI__builtin_neon_splat_lane_v:
1162 case NEON::BI__builtin_neon_splat_laneq_v:
1163 case NEON::BI__builtin_neon_splatq_lane_v:
1164 case NEON::BI__builtin_neon_splatq_laneq_v: {
1165 auto NumElements = VTy->getElementCount();
1166 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
1167 NumElements = NumElements * 2;
1168 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
1169 NumElements = NumElements.divideCoefficientBy(2);
1170
1171 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
1172 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements);
1173 }
1174 case NEON::BI__builtin_neon_vpadd_v:
1175 case NEON::BI__builtin_neon_vpaddq_v:
1176 // We don't allow fp/int overloading of intrinsics.
1177 if (VTy->getElementType()->isFloatingPointTy() &&
1178 Int == Intrinsic::aarch64_neon_addp)
1179 Int = Intrinsic::aarch64_neon_faddp;
1180 break;
1181 case NEON::BI__builtin_neon_vabs_v:
1182 case NEON::BI__builtin_neon_vabsq_v:
1183 if (VTy->getElementType()->isFloatingPointTy())
1184 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs");
1185 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs");
1186 case NEON::BI__builtin_neon_vadd_v:
1187 case NEON::BI__builtin_neon_vaddq_v: {
1188 llvm::Type *VTy = llvm::FixedVectorType::get(Int8Ty, Quad ? 16 : 8);
1189 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
1190 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
1191 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]);
1192 return Builder.CreateBitCast(Ops[0], Ty);
1193 }
1194 case NEON::BI__builtin_neon_vaddhn_v: {
1195 llvm::FixedVectorType *SrcTy =
1196 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1197
1198 // %sum = add <4 x i32> %lhs, %rhs
1199 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
1200 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
1201 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn");
1202
1203 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
1204 Constant *ShiftAmt =
1205 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
1206 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn");
1207
1208 // %res = trunc <4 x i32> %high to <4 x i16>
1209 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn");
1210 }
1211 case NEON::BI__builtin_neon_vcale_v:
1212 case NEON::BI__builtin_neon_vcaleq_v:
1213 case NEON::BI__builtin_neon_vcalt_v:
1214 case NEON::BI__builtin_neon_vcaltq_v:
1215 std::swap(Ops[0], Ops[1]);
1216 [[fallthrough]];
1217 case NEON::BI__builtin_neon_vcage_v:
1218 case NEON::BI__builtin_neon_vcageq_v:
1219 case NEON::BI__builtin_neon_vcagt_v:
1220 case NEON::BI__builtin_neon_vcagtq_v: {
1221 llvm::Type *Ty;
1222 switch (VTy->getScalarSizeInBits()) {
1223 default: llvm_unreachable("unexpected type");
1224 case 32:
1225 Ty = FloatTy;
1226 break;
1227 case 64:
1228 Ty = DoubleTy;
1229 break;
1230 case 16:
1231 Ty = HalfTy;
1232 break;
1233 }
1234 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
1235 llvm::Type *Tys[] = { VTy, VecFlt };
1236 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
1237 return EmitNeonCall(F, Ops, NameHint);
1238 }
1239 case NEON::BI__builtin_neon_vceqz_v:
1240 case NEON::BI__builtin_neon_vceqzq_v:
1241 return EmitAArch64CompareBuiltinExpr(
1242 Ops[0], Ty, Floating ? ICmpInst::FCMP_OEQ : ICmpInst::ICMP_EQ, "vceqz");
1243 case NEON::BI__builtin_neon_vcgez_v:
1244 case NEON::BI__builtin_neon_vcgezq_v:
1245 return EmitAArch64CompareBuiltinExpr(
1246 Ops[0], Ty, Floating ? ICmpInst::FCMP_OGE : ICmpInst::ICMP_SGE,
1247 "vcgez");
1248 case NEON::BI__builtin_neon_vclez_v:
1249 case NEON::BI__builtin_neon_vclezq_v:
1250 return EmitAArch64CompareBuiltinExpr(
1251 Ops[0], Ty, Floating ? ICmpInst::FCMP_OLE : ICmpInst::ICMP_SLE,
1252 "vclez");
1253 case NEON::BI__builtin_neon_vcgtz_v:
1254 case NEON::BI__builtin_neon_vcgtzq_v:
1255 return EmitAArch64CompareBuiltinExpr(
1256 Ops[0], Ty, Floating ? ICmpInst::FCMP_OGT : ICmpInst::ICMP_SGT,
1257 "vcgtz");
1258 case NEON::BI__builtin_neon_vcltz_v:
1259 case NEON::BI__builtin_neon_vcltzq_v:
1260 return EmitAArch64CompareBuiltinExpr(
1261 Ops[0], Ty, Floating ? ICmpInst::FCMP_OLT : ICmpInst::ICMP_SLT,
1262 "vcltz");
1263 case NEON::BI__builtin_neon_vclz_v:
1264 case NEON::BI__builtin_neon_vclzq_v:
1265 // We generate target-independent intrinsic, which needs a second argument
1266 // for whether or not clz of zero is undefined; on ARM it isn't.
1267 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
1268 break;
1269 case NEON::BI__builtin_neon_vcvt_f32_v:
1270 case NEON::BI__builtin_neon_vcvtq_f32_v:
1271 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1272 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad),
1273 HasFastHalfType);
1274 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
1275 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
1276 case NEON::BI__builtin_neon_vcvt_f16_s16:
1277 case NEON::BI__builtin_neon_vcvt_f16_u16:
1278 case NEON::BI__builtin_neon_vcvtq_f16_s16:
1279 case NEON::BI__builtin_neon_vcvtq_f16_u16:
1280 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1281 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float16, false, Quad),
1282 HasFastHalfType);
1283 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
1284 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
1285 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
1286 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
1287 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
1288 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
1289 llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty };
1290 Function *F = CGM.getIntrinsic(Int, Tys);
1291 return EmitNeonCall(F, Ops, "vcvt_n");
1292 }
1293 case NEON::BI__builtin_neon_vcvt_n_f32_v:
1294 case NEON::BI__builtin_neon_vcvt_n_f64_v:
1295 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
1296 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
1297 llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty };
1298 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
1299 Function *F = CGM.getIntrinsic(Int, Tys);
1300 return EmitNeonCall(F, Ops, "vcvt_n");
1301 }
1302 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
1303 case NEON::BI__builtin_neon_vcvt_n_s32_v:
1304 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
1305 case NEON::BI__builtin_neon_vcvt_n_u32_v:
1306 case NEON::BI__builtin_neon_vcvt_n_s64_v:
1307 case NEON::BI__builtin_neon_vcvt_n_u64_v:
1308 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
1309 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
1310 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
1311 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
1312 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
1313 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
1314 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
1315 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
1316 return EmitNeonCall(F, Ops, "vcvt_n");
1317 }
1318 case NEON::BI__builtin_neon_vcvt_s32_v:
1319 case NEON::BI__builtin_neon_vcvt_u32_v:
1320 case NEON::BI__builtin_neon_vcvt_s64_v:
1321 case NEON::BI__builtin_neon_vcvt_u64_v:
1322 case NEON::BI__builtin_neon_vcvt_s16_f16:
1323 case NEON::BI__builtin_neon_vcvt_u16_f16:
1324 case NEON::BI__builtin_neon_vcvtq_s32_v:
1325 case NEON::BI__builtin_neon_vcvtq_u32_v:
1326 case NEON::BI__builtin_neon_vcvtq_s64_v:
1327 case NEON::BI__builtin_neon_vcvtq_u64_v:
1328 case NEON::BI__builtin_neon_vcvtq_s16_f16:
1329 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
1330 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
1331 if (Int) {
1332 // AArch64: use fptosi.sat/fptoui.sat unless under strict FP.
1333 if (!Builder.getIsFPConstrained())
1334 Int = Usgn ? Intrinsic::fptoui_sat : Intrinsic::fptosi_sat;
1335 llvm::Type *Tys[2] = {Ty, Ops[0]->getType()};
1336 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtz");
1337 }
1338 // FIXME: ARM uses plain fptoui/fptosi which have UB on out-of-range
1339 // values. These should also use saturating intrinsics.
1340 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
1341 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
1342 }
1343 case NEON::BI__builtin_neon_vcvta_s16_f16:
1344 case NEON::BI__builtin_neon_vcvta_s32_v:
1345 case NEON::BI__builtin_neon_vcvta_s64_v:
1346 case NEON::BI__builtin_neon_vcvta_u16_f16:
1347 case NEON::BI__builtin_neon_vcvta_u32_v:
1348 case NEON::BI__builtin_neon_vcvta_u64_v:
1349 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
1350 case NEON::BI__builtin_neon_vcvtaq_s32_v:
1351 case NEON::BI__builtin_neon_vcvtaq_s64_v:
1352 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
1353 case NEON::BI__builtin_neon_vcvtaq_u32_v:
1354 case NEON::BI__builtin_neon_vcvtaq_u64_v:
1355 case NEON::BI__builtin_neon_vcvtn_s16_f16:
1356 case NEON::BI__builtin_neon_vcvtn_s32_v:
1357 case NEON::BI__builtin_neon_vcvtn_s64_v:
1358 case NEON::BI__builtin_neon_vcvtn_u16_f16:
1359 case NEON::BI__builtin_neon_vcvtn_u32_v:
1360 case NEON::BI__builtin_neon_vcvtn_u64_v:
1361 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
1362 case NEON::BI__builtin_neon_vcvtnq_s32_v:
1363 case NEON::BI__builtin_neon_vcvtnq_s64_v:
1364 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
1365 case NEON::BI__builtin_neon_vcvtnq_u32_v:
1366 case NEON::BI__builtin_neon_vcvtnq_u64_v:
1367 case NEON::BI__builtin_neon_vcvtp_s16_f16:
1368 case NEON::BI__builtin_neon_vcvtp_s32_v:
1369 case NEON::BI__builtin_neon_vcvtp_s64_v:
1370 case NEON::BI__builtin_neon_vcvtp_u16_f16:
1371 case NEON::BI__builtin_neon_vcvtp_u32_v:
1372 case NEON::BI__builtin_neon_vcvtp_u64_v:
1373 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
1374 case NEON::BI__builtin_neon_vcvtpq_s32_v:
1375 case NEON::BI__builtin_neon_vcvtpq_s64_v:
1376 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
1377 case NEON::BI__builtin_neon_vcvtpq_u32_v:
1378 case NEON::BI__builtin_neon_vcvtpq_u64_v:
1379 case NEON::BI__builtin_neon_vcvtm_s16_f16:
1380 case NEON::BI__builtin_neon_vcvtm_s32_v:
1381 case NEON::BI__builtin_neon_vcvtm_s64_v:
1382 case NEON::BI__builtin_neon_vcvtm_u16_f16:
1383 case NEON::BI__builtin_neon_vcvtm_u32_v:
1384 case NEON::BI__builtin_neon_vcvtm_u64_v:
1385 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
1386 case NEON::BI__builtin_neon_vcvtmq_s32_v:
1387 case NEON::BI__builtin_neon_vcvtmq_s64_v:
1388 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
1389 case NEON::BI__builtin_neon_vcvtmq_u32_v:
1390 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
1391 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
1392 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
1393 }
1394 case NEON::BI__builtin_neon_vcvtx_f32_v: {
1395 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
1396 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
1397
1398 }
1399 case NEON::BI__builtin_neon_vext_v:
1400 case NEON::BI__builtin_neon_vextq_v: {
1401 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
1402 SmallVector<int, 16> Indices;
1403 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
1404 Indices.push_back(i+CV);
1405
1406 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1407 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
1408 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext");
1409 }
1410 case NEON::BI__builtin_neon_vfma_v:
1411 case NEON::BI__builtin_neon_vfmaq_v: {
1412 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1413 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
1414 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
1415
1416 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
1417 return emitCallMaybeConstrainedFPBuiltin(
1418 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
1419 {Ops[1], Ops[2], Ops[0]});
1420 }
1421 case NEON::BI__builtin_neon_vld1_x2_v:
1422 case NEON::BI__builtin_neon_vld1q_x2_v:
1423 case NEON::BI__builtin_neon_vld1_x3_v:
1424 case NEON::BI__builtin_neon_vld1q_x3_v:
1425 case NEON::BI__builtin_neon_vld1_x4_v:
1426 case NEON::BI__builtin_neon_vld1q_x4_v: {
1427 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
1428 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
1429 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN");
1430 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
1431 }
1432 case NEON::BI__builtin_neon_vld1_v:
1433 case NEON::BI__builtin_neon_vld1q_v: {
1434 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1435 Ops.push_back(getAlignmentValue32(PtrOp0));
1436 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1");
1437 }
1438 case NEON::BI__builtin_neon_vld2_v:
1439 case NEON::BI__builtin_neon_vld2q_v:
1440 case NEON::BI__builtin_neon_vld3_v:
1441 case NEON::BI__builtin_neon_vld3q_v:
1442 case NEON::BI__builtin_neon_vld4_v:
1443 case NEON::BI__builtin_neon_vld4q_v:
1444 case NEON::BI__builtin_neon_vld2_dup_v:
1445 case NEON::BI__builtin_neon_vld2q_dup_v:
1446 case NEON::BI__builtin_neon_vld3_dup_v:
1447 case NEON::BI__builtin_neon_vld3q_dup_v:
1448 case NEON::BI__builtin_neon_vld4_dup_v:
1449 case NEON::BI__builtin_neon_vld4q_dup_v: {
1450 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1451 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
1452 Value *Align = getAlignmentValue32(PtrOp1);
1453 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
1454 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
1455 }
1456 case NEON::BI__builtin_neon_vld1_dup_v:
1457 case NEON::BI__builtin_neon_vld1q_dup_v: {
1458 Value *V = PoisonValue::get(Ty);
1459 PtrOp0 = PtrOp0.withElementType(VTy->getElementType());
1460 LoadInst *Ld = Builder.CreateLoad(PtrOp0);
1461 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
1462 Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
1463 return EmitNeonSplat(Ops[0], CI);
1464 }
1465 case NEON::BI__builtin_neon_vld2_lane_v:
1466 case NEON::BI__builtin_neon_vld2q_lane_v:
1467 case NEON::BI__builtin_neon_vld3_lane_v:
1468 case NEON::BI__builtin_neon_vld3q_lane_v:
1469 case NEON::BI__builtin_neon_vld4_lane_v:
1470 case NEON::BI__builtin_neon_vld4q_lane_v: {
1471 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1472 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
1473 for (unsigned I = 2; I < Ops.size() - 1; ++I)
1474 Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
1475 Ops.push_back(getAlignmentValue32(PtrOp1));
1476 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), NameHint);
1477 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
1478 }
1479 case NEON::BI__builtin_neon_vmovl_v: {
1480 llvm::FixedVectorType *DTy =
1481 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
1482 Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
1483 if (Usgn)
1484 return Builder.CreateZExt(Ops[0], Ty, "vmovl");
1485 return Builder.CreateSExt(Ops[0], Ty, "vmovl");
1486 }
1487 case NEON::BI__builtin_neon_vmovn_v: {
1488 llvm::FixedVectorType *QTy =
1489 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1490 Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
1491 return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
1492 }
1493 case NEON::BI__builtin_neon_vmull_v:
1494 // FIXME: the integer vmull operations could be emitted in terms of pure
1495 // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of
1496 // hoisting the exts outside loops. Until global ISel comes along that can
1497 // see through such movement this leads to bad CodeGen. So we need an
1498 // intrinsic for now.
1499 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
1500 Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
1501 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
1502 case NEON::BI__builtin_neon_vpadal_v:
1503 case NEON::BI__builtin_neon_vpadalq_v: {
1504 // The source operand type has twice as many elements of half the size.
1505 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
1506 llvm::Type *EltTy =
1507 llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
1508 auto *NarrowTy =
1509 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
1510 llvm::Type *Tys[2] = { Ty, NarrowTy };
1511 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
1512 }
1513 case NEON::BI__builtin_neon_vpaddl_v:
1514 case NEON::BI__builtin_neon_vpaddlq_v: {
1515 // The source operand type has twice as many elements of half the size.
1516 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
1517 llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
1518 auto *NarrowTy =
1519 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
1520 llvm::Type *Tys[2] = { Ty, NarrowTy };
1521 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl");
1522 }
1523 case NEON::BI__builtin_neon_vqdmlal_v:
1524 case NEON::BI__builtin_neon_vqdmlsl_v: {
1525 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
1526 Ops[1] =
1527 EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps, "vqdmlal");
1528 Ops.resize(2);
1529 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint);
1530 }
1531 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
1532 case NEON::BI__builtin_neon_vqdmulh_lane_v:
1533 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
1534 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
1535 auto *RTy = cast<llvm::FixedVectorType>(Ty);
1536 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
1537 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
1538 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
1539 RTy->getNumElements() * 2);
1540 llvm::Type *Tys[2] = {
1541 RTy, GetNeonType(this, NeonTypeFlags(Type.getEltType(), false,
1542 /*isQuad*/ false))};
1543 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
1544 }
1545 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
1546 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
1547 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
1548 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
1549 llvm::Type *Tys[2] = {
1550 Ty, GetNeonType(this, NeonTypeFlags(Type.getEltType(), false,
1551 /*isQuad*/ true))};
1552 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
1553 }
1554 case NEON::BI__builtin_neon_vqshl_n_v:
1555 case NEON::BI__builtin_neon_vqshlq_n_v:
1556 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n",
1557 1, false);
1558 case NEON::BI__builtin_neon_vqshlu_n_v:
1559 case NEON::BI__builtin_neon_vqshluq_n_v:
1560 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n",
1561 1, false);
1562 case NEON::BI__builtin_neon_vrecpe_v:
1563 case NEON::BI__builtin_neon_vrecpeq_v:
1564 case NEON::BI__builtin_neon_vrsqrte_v:
1565 case NEON::BI__builtin_neon_vrsqrteq_v:
1566 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
1567 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
1568 case NEON::BI__builtin_neon_vrndi_v:
1569 case NEON::BI__builtin_neon_vrndiq_v:
1570 Int = Builder.getIsFPConstrained()
1571 ? Intrinsic::experimental_constrained_nearbyint
1572 : Intrinsic::nearbyint;
1573 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
1574 case NEON::BI__builtin_neon_vrshr_n_v:
1575 case NEON::BI__builtin_neon_vrshrq_n_v:
1576 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n",
1577 1, true);
1578 case NEON::BI__builtin_neon_vsha512hq_u64:
1579 case NEON::BI__builtin_neon_vsha512h2q_u64:
1580 case NEON::BI__builtin_neon_vsha512su0q_u64:
1581 case NEON::BI__builtin_neon_vsha512su1q_u64: {
1582 Function *F = CGM.getIntrinsic(Int);
1583 return EmitNeonCall(F, Ops, "");
1584 }
1585 case NEON::BI__builtin_neon_vshl_n_v:
1586 case NEON::BI__builtin_neon_vshlq_n_v:
1587 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false);
1588 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
1589 "vshl_n");
1590 case NEON::BI__builtin_neon_vshll_n_v: {
1591 llvm::FixedVectorType *SrcTy =
1592 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
1593 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
1594 if (Usgn)
1595 Ops[0] = Builder.CreateZExt(Ops[0], VTy);
1596 else
1597 Ops[0] = Builder.CreateSExt(Ops[0], VTy);
1598 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false);
1599 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n");
1600 }
1601 case NEON::BI__builtin_neon_vshrn_n_v: {
1602 llvm::FixedVectorType *SrcTy =
1603 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1604 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
1605 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false);
1606 if (Usgn)
1607 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
1608 else
1609 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
1610 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n");
1611 }
1612 case NEON::BI__builtin_neon_vshr_n_v:
1613 case NEON::BI__builtin_neon_vshrq_n_v:
1614 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n");
1615 case NEON::BI__builtin_neon_vst1_v:
1616 case NEON::BI__builtin_neon_vst1q_v:
1617 case NEON::BI__builtin_neon_vst2_v:
1618 case NEON::BI__builtin_neon_vst2q_v:
1619 case NEON::BI__builtin_neon_vst3_v:
1620 case NEON::BI__builtin_neon_vst3q_v:
1621 case NEON::BI__builtin_neon_vst4_v:
1622 case NEON::BI__builtin_neon_vst4q_v:
1623 case NEON::BI__builtin_neon_vst2_lane_v:
1624 case NEON::BI__builtin_neon_vst2q_lane_v:
1625 case NEON::BI__builtin_neon_vst3_lane_v:
1626 case NEON::BI__builtin_neon_vst3q_lane_v:
1627 case NEON::BI__builtin_neon_vst4_lane_v:
1628 case NEON::BI__builtin_neon_vst4q_lane_v: {
1629 llvm::Type *Tys[] = {Int8PtrTy, Ty};
1630 Ops.push_back(getAlignmentValue32(PtrOp0));
1631 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "");
1632 }
1633 case NEON::BI__builtin_neon_vsm3partw1q_u32:
1634 case NEON::BI__builtin_neon_vsm3partw2q_u32:
1635 case NEON::BI__builtin_neon_vsm3ss1q_u32:
1636 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
1637 case NEON::BI__builtin_neon_vsm4eq_u32: {
1638 Function *F = CGM.getIntrinsic(Int);
1639 return EmitNeonCall(F, Ops, "");
1640 }
1641 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
1642 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
1643 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
1644 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
1645 Function *F = CGM.getIntrinsic(Int);
1646 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
1647 return EmitNeonCall(F, Ops, "");
1648 }
1649 case NEON::BI__builtin_neon_vst1_x2_v:
1650 case NEON::BI__builtin_neon_vst1q_x2_v:
1651 case NEON::BI__builtin_neon_vst1_x3_v:
1652 case NEON::BI__builtin_neon_vst1q_x3_v:
1653 case NEON::BI__builtin_neon_vst1_x4_v:
1654 case NEON::BI__builtin_neon_vst1q_x4_v: {
1655 // TODO: Currently in AArch32 mode the pointer operand comes first, whereas
1656 // in AArch64 it comes last. We may want to stick to one or another.
1657 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
1658 Arch == llvm::Triple::aarch64_32) {
1659 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
1660 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
1661 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "");
1662 }
1663 llvm::Type *Tys[2] = {DefaultPtrTy, VTy};
1664 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "");
1665 }
1666 case NEON::BI__builtin_neon_vsubhn_v: {
1667 llvm::FixedVectorType *SrcTy =
1668 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1669
1670 // %sum = add <4 x i32> %lhs, %rhs
1671 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
1672 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
1673 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn");
1674
1675 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
1676 Constant *ShiftAmt =
1677 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
1678 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn");
1679
1680 // %res = trunc <4 x i32> %high to <4 x i16>
1681 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn");
1682 }
1683 case NEON::BI__builtin_neon_vtrn_v:
1684 case NEON::BI__builtin_neon_vtrnq_v: {
1685 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
1686 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
1687 Value *SV = nullptr;
1688
1689 for (unsigned vi = 0; vi != 2; ++vi) {
1690 SmallVector<int, 16> Indices;
1691 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
1692 Indices.push_back(i+vi);
1693 Indices.push_back(i+e+vi);
1694 }
1695 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
1696 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
1697 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
1698 }
1699 return SV;
1700 }
1701 case NEON::BI__builtin_neon_vtst_v:
1702 case NEON::BI__builtin_neon_vtstq_v: {
1703 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1704 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
1705 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
1706 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
1707 ConstantAggregateZero::get(Ty));
1708 return Builder.CreateSExt(Ops[0], Ty, "vtst");
1709 }
1710 case NEON::BI__builtin_neon_vuzp_v:
1711 case NEON::BI__builtin_neon_vuzpq_v: {
1712 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
1713 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
1714 Value *SV = nullptr;
1715
1716 for (unsigned vi = 0; vi != 2; ++vi) {
1717 SmallVector<int, 16> Indices;
1718 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
1719 Indices.push_back(2*i+vi);
1720
1721 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
1722 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
1723 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
1724 }
1725 return SV;
1726 }
1727 case NEON::BI__builtin_neon_vxarq_u64: {
1728 Function *F = CGM.getIntrinsic(Int);
1729 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
1730 return EmitNeonCall(F, Ops, "");
1731 }
1732 case NEON::BI__builtin_neon_vzip_v:
1733 case NEON::BI__builtin_neon_vzipq_v: {
1734 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
1735 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
1736 Value *SV = nullptr;
1737
1738 for (unsigned vi = 0; vi != 2; ++vi) {
1739 SmallVector<int, 16> Indices;
1740 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
1741 Indices.push_back((i + vi*e) >> 1);
1742 Indices.push_back(((i + vi*e) >> 1)+e);
1743 }
1744 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
1745 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
1746 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
1747 }
1748 return SV;
1749 }
1750 case NEON::BI__builtin_neon_vdot_s32:
1751 case NEON::BI__builtin_neon_vdot_u32:
1752 case NEON::BI__builtin_neon_vdotq_s32:
1753 case NEON::BI__builtin_neon_vdotq_u32: {
1754 auto *InputTy =
1755 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
1756 llvm::Type *Tys[2] = { Ty, InputTy };
1757 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vdot");
1758 }
1759 case NEON::BI__builtin_neon_vfmlal_low_f16:
1760 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
1761 auto *InputTy =
1762 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1763 llvm::Type *Tys[2] = { Ty, InputTy };
1764 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_low");
1765 }
1766 case NEON::BI__builtin_neon_vfmlsl_low_f16:
1767 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
1768 auto *InputTy =
1769 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1770 llvm::Type *Tys[2] = { Ty, InputTy };
1771 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_low");
1772 }
1773 case NEON::BI__builtin_neon_vfmlal_high_f16:
1774 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
1775 auto *InputTy =
1776 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1777 llvm::Type *Tys[2] = { Ty, InputTy };
1778 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_high");
1779 }
1780 case NEON::BI__builtin_neon_vfmlsl_high_f16:
1781 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
1782 auto *InputTy =
1783 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1784 llvm::Type *Tys[2] = { Ty, InputTy };
1785 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_high");
1786 }
1787 case NEON::BI__builtin_neon_vmmlaq_s32:
1788 case NEON::BI__builtin_neon_vmmlaq_u32: {
1789 auto *InputTy =
1790 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
1791 llvm::Type *Tys[2] = { Ty, InputTy };
1792 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vmmla");
1793 }
1794 case NEON::BI__builtin_neon_vmmlaq_f16:
1795 case NEON::BI__builtin_neon_vmmlaq_f32_f16: {
1796 auto *InputTy =
1797 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1798 llvm::Type *Tys[2] = {Ty, InputTy};
1799 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fmmla");
1800 }
1801 case NEON::BI__builtin_neon_vusmmlaq_s32: {
1802 auto *InputTy =
1803 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
1804 llvm::Type *Tys[2] = { Ty, InputTy };
1805 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusmmla");
1806 }
1807 case NEON::BI__builtin_neon_vusdot_s32:
1808 case NEON::BI__builtin_neon_vusdotq_s32: {
1809 auto *InputTy =
1810 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
1811 llvm::Type *Tys[2] = { Ty, InputTy };
1812 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusdot");
1813 }
1814 case NEON::BI__builtin_neon_vbfdot_f32:
1815 case NEON::BI__builtin_neon_vbfdotq_f32: {
1816 llvm::Type *InputTy =
1817 llvm::FixedVectorType::get(BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
1818 llvm::Type *Tys[2] = { Ty, InputTy };
1819 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vbfdot");
1820 }
1821 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
1822 llvm::Type *Tys[1] = { Ty };
1823 Function *F = CGM.getIntrinsic(Int, Tys);
1824 return EmitNeonCall(F, Ops, "vcvtfp2bf");
1825 }
1826
1827 }
1828
1829 assert(Int && "Expected valid intrinsic number");
1830
1831 // Determine the type(s) of this overloaded AArch64 intrinsic.
1832 Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E);
1833
1834 Value *Result = EmitNeonCall(F, Ops, NameHint);
1835 llvm::Type *ResultType = ConvertType(E->getType());
1836 // AArch64 intrinsic one-element vector type cast to
1837 // scalar type expected by the builtin
1838 return Builder.CreateBitCast(Result, ResultType, NameHint);
1839}
1840
1841Value *
1842CodeGenFunction::EmitAArch64CompareBuiltinExpr(Value *Op, llvm::Type *Ty,
1843 const CmpInst::Predicate Pred,
1844 const Twine &Name) {
1845
1846 if (isa<FixedVectorType>(Ty)) {
1847 // Vector types are cast to i8 vectors. Recover original type.
1848 Op = Builder.CreateBitCast(Op, Ty);
1849 }
1850
1851 Constant *zero = Constant::getNullValue(Op->getType());
1852
1853 if (CmpInst::isFPPredicate(Pred)) {
1854 if (Pred == CmpInst::FCMP_OEQ)
1855 Op = Builder.CreateFCmp(Pred, Op, zero);
1856 else
1857 Op = Builder.CreateFCmpS(Pred, Op, zero);
1858 } else {
1859 Op = Builder.CreateICmp(Pred, Op, zero);
1860 }
1861
1862 llvm::Type *ResTy = Ty;
1863 if (auto *VTy = dyn_cast<FixedVectorType>(Ty))
1864 ResTy = FixedVectorType::get(
1865 IntegerType::get(getLLVMContext(), VTy->getScalarSizeInBits()),
1866 VTy->getNumElements());
1867
1868 return Builder.CreateSExt(Op, ResTy, Name);
1869}
1870
1871static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
1872 Value *ExtOp, Value *IndexOp,
1873 llvm::Type *ResTy, unsigned IntID,
1874 const char *Name) {
1875 SmallVector<Value *, 2> TblOps;
1876 if (ExtOp)
1877 TblOps.push_back(ExtOp);
1878
1879 // Build a vector containing sequential number like (0, 1, 2, ..., 15)
1880 SmallVector<int, 16> Indices;
1881 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
1882 for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
1883 Indices.push_back(2*i);
1884 Indices.push_back(2*i+1);
1885 }
1886
1887 int PairPos = 0, End = Ops.size() - 1;
1888 while (PairPos < End) {
1889 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
1890 Ops[PairPos+1], Indices,
1891 Name));
1892 PairPos += 2;
1893 }
1894
1895 // If there's an odd number of 64-bit lookup table, fill the high 64-bit
1896 // of the 128-bit lookup table with zero.
1897 if (PairPos == End) {
1898 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
1899 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
1900 ZeroTbl, Indices, Name));
1901 }
1902
1903 Function *TblF;
1904 TblOps.push_back(IndexOp);
1905 TblF = CGF.CGM.getIntrinsic(IntID, ResTy);
1906
1907 return CGF.EmitNeonCall(TblF, TblOps, Name);
1908}
1909
1910Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) {
1911 unsigned Value;
1912 switch (BuiltinID) {
1913 default:
1914 return nullptr;
1915 case clang::ARM::BI__builtin_arm_nop:
1916 Value = 0;
1917 break;
1918 case clang::ARM::BI__builtin_arm_yield:
1919 case clang::ARM::BI__yield:
1920 Value = 1;
1921 break;
1922 case clang::ARM::BI__builtin_arm_wfe:
1923 case clang::ARM::BI__wfe:
1924 Value = 2;
1925 break;
1926 case clang::ARM::BI__builtin_arm_wfi:
1927 case clang::ARM::BI__wfi:
1928 Value = 3;
1929 break;
1930 case clang::ARM::BI__builtin_arm_sev:
1931 case clang::ARM::BI__sev:
1932 Value = 4;
1933 break;
1934 case clang::ARM::BI__builtin_arm_sevl:
1935 case clang::ARM::BI__sevl:
1936 Value = 5;
1937 break;
1938 }
1939
1940 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
1941 llvm::ConstantInt::get(Int32Ty, Value));
1942}
1943
1949
1950// Generates the IR for the read/write special register builtin,
1951// ValueType is the type of the value that is to be written or read,
1952// RegisterType is the type of the register being written to or read from.
1953static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF,
1954 const CallExpr *E,
1955 llvm::Type *RegisterType,
1956 llvm::Type *ValueType,
1957 SpecialRegisterAccessKind AccessKind,
1958 StringRef SysReg = "") {
1959 // write and register intrinsics only support 32, 64 and 128 bit operations.
1960 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64) ||
1961 RegisterType->isIntegerTy(128)) &&
1962 "Unsupported size for register.");
1963
1964 CodeGen::CGBuilderTy &Builder = CGF.Builder;
1965 CodeGen::CodeGenModule &CGM = CGF.CGM;
1966 LLVMContext &Context = CGM.getLLVMContext();
1967
1968 if (SysReg.empty()) {
1969 const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts();
1970 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
1971 }
1972
1973 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
1974 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
1975 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
1976
1977 llvm::Type *Types[] = { RegisterType };
1978
1979 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
1980 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
1981 && "Can't fit 64-bit value in 32-bit register");
1982
1983 if (AccessKind != Write) {
1984 assert(AccessKind == NormalRead || AccessKind == VolatileRead);
1985 llvm::Function *F = CGM.getIntrinsic(
1986 AccessKind == VolatileRead ? Intrinsic::read_volatile_register
1987 : Intrinsic::read_register,
1988 Types);
1989 llvm::Value *Call = Builder.CreateCall(F, Metadata);
1990
1991 if (MixedTypes)
1992 // Read into 64 bit register and then truncate result to 32 bit.
1993 return Builder.CreateTrunc(Call, ValueType);
1994
1995 if (ValueType->isPointerTy())
1996 // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*).
1997 return Builder.CreateIntToPtr(Call, ValueType);
1998
1999 return Call;
2000 }
2001
2002 llvm::Function *F = CGM.getIntrinsic(Intrinsic::write_register, Types);
2003 llvm::Value *ArgValue = CGF.EmitScalarExpr(E->getArg(1));
2004 if (MixedTypes) {
2005 // Extend 32 bit write value to 64 bit to pass to write.
2006 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
2007 return Builder.CreateCall(F, { Metadata, ArgValue });
2008 }
2009
2010 if (ValueType->isPointerTy()) {
2011 // Have VoidPtrTy ArgValue but want to return an i32/i64.
2012 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
2013 return Builder.CreateCall(F, { Metadata, ArgValue });
2014 }
2015
2016 return Builder.CreateCall(F, { Metadata, ArgValue });
2017}
2018
2019static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID,
2020 const CallExpr *E) {
2021 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2022 CodeGen::CodeGenModule &CGM = CGF.CGM;
2023 SmallVector<llvm::Value *, 4> Ops;
2024
2025 auto getIntArg = [&](unsigned ArgNo) {
2026 Expr::EvalResult Result;
2027 if (!E->getArg(ArgNo)->EvaluateAsInt(Result, CGM.getContext()))
2028 llvm_unreachable("Expected constant argument to range prefetch.");
2029 return Result.Val.getInt().getExtValue();
2030 };
2031
2032 Ops.push_back(CGF.EmitScalarExpr(E->getArg(0))); /*Addr*/
2033 Ops.push_back(CGF.EmitScalarExpr(E->getArg(1))); /*Access Kind*/
2034 Ops.push_back(CGF.EmitScalarExpr(E->getArg(2))); /*Policy*/
2035
2036 if (BuiltinID == clang::AArch64::BI__builtin_arm_range_prefetch_x) {
2037 auto Length = getIntArg(3);
2038 auto Count = getIntArg(4) - 1;
2039 auto Stride = getIntArg(5);
2040 auto Distance = getIntArg(6);
2041
2042 // Map ReuseDistance given in bytes to four bits representing decreasing
2043 // powers of two in the range 512MiB (0b0001) to 32KiB (0b1111). Values
2044 // are rounded up to the nearest power of 2, starting at 32KiB. Any value
2045 // over the maximum is represented by 0 (distance not known).
2046 if (Distance > 0) {
2047 Distance = llvm::Log2_32_Ceil(Distance);
2048 if (Distance < 15)
2049 Distance = 15;
2050 else if (Distance > 29)
2051 Distance = 0;
2052 else
2053 Distance = 30 - Distance;
2054 }
2055
2056 uint64_t Mask22 = (1ULL << 22) - 1;
2057 uint64_t Mask16 = (1ULL << 16) - 1;
2058 uint64_t Metadata = (Distance << 60) | ((Stride & Mask22) << 38) |
2059 ((Count & Mask16) << 22) | (Length & Mask22);
2060
2061 Ops.push_back(llvm::ConstantInt::get(Builder.getInt64Ty(), Metadata));
2062 } else
2063 Ops.push_back(CGF.EmitScalarExpr(E->getArg(3)));
2064
2065 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_range_prefetch),
2066 Ops);
2067}
2068
2069static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
2070 unsigned BuiltinID,
2071 const CallExpr *E) {
2072 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2073 CodeGen::CodeGenModule &CGM = CGF.CGM;
2074 Expr::EvalResult Result;
2075 if (!E->getArg(2)->EvaluateAsInt(Result, CGM.getContext()))
2076 llvm_unreachable(
2077 "Expected integer policy argument to atomic store with hint.");
2078
2079 const Expr *Ptr = E->getArg(0);
2080 Address Addr = CGF.EmitPointerWithAlignment(Ptr);
2081 Addr = Addr.withElementType(
2082 CGF.ConvertTypeForMem(Ptr->getType()->getPointeeType()));
2083
2084 const Expr *Data = E->getArg(1);
2085 Value *DataVal = CGF.EmitToMemory(CGF.EmitScalarExpr(Data), Data->getType());
2086
2087 StoreInst *Store = Builder.CreateStore(DataVal, Addr);
2088 Store->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified());
2089
2090 AtomicOrdering Ordering;
2091 unsigned OrderingArg = Result.Val.getInt().getExtValue();
2092 assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering");
2093
2094 switch (static_cast<AtomicOrderingCABI>(OrderingArg)) {
2095 default:
2096 llvm_unreachable("Unsupported atomic ordering found.");
2097 case AtomicOrderingCABI::relaxed:
2098 Ordering = AtomicOrdering::Monotonic;
2099 break;
2100 case AtomicOrderingCABI::release:
2101 Ordering = AtomicOrdering::Release;
2102 break;
2103 case AtomicOrderingCABI::seq_cst:
2104 Ordering = AtomicOrdering::SequentiallyConsistent;
2105 break;
2106 }
2107 Store->setAtomic(Ordering);
2108
2109 if (!E->getArg(3)->EvaluateAsInt(Result, CGM.getContext()))
2110 llvm_unreachable(
2111 "Expected integer hint argument to atomic store with hint.");
2112 unsigned HintArg = Result.Val.getInt().getExtValue();
2113
2114 // Attach the hint if valid
2115 if (toAArch64MemoryHint(HintArg) != AArch64MemoryHint::NONE) {
2116 LLVMContext &Ctx = CGM.getLLVMContext();
2117 MDNode *MemHint = MDNode::get(
2118 Ctx, {MDString::get(Ctx, "aarch64.mem_hint"),
2119 llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))});
2120 MDNode *HintNode = MDNode::get(
2121 CGM.getLLVMContext(),
2122 {llvm::ConstantAsMetadata::get(Builder.getInt32(1)), MemHint});
2123
2124 Store->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode);
2125 }
2126
2127 return Store;
2128}
2129
2130/// Return true if BuiltinID is an overloaded Neon intrinsic with an extra
2131/// argument that specifies the vector type. The additional argument is meant
2132/// for Sema checking (see `CheckNeonBuiltinFunctionCall`) and this function
2133/// should be kept consistent with the logic in Sema.
2134/// TODO: Make this return false for SISD builtins.
2135static bool HasExtraNeonArgument(unsigned BuiltinID) {
2136 // Required by the headers included below, but not in this particular
2137 // function.
2138 [[maybe_unused]] int PtrArgNum = -1;
2139 [[maybe_unused]] bool HasConstPtr = false;
2140
2141 // The mask encodes the type. We don't care about the actual value. Instead,
2142 // we just check whether its been set.
2143 uint64_t mask = 0;
2144 switch (BuiltinID) {
2145#define GET_NEON_OVERLOAD_CHECK
2146#include "clang/Basic/arm_fp16.inc"
2147#include "clang/Basic/arm_neon.inc"
2148#undef GET_NEON_OVERLOAD_CHECK
2149 // Non-neon builtins for controling VFP that take extra argument for
2150 // discriminating the type.
2151 case ARM::BI__builtin_arm_vcvtr_f:
2152 case ARM::BI__builtin_arm_vcvtr_d:
2153 mask = 1;
2154 }
2155
2156 if (mask)
2157 return true;
2158
2159 return false;
2160}
2161
2162Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
2163 const CallExpr *E,
2164 ReturnValueSlot ReturnValue,
2165 llvm::Triple::ArchType Arch) {
2166 if (auto Hint = GetValueForARMHint(BuiltinID))
2167 return Hint;
2168
2169 if (BuiltinID == clang::ARM::BI__emit) {
2170 bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
2171 llvm::FunctionType *FTy =
2172 llvm::FunctionType::get(VoidTy, /*Variadic=*/false);
2173
2174 Expr::EvalResult Result;
2175 if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
2176 llvm_unreachable("Sema will ensure that the parameter is constant");
2177
2178 llvm::APSInt Value = Result.Val.getInt();
2179 uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
2180
2181 llvm::InlineAsm *Emit =
2182 IsThumb ? InlineAsm::get(FTy, ".inst.n 0x" + utohexstr(ZExtValue), "",
2183 /*hasSideEffects=*/true)
2184 : InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "",
2185 /*hasSideEffects=*/true);
2186
2187 return Builder.CreateCall(Emit);
2188 }
2189
2190 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
2191 Value *Option = EmitScalarExpr(E->getArg(0));
2192 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
2193 }
2194
2195 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
2196 Value *Address = EmitScalarExpr(E->getArg(0));
2197 Value *RW = EmitScalarExpr(E->getArg(1));
2198 Value *IsData = EmitScalarExpr(E->getArg(2));
2199
2200 // Locality is not supported on ARM target
2201 Value *Locality = llvm::ConstantInt::get(Int32Ty, 3);
2202
2203 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
2204 return Builder.CreateCall(F, {Address, RW, Locality, IsData});
2205 }
2206
2207 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
2208 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
2209 return Builder.CreateCall(
2210 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
2211 }
2212
2213 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
2214 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
2215 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
2216 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Arg->getType());
2217 Value *Res = Builder.CreateCall(F, {Arg, Builder.getInt1(false)});
2218 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
2219 Res = Builder.CreateTrunc(Res, Builder.getInt32Ty());
2220 return Res;
2221 }
2222
2223
2224 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
2225 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
2226 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls), Arg, "cls");
2227 }
2228 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
2229 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
2230 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls64), Arg,
2231 "cls");
2232 }
2233
2234 if (BuiltinID == clang::ARM::BI__clear_cache) {
2235 Value *Begin = EmitScalarExpr(E->getArg(0));
2236 Value *End = EmitScalarExpr(E->getArg(1));
2237 Function *F = CGM.getIntrinsic(Intrinsic::clear_cache, {CGM.DefaultPtrTy});
2238 return Builder.CreateCall(F, {Begin, End});
2239 }
2240
2241 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
2242 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
2243 Function *F;
2244
2245 switch (BuiltinID) {
2246 default: llvm_unreachable("unexpected builtin");
2247 case clang::ARM::BI__builtin_arm_mcrr:
2248 F = CGM.getIntrinsic(Intrinsic::arm_mcrr);
2249 break;
2250 case clang::ARM::BI__builtin_arm_mcrr2:
2251 F = CGM.getIntrinsic(Intrinsic::arm_mcrr2);
2252 break;
2253 }
2254
2255 // MCRR{2} instruction has 5 operands but
2256 // the intrinsic has 4 because Rt and Rt2
2257 // are represented as a single unsigned 64
2258 // bit integer in the intrinsic definition
2259 // but internally it's represented as 2 32
2260 // bit integers.
2261
2262 Value *Coproc = EmitScalarExpr(E->getArg(0));
2263 Value *Opc1 = EmitScalarExpr(E->getArg(1));
2264 Value *RtAndRt2 = EmitScalarExpr(E->getArg(2));
2265 Value *CRm = EmitScalarExpr(E->getArg(3));
2266
2267 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
2268 Value *Rt = Builder.CreateTruncOrBitCast(RtAndRt2, Int32Ty);
2269 Value *Rt2 = Builder.CreateLShr(RtAndRt2, C1);
2270 Rt2 = Builder.CreateTruncOrBitCast(Rt2, Int32Ty);
2271
2272 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
2273 }
2274
2275 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
2276 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
2277 Function *F;
2278
2279 switch (BuiltinID) {
2280 default: llvm_unreachable("unexpected builtin");
2281 case clang::ARM::BI__builtin_arm_mrrc:
2282 F = CGM.getIntrinsic(Intrinsic::arm_mrrc);
2283 break;
2284 case clang::ARM::BI__builtin_arm_mrrc2:
2285 F = CGM.getIntrinsic(Intrinsic::arm_mrrc2);
2286 break;
2287 }
2288
2289 Value *Coproc = EmitScalarExpr(E->getArg(0));
2290 Value *Opc1 = EmitScalarExpr(E->getArg(1));
2291 Value *CRm = EmitScalarExpr(E->getArg(2));
2292 Value *RtAndRt2 = Builder.CreateCall(F, {Coproc, Opc1, CRm});
2293
2294 // Returns an unsigned 64 bit integer, represented
2295 // as two 32 bit integers.
2296
2297 Value *Rt = Builder.CreateExtractValue(RtAndRt2, 1);
2298 Value *Rt1 = Builder.CreateExtractValue(RtAndRt2, 0);
2299 Rt = Builder.CreateZExt(Rt, Int64Ty);
2300 Rt1 = Builder.CreateZExt(Rt1, Int64Ty);
2301
2302 Value *ShiftCast = llvm::ConstantInt::get(Int64Ty, 32);
2303 RtAndRt2 = Builder.CreateShl(Rt, ShiftCast, "shl", true);
2304 RtAndRt2 = Builder.CreateOr(RtAndRt2, Rt1);
2305
2306 return Builder.CreateBitCast(RtAndRt2, ConvertType(E->getType()));
2307 }
2308
2309 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
2310 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
2311 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
2312 getContext().getTypeSize(E->getType()) == 64) ||
2313 BuiltinID == clang::ARM::BI__ldrexd) {
2314 Function *F;
2315
2316 switch (BuiltinID) {
2317 default: llvm_unreachable("unexpected builtin");
2318 case clang::ARM::BI__builtin_arm_ldaex:
2319 F = CGM.getIntrinsic(Intrinsic::arm_ldaexd);
2320 break;
2321 case clang::ARM::BI__builtin_arm_ldrexd:
2322 case clang::ARM::BI__builtin_arm_ldrex:
2323 case clang::ARM::BI__ldrexd:
2324 F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
2325 break;
2326 }
2327
2328 Value *LdPtr = EmitScalarExpr(E->getArg(0));
2329 Value *Val = Builder.CreateCall(F, LdPtr, "ldrexd");
2330
2331 Value *Val0 = Builder.CreateExtractValue(Val, 1);
2332 Value *Val1 = Builder.CreateExtractValue(Val, 0);
2333 Val0 = Builder.CreateZExt(Val0, Int64Ty);
2334 Val1 = Builder.CreateZExt(Val1, Int64Ty);
2335
2336 Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32);
2337 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
2338 Val = Builder.CreateOr(Val, Val1);
2339 return Builder.CreateBitCast(Val, ConvertType(E->getType()));
2340 }
2341
2342 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
2343 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
2344 Value *LoadAddr = EmitScalarExpr(E->getArg(0));
2345
2346 QualType Ty = E->getType();
2347 llvm::Type *RealResTy = ConvertType(Ty);
2348 llvm::Type *IntTy =
2349 llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
2350
2351 Function *F = CGM.getIntrinsic(
2352 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
2353 : Intrinsic::arm_ldrex,
2354 DefaultPtrTy);
2355 CallInst *Val = Builder.CreateCall(F, LoadAddr, "ldrex");
2356 Val->addParamAttr(
2357 0, Attribute::get(getLLVMContext(), Attribute::ElementType, IntTy));
2358
2359 if (RealResTy->isPointerTy())
2360 return Builder.CreateIntToPtr(Val, RealResTy);
2361 else {
2362 llvm::Type *IntResTy = llvm::IntegerType::get(
2363 getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
2364 return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
2365 RealResTy);
2366 }
2367 }
2368
2369 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
2370 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
2371 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
2372 getContext().getTypeSize(E->getArg(0)->getType()) == 64)) {
2373 Function *F = CGM.getIntrinsic(
2374 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
2375 : Intrinsic::arm_strexd);
2376 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty);
2377
2378 Address Tmp = CreateMemTempWithoutCast(E->getArg(0)->getType());
2379 Value *Val = EmitScalarExpr(E->getArg(0));
2380 Builder.CreateStore(Val, Tmp);
2381
2382 Address LdPtr = Tmp.withElementType(STy);
2383 Val = Builder.CreateLoad(LdPtr);
2384
2385 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
2386 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
2387 Value *StPtr = EmitScalarExpr(E->getArg(1));
2388 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd");
2389 }
2390
2391 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
2392 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
2393 Value *StoreVal = EmitScalarExpr(E->getArg(0));
2394 Value *StoreAddr = EmitScalarExpr(E->getArg(1));
2395
2396 QualType Ty = E->getArg(0)->getType();
2397 llvm::Type *StoreTy =
2398 llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
2399
2400 if (StoreVal->getType()->isPointerTy())
2401 StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty);
2402 else {
2403 llvm::Type *IntTy = llvm::IntegerType::get(
2404 getLLVMContext(),
2405 CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
2406 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
2407 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty);
2408 }
2409
2410 Function *F = CGM.getIntrinsic(
2411 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
2412 : Intrinsic::arm_strex,
2413 StoreAddr->getType());
2414
2415 CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex");
2416 CI->addParamAttr(
2417 1, Attribute::get(getLLVMContext(), Attribute::ElementType, StoreTy));
2418 return CI;
2419 }
2420
2421 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
2422 Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
2423 return Builder.CreateCall(F);
2424 }
2425
2426 // CRC32
2427 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
2428 switch (BuiltinID) {
2429 case clang::ARM::BI__builtin_arm_crc32b:
2430 CRCIntrinsicID = Intrinsic::arm_crc32b; break;
2431 case clang::ARM::BI__builtin_arm_crc32cb:
2432 CRCIntrinsicID = Intrinsic::arm_crc32cb; break;
2433 case clang::ARM::BI__builtin_arm_crc32h:
2434 CRCIntrinsicID = Intrinsic::arm_crc32h; break;
2435 case clang::ARM::BI__builtin_arm_crc32ch:
2436 CRCIntrinsicID = Intrinsic::arm_crc32ch; break;
2437 case clang::ARM::BI__builtin_arm_crc32w:
2438 case clang::ARM::BI__builtin_arm_crc32d:
2439 CRCIntrinsicID = Intrinsic::arm_crc32w; break;
2440 case clang::ARM::BI__builtin_arm_crc32cw:
2441 case clang::ARM::BI__builtin_arm_crc32cd:
2442 CRCIntrinsicID = Intrinsic::arm_crc32cw; break;
2443 }
2444
2445 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
2446 Value *Arg0 = EmitScalarExpr(E->getArg(0));
2447 Value *Arg1 = EmitScalarExpr(E->getArg(1));
2448
2449 // crc32{c,}d intrinsics are implemented as two calls to crc32{c,}w
2450 // intrinsics, hence we need different codegen for these cases.
2451 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
2452 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
2453 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
2454 Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty);
2455 Value *Arg1b = Builder.CreateLShr(Arg1, C1);
2456 Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty);
2457
2458 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
2459 Value *Res = Builder.CreateCall(F, {Arg0, Arg1a});
2460 return Builder.CreateCall(F, {Res, Arg1b});
2461 } else {
2462 Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty);
2463
2464 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
2465 return Builder.CreateCall(F, {Arg0, Arg1});
2466 }
2467 }
2468
2469 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
2470 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2471 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
2472 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
2473 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
2474 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
2475
2476 SpecialRegisterAccessKind AccessKind = Write;
2477 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
2478 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2479 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
2480 AccessKind = VolatileRead;
2481
2482 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
2483 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
2484
2485 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2486 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
2487
2488 llvm::Type *ValueType;
2489 llvm::Type *RegisterType;
2490 if (IsPointerBuiltin) {
2491 ValueType = VoidPtrTy;
2492 RegisterType = Int32Ty;
2493 } else if (Is64Bit) {
2494 ValueType = RegisterType = Int64Ty;
2495 } else {
2496 ValueType = RegisterType = Int32Ty;
2497 }
2498
2499 return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType,
2500 AccessKind);
2501 }
2502
2503 if (BuiltinID == ARM::BI__builtin_sponentry) {
2504 llvm::Function *F = CGM.getIntrinsic(Intrinsic::sponentry, AllocaInt8PtrTy);
2505 return Builder.CreateCall(F);
2506 }
2507
2508 // Handle MSVC intrinsics before argument evaluation to prevent double
2509 // evaluation.
2510 if (std::optional<MSVCIntrin> MsvcIntId = translateArmToMsvcIntrin(BuiltinID))
2511 return EmitMSVCBuiltinExpr(*MsvcIntId, E);
2512
2513 // Deal with MVE builtins
2514 if (Value *Result = EmitARMMVEBuiltinExpr(BuiltinID, E, ReturnValue, Arch))
2515 return Result;
2516 // Handle CDE builtins
2517 if (Value *Result = EmitARMCDEBuiltinExpr(BuiltinID, E, ReturnValue, Arch))
2518 return Result;
2519
2520 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
2521 auto It = llvm::find_if(NEONEquivalentIntrinsicMap, [BuiltinID](auto &P) {
2522 return P.first == BuiltinID;
2523 });
2524 if (It != end(NEONEquivalentIntrinsicMap))
2525 BuiltinID = It->second;
2526
2527 // Find out if any arguments are required to be integer constant
2528 // expressions.
2529 unsigned ICEArguments = 0;
2530 ASTContext::GetBuiltinTypeError Error;
2531 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
2532 assert(Error == ASTContext::GE_None && "Should not codegen an error");
2533
2534 auto getAlignmentValue32 = [&](Address addr) -> Value* {
2535 return Builder.getInt32(addr.getAlignment().getQuantity());
2536 };
2537
2538 Address PtrOp0 = Address::invalid();
2539 Address PtrOp1 = Address::invalid();
2540 SmallVector<Value*, 4> Ops;
2541 bool HasExtraArg = HasExtraNeonArgument(BuiltinID);
2542 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
2543 for (unsigned i = 0, e = NumArgs; i != e; i++) {
2544 if (i == 0) {
2545 switch (BuiltinID) {
2546 case NEON::BI__builtin_neon_vld1_v:
2547 case NEON::BI__builtin_neon_vld1q_v:
2548 case NEON::BI__builtin_neon_vld1q_lane_v:
2549 case NEON::BI__builtin_neon_vld1_lane_v:
2550 case NEON::BI__builtin_neon_vld1_dup_v:
2551 case NEON::BI__builtin_neon_vld1q_dup_v:
2552 case NEON::BI__builtin_neon_vst1_v:
2553 case NEON::BI__builtin_neon_vst1q_v:
2554 case NEON::BI__builtin_neon_vst1q_lane_v:
2555 case NEON::BI__builtin_neon_vst1_lane_v:
2556 case NEON::BI__builtin_neon_vst2_v:
2557 case NEON::BI__builtin_neon_vst2q_v:
2558 case NEON::BI__builtin_neon_vst2_lane_v:
2559 case NEON::BI__builtin_neon_vst2q_lane_v:
2560 case NEON::BI__builtin_neon_vst3_v:
2561 case NEON::BI__builtin_neon_vst3q_v:
2562 case NEON::BI__builtin_neon_vst3_lane_v:
2563 case NEON::BI__builtin_neon_vst3q_lane_v:
2564 case NEON::BI__builtin_neon_vst4_v:
2565 case NEON::BI__builtin_neon_vst4q_v:
2566 case NEON::BI__builtin_neon_vst4_lane_v:
2567 case NEON::BI__builtin_neon_vst4q_lane_v:
2568 // Get the alignment for the argument in addition to the value;
2569 // we'll use it later.
2570 PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
2571 Ops.push_back(PtrOp0.emitRawPointer(*this));
2572 continue;
2573 }
2574 }
2575 if (i == 1) {
2576 switch (BuiltinID) {
2577 case NEON::BI__builtin_neon_vld2_v:
2578 case NEON::BI__builtin_neon_vld2q_v:
2579 case NEON::BI__builtin_neon_vld3_v:
2580 case NEON::BI__builtin_neon_vld3q_v:
2581 case NEON::BI__builtin_neon_vld4_v:
2582 case NEON::BI__builtin_neon_vld4q_v:
2583 case NEON::BI__builtin_neon_vld2_lane_v:
2584 case NEON::BI__builtin_neon_vld2q_lane_v:
2585 case NEON::BI__builtin_neon_vld3_lane_v:
2586 case NEON::BI__builtin_neon_vld3q_lane_v:
2587 case NEON::BI__builtin_neon_vld4_lane_v:
2588 case NEON::BI__builtin_neon_vld4q_lane_v:
2589 case NEON::BI__builtin_neon_vld2_dup_v:
2590 case NEON::BI__builtin_neon_vld2q_dup_v:
2591 case NEON::BI__builtin_neon_vld3_dup_v:
2592 case NEON::BI__builtin_neon_vld3q_dup_v:
2593 case NEON::BI__builtin_neon_vld4_dup_v:
2594 case NEON::BI__builtin_neon_vld4q_dup_v:
2595 // Get the alignment for the argument in addition to the value;
2596 // we'll use it later.
2597 PtrOp1 = EmitPointerWithAlignment(E->getArg(1));
2598 Ops.push_back(PtrOp1.emitRawPointer(*this));
2599 continue;
2600 }
2601 }
2602
2603 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E));
2604 }
2605
2606 switch (BuiltinID) {
2607 default: break;
2608
2609 case NEON::BI__builtin_neon_vget_lane_i8:
2610 case NEON::BI__builtin_neon_vget_lane_i16:
2611 case NEON::BI__builtin_neon_vget_lane_i32:
2612 case NEON::BI__builtin_neon_vget_lane_i64:
2613 case NEON::BI__builtin_neon_vget_lane_bf16:
2614 case NEON::BI__builtin_neon_vget_lane_f32:
2615 case NEON::BI__builtin_neon_vgetq_lane_i8:
2616 case NEON::BI__builtin_neon_vgetq_lane_i16:
2617 case NEON::BI__builtin_neon_vgetq_lane_i32:
2618 case NEON::BI__builtin_neon_vgetq_lane_i64:
2619 case NEON::BI__builtin_neon_vgetq_lane_bf16:
2620 case NEON::BI__builtin_neon_vgetq_lane_f32:
2621 case NEON::BI__builtin_neon_vduph_lane_bf16:
2622 case NEON::BI__builtin_neon_vduph_laneq_bf16:
2623 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
2624
2625 case NEON::BI__builtin_neon_vrndns_f32: {
2626 Value *Arg = EmitScalarExpr(E->getArg(0));
2627 llvm::Type *Tys[] = {Arg->getType()};
2628 Function *F = CGM.getIntrinsic(Intrinsic::roundeven, Tys);
2629 return Builder.CreateCall(F, {Arg}, "vrndn"); }
2630
2631 case NEON::BI__builtin_neon_vset_lane_i8:
2632 case NEON::BI__builtin_neon_vset_lane_i16:
2633 case NEON::BI__builtin_neon_vset_lane_i32:
2634 case NEON::BI__builtin_neon_vset_lane_i64:
2635 case NEON::BI__builtin_neon_vset_lane_bf16:
2636 case NEON::BI__builtin_neon_vset_lane_f32:
2637 case NEON::BI__builtin_neon_vsetq_lane_i8:
2638 case NEON::BI__builtin_neon_vsetq_lane_i16:
2639 case NEON::BI__builtin_neon_vsetq_lane_i32:
2640 case NEON::BI__builtin_neon_vsetq_lane_i64:
2641 case NEON::BI__builtin_neon_vsetq_lane_bf16:
2642 case NEON::BI__builtin_neon_vsetq_lane_f32:
2643 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
2644
2645 case NEON::BI__builtin_neon_vsha1h_u32:
2646 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops,
2647 "vsha1h");
2648 case NEON::BI__builtin_neon_vsha1cq_u32:
2649 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops,
2650 "vsha1h");
2651 case NEON::BI__builtin_neon_vsha1pq_u32:
2652 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops,
2653 "vsha1h");
2654 case NEON::BI__builtin_neon_vsha1mq_u32:
2655 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
2656 "vsha1h");
2657
2658 case NEON::BI__builtin_neon_vcvth_bf16_f32:
2659 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvtbfp2bf), Ops,
2660 "vcvtbfp2bf");
2661 case NEON::BI__builtin_neon_vcvt_f16_f32:
2662 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvtfp2hf), Ops,
2663 "vcvtfp2hf");
2664 case NEON::BI__builtin_neon_vcvt_f32_f16:
2665 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvthf2fp), Ops,
2666 "vcvthf2fp");
2667
2668 // The ARM _MoveToCoprocessor builtins put the input register value as
2669 // the first argument, but the LLVM intrinsic expects it as the third one.
2670 case clang::ARM::BI_MoveToCoprocessor:
2671 case clang::ARM::BI_MoveToCoprocessor2: {
2672 Function *F = CGM.getIntrinsic(BuiltinID == clang::ARM::BI_MoveToCoprocessor
2673 ? Intrinsic::arm_mcr
2674 : Intrinsic::arm_mcr2);
2675 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
2676 Ops[3], Ops[4], Ops[5]});
2677 }
2678 }
2679
2680 // Get the last argument, which specifies the vector type.
2681 assert(HasExtraArg);
2682 const Expr *Arg = E->getArg(E->getNumArgs()-1);
2683 std::optional<llvm::APSInt> Result =
2684 Arg->getIntegerConstantExpr(getContext());
2685 if (!Result)
2686 return nullptr;
2687
2688 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
2689 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
2690 // Determine the overloaded type of this builtin.
2691 llvm::Type *Ty;
2692 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
2693 Ty = FloatTy;
2694 else
2695 Ty = DoubleTy;
2696
2697 // Determine whether this is an unsigned conversion or not.
2698 bool usgn = Result->getZExtValue() == 1;
2699 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
2700
2701 // Call the appropriate intrinsic.
2702 Function *F = CGM.getIntrinsic(Int, Ty);
2703 return Builder.CreateCall(F, Ops, "vcvtr");
2704 }
2705
2706 // Determine the type of this overloaded NEON intrinsic.
2707 NeonTypeFlags Type = Result->getZExtValue();
2708 bool usgn = Type.isUnsigned();
2709 bool rightShift = false;
2710
2711 llvm::FixedVectorType *VTy =
2712 GetNeonType(this, Type, getTarget().hasFastHalfType(), false,
2713 getTarget().hasBFloat16Type());
2714 llvm::Type *Ty = VTy;
2715 if (!Ty)
2716 return nullptr;
2717
2718 // Many NEON builtins have identical semantics and uses in ARM and
2719 // AArch64. Emit these in a single function.
2720 auto IntrinsicMap = ArrayRef(ARMSIMDIntrinsicMap);
2721 const ARMNeonVectorIntrinsicInfo *Builtin = findARMVectorIntrinsicInMap(
2722 IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted);
2723 if (Builtin)
2724 return EmitCommonNeonBuiltinExpr(
2725 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
2726 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
2727
2728 unsigned Int;
2729 switch (BuiltinID) {
2730 default: return nullptr;
2731 case NEON::BI__builtin_neon_vld1q_lane_v:
2732 // Handle 64-bit integer elements as a special case. Use shuffles of
2733 // one-element vectors to avoid poor code for i64 in the backend.
2734 if (VTy->getElementType()->isIntegerTy(64)) {
2735 // Extract the other lane.
2736 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2737 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
2738 Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
2739 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
2740 // Load the value as a one-element vector.
2741 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
2742 llvm::Type *Tys[] = {Ty, Int8PtrTy};
2743 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys);
2744 Value *Align = getAlignmentValue32(PtrOp0);
2745 Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
2746 // Combine them.
2747 int Indices[] = {1 - Lane, Lane};
2748 return Builder.CreateShuffleVector(Ops[1], Ld, Indices, "vld1q_lane");
2749 }
2750 [[fallthrough]];
2751 case NEON::BI__builtin_neon_vld1_lane_v: {
2752 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2753 PtrOp0 = PtrOp0.withElementType(VTy->getElementType());
2754 Value *Ld = Builder.CreateLoad(PtrOp0);
2755 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
2756 }
2757 case NEON::BI__builtin_neon_vqrshrn_n_v:
2758 Int =
2759 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
2760 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n",
2761 1, true);
2762 case NEON::BI__builtin_neon_vqrshrun_n_v:
2763 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
2764 Ops, "vqrshrun_n", 1, true);
2765 case NEON::BI__builtin_neon_vqshrn_n_v:
2766 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
2767 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n",
2768 1, true);
2769 case NEON::BI__builtin_neon_vqshrun_n_v:
2770 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
2771 Ops, "vqshrun_n", 1, true);
2772 case NEON::BI__builtin_neon_vrecpe_v:
2773 case NEON::BI__builtin_neon_vrecpeq_v:
2774 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty),
2775 Ops, "vrecpe");
2776 case NEON::BI__builtin_neon_vrshrn_n_v:
2777 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
2778 Ops, "vrshrn_n", 1, true);
2779 case NEON::BI__builtin_neon_vrsra_n_v:
2780 case NEON::BI__builtin_neon_vrsraq_n_v:
2781 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2782 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2783 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true);
2784 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
2785 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]});
2786 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
2787 case NEON::BI__builtin_neon_vsri_n_v:
2788 case NEON::BI__builtin_neon_vsriq_n_v:
2789 rightShift = true;
2790 [[fallthrough]];
2791 case NEON::BI__builtin_neon_vsli_n_v:
2792 case NEON::BI__builtin_neon_vsliq_n_v:
2793 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift);
2794 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
2795 Ops, "vsli_n");
2796 case NEON::BI__builtin_neon_vsra_n_v:
2797 case NEON::BI__builtin_neon_vsraq_n_v:
2798 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2799 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
2800 return Builder.CreateAdd(Ops[0], Ops[1]);
2801 case NEON::BI__builtin_neon_vst1q_lane_v:
2802 // Handle 64-bit integer elements as a special case. Use a shuffle to get
2803 // a one-element vector and avoid poor code for i64 in the backend.
2804 if (VTy->getElementType()->isIntegerTy(64)) {
2805 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2806 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
2807 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
2808 Ops[2] = getAlignmentValue32(PtrOp0);
2809 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
2810 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
2811 Tys), Ops);
2812 }
2813 [[fallthrough]];
2814 case NEON::BI__builtin_neon_vst1_lane_v: {
2815 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2816 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
2817 return Builder.CreateStore(Ops[1],
2818 PtrOp0.withElementType(Ops[1]->getType()));
2819 }
2820 case NEON::BI__builtin_neon_vtbl1_v:
2821 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
2822 Ops, "vtbl1");
2823 case NEON::BI__builtin_neon_vtbl2_v:
2824 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2),
2825 Ops, "vtbl2");
2826 case NEON::BI__builtin_neon_vtbl3_v:
2827 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3),
2828 Ops, "vtbl3");
2829 case NEON::BI__builtin_neon_vtbl4_v:
2830 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4),
2831 Ops, "vtbl4");
2832 case NEON::BI__builtin_neon_vtbx1_v:
2833 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1),
2834 Ops, "vtbx1");
2835 case NEON::BI__builtin_neon_vtbx2_v:
2836 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2),
2837 Ops, "vtbx2");
2838 case NEON::BI__builtin_neon_vtbx3_v:
2839 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3),
2840 Ops, "vtbx3");
2841 case NEON::BI__builtin_neon_vtbx4_v:
2842 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4),
2843 Ops, "vtbx4");
2844 }
2845}
2846
2847template<typename Integer>
2848static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context) {
2849 return E->getIntegerConstantExpr(Context)->getExtValue();
2850}
2851
2852static llvm::Value *SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V,
2853 llvm::Type *T, bool Unsigned) {
2854 // Helper function called by Tablegen-constructed ARM MVE builtin codegen,
2855 // which finds it convenient to specify signed/unsigned as a boolean flag.
2856 return Unsigned ? Builder.CreateZExt(V, T) : Builder.CreateSExt(V, T);
2857}
2858
2859static llvm::Value *MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V,
2860 uint32_t Shift, bool Unsigned) {
2861 // MVE helper function for integer shift right. This must handle signed vs
2862 // unsigned, and also deal specially with the case where the shift count is
2863 // equal to the lane size. In LLVM IR, an LShr with that parameter would be
2864 // undefined behavior, but in MVE it's legal, so we must convert it to code
2865 // that is not undefined in IR.
2866 unsigned LaneBits = cast<llvm::VectorType>(V->getType())
2867 ->getElementType()
2868 ->getPrimitiveSizeInBits();
2869 if (Shift == LaneBits) {
2870 // An unsigned shift of the full lane size always generates zero, so we can
2871 // simply emit a zero vector. A signed shift of the full lane size does the
2872 // same thing as shifting by one bit fewer.
2873 if (Unsigned)
2874 return llvm::Constant::getNullValue(V->getType());
2875 else
2876 --Shift;
2877 }
2878 return Unsigned ? Builder.CreateLShr(V, Shift) : Builder.CreateAShr(V, Shift);
2879}
2880
2881static llvm::Value *ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V) {
2882 // MVE-specific helper function for a vector splat, which infers the element
2883 // count of the output vector by knowing that MVE vectors are all 128 bits
2884 // wide.
2885 unsigned Elements = 128 / V->getType()->getPrimitiveSizeInBits();
2886 return Builder.CreateVectorSplat(Elements, V);
2887}
2888
2889static llvm::Value *ARMMVEVectorReinterpret(CGBuilderTy &Builder,
2890 CodeGenFunction *CGF,
2891 llvm::Value *V,
2892 llvm::Type *DestType) {
2893 // Convert one MVE vector type into another by reinterpreting its in-register
2894 // format.
2895 //
2896 // Little-endian, this is identical to a bitcast (which reinterprets the
2897 // memory format). But big-endian, they're not necessarily the same, because
2898 // the register and memory formats map to each other differently depending on
2899 // the lane size.
2900 //
2901 // We generate a bitcast whenever we can (if we're little-endian, or if the
2902 // lane sizes are the same anyway). Otherwise we fall back to an IR intrinsic
2903 // that performs the different kind of reinterpretation.
2904 if (CGF->getTarget().isBigEndian() &&
2905 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
2906 return Builder.CreateCall(
2907 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vreinterpretq,
2908 {DestType, V->getType()}),
2909 V);
2910 } else {
2911 return Builder.CreateBitCast(V, DestType);
2912 }
2913}
2914
2915static llvm::Value *VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd) {
2916 // Make a shufflevector that extracts every other element of a vector (evens
2917 // or odds, as desired).
2918 SmallVector<int, 16> Indices;
2919 unsigned InputElements =
2920 cast<llvm::FixedVectorType>(V->getType())->getNumElements();
2921 for (unsigned i = 0; i < InputElements; i += 2)
2922 Indices.push_back(i + Odd);
2923 return Builder.CreateShuffleVector(V, Indices);
2924}
2925
2926static llvm::Value *VectorZip(CGBuilderTy &Builder, llvm::Value *V0,
2927 llvm::Value *V1) {
2928 // Make a shufflevector that interleaves two vectors element by element.
2929 assert(V0->getType() == V1->getType() && "Can't zip different vector types");
2930 SmallVector<int, 16> Indices;
2931 unsigned InputElements =
2932 cast<llvm::FixedVectorType>(V0->getType())->getNumElements();
2933 for (unsigned i = 0; i < InputElements; i++) {
2934 Indices.push_back(i);
2935 Indices.push_back(i + InputElements);
2936 }
2937 return Builder.CreateShuffleVector(V0, V1, Indices);
2938}
2939
2940template<unsigned HighBit, unsigned OtherBits>
2941static llvm::Value *ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT) {
2942 // MVE-specific helper function to make a vector splat of a constant such as
2943 // UINT_MAX or INT_MIN, in which all bits below the highest one are equal.
2944 llvm::Type *T = cast<llvm::VectorType>(VT)->getElementType();
2945 unsigned LaneBits = T->getPrimitiveSizeInBits();
2946 uint32_t Value = HighBit << (LaneBits - 1);
2947 if (OtherBits)
2948 Value |= (1UL << (LaneBits - 1)) - 1;
2949 llvm::Value *Lane = llvm::ConstantInt::get(T, Value);
2950 return ARMMVEVectorSplat(Builder, Lane);
2951}
2952
2953static llvm::Value *ARMMVEVectorElementReverse(CGBuilderTy &Builder,
2954 llvm::Value *V,
2955 unsigned ReverseWidth) {
2956 // MVE-specific helper function which reverses the elements of a
2957 // vector within every (ReverseWidth)-bit collection of lanes.
2958 SmallVector<int, 16> Indices;
2959 unsigned LaneSize = V->getType()->getScalarSizeInBits();
2960 unsigned Elements = 128 / LaneSize;
2961 unsigned Mask = ReverseWidth / LaneSize - 1;
2962 for (unsigned i = 0; i < Elements; i++)
2963 Indices.push_back(i ^ Mask);
2964 return Builder.CreateShuffleVector(V, Indices);
2965}
2966
2967static llvm::Value *ARMMVECreateSIToFP(CGBuilderTy &Builder,
2968 CodeGenFunction *CGF, llvm::Value *V,
2969 llvm::Type *Ty) {
2970 return Builder.CreateCall(
2971 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vcvt_fp_int, {Ty, V->getType()}),
2972 {V, llvm::ConstantInt::get(Builder.getInt32Ty(), 0)});
2973}
2974
2975static llvm::Value *ARMMVECreateUIToFP(CGBuilderTy &Builder,
2976 CodeGenFunction *CGF, llvm::Value *V,
2977 llvm::Type *Ty) {
2978 return Builder.CreateCall(
2979 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vcvt_fp_int, {Ty, V->getType()}),
2980 {V, llvm::ConstantInt::get(Builder.getInt32Ty(), 1)});
2981}
2982
2983static llvm::Value *ARMMVECreateFPToSI(CGBuilderTy &Builder,
2984 CodeGenFunction *CGF, llvm::Value *V,
2985 llvm::Type *Ty) {
2986 return Builder.CreateCall(
2987 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vcvt_int_fp, {Ty, V->getType()}),
2988 {V, llvm::ConstantInt::get(Builder.getInt32Ty(), 0)});
2989}
2990
2991static llvm::Value *ARMMVECreateFPToUI(CGBuilderTy &Builder,
2992 CodeGenFunction *CGF, llvm::Value *V,
2993 llvm::Type *Ty) {
2994 return Builder.CreateCall(
2995 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vcvt_int_fp, {Ty, V->getType()}),
2996 {V, llvm::ConstantInt::get(Builder.getInt32Ty(), 1)});
2997}
2998
2999Value *CodeGenFunction::EmitARMMVEBuiltinExpr(unsigned BuiltinID,
3000 const CallExpr *E,
3001 ReturnValueSlot ReturnValue,
3002 llvm::Triple::ArchType Arch) {
3003 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
3004 Intrinsic::ID IRIntr;
3005 unsigned NumVectors;
3006
3007 // Code autogenerated by Tablegen will handle all the simple builtins.
3008 switch (BuiltinID) {
3009 #include "clang/Basic/arm_mve_builtin_cg.inc"
3010
3011 // If we didn't match an MVE builtin id at all, go back to the
3012 // main EmitARMBuiltinExpr.
3013 default:
3014 return nullptr;
3015 }
3016
3017 // Anything that breaks from that switch is an MVE builtin that
3018 // needs handwritten code to generate.
3019
3020 switch (CustomCodeGenType) {
3021
3022 case CustomCodeGen::VLD24: {
3023 llvm::SmallVector<Value *, 4> Ops;
3024 llvm::SmallVector<llvm::Type *, 4> Tys;
3025
3026 auto MvecCType = E->getType();
3027 auto MvecLType = ConvertType(MvecCType);
3028 assert(MvecLType->isStructTy() &&
3029 "Return type for vld[24]q should be a struct");
3030 assert(MvecLType->getStructNumElements() == 1 &&
3031 "Return-type struct for vld[24]q should have one element");
3032 auto MvecLTypeInner = MvecLType->getStructElementType(0);
3033 assert(MvecLTypeInner->isArrayTy() &&
3034 "Return-type struct for vld[24]q should contain an array");
3035 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
3036 "Array member of return-type struct vld[24]q has wrong length");
3037 auto VecLType = MvecLTypeInner->getArrayElementType();
3038
3039 Tys.push_back(VecLType);
3040
3041 auto Addr = E->getArg(0);
3042 Ops.push_back(EmitScalarExpr(Addr));
3043 Tys.push_back(ConvertType(Addr->getType()));
3044
3045 Function *F = CGM.getIntrinsic(IRIntr, ArrayRef(Tys));
3046 Value *LoadResult = Builder.CreateCall(F, Ops);
3047 Value *MvecOut = PoisonValue::get(MvecLType);
3048 for (unsigned i = 0; i < NumVectors; ++i) {
3049 Value *Vec = Builder.CreateExtractValue(LoadResult, i);
3050 MvecOut = Builder.CreateInsertValue(MvecOut, Vec, {0, i});
3051 }
3052
3053 if (ReturnValue.isNull())
3054 return MvecOut;
3055 else
3056 return Builder.CreateStore(MvecOut, ReturnValue.getAddress());
3057 }
3058
3059 case CustomCodeGen::VST24: {
3060 llvm::SmallVector<Value *, 4> Ops;
3061 llvm::SmallVector<llvm::Type *, 4> Tys;
3062
3063 auto Addr = E->getArg(0);
3064 Ops.push_back(EmitScalarExpr(Addr));
3065 Tys.push_back(ConvertType(Addr->getType()));
3066
3067 auto MvecCType = E->getArg(1)->getType();
3068 auto MvecLType = ConvertType(MvecCType);
3069 assert(MvecLType->isStructTy() && "Data type for vst2q should be a struct");
3070 assert(MvecLType->getStructNumElements() == 1 &&
3071 "Data-type struct for vst2q should have one element");
3072 auto MvecLTypeInner = MvecLType->getStructElementType(0);
3073 assert(MvecLTypeInner->isArrayTy() &&
3074 "Data-type struct for vst2q should contain an array");
3075 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
3076 "Array member of return-type struct vld[24]q has wrong length");
3077 auto VecLType = MvecLTypeInner->getArrayElementType();
3078
3079 Tys.push_back(VecLType);
3080
3081 AggValueSlot MvecSlot = CreateAggTemp(MvecCType);
3082 EmitAggExpr(E->getArg(1), MvecSlot);
3083 auto Mvec = Builder.CreateLoad(MvecSlot.getAddress());
3084 for (unsigned i = 0; i < NumVectors; i++)
3085 Ops.push_back(Builder.CreateExtractValue(Mvec, {0, i}));
3086
3087 Function *F = CGM.getIntrinsic(IRIntr, ArrayRef(Tys));
3088 Value *ToReturn = nullptr;
3089 for (unsigned i = 0; i < NumVectors; i++) {
3090 Ops.push_back(llvm::ConstantInt::get(Int32Ty, i));
3091 ToReturn = Builder.CreateCall(F, Ops);
3092 Ops.pop_back();
3093 }
3094 return ToReturn;
3095 }
3096 }
3097 llvm_unreachable("unknown custom codegen type.");
3098}
3099
3100Value *CodeGenFunction::EmitARMCDEBuiltinExpr(unsigned BuiltinID,
3101 const CallExpr *E,
3102 ReturnValueSlot ReturnValue,
3103 llvm::Triple::ArchType Arch) {
3104 switch (BuiltinID) {
3105 default:
3106 return nullptr;
3107#include "clang/Basic/arm_cde_builtin_cg.inc"
3108 }
3109}
3110
3111static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID,
3112 const CallExpr *E,
3113 SmallVectorImpl<Value *> &Ops,
3114 llvm::Triple::ArchType Arch) {
3115 unsigned int Int = 0;
3116 const char *s = nullptr;
3117
3118 switch (BuiltinID) {
3119 default:
3120 return nullptr;
3121 case NEON::BI__builtin_neon_vtbl1_v:
3122 case NEON::BI__builtin_neon_vqtbl1_v:
3123 case NEON::BI__builtin_neon_vqtbl1q_v:
3124 case NEON::BI__builtin_neon_vtbl2_v:
3125 case NEON::BI__builtin_neon_vqtbl2_v:
3126 case NEON::BI__builtin_neon_vqtbl2q_v:
3127 case NEON::BI__builtin_neon_vtbl3_v:
3128 case NEON::BI__builtin_neon_vqtbl3_v:
3129 case NEON::BI__builtin_neon_vqtbl3q_v:
3130 case NEON::BI__builtin_neon_vtbl4_v:
3131 case NEON::BI__builtin_neon_vqtbl4_v:
3132 case NEON::BI__builtin_neon_vqtbl4q_v:
3133 break;
3134 case NEON::BI__builtin_neon_vtbx1_v:
3135 case NEON::BI__builtin_neon_vqtbx1_v:
3136 case NEON::BI__builtin_neon_vqtbx1q_v:
3137 case NEON::BI__builtin_neon_vtbx2_v:
3138 case NEON::BI__builtin_neon_vqtbx2_v:
3139 case NEON::BI__builtin_neon_vqtbx2q_v:
3140 case NEON::BI__builtin_neon_vtbx3_v:
3141 case NEON::BI__builtin_neon_vqtbx3_v:
3142 case NEON::BI__builtin_neon_vqtbx3q_v:
3143 case NEON::BI__builtin_neon_vtbx4_v:
3144 case NEON::BI__builtin_neon_vqtbx4_v:
3145 case NEON::BI__builtin_neon_vqtbx4q_v:
3146 break;
3147 }
3148
3149 assert(E->getNumArgs() >= 3);
3150
3151 // Get the last argument, which specifies the vector type.
3152 const Expr *Arg = E->getArg(E->getNumArgs() - 1);
3153 std::optional<llvm::APSInt> Result =
3154 Arg->getIntegerConstantExpr(CGF.getContext());
3155 if (!Result)
3156 return nullptr;
3157
3158 // Determine the type of this overloaded NEON intrinsic.
3159 NeonTypeFlags Type = Result->getZExtValue();
3160 llvm::FixedVectorType *Ty = GetNeonType(&CGF, Type);
3161 if (!Ty)
3162 return nullptr;
3163
3164 CodeGen::CGBuilderTy &Builder = CGF.Builder;
3165
3166 // AArch64 scalar builtins are not overloaded, they do not have an extra
3167 // argument that specifies the vector type, need to handle each case.
3168 switch (BuiltinID) {
3169 case NEON::BI__builtin_neon_vtbl1_v: {
3170 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 1), nullptr, Ops[1],
3171 Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1");
3172 }
3173 case NEON::BI__builtin_neon_vtbl2_v: {
3174 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 2), nullptr, Ops[2],
3175 Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1");
3176 }
3177 case NEON::BI__builtin_neon_vtbl3_v: {
3178 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 3), nullptr, Ops[3],
3179 Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2");
3180 }
3181 case NEON::BI__builtin_neon_vtbl4_v: {
3182 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 4), nullptr, Ops[4],
3183 Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2");
3184 }
3185 case NEON::BI__builtin_neon_vtbx1_v: {
3186 Value *TblRes =
3187 packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 1), nullptr, Ops[2], Ty,
3188 Intrinsic::aarch64_neon_tbl1, "vtbl1");
3189
3190 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
3191 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
3192 CmpRes = Builder.CreateSExt(CmpRes, Ty);
3193
3194 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
3195 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
3196 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
3197 }
3198 case NEON::BI__builtin_neon_vtbx2_v: {
3199 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 2), Ops[0], Ops[3],
3200 Ty, Intrinsic::aarch64_neon_tbx1, "vtbx1");
3201 }
3202 case NEON::BI__builtin_neon_vtbx3_v: {
3203 Value *TblRes =
3204 packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 3), nullptr, Ops[4], Ty,
3205 Intrinsic::aarch64_neon_tbl2, "vtbl2");
3206
3207 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
3208 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
3209 TwentyFourV);
3210 CmpRes = Builder.CreateSExt(CmpRes, Ty);
3211
3212 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
3213 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
3214 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
3215 }
3216 case NEON::BI__builtin_neon_vtbx4_v: {
3217 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 4), Ops[0], Ops[5],
3218 Ty, Intrinsic::aarch64_neon_tbx2, "vtbx2");
3219 }
3220 case NEON::BI__builtin_neon_vqtbl1_v:
3221 case NEON::BI__builtin_neon_vqtbl1q_v:
3222 Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break;
3223 case NEON::BI__builtin_neon_vqtbl2_v:
3224 case NEON::BI__builtin_neon_vqtbl2q_v: {
3225 Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break;
3226 case NEON::BI__builtin_neon_vqtbl3_v:
3227 case NEON::BI__builtin_neon_vqtbl3q_v:
3228 Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break;
3229 case NEON::BI__builtin_neon_vqtbl4_v:
3230 case NEON::BI__builtin_neon_vqtbl4q_v:
3231 Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break;
3232 case NEON::BI__builtin_neon_vqtbx1_v:
3233 case NEON::BI__builtin_neon_vqtbx1q_v:
3234 Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break;
3235 case NEON::BI__builtin_neon_vqtbx2_v:
3236 case NEON::BI__builtin_neon_vqtbx2q_v:
3237 Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break;
3238 case NEON::BI__builtin_neon_vqtbx3_v:
3239 case NEON::BI__builtin_neon_vqtbx3q_v:
3240 Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break;
3241 case NEON::BI__builtin_neon_vqtbx4_v:
3242 case NEON::BI__builtin_neon_vqtbx4q_v:
3243 Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break;
3244 }
3245 }
3246
3247 if (!Int)
3248 return nullptr;
3249
3250 Function *F = CGF.CGM.getIntrinsic(Int, Ty);
3251 return CGF.EmitNeonCall(F, Ops, s);
3252}
3253
3254Value *CodeGenFunction::vectorWrapScalar16(Value *Op) {
3255 auto *VTy = llvm::FixedVectorType::get(Int16Ty, 4);
3256 Op = Builder.CreateBitCast(Op, Int16Ty);
3257 Value *V = PoisonValue::get(VTy);
3258 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
3259 Op = Builder.CreateInsertElement(V, Op, CI);
3260 return Op;
3261}
3262
3263/// SVEBuiltinMemEltTy - Returns the memory element type for this memory
3264/// access builtin. Only required if it can't be inferred from the base pointer
3265/// operand.
3266llvm::Type *CodeGenFunction::SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags) {
3267 switch (TypeFlags.getMemEltType()) {
3268 case SVETypeFlags::MemEltTyDefault:
3269 return getEltType(TypeFlags);
3270 case SVETypeFlags::MemEltTyInt8:
3271 return Builder.getInt8Ty();
3272 case SVETypeFlags::MemEltTyInt16:
3273 return Builder.getInt16Ty();
3274 case SVETypeFlags::MemEltTyInt32:
3275 return Builder.getInt32Ty();
3276 case SVETypeFlags::MemEltTyInt64:
3277 return Builder.getInt64Ty();
3278 }
3279 llvm_unreachable("Unknown MemEltType");
3280}
3281
3282llvm::Type *CodeGenFunction::getEltType(const SVETypeFlags &TypeFlags) {
3283 switch (TypeFlags.getEltType()) {
3284 default:
3285 llvm_unreachable("Invalid SVETypeFlag!");
3286
3287 case SVETypeFlags::EltTyMFloat8:
3288 case SVETypeFlags::EltTyInt8:
3289 return Builder.getInt8Ty();
3290 case SVETypeFlags::EltTyInt16:
3291 return Builder.getInt16Ty();
3292 case SVETypeFlags::EltTyInt32:
3293 return Builder.getInt32Ty();
3294 case SVETypeFlags::EltTyInt64:
3295 return Builder.getInt64Ty();
3296 case SVETypeFlags::EltTyInt128:
3297 return Builder.getInt128Ty();
3298
3299 case SVETypeFlags::EltTyFloat16:
3300 return Builder.getHalfTy();
3301 case SVETypeFlags::EltTyFloat32:
3302 return Builder.getFloatTy();
3303 case SVETypeFlags::EltTyFloat64:
3304 return Builder.getDoubleTy();
3305
3306 case SVETypeFlags::EltTyBFloat16:
3307 return Builder.getBFloatTy();
3308
3309 case SVETypeFlags::EltTyBool8:
3310 case SVETypeFlags::EltTyBool16:
3311 case SVETypeFlags::EltTyBool32:
3312 case SVETypeFlags::EltTyBool64:
3313 return Builder.getInt1Ty();
3314 }
3315}
3316
3317// Return the llvm predicate vector type corresponding to the specified element
3318// TypeFlags.
3319llvm::ScalableVectorType *
3320CodeGenFunction::getSVEPredType(const SVETypeFlags &TypeFlags) {
3321 switch (TypeFlags.getEltType()) {
3322 default: llvm_unreachable("Unhandled SVETypeFlag!");
3323
3324 case SVETypeFlags::EltTyInt8:
3325 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
3326 case SVETypeFlags::EltTyInt16:
3327 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3328 case SVETypeFlags::EltTyInt32:
3329 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
3330 case SVETypeFlags::EltTyInt64:
3331 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
3332
3333 case SVETypeFlags::EltTyBFloat16:
3334 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3335 case SVETypeFlags::EltTyFloat16:
3336 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3337 case SVETypeFlags::EltTyFloat32:
3338 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
3339 case SVETypeFlags::EltTyFloat64:
3340 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
3341
3342 case SVETypeFlags::EltTyBool8:
3343 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
3344 case SVETypeFlags::EltTyBool16:
3345 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3346 case SVETypeFlags::EltTyBool32:
3347 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
3348 case SVETypeFlags::EltTyBool64:
3349 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
3350 }
3351}
3352
3353// Return the llvm vector type corresponding to the specified element TypeFlags.
3354llvm::ScalableVectorType *
3355CodeGenFunction::getSVEType(const SVETypeFlags &TypeFlags) {
3356 switch (TypeFlags.getEltType()) {
3357 default:
3358 llvm_unreachable("Invalid SVETypeFlag!");
3359
3360 case SVETypeFlags::EltTyInt8:
3361 return llvm::ScalableVectorType::get(Builder.getInt8Ty(), 16);
3362 case SVETypeFlags::EltTyInt16:
3363 return llvm::ScalableVectorType::get(Builder.getInt16Ty(), 8);
3364 case SVETypeFlags::EltTyInt32:
3365 return llvm::ScalableVectorType::get(Builder.getInt32Ty(), 4);
3366 case SVETypeFlags::EltTyInt64:
3367 return llvm::ScalableVectorType::get(Builder.getInt64Ty(), 2);
3368
3369 case SVETypeFlags::EltTyMFloat8:
3370 return llvm::ScalableVectorType::get(Builder.getInt8Ty(), 16);
3371 case SVETypeFlags::EltTyFloat16:
3372 return llvm::ScalableVectorType::get(Builder.getHalfTy(), 8);
3373 case SVETypeFlags::EltTyBFloat16:
3374 return llvm::ScalableVectorType::get(Builder.getBFloatTy(), 8);
3375 case SVETypeFlags::EltTyFloat32:
3376 return llvm::ScalableVectorType::get(Builder.getFloatTy(), 4);
3377 case SVETypeFlags::EltTyFloat64:
3378 return llvm::ScalableVectorType::get(Builder.getDoubleTy(), 2);
3379
3380 case SVETypeFlags::EltTyBool8:
3381 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
3382 case SVETypeFlags::EltTyBool16:
3383 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3384 case SVETypeFlags::EltTyBool32:
3385 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
3386 case SVETypeFlags::EltTyBool64:
3387 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
3388 }
3389}
3390
3391constexpr unsigned SVEBitsPerBlock = 128;
3392
3393static llvm::ScalableVectorType *getSVEVectorForElementType(llvm::Type *EltTy) {
3394 unsigned NumElts = SVEBitsPerBlock / EltTy->getScalarSizeInBits();
3395 return llvm::ScalableVectorType::get(EltTy, NumElts);
3396}
3397
3398// Reinterpret the input predicate so that it can be used to correctly isolate
3399// the elements of the specified datatype.
3400Value *CodeGenFunction::EmitSVEPredicateCast(Value *Pred,
3401 llvm::ScalableVectorType *VTy) {
3402
3403 if (isa<TargetExtType>(Pred->getType()) &&
3404 cast<TargetExtType>(Pred->getType())->getName() == "aarch64.svcount")
3405 return Pred;
3406
3407 auto *RTy = llvm::VectorType::get(IntegerType::get(getLLVMContext(), 1), VTy);
3408 if (Pred->getType() == RTy)
3409 return Pred;
3410
3411 unsigned IntID;
3412 llvm::Type *IntrinsicTy;
3413 switch (VTy->getMinNumElements()) {
3414 default:
3415 llvm_unreachable("unsupported element count!");
3416 case 1:
3417 case 2:
3418 case 4:
3419 case 8:
3420 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
3421 IntrinsicTy = RTy;
3422 break;
3423 case 16:
3424 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
3425 IntrinsicTy = Pred->getType();
3426 break;
3427 }
3428
3429 Function *F = CGM.getIntrinsic(IntID, IntrinsicTy);
3430 Value *C = Builder.CreateCall(F, Pred);
3431 assert(C->getType() == RTy && "Unexpected return type!");
3432 return C;
3433}
3434
3435Value *CodeGenFunction::EmitSVEPredicateTupleCast(Value *PredTuple,
3436 llvm::StructType *Ty) {
3437 if (PredTuple->getType() == Ty)
3438 return PredTuple;
3439
3440 Value *Ret = llvm::PoisonValue::get(Ty);
3441 for (unsigned I = 0; I < Ty->getNumElements(); ++I) {
3442 Value *Pred = Builder.CreateExtractValue(PredTuple, I);
3443 Pred = EmitSVEPredicateCast(
3444 Pred, cast<llvm::ScalableVectorType>(Ty->getTypeAtIndex(I)));
3445 Ret = Builder.CreateInsertValue(Ret, Pred, I);
3446 }
3447
3448 return Ret;
3449}
3450
3451Value *CodeGenFunction::EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,
3452 SmallVectorImpl<Value *> &Ops,
3453 unsigned IntID) {
3454 auto *ResultTy = getSVEType(TypeFlags);
3455 auto *OverloadedTy =
3456 llvm::ScalableVectorType::get(SVEBuiltinMemEltTy(TypeFlags), ResultTy);
3457 Function *F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[1]->getType()});
3458
3459 // At the ACLE level there's only one predicate type, svbool_t, which is
3460 // mapped to <n x 16 x i1>. However, this might be incompatible with the
3461 // actual type being loaded. For example, when loading doubles (i64) the
3462 // predicate should be <n x 2 x i1> instead. At the IR level the type of
3463 // the predicate and the data being loaded must match. Cast to the type
3464 // expected by the intrinsic. The intrinsic itself should be defined in
3465 // a way than enforces relations between parameter types.
3466 Ops[0] = EmitSVEPredicateCast(
3467 Ops[0], cast<llvm::ScalableVectorType>(F->getArg(0)->getType()));
3468
3469 // Pass 0 when the offset is missing. This can only be applied when using
3470 // the "vector base" addressing mode for which ACLE allows no offset. The
3471 // corresponding LLVM IR always requires an offset.
3472 if (Ops.size() == 2) {
3473 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
3474 Ops.push_back(ConstantInt::get(Int64Ty, 0));
3475 }
3476
3477 // For "vector base, scalar index" scale the index so that it becomes a
3478 // scalar offset.
3479 if (!TypeFlags.isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
3480 unsigned BytesPerElt =
3481 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
3482 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
3483 }
3484
3485 Value *Call = Builder.CreateCall(F, Ops);
3486
3487 // The following sext/zext is only needed when ResultTy != OverloadedTy. In
3488 // other cases it's folded into a nop.
3489 return TypeFlags.isZExtReturn() ? Builder.CreateZExt(Call, ResultTy)
3490 : Builder.CreateSExt(Call, ResultTy);
3491}
3492
3493Value *CodeGenFunction::EmitSVEScatterStore(const SVETypeFlags &TypeFlags,
3494 SmallVectorImpl<Value *> &Ops,
3495 unsigned IntID) {
3496 auto *SrcDataTy = getSVEType(TypeFlags);
3497 auto *OverloadedTy =
3498 llvm::ScalableVectorType::get(SVEBuiltinMemEltTy(TypeFlags), SrcDataTy);
3499
3500 // In ACLE the source data is passed in the last argument, whereas in LLVM IR
3501 // it's the first argument. Move it accordingly.
3502 Ops.insert(Ops.begin(), Ops.pop_back_val());
3503
3504 Function *F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[2]->getType()});
3505
3506 // Pass 0 when the offset is missing. This can only be applied when using
3507 // the "vector base" addressing mode for which ACLE allows no offset. The
3508 // corresponding LLVM IR always requires an offset.
3509 if (Ops.size() == 3) {
3510 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
3511 Ops.push_back(ConstantInt::get(Int64Ty, 0));
3512 }
3513
3514 // Truncation is needed when SrcDataTy != OverloadedTy. In other cases it's
3515 // folded into a nop.
3516 Ops[0] = Builder.CreateTrunc(Ops[0], OverloadedTy);
3517
3518 // At the ACLE level there's only one predicate type, svbool_t, which is
3519 // mapped to <n x 16 x i1>. However, this might be incompatible with the
3520 // actual type being stored. For example, when storing doubles (i64) the
3521 // predicated should be <n x 2 x i1> instead. At the IR level the type of
3522 // the predicate and the data being stored must match. Cast to the type
3523 // expected by the intrinsic. The intrinsic itself should be defined in
3524 // a way that enforces relations between parameter types.
3525 Ops[1] = EmitSVEPredicateCast(
3526 Ops[1], cast<llvm::ScalableVectorType>(F->getArg(1)->getType()));
3527
3528 // For "vector base, scalar index" scale the index so that it becomes a
3529 // scalar offset.
3530 if (!TypeFlags.isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
3531 unsigned BytesPerElt =
3532 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
3533 Ops[3] = Builder.CreateShl(Ops[3], Log2_32(BytesPerElt));
3534 }
3535
3536 return Builder.CreateCall(F, Ops);
3537}
3538
3539Value *CodeGenFunction::EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
3540 SmallVectorImpl<Value *> &Ops,
3541 unsigned IntID) {
3542 // The gather prefetches are overloaded on the vector input - this can either
3543 // be the vector of base addresses or vector of offsets.
3544 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType());
3545 if (!OverloadedTy)
3546 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType());
3547
3548 // Cast the predicate from svbool_t to the right number of elements.
3549 Ops[0] = EmitSVEPredicateCast(Ops[0], OverloadedTy);
3550
3551 // vector + imm addressing modes
3552 if (Ops[1]->getType()->isVectorTy()) {
3553 if (Ops.size() == 3) {
3554 // Pass 0 for 'vector+imm' when the index is omitted.
3555 Ops.push_back(ConstantInt::get(Int64Ty, 0));
3556
3557 // The sv_prfop is the last operand in the builtin and IR intrinsic.
3558 std::swap(Ops[2], Ops[3]);
3559 } else {
3560 // Index needs to be passed as scaled offset.
3561 llvm::Type *MemEltTy = SVEBuiltinMemEltTy(TypeFlags);
3562 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
3563 if (BytesPerElt > 1)
3564 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
3565 }
3566
3567 Function *F = CGM.getIntrinsic(IntID, OverloadedTy);
3568 return Builder.CreateCall(F, Ops);
3569 }
3570
3571 Function *F = CGM.getIntrinsic(IntID, {Ops[1]->getType(), OverloadedTy});
3572 return Builder.CreateCall(F, Ops);
3573}
3574
3575Value *CodeGenFunction::EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
3576 SmallVectorImpl<Value*> &Ops,
3577 unsigned IntID) {
3578 llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
3579 Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy);
3580 Value *BasePtr = Ops[1];
3581
3582 // Does the load have an offset?
3583 if (Ops.size() > 2)
3584 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]);
3585
3586 Function *F = CGM.getIntrinsic(IntID, {VTy, BasePtr->getType()});
3587 return Builder.CreateCall(F, {Predicate, BasePtr});
3588}
3589
3590Value *CodeGenFunction::EmitSVEStructStore(const SVETypeFlags &TypeFlags,
3591 SmallVectorImpl<Value*> &Ops,
3592 unsigned IntID) {
3593 llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
3594
3595 unsigned N;
3596 switch (IntID) {
3597 case Intrinsic::aarch64_sve_st2:
3598 case Intrinsic::aarch64_sve_st1_pn_x2:
3599 case Intrinsic::aarch64_sve_stnt1_pn_x2:
3600 case Intrinsic::aarch64_sve_st2q:
3601 N = 2;
3602 break;
3603 case Intrinsic::aarch64_sve_st3:
3604 case Intrinsic::aarch64_sve_st3q:
3605 N = 3;
3606 break;
3607 case Intrinsic::aarch64_sve_st4:
3608 case Intrinsic::aarch64_sve_st1_pn_x4:
3609 case Intrinsic::aarch64_sve_stnt1_pn_x4:
3610 case Intrinsic::aarch64_sve_st4q:
3611 N = 4;
3612 break;
3613 default:
3614 llvm_unreachable("unknown intrinsic!");
3615 }
3616
3617 Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy);
3618 Value *BasePtr = Ops[1];
3619
3620 // Does the store have an offset?
3621 if (Ops.size() > (2 + N))
3622 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]);
3623
3624 // The llvm.aarch64.sve.st2/3/4 intrinsics take legal part vectors, so we
3625 // need to break up the tuple vector.
3626 SmallVector<llvm::Value*, 5> Operands;
3627 for (unsigned I = Ops.size() - N; I < Ops.size(); ++I)
3628 Operands.push_back(Ops[I]);
3629 Operands.append({Predicate, BasePtr});
3630 Function *F = CGM.getIntrinsic(IntID, {VTy, BasePtr->getType()});
3631
3632 return Builder.CreateCall(F, Operands);
3633}
3634
3635// SVE2's svpmullb and svpmullt builtins are similar to the svpmullb_pair and
3636// svpmullt_pair intrinsics, with the exception that their results are bitcast
3637// to a wider type.
3638Value *CodeGenFunction::EmitSVEPMull(const SVETypeFlags &TypeFlags,
3639 SmallVectorImpl<Value *> &Ops,
3640 unsigned BuiltinID) {
3641 // Splat scalar operand to vector (intrinsics with _n infix)
3642 if (TypeFlags.hasSplatOperand()) {
3643 unsigned OpNo = TypeFlags.getSplatOperand();
3644 Ops[OpNo] = EmitSVEDupX(Ops[OpNo]);
3645 }
3646
3647 // The pair-wise function has a narrower overloaded type.
3648 Function *F = CGM.getIntrinsic(BuiltinID, Ops[0]->getType());
3649 Value *Call = Builder.CreateCall(F, {Ops[0], Ops[1]});
3650
3651 // Now bitcast to the wider result type.
3652 llvm::ScalableVectorType *Ty = getSVEType(TypeFlags);
3653 return EmitSVEReinterpret(Call, Ty);
3654}
3655
3656Value *CodeGenFunction::EmitSVEMovl(const SVETypeFlags &TypeFlags,
3657 ArrayRef<Value *> Ops, unsigned BuiltinID) {
3658 llvm::Type *OverloadedTy = getSVEType(TypeFlags);
3659 Function *F = CGM.getIntrinsic(BuiltinID, OverloadedTy);
3660 return Builder.CreateCall(F, {Ops[0], Builder.getInt32(0)});
3661}
3662
3663Value *CodeGenFunction::EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
3664 SmallVectorImpl<Value *> &Ops,
3665 unsigned BuiltinID) {
3666 auto *MemEltTy = SVEBuiltinMemEltTy(TypeFlags);
3667 auto *VectorTy = getSVEVectorForElementType(MemEltTy);
3668 auto *MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
3669
3670 Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy);
3671 Value *BasePtr = Ops[1];
3672
3673 // Implement the index operand if not omitted.
3674 if (Ops.size() > 3)
3675 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
3676
3677 Value *PrfOp = Ops.back();
3678
3679 llvm::Type *Tys[2] = {Predicate->getType(), BasePtr->getType()};
3680 Function *F = CGM.getIntrinsic(BuiltinID, Tys);
3681 return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
3682}
3683
3684Value *CodeGenFunction::EmitSVEMaskedLoad(const CallExpr *E,
3685 llvm::Type *ReturnTy,
3686 SmallVectorImpl<Value *> &Ops,
3687 unsigned IntrinsicID,
3688 bool IsZExtReturn) {
3689 QualType LangPTy = E->getArg(1)->getType();
3690 llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
3691 LangPTy->castAs<PointerType>()->getPointeeType());
3692
3693 // Mfloat8 types is stored as a vector, so extra work
3694 // to extract sclar element type is necessary.
3695 if (MemEltTy->isVectorTy()) {
3696 assert(MemEltTy == FixedVectorType::get(Int8Ty, 1) &&
3697 "Only <1 x i8> expected");
3698 MemEltTy = cast<llvm::VectorType>(MemEltTy)->getElementType();
3699 }
3700
3701 // The vector type that is returned may be different from the
3702 // eventual type loaded from memory.
3703 auto VectorTy = cast<llvm::ScalableVectorType>(ReturnTy);
3704 llvm::ScalableVectorType *MemoryTy = nullptr;
3705 llvm::ScalableVectorType *PredTy = nullptr;
3706 bool IsQuadLoad = false;
3707 switch (IntrinsicID) {
3708 case Intrinsic::aarch64_sve_ld1uwq:
3709 case Intrinsic::aarch64_sve_ld1udq:
3710 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
3711 PredTy = llvm::ScalableVectorType::get(
3712 llvm::Type::getInt1Ty(getLLVMContext()), 1);
3713 IsQuadLoad = true;
3714 break;
3715 default:
3716 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
3717 PredTy = MemoryTy;
3718 break;
3719 }
3720
3721 Value *Predicate = EmitSVEPredicateCast(Ops[0], PredTy);
3722 Value *BasePtr = Ops[1];
3723
3724 // Does the load have an offset?
3725 if (Ops.size() > 2)
3726 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
3727
3728 llvm::Type *Tys[2] = {IsQuadLoad ? VectorTy : MemoryTy, BasePtr->getType()};
3729 Function *F = CGM.getIntrinsic(IntrinsicID, Tys);
3730 auto *Load = Builder.CreateCall(F, {Predicate, BasePtr});
3731 auto TBAAInfo = CGM.getTBAAAccessInfo(LangPTy->getPointeeType());
3732 CGM.DecorateInstructionWithTBAA(Load, TBAAInfo);
3733
3734 if (IsQuadLoad)
3735 return Load;
3736
3737 return IsZExtReturn ? Builder.CreateZExt(Load, VectorTy)
3738 : Builder.CreateSExt(Load, VectorTy);
3739}
3740
3741Value *CodeGenFunction::EmitSVEMaskedStore(const CallExpr *E,
3742 SmallVectorImpl<Value *> &Ops,
3743 unsigned IntrinsicID) {
3744 QualType LangPTy = E->getArg(1)->getType();
3745 llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
3746 LangPTy->castAs<PointerType>()->getPointeeType());
3747
3748 // Mfloat8 types is stored as a vector, so extra work
3749 // to extract sclar element type is necessary.
3750 if (MemEltTy->isVectorTy()) {
3751 assert(MemEltTy == FixedVectorType::get(Int8Ty, 1) &&
3752 "Only <1 x i8> expected");
3753 MemEltTy = cast<llvm::VectorType>(MemEltTy)->getElementType();
3754 }
3755
3756 // The vector type that is stored may be different from the
3757 // eventual type stored to memory.
3758 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType());
3759 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
3760
3761 auto PredTy = MemoryTy;
3762 auto AddrMemoryTy = MemoryTy;
3763 bool IsQuadStore = false;
3764
3765 switch (IntrinsicID) {
3766 case Intrinsic::aarch64_sve_st1wq:
3767 case Intrinsic::aarch64_sve_st1dq:
3768 AddrMemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
3769 PredTy =
3770 llvm::ScalableVectorType::get(IntegerType::get(getLLVMContext(), 1), 1);
3771 IsQuadStore = true;
3772 break;
3773 default:
3774 break;
3775 }
3776 Value *Predicate = EmitSVEPredicateCast(Ops[0], PredTy);
3777 Value *BasePtr = Ops[1];
3778
3779 // Does the store have an offset?
3780 if (Ops.size() == 4)
3781 BasePtr = Builder.CreateGEP(AddrMemoryTy, BasePtr, Ops[2]);
3782
3783 // Last value is always the data
3784 Value *Val =
3785 IsQuadStore ? Ops.back() : Builder.CreateTrunc(Ops.back(), MemoryTy);
3786
3787 llvm::Type *Tys[2] = {IsQuadStore ? VectorTy : MemoryTy, BasePtr->getType()};
3788 Function *F = CGM.getIntrinsic(IntrinsicID, Tys);
3789 auto *Store = Builder.CreateCall(F, {Val, Predicate, BasePtr});
3790 auto TBAAInfo = CGM.getTBAAAccessInfo(LangPTy->getPointeeType());
3791 CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
3792 return Store;
3793}
3794
3795Value *CodeGenFunction::EmitSMELd1St1(const SVETypeFlags &TypeFlags,
3796 SmallVectorImpl<Value *> &Ops,
3797 unsigned IntID) {
3798 Ops[2] = EmitSVEPredicateCast(
3799 Ops[2], getSVEVectorForElementType(SVEBuiltinMemEltTy(TypeFlags)));
3800
3801 SmallVector<Value *> NewOps;
3802 NewOps.push_back(Ops[2]);
3803
3804 llvm::Value *BasePtr = Ops[3];
3805 llvm::Value *RealSlice = Ops[1];
3806 // If the intrinsic contains the vnum parameter, multiply it with the vector
3807 // size in bytes.
3808 if (Ops.size() == 5) {
3809 Function *StreamingVectorLength =
3810 CGM.getIntrinsic(Intrinsic::aarch64_sme_cntsd);
3811 llvm::Value *StreamingVectorLengthCall =
3812 Builder.CreateMul(Builder.CreateCall(StreamingVectorLength),
3813 llvm::ConstantInt::get(Int64Ty, 8), "svl",
3814 /* HasNUW */ true, /* HasNSW */ true);
3815 llvm::Value *Mulvl =
3816 Builder.CreateMul(StreamingVectorLengthCall, Ops[4], "mulvl");
3817 // The type of the ptr parameter is void *, so use Int8Ty here.
3818 BasePtr = Builder.CreateGEP(Int8Ty, Ops[3], Mulvl);
3819 RealSlice = Builder.CreateZExt(RealSlice, Int64Ty);
3820 RealSlice = Builder.CreateAdd(RealSlice, Ops[4]);
3821 RealSlice = Builder.CreateTrunc(RealSlice, Int32Ty);
3822 }
3823 NewOps.push_back(BasePtr);
3824 NewOps.push_back(Ops[0]);
3825 NewOps.push_back(RealSlice);
3826 Function *F = CGM.getIntrinsic(IntID, BasePtr->getType());
3827 return Builder.CreateCall(F, NewOps);
3828}
3829
3830Value *CodeGenFunction::EmitSMEReadWrite(const SVETypeFlags &TypeFlags,
3831 SmallVectorImpl<Value *> &Ops,
3832 unsigned IntID) {
3833 auto *VecTy = getSVEType(TypeFlags);
3834 Function *F = CGM.getIntrinsic(IntID, VecTy);
3835 if (TypeFlags.isReadZA())
3836 Ops[1] = EmitSVEPredicateCast(Ops[1], VecTy);
3837 else if (TypeFlags.isWriteZA())
3838 Ops[2] = EmitSVEPredicateCast(Ops[2], VecTy);
3839 return Builder.CreateCall(F, Ops);
3840}
3841
3842Value *CodeGenFunction::EmitSMEZero(const SVETypeFlags &TypeFlags,
3843 SmallVectorImpl<Value *> &Ops,
3844 unsigned IntID) {
3845 // svzero_za() intrinsic zeros the entire za tile and has no paramters.
3846 if (Ops.size() == 0)
3847 Ops.push_back(llvm::ConstantInt::get(Int32Ty, 255));
3848 Function *F = CGM.getIntrinsic(IntID, {});
3849 return Builder.CreateCall(F, Ops);
3850}
3851
3852Value *CodeGenFunction::EmitSMELdrStr(const SVETypeFlags &TypeFlags,
3853 SmallVectorImpl<Value *> &Ops,
3854 unsigned IntID) {
3855 if (Ops.size() == 2)
3856 Ops.push_back(Builder.getInt32(0));
3857 else
3858 Ops[2] = Builder.CreateIntCast(Ops[2], Int32Ty, true);
3859 Function *F = CGM.getIntrinsic(IntID, Ops[1]->getType());
3860 return Builder.CreateCall(F, Ops);
3861}
3862
3863// Limit the usage of scalable llvm IR generated by the ACLE by using the
3864// sve dup.x intrinsic instead of IRBuilder::CreateVectorSplat.
3865Value *CodeGenFunction::EmitSVEDupX(Value *Scalar, llvm::Type *Ty) {
3866 return Builder.CreateVectorSplat(
3867 cast<llvm::VectorType>(Ty)->getElementCount(), Scalar);
3868}
3869
3870Value *CodeGenFunction::EmitSVEDupX(Value *Scalar) {
3871 if (auto *Ty = Scalar->getType(); Ty->isVectorTy()) {
3872#ifndef NDEBUG
3873 auto *VecTy = cast<llvm::VectorType>(Ty);
3874 ElementCount EC = VecTy->getElementCount();
3875 assert(EC.isScalar() && VecTy->getElementType() == Int8Ty &&
3876 "Only <1 x i8> expected");
3877#endif
3878 Scalar = Builder.CreateExtractElement(Scalar, uint64_t(0));
3879 }
3880 return EmitSVEDupX(Scalar, getSVEVectorForElementType(Scalar->getType()));
3881}
3882
3883Value *CodeGenFunction::EmitSVEReinterpret(Value *Val, llvm::Type *Ty) {
3884 // FIXME: For big endian this needs an additional REV, or needs a separate
3885 // intrinsic that is code-generated as a no-op, because the LLVM bitcast
3886 // instruction is defined as 'bitwise' equivalent from memory point of
3887 // view (when storing/reloading), whereas the svreinterpret builtin
3888 // implements bitwise equivalent cast from register point of view.
3889 // LLVM CodeGen for a bitcast must add an explicit REV for big-endian.
3890
3891 if (auto *StructTy = dyn_cast<StructType>(Ty)) {
3892 Value *Tuple = llvm::PoisonValue::get(Ty);
3893
3894 for (unsigned I = 0; I < StructTy->getNumElements(); ++I) {
3895 Value *In = Builder.CreateExtractValue(Val, I);
3896 Value *Out = Builder.CreateBitCast(In, StructTy->getTypeAtIndex(I));
3897 Tuple = Builder.CreateInsertValue(Tuple, Out, I);
3898 }
3899
3900 return Tuple;
3901 }
3902
3903 return Builder.CreateBitCast(Val, Ty);
3904}
3905
3906static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty,
3907 SmallVectorImpl<Value *> &Ops) {
3908 auto *SplatZero = Constant::getNullValue(Ty);
3909 Ops.insert(Ops.begin(), SplatZero);
3910}
3911
3912static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty,
3913 SmallVectorImpl<Value *> &Ops) {
3914 auto *SplatUndef = UndefValue::get(Ty);
3915 Ops.insert(Ops.begin(), SplatUndef);
3916}
3917
3918SmallVector<llvm::Type *, 2>
3919CodeGenFunction::getSVEOverloadTypes(const SVETypeFlags &TypeFlags,
3920 llvm::Type *ResultType,
3921 ArrayRef<Value *> Ops) {
3922 if (TypeFlags.isOverloadNone())
3923 return {};
3924
3925 llvm::Type *DefaultType = getSVEType(TypeFlags);
3926
3927 if (TypeFlags.isOverloadWhileOrMultiVecCvt())
3928 return {DefaultType, Ops[1]->getType()};
3929
3930 if (TypeFlags.isOverloadWhileRW())
3931 return {getSVEPredType(TypeFlags), Ops[0]->getType()};
3932
3933 if (TypeFlags.isOverloadDefaultAndOp0())
3934 return {DefaultType, Ops[0]->getType()};
3935
3936 if (TypeFlags.isOverloadFirstandLast())
3937 return {Ops[0]->getType(), Ops.back()->getType()};
3938
3939 if (TypeFlags.isReductionQV())
3940 return {ResultType, Ops[1]->getType()};
3941
3942 assert(TypeFlags.isOverloadDefault() && "Unexpected value for overloads");
3943 return {DefaultType};
3944}
3945
3946Value *CodeGenFunction::EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,
3947 ArrayRef<Value *> Ops) {
3948 assert((TypeFlags.isTupleSet() || TypeFlags.isTupleGet()) &&
3949 "Expects TypleFlags.isTupleSet() or TypeFlags.isTupleGet()");
3950 unsigned Idx = cast<ConstantInt>(Ops[1])->getZExtValue();
3951
3952 if (TypeFlags.isTupleSet())
3953 return Builder.CreateInsertValue(Ops[0], Ops[2], Idx);
3954 return Builder.CreateExtractValue(Ops[0], Idx);
3955}
3956
3957Value *CodeGenFunction::EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
3958 llvm::Type *Ty,
3959 ArrayRef<Value *> Ops) {
3960 assert(TypeFlags.isTupleCreate() && "Expects TypleFlag isTupleCreate");
3961
3962 Value *Tuple = llvm::PoisonValue::get(Ty);
3963 for (unsigned Idx = 0; Idx < Ops.size(); Idx++)
3964 Tuple = Builder.CreateInsertValue(Tuple, Ops[Idx], Idx);
3965
3966 return Tuple;
3967}
3968
3969void CodeGenFunction::GetAArch64SVEProcessedOperands(
3970 unsigned BuiltinID, const CallExpr *E, SmallVectorImpl<Value *> &Ops,
3971 SVETypeFlags TypeFlags) {
3972 // Find out if any arguments are required to be integer constant expressions.
3973 unsigned ICEArguments = 0;
3974 ASTContext::GetBuiltinTypeError Error;
3975 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
3976 assert(Error == ASTContext::GE_None && "Should not codegen an error");
3977
3978 // Tuple set/get only requires one insert/extract vector, which is
3979 // created by EmitSVETupleSetOrGet.
3980 bool IsTupleGetOrSet = TypeFlags.isTupleSet() || TypeFlags.isTupleGet();
3981
3982 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
3983 bool IsICE = ICEArguments & (1 << i);
3984 Value *Arg = EmitScalarExpr(E->getArg(i));
3985
3986 if (IsICE) {
3987 // If this is required to be a constant, constant fold it so that we know
3988 // that the generated intrinsic gets a ConstantInt.
3989 std::optional<llvm::APSInt> Result =
3990 E->getArg(i)->getIntegerConstantExpr(getContext());
3991 assert(Result && "Expected argument to be a constant");
3992
3993 // Immediates for SVE llvm intrinsics are always 32bit. We can safely
3994 // truncate because the immediate has been range checked and no valid
3995 // immediate requires more than a handful of bits.
3996 *Result = Result->extOrTrunc(32);
3997 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), *Result));
3998 continue;
3999 }
4000
4001 if (isa<StructType>(Arg->getType()) && !IsTupleGetOrSet) {
4002 for (unsigned I = 0; I < Arg->getType()->getStructNumElements(); ++I)
4003 Ops.push_back(Builder.CreateExtractValue(Arg, I));
4004
4005 continue;
4006 }
4007
4008 Ops.push_back(Arg);
4009 }
4010}
4011
4012Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
4013 const CallExpr *E) {
4014 llvm::Type *Ty = ConvertType(E->getType());
4015 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
4016 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
4017 Value *Val = EmitScalarExpr(E->getArg(0));
4018 return EmitSVEReinterpret(Val, Ty);
4019 }
4020
4021 auto *Builtin =
4022 findARMVectorIntrinsicInMap(ArrayRef(AArch64SVEIntrinsicMap), BuiltinID,
4023 AArch64SVEIntrinsicsProvenSorted);
4024
4025 llvm::SmallVector<Value *, 4> Ops;
4026 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4027 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags);
4028
4029 if (TypeFlags.isLoad())
4030 return EmitSVEMaskedLoad(E, Ty, Ops, Builtin->LLVMIntrinsic,
4031 TypeFlags.isZExtReturn());
4032 if (TypeFlags.isStore())
4033 return EmitSVEMaskedStore(E, Ops, Builtin->LLVMIntrinsic);
4034 if (TypeFlags.isGatherLoad())
4035 return EmitSVEGatherLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4036 if (TypeFlags.isScatterStore())
4037 return EmitSVEScatterStore(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4038 if (TypeFlags.isPrefetch())
4039 return EmitSVEPrefetchLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4040 if (TypeFlags.isGatherPrefetch())
4041 return EmitSVEGatherPrefetch(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4042 if (TypeFlags.isStructLoad())
4043 return EmitSVEStructLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4044 if (TypeFlags.isStructStore())
4045 return EmitSVEStructStore(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4046 if (TypeFlags.isTupleSet() || TypeFlags.isTupleGet())
4047 return EmitSVETupleSetOrGet(TypeFlags, Ops);
4048 if (TypeFlags.isTupleCreate())
4049 return EmitSVETupleCreate(TypeFlags, Ty, Ops);
4050 if (TypeFlags.isUndef())
4051 return UndefValue::get(Ty);
4052
4053 // Handle built-ins for which there is a corresponding LLVM Intrinsic.
4054 // -------------------------------------------------------------------
4055 if (Builtin->LLVMIntrinsic != 0) {
4056 // Emit set FPMR for intrinsics that require it
4057 if (TypeFlags.setsFPMR())
4058 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_set_fpmr),
4059 Ops.pop_back_val());
4060 if (TypeFlags.getMergeType() == SVETypeFlags::MergeZeroExp)
4061 InsertExplicitZeroOperand(Builder, Ty, Ops);
4062
4063 if (TypeFlags.getMergeType() == SVETypeFlags::MergeAnyExp)
4064 InsertExplicitUndefOperand(Builder, Ty, Ops);
4065
4066 // Some ACLE builtins leave out the argument to specify the predicate
4067 // pattern, which is expected to be expanded to an SV_ALL pattern.
4068 if (TypeFlags.isAppendSVALL())
4069 Ops.push_back(Builder.getInt32(/*SV_ALL*/ 31));
4070 if (TypeFlags.isInsertOp1SVALL())
4071 Ops.insert(&Ops[1], Builder.getInt32(/*SV_ALL*/ 31));
4072
4073 // Predicates must match the main datatype.
4074 for (Value *&Op : Ops)
4075 if (auto PredTy = dyn_cast<llvm::VectorType>(Op->getType()))
4076 if (PredTy->getElementType()->isIntegerTy(1))
4077 Op = EmitSVEPredicateCast(Op, getSVEType(TypeFlags));
4078
4079 // Splat scalar operand to vector (intrinsics with _n infix)
4080 if (TypeFlags.hasSplatOperand()) {
4081 unsigned OpNo = TypeFlags.getSplatOperand();
4082 Ops[OpNo] = EmitSVEDupX(Ops[OpNo]);
4083 }
4084
4085 if (TypeFlags.isReverseCompare())
4086 std::swap(Ops[1], Ops[2]);
4087 else if (TypeFlags.isReverseUSDOT())
4088 std::swap(Ops[1], Ops[2]);
4089 else if (TypeFlags.isReverseMergeAnyBinOp() &&
4090 TypeFlags.getMergeType() == SVETypeFlags::MergeAny)
4091 std::swap(Ops[1], Ops[2]);
4092 else if (TypeFlags.isReverseMergeAnyAccOp() &&
4093 TypeFlags.getMergeType() == SVETypeFlags::MergeAny)
4094 std::swap(Ops[1], Ops[3]);
4095
4096 // Predicated intrinsics with _z suffix need a select w/ zeroinitializer.
4097 if (TypeFlags.getMergeType() == SVETypeFlags::MergeZero) {
4098 llvm::Type *OpndTy = Ops[1]->getType();
4099 auto *SplatZero = Constant::getNullValue(OpndTy);
4100 Ops[1] = Builder.CreateSelect(Ops[0], Ops[1], SplatZero);
4101 }
4102
4103 Function *F = CGM.getIntrinsic(Builtin->LLVMIntrinsic,
4104 getSVEOverloadTypes(TypeFlags, Ty, Ops));
4105 Value *Call = Builder.CreateCall(F, Ops);
4106
4107 if (Call->getType() == Ty)
4108 return Call;
4109
4110 // Predicate results must be converted to svbool_t.
4111 if (auto PredTy = dyn_cast<llvm::ScalableVectorType>(Ty))
4112 return EmitSVEPredicateCast(Call, PredTy);
4113 if (auto PredTupleTy = dyn_cast<llvm::StructType>(Ty))
4114 return EmitSVEPredicateTupleCast(Call, PredTupleTy);
4115
4116 llvm_unreachable("unsupported element count!");
4117 }
4118
4119 switch (BuiltinID) {
4120 default:
4121 return nullptr;
4122
4123 case SVE::BI__builtin_sve_svreinterpret_b: {
4124 Function *CastFromSVCountF =
4125 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svcount);
4126 return Builder.CreateCall(CastFromSVCountF, Ops[0]);
4127 }
4128 case SVE::BI__builtin_sve_svreinterpret_c: {
4129 Function *CastToSVCountF =
4130 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_to_svcount);
4131 return Builder.CreateCall(CastToSVCountF, Ops[0]);
4132 }
4133
4134 case SVE::BI__builtin_sve_svpsel_lane_b8:
4135 case SVE::BI__builtin_sve_svpsel_lane_b16:
4136 case SVE::BI__builtin_sve_svpsel_lane_b32:
4137 case SVE::BI__builtin_sve_svpsel_lane_b64:
4138 case SVE::BI__builtin_sve_svpsel_lane_c8:
4139 case SVE::BI__builtin_sve_svpsel_lane_c16:
4140 case SVE::BI__builtin_sve_svpsel_lane_c32:
4141 case SVE::BI__builtin_sve_svpsel_lane_c64: {
4142 auto OverloadedTy = getSVEType(SVETypeFlags(Builtin->TypeModifier));
4143 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_psel,
4144 {Ops[0]->getType(), OverloadedTy});
4145 llvm::Value *Ops1 = EmitSVEPredicateCast(Ops[1], OverloadedTy);
4146 return Builder.CreateCall(F, {Ops[0], Ops1, Ops[2]});
4147 }
4148 case SVE::BI__builtin_sve_svmov_b_z: {
4149 // svmov_b_z(pg, op) <=> svand_b_z(pg, op, op)
4150 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4151 llvm::Type* OverloadedTy = getSVEType(TypeFlags);
4152 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_and_z, OverloadedTy);
4153 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
4154 }
4155
4156 case SVE::BI__builtin_sve_svnot_b_z: {
4157 // svnot_b_z(pg, op) <=> sveor_b_z(pg, op, pg)
4158 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4159 llvm::Type* OverloadedTy = getSVEType(TypeFlags);
4160 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_eor_z, OverloadedTy);
4161 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
4162 }
4163
4164 case SVE::BI__builtin_sve_svmovlb_u16:
4165 case SVE::BI__builtin_sve_svmovlb_u32:
4166 case SVE::BI__builtin_sve_svmovlb_u64:
4167 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
4168
4169 case SVE::BI__builtin_sve_svmovlb_s16:
4170 case SVE::BI__builtin_sve_svmovlb_s32:
4171 case SVE::BI__builtin_sve_svmovlb_s64:
4172 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
4173
4174 case SVE::BI__builtin_sve_svmovlt_u16:
4175 case SVE::BI__builtin_sve_svmovlt_u32:
4176 case SVE::BI__builtin_sve_svmovlt_u64:
4177 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
4178
4179 case SVE::BI__builtin_sve_svmovlt_s16:
4180 case SVE::BI__builtin_sve_svmovlt_s32:
4181 case SVE::BI__builtin_sve_svmovlt_s64:
4182 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
4183
4184 case SVE::BI__builtin_sve_svpmullt_u16:
4185 case SVE::BI__builtin_sve_svpmullt_u64:
4186 case SVE::BI__builtin_sve_svpmullt_n_u16:
4187 case SVE::BI__builtin_sve_svpmullt_n_u64:
4188 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
4189
4190 case SVE::BI__builtin_sve_svpmullb_u16:
4191 case SVE::BI__builtin_sve_svpmullb_u64:
4192 case SVE::BI__builtin_sve_svpmullb_n_u16:
4193 case SVE::BI__builtin_sve_svpmullb_n_u64:
4194 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
4195
4196 case SVE::BI__builtin_sve_svdup_n_b8:
4197 case SVE::BI__builtin_sve_svdup_n_b16:
4198 case SVE::BI__builtin_sve_svdup_n_b32:
4199 case SVE::BI__builtin_sve_svdup_n_b64: {
4200 llvm::ScalableVectorType *OverloadedTy = getSVEType(TypeFlags);
4201 Value *Dup = EmitSVEDupX(Ops[0], OverloadedTy);
4202 return EmitSVEPredicateCast(Dup, cast<llvm::ScalableVectorType>(Ty));
4203 }
4204
4205 case SVE::BI__builtin_sve_svdupq_n_b8:
4206 case SVE::BI__builtin_sve_svdupq_n_b16:
4207 case SVE::BI__builtin_sve_svdupq_n_b32:
4208 case SVE::BI__builtin_sve_svdupq_n_b64:
4209 case SVE::BI__builtin_sve_svdupq_n_u8:
4210 case SVE::BI__builtin_sve_svdupq_n_s8:
4211 case SVE::BI__builtin_sve_svdupq_n_u64:
4212 case SVE::BI__builtin_sve_svdupq_n_f64:
4213 case SVE::BI__builtin_sve_svdupq_n_s64:
4214 case SVE::BI__builtin_sve_svdupq_n_u16:
4215 case SVE::BI__builtin_sve_svdupq_n_f16:
4216 case SVE::BI__builtin_sve_svdupq_n_bf16:
4217 case SVE::BI__builtin_sve_svdupq_n_s16:
4218 case SVE::BI__builtin_sve_svdupq_n_u32:
4219 case SVE::BI__builtin_sve_svdupq_n_f32:
4220 case SVE::BI__builtin_sve_svdupq_n_s32: {
4221 // These builtins are implemented by storing each element to an array and using
4222 // ld1rq to materialize a vector.
4223 unsigned NumOpnds = Ops.size();
4224
4225 bool IsBoolTy =
4226 cast<llvm::VectorType>(Ty)->getElementType()->isIntegerTy(1);
4227
4228 // For svdupq_n_b* the element type of is an integer of type 128/numelts,
4229 // so that the compare can use the width that is natural for the expected
4230 // number of predicate lanes.
4231 llvm::Type *EltTy = Ops[0]->getType();
4232 if (IsBoolTy)
4233 EltTy = IntegerType::get(getLLVMContext(), SVEBitsPerBlock / NumOpnds);
4234
4235 SmallVector<llvm::Value *, 16> VecOps;
4236 for (unsigned I = 0; I < NumOpnds; ++I)
4237 VecOps.push_back(Builder.CreateZExt(Ops[I], EltTy));
4238 Value *Vec = BuildVector(VecOps);
4239
4240 llvm::Type *OverloadedTy = getSVEVectorForElementType(EltTy);
4241 Value *InsertSubVec = Builder.CreateInsertVector(
4242 OverloadedTy, PoisonValue::get(OverloadedTy), Vec, uint64_t(0));
4243
4244 Function *F =
4245 CGM.getIntrinsic(Intrinsic::aarch64_sve_dupq_lane, OverloadedTy);
4246 Value *DupQLane =
4247 Builder.CreateCall(F, {InsertSubVec, Builder.getInt64(0)});
4248
4249 if (!IsBoolTy)
4250 return DupQLane;
4251
4252 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4253 Constant *Pred = ConstantInt::getTrue(getSVEPredType(TypeFlags));
4254
4255 // For svdupq_n_b* we need to add an additional 'cmpne' with '0'.
4256 F = CGM.getIntrinsic(NumOpnds == 2 ? Intrinsic::aarch64_sve_cmpne
4257 : Intrinsic::aarch64_sve_cmpne_wide,
4258 OverloadedTy);
4259 Value *Call = Builder.CreateCall(
4260 F, {Pred, DupQLane, EmitSVEDupX(Builder.getInt64(0))});
4261 return EmitSVEPredicateCast(Call, cast<llvm::ScalableVectorType>(Ty));
4262 }
4263
4264 case SVE::BI__builtin_sve_svpfalse_b:
4265 return ConstantInt::getFalse(Ty);
4266
4267 case SVE::BI__builtin_sve_svpfalse_c:
4268 return Constant::getNullValue(Ty);
4269
4270 case SVE::BI__builtin_sve_svlen_bf16:
4271 case SVE::BI__builtin_sve_svlen_f16:
4272 case SVE::BI__builtin_sve_svlen_f32:
4273 case SVE::BI__builtin_sve_svlen_f64:
4274 case SVE::BI__builtin_sve_svlen_s8:
4275 case SVE::BI__builtin_sve_svlen_s16:
4276 case SVE::BI__builtin_sve_svlen_s32:
4277 case SVE::BI__builtin_sve_svlen_s64:
4278 case SVE::BI__builtin_sve_svlen_u8:
4279 case SVE::BI__builtin_sve_svlen_u16:
4280 case SVE::BI__builtin_sve_svlen_u32:
4281 case SVE::BI__builtin_sve_svlen_u64: {
4282 SVETypeFlags TF(Builtin->TypeModifier);
4283 return Builder.CreateElementCount(Ty, getSVEType(TF)->getElementCount());
4284 }
4285
4286 case SVE::BI__builtin_sve_svtbl2_u8:
4287 case SVE::BI__builtin_sve_svtbl2_s8:
4288 case SVE::BI__builtin_sve_svtbl2_u16:
4289 case SVE::BI__builtin_sve_svtbl2_s16:
4290 case SVE::BI__builtin_sve_svtbl2_u32:
4291 case SVE::BI__builtin_sve_svtbl2_s32:
4292 case SVE::BI__builtin_sve_svtbl2_u64:
4293 case SVE::BI__builtin_sve_svtbl2_s64:
4294 case SVE::BI__builtin_sve_svtbl2_f16:
4295 case SVE::BI__builtin_sve_svtbl2_bf16:
4296 case SVE::BI__builtin_sve_svtbl2_f32:
4297 case SVE::BI__builtin_sve_svtbl2_f64: {
4298 SVETypeFlags TF(Builtin->TypeModifier);
4299 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_tbl2, getSVEType(TF));
4300 return Builder.CreateCall(F, Ops);
4301 }
4302
4303 case SVE::BI__builtin_sve_svset_neonq_s8:
4304 case SVE::BI__builtin_sve_svset_neonq_s16:
4305 case SVE::BI__builtin_sve_svset_neonq_s32:
4306 case SVE::BI__builtin_sve_svset_neonq_s64:
4307 case SVE::BI__builtin_sve_svset_neonq_u8:
4308 case SVE::BI__builtin_sve_svset_neonq_u16:
4309 case SVE::BI__builtin_sve_svset_neonq_u32:
4310 case SVE::BI__builtin_sve_svset_neonq_u64:
4311 case SVE::BI__builtin_sve_svset_neonq_f16:
4312 case SVE::BI__builtin_sve_svset_neonq_f32:
4313 case SVE::BI__builtin_sve_svset_neonq_f64:
4314 case SVE::BI__builtin_sve_svset_neonq_bf16:
4315 case SVE::BI__builtin_sve_svset_neonq_mf8: {
4316 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], uint64_t(0));
4317 }
4318
4319 case SVE::BI__builtin_sve_svget_neonq_s8:
4320 case SVE::BI__builtin_sve_svget_neonq_s16:
4321 case SVE::BI__builtin_sve_svget_neonq_s32:
4322 case SVE::BI__builtin_sve_svget_neonq_s64:
4323 case SVE::BI__builtin_sve_svget_neonq_u8:
4324 case SVE::BI__builtin_sve_svget_neonq_u16:
4325 case SVE::BI__builtin_sve_svget_neonq_u32:
4326 case SVE::BI__builtin_sve_svget_neonq_u64:
4327 case SVE::BI__builtin_sve_svget_neonq_f16:
4328 case SVE::BI__builtin_sve_svget_neonq_f32:
4329 case SVE::BI__builtin_sve_svget_neonq_f64:
4330 case SVE::BI__builtin_sve_svget_neonq_bf16:
4331 case SVE::BI__builtin_sve_svget_neonq_mf8: {
4332 return Builder.CreateExtractVector(Ty, Ops[0], uint64_t(0));
4333 }
4334
4335 case SVE::BI__builtin_sve_svdup_neonq_s8:
4336 case SVE::BI__builtin_sve_svdup_neonq_s16:
4337 case SVE::BI__builtin_sve_svdup_neonq_s32:
4338 case SVE::BI__builtin_sve_svdup_neonq_s64:
4339 case SVE::BI__builtin_sve_svdup_neonq_u8:
4340 case SVE::BI__builtin_sve_svdup_neonq_u16:
4341 case SVE::BI__builtin_sve_svdup_neonq_u32:
4342 case SVE::BI__builtin_sve_svdup_neonq_u64:
4343 case SVE::BI__builtin_sve_svdup_neonq_f16:
4344 case SVE::BI__builtin_sve_svdup_neonq_f32:
4345 case SVE::BI__builtin_sve_svdup_neonq_f64:
4346 case SVE::BI__builtin_sve_svdup_neonq_bf16:
4347 case SVE::BI__builtin_sve_svdup_neonq_mf8: {
4348 Value *Insert = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0],
4349 uint64_t(0));
4350 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
4351 {Insert, Builder.getInt64(0)});
4352 }
4353 }
4354
4355 /// Should not happen
4356 return nullptr;
4357}
4358
4359static void swapCommutativeSMEOperands(unsigned BuiltinID,
4360 SmallVectorImpl<Value *> &Ops) {
4361 unsigned MultiVec;
4362 switch (BuiltinID) {
4363 default:
4364 return;
4365 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1:
4366 MultiVec = 1;
4367 break;
4368 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2:
4369 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2:
4370 MultiVec = 2;
4371 break;
4372 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4:
4373 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4:
4374 MultiVec = 4;
4375 break;
4376 }
4377
4378 if (MultiVec > 0)
4379 for (unsigned I = 0; I < MultiVec; ++I)
4380 std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]);
4381}
4382
4383Value *CodeGenFunction::EmitAArch64SMEBuiltinExpr(unsigned BuiltinID,
4384 const CallExpr *E) {
4385 auto *Builtin =
4386 findARMVectorIntrinsicInMap(ArrayRef(AArch64SMEIntrinsicMap), BuiltinID,
4387 AArch64SMEIntrinsicsProvenSorted);
4388
4389 llvm::SmallVector<Value *, 4> Ops;
4390 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4391 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags);
4392
4393 if (TypeFlags.isLoad() || TypeFlags.isStore())
4394 return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4395 if (TypeFlags.isReadZA() || TypeFlags.isWriteZA())
4396 return EmitSMEReadWrite(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4397 if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
4398 BuiltinID == SME::BI__builtin_sme_svzero_za)
4399 return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4400 if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
4401 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za ||
4402 BuiltinID == SME::BI__builtin_sme_svldr_za ||
4403 BuiltinID == SME::BI__builtin_sme_svstr_za)
4404 return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4405
4406 // Emit set FPMR for intrinsics that require it
4407 if (TypeFlags.setsFPMR())
4408 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_set_fpmr),
4409 Ops.pop_back_val());
4410 // Handle builtins which require their multi-vector operands to be swapped
4411 swapCommutativeSMEOperands(BuiltinID, Ops);
4412
4413 auto isCntsBuiltin = [&]() {
4414 switch (BuiltinID) {
4415 default:
4416 return 0;
4417 case SME::BI__builtin_sme_svcntsb:
4418 return 8;
4419 case SME::BI__builtin_sme_svcntsh:
4420 return 4;
4421 case SME::BI__builtin_sme_svcntsw:
4422 return 2;
4423 }
4424 };
4425
4426 if (auto Mul = isCntsBuiltin()) {
4427 llvm::Value *Cntd =
4428 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_sme_cntsd));
4429 return Builder.CreateMul(Cntd, llvm::ConstantInt::get(Int64Ty, Mul),
4430 "mulsvl", /* HasNUW */ true, /* HasNSW */ true);
4431 }
4432
4433 // Should not happen!
4434 if (Builtin->LLVMIntrinsic == 0)
4435 return nullptr;
4436
4437 // Predicates must match the main datatype.
4438 for (Value *&Op : Ops)
4439 if (auto PredTy = dyn_cast<llvm::VectorType>(Op->getType()))
4440 if (PredTy->getElementType()->isIntegerTy(1))
4441 Op = EmitSVEPredicateCast(Op, getSVEType(TypeFlags));
4442
4443 if (BuiltinID == SME::BI__builtin_sme_svldr_zt ||
4444 BuiltinID == SME::BI__builtin_sme_svstr_zt) {
4445 Function *F = CGM.getIntrinsic(Builtin->LLVMIntrinsic, Ops[1]->getType());
4446 return Builder.CreateCall(F, Ops);
4447 }
4448
4449 Function *F =
4450 TypeFlags.isOverloadNone()
4451 ? CGM.getIntrinsic(Builtin->LLVMIntrinsic)
4452 : CGM.getIntrinsic(Builtin->LLVMIntrinsic, {getSVEType(TypeFlags)});
4453
4454 return Builder.CreateCall(F, Ops);
4455}
4456
4457/// Helper for the read/write/add/inc X18 builtins: read the X18 register and
4458/// return it as an i8 pointer.
4459Value *readX18AsPtr(CodeGenFunction &CGF) {
4460 LLVMContext &Context = CGF.CGM.getLLVMContext();
4461 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")};
4462 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
4463 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
4464 llvm::Function *F =
4465 CGF.CGM.getIntrinsic(Intrinsic::read_register, {CGF.Int64Ty});
4466 llvm::Value *X18 = CGF.Builder.CreateCall(F, Metadata);
4467 return CGF.Builder.CreateIntToPtr(X18, CGF.Int8PtrTy);
4468}
4469
4470Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
4471 const CallExpr *E,
4472 llvm::Triple::ArchType Arch) {
4473 if (BuiltinID >= clang::AArch64::FirstSVEBuiltin &&
4474 BuiltinID <= clang::AArch64::LastSVEBuiltin)
4475 return EmitAArch64SVEBuiltinExpr(BuiltinID, E);
4476
4477 if (BuiltinID >= clang::AArch64::FirstSMEBuiltin &&
4478 BuiltinID <= clang::AArch64::LastSMEBuiltin)
4479 return EmitAArch64SMEBuiltinExpr(BuiltinID, E);
4480
4481 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
4482 return EmitAArch64CpuSupports(E);
4483
4484 unsigned HintID = static_cast<unsigned>(-1);
4485 switch (BuiltinID) {
4486 default: break;
4487 case clang::AArch64::BI__builtin_arm_nop:
4488 HintID = 0;
4489 break;
4490 case clang::AArch64::BI__builtin_arm_yield:
4491 case clang::AArch64::BI__yield:
4492 HintID = 1;
4493 break;
4494 case clang::AArch64::BI__builtin_arm_wfe:
4495 case clang::AArch64::BI__wfe:
4496 HintID = 2;
4497 break;
4498 case clang::AArch64::BI__builtin_arm_wfi:
4499 case clang::AArch64::BI__wfi:
4500 HintID = 3;
4501 break;
4502 case clang::AArch64::BI__builtin_arm_sev:
4503 case clang::AArch64::BI__sev:
4504 HintID = 4;
4505 break;
4506 case clang::AArch64::BI__builtin_arm_sevl:
4507 case clang::AArch64::BI__sevl:
4508 HintID = 5;
4509 break;
4510 }
4511
4512 if (HintID != static_cast<unsigned>(-1)) {
4513 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint);
4514 return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
4515 }
4516
4517 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
4518 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_break);
4519 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4520 return Builder.CreateCall(F, Builder.CreateZExt(Arg, CGM.Int32Ty));
4521 }
4522
4523 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
4524 // Create call to __arm_sme_state and store the results to the two pointers.
4525 CallInst *CI = EmitRuntimeCall(CGM.CreateRuntimeFunction(
4526 llvm::FunctionType::get(StructType::get(CGM.Int64Ty, CGM.Int64Ty), {},
4527 false),
4528 "__arm_sme_state"));
4529 auto Attrs = AttributeList().addFnAttribute(getLLVMContext(),
4530 "aarch64_pstate_sm_compatible");
4531 CI->setAttributes(Attrs);
4532 CI->setCallingConv(
4533 llvm::CallingConv::
4534 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
4535 Builder.CreateStore(Builder.CreateExtractValue(CI, 0),
4536 EmitPointerWithAlignment(E->getArg(0)));
4537 return Builder.CreateStore(Builder.CreateExtractValue(CI, 1),
4538 EmitPointerWithAlignment(E->getArg(1)));
4539 }
4540
4541 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
4542 assert((getContext().getTypeSize(E->getType()) == 32) &&
4543 "rbit of unusual size!");
4544 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4545 return Builder.CreateCall(
4546 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
4547 }
4548 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
4549 assert((getContext().getTypeSize(E->getType()) == 64) &&
4550 "rbit of unusual size!");
4551 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4552 return Builder.CreateCall(
4553 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
4554 }
4555
4556 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
4557 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
4558 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4559 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Arg->getType());
4560 Value *Res = Builder.CreateCall(F, {Arg, Builder.getInt1(false)});
4561 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
4562 Res = Builder.CreateTrunc(Res, Builder.getInt32Ty());
4563 return Res;
4564 }
4565
4566 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
4567 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4568 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls), Arg,
4569 "cls");
4570 }
4571 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
4572 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4573 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls64), Arg,
4574 "cls");
4575 }
4576
4577 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
4578 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
4579 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4580 llvm::Type *Ty = Arg->getType();
4581 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint32z, Ty),
4582 Arg, "frint32z");
4583 }
4584
4585 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
4586 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
4587 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4588 llvm::Type *Ty = Arg->getType();
4589 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint64z, Ty),
4590 Arg, "frint64z");
4591 }
4592
4593 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
4594 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
4595 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4596 llvm::Type *Ty = Arg->getType();
4597 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint32x, Ty),
4598 Arg, "frint32x");
4599 }
4600
4601 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
4602 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
4603 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4604 llvm::Type *Ty = Arg->getType();
4605 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint64x, Ty),
4606 Arg, "frint64x");
4607 }
4608
4609 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
4610 assert((getContext().getTypeSize(E->getType()) == 32) &&
4611 "__jcvt of unusual size!");
4612 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4613 return Builder.CreateCall(
4614 CGM.getIntrinsic(Intrinsic::aarch64_fjcvtzs), Arg);
4615 }
4616
4617 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
4618 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
4619 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
4620 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
4621 llvm::Value *MemAddr = EmitScalarExpr(E->getArg(0));
4622 llvm::Value *ValPtr = EmitScalarExpr(E->getArg(1));
4623
4624 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
4625 // Load from the address via an LLVM intrinsic, receiving a
4626 // tuple of 8 i64 words, and store each one to ValPtr.
4627 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_ld64b);
4628 llvm::Value *Val = Builder.CreateCall(F, MemAddr);
4629 llvm::Value *ToRet;
4630 for (size_t i = 0; i < 8; i++) {
4631 llvm::Value *ValOffsetPtr =
4632 Builder.CreateGEP(Int64Ty, ValPtr, Builder.getInt32(i));
4633 Address Addr =
4634 Address(ValOffsetPtr, Int64Ty, CharUnits::fromQuantity(8));
4635 ToRet = Builder.CreateStore(Builder.CreateExtractValue(Val, i), Addr);
4636 }
4637 return ToRet;
4638 }
4639
4640 // Load 8 i64 words from ValPtr, and store them to the address
4641 // via an LLVM intrinsic.
4642 SmallVector<llvm::Value *, 9> Args;
4643 Args.push_back(MemAddr);
4644 for (size_t i = 0; i < 8; i++) {
4645 llvm::Value *ValOffsetPtr =
4646 Builder.CreateGEP(Int64Ty, ValPtr, Builder.getInt32(i));
4647 Address Addr = Address(ValOffsetPtr, Int64Ty, CharUnits::fromQuantity(8));
4648 Args.push_back(Builder.CreateLoad(Addr));
4649 }
4650
4651 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
4652 ? Intrinsic::aarch64_st64b
4653 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
4654 ? Intrinsic::aarch64_st64bv
4655 : Intrinsic::aarch64_st64bv0);
4656 Function *F = CGM.getIntrinsic(Intr);
4657 return Builder.CreateCall(F, Args);
4658 }
4659
4660 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
4661 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
4662
4663 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
4664 ? Intrinsic::aarch64_rndr
4665 : Intrinsic::aarch64_rndrrs);
4666 Function *F = CGM.getIntrinsic(Intr);
4667 llvm::Value *Val = Builder.CreateCall(F);
4668 Value *RandomValue = Builder.CreateExtractValue(Val, 0);
4669 Value *Status = Builder.CreateExtractValue(Val, 1);
4670
4671 Address MemAddress = EmitPointerWithAlignment(E->getArg(0));
4672 Builder.CreateStore(RandomValue, MemAddress);
4673 Status = Builder.CreateZExt(Status, Int32Ty);
4674 return Status;
4675 }
4676
4677 if (BuiltinID == clang::AArch64::BI__clear_cache) {
4678 Value *Begin = EmitScalarExpr(E->getArg(0));
4679 Value *End = EmitScalarExpr(E->getArg(1));
4680 Function *F = CGM.getIntrinsic(Intrinsic::clear_cache, {CGM.DefaultPtrTy});
4681 return Builder.CreateCall(F, {Begin, End});
4682 }
4683
4684 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
4685 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
4686 getContext().getTypeSize(E->getType()) == 128) {
4687 Function *F =
4688 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
4689 ? Intrinsic::aarch64_ldaxp
4690 : Intrinsic::aarch64_ldxp);
4691
4692 Value *LdPtr = EmitScalarExpr(E->getArg(0));
4693 Value *Val = Builder.CreateCall(F, LdPtr, "ldxp");
4694
4695 Value *Val0 = Builder.CreateExtractValue(Val, 1);
4696 Value *Val1 = Builder.CreateExtractValue(Val, 0);
4697 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
4698 Val0 = Builder.CreateZExt(Val0, Int128Ty);
4699 Val1 = Builder.CreateZExt(Val1, Int128Ty);
4700
4701 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
4702 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
4703 Val = Builder.CreateOr(Val, Val1);
4704 return Builder.CreateBitCast(Val, ConvertType(E->getType()));
4705 } else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
4706 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
4707 Value *LoadAddr = EmitScalarExpr(E->getArg(0));
4708
4709 QualType Ty = E->getType();
4710 llvm::Type *RealResTy = ConvertType(Ty);
4711 llvm::Type *IntTy =
4712 llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
4713
4714 Function *F =
4715 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
4716 ? Intrinsic::aarch64_ldaxr
4717 : Intrinsic::aarch64_ldxr,
4718 DefaultPtrTy);
4719 CallInst *Val = Builder.CreateCall(F, LoadAddr, "ldxr");
4720 Val->addParamAttr(
4721 0, Attribute::get(getLLVMContext(), Attribute::ElementType, IntTy));
4722
4723 if (RealResTy->isPointerTy())
4724 return Builder.CreateIntToPtr(Val, RealResTy);
4725
4726 llvm::Type *IntResTy = llvm::IntegerType::get(
4727 getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
4728 return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
4729 RealResTy);
4730 }
4731
4732 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
4733 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
4734 getContext().getTypeSize(E->getArg(0)->getType()) == 128) {
4735 Function *F =
4736 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_stlex
4737 ? Intrinsic::aarch64_stlxp
4738 : Intrinsic::aarch64_stxp);
4739 llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty);
4740
4741 Address Tmp = CreateMemTempWithoutCast(E->getArg(0)->getType());
4742 EmitAnyExprToMem(E->getArg(0), Tmp, Qualifiers(), /*init*/ true);
4743
4744 Tmp = Tmp.withElementType(STy);
4745 llvm::Value *Val = Builder.CreateLoad(Tmp);
4746
4747 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
4748 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
4749 Value *StPtr = EmitScalarExpr(E->getArg(1));
4750 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp");
4751 }
4752
4753 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
4754 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
4755 Value *StoreVal = EmitScalarExpr(E->getArg(0));
4756 Value *StoreAddr = EmitScalarExpr(E->getArg(1));
4757
4758 QualType Ty = E->getArg(0)->getType();
4759 llvm::Type *StoreTy =
4760 llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
4761
4762 if (StoreVal->getType()->isPointerTy())
4763 StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty);
4764 else {
4765 llvm::Type *IntTy = llvm::IntegerType::get(
4766 getLLVMContext(),
4767 CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
4768 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
4769 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
4770 }
4771
4772 Function *F =
4773 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_stlex
4774 ? Intrinsic::aarch64_stlxr
4775 : Intrinsic::aarch64_stxr,
4776 StoreAddr->getType());
4777 CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
4778 CI->addParamAttr(
4779 1, Attribute::get(getLLVMContext(), Attribute::ElementType, StoreTy));
4780 return CI;
4781 }
4782
4783 if (BuiltinID == clang::AArch64::BI__getReg ||
4784 BuiltinID == clang::AArch64::BI__setReg) {
4785 Expr::EvalResult Result;
4786 if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
4787 llvm_unreachable("Sema will ensure that the parameter is constant");
4788
4789 llvm::APSInt Value = Result.Val.getInt();
4790 LLVMContext &Context = CGM.getLLVMContext();
4791 std::string Reg = Value == 31 ? "sp" : "x" + toString(Value, 10);
4792
4793 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
4794 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
4795 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
4796
4797 CallInst *CI;
4798 if (BuiltinID == clang::AArch64::BI__getReg) {
4799 llvm::Function *F =
4800 CGM.getIntrinsic(Intrinsic::read_volatile_register, {Int64Ty});
4801 CI = Builder.CreateCall(F, Metadata);
4802 } else {
4803 llvm::Function *F =
4804 CGM.getIntrinsic(Intrinsic::write_volatile_register, {Int64Ty});
4805 CI = Builder.CreateCall(F, {Metadata, EmitScalarExpr(E->getArg(1))});
4806 }
4807 return CI;
4808 }
4809
4810 if (BuiltinID == clang::AArch64::BI__getRegFp ||
4811 BuiltinID == clang::AArch64::BI__setRegFp) {
4812 Expr::EvalResult Result;
4813 if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
4814 llvm_unreachable("Sema will ensure that the parameter is constant");
4815
4816 llvm::APSInt Value = Result.Val.getInt();
4817 LLVMContext &Context = CGM.getLLVMContext();
4818 std::string Reg = "d" + toString(Value, 10);
4819
4820 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
4821 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
4822 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
4823
4824 llvm::Value *Ret;
4825 if (BuiltinID == clang::AArch64::BI__getRegFp) {
4826 llvm::Function *F =
4827 CGM.getIntrinsic(Intrinsic::read_volatile_register, {Int64Ty});
4828 llvm::Value *Bits = Builder.CreateCall(F, Metadata);
4829 Ret = Builder.CreateBitCast(Bits, llvm::Type::getDoubleTy(Context));
4830 } else {
4831 llvm::Value *Val = EmitScalarExpr(E->getArg(1));
4832 llvm::Value *Bits = Builder.CreateBitCast(Val, Int64Ty);
4833 llvm::Function *F =
4834 CGM.getIntrinsic(Intrinsic::write_volatile_register, {Int64Ty});
4835 Ret = Builder.CreateCall(F, {Metadata, Bits});
4836 }
4837 return Ret;
4838 }
4839
4840 if (BuiltinID == clang::AArch64::BI__break) {
4841 Expr::EvalResult Result;
4842 if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
4843 llvm_unreachable("Sema will ensure that the parameter is constant");
4844
4845 llvm::Function *F = CGM.getIntrinsic(Intrinsic::aarch64_break);
4846 return Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0))});
4847 }
4848
4849 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
4850 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
4851 return Builder.CreateCall(F);
4852 }
4853
4854 if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
4855 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
4856 llvm::SyncScope::SingleThread);
4857
4858 // CRC32
4859 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
4860 switch (BuiltinID) {
4861 case clang::AArch64::BI__builtin_arm_crc32b:
4862 CRCIntrinsicID = Intrinsic::aarch64_crc32b; break;
4863 case clang::AArch64::BI__builtin_arm_crc32cb:
4864 CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break;
4865 case clang::AArch64::BI__builtin_arm_crc32h:
4866 CRCIntrinsicID = Intrinsic::aarch64_crc32h; break;
4867 case clang::AArch64::BI__builtin_arm_crc32ch:
4868 CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break;
4869 case clang::AArch64::BI__builtin_arm_crc32w:
4870 CRCIntrinsicID = Intrinsic::aarch64_crc32w; break;
4871 case clang::AArch64::BI__builtin_arm_crc32cw:
4872 CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break;
4873 case clang::AArch64::BI__builtin_arm_crc32d:
4874 CRCIntrinsicID = Intrinsic::aarch64_crc32x; break;
4875 case clang::AArch64::BI__builtin_arm_crc32cd:
4876 CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break;
4877 }
4878
4879 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
4880 Value *Arg0 = EmitScalarExpr(E->getArg(0));
4881 Value *Arg1 = EmitScalarExpr(E->getArg(1));
4882 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
4883
4884 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
4885 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
4886
4887 return Builder.CreateCall(F, {Arg0, Arg1});
4888 }
4889
4890 // Memory Operations (MOPS)
4891 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
4892 Value *Dst = EmitScalarExpr(E->getArg(0));
4893 Value *Val = EmitScalarExpr(E->getArg(1));
4894 Value *Size = EmitScalarExpr(E->getArg(2));
4895 Val = Builder.CreateTrunc(Val, Int8Ty);
4896 Size = Builder.CreateIntCast(Size, Int64Ty, false);
4897 return Builder.CreateCall(
4898 CGM.getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
4899 }
4900
4901 if (BuiltinID == AArch64::BI__builtin_arm_range_prefetch ||
4902 BuiltinID == AArch64::BI__builtin_arm_range_prefetch_x)
4903 return EmitRangePrefetchBuiltin(*this, BuiltinID, E);
4904
4905 if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint)
4906 return EmitAtomicStoreWithHintBuiltin(*this, BuiltinID, E);
4907
4908 // Memory Tagging Extensions (MTE) Intrinsics
4909 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
4910 switch (BuiltinID) {
4911 case clang::AArch64::BI__builtin_arm_irg:
4912 MTEIntrinsicID = Intrinsic::aarch64_irg; break;
4913 case clang::AArch64::BI__builtin_arm_addg:
4914 MTEIntrinsicID = Intrinsic::aarch64_addg; break;
4915 case clang::AArch64::BI__builtin_arm_gmi:
4916 MTEIntrinsicID = Intrinsic::aarch64_gmi; break;
4917 case clang::AArch64::BI__builtin_arm_ldg:
4918 MTEIntrinsicID = Intrinsic::aarch64_ldg; break;
4919 case clang::AArch64::BI__builtin_arm_stg:
4920 MTEIntrinsicID = Intrinsic::aarch64_stg; break;
4921 case clang::AArch64::BI__builtin_arm_subp:
4922 MTEIntrinsicID = Intrinsic::aarch64_subp; break;
4923 }
4924
4925 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
4926 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
4927 Value *Pointer = EmitScalarExpr(E->getArg(0));
4928 Value *Mask = EmitScalarExpr(E->getArg(1));
4929 assert(Mask->getType()->getScalarSizeInBits() == 64 &&
4930 "SemaARM::BuiltinARMMemoryTaggingCall() enforces this");
4931 return Builder.CreateCall(CGM.getIntrinsic(MTEIntrinsicID),
4932 {Pointer, Mask});
4933 }
4934 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
4935 Value *Pointer = EmitScalarExpr(E->getArg(0));
4936 Value *TagOffset = EmitScalarExpr(E->getArg(1));
4937
4938 TagOffset = Builder.CreateZExt(TagOffset, Int64Ty);
4939 return Builder.CreateCall(CGM.getIntrinsic(MTEIntrinsicID),
4940 {Pointer, TagOffset});
4941 }
4942 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
4943 Value *Pointer = EmitScalarExpr(E->getArg(0));
4944 Value *ExcludedMask = EmitScalarExpr(E->getArg(1));
4945 assert(ExcludedMask->getType()->getScalarSizeInBits() == 64 &&
4946 "SemaARM::BuiltinARMMemoryTaggingCall() enforces this");
4947 return Builder.CreateCall(CGM.getIntrinsic(MTEIntrinsicID),
4948 {Pointer, ExcludedMask});
4949 }
4950 // Although it is possible to supply a different return
4951 // address (first arg) to this intrinsic, for now we set
4952 // return address same as input address.
4953 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
4954 Value *TagAddress = EmitScalarExpr(E->getArg(0));
4955 return Builder.CreateCall(CGM.getIntrinsic(MTEIntrinsicID),
4956 {TagAddress, TagAddress});
4957 }
4958 // Although it is possible to supply a different tag (to set)
4959 // to this intrinsic (as first arg), for now we supply
4960 // the tag that is in input address arg (common use case).
4961 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
4962 Value *TagAddress = EmitScalarExpr(E->getArg(0));
4963 return Builder.CreateCall(CGM.getIntrinsic(MTEIntrinsicID),
4964 {TagAddress, TagAddress});
4965 }
4966 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
4967 Value *PointerA = EmitScalarExpr(E->getArg(0));
4968 Value *PointerB = EmitScalarExpr(E->getArg(1));
4969 return Builder.CreateCall(
4970 CGM.getIntrinsic(MTEIntrinsicID), {PointerA, PointerB});
4971 }
4972 }
4973
4974 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
4975 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
4976 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
4977 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
4978 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
4979 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
4980 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
4981 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
4982
4983 SpecialRegisterAccessKind AccessKind = Write;
4984 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
4985 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
4986 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
4987 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
4988 AccessKind = VolatileRead;
4989
4990 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
4991 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
4992
4993 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
4994 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
4995
4996 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
4997 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
4998
4999 llvm::Type *ValueType;
5000 llvm::Type *RegisterType = Int64Ty;
5001 if (Is32Bit) {
5002 ValueType = Int32Ty;
5003 } else if (Is128Bit) {
5004 llvm::Type *Int128Ty =
5005 llvm::IntegerType::getInt128Ty(CGM.getLLVMContext());
5006 ValueType = Int128Ty;
5007 RegisterType = Int128Ty;
5008 } else if (IsPointerBuiltin) {
5009 ValueType = VoidPtrTy;
5010 } else {
5011 ValueType = Int64Ty;
5012 };
5013
5014 return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType,
5015 AccessKind);
5016 }
5017
5018 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
5019 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
5020 LLVMContext &Context = CGM.getLLVMContext();
5021
5022 unsigned SysReg =
5023 E->getArg(0)->EvaluateKnownConstInt(getContext()).getZExtValue();
5024
5025 std::string SysRegStr;
5026 llvm::raw_string_ostream(SysRegStr)
5027 << (0b10 | SysReg >> 14) << ":" << ((SysReg >> 11) & 7) << ":"
5028 << ((SysReg >> 7) & 15) << ":" << ((SysReg >> 3) & 15) << ":"
5029 << (SysReg & 7);
5030
5031 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
5032 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
5033 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
5034
5035 llvm::Type *RegisterType = Int64Ty;
5036 llvm::Type *Types[] = { RegisterType };
5037
5038 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
5039 llvm::Function *F = CGM.getIntrinsic(Intrinsic::read_register, Types);
5040
5041 return Builder.CreateCall(F, Metadata);
5042 }
5043
5044 llvm::Function *F = CGM.getIntrinsic(Intrinsic::write_register, Types);
5045 llvm::Value *ArgValue = EmitScalarExpr(E->getArg(1));
5046 llvm::Value *Result = Builder.CreateCall(F, {Metadata, ArgValue});
5047
5048 return Result;
5049 }
5050
5051 if (BuiltinID == clang::AArch64::BI__sys) {
5052 unsigned SysReg =
5053 E->getArg(0)->EvaluateKnownConstInt(getContext()).getZExtValue();
5054 const unsigned Op1 = SysReg >> 11;
5055 const unsigned CRn = (SysReg >> 7) & 0xf;
5056 const unsigned CRm = (SysReg >> 3) & 0xf;
5057 const unsigned Op2 = SysReg & 0x7;
5058
5059 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_sys),
5060 {Builder.getInt32(Op1), Builder.getInt32(CRn),
5061 Builder.getInt32(CRm), Builder.getInt32(Op2),
5062 EmitScalarExpr(E->getArg(1))});
5063
5064 // Return 0 for convenience, even though MSVC returns some other undefined
5065 // value.
5066 return ConstantInt::get(Builder.getInt32Ty(), 0);
5067 }
5068
5069 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
5070 llvm::Function *F =
5071 CGM.getIntrinsic(Intrinsic::addressofreturnaddress, AllocaInt8PtrTy);
5072 return Builder.CreateCall(F);
5073 }
5074
5075 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
5076 llvm::Function *F = CGM.getIntrinsic(Intrinsic::sponentry, AllocaInt8PtrTy);
5077 return Builder.CreateCall(F);
5078 }
5079
5080 if (BuiltinID == clang::AArch64::BI__mulh ||
5081 BuiltinID == clang::AArch64::BI__umulh) {
5082 llvm::Type *ResType = ConvertType(E->getType());
5083 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
5084
5085 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
5086 Value *LHS =
5087 Builder.CreateIntCast(EmitScalarExpr(E->getArg(0)), Int128Ty, IsSigned);
5088 Value *RHS =
5089 Builder.CreateIntCast(EmitScalarExpr(E->getArg(1)), Int128Ty, IsSigned);
5090
5091 Value *MulResult, *HigherBits;
5092 if (IsSigned) {
5093 MulResult = Builder.CreateNSWMul(LHS, RHS);
5094 HigherBits = Builder.CreateAShr(MulResult, 64);
5095 } else {
5096 MulResult = Builder.CreateNUWMul(LHS, RHS);
5097 HigherBits = Builder.CreateLShr(MulResult, 64);
5098 }
5099 HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
5100
5101 return HigherBits;
5102 }
5103
5104 if (BuiltinID == AArch64::BI__writex18byte ||
5105 BuiltinID == AArch64::BI__writex18word ||
5106 BuiltinID == AArch64::BI__writex18dword ||
5107 BuiltinID == AArch64::BI__writex18qword) {
5108 // Process the args first
5109 Value *OffsetArg = EmitScalarExpr(E->getArg(0));
5110 Value *DataArg = EmitScalarExpr(E->getArg(1));
5111
5112 // Read x18 as i8*
5113 llvm::Value *X18 = readX18AsPtr(*this);
5114
5115 // Store val at x18 + offset
5116 Value *Offset = Builder.CreateZExt(OffsetArg, Int64Ty);
5117 Value *Ptr = Builder.CreateGEP(Int8Ty, X18, Offset);
5118 StoreInst *Store =
5119 Builder.CreateAlignedStore(DataArg, Ptr, CharUnits::One());
5120 return Store;
5121 }
5122
5123 if (BuiltinID == AArch64::BI__readx18byte ||
5124 BuiltinID == AArch64::BI__readx18word ||
5125 BuiltinID == AArch64::BI__readx18dword ||
5126 BuiltinID == AArch64::BI__readx18qword) {
5127 // Process the args first
5128 Value *OffsetArg = EmitScalarExpr(E->getArg(0));
5129
5130 // Read x18 as i8*
5131 llvm::Value *X18 = readX18AsPtr(*this);
5132
5133 // Load x18 + offset
5134 Value *Offset = Builder.CreateZExt(OffsetArg, Int64Ty);
5135 Value *Ptr = Builder.CreateGEP(Int8Ty, X18, Offset);
5136 llvm::Type *IntTy = ConvertType(E->getType());
5137 LoadInst *Load = Builder.CreateAlignedLoad(IntTy, Ptr, CharUnits::One());
5138 return Load;
5139 }
5140
5141 if (BuiltinID == AArch64::BI__addx18byte ||
5142 BuiltinID == AArch64::BI__addx18word ||
5143 BuiltinID == AArch64::BI__addx18dword ||
5144 BuiltinID == AArch64::BI__addx18qword ||
5145 BuiltinID == AArch64::BI__incx18byte ||
5146 BuiltinID == AArch64::BI__incx18word ||
5147 BuiltinID == AArch64::BI__incx18dword ||
5148 BuiltinID == AArch64::BI__incx18qword) {
5149 llvm::Type *IntTy;
5150 bool isIncrement;
5151 switch (BuiltinID) {
5152 case AArch64::BI__incx18byte:
5153 IntTy = Int8Ty;
5154 isIncrement = true;
5155 break;
5156 case AArch64::BI__incx18word:
5157 IntTy = Int16Ty;
5158 isIncrement = true;
5159 break;
5160 case AArch64::BI__incx18dword:
5161 IntTy = Int32Ty;
5162 isIncrement = true;
5163 break;
5164 case AArch64::BI__incx18qword:
5165 IntTy = Int64Ty;
5166 isIncrement = true;
5167 break;
5168 default:
5169 IntTy = ConvertType(E->getArg(1)->getType());
5170 isIncrement = false;
5171 break;
5172 }
5173 // Process the args first
5174 Value *OffsetArg = EmitScalarExpr(E->getArg(0));
5175 Value *ValToAdd =
5176 isIncrement ? ConstantInt::get(IntTy, 1) : EmitScalarExpr(E->getArg(1));
5177
5178 // Read x18 as i8*
5179 llvm::Value *X18 = readX18AsPtr(*this);
5180
5181 // Load x18 + offset
5182 Value *Offset = Builder.CreateZExt(OffsetArg, Int64Ty);
5183 Value *Ptr = Builder.CreateGEP(Int8Ty, X18, Offset);
5184 LoadInst *Load = Builder.CreateAlignedLoad(IntTy, Ptr, CharUnits::One());
5185
5186 // Add values
5187 Value *AddResult = Builder.CreateAdd(Load, ValToAdd);
5188
5189 // Store val at x18 + offset
5190 StoreInst *Store =
5191 Builder.CreateAlignedStore(AddResult, Ptr, CharUnits::One());
5192 return Store;
5193 }
5194
5195 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
5196 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
5197 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
5198 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
5199 Value *Arg = EmitScalarExpr(E->getArg(0));
5200 llvm::Type *RetTy = ConvertType(E->getType());
5201 return Builder.CreateBitCast(Arg, RetTy);
5202 }
5203
5204 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
5205 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
5206 BuiltinID == AArch64::BI_CountLeadingZeros ||
5207 BuiltinID == AArch64::BI_CountLeadingZeros64) {
5208 Value *Arg = EmitScalarExpr(E->getArg(0));
5209 llvm::Type *ArgType = Arg->getType();
5210
5211 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
5212 BuiltinID == AArch64::BI_CountLeadingOnes64)
5213 Arg = Builder.CreateXor(Arg, Constant::getAllOnesValue(ArgType));
5214
5215 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
5216 Value *Result = Builder.CreateCall(F, {Arg, Builder.getInt1(false)});
5217
5218 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
5219 BuiltinID == AArch64::BI_CountLeadingZeros64)
5220 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
5221 return Result;
5222 }
5223
5224 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
5225 BuiltinID == AArch64::BI_CountLeadingSigns64) {
5226 Value *Arg = EmitScalarExpr(E->getArg(0));
5227
5228 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
5229 ? CGM.getIntrinsic(Intrinsic::aarch64_cls)
5230 : CGM.getIntrinsic(Intrinsic::aarch64_cls64);
5231
5232 Value *Result = Builder.CreateCall(F, Arg, "cls");
5233 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
5234 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
5235 return Result;
5236 }
5237
5238 if (BuiltinID == AArch64::BI_CountOneBits ||
5239 BuiltinID == AArch64::BI_CountOneBits64) {
5240 Value *ArgValue = EmitScalarExpr(E->getArg(0));
5241 llvm::Type *ArgType = ArgValue->getType();
5242 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
5243
5244 Value *Result = Builder.CreateCall(F, ArgValue);
5245 if (BuiltinID == AArch64::BI_CountOneBits64)
5246 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
5247 return Result;
5248 }
5249
5250 if (BuiltinID == AArch64::BI_CountTrailingZeros ||
5251 BuiltinID == AArch64::BI_CountTrailingZeros64) {
5252 Value *ArgValue = EmitScalarExpr(E->getArg(0));
5253 llvm::Type *ArgType = ArgValue->getType();
5254 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
5255
5256 // MSVC leaves 0 undefined; use false for predictable codegen
5257 Value *Result = Builder.CreateCall(F, {ArgValue, Builder.getInt1(false)});
5258 if (BuiltinID == AArch64::BI_CountTrailingZeros64)
5259 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
5260 return Result;
5261 }
5262
5263 if (BuiltinID == AArch64::BI__prefetch) {
5264 Value *Address = EmitScalarExpr(E->getArg(0));
5265 Value *RW = llvm::ConstantInt::get(Int32Ty, 0);
5266 Value *Locality = ConstantInt::get(Int32Ty, 3);
5267 Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
5268 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
5269 return Builder.CreateCall(F, {Address, RW, Locality, Data});
5270 }
5271
5272 if (BuiltinID == AArch64::BI__prefetch2) {
5273 Value *Address = EmitScalarExpr(E->getArg(0));
5274 llvm::APSInt PrfOp = E->getArg(1)->EvaluateKnownConstInt(CGM.getContext());
5275 // Decode 5-bit PRFM encoding: bits[4:3]=type, bits[2:1]=target,
5276 // bit[0]=policy
5277 // type: PLD=0(load), PLI=1(instr), PST=2(store)
5278 // target: L1=0, L2=1, L3=2
5279 // policy: KEEP=0, STRM=1
5280 uint64_t Op = PrfOp.getZExtValue();
5281 uint64_t Type = (Op >> 3) & 0x3;
5282 uint64_t Target = (Op >> 1) & 0x3;
5283 uint64_t Policy = Op & 0x1;
5284 Value *RW = Builder.getInt32(Type == 2 ? 1 : 0);
5285 Value *Local = Builder.getInt32(Target);
5286 Value *IsStream = Builder.getInt32(Policy);
5287 Value *IsData = Builder.getInt32(Type == 1 ? 0 : 1);
5288 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_prefetch);
5289 return Builder.CreateCall(F, {Address, RW, Local, IsStream, IsData});
5290 }
5291
5292 if (BuiltinID == AArch64::BI__hlt) {
5293 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hlt);
5294 Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0))});
5295
5296 // FIXME: MSVC documents __hlt as taking further arguments in X0-X3 and
5297 // returning the value in X0, like __hvc/__svc below. This ignores the
5298 // extra arguments and returns 0.
5299 return ConstantInt::get(Builder.getInt32Ty(), 0);
5300 }
5301
5302 if (BuiltinID == AArch64::BI__hvc || BuiltinID == AArch64::BI__svc) {
5303 unsigned IID = BuiltinID == AArch64::BI__svc ? Intrinsic::aarch64_svc
5304 : Intrinsic::aarch64_hvc;
5305 // The first argument is the instruction immediate; it must be a constant
5306 // (ImmArg on the intrinsic, encoded in the instruction). The remaining
5307 // arguments (at most four, enforced by Sema) are passed in X0-X3, widened
5308 // to 64 bits. The intrinsic takes exactly four register operands, so any
5309 // unused trailing ones are passed as poison and dropped during lowering.
5310 SmallVector<Value *, 5> Args{Builder.getInt32(
5311 GetIntegerConstantValue<uint32_t>(E->getArg(0), getContext()))};
5312 for (unsigned I = 1, N = E->getNumArgs(); I < N; ++I) {
5313 Value *Arg = EmitScalarExpr(E->getArg(I));
5314 llvm::Type *ArgTy = Arg->getType();
5315 if (ArgTy->isPointerTy())
5316 Arg = Builder.CreatePtrToInt(Arg, Int64Ty);
5317 else if (ArgTy->isFloatingPointTy())
5318 // Reinterpret the bits into the integer register, matching MSVC (e.g.
5319 // "fmov x0, d0" for a double).
5320 Arg = Builder.CreateZExtOrTrunc(
5321 Builder.CreateBitCast(
5322 Arg, Builder.getIntNTy(ArgTy->getPrimitiveSizeInBits())),
5323 Int64Ty);
5324 else
5325 Arg = Builder.CreateIntCast(
5326 Arg, Int64Ty, E->getArg(I)->getType()->isSignedIntegerType());
5327 Args.push_back(Arg);
5328 }
5329 while (Args.size() < 5)
5330 Args.push_back(llvm::PoisonValue::get(Int64Ty));
5331 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Args);
5332 // MSVC returns unsigned int, i.e. the low 32 bits of the X0 result.
5333 return Builder.CreateTrunc(Call, Int32Ty);
5334 }
5335
5336 if (BuiltinID == NEON::BI__builtin_neon_vcvth_bf16_f32)
5337 return Builder.CreateFPTrunc(
5338 Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)),
5339 Builder.getFloatTy()),
5340 Builder.getBFloatTy());
5341
5342 // Handle MSVC intrinsics before argument evaluation to prevent double
5343 // evaluation.
5344 if (std::optional<MSVCIntrin> MsvcIntId =
5345 translateAarch64ToMsvcIntrin(BuiltinID))
5346 return EmitMSVCBuiltinExpr(*MsvcIntId, E);
5347
5348 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
5349 auto It = llvm::find_if(NEONEquivalentIntrinsicMap, [BuiltinID](auto &P) {
5350 return P.first == BuiltinID;
5351 });
5352 if (It != end(NEONEquivalentIntrinsicMap))
5353 BuiltinID = It->second;
5354
5355 // Check whether this is an SISD builtin.
5356 auto SISDMap = ArrayRef(AArch64SISDIntrinsicMap);
5357 const ARMNeonVectorIntrinsicInfo *Builtin = findARMVectorIntrinsicInMap(
5358 SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted);
5359 bool IsSISD = (Builtin != nullptr);
5360
5361 // Find out if any arguments are required to be integer constant
5362 // expressions.
5363 unsigned ICEArguments = 0;
5364 ASTContext::GetBuiltinTypeError Error;
5365 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
5366 assert(Error == ASTContext::GE_None && "Should not codegen an error");
5367
5368 llvm::SmallVector<Value*, 4> Ops;
5369 Address PtrOp0 = Address::invalid();
5370 // Note the assumption that SISD intrinsics do not contain extra arguments.
5371 // TODO: Fold this into a single function call instead of, effectively, two
5372 // separate checks.
5373 bool HasExtraArg = !IsSISD && HasExtraNeonArgument(BuiltinID);
5374 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
5375 for (unsigned i = 0, e = NumArgs; i != e; i++) {
5376 if (i == 0) {
5377 switch (BuiltinID) {
5378 case NEON::BI__builtin_neon_vld1_v:
5379 case NEON::BI__builtin_neon_vld1q_v:
5380 case NEON::BI__builtin_neon_vld1_dup_v:
5381 case NEON::BI__builtin_neon_vld1q_dup_v:
5382 case NEON::BI__builtin_neon_vld1_lane_v:
5383 case NEON::BI__builtin_neon_vld1q_lane_v:
5384 case NEON::BI__builtin_neon_vst1_v:
5385 case NEON::BI__builtin_neon_vst1q_v:
5386 case NEON::BI__builtin_neon_vst1_lane_v:
5387 case NEON::BI__builtin_neon_vst1q_lane_v:
5388 case NEON::BI__builtin_neon_vldap1_lane_s64:
5389 case NEON::BI__builtin_neon_vldap1q_lane_s64:
5390 case NEON::BI__builtin_neon_vstl1_lane_s64:
5391 case NEON::BI__builtin_neon_vstl1q_lane_s64:
5392 // Get the alignment for the argument in addition to the value;
5393 // we'll use it later.
5394 PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
5395 Ops.push_back(PtrOp0.emitRawPointer(*this));
5396 continue;
5397 }
5398 }
5399 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E));
5400 }
5401
5402 if (Builtin) {
5403 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E);
5404 assert(Result && "SISD intrinsic should have been handled");
5405 return Result;
5406 }
5407
5408 const Expr *Arg = E->getArg(E->getNumArgs()-1);
5409 NeonTypeFlags Type(0);
5410 if (std::optional<llvm::APSInt> Result =
5411 Arg->getIntegerConstantExpr(getContext()))
5412 // Determine the type of this overloaded NEON intrinsic.
5413 Type = NeonTypeFlags(Result->getZExtValue());
5414
5415 bool usgn = Type.isUnsigned();
5416 bool quad = Type.isQuad();
5417 unsigned Int;
5418
5419 // Not all intrinsics handled by the common case work for AArch64 yet, so only
5420 // defer to common code if it's been added to our special map.
5421 Builtin =
5422 findARMVectorIntrinsicInMap(ArrayRef(AArch64SIMDIntrinsicMap), BuiltinID,
5423 AArch64SIMDIntrinsicsProvenSorted);
5424
5425 if (Builtin)
5426 return EmitCommonNeonBuiltinExpr(
5427 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
5428 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
5429 /*never use addresses*/ Address::invalid(), Address::invalid(), Arch);
5430
5431 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops, Arch))
5432 return V;
5433
5434 // Handle non-overloaded intrinsics first.
5435 switch (BuiltinID) {
5436 default: break;
5437 case NEON::BI__builtin_neon_vabsh_f16:
5438 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, HalfTy), Ops, "vabs");
5439 case NEON::BI__builtin_neon_vaddq_p128: {
5440 llvm::Type *Ty = GetNeonType(this, NeonTypeFlags::Poly128);
5441 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5442 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5443 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]);
5444 llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
5445 return Builder.CreateBitCast(Ops[0], Int128Ty);
5446 }
5447 case NEON::BI__builtin_neon_vldrq_p128: {
5448 llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
5449 return Builder.CreateAlignedLoad(Int128Ty, Ops[0],
5450 CharUnits::fromQuantity(16));
5451 }
5452 case NEON::BI__builtin_neon_vstrq_p128: {
5453 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5454 }
5455 case NEON::BI__builtin_neon_vcvts_f32_u32:
5456 case NEON::BI__builtin_neon_vcvtd_f64_u64:
5457 usgn = true;
5458 [[fallthrough]];
5459 case NEON::BI__builtin_neon_vcvts_f32_s32:
5460 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
5461 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5462 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
5463 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
5464 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
5465 if (usgn)
5466 return Builder.CreateUIToFP(Ops[0], FTy);
5467 return Builder.CreateSIToFP(Ops[0], FTy);
5468 }
5469 case NEON::BI__builtin_neon_vcvth_f16_u16:
5470 case NEON::BI__builtin_neon_vcvth_f16_u32:
5471 case NEON::BI__builtin_neon_vcvth_f16_u64:
5472 usgn = true;
5473 [[fallthrough]];
5474 case NEON::BI__builtin_neon_vcvth_f16_s16:
5475 case NEON::BI__builtin_neon_vcvth_f16_s32:
5476 case NEON::BI__builtin_neon_vcvth_f16_s64: {
5477 llvm::Type *FTy = HalfTy;
5478 llvm::Type *InTy;
5479 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
5480 InTy = Int64Ty;
5481 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
5482 InTy = Int32Ty;
5483 else
5484 InTy = Int16Ty;
5485 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
5486 if (usgn)
5487 return Builder.CreateUIToFP(Ops[0], FTy);
5488 return Builder.CreateSIToFP(Ops[0], FTy);
5489 }
5490 case NEON::BI__builtin_neon_vcvtah_u16_f16:
5491 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
5492 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
5493 case NEON::BI__builtin_neon_vcvtph_u16_f16:
5494 case NEON::BI__builtin_neon_vcvtah_s16_f16:
5495 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
5496 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
5497 case NEON::BI__builtin_neon_vcvtph_s16_f16: {
5498 llvm::Type *InTy = Int16Ty;
5499 llvm::Type* FTy = HalfTy;
5500 llvm::Type *Tys[2] = {InTy, FTy};
5501 switch (BuiltinID) {
5502 default: llvm_unreachable("missing builtin ID in switch!");
5503 case NEON::BI__builtin_neon_vcvtah_u16_f16:
5504 Int = Intrinsic::aarch64_neon_fcvtau; break;
5505 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
5506 Int = Intrinsic::aarch64_neon_fcvtmu; break;
5507 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
5508 Int = Intrinsic::aarch64_neon_fcvtnu; break;
5509 case NEON::BI__builtin_neon_vcvtph_u16_f16:
5510 Int = Intrinsic::aarch64_neon_fcvtpu; break;
5511 case NEON::BI__builtin_neon_vcvtah_s16_f16:
5512 Int = Intrinsic::aarch64_neon_fcvtas; break;
5513 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
5514 Int = Intrinsic::aarch64_neon_fcvtms; break;
5515 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
5516 Int = Intrinsic::aarch64_neon_fcvtns; break;
5517 case NEON::BI__builtin_neon_vcvtph_s16_f16:
5518 Int = Intrinsic::aarch64_neon_fcvtps; break;
5519 }
5520 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvt");
5521 }
5522 case NEON::BI__builtin_neon_vcaleh_f16:
5523 case NEON::BI__builtin_neon_vcalth_f16:
5524 case NEON::BI__builtin_neon_vcageh_f16:
5525 case NEON::BI__builtin_neon_vcagth_f16: {
5526 llvm::Type* InTy = Int32Ty;
5527 llvm::Type* FTy = HalfTy;
5528 llvm::Type *Tys[2] = {InTy, FTy};
5529 switch (BuiltinID) {
5530 default: llvm_unreachable("missing builtin ID in switch!");
5531 case NEON::BI__builtin_neon_vcageh_f16:
5532 Int = Intrinsic::aarch64_neon_facge; break;
5533 case NEON::BI__builtin_neon_vcagth_f16:
5534 Int = Intrinsic::aarch64_neon_facgt; break;
5535 case NEON::BI__builtin_neon_vcaleh_f16:
5536 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]); break;
5537 case NEON::BI__builtin_neon_vcalth_f16:
5538 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]); break;
5539 }
5540 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "facg");
5541 return Builder.CreateTrunc(Ops[0], Int16Ty);
5542 }
5543 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
5544 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
5545 llvm::Type* InTy = Int32Ty;
5546 llvm::Type* FTy = HalfTy;
5547 llvm::Type *Tys[2] = {InTy, FTy};
5548 switch (BuiltinID) {
5549 default: llvm_unreachable("missing builtin ID in switch!");
5550 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
5551 Int = Intrinsic::aarch64_neon_vcvtfp2fxs; break;
5552 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
5553 Int = Intrinsic::aarch64_neon_vcvtfp2fxu; break;
5554 }
5555 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n");
5556 return Builder.CreateTrunc(Ops[0], Int16Ty);
5557 }
5558 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
5559 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
5560 llvm::Type* FTy = HalfTy;
5561 llvm::Type* InTy = Int32Ty;
5562 llvm::Type *Tys[2] = {FTy, InTy};
5563 switch (BuiltinID) {
5564 default: llvm_unreachable("missing builtin ID in switch!");
5565 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
5566 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
5567 Ops[0] = Builder.CreateSExt(Ops[0], InTy, "sext");
5568 break;
5569 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
5570 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
5571 Ops[0] = Builder.CreateZExt(Ops[0], InTy);
5572 break;
5573 }
5574 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n");
5575 }
5576 case NEON::BI__builtin_neon_vpaddd_s64: {
5577 // TODO: Isn't this handled by
5578 // EmitCommonNeonSISDBuiltinExpr?
5579 auto *Ty = llvm::FixedVectorType::get(Int64Ty, 2);
5580 // The vector is v2f64, so make sure it's bitcast to that.
5581 Ops[0] = Builder.CreateBitCast(Ops[0], Ty, "v2i64");
5582 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
5583 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
5584 Value *Op0 = Builder.CreateExtractElement(Ops[0], Idx0, "lane0");
5585 Value *Op1 = Builder.CreateExtractElement(Ops[0], Idx1, "lane1");
5586 // Pairwise addition of a v2f64 into a scalar f64.
5587 return Builder.CreateAdd(Op0, Op1, "vpaddd");
5588 }
5589 case NEON::BI__builtin_neon_vpaddd_f64: {
5590 auto *Ty = llvm::FixedVectorType::get(DoubleTy, 2);
5591 // The vector is v2f64, so make sure it's bitcast to that.
5592 Ops[0] = Builder.CreateBitCast(Ops[0], Ty, "v2f64");
5593 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
5594 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
5595 Value *Op0 = Builder.CreateExtractElement(Ops[0], Idx0, "lane0");
5596 Value *Op1 = Builder.CreateExtractElement(Ops[0], Idx1, "lane1");
5597 // Pairwise addition of a v2f64 into a scalar f64.
5598 return Builder.CreateFAdd(Op0, Op1, "vpaddd");
5599 }
5600 case NEON::BI__builtin_neon_vpadds_f32: {
5601 auto *Ty = llvm::FixedVectorType::get(FloatTy, 2);
5602 // The vector is v2f32, so make sure it's bitcast to that.
5603 Ops[0] = Builder.CreateBitCast(Ops[0], Ty, "v2f32");
5604 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
5605 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
5606 Value *Op0 = Builder.CreateExtractElement(Ops[0], Idx0, "lane0");
5607 Value *Op1 = Builder.CreateExtractElement(Ops[0], Idx1, "lane1");
5608 // Pairwise addition of a v2f32 into a scalar f32.
5609 return Builder.CreateFAdd(Op0, Op1, "vpaddd");
5610 }
5611 case NEON::BI__builtin_neon_vceqzd_s64:
5612 return EmitAArch64CompareBuiltinExpr(
5613 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5614 ICmpInst::ICMP_EQ, "vceqz");
5615 case NEON::BI__builtin_neon_vceqzd_f64:
5616 case NEON::BI__builtin_neon_vceqzs_f32:
5617 case NEON::BI__builtin_neon_vceqzh_f16:
5618 return EmitAArch64CompareBuiltinExpr(
5619 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5620 ICmpInst::FCMP_OEQ, "vceqz");
5621 case NEON::BI__builtin_neon_vcgezd_s64:
5622 return EmitAArch64CompareBuiltinExpr(
5623 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5624 ICmpInst::ICMP_SGE, "vcgez");
5625 case NEON::BI__builtin_neon_vcgezd_f64:
5626 case NEON::BI__builtin_neon_vcgezs_f32:
5627 case NEON::BI__builtin_neon_vcgezh_f16:
5628 return EmitAArch64CompareBuiltinExpr(
5629 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5630 ICmpInst::FCMP_OGE, "vcgez");
5631 case NEON::BI__builtin_neon_vclezd_s64:
5632 return EmitAArch64CompareBuiltinExpr(
5633 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5634 ICmpInst::ICMP_SLE, "vclez");
5635 case NEON::BI__builtin_neon_vclezd_f64:
5636 case NEON::BI__builtin_neon_vclezs_f32:
5637 case NEON::BI__builtin_neon_vclezh_f16:
5638 return EmitAArch64CompareBuiltinExpr(
5639 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5640 ICmpInst::FCMP_OLE, "vclez");
5641 case NEON::BI__builtin_neon_vcgtzd_s64:
5642 return EmitAArch64CompareBuiltinExpr(
5643 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5644 ICmpInst::ICMP_SGT, "vcgtz");
5645 case NEON::BI__builtin_neon_vcgtzd_f64:
5646 case NEON::BI__builtin_neon_vcgtzs_f32:
5647 case NEON::BI__builtin_neon_vcgtzh_f16:
5648 return EmitAArch64CompareBuiltinExpr(
5649 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5650 ICmpInst::FCMP_OGT, "vcgtz");
5651 case NEON::BI__builtin_neon_vcltzd_s64:
5652 return EmitAArch64CompareBuiltinExpr(
5653 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5654 ICmpInst::ICMP_SLT, "vcltz");
5655
5656 case NEON::BI__builtin_neon_vcltzd_f64:
5657 case NEON::BI__builtin_neon_vcltzs_f32:
5658 case NEON::BI__builtin_neon_vcltzh_f16:
5659 return EmitAArch64CompareBuiltinExpr(
5660 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5661 ICmpInst::FCMP_OLT, "vcltz");
5662
5663 case NEON::BI__builtin_neon_vceqzd_u64: {
5664 return EmitAArch64CompareBuiltinExpr(
5665 Ops[0], ConvertType(E->getCallReturnType(getContext())),
5666 ICmpInst::ICMP_EQ, "vceqzd");
5667 }
5668 case NEON::BI__builtin_neon_vceqd_f64:
5669 case NEON::BI__builtin_neon_vcled_f64:
5670 case NEON::BI__builtin_neon_vcltd_f64:
5671 case NEON::BI__builtin_neon_vcged_f64:
5672 case NEON::BI__builtin_neon_vcgtd_f64: {
5673 llvm::CmpInst::Predicate P;
5674 switch (BuiltinID) {
5675 default: llvm_unreachable("missing builtin ID in switch!");
5676 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break;
5677 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break;
5678 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break;
5679 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break;
5680 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break;
5681 }
5682 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
5683 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
5684 if (P == llvm::FCmpInst::FCMP_OEQ)
5685 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
5686 else
5687 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
5688 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd");
5689 }
5690 case NEON::BI__builtin_neon_vceqs_f32:
5691 case NEON::BI__builtin_neon_vcles_f32:
5692 case NEON::BI__builtin_neon_vclts_f32:
5693 case NEON::BI__builtin_neon_vcges_f32:
5694 case NEON::BI__builtin_neon_vcgts_f32: {
5695 llvm::CmpInst::Predicate P;
5696 switch (BuiltinID) {
5697 default: llvm_unreachable("missing builtin ID in switch!");
5698 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break;
5699 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break;
5700 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break;
5701 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break;
5702 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break;
5703 }
5704 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
5705 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
5706 if (P == llvm::FCmpInst::FCMP_OEQ)
5707 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
5708 else
5709 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
5710 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd");
5711 }
5712 case NEON::BI__builtin_neon_vceqh_f16:
5713 case NEON::BI__builtin_neon_vcleh_f16:
5714 case NEON::BI__builtin_neon_vclth_f16:
5715 case NEON::BI__builtin_neon_vcgeh_f16:
5716 case NEON::BI__builtin_neon_vcgth_f16: {
5717 llvm::CmpInst::Predicate P;
5718 switch (BuiltinID) {
5719 default: llvm_unreachable("missing builtin ID in switch!");
5720 case NEON::BI__builtin_neon_vceqh_f16: P = llvm::FCmpInst::FCMP_OEQ; break;
5721 case NEON::BI__builtin_neon_vcleh_f16: P = llvm::FCmpInst::FCMP_OLE; break;
5722 case NEON::BI__builtin_neon_vclth_f16: P = llvm::FCmpInst::FCMP_OLT; break;
5723 case NEON::BI__builtin_neon_vcgeh_f16: P = llvm::FCmpInst::FCMP_OGE; break;
5724 case NEON::BI__builtin_neon_vcgth_f16: P = llvm::FCmpInst::FCMP_OGT; break;
5725 }
5726 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
5727 Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy);
5728 if (P == llvm::FCmpInst::FCMP_OEQ)
5729 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
5730 else
5731 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
5732 return Builder.CreateSExt(Ops[0], Int16Ty, "vcmpd");
5733 }
5734 case NEON::BI__builtin_neon_vceqd_s64:
5735 case NEON::BI__builtin_neon_vceqd_u64:
5736 case NEON::BI__builtin_neon_vcgtd_s64:
5737 case NEON::BI__builtin_neon_vcgtd_u64:
5738 case NEON::BI__builtin_neon_vcltd_s64:
5739 case NEON::BI__builtin_neon_vcltd_u64:
5740 case NEON::BI__builtin_neon_vcged_u64:
5741 case NEON::BI__builtin_neon_vcged_s64:
5742 case NEON::BI__builtin_neon_vcled_u64:
5743 case NEON::BI__builtin_neon_vcled_s64: {
5744 llvm::CmpInst::Predicate P;
5745 switch (BuiltinID) {
5746 default: llvm_unreachable("missing builtin ID in switch!");
5747 case NEON::BI__builtin_neon_vceqd_s64:
5748 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break;
5749 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break;
5750 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break;
5751 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break;
5752 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break;
5753 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break;
5754 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break;
5755 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break;
5756 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break;
5757 }
5758 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
5759 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
5760 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
5761 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd");
5762 }
5763 case NEON::BI__builtin_neon_vnegd_s64:
5764 return Builder.CreateNeg(Ops[0], "vnegd");
5765 case NEON::BI__builtin_neon_vnegh_f16:
5766 return Builder.CreateFNeg(Ops[0], "vnegh");
5767 case NEON::BI__builtin_neon_vtstd_s64:
5768 case NEON::BI__builtin_neon_vtstd_u64: {
5769 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
5770 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
5771 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
5772 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
5773 llvm::Constant::getNullValue(Int64Ty));
5774 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd");
5775 }
5776 case NEON::BI__builtin_neon_vset_lane_i8:
5777 case NEON::BI__builtin_neon_vset_lane_i16:
5778 case NEON::BI__builtin_neon_vset_lane_i32:
5779 case NEON::BI__builtin_neon_vset_lane_i64:
5780 case NEON::BI__builtin_neon_vset_lane_bf16:
5781 case NEON::BI__builtin_neon_vset_lane_f32:
5782 case NEON::BI__builtin_neon_vsetq_lane_i8:
5783 case NEON::BI__builtin_neon_vsetq_lane_i16:
5784 case NEON::BI__builtin_neon_vsetq_lane_i32:
5785 case NEON::BI__builtin_neon_vsetq_lane_i64:
5786 case NEON::BI__builtin_neon_vsetq_lane_bf16:
5787 case NEON::BI__builtin_neon_vsetq_lane_f32:
5788 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
5789 case NEON::BI__builtin_neon_vset_lane_f64:
5790 // The vector type needs a cast for the v1f64 variant.
5791 Ops[1] =
5792 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 1));
5793 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
5794 case NEON::BI__builtin_neon_vset_lane_mf8:
5795 case NEON::BI__builtin_neon_vsetq_lane_mf8:
5796 // The input vector type needs a cast to scalar type.
5797 Ops[0] =
5798 Builder.CreateBitCast(Ops[0], llvm::Type::getInt8Ty(getLLVMContext()));
5799 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
5800 case NEON::BI__builtin_neon_vsetq_lane_f64:
5801 // The vector type needs a cast for the v2f64 variant.
5802 Ops[1] =
5803 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 2));
5804 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
5805
5806 case NEON::BI__builtin_neon_vget_lane_i8:
5807 case NEON::BI__builtin_neon_vdupb_lane_i8:
5808 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5809 case NEON::BI__builtin_neon_vgetq_lane_i8:
5810 case NEON::BI__builtin_neon_vdupb_laneq_i8:
5811 return Builder.CreateExtractElement(Ops[0], Ops[1], "vgetq_lane");
5812 case NEON::BI__builtin_neon_vget_lane_mf8:
5813 case NEON::BI__builtin_neon_vdupb_lane_mf8:
5814 case NEON::BI__builtin_neon_vgetq_lane_mf8:
5815 case NEON::BI__builtin_neon_vdupb_laneq_mf8:
5816 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5817 case NEON::BI__builtin_neon_vget_lane_i16:
5818 case NEON::BI__builtin_neon_vduph_lane_i16:
5819 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5820 case NEON::BI__builtin_neon_vgetq_lane_i16:
5821 case NEON::BI__builtin_neon_vduph_laneq_i16:
5822 return Builder.CreateExtractElement(Ops[0], Ops[1], "vgetq_lane");
5823 case NEON::BI__builtin_neon_vget_lane_i32:
5824 case NEON::BI__builtin_neon_vdups_lane_i32:
5825 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5826 case NEON::BI__builtin_neon_vdups_lane_f32:
5827 return Builder.CreateExtractElement(Ops[0], Ops[1], "vdups_lane");
5828 case NEON::BI__builtin_neon_vgetq_lane_i32:
5829 case NEON::BI__builtin_neon_vdups_laneq_i32:
5830 return Builder.CreateExtractElement(Ops[0], Ops[1], "vgetq_lane");
5831 case NEON::BI__builtin_neon_vget_lane_i64:
5832 case NEON::BI__builtin_neon_vdupd_lane_i64:
5833 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5834 case NEON::BI__builtin_neon_vdupd_lane_f64:
5835 return Builder.CreateExtractElement(Ops[0], Ops[1], "vdupd_lane");
5836 case NEON::BI__builtin_neon_vgetq_lane_i64:
5837 case NEON::BI__builtin_neon_vdupd_laneq_i64:
5838 return Builder.CreateExtractElement(Ops[0], Ops[1], "vgetq_lane");
5839 case NEON::BI__builtin_neon_vget_lane_f32:
5840 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5841 case NEON::BI__builtin_neon_vget_lane_f64:
5842 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5843 case NEON::BI__builtin_neon_vgetq_lane_f32:
5844 case NEON::BI__builtin_neon_vdups_laneq_f32:
5845 return Builder.CreateExtractElement(Ops[0], Ops[1], "vgetq_lane");
5846 case NEON::BI__builtin_neon_vgetq_lane_f64:
5847 case NEON::BI__builtin_neon_vdupd_laneq_f64:
5848 return Builder.CreateExtractElement(Ops[0], Ops[1], "vgetq_lane");
5849 case NEON::BI__builtin_neon_vaddh_f16:
5850 return Builder.CreateFAdd(Ops[0], Ops[1], "vaddh");
5851 case NEON::BI__builtin_neon_vsubh_f16:
5852 return Builder.CreateFSub(Ops[0], Ops[1], "vsubh");
5853 case NEON::BI__builtin_neon_vmulh_f16:
5854 return Builder.CreateFMul(Ops[0], Ops[1], "vmulh");
5855 case NEON::BI__builtin_neon_vdivh_f16:
5856 return Builder.CreateFDiv(Ops[0], Ops[1], "vdivh");
5857 case NEON::BI__builtin_neon_vfmah_f16:
5858 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
5859 return emitCallMaybeConstrainedFPBuiltin(
5860 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, HalfTy,
5861 {Ops[1], Ops[2], Ops[0]});
5862 case NEON::BI__builtin_neon_vfmsh_f16: {
5863 Value *Neg = Builder.CreateFNeg(Ops[1], "vsubh");
5864
5865 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
5866 return emitCallMaybeConstrainedFPBuiltin(
5867 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, HalfTy,
5868 {Neg, Ops[2], Ops[0]});
5869 }
5870 case NEON::BI__builtin_neon_vaddd_s64:
5871 case NEON::BI__builtin_neon_vaddd_u64:
5872 return Builder.CreateAdd(Ops[0], Ops[1], "vaddd");
5873 case NEON::BI__builtin_neon_vsubd_s64:
5874 case NEON::BI__builtin_neon_vsubd_u64:
5875 return Builder.CreateSub(Ops[0], Ops[1], "vsubd");
5876 case NEON::BI__builtin_neon_vqdmlalh_s16:
5877 case NEON::BI__builtin_neon_vqdmlslh_s16: {
5878 SmallVector<Value *, 2> ProductOps;
5879 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
5880 ProductOps.push_back(vectorWrapScalar16(Ops[2]));
5881 auto *VTy = llvm::FixedVectorType::get(Int32Ty, 4);
5882 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
5883 ProductOps, "vqdmlXl");
5884 Constant *CI = ConstantInt::get(SizeTy, 0);
5885 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
5886
5887 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
5888 ? Intrinsic::aarch64_neon_sqadd
5889 : Intrinsic::aarch64_neon_sqsub;
5890 // Drop the 2nd multiplication argument before the accumulation
5891 Ops.pop_back();
5892 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl");
5893 }
5894 case NEON::BI__builtin_neon_vqshlud_n_s64: {
5895 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
5896 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty),
5897 Ops, "vqshlu_n");
5898 }
5899 case NEON::BI__builtin_neon_vqshld_n_u64:
5900 case NEON::BI__builtin_neon_vqshld_n_s64: {
5901 Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
5902 ? Intrinsic::aarch64_neon_uqshl
5903 : Intrinsic::aarch64_neon_sqshl;
5904 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
5905 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n");
5906 }
5907 case NEON::BI__builtin_neon_vrshrd_n_u64:
5908 case NEON::BI__builtin_neon_vrshrd_n_s64: {
5909 Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
5910 ? Intrinsic::aarch64_neon_urshl
5911 : Intrinsic::aarch64_neon_srshl;
5912 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
5913 Ops[1] = ConstantInt::get(Int64Ty, -SV);
5914 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n");
5915 }
5916 case NEON::BI__builtin_neon_vrsrad_n_u64:
5917 case NEON::BI__builtin_neon_vrsrad_n_s64: {
5918 Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
5919 ? Intrinsic::aarch64_neon_urshl
5920 : Intrinsic::aarch64_neon_srshl;
5921 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
5922 Ops[2] = Builder.CreateNeg(Ops[2]);
5923 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty),
5924 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
5925 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
5926 }
5927 case NEON::BI__builtin_neon_vshld_n_s64:
5928 case NEON::BI__builtin_neon_vshld_n_u64: {
5929 llvm::ConstantInt *Amt = cast<ConstantInt>(Ops[1]);
5930 return Builder.CreateShl(
5931 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n");
5932 }
5933 case NEON::BI__builtin_neon_vshrd_n_s64: {
5934 llvm::ConstantInt *Amt = cast<ConstantInt>(Ops[1]);
5935 return Builder.CreateAShr(
5936 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
5937 Amt->getZExtValue())),
5938 "shrd_n");
5939 }
5940 case NEON::BI__builtin_neon_vshrd_n_u64: {
5941 llvm::ConstantInt *Amt = cast<ConstantInt>(Ops[1]);
5942 uint64_t ShiftAmt = Amt->getZExtValue();
5943 // Right-shifting an unsigned value by its size yields 0.
5944 if (ShiftAmt == 64)
5945 return ConstantInt::get(Int64Ty, 0);
5946 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt),
5947 "shrd_n");
5948 }
5949 case NEON::BI__builtin_neon_vsrad_n_s64: {
5950 llvm::ConstantInt *Amt = cast<ConstantInt>(Ops[2]);
5951 Ops[1] = Builder.CreateAShr(
5952 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
5953 Amt->getZExtValue())),
5954 "shrd_n");
5955 return Builder.CreateAdd(Ops[0], Ops[1]);
5956 }
5957 case NEON::BI__builtin_neon_vsrad_n_u64: {
5958 llvm::ConstantInt *Amt = cast<ConstantInt>(Ops[2]);
5959 uint64_t ShiftAmt = Amt->getZExtValue();
5960 // Right-shifting an unsigned value by its size yields 0.
5961 // As Op + 0 = Op, return Ops[0] directly.
5962 if (ShiftAmt == 64)
5963 return Ops[0];
5964 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
5965 "shrd_n");
5966 return Builder.CreateAdd(Ops[0], Ops[1]);
5967 }
5968 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
5969 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
5970 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
5971 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
5972 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "lane");
5973 SmallVector<Value *, 2> ProductOps;
5974 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
5975 ProductOps.push_back(vectorWrapScalar16(Ops[2]));
5976 auto *VTy = llvm::FixedVectorType::get(Int32Ty, 4);
5977 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
5978 ProductOps, "vqdmlXl");
5979 Constant *CI = ConstantInt::get(SizeTy, 0);
5980 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
5981 // Drop lane-selection and the corresponding vector argument (these have
5982 // already been used)
5983 Ops.pop_back_n(2);
5984
5985 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
5986 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
5987 ? Intrinsic::aarch64_neon_sqadd
5988 : Intrinsic::aarch64_neon_sqsub;
5989 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl");
5990 }
5991 case NEON::BI__builtin_neon_vqdmlals_s32:
5992 case NEON::BI__builtin_neon_vqdmlsls_s32: {
5993 SmallVector<Value *, 2> ProductOps;
5994 ProductOps.push_back(Ops[1]);
5995 ProductOps.push_back(Ops[2]);
5996 Ops[1] =
5997 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
5998 ProductOps, "vqdmlXl");
5999
6000 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
6001 ? Intrinsic::aarch64_neon_sqadd
6002 : Intrinsic::aarch64_neon_sqsub;
6003 // Drop the 2nd multiplication argument before the accumulation
6004 Ops.pop_back();
6005 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl");
6006 }
6007 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
6008 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
6009 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
6010 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
6011 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "lane");
6012 SmallVector<Value *, 2> ProductOps;
6013 ProductOps.push_back(Ops[1]);
6014 ProductOps.push_back(Ops[2]);
6015 Ops[1] =
6016 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
6017 ProductOps, "vqdmlXl");
6018 // Drop lane-selection and the corresponding vector argument (these have
6019 // already been used)
6020 Ops.pop_back_n(2);
6021
6022 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
6023 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
6024 ? Intrinsic::aarch64_neon_sqadd
6025 : Intrinsic::aarch64_neon_sqsub;
6026 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl");
6027 }
6028 case NEON::BI__builtin_neon_vget_lane_bf16:
6029 case NEON::BI__builtin_neon_vduph_lane_bf16:
6030 case NEON::BI__builtin_neon_vduph_lane_f16: {
6031 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
6032 }
6033 case NEON::BI__builtin_neon_vgetq_lane_bf16:
6034 case NEON::BI__builtin_neon_vduph_laneq_bf16:
6035 case NEON::BI__builtin_neon_vduph_laneq_f16: {
6036 return Builder.CreateExtractElement(Ops[0], Ops[1], "vgetq_lane");
6037 }
6038 case NEON::BI__builtin_neon_vcvt_bf16_f32: {
6039 llvm::Type *V4F32 = FixedVectorType::get(Builder.getFloatTy(), 4);
6040 llvm::Type *V4BF16 = FixedVectorType::get(Builder.getBFloatTy(), 4);
6041 return Builder.CreateFPTrunc(Builder.CreateBitCast(Ops[0], V4F32), V4BF16);
6042 }
6043 case NEON::BI__builtin_neon_vcvtq_low_bf16_f32: {
6044 SmallVector<int, 16> ConcatMask(8);
6045 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
6046 llvm::Type *V4F32 = FixedVectorType::get(Builder.getFloatTy(), 4);
6047 llvm::Type *V4BF16 = FixedVectorType::get(Builder.getBFloatTy(), 4);
6048 llvm::Value *Trunc =
6049 Builder.CreateFPTrunc(Builder.CreateBitCast(Ops[0], V4F32), V4BF16);
6050 return Builder.CreateShuffleVector(
6051 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
6052 }
6053 case NEON::BI__builtin_neon_vcvtq_high_bf16_f32: {
6054 SmallVector<int, 16> ConcatMask(8);
6055 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
6056 SmallVector<int, 16> LoMask(4);
6057 std::iota(LoMask.begin(), LoMask.end(), 0);
6058 llvm::Type *V4F32 = FixedVectorType::get(Builder.getFloatTy(), 4);
6059 llvm::Type *V4BF16 = FixedVectorType::get(Builder.getBFloatTy(), 4);
6060 llvm::Type *V8BF16 = FixedVectorType::get(Builder.getBFloatTy(), 8);
6061 llvm::Value *Inactive = Builder.CreateShuffleVector(
6062 Builder.CreateBitCast(Ops[0], V8BF16), LoMask);
6063 llvm::Value *Trunc =
6064 Builder.CreateFPTrunc(Builder.CreateBitCast(Ops[1], V4F32), V4BF16);
6065 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
6066 }
6067 case NEON::BI__builtin_neon_vcvt_f16_f32: {
6068 llvm::Type *V4F32 = FixedVectorType::get(Builder.getFloatTy(), 4);
6069 llvm::Type *V4F16 = FixedVectorType::get(Builder.getHalfTy(), 4);
6070 return Builder.CreateFPTrunc(Builder.CreateBitCast(Ops[0], V4F32), V4F16);
6071 }
6072 case NEON::BI__builtin_neon_vcvt_f32_f16: {
6073 llvm::Type *V4F32 = FixedVectorType::get(Builder.getFloatTy(), 4);
6074 llvm::Type *V4F16 = FixedVectorType::get(Builder.getHalfTy(), 4);
6075 return Builder.CreateFPExt(Builder.CreateBitCast(Ops[0], V4F16), V4F32);
6076 }
6077
6078 case clang::AArch64::BI_InterlockedAdd:
6079 case clang::AArch64::BI_InterlockedAdd_acq:
6080 case clang::AArch64::BI_InterlockedAdd_rel:
6081 case clang::AArch64::BI_InterlockedAdd_nf:
6082 case clang::AArch64::BI_InterlockedAdd64:
6083 case clang::AArch64::BI_InterlockedAdd64_acq:
6084 case clang::AArch64::BI_InterlockedAdd64_rel:
6085 case clang::AArch64::BI_InterlockedAdd64_nf: {
6086 Address DestAddr = CheckAtomicAlignment(*this, E);
6087 Value *Val = Ops[1];
6088 llvm::AtomicOrdering Ordering;
6089 switch (BuiltinID) {
6090 case clang::AArch64::BI_InterlockedAdd:
6091 case clang::AArch64::BI_InterlockedAdd64:
6092 Ordering = llvm::AtomicOrdering::SequentiallyConsistent;
6093 break;
6094 case clang::AArch64::BI_InterlockedAdd_acq:
6095 case clang::AArch64::BI_InterlockedAdd64_acq:
6096 Ordering = llvm::AtomicOrdering::Acquire;
6097 break;
6098 case clang::AArch64::BI_InterlockedAdd_rel:
6099 case clang::AArch64::BI_InterlockedAdd64_rel:
6100 Ordering = llvm::AtomicOrdering::Release;
6101 break;
6102 case clang::AArch64::BI_InterlockedAdd_nf:
6103 case clang::AArch64::BI_InterlockedAdd64_nf:
6104 Ordering = llvm::AtomicOrdering::Monotonic;
6105 break;
6106 default:
6107 llvm_unreachable("missing builtin ID in switch!");
6108 }
6109 AtomicRMWInst *RMWI =
6110 Builder.CreateAtomicRMW(AtomicRMWInst::Add, DestAddr, Val, Ordering);
6111 return Builder.CreateAdd(RMWI, Val);
6112 }
6113 }
6114
6115 llvm::FixedVectorType *VTy = GetNeonType(this, Type);
6116 llvm::Type *Ty = VTy;
6117 if (!Ty)
6118 return nullptr;
6119
6120 bool ExtractLow = false;
6121 bool ExtendLaneArg = false;
6122 switch (BuiltinID) {
6123 default: return nullptr;
6124 case NEON::BI__builtin_neon_vbsl_v:
6125 case NEON::BI__builtin_neon_vbslq_v: {
6126 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
6127 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl");
6128 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl");
6129 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl");
6130
6131 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl");
6132 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl");
6133 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl");
6134 return Builder.CreateBitCast(Ops[0], Ty);
6135 }
6136 case NEON::BI__builtin_neon_vfma_lane_v:
6137 case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types
6138 // The ARM builtins (and instructions) have the addend as the first
6139 // operand, but the 'fma' intrinsics have it last. Swap it around here.
6140 Value *Addend = Ops[0];
6141 Value *Multiplicand = Ops[1];
6142 Value *LaneSource = Ops[2];
6143 Ops[0] = Multiplicand;
6144 Ops[1] = LaneSource;
6145 Ops[2] = Addend;
6146
6147 // Now adjust things to handle the lane access.
6148 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
6149 ? llvm::FixedVectorType::get(VTy->getElementType(),
6150 VTy->getNumElements() / 2)
6151 : VTy;
6152 llvm::Constant *cst = cast<Constant>(Ops[3]);
6153 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
6154 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
6155 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
6156
6157 Ops.pop_back();
6158 Int = Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
6159 : Intrinsic::fma;
6160 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla");
6161 }
6162 case NEON::BI__builtin_neon_vfma_laneq_v: {
6163 auto *VTy = cast<llvm::FixedVectorType>(Ty);
6164 // v1f64 fma should be mapped to Neon scalar f64 fma
6165 if (VTy && VTy->getElementType() == DoubleTy) {
6166 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
6167 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
6168 llvm::FixedVectorType *VTy =
6169 GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, true));
6170 Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
6171 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
6172 Value *Result;
6173 Result = emitCallMaybeConstrainedFPBuiltin(
6174 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
6175 DoubleTy, {Ops[1], Ops[2], Ops[0]});
6176 return Builder.CreateBitCast(Result, Ty);
6177 }
6178 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6179 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6180
6181 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
6182 VTy->getNumElements() * 2);
6183 Ops[2] = Builder.CreateBitCast(Ops[2], STy);
6184 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
6185 cast<ConstantInt>(Ops[3]));
6186 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
6187
6188 return emitCallMaybeConstrainedFPBuiltin(
6189 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
6190 {Ops[2], Ops[1], Ops[0]});
6191 }
6192 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
6193 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6194 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6195
6196 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6197 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
6198 return emitCallMaybeConstrainedFPBuiltin(
6199 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
6200 {Ops[2], Ops[1], Ops[0]});
6201 }
6202 case NEON::BI__builtin_neon_vfmah_lane_f16:
6203 case NEON::BI__builtin_neon_vfmas_lane_f32:
6204 case NEON::BI__builtin_neon_vfmah_laneq_f16:
6205 case NEON::BI__builtin_neon_vfmas_laneq_f32:
6206 case NEON::BI__builtin_neon_vfmad_lane_f64:
6207 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
6208 llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
6209 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
6210 return emitCallMaybeConstrainedFPBuiltin(
6211 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
6212 {Ops[1], Ops[2], Ops[0]});
6213 }
6214 case NEON::BI__builtin_neon_vmull_v:
6215 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6216 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
6217 if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
6218 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
6219 case NEON::BI__builtin_neon_vmax_v:
6220 case NEON::BI__builtin_neon_vmaxq_v:
6221 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6222 Int = usgn ? Intrinsic::umax : Intrinsic::smax;
6223 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
6224 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax");
6225 case NEON::BI__builtin_neon_vmaxh_f16: {
6226 Int = Intrinsic::aarch64_neon_fmax;
6227 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmax");
6228 }
6229 case NEON::BI__builtin_neon_vmin_v:
6230 case NEON::BI__builtin_neon_vminq_v:
6231 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6232 Int = usgn ? Intrinsic::umin : Intrinsic::smin;
6233 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
6234 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin");
6235 case NEON::BI__builtin_neon_vminh_f16: {
6236 Int = Intrinsic::aarch64_neon_fmin;
6237 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmin");
6238 }
6239 case NEON::BI__builtin_neon_vabd_v:
6240 case NEON::BI__builtin_neon_vabdq_v:
6241 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6242 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
6243 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
6244 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd");
6245 case NEON::BI__builtin_neon_vpadal_v:
6246 case NEON::BI__builtin_neon_vpadalq_v: {
6247 unsigned ArgElts = VTy->getNumElements();
6248 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
6249 unsigned BitWidth = EltTy->getBitWidth();
6250 auto *ArgTy = llvm::FixedVectorType::get(
6251 llvm::IntegerType::get(getLLVMContext(), BitWidth / 2), 2 * ArgElts);
6252 llvm::Type* Tys[2] = { VTy, ArgTy };
6253 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
6254 SmallVector<llvm::Value*, 1> TmpOps;
6255 TmpOps.push_back(Ops[1]);
6256 Function *F = CGM.getIntrinsic(Int, Tys);
6257 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal");
6258 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
6259 return Builder.CreateAdd(tmp, addend);
6260 }
6261 case NEON::BI__builtin_neon_vpmin_v:
6262 case NEON::BI__builtin_neon_vpminq_v:
6263 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6264 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
6265 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
6266 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin");
6267 case NEON::BI__builtin_neon_vpmax_v:
6268 case NEON::BI__builtin_neon_vpmaxq_v:
6269 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6270 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
6271 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
6272 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax");
6273 case NEON::BI__builtin_neon_vminnm_v:
6274 case NEON::BI__builtin_neon_vminnmq_v:
6275 Int = Intrinsic::aarch64_neon_fminnm;
6276 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm");
6277 case NEON::BI__builtin_neon_vminnmh_f16:
6278 Int = Intrinsic::aarch64_neon_fminnm;
6279 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vminnm");
6280 case NEON::BI__builtin_neon_vmaxnm_v:
6281 case NEON::BI__builtin_neon_vmaxnmq_v:
6282 Int = Intrinsic::aarch64_neon_fmaxnm;
6283 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm");
6284 case NEON::BI__builtin_neon_vmaxnmh_f16:
6285 Int = Intrinsic::aarch64_neon_fmaxnm;
6286 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmaxnm");
6287 case NEON::BI__builtin_neon_vrecpss_f32: {
6288 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy),
6289 Ops, "vrecps");
6290 }
6291 case NEON::BI__builtin_neon_vrecpsd_f64:
6292 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy),
6293 Ops, "vrecps");
6294 case NEON::BI__builtin_neon_vrecpsh_f16:
6295 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, HalfTy),
6296 Ops, "vrecps");
6297 case NEON::BI__builtin_neon_vqshrun_n_v:
6298 Int = Intrinsic::aarch64_neon_sqshrun;
6299 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n");
6300 case NEON::BI__builtin_neon_vqrshrun_n_v:
6301 Int = Intrinsic::aarch64_neon_sqrshrun;
6302 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n");
6303 case NEON::BI__builtin_neon_vqshrn_n_v:
6304 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
6305 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n");
6306 case NEON::BI__builtin_neon_vrshrn_n_v:
6307 Int = Intrinsic::aarch64_neon_rshrn;
6308 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n");
6309 case NEON::BI__builtin_neon_vqrshrn_n_v:
6310 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
6311 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n");
6312 case NEON::BI__builtin_neon_vrndah_f16: {
6313 Int = Builder.getIsFPConstrained()
6314 ? Intrinsic::experimental_constrained_round
6315 : Intrinsic::round;
6316 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrnda");
6317 }
6318 case NEON::BI__builtin_neon_vrnda_v:
6319 case NEON::BI__builtin_neon_vrndaq_v: {
6320 Int = Builder.getIsFPConstrained()
6321 ? Intrinsic::experimental_constrained_round
6322 : Intrinsic::round;
6323 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda");
6324 }
6325 case NEON::BI__builtin_neon_vrndih_f16: {
6326 Int = Builder.getIsFPConstrained()
6327 ? Intrinsic::experimental_constrained_nearbyint
6328 : Intrinsic::nearbyint;
6329 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndi");
6330 }
6331 case NEON::BI__builtin_neon_vrndmh_f16: {
6332 Int = Builder.getIsFPConstrained()
6333 ? Intrinsic::experimental_constrained_floor
6334 : Intrinsic::floor;
6335 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndm");
6336 }
6337 case NEON::BI__builtin_neon_vrndm_v:
6338 case NEON::BI__builtin_neon_vrndmq_v: {
6339 Int = Builder.getIsFPConstrained()
6340 ? Intrinsic::experimental_constrained_floor
6341 : Intrinsic::floor;
6342 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm");
6343 }
6344 case NEON::BI__builtin_neon_vrndnh_f16: {
6345 Int = Builder.getIsFPConstrained()
6346 ? Intrinsic::experimental_constrained_roundeven
6347 : Intrinsic::roundeven;
6348 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndn");
6349 }
6350 case NEON::BI__builtin_neon_vrndn_v:
6351 case NEON::BI__builtin_neon_vrndnq_v: {
6352 Int = Builder.getIsFPConstrained()
6353 ? Intrinsic::experimental_constrained_roundeven
6354 : Intrinsic::roundeven;
6355 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn");
6356 }
6357 case NEON::BI__builtin_neon_vrndns_f32: {
6358 Int = Builder.getIsFPConstrained()
6359 ? Intrinsic::experimental_constrained_roundeven
6360 : Intrinsic::roundeven;
6361 return EmitNeonCall(CGM.getIntrinsic(Int, FloatTy), Ops, "vrndn");
6362 }
6363 case NEON::BI__builtin_neon_vrndph_f16: {
6364 Int = Builder.getIsFPConstrained()
6365 ? Intrinsic::experimental_constrained_ceil
6366 : Intrinsic::ceil;
6367 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndp");
6368 }
6369 case NEON::BI__builtin_neon_vrndp_v:
6370 case NEON::BI__builtin_neon_vrndpq_v: {
6371 Int = Builder.getIsFPConstrained()
6372 ? Intrinsic::experimental_constrained_ceil
6373 : Intrinsic::ceil;
6374 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp");
6375 }
6376 case NEON::BI__builtin_neon_vrndxh_f16: {
6377 Int = Builder.getIsFPConstrained()
6378 ? Intrinsic::experimental_constrained_rint
6379 : Intrinsic::rint;
6380 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndx");
6381 }
6382 case NEON::BI__builtin_neon_vrndx_v:
6383 case NEON::BI__builtin_neon_vrndxq_v: {
6384 Int = Builder.getIsFPConstrained()
6385 ? Intrinsic::experimental_constrained_rint
6386 : Intrinsic::rint;
6387 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx");
6388 }
6389 case NEON::BI__builtin_neon_vrndh_f16: {
6390 Int = Builder.getIsFPConstrained()
6391 ? Intrinsic::experimental_constrained_trunc
6392 : Intrinsic::trunc;
6393 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndz");
6394 }
6395 case NEON::BI__builtin_neon_vrnd_v:
6396 case NEON::BI__builtin_neon_vrndq_v: {
6397 Int = Builder.getIsFPConstrained()
6398 ? Intrinsic::experimental_constrained_trunc
6399 : Intrinsic::trunc;
6400 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz");
6401 }
6402 case NEON::BI__builtin_neon_vcvt_f64_v:
6403 case NEON::BI__builtin_neon_vcvtq_f64_v:
6404 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6405 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
6406 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
6407 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
6408 case NEON::BI__builtin_neon_vcvt_f64_f32: {
6409 assert(Type.getEltType() == NeonTypeFlags::Float64 && quad &&
6410 "unexpected vcvt_f64_f32 builtin");
6411 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false);
6412 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
6413
6414 return Builder.CreateFPExt(Ops[0], Ty, "vcvt");
6415 }
6416 case NEON::BI__builtin_neon_vcvt_f32_f64: {
6417 assert(Type.getEltType() == NeonTypeFlags::Float32 &&
6418 "unexpected vcvt_f32_f64 builtin");
6419 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true);
6420 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
6421
6422 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt");
6423 }
6424 case NEON::BI__builtin_neon_vcvta_s16_f16:
6425 case NEON::BI__builtin_neon_vcvta_u16_f16:
6426 case NEON::BI__builtin_neon_vcvta_s32_v:
6427 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
6428 case NEON::BI__builtin_neon_vcvtaq_s32_v:
6429 case NEON::BI__builtin_neon_vcvta_u32_v:
6430 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
6431 case NEON::BI__builtin_neon_vcvtaq_u32_v:
6432 case NEON::BI__builtin_neon_vcvta_s64_v:
6433 case NEON::BI__builtin_neon_vcvtaq_s64_v:
6434 case NEON::BI__builtin_neon_vcvta_u64_v:
6435 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
6436 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
6437 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6438 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta");
6439 }
6440 case NEON::BI__builtin_neon_vcvtm_s16_f16:
6441 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
6442 case NEON::BI__builtin_neon_vcvtm_u16_f16:
6443 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
6444 case NEON::BI__builtin_neon_vcvtm_s32_v:
6445 case NEON::BI__builtin_neon_vcvtmq_s32_v:
6446 case NEON::BI__builtin_neon_vcvtm_u32_v:
6447 case NEON::BI__builtin_neon_vcvtmq_u32_v:
6448 case NEON::BI__builtin_neon_vcvtm_s64_v:
6449 case NEON::BI__builtin_neon_vcvtmq_s64_v:
6450 case NEON::BI__builtin_neon_vcvtm_u64_v:
6451 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
6452 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
6453 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6454 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm");
6455 }
6456 case NEON::BI__builtin_neon_vcvtn_s16_f16:
6457 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
6458 case NEON::BI__builtin_neon_vcvtn_u16_f16:
6459 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
6460 case NEON::BI__builtin_neon_vcvtn_s32_v:
6461 case NEON::BI__builtin_neon_vcvtnq_s32_v:
6462 case NEON::BI__builtin_neon_vcvtn_u32_v:
6463 case NEON::BI__builtin_neon_vcvtnq_u32_v:
6464 case NEON::BI__builtin_neon_vcvtn_s64_v:
6465 case NEON::BI__builtin_neon_vcvtnq_s64_v:
6466 case NEON::BI__builtin_neon_vcvtn_u64_v:
6467 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
6468 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
6469 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6470 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn");
6471 }
6472 case NEON::BI__builtin_neon_vcvtp_s16_f16:
6473 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
6474 case NEON::BI__builtin_neon_vcvtp_u16_f16:
6475 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
6476 case NEON::BI__builtin_neon_vcvtp_s32_v:
6477 case NEON::BI__builtin_neon_vcvtpq_s32_v:
6478 case NEON::BI__builtin_neon_vcvtp_u32_v:
6479 case NEON::BI__builtin_neon_vcvtpq_u32_v:
6480 case NEON::BI__builtin_neon_vcvtp_s64_v:
6481 case NEON::BI__builtin_neon_vcvtpq_s64_v:
6482 case NEON::BI__builtin_neon_vcvtp_u64_v:
6483 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
6484 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
6485 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6486 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp");
6487 }
6488 case NEON::BI__builtin_neon_vmulx_v:
6489 case NEON::BI__builtin_neon_vmulxq_v: {
6490 Int = Intrinsic::aarch64_neon_fmulx;
6491 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx");
6492 }
6493 case NEON::BI__builtin_neon_vmulxh_lane_f16:
6494 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
6495 // vmulx_lane should be mapped to Neon scalar mulx after
6496 // extracting the scalar element
6497 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
6498 Ops.pop_back();
6499 Int = Intrinsic::aarch64_neon_fmulx;
6500 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmulx");
6501 }
6502 case NEON::BI__builtin_neon_vmul_lane_v:
6503 case NEON::BI__builtin_neon_vmul_laneq_v: {
6504 // v1f64 vmul_lane should be mapped to Neon scalar mul lane
6505 bool Quad = false;
6506 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
6507 Quad = true;
6508 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
6509 llvm::FixedVectorType *VTy =
6510 GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, Quad));
6511 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
6512 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
6513 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
6514 return Builder.CreateBitCast(Result, Ty);
6515 }
6516 case NEON::BI__builtin_neon_vpmaxnm_v:
6517 case NEON::BI__builtin_neon_vpmaxnmq_v: {
6518 Int = Intrinsic::aarch64_neon_fmaxnmp;
6519 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm");
6520 }
6521 case NEON::BI__builtin_neon_vpminnm_v:
6522 case NEON::BI__builtin_neon_vpminnmq_v: {
6523 Int = Intrinsic::aarch64_neon_fminnmp;
6524 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm");
6525 }
6526 case NEON::BI__builtin_neon_vsqrth_f16: {
6527 Int = Builder.getIsFPConstrained()
6528 ? Intrinsic::experimental_constrained_sqrt
6529 : Intrinsic::sqrt;
6530 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vsqrt");
6531 }
6532 case NEON::BI__builtin_neon_vsqrt_v:
6533 case NEON::BI__builtin_neon_vsqrtq_v: {
6534 Int = Builder.getIsFPConstrained()
6535 ? Intrinsic::experimental_constrained_sqrt
6536 : Intrinsic::sqrt;
6537 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6538 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt");
6539 }
6540 case NEON::BI__builtin_neon_vrbit_v:
6541 case NEON::BI__builtin_neon_vrbitq_v: {
6542 Int = Intrinsic::bitreverse;
6543 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit");
6544 }
6545 case NEON::BI__builtin_neon_vmaxv_f16: {
6546 Int = Intrinsic::aarch64_neon_fmaxv;
6547 Ty = HalfTy;
6548 VTy = llvm::FixedVectorType::get(HalfTy, 4);
6549 llvm::Type *Tys[2] = {Ty, VTy};
6550 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6551 }
6552 case NEON::BI__builtin_neon_vmaxvq_f16: {
6553 Int = Intrinsic::aarch64_neon_fmaxv;
6554 Ty = HalfTy;
6555 VTy = llvm::FixedVectorType::get(HalfTy, 8);
6556 llvm::Type *Tys[2] = {Ty, VTy};
6557 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6558 }
6559 case NEON::BI__builtin_neon_vminv_f16: {
6560 Int = Intrinsic::aarch64_neon_fminv;
6561 Ty = HalfTy;
6562 VTy = llvm::FixedVectorType::get(HalfTy, 4);
6563 llvm::Type *Tys[2] = {Ty, VTy};
6564 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6565 }
6566 case NEON::BI__builtin_neon_vminvq_f16: {
6567 Int = Intrinsic::aarch64_neon_fminv;
6568 Ty = HalfTy;
6569 VTy = llvm::FixedVectorType::get(HalfTy, 8);
6570 llvm::Type *Tys[2] = {Ty, VTy};
6571 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6572 }
6573 case NEON::BI__builtin_neon_vmaxnmv_f16: {
6574 Int = Intrinsic::aarch64_neon_fmaxnmv;
6575 Ty = HalfTy;
6576 VTy = llvm::FixedVectorType::get(HalfTy, 4);
6577 llvm::Type *Tys[2] = {Ty, VTy};
6578 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv");
6579 }
6580 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
6581 Int = Intrinsic::aarch64_neon_fmaxnmv;
6582 Ty = HalfTy;
6583 VTy = llvm::FixedVectorType::get(HalfTy, 8);
6584 llvm::Type *Tys[2] = {Ty, VTy};
6585 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv");
6586 }
6587 case NEON::BI__builtin_neon_vminnmv_f16: {
6588 Int = Intrinsic::aarch64_neon_fminnmv;
6589 Ty = HalfTy;
6590 VTy = llvm::FixedVectorType::get(HalfTy, 4);
6591 llvm::Type *Tys[2] = {Ty, VTy};
6592 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv");
6593 }
6594 case NEON::BI__builtin_neon_vminnmvq_f16: {
6595 Int = Intrinsic::aarch64_neon_fminnmv;
6596 Ty = HalfTy;
6597 VTy = llvm::FixedVectorType::get(HalfTy, 8);
6598 llvm::Type *Tys[2] = {Ty, VTy};
6599 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv");
6600 }
6601 case NEON::BI__builtin_neon_vmul_n_f64: {
6602 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
6603 Value *RHS = Builder.CreateBitCast(Ops[1], DoubleTy);
6604 return Builder.CreateFMul(Ops[0], RHS);
6605 }
6606 case NEON::BI__builtin_neon_vaddlv_u8:
6607 case NEON::BI__builtin_neon_vaddlvq_u8:
6608 case NEON::BI__builtin_neon_vaddlv_u16:
6609 case NEON::BI__builtin_neon_vaddlvq_u16: {
6610 Int = Intrinsic::aarch64_neon_uaddlv;
6611 Ty = Int32Ty;
6612 VTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
6613 llvm::Type *Tys[2] = {Ty, VTy};
6614 Value *Result = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6615 if (VTy->getElementType()->getPrimitiveSizeInBits() == 8)
6616 return Builder.CreateTrunc(Result, Int16Ty);
6617 return Result;
6618 }
6619 case NEON::BI__builtin_neon_vaddlv_s8:
6620 case NEON::BI__builtin_neon_vaddlvq_s8:
6621 case NEON::BI__builtin_neon_vaddlv_s16:
6622 case NEON::BI__builtin_neon_vaddlvq_s16: {
6623 Int = Intrinsic::aarch64_neon_saddlv;
6624 Ty = Int32Ty;
6625 VTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
6626 llvm::Type *Tys[2] = {Ty, VTy};
6627 Value *Result = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6628 if (VTy->getElementType()->getPrimitiveSizeInBits() == 8)
6629 return Builder.CreateTrunc(Result, Int16Ty);
6630 return Result;
6631 }
6632 case NEON::BI__builtin_neon_vsri_n_v:
6633 case NEON::BI__builtin_neon_vsriq_n_v: {
6634 Int = Intrinsic::aarch64_neon_vsri;
6635 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
6636 return EmitNeonCall(Intrin, Ops, "vsri_n");
6637 }
6638 case NEON::BI__builtin_neon_vsli_n_v:
6639 case NEON::BI__builtin_neon_vsliq_n_v: {
6640 Int = Intrinsic::aarch64_neon_vsli;
6641 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
6642 return EmitNeonCall(Intrin, Ops, "vsli_n");
6643 }
6644 case NEON::BI__builtin_neon_vsra_n_v:
6645 case NEON::BI__builtin_neon_vsraq_n_v:
6646 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6647 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
6648 return Builder.CreateAdd(Ops[0], Ops[1]);
6649 case NEON::BI__builtin_neon_vrsra_n_v:
6650 case NEON::BI__builtin_neon_vrsraq_n_v: {
6651 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
6652 SmallVector<llvm::Value*,2> TmpOps;
6653 TmpOps.push_back(Ops[1]);
6654 TmpOps.push_back(Ops[2]);
6655 Function* F = CGM.getIntrinsic(Int, Ty);
6656 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true);
6657 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
6658 return Builder.CreateAdd(Ops[0], tmp);
6659 }
6660 case NEON::BI__builtin_neon_vld1_v:
6661 case NEON::BI__builtin_neon_vld1q_v: {
6662 return Builder.CreateAlignedLoad(VTy, Ops[0], PtrOp0.getAlignment());
6663 }
6664 case NEON::BI__builtin_neon_vst1_v:
6665 case NEON::BI__builtin_neon_vst1q_v:
6666 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
6667 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment());
6668 case NEON::BI__builtin_neon_vld1_lane_v:
6669 case NEON::BI__builtin_neon_vld1q_lane_v: {
6670 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6671 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
6672 PtrOp0.getAlignment());
6673 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane");
6674 }
6675 case NEON::BI__builtin_neon_vldap1_lane_s64:
6676 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
6677 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6678 llvm::LoadInst *LI = Builder.CreateAlignedLoad(
6679 VTy->getElementType(), Ops[0], PtrOp0.getAlignment());
6680 LI->setAtomic(llvm::AtomicOrdering::Acquire);
6681 Ops[0] = LI;
6682 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vldap1_lane");
6683 }
6684 case NEON::BI__builtin_neon_vld1_dup_v:
6685 case NEON::BI__builtin_neon_vld1q_dup_v: {
6686 Value *V = PoisonValue::get(Ty);
6687 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
6688 PtrOp0.getAlignment());
6689 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
6690 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
6691 return EmitNeonSplat(Ops[0], CI);
6692 }
6693 case NEON::BI__builtin_neon_vst1_lane_v:
6694 case NEON::BI__builtin_neon_vst1q_lane_v:
6695 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6696 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
6697 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment());
6698 case NEON::BI__builtin_neon_vstl1_lane_s64:
6699 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
6700 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6701 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
6702 llvm::StoreInst *SI =
6703 Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment());
6704 SI->setAtomic(llvm::AtomicOrdering::Release);
6705 return SI;
6706 }
6707 case NEON::BI__builtin_neon_vld2_v:
6708 case NEON::BI__builtin_neon_vld2q_v: {
6709 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6710 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys);
6711 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
6712 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6713 }
6714 case NEON::BI__builtin_neon_vld3_v:
6715 case NEON::BI__builtin_neon_vld3q_v: {
6716 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6717 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys);
6718 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
6719 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6720 }
6721 case NEON::BI__builtin_neon_vld4_v:
6722 case NEON::BI__builtin_neon_vld4q_v: {
6723 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6724 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys);
6725 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
6726 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6727 }
6728 case NEON::BI__builtin_neon_vld2_dup_v:
6729 case NEON::BI__builtin_neon_vld2q_dup_v: {
6730 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6731 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys);
6732 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
6733 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6734 }
6735 case NEON::BI__builtin_neon_vld3_dup_v:
6736 case NEON::BI__builtin_neon_vld3q_dup_v: {
6737 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6738 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys);
6739 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
6740 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6741 }
6742 case NEON::BI__builtin_neon_vld4_dup_v:
6743 case NEON::BI__builtin_neon_vld4q_dup_v: {
6744 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6745 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys);
6746 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
6747 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6748 }
6749 case NEON::BI__builtin_neon_vld2_lane_v:
6750 case NEON::BI__builtin_neon_vld2q_lane_v: {
6751 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6752 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
6753 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
6754 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6755 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6756 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
6757 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld2_lane");
6758 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6759 }
6760 case NEON::BI__builtin_neon_vld3_lane_v:
6761 case NEON::BI__builtin_neon_vld3q_lane_v: {
6762 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6763 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
6764 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
6765 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6766 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6767 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
6768 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
6769 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld3_lane");
6770 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6771 }
6772 case NEON::BI__builtin_neon_vld4_lane_v:
6773 case NEON::BI__builtin_neon_vld4q_lane_v: {
6774 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6775 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
6776 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
6777 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6778 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6779 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
6780 Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
6781 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty);
6782 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld4_lane");
6783 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6784 }
6785 case NEON::BI__builtin_neon_vst2_v:
6786 case NEON::BI__builtin_neon_vst2q_v: {
6787 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6788 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
6789 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys),
6790 Ops, "");
6791 }
6792 case NEON::BI__builtin_neon_vst2_lane_v:
6793 case NEON::BI__builtin_neon_vst2q_lane_v: {
6794 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6795 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
6796 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6797 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys),
6798 Ops, "");
6799 }
6800 case NEON::BI__builtin_neon_vst3_v:
6801 case NEON::BI__builtin_neon_vst3q_v: {
6802 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6803 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6804 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys),
6805 Ops, "");
6806 }
6807 case NEON::BI__builtin_neon_vst3_lane_v:
6808 case NEON::BI__builtin_neon_vst3q_lane_v: {
6809 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6810 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
6811 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6812 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys),
6813 Ops, "");
6814 }
6815 case NEON::BI__builtin_neon_vst4_v:
6816 case NEON::BI__builtin_neon_vst4q_v: {
6817 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6818 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6819 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys),
6820 Ops, "");
6821 }
6822 case NEON::BI__builtin_neon_vst4_lane_v:
6823 case NEON::BI__builtin_neon_vst4q_lane_v: {
6824 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6825 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
6826 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
6827 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys),
6828 Ops, "");
6829 }
6830 case NEON::BI__builtin_neon_vtrn_v:
6831 case NEON::BI__builtin_neon_vtrnq_v: {
6832 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6833 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6834 Value *SV = nullptr;
6835
6836 for (unsigned vi = 0; vi != 2; ++vi) {
6837 SmallVector<int, 16> Indices;
6838 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6839 Indices.push_back(i+vi);
6840 Indices.push_back(i+e+vi);
6841 }
6842 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6843 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
6844 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
6845 }
6846 return SV;
6847 }
6848 case NEON::BI__builtin_neon_vuzp_v:
6849 case NEON::BI__builtin_neon_vuzpq_v: {
6850 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6851 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6852 Value *SV = nullptr;
6853
6854 for (unsigned vi = 0; vi != 2; ++vi) {
6855 SmallVector<int, 16> Indices;
6856 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
6857 Indices.push_back(2*i+vi);
6858
6859 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6860 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
6861 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
6862 }
6863 return SV;
6864 }
6865 case NEON::BI__builtin_neon_vzip_v:
6866 case NEON::BI__builtin_neon_vzipq_v: {
6867 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6868 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6869 Value *SV = nullptr;
6870
6871 for (unsigned vi = 0; vi != 2; ++vi) {
6872 SmallVector<int, 16> Indices;
6873 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6874 Indices.push_back((i + vi*e) >> 1);
6875 Indices.push_back(((i + vi*e) >> 1)+e);
6876 }
6877 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6878 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
6879 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
6880 }
6881 return SV;
6882 }
6883 case NEON::BI__builtin_neon_vqtbl1q_v: {
6884 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty),
6885 Ops, "vtbl1");
6886 }
6887 case NEON::BI__builtin_neon_vqtbl2q_v: {
6888 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty),
6889 Ops, "vtbl2");
6890 }
6891 case NEON::BI__builtin_neon_vqtbl3q_v: {
6892 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty),
6893 Ops, "vtbl3");
6894 }
6895 case NEON::BI__builtin_neon_vqtbl4q_v: {
6896 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty),
6897 Ops, "vtbl4");
6898 }
6899 case NEON::BI__builtin_neon_vqtbx1q_v: {
6900 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty),
6901 Ops, "vtbx1");
6902 }
6903 case NEON::BI__builtin_neon_vqtbx2q_v: {
6904 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty),
6905 Ops, "vtbx2");
6906 }
6907 case NEON::BI__builtin_neon_vqtbx3q_v: {
6908 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty),
6909 Ops, "vtbx3");
6910 }
6911 case NEON::BI__builtin_neon_vqtbx4q_v: {
6912 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty),
6913 Ops, "vtbx4");
6914 }
6915 case NEON::BI__builtin_neon_vsqadd_v:
6916 case NEON::BI__builtin_neon_vsqaddq_v: {
6917 Int = Intrinsic::aarch64_neon_usqadd;
6918 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd");
6919 }
6920 case NEON::BI__builtin_neon_vuqadd_v:
6921 case NEON::BI__builtin_neon_vuqaddq_v: {
6922 Int = Intrinsic::aarch64_neon_suqadd;
6923 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd");
6924 }
6925
6926 case NEON::BI__builtin_neon_vluti2_laneq_mf8:
6927 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
6928 case NEON::BI__builtin_neon_vluti2_laneq_f16:
6929 case NEON::BI__builtin_neon_vluti2_laneq_p16:
6930 case NEON::BI__builtin_neon_vluti2_laneq_p8:
6931 case NEON::BI__builtin_neon_vluti2_laneq_s16:
6932 case NEON::BI__builtin_neon_vluti2_laneq_s8:
6933 case NEON::BI__builtin_neon_vluti2_laneq_u16:
6934 case NEON::BI__builtin_neon_vluti2_laneq_u8: {
6935 Int = Intrinsic::aarch64_neon_vluti2_laneq;
6936 llvm::Type *Tys[2];
6937 Tys[0] = Ty;
6938 Tys[1] = GetNeonType(this, NeonTypeFlags(Type.getEltType(), false,
6939 /*isQuad*/ false));
6940 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vluti2_laneq");
6941 }
6942 case NEON::BI__builtin_neon_vluti2q_laneq_mf8:
6943 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
6944 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
6945 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
6946 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
6947 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
6948 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
6949 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
6950 case NEON::BI__builtin_neon_vluti2q_laneq_u8: {
6951 Int = Intrinsic::aarch64_neon_vluti2_laneq;
6952 llvm::Type *Tys[2];
6953 Tys[0] = Ty;
6954 Tys[1] = GetNeonType(this, NeonTypeFlags(Type.getEltType(), false,
6955 /*isQuad*/ true));
6956 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vluti2_laneq");
6957 }
6958 case NEON::BI__builtin_neon_vluti2_lane_mf8:
6959 case NEON::BI__builtin_neon_vluti2_lane_bf16:
6960 case NEON::BI__builtin_neon_vluti2_lane_f16:
6961 case NEON::BI__builtin_neon_vluti2_lane_p16:
6962 case NEON::BI__builtin_neon_vluti2_lane_p8:
6963 case NEON::BI__builtin_neon_vluti2_lane_s16:
6964 case NEON::BI__builtin_neon_vluti2_lane_s8:
6965 case NEON::BI__builtin_neon_vluti2_lane_u16:
6966 case NEON::BI__builtin_neon_vluti2_lane_u8: {
6967 Int = Intrinsic::aarch64_neon_vluti2_lane;
6968 llvm::Type *Tys[2];
6969 Tys[0] = Ty;
6970 Tys[1] = GetNeonType(this, NeonTypeFlags(Type.getEltType(), false,
6971 /*isQuad*/ false));
6972 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vluti2_lane");
6973 }
6974 case NEON::BI__builtin_neon_vluti2q_lane_mf8:
6975 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
6976 case NEON::BI__builtin_neon_vluti2q_lane_f16:
6977 case NEON::BI__builtin_neon_vluti2q_lane_p16:
6978 case NEON::BI__builtin_neon_vluti2q_lane_p8:
6979 case NEON::BI__builtin_neon_vluti2q_lane_s16:
6980 case NEON::BI__builtin_neon_vluti2q_lane_s8:
6981 case NEON::BI__builtin_neon_vluti2q_lane_u16:
6982 case NEON::BI__builtin_neon_vluti2q_lane_u8: {
6983 Int = Intrinsic::aarch64_neon_vluti2_lane;
6984 llvm::Type *Tys[2];
6985 Tys[0] = Ty;
6986 Tys[1] = GetNeonType(this, NeonTypeFlags(Type.getEltType(), false,
6987 /*isQuad*/ true));
6988 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vluti2_lane");
6989 }
6990 case NEON::BI__builtin_neon_vluti4q_lane_mf8:
6991 case NEON::BI__builtin_neon_vluti4q_lane_p8:
6992 case NEON::BI__builtin_neon_vluti4q_lane_s8:
6993 case NEON::BI__builtin_neon_vluti4q_lane_u8: {
6994 Int = Intrinsic::aarch64_neon_vluti4q_lane;
6995 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vluti4q_lane");
6996 }
6997 case NEON::BI__builtin_neon_vluti4q_laneq_mf8:
6998 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
6999 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
7000 case NEON::BI__builtin_neon_vluti4q_laneq_u8: {
7001 Int = Intrinsic::aarch64_neon_vluti4q_laneq;
7002 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vluti4q_laneq");
7003 }
7004 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
7005 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
7006 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
7007 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
7008 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2: {
7009 Int = Intrinsic::aarch64_neon_vluti4q_lane_x2;
7010 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vluti4q_lane_x2");
7011 }
7012 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
7013 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
7014 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
7015 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
7016 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2: {
7017 Int = Intrinsic::aarch64_neon_vluti4q_laneq_x2;
7018 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vluti4q_laneq_x2");
7019 }
7020 case NEON::BI__builtin_neon_vmmlaq_f16_mf8_fpm:
7021 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fmmla,
7022 {llvm::FixedVectorType::get(HalfTy, 8),
7023 llvm::FixedVectorType::get(Int8Ty, 16)},
7024 Ops, E, "fmmla");
7025 case NEON::BI__builtin_neon_vmmlaq_f32_mf8_fpm:
7026 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fmmla,
7027 {llvm::FixedVectorType::get(FloatTy, 4),
7028 llvm::FixedVectorType::get(Int8Ty, 16)},
7029 Ops, E, "fmmla");
7030 case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
7031 ExtractLow = true;
7032 [[fallthrough]];
7033 case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
7034 case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
7035 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_cvtl1,
7036 llvm::FixedVectorType::get(BFloatTy, 8),
7037 Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt1");
7038 case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
7039 ExtractLow = true;
7040 [[fallthrough]];
7041 case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
7042 case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
7043 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_cvtl2,
7044 llvm::FixedVectorType::get(BFloatTy, 8),
7045 Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt2");
7046 case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
7047 ExtractLow = true;
7048 [[fallthrough]];
7049 case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
7050 case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
7051 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_cvtl1,
7052 llvm::FixedVectorType::get(HalfTy, 8),
7053 Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt1");
7054 case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
7055 ExtractLow = true;
7056 [[fallthrough]];
7057 case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
7058 case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
7059 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_cvtl2,
7060 llvm::FixedVectorType::get(HalfTy, 8),
7061 Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt2");
7062 case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
7063 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_fcvtn,
7064 llvm::FixedVectorType::get(Int8Ty, 8),
7065 Ops[0]->getType(), false, Ops, E, "vfcvtn");
7066 case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
7067 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_fcvtn,
7068 llvm::FixedVectorType::get(Int8Ty, 8),
7069 llvm::FixedVectorType::get(HalfTy, 4), false, Ops,
7070 E, "vfcvtn");
7071 case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
7072 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_fcvtn,
7073 llvm::FixedVectorType::get(Int8Ty, 16),
7074 llvm::FixedVectorType::get(HalfTy, 8), false, Ops,
7075 E, "vfcvtn");
7076 case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm: {
7077 llvm::Type *Ty = llvm::FixedVectorType::get(Int8Ty, 16);
7078 Ops[0] = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0],
7079 uint64_t(0));
7080 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_fcvtn2, Ty,
7081 Ops[1]->getType(), false, Ops, E, "vfcvtn2");
7082 }
7083
7084 case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
7085 case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
7086 return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot2, false, HalfTy,
7087 Ops, E, "fdot2");
7088 case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
7089 case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
7090 ExtendLaneArg = true;
7091 [[fallthrough]];
7092 case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
7093 case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
7094 return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot2_lane,
7095 ExtendLaneArg, HalfTy, Ops, E, "fdot2_lane");
7096 case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
7097 case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
7098 return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot4, false,
7099 FloatTy, Ops, E, "fdot4");
7100 case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
7101 case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
7102 ExtendLaneArg = true;
7103 [[fallthrough]];
7104 case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
7105 case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
7106 return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot4_lane,
7107 ExtendLaneArg, FloatTy, Ops, E, "fdot4_lane");
7108
7109 case NEON::BI__builtin_neon_vdot_f32_f16:
7110 case NEON::BI__builtin_neon_vdotq_f32_f16: {
7111 llvm::Type *InputTy =
7112 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
7113 llvm::Type *Tys[2] = {Ty, InputTy};
7114 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_fdot, Tys),
7115 Ops, "vdot");
7116 }
7117
7118 case NEON::BI__builtin_neon_vdot_lane_f32_f16:
7119 case NEON::BI__builtin_neon_vdot_laneq_f32_f16:
7120 case NEON::BI__builtin_neon_vdotq_lane_f32_f16:
7121 case NEON::BI__builtin_neon_vdotq_laneq_f32_f16: {
7122 llvm::FixedVectorType *InputTy =
7123 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
7124 llvm::FixedVectorType *LaneTy = llvm::FixedVectorType::get(
7125 HalfTy, Ops[2]->getType()->getPrimitiveSizeInBits() / 16);
7126 // Treat the lane argument as a splat and use non-lane version of the
7127 // intrinsic.
7128 Ops[2] = Builder.CreateBitCast(Ops[2], LaneTy);
7129 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]),
7130 InputTy->getElementCount());
7131 llvm::Type *Tys[2] = {Ty, InputTy};
7132 Ops.pop_back();
7133 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_fdot, Tys),
7134 Ops, "vdot");
7135 }
7136
7137 case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
7138 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalb,
7139 {llvm::FixedVectorType::get(HalfTy, 8)}, Ops, E,
7140 "vmlal");
7141 case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
7142 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalt,
7143 {llvm::FixedVectorType::get(HalfTy, 8)}, Ops, E,
7144 "vmlal");
7145 case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
7146 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlallbb,
7147 {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
7148 "vmlall");
7149 case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
7150 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlallbt,
7151 {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
7152 "vmlall");
7153 case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
7154 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalltb,
7155 {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
7156 "vmlall");
7157 case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
7158 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalltt,
7159 {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
7160 "vmlall");
7161 case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
7162 ExtendLaneArg = true;
7163 [[fallthrough]];
7164 case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
7165 return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlalb_lane,
7166 ExtendLaneArg, HalfTy, Ops, E, "vmlal_lane");
7167 case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
7168 ExtendLaneArg = true;
7169 [[fallthrough]];
7170 case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
7171 return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlalt_lane,
7172 ExtendLaneArg, HalfTy, Ops, E, "vmlal_lane");
7173 case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
7174 ExtendLaneArg = true;
7175 [[fallthrough]];
7176 case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
7177 return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlallbb_lane,
7178 ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
7179 case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
7180 ExtendLaneArg = true;
7181 [[fallthrough]];
7182 case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
7183 return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlallbt_lane,
7184 ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
7185 case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
7186 ExtendLaneArg = true;
7187 [[fallthrough]];
7188 case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
7189 return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlalltb_lane,
7190 ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
7191 case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
7192 ExtendLaneArg = true;
7193 [[fallthrough]];
7194 case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
7195 return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlalltt_lane,
7196 ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
7197 case NEON::BI__builtin_neon_vamin_f16:
7198 case NEON::BI__builtin_neon_vaminq_f16:
7199 case NEON::BI__builtin_neon_vamin_f32:
7200 case NEON::BI__builtin_neon_vaminq_f32:
7201 case NEON::BI__builtin_neon_vaminq_f64: {
7202 Int = Intrinsic::aarch64_neon_famin;
7203 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "famin");
7204 }
7205 case NEON::BI__builtin_neon_vamax_f16:
7206 case NEON::BI__builtin_neon_vamaxq_f16:
7207 case NEON::BI__builtin_neon_vamax_f32:
7208 case NEON::BI__builtin_neon_vamaxq_f32:
7209 case NEON::BI__builtin_neon_vamaxq_f64: {
7210 Int = Intrinsic::aarch64_neon_famax;
7211 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "famax");
7212 }
7213 case NEON::BI__builtin_neon_vscale_f16:
7214 case NEON::BI__builtin_neon_vscaleq_f16:
7215 case NEON::BI__builtin_neon_vscale_f32:
7216 case NEON::BI__builtin_neon_vscaleq_f32:
7217 case NEON::BI__builtin_neon_vscaleq_f64: {
7218 Int = Intrinsic::aarch64_neon_fp8_fscale;
7219 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fscale");
7220 }
7221 }
7222}
7223
7224Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID,
7225 const CallExpr *E) {
7226 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
7227 BuiltinID == BPF::BI__builtin_btf_type_id ||
7228 BuiltinID == BPF::BI__builtin_preserve_type_info ||
7229 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
7230 "unexpected BPF builtin");
7231
7232 // A sequence number, injected into IR builtin functions, to
7233 // prevent CSE given the only difference of the function
7234 // may just be the debuginfo metadata.
7235 static uint32_t BuiltinSeqNum;
7236
7237 switch (BuiltinID) {
7238 default:
7239 llvm_unreachable("Unexpected BPF builtin");
7240 case BPF::BI__builtin_preserve_field_info: {
7241 const Expr *Arg = E->getArg(0);
7242 bool IsBitField = Arg->IgnoreParens()->getObjectKind() == OK_BitField;
7243
7244 if (!getDebugInfo()) {
7245 CGM.Error(E->getExprLoc(),
7246 "using __builtin_preserve_field_info() without -g");
7247 return IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this)
7248 : EmitLValue(Arg).emitRawPointer(*this);
7249 }
7250
7251 // Enable underlying preserve_*_access_index() generation.
7252 bool OldIsInPreservedAIRegion = IsInPreservedAIRegion;
7253 IsInPreservedAIRegion = true;
7254 Value *FieldAddr = IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this)
7255 : EmitLValue(Arg).emitRawPointer(*this);
7256 IsInPreservedAIRegion = OldIsInPreservedAIRegion;
7257
7258 ConstantInt *C = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
7259 Value *InfoKind = ConstantInt::get(Int64Ty, C->getSExtValue());
7260
7261 // Built the IR for the preserve_field_info intrinsic.
7262 llvm::Function *FnGetFieldInfo = Intrinsic::getOrInsertDeclaration(
7263 &CGM.getModule(), Intrinsic::bpf_preserve_field_info,
7264 {FieldAddr->getType()});
7265 return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
7266 }
7267 case BPF::BI__builtin_btf_type_id:
7268 case BPF::BI__builtin_preserve_type_info: {
7269 if (!getDebugInfo()) {
7270 CGM.Error(E->getExprLoc(), "using builtin function without -g");
7271 return nullptr;
7272 }
7273
7274 const Expr *Arg0 = E->getArg(0);
7275 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(
7276 Arg0->getType(), Arg0->getExprLoc());
7277
7278 ConstantInt *Flag = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
7279 Value *FlagValue = ConstantInt::get(Int64Ty, Flag->getSExtValue());
7280 Value *SeqNumVal = ConstantInt::get(Int32Ty, BuiltinSeqNum++);
7281
7282 llvm::Function *FnDecl;
7283 if (BuiltinID == BPF::BI__builtin_btf_type_id)
7284 FnDecl = Intrinsic::getOrInsertDeclaration(
7285 &CGM.getModule(), Intrinsic::bpf_btf_type_id, {});
7286 else
7287 FnDecl = Intrinsic::getOrInsertDeclaration(
7288 &CGM.getModule(), Intrinsic::bpf_preserve_type_info, {});
7289 CallInst *Fn = Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
7290 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
7291 return Fn;
7292 }
7293 case BPF::BI__builtin_preserve_enum_value: {
7294 if (!getDebugInfo()) {
7295 CGM.Error(E->getExprLoc(), "using builtin function without -g");
7296 return nullptr;
7297 }
7298
7299 const Expr *Arg0 = E->getArg(0);
7300 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(
7301 Arg0->getType(), Arg0->getExprLoc());
7302
7303 // Find enumerator
7304 const auto *UO = cast<UnaryOperator>(Arg0->IgnoreParens());
7305 const auto *CE = cast<CStyleCastExpr>(UO->getSubExpr());
7306 const auto *DR = cast<DeclRefExpr>(CE->getSubExpr());
7307 const auto *Enumerator = cast<EnumConstantDecl>(DR->getDecl());
7308
7309 auto InitVal = Enumerator->getInitVal();
7310 std::string InitValStr;
7311 if (InitVal.isNegative() || InitVal > uint64_t(INT64_MAX))
7312 InitValStr = std::to_string(InitVal.getSExtValue());
7313 else
7314 InitValStr = std::to_string(InitVal.getZExtValue());
7315 std::string EnumStr = Enumerator->getNameAsString() + ":" + InitValStr;
7316 Value *EnumStrVal = Builder.CreateGlobalString(EnumStr);
7317
7318 ConstantInt *Flag = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
7319 Value *FlagValue = ConstantInt::get(Int64Ty, Flag->getSExtValue());
7320 Value *SeqNumVal = ConstantInt::get(Int32Ty, BuiltinSeqNum++);
7321
7322 llvm::Function *IntrinsicFn = Intrinsic::getOrInsertDeclaration(
7323 &CGM.getModule(), Intrinsic::bpf_preserve_enum_value, {});
7324 CallInst *Fn =
7325 Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
7326 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
7327 return Fn;
7328 }
7329 }
7330}
7331
7332llvm::Value *CodeGenFunction::
7333BuildVector(ArrayRef<llvm::Value*> Ops) {
7334 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
7335 "Not a power-of-two sized vector!");
7336 bool AllConstants = true;
7337 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
7338 AllConstants &= isa<Constant>(Ops[i]);
7339
7340 // If this is a constant vector, create a ConstantVector.
7341 if (AllConstants) {
7342 SmallVector<llvm::Constant*, 16> CstOps;
7343 for (llvm::Value *Op : Ops)
7344 CstOps.push_back(cast<Constant>(Op));
7345 return llvm::ConstantVector::get(CstOps);
7346 }
7347
7348 // Otherwise, insertelement the values to build the vector.
7349 Value *Result = llvm::PoisonValue::get(
7350 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size()));
7351
7352 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
7353 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt64(i));
7354
7355 return Result;
7356}
7357
7358Value *CodeGenFunction::EmitAArch64CpuInit() {
7359 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
7360 llvm::FunctionCallee Func =
7361 CGM.CreateRuntimeFunction(FTy, "__init_cpu_features_resolver");
7362 cast<llvm::GlobalValue>(Func.getCallee())->setDSOLocal(true);
7363 cast<llvm::GlobalValue>(Func.getCallee())
7364 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
7365 return Builder.CreateCall(Func);
7366}
7367
7368Value *CodeGenFunction::EmitAArch64CpuSupports(const CallExpr *E) {
7369 const Expr *ArgExpr = E->getArg(0)->IgnoreParenCasts();
7370 StringRef ArgStr = cast<StringLiteral>(ArgExpr)->getString();
7371 llvm::SmallVector<StringRef, 8> OrigFeatures;
7372 ArgStr.split(OrigFeatures, "+");
7373 llvm::SmallVector<StringRef, 8> Features;
7374 for (StringRef Feature : OrigFeatures) {
7375 Feature = Feature.trim();
7376 if (!llvm::AArch64::parseFMVExtension(Feature))
7377 return Builder.getFalse();
7378 if (Feature != "default")
7379 Features.push_back(Feature);
7380 }
7381 return EmitAArch64CpuSupports(Features);
7382}
7383
7384llvm::Value *
7385CodeGenFunction::EmitAArch64CpuSupports(ArrayRef<StringRef> FeaturesStrs) {
7386 llvm::APInt FeaturesMask = llvm::AArch64::getCpuSupportsMask(FeaturesStrs);
7387 Value *Result = Builder.getTrue();
7388 if (FeaturesMask != 0) {
7389 // Get features from structure in runtime library
7390 // struct {
7391 // unsigned long long features;
7392 // } __aarch64_cpu_features;
7393 llvm::Type *STy = llvm::StructType::get(Int64Ty);
7394 llvm::Constant *AArch64CPUFeatures =
7395 CGM.CreateRuntimeVariable(STy, "__aarch64_cpu_features");
7396 cast<llvm::GlobalValue>(AArch64CPUFeatures)->setDSOLocal(true);
7397 llvm::Value *CpuFeatures = Builder.CreateGEP(
7398 STy, AArch64CPUFeatures,
7399 {ConstantInt::get(Int32Ty, 0), ConstantInt::get(Int32Ty, 0)});
7400 Value *Features = Builder.CreateAlignedLoad(Int64Ty, CpuFeatures,
7401 CharUnits::fromQuantity(8));
7402 Value *Mask = Builder.getInt(FeaturesMask.trunc(64));
7403 Value *Bitset = Builder.CreateAnd(Features, Mask);
7404 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
7405 Result = Builder.CreateAnd(Result, Cmp);
7406 }
7407 return Result;
7408}
AArch64CodeGenUtils.h
Utilities used for generating code for AArch64 that are shared between the classic and ClangIR code-g...
NEONMAP2
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
Definition AArch64CodeGenUtils.h:113
NEONMAP1
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
Definition AArch64CodeGenUtils.h:109
NEONMAP0
#define NEONMAP0(NameBase)
Definition AArch64CodeGenUtils.h:106
V
#define V(N, I)
Definition ASTContext.h:3855
CheckAtomicAlignment
Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E)
Definition CGBuiltin.cpp:301
getSVEVectorForElementType
static cir::VectorType getSVEVectorForElementType(CIRGenModule &cgm, mlir::Type eltTy)
Definition CIRGenBuiltinAArch64.cpp:1643
findARMVectorIntrinsicInMap
static const IntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< IntrinsicInfo > intrinsicMap, unsigned builtinID, bool &mapProvenSorted)
Definition CIRGenBuiltinAArch64.cpp:66
EmitSpecialRegisterBuiltin
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, SpecialRegisterAccessKind AccessKind, StringRef SysReg="")
Definition ARM.cpp:1953
ARMMVEVectorReinterpret
static llvm::Value * ARMMVEVectorReinterpret(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *DestType)
Definition ARM.cpp:2889
GetFloatNeonType
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
Definition ARM.cpp:402
MVEImmediateShr
static llvm::Value * MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V, uint32_t Shift, bool Unsigned)
Definition ARM.cpp:2859
SignOrZeroExtend
static llvm::Value * SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V, llvm::Type *T, bool Unsigned)
Definition ARM.cpp:2852
InsertExplicitZeroOperand
static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
Definition ARM.cpp:3906
EmitAArch64TblBuiltinExpr
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops, llvm::Triple::ArchType Arch)
Definition ARM.cpp:3111
swapCommutativeSMEOperands
static void swapCommutativeSMEOperands(unsigned BuiltinID, SmallVectorImpl< Value * > &Ops)
Definition ARM.cpp:4359
AArch64SISDIntrinsicsProvenSorted
static bool AArch64SISDIntrinsicsProvenSorted
Definition ARM.cpp:987
ARMMVECreateFPToSI
static llvm::Value * ARMMVECreateFPToSI(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *Ty)
Definition ARM.cpp:2983
HasExtraNeonArgument
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
Definition ARM.cpp:2135
EmitCommonNeonSISDBuiltinExpr
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const ARMNeonVectorIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
Definition ARM.cpp:1055
ARMMVECreateFPToUI
static llvm::Value * ARMMVECreateFPToUI(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *Ty)
Definition ARM.cpp:2991
ARMMVECreateSIToFP
static llvm::Value * ARMMVECreateSIToFP(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *Ty)
Definition ARM.cpp:2967
AArch64SVEIntrinsicsProvenSorted
static bool AArch64SVEIntrinsicsProvenSorted
Definition ARM.cpp:988
InsertExplicitUndefOperand
static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
Definition ARM.cpp:3912
GetIntegerConstantValue
static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context)
Definition ARM.cpp:2848
AArch64SMEIntrinsicsProvenSorted
static bool AArch64SMEIntrinsicsProvenSorted
Definition ARM.cpp:989
VectorZip
static llvm::Value * VectorZip(CGBuilderTy &Builder, llvm::Value *V0, llvm::Value *V1)
Definition ARM.cpp:2926
SVEBitsPerBlock
constexpr unsigned SVEBitsPerBlock
Definition ARM.cpp:3391
NEONEquivalentIntrinsicMap
static const std::pair< unsigned, unsigned > NEONEquivalentIntrinsicMap[]
Definition ARM.cpp:861
GetNeonType
static llvm::FixedVectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasFastHalfType=true, bool V1Ty=false, bool AllowBFloatArgsAndRet=true)
Definition ARM.cpp:362
readX18AsPtr
Value * readX18AsPtr(CodeGenFunction &CGF)
Helper for the read/write/add/inc X18 builtins: read the X18 register and return it as an i8 pointer.
Definition ARM.cpp:4459
ARMMVEVectorElementReverse
static llvm::Value * ARMMVEVectorElementReverse(CGBuilderTy &Builder, llvm::Value *V, unsigned ReverseWidth)
Definition ARM.cpp:2953
translateAarch64ToMsvcIntrin
static std::optional< CodeGenFunction::MSVCIntrin > translateAarch64ToMsvcIntrin(unsigned BuiltinID)
Definition ARM.cpp:34
translateArmToMsvcIntrin
static std::optional< CodeGenFunction::MSVCIntrin > translateArmToMsvcIntrin(unsigned BuiltinID)
Definition ARM.cpp:193
ARMMVECreateUIToFP
static llvm::Value * ARMMVECreateUIToFP(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *Ty)
Definition ARM.cpp:2975
VectorUnzip
static llvm::Value * VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd)
Definition ARM.cpp:2915
EmitAtomicStoreWithHintBuiltin
static Value * EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Definition ARM.cpp:2069
ARMMVEConstantSplat
static llvm::Value * ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT)
Definition ARM.cpp:2941
SpecialRegisterAccessKind
SpecialRegisterAccessKind
Definition ARM.cpp:1944
VolatileRead
@ VolatileRead
Definition ARM.cpp:1946
NormalRead
@ NormalRead
Definition ARM.cpp:1945
Write
@ Write
Definition ARM.cpp:1947
AArch64SMEIntrinsicMap
static const AArch64SVEAndSMEVectorIntrinsicInfo AArch64SMEIntrinsicMap[]
Definition ARM.cpp:975
ARMMVEVectorSplat
static llvm::Value * ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V)
Definition ARM.cpp:2881
NEONSIMDIntrinsicsProvenSorted
static bool NEONSIMDIntrinsicsProvenSorted
Definition ARM.cpp:984
emitCallMaybeConstrainedFPBuiltin
static Value * emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, llvm::Type *Ty, ArrayRef< Value * > Args)
Definition ARM.cpp:345
EmitRangePrefetchBuiltin
static Value * EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Definition ARM.cpp:2019
packTBLDVectorList
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
Definition ARM.cpp:1871
AArch64SIMDIntrinsicsProvenSorted
static bool AArch64SIMDIntrinsicsProvenSorted
Definition ARM.cpp:986
ARMSIMDIntrinsicMap
static const ARMNeonVectorIntrinsicInfo ARMSIMDIntrinsicMap[]
Definition ARM.cpp:541
AArch64SVEIntrinsicMap
static const AArch64SVEAndSMEVectorIntrinsicInfo AArch64SVEIntrinsicMap[]
Definition ARM.cpp:960
getType
TokenType getType() const
Returns the token's type, e.g.
Definition FormatToken.h:999
Result
Result
Implement __builtin_bit_cast and related operations.
Definition InterpBuiltinBitCast.cpp:38
toString
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
Definition SanitizerArgs.cpp:1367
RegisterType
HLSLResourceBindingAttr::RegisterType RegisterType
Definition SemaHLSL.cpp:60
TargetBuiltins.h
Enumerates target-specific builtins in their own namespaces within namespace clang.
clang::ASTContext
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition ASTContext.h:239
clang::ASTContext::GetBuiltinType
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
Definition ASTContext.cpp:13078
clang::ASTContext::GE_None
@ GE_None
No error.
Definition ASTContext.h:2746
clang::CallExpr
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Definition Expr.h:2987
clang::CallExpr::getArg
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
Definition Expr.h:3191
clang::CallExpr::getNumArgs
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
Definition Expr.h:3178
clang::CallExpr::getCallReturnType
QualType getCallReturnType(const ASTContext &Ctx) const
getCallReturnType - Get the return type of the call expr.
Definition Expr.cpp:1635
clang::CharUnits::One
static CharUnits One()
Construct a CharUnits quantity of one.
Definition CharUnits.h:55
clang::CharUnits::fromQuantity
static CharUnits fromQuantity(QuantityType Quantity)
Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:58
clang::CodeGen::Address
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
Definition Address.h:128
clang::CodeGen::Address::invalid
static Address invalid()
Definition Address.h:176
clang::CodeGen::Address::emitRawPointer
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
Definition Address.h:253
clang::CodeGen::Address::getAlignment
CharUnits getAlignment() const
Definition Address.h:194
clang::CodeGen::Address::withElementType
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Definition Address.h:276
clang::CodeGen::Address::getType
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition Address.h:204
clang::CodeGen::AggValueSlot
An aggregate value slot.
Definition CGValue.h:551
clang::CodeGen::AggValueSlot::getAddress
Address getAddress() const
Definition CGValue.h:691
clang::CodeGen::CGDebugInfo::getOrCreateStandaloneType
llvm::DIType * getOrCreateStandaloneType(QualType Ty, SourceLocation Loc)
Emit standalone debug info for a type.
Definition CGDebugInfo.cpp:3108
clang::CodeGen::CodeGenFunction
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Definition CodeGenFunction.h:259
clang::CodeGen::CodeGenFunction::EmitSVEPredicateCast
llvm::Value * EmitSVEPredicateCast(llvm::Value *Pred, llvm::ScalableVectorType *VTy)
Definition ARM.cpp:3400
clang::CodeGen::CodeGenFunction::EmitFP8NeonFMLACall
llvm::Value * EmitFP8NeonFMLACall(unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy, SmallVectorImpl< llvm::Value * > &Ops, const CallExpr *E, const char *name)
Definition ARM.cpp:473
clang::CodeGen::CodeGenFunction::BuildVector
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
Definition ARM.cpp:7333
clang::CodeGen::CodeGenFunction::EmitScalarOrConstFoldImmArg
llvm::Value * EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx, const CallExpr *E)
Definition CodeGenFunction.cpp:2216
clang::CodeGen::CodeGenFunction::EmitSVEStructLoad
llvm::Value * EmitSVEStructLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3575
clang::CodeGen::CodeGenFunction::EmitSVEMaskedLoad
llvm::Value * EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID, bool IsZExtReturn)
Definition ARM.cpp:3684
clang::CodeGen::CodeGenFunction::EmitFP8NeonCall
llvm::Value * EmitFP8NeonCall(unsigned IID, ArrayRef< llvm::Type * > Tys, SmallVectorImpl< llvm::Value * > &O, const CallExpr *E, const char *name)
Definition ARM.cpp:448
clang::CodeGen::CodeGenFunction::ConvertType
llvm::Type * ConvertType(QualType T)
Definition CodeGenFunction.cpp:230
clang::CodeGen::CodeGenFunction::EmitSVEGatherPrefetch
llvm::Value * EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3539
clang::CodeGen::CodeGenFunction::EmitSMEReadWrite
llvm::Value * EmitSMEReadWrite(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3830
clang::CodeGen::CodeGenFunction::SVEBuiltinMemEltTy
llvm::Type * SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags)
SVEBuiltinMemEltTy - Returns the memory element type for this memory access builtin.
Definition ARM.cpp:3266
clang::CodeGen::CodeGenFunction::EmitSVEScatterStore
llvm::Value * EmitSVEScatterStore(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3493
clang::CodeGen::CodeGenFunction::EmitSVEMaskedStore
llvm::Value * EmitSVEMaskedStore(const CallExpr *, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
Definition ARM.cpp:3741
clang::CodeGen::CodeGenFunction::EmitAArch64SMEBuiltinExpr
llvm::Value * EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Definition ARM.cpp:4383
clang::CodeGen::CodeGenFunction::GetAArch64SVEProcessedOperands
void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, SVETypeFlags TypeFlags)
Definition ARM.cpp:3969
clang::CodeGen::CodeGenFunction::EmitSVEGatherLoad
llvm::Value * EmitSVEGatherLoad(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3451
clang::CodeGen::CodeGenFunction::LookupNeonLLVMIntrinsic
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
Definition ARM.cpp:1013
clang::CodeGen::CodeGenFunction::getEltType
llvm::Type * getEltType(const SVETypeFlags &TypeFlags)
Definition ARM.cpp:3282
clang::CodeGen::CodeGenFunction::EmitCommonNeonBuiltinExpr
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
Definition ARM.cpp:1121
clang::CodeGen::CodeGenFunction::EmitNeonSplat
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx, const llvm::ElementCount &Count)
clang::CodeGen::CodeGenFunction::EmitSVEDupX
llvm::Value * EmitSVEDupX(llvm::Value *Scalar)
clang::CodeGen::CodeGenFunction::getTarget
const TargetInfo & getTarget() const
Definition CodeGenFunction.h:2284
clang::CodeGen::CodeGenFunction::EmitAArch64SVEBuiltinExpr
llvm::Value * EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Definition ARM.cpp:4012
clang::CodeGen::CodeGenFunction::EmitFP8NeonCvtCall
llvm::Value * EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0, llvm::Type *Ty1, bool Extract, SmallVectorImpl< llvm::Value * > &Ops, const CallExpr *E, const char *name)
Definition ARM.cpp:494
clang::CodeGen::CodeGenFunction::EmitARMBuiltinExpr
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
Definition ARM.cpp:2162
clang::CodeGen::CodeGenFunction::getSVEType
llvm::ScalableVectorType * getSVEType(const SVETypeFlags &TypeFlags)
Definition ARM.cpp:3355
clang::CodeGen::CodeGenFunction::EmitBPFBuiltinExpr
llvm::Value * EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Definition ARM.cpp:7224
clang::CodeGen::CodeGenFunction::EmitSMELdrStr
llvm::Value * EmitSMELdrStr(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3852
clang::CodeGen::CodeGenFunction::EmitSVETupleCreate
llvm::Value * EmitSVETupleCreate(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
Definition ARM.cpp:3957
clang::CodeGen::CodeGenFunction::EmitSVEPMull
llvm::Value * EmitSVEPMull(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
Definition ARM.cpp:3638
clang::CodeGen::CodeGenFunction::EmitARMMVEBuiltinExpr
llvm::Value * EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
Definition ARM.cpp:2999
clang::CodeGen::CodeGenFunction::CreateAggTemp
AggValueSlot CreateAggTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateAggTemp - Create a temporary memory object for the given aggregate type.
Definition CodeGenFunction.h:3025
clang::CodeGen::CodeGenFunction::EmitNeonRShiftImm
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
Definition ARM.cpp:510
clang::CodeGen::CodeGenFunction::getTypeSize
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
Definition CGStmtOpenMP.cpp:443
clang::CodeGen::CodeGenFunction::EmitToMemory
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
Definition CGExpr.cpp:2270
clang::CodeGen::CodeGenFunction::getSVEOverloadTypes
SmallVector< llvm::Type *, 2 > getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
Definition ARM.cpp:3919
clang::CodeGen::CodeGenFunction::getTargetHooks
const TargetCodeGenInfo & getTargetHooks() const
Definition CodeGenFunction.h:2297
clang::CodeGen::CodeGenFunction::CreateMemTempWithoutCast
RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen without...
Definition CGExpr.cpp:234
clang::CodeGen::CodeGenFunction::EmitNeonShiftVector
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
Definition ARM.cpp:488
clang::CodeGen::CodeGenFunction::IsInPreservedAIRegion
bool IsInPreservedAIRegion
True if CodeGen currently emits code inside presereved access index region.
Definition CodeGenFunction.h:627
clang::CodeGen::CodeGenFunction::EmitAArch64BuiltinExpr
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
Definition ARM.cpp:4470
clang::CodeGen::CodeGenFunction::EmitMSVCBuiltinExpr
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
Definition CGBuiltin.cpp:1851
clang::CodeGen::CodeGenFunction::EmitFP8NeonFDOTCall
llvm::Value * EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy, SmallVectorImpl< llvm::Value * > &Ops, const CallExpr *E, const char *name)
Definition ARM.cpp:457
clang::CodeGen::CodeGenFunction::vectorWrapScalar16
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
Definition ARM.cpp:3254
clang::CodeGen::CodeGenFunction::EmitARMCDEBuiltinExpr
llvm::Value * EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
Definition ARM.cpp:3100
clang::CodeGen::CodeGenFunction::EmitAArch64CompareBuiltinExpr
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Pred, const llvm::Twine &Name="")
Definition ARM.cpp:1842
clang::CodeGen::CodeGenFunction::EmitAnyExprToMem
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
Definition CGExpr.cpp:312
clang::CodeGen::CodeGenFunction::EmitRuntimeCall
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
clang::CodeGen::CodeGenFunction::ConvertTypeForMem
llvm::Type * ConvertTypeForMem(QualType T)
Definition CodeGenFunction.cpp:226
clang::CodeGen::CodeGenFunction::EmitSVEMovl
llvm::Value * EmitSVEMovl(const SVETypeFlags &TypeFlags, llvm::ArrayRef< llvm::Value * > Ops, unsigned BuiltinID)
Definition ARM.cpp:3656
clang::CodeGen::CodeGenFunction::EmitSVEPredicateTupleCast
llvm::Value * EmitSVEPredicateTupleCast(llvm::Value *PredTuple, llvm::StructType *Ty)
Definition ARM.cpp:3435
clang::CodeGen::CodeGenFunction::EmitSVEPrefetchLoad
llvm::Value * EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
Definition ARM.cpp:3663
clang::CodeGen::CodeGenFunction::EmitSMEZero
llvm::Value * EmitSMEZero(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3842
clang::CodeGen::CodeGenFunction::EmitPointerWithAlignment
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
Definition CGExpr.cpp:1624
clang::CodeGen::CodeGenFunction::EmitSVEStructStore
llvm::Value * EmitSVEStructStore(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3590
clang::CodeGen::CodeGenFunction::EmitSMELd1St1
llvm::Value * EmitSMELd1St1(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3795
clang::CodeGen::CodeGenFunction::EmitAggExpr
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
Definition CGExprAgg.cpp:2171
clang::CodeGen::CodeGenFunction::EmitScalarExpr
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
Definition CGExprScalar.cpp:6199
clang::CodeGen::CodeGenFunction::EmitSVEReinterpret
llvm::Value * EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty)
Definition ARM.cpp:3883
clang::CodeGen::CodeGenFunction::ReturnValue
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
Definition CodeGenFunction.h:427
clang::CodeGen::CodeGenFunction::EmitLValue
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
Definition CGExpr.cpp:1740
clang::CodeGen::CodeGenFunction::getLLVMContext
llvm::LLVMContext & getLLVMContext()
Definition CodeGenFunction.h:2285
clang::CodeGen::CodeGenFunction::getSVEPredType
llvm::ScalableVectorType * getSVEPredType(const SVETypeFlags &TypeFlags)
Definition ARM.cpp:3320
clang::CodeGen::CodeGenFunction::EmitNeonCall
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
Definition ARM.cpp:428
clang::CodeGen::CodeGenFunction::EmitSVETupleSetOrGet
llvm::Value * EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags, ArrayRef< llvm::Value * > Ops)
Definition ARM.cpp:3946
clang::CodeGen::CodeGenModule
This class organizes the cross-function state that is used while generating LLVM code.
Definition CodeGenModule.h:333
clang::CodeGen::CodeGenModule::CreateRuntimeFunction
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
Definition CodeGenModule.cpp:6016
clang::CodeGen::CodeGenModule::getContext
ASTContext & getContext() const
Definition CodeGenModule.h:900
clang::CodeGen::CodeGenModule::getLLVMContext
llvm::LLVMContext & getLLVMContext()
Definition CodeGenModule.h:938
clang::CodeGen::CodeGenModule::getIntrinsic
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys={})
Definition CodeGenModule.cpp:7299
clang::CodeGen::LValue::getRawBitFieldPointer
llvm::Value * getRawBitFieldPointer(CodeGenFunction &CGF) const
Definition CGValue.h:441
clang::CodeGen::LValue::emitRawPointer
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Definition CGPointerAuth.cpp:790
clang::CodeGen::ReturnValueSlot
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
Definition CGCall.h:384
clang::Expr
This represents one expression.
Definition Expr.h:113
clang::Expr::EvaluateAsInt
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Definition ExprConstant.cpp:22119
clang::Expr::IgnoreParenCasts
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
Definition Expr.cpp:3131
clang::Expr::EvaluateKnownConstInt
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Definition ExprConstant.cpp:22449
clang::Expr::IgnoreParens
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
Definition Expr.cpp:3122
clang::Expr::getObjectKind
ExprObjectKind getObjectKind() const
getObjectKind - The object kind that this expression produces.
Definition Expr.h:455
clang::Expr::getIntegerConstantExpr
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, bool AllowRelaxedEval=false) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
Definition ExprConstant.cpp:23076
clang::Expr::getExprLoc
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition Expr.cpp:283
clang::Expr::getType
QualType getType() const
Definition Expr.h:145
clang::NeonTypeFlags
Flags to identify the types for overloaded Neon builtins.
Definition TargetBuiltins.h:254
clang::NeonTypeFlags::getEltType
EltType getEltType() const
Definition TargetBuiltins.h:287
clang::PointerType
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition TypeBase.h:3403
clang::PointerType::getPointeeType
QualType getPointeeType() const
Definition TypeBase.h:3413
clang::QualType
A (possibly-)qualified type.
Definition TypeBase.h:938
clang::Qualifiers
The collection of all-type qualifiers we support.
Definition TypeBase.h:332
clang::SVETypeFlags
Flags to identify the types for overloaded SVE builtins.
Definition TargetBuiltins.h:331
clang::SVETypeFlags::isZExtReturn
bool isZExtReturn() const
Definition TargetBuiltins.h:394
clang::SVETypeFlags::isReverseUSDOT
bool isReverseUSDOT() const
Definition TargetBuiltins.h:413
clang::SVETypeFlags::isOverloadNone
bool isOverloadNone() const
Definition TargetBuiltins.h:396
clang::SVETypeFlags::getMemEltType
MemEltType getMemEltType() const
Definition TargetBuiltins.h:372
clang::SVETypeFlags::isGatherLoad
bool isGatherLoad() const
Definition TargetBuiltins.h:390
clang::SVETypeFlags::getEltType
EltType getEltType() const
Definition TargetBuiltins.h:368
clang::SVETypeFlags::isOverloadFirstandLast
bool isOverloadFirstandLast() const
Definition TargetBuiltins.h:402
clang::SVETypeFlags::isOverloadDefault
bool isOverloadDefault() const
Definition TargetBuiltins.h:400
clang::SVETypeFlags::isPrefetch
bool isPrefetch() const
Definition TargetBuiltins.h:408
clang::SVETypeFlags::isOverloadWhileRW
bool isOverloadWhileRW() const
Definition TargetBuiltins.h:401
clang::SVETypeFlags::isTupleSet
bool isTupleSet() const
Definition TargetBuiltins.h:419
clang::SVETypeFlags::isReverseMergeAnyAccOp
bool isReverseMergeAnyAccOp() const
Definition TargetBuiltins.h:415
clang::SVETypeFlags::isReductionQV
bool isReductionQV() const
Definition TargetBuiltins.h:423
clang::SVETypeFlags::isTupleGet
bool isTupleGet() const
Definition TargetBuiltins.h:418
clang::SVETypeFlags::isInsertOp1SVALL
bool isInsertOp1SVALL() const
Definition TargetBuiltins.h:411
clang::SVETypeFlags::isAppendSVALL
bool isAppendSVALL() const
Definition TargetBuiltins.h:410
clang::SVETypeFlags::isReverseMergeAnyBinOp
bool isReverseMergeAnyBinOp() const
Definition TargetBuiltins.h:414
clang::SVETypeFlags::isStructStore
bool isStructStore() const
Definition TargetBuiltins.h:393
clang::SVETypeFlags::isOverloadDefaultAndOp0
bool isOverloadDefaultAndOp0() const
Definition TargetBuiltins.h:405
clang::SVETypeFlags::isTupleCreate
bool isTupleCreate() const
Definition TargetBuiltins.h:417
clang::SVETypeFlags::isGatherPrefetch
bool isGatherPrefetch() const
Definition TargetBuiltins.h:412
clang::SVETypeFlags::hasSplatOperand
bool hasSplatOperand() const
Definition TargetBuiltins.h:384
clang::SVETypeFlags::getMergeType
MergeType getMergeType() const
Definition TargetBuiltins.h:376
clang::SVETypeFlags::isByteIndexed
bool isByteIndexed() const
Definition TargetBuiltins.h:395
clang::SVETypeFlags::isStructLoad
bool isStructLoad() const
Definition TargetBuiltins.h:392
clang::SVETypeFlags::isOverloadWhileOrMultiVecCvt
bool isOverloadWhileOrMultiVecCvt() const
Definition TargetBuiltins.h:397
clang::SVETypeFlags::getSplatOperand
unsigned getSplatOperand() const
Definition TargetBuiltins.h:380
clang::SVETypeFlags::isScatterStore
bool isScatterStore() const
Definition TargetBuiltins.h:391
clang::SVETypeFlags::isReverseCompare
bool isReverseCompare() const
Definition TargetBuiltins.h:409
clang::TargetInfo::getTriple
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
Definition TargetInfo.h:1314
clang::TargetInfo::hasFastHalfType
virtual bool hasFastHalfType() const
Determine whether the target has fast native support for operations on half types.
Definition TargetInfo.h:705
clang::TargetInfo::isBigEndian
bool isBigEndian() const
Definition TargetInfo.h:1745
clang::Type
The base class of the type hierarchy.
Definition TypeBase.h:1879
clang::Type::isSignedIntegerType
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
Definition Type.cpp:2392
clang::Type::castAs
const T * castAs() const
Member-template castAs<specific type>.
Definition TypeBase.h:9366
clang::Type::getPointeeType
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
Definition Type.cpp:885
clang::Value::getType
QualType getType() const
Definition Value.cpp:238
clang::CodeGen::AlignmentSource::Type
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
Definition CGValue.h:155
clang::aarch64::AArch64SISDIntrinsicMap
const ARMNeonVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
Definition AArch64CodeGenUtils.h:417
clang::aarch64::AArch64SIMDIntrinsicMap
const ARMNeonVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
Definition AArch64CodeGenUtils.h:119
clang
Top level wrappers for InstallAPI frontend operations.
Definition CalledOnceCheck.h:17
clang::isa
bool isa(CodeGen::Address addr)
Definition Address.h:330
clang::OK_BitField
@ OK_BitField
A bitfield object is a bitfield on a C or C++ record.
Definition Specifiers.h:158
clang::ObjCSubstitutionContext::Result
@ Result
The result type of a method or function.
Definition TypeBase.h:906
clang::T
const FunctionProtoType * T
Definition RecursiveASTVisitor.h:1425
clang::cast
U cast(CodeGen::Address addr)
Definition Address.h:327
clang::CCEKind::Enumerator
@ Enumerator
Enumerator value with fixed underlying type.
Definition Sema.h:834
llvm
Diagnostic wrappers for TextAPI types for error reporting.
Definition Dominators.h:30
uint32_t
int32_t uint32_t
Definition riscv_packed_simd.h:446
clang::CodeGen::CodeGenTypeCache::Int8Ty
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
Definition CodeGenTypeCache.h:37
clang::CodeGen::CodeGenTypeCache::HalfTy
llvm::Type * HalfTy
half, bfloat, float, double
Definition CodeGenTypeCache.h:39
clang::Expr::EvalResult
EvalResult is a struct with detailed info about an evaluated expression.
Definition Expr.h:666
clang::aarch64::AArch64SVEAndSMEVectorIntrinsicInfo
Describes an AArch64 SVE or SME intrinsic.
Definition AArch64CodeGenUtils.h:91
clang::aarch64::ARMNeonVectorIntrinsicInfo
Describes an ARM or AArch64 NEON intrinsic, or an AArch64 SISD intrinsic.
Definition AArch64CodeGenUtils.h:70
trunc
#define trunc(__x)
Definition tgmath.h:1216
round
#define round(__x)
Definition tgmath.h:1148
rint
#define rint(__x)
Definition tgmath.h:1131
floor
#define floor(__x)
Definition tgmath.h:722
ceil
#define ceil(__x)
Definition tgmath.h:601
Generated on for clang by doxygen 1.14.0