clang 24.0.0git
ARM.cpp
Go to the documentation of this file.
1//===---------- ARM.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfo.h"
14#include "CGBuiltin.h"
15#include "CGDebugInfo.h"
16#include "TargetInfo.h"
20#include "llvm/IR/InlineAsm.h"
21#include "llvm/IR/IntrinsicsAArch64.h"
22#include "llvm/IR/IntrinsicsARM.h"
23#include "llvm/IR/IntrinsicsBPF.h"
24#include "llvm/Support/AArch64MemoryHints.h"
25#include "llvm/TargetParser/AArch64TargetParser.h"
26
27#include <numeric>
28
29using namespace clang;
30using namespace CodeGen;
31using namespace llvm;
32using namespace clang::aarch64;
33
34static std::optional<CodeGenFunction::MSVCIntrin>
35translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
36 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
37 switch (BuiltinID) {
38 default:
39 return std::nullopt;
40 case clang::AArch64::BI_BitScanForward:
41 case clang::AArch64::BI_BitScanForward64:
42 return MSVCIntrin::_BitScanForward;
43 case clang::AArch64::BI_BitScanReverse:
44 case clang::AArch64::BI_BitScanReverse64:
45 return MSVCIntrin::_BitScanReverse;
46 case clang::AArch64::BI_InterlockedAnd64:
47 return MSVCIntrin::_InterlockedAnd;
48 case clang::AArch64::BI_InterlockedExchange64:
49 return MSVCIntrin::_InterlockedExchange;
50 case clang::AArch64::BI_InterlockedExchangeAdd64:
51 return MSVCIntrin::_InterlockedExchangeAdd;
52 case clang::AArch64::BI_InterlockedExchangeSub64:
53 return MSVCIntrin::_InterlockedExchangeSub;
54 case clang::AArch64::BI_InterlockedOr64:
55 return MSVCIntrin::_InterlockedOr;
56 case clang::AArch64::BI_InterlockedXor64:
57 return MSVCIntrin::_InterlockedXor;
58 case clang::AArch64::BI_InterlockedDecrement64:
59 return MSVCIntrin::_InterlockedDecrement;
60 case clang::AArch64::BI_InterlockedIncrement64:
61 return MSVCIntrin::_InterlockedIncrement;
62 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
63 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
64 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
65 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
66 return MSVCIntrin::_InterlockedExchangeAdd_acq;
67 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
68 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
69 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
70 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
71 return MSVCIntrin::_InterlockedExchangeAdd_rel;
72 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
73 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
74 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
75 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
76 return MSVCIntrin::_InterlockedExchangeAdd_nf;
77 case clang::AArch64::BI_InterlockedExchange8_acq:
78 case clang::AArch64::BI_InterlockedExchange16_acq:
79 case clang::AArch64::BI_InterlockedExchange_acq:
80 case clang::AArch64::BI_InterlockedExchange64_acq:
81 case clang::AArch64::BI_InterlockedExchangePointer_acq:
82 return MSVCIntrin::_InterlockedExchange_acq;
83 case clang::AArch64::BI_InterlockedExchange8_rel:
84 case clang::AArch64::BI_InterlockedExchange16_rel:
85 case clang::AArch64::BI_InterlockedExchange_rel:
86 case clang::AArch64::BI_InterlockedExchange64_rel:
87 case clang::AArch64::BI_InterlockedExchangePointer_rel:
88 return MSVCIntrin::_InterlockedExchange_rel;
89 case clang::AArch64::BI_InterlockedExchange8_nf:
90 case clang::AArch64::BI_InterlockedExchange16_nf:
91 case clang::AArch64::BI_InterlockedExchange_nf:
92 case clang::AArch64::BI_InterlockedExchange64_nf:
93 case clang::AArch64::BI_InterlockedExchangePointer_nf:
94 return MSVCIntrin::_InterlockedExchange_nf;
95 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
96 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
97 case clang::AArch64::BI_InterlockedCompareExchange_acq:
98 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
99 case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
100 return MSVCIntrin::_InterlockedCompareExchange_acq;
101 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
102 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
103 case clang::AArch64::BI_InterlockedCompareExchange_rel:
104 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
105 case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
106 return MSVCIntrin::_InterlockedCompareExchange_rel;
107 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
108 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
109 case clang::AArch64::BI_InterlockedCompareExchange_nf:
110 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
111 return MSVCIntrin::_InterlockedCompareExchange_nf;
112 case clang::AArch64::BI_InterlockedCompareExchange128:
113 return MSVCIntrin::_InterlockedCompareExchange128;
114 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
115 return MSVCIntrin::_InterlockedCompareExchange128_acq;
116 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
117 return MSVCIntrin::_InterlockedCompareExchange128_nf;
118 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
119 return MSVCIntrin::_InterlockedCompareExchange128_rel;
120 case clang::AArch64::BI_InterlockedOr8_acq:
121 case clang::AArch64::BI_InterlockedOr16_acq:
122 case clang::AArch64::BI_InterlockedOr_acq:
123 case clang::AArch64::BI_InterlockedOr64_acq:
124 return MSVCIntrin::_InterlockedOr_acq;
125 case clang::AArch64::BI_InterlockedOr8_rel:
126 case clang::AArch64::BI_InterlockedOr16_rel:
127 case clang::AArch64::BI_InterlockedOr_rel:
128 case clang::AArch64::BI_InterlockedOr64_rel:
129 return MSVCIntrin::_InterlockedOr_rel;
130 case clang::AArch64::BI_InterlockedOr8_nf:
131 case clang::AArch64::BI_InterlockedOr16_nf:
132 case clang::AArch64::BI_InterlockedOr_nf:
133 case clang::AArch64::BI_InterlockedOr64_nf:
134 return MSVCIntrin::_InterlockedOr_nf;
135 case clang::AArch64::BI_InterlockedXor8_acq:
136 case clang::AArch64::BI_InterlockedXor16_acq:
137 case clang::AArch64::BI_InterlockedXor_acq:
138 case clang::AArch64::BI_InterlockedXor64_acq:
139 return MSVCIntrin::_InterlockedXor_acq;
140 case clang::AArch64::BI_InterlockedXor8_rel:
141 case clang::AArch64::BI_InterlockedXor16_rel:
142 case clang::AArch64::BI_InterlockedXor_rel:
143 case clang::AArch64::BI_InterlockedXor64_rel:
144 return MSVCIntrin::_InterlockedXor_rel;
145 case clang::AArch64::BI_InterlockedXor8_nf:
146 case clang::AArch64::BI_InterlockedXor16_nf:
147 case clang::AArch64::BI_InterlockedXor_nf:
148 case clang::AArch64::BI_InterlockedXor64_nf:
149 return MSVCIntrin::_InterlockedXor_nf;
150 case clang::AArch64::BI_InterlockedAnd8_acq:
151 case clang::AArch64::BI_InterlockedAnd16_acq:
152 case clang::AArch64::BI_InterlockedAnd_acq:
153 case clang::AArch64::BI_InterlockedAnd64_acq:
154 return MSVCIntrin::_InterlockedAnd_acq;
155 case clang::AArch64::BI_InterlockedAnd8_rel:
156 case clang::AArch64::BI_InterlockedAnd16_rel:
157 case clang::AArch64::BI_InterlockedAnd_rel:
158 case clang::AArch64::BI_InterlockedAnd64_rel:
159 return MSVCIntrin::_InterlockedAnd_rel;
160 case clang::AArch64::BI_InterlockedAnd8_nf:
161 case clang::AArch64::BI_InterlockedAnd16_nf:
162 case clang::AArch64::BI_InterlockedAnd_nf:
163 case clang::AArch64::BI_InterlockedAnd64_nf:
164 return MSVCIntrin::_InterlockedAnd_nf;
165 case clang::AArch64::BI_InterlockedIncrement16_acq:
166 case clang::AArch64::BI_InterlockedIncrement_acq:
167 case clang::AArch64::BI_InterlockedIncrement64_acq:
168 return MSVCIntrin::_InterlockedIncrement_acq;
169 case clang::AArch64::BI_InterlockedIncrement16_rel:
170 case clang::AArch64::BI_InterlockedIncrement_rel:
171 case clang::AArch64::BI_InterlockedIncrement64_rel:
172 return MSVCIntrin::_InterlockedIncrement_rel;
173 case clang::AArch64::BI_InterlockedIncrement16_nf:
174 case clang::AArch64::BI_InterlockedIncrement_nf:
175 case clang::AArch64::BI_InterlockedIncrement64_nf:
176 return MSVCIntrin::_InterlockedIncrement_nf;
177 case clang::AArch64::BI_InterlockedDecrement16_acq:
178 case clang::AArch64::BI_InterlockedDecrement_acq:
179 case clang::AArch64::BI_InterlockedDecrement64_acq:
180 return MSVCIntrin::_InterlockedDecrement_acq;
181 case clang::AArch64::BI_InterlockedDecrement16_rel:
182 case clang::AArch64::BI_InterlockedDecrement_rel:
183 case clang::AArch64::BI_InterlockedDecrement64_rel:
184 return MSVCIntrin::_InterlockedDecrement_rel;
185 case clang::AArch64::BI_InterlockedDecrement16_nf:
186 case clang::AArch64::BI_InterlockedDecrement_nf:
187 case clang::AArch64::BI_InterlockedDecrement64_nf:
188 return MSVCIntrin::_InterlockedDecrement_nf;
189 case clang::AArch64::BI_ReadWriteBarrier:
190 return MSVCIntrin::_ReadWriteBarrier;
191 }
192 llvm_unreachable("must return from switch");
193}
194
195static std::optional<CodeGenFunction::MSVCIntrin>
196translateArmToMsvcIntrin(unsigned BuiltinID) {
197 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
198 switch (BuiltinID) {
199 default:
200 return std::nullopt;
201 case clang::ARM::BI_BitScanForward:
202 case clang::ARM::BI_BitScanForward64:
203 return MSVCIntrin::_BitScanForward;
204 case clang::ARM::BI_BitScanReverse:
205 case clang::ARM::BI_BitScanReverse64:
206 return MSVCIntrin::_BitScanReverse;
207 case clang::ARM::BI_InterlockedAnd64:
208 return MSVCIntrin::_InterlockedAnd;
209 case clang::ARM::BI_InterlockedExchange64:
210 return MSVCIntrin::_InterlockedExchange;
211 case clang::ARM::BI_InterlockedExchangeAdd64:
212 return MSVCIntrin::_InterlockedExchangeAdd;
213 case clang::ARM::BI_InterlockedExchangeSub64:
214 return MSVCIntrin::_InterlockedExchangeSub;
215 case clang::ARM::BI_InterlockedOr64:
216 return MSVCIntrin::_InterlockedOr;
217 case clang::ARM::BI_InterlockedXor64:
218 return MSVCIntrin::_InterlockedXor;
219 case clang::ARM::BI_InterlockedDecrement64:
220 return MSVCIntrin::_InterlockedDecrement;
221 case clang::ARM::BI_InterlockedIncrement64:
222 return MSVCIntrin::_InterlockedIncrement;
223 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
224 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
225 case clang::ARM::BI_InterlockedExchangeAdd_acq:
226 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
227 return MSVCIntrin::_InterlockedExchangeAdd_acq;
228 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
229 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
230 case clang::ARM::BI_InterlockedExchangeAdd_rel:
231 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
232 return MSVCIntrin::_InterlockedExchangeAdd_rel;
233 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
234 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
235 case clang::ARM::BI_InterlockedExchangeAdd_nf:
236 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
237 return MSVCIntrin::_InterlockedExchangeAdd_nf;
238 case clang::ARM::BI_InterlockedExchange8_acq:
239 case clang::ARM::BI_InterlockedExchange16_acq:
240 case clang::ARM::BI_InterlockedExchange_acq:
241 case clang::ARM::BI_InterlockedExchange64_acq:
242 case clang::ARM::BI_InterlockedExchangePointer_acq:
243 return MSVCIntrin::_InterlockedExchange_acq;
244 case clang::ARM::BI_InterlockedExchange8_rel:
245 case clang::ARM::BI_InterlockedExchange16_rel:
246 case clang::ARM::BI_InterlockedExchange_rel:
247 case clang::ARM::BI_InterlockedExchange64_rel:
248 case clang::ARM::BI_InterlockedExchangePointer_rel:
249 return MSVCIntrin::_InterlockedExchange_rel;
250 case clang::ARM::BI_InterlockedExchange8_nf:
251 case clang::ARM::BI_InterlockedExchange16_nf:
252 case clang::ARM::BI_InterlockedExchange_nf:
253 case clang::ARM::BI_InterlockedExchange64_nf:
254 case clang::ARM::BI_InterlockedExchangePointer_nf:
255 return MSVCIntrin::_InterlockedExchange_nf;
256 case clang::ARM::BI_InterlockedCompareExchange8_acq:
257 case clang::ARM::BI_InterlockedCompareExchange16_acq:
258 case clang::ARM::BI_InterlockedCompareExchange_acq:
259 case clang::ARM::BI_InterlockedCompareExchange64_acq:
260 case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
261 return MSVCIntrin::_InterlockedCompareExchange_acq;
262 case clang::ARM::BI_InterlockedCompareExchange8_rel:
263 case clang::ARM::BI_InterlockedCompareExchange16_rel:
264 case clang::ARM::BI_InterlockedCompareExchange_rel:
265 case clang::ARM::BI_InterlockedCompareExchange64_rel:
266 case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
267 return MSVCIntrin::_InterlockedCompareExchange_rel;
268 case clang::ARM::BI_InterlockedCompareExchange8_nf:
269 case clang::ARM::BI_InterlockedCompareExchange16_nf:
270 case clang::ARM::BI_InterlockedCompareExchange_nf:
271 case clang::ARM::BI_InterlockedCompareExchange64_nf:
272 return MSVCIntrin::_InterlockedCompareExchange_nf;
273 case clang::ARM::BI_InterlockedOr8_acq:
274 case clang::ARM::BI_InterlockedOr16_acq:
275 case clang::ARM::BI_InterlockedOr_acq:
276 case clang::ARM::BI_InterlockedOr64_acq:
277 return MSVCIntrin::_InterlockedOr_acq;
278 case clang::ARM::BI_InterlockedOr8_rel:
279 case clang::ARM::BI_InterlockedOr16_rel:
280 case clang::ARM::BI_InterlockedOr_rel:
281 case clang::ARM::BI_InterlockedOr64_rel:
282 return MSVCIntrin::_InterlockedOr_rel;
283 case clang::ARM::BI_InterlockedOr8_nf:
284 case clang::ARM::BI_InterlockedOr16_nf:
285 case clang::ARM::BI_InterlockedOr_nf:
286 case clang::ARM::BI_InterlockedOr64_nf:
287 return MSVCIntrin::_InterlockedOr_nf;
288 case clang::ARM::BI_InterlockedXor8_acq:
289 case clang::ARM::BI_InterlockedXor16_acq:
290 case clang::ARM::BI_InterlockedXor_acq:
291 case clang::ARM::BI_InterlockedXor64_acq:
292 return MSVCIntrin::_InterlockedXor_acq;
293 case clang::ARM::BI_InterlockedXor8_rel:
294 case clang::ARM::BI_InterlockedXor16_rel:
295 case clang::ARM::BI_InterlockedXor_rel:
296 case clang::ARM::BI_InterlockedXor64_rel:
297 return MSVCIntrin::_InterlockedXor_rel;
298 case clang::ARM::BI_InterlockedXor8_nf:
299 case clang::ARM::BI_InterlockedXor16_nf:
300 case clang::ARM::BI_InterlockedXor_nf:
301 case clang::ARM::BI_InterlockedXor64_nf:
302 return MSVCIntrin::_InterlockedXor_nf;
303 case clang::ARM::BI_InterlockedAnd8_acq:
304 case clang::ARM::BI_InterlockedAnd16_acq:
305 case clang::ARM::BI_InterlockedAnd_acq:
306 case clang::ARM::BI_InterlockedAnd64_acq:
307 return MSVCIntrin::_InterlockedAnd_acq;
308 case clang::ARM::BI_InterlockedAnd8_rel:
309 case clang::ARM::BI_InterlockedAnd16_rel:
310 case clang::ARM::BI_InterlockedAnd_rel:
311 case clang::ARM::BI_InterlockedAnd64_rel:
312 return MSVCIntrin::_InterlockedAnd_rel;
313 case clang::ARM::BI_InterlockedAnd8_nf:
314 case clang::ARM::BI_InterlockedAnd16_nf:
315 case clang::ARM::BI_InterlockedAnd_nf:
316 case clang::ARM::BI_InterlockedAnd64_nf:
317 return MSVCIntrin::_InterlockedAnd_nf;
318 case clang::ARM::BI_InterlockedIncrement16_acq:
319 case clang::ARM::BI_InterlockedIncrement_acq:
320 case clang::ARM::BI_InterlockedIncrement64_acq:
321 return MSVCIntrin::_InterlockedIncrement_acq;
322 case clang::ARM::BI_InterlockedIncrement16_rel:
323 case clang::ARM::BI_InterlockedIncrement_rel:
324 case clang::ARM::BI_InterlockedIncrement64_rel:
325 return MSVCIntrin::_InterlockedIncrement_rel;
326 case clang::ARM::BI_InterlockedIncrement16_nf:
327 case clang::ARM::BI_InterlockedIncrement_nf:
328 case clang::ARM::BI_InterlockedIncrement64_nf:
329 return MSVCIntrin::_InterlockedIncrement_nf;
330 case clang::ARM::BI_InterlockedDecrement16_acq:
331 case clang::ARM::BI_InterlockedDecrement_acq:
332 case clang::ARM::BI_InterlockedDecrement64_acq:
333 return MSVCIntrin::_InterlockedDecrement_acq;
334 case clang::ARM::BI_InterlockedDecrement16_rel:
335 case clang::ARM::BI_InterlockedDecrement_rel:
336 case clang::ARM::BI_InterlockedDecrement64_rel:
337 return MSVCIntrin::_InterlockedDecrement_rel;
338 case clang::ARM::BI_InterlockedDecrement16_nf:
339 case clang::ARM::BI_InterlockedDecrement_nf:
340 case clang::ARM::BI_InterlockedDecrement64_nf:
341 return MSVCIntrin::_InterlockedDecrement_nf;
342 case clang::ARM::BI_ReadWriteBarrier:
343 case clang::ARM::BI_ReadBarrier:
344 case clang::ARM::BI_WriteBarrier:
345 return MSVCIntrin::_ReadWriteBarrier;
346 }
347 llvm_unreachable("must return from switch");
348}
349
350// Emit an intrinsic where all operands are of the same type as the result.
351// Depending on mode, this may be a constrained floating-point intrinsic.
353 unsigned IntrinsicID,
354 unsigned ConstrainedIntrinsicID,
355 llvm::Type *Ty,
356 ArrayRef<Value *> Args) {
357 Function *F;
358 if (CGF.Builder.getIsFPConstrained())
359 F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Ty);
360 else
361 F = CGF.CGM.getIntrinsic(IntrinsicID, Ty);
362
363 if (CGF.Builder.getIsFPConstrained())
364 return CGF.Builder.CreateConstrainedFPCall(F, Args);
365
366 return CGF.Builder.CreateCall(F, Args);
367}
368
369static llvm::FixedVectorType *GetNeonType(CodeGenFunction *CGF,
370 NeonTypeFlags TypeFlags,
371 bool HasFastHalfType = true,
372 bool V1Ty = false,
373 bool AllowBFloatArgsAndRet = true) {
374 int IsQuad = TypeFlags.isQuad();
375 switch (TypeFlags.getEltType()) {
379 return llvm::FixedVectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad));
382 return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
384 if (AllowBFloatArgsAndRet)
385 return llvm::FixedVectorType::get(CGF->BFloatTy, V1Ty ? 1 : (4 << IsQuad));
386 return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
388 if (HasFastHalfType)
389 return llvm::FixedVectorType::get(CGF->HalfTy, V1Ty ? 1 : (4 << IsQuad));
390 return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
392 return llvm::FixedVectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad));
395 return llvm::FixedVectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad));
397 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
398 // There is a lot of i128 and f128 API missing.
399 // so we use v16i8 to represent poly128 and get pattern matched.
400 return llvm::FixedVectorType::get(CGF->Int8Ty, 16);
402 return llvm::FixedVectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad));
404 return llvm::FixedVectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad));
405 }
406 llvm_unreachable("Unknown vector element type!");
407}
408
409static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF,
410 NeonTypeFlags IntTypeFlags) {
411 int IsQuad = IntTypeFlags.isQuad();
412 switch (IntTypeFlags.getEltType()) {
414 return llvm::FixedVectorType::get(CGF->HalfTy, (4 << IsQuad));
416 return llvm::FixedVectorType::get(CGF->FloatTy, (2 << IsQuad));
418 return llvm::FixedVectorType::get(CGF->DoubleTy, (1 << IsQuad));
419 default:
420 llvm_unreachable("Type can't be converted to floating-point!");
421 }
422}
423
425 const ElementCount &Count) {
426 Value *SV = llvm::ConstantVector::getSplat(Count, C);
427 return Builder.CreateShuffleVector(V, V, SV, "lane");
428}
429
431 ElementCount EC = cast<llvm::VectorType>(V->getType())->getElementCount();
432 return EmitNeonSplat(V, C, EC);
433}
434
436 const char *name,
437 unsigned shift, bool rightshift) {
438 unsigned j = 0;
439 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
440 ai != ae; ++ai, ++j) {
441 if (F->isConstrainedFPIntrinsic())
442 if (ai->getType()->isMetadataTy())
443 continue;
444 if (shift > 0 && shift == j)
445 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
446 else
447 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
448 }
449
450 if (F->isConstrainedFPIntrinsic())
451 return Builder.CreateConstrainedFPCall(F, Ops, name);
452 return Builder.CreateCall(F, Ops, name);
453}
454
458 const CallExpr *E, const char *name) {
459 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_set_fpmr),
460 Ops.pop_back_val());
461 return EmitNeonCall(CGM.getIntrinsic(IID, Tys), Ops, name);
462}
463
465 unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy,
466 SmallVectorImpl<llvm::Value *> &Ops, const CallExpr *E, const char *name) {
467
468 const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
469 RetTy->getPrimitiveSizeInBits();
470 llvm::Type *Tys[] = {llvm::FixedVectorType::get(RetTy, ElemCount),
471 Ops[1]->getType()};
472 if (ExtendLaneArg) {
473 auto *VT = llvm::FixedVectorType::get(Int8Ty, 16);
474 Ops[2] = Builder.CreateInsertVector(VT, PoisonValue::get(VT), Ops[2],
475 uint64_t(0));
476 }
477 return EmitFP8NeonCall(IID, Tys, Ops, E, name);
478}
479
481 unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy,
482 SmallVectorImpl<llvm::Value *> &Ops, const CallExpr *E, const char *name) {
483
484 if (ExtendLaneArg) {
485 auto *VT = llvm::FixedVectorType::get(Int8Ty, 16);
486 Ops[2] = Builder.CreateInsertVector(VT, PoisonValue::get(VT), Ops[2],
487 uint64_t(0));
488 }
489 const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
490 RetTy->getPrimitiveSizeInBits();
491 return EmitFP8NeonCall(IID, {llvm::FixedVectorType::get(RetTy, ElemCount)},
492 Ops, E, name);
493}
494
496 bool neg) {
497 int SV = cast<ConstantInt>(V)->getSExtValue();
498 return ConstantInt::getSigned(Ty, neg ? -SV : SV);
499}
500
501Value *CodeGenFunction::EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
502 llvm::Type *Ty1, bool Extract,
504 const CallExpr *E,
505 const char *name) {
506 llvm::Type *Tys[] = {Ty0, Ty1};
507 if (Extract) {
508 // Op[0] is mfloat8x16_t, but the intrinsic converts only the lower part of
509 // the vector.
510 Tys[1] = llvm::FixedVectorType::get(Int8Ty, 8);
511 Ops[0] = Builder.CreateExtractVector(Tys[1], Ops[0], uint64_t(0));
512 }
513 return EmitFP8NeonCall(IID, Tys, Ops, E, name);
514}
515
516// Right-shift a vector by a constant.
518 llvm::Type *Ty, bool usgn,
519 const char *name) {
520 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
521
522 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
523 int EltSize = VTy->getScalarSizeInBits();
524
525 Vec = Builder.CreateBitCast(Vec, Ty);
526
527 // lshr/ashr are undefined when the shift amount is equal to the vector
528 // element size.
529 if (ShiftAmt == EltSize) {
530 if (usgn) {
531 // Right-shifting an unsigned value by its size yields 0.
532 return llvm::ConstantAggregateZero::get(VTy);
533 } else {
534 // Right-shifting a signed value by its size is equivalent
535 // to a shift of size-1.
536 --ShiftAmt;
537 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
538 }
539 }
540
541 Shift = EmitNeonShiftVector(Shift, Ty, false);
542 if (usgn)
543 return Builder.CreateLShr(Vec, Shift, name);
544 return Builder.CreateAShr(Vec, Shift, name);
545}
546
547// clang-format off
549 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
550 NEONMAP0(splat_lane_v),
551 NEONMAP0(splat_laneq_v),
552 NEONMAP0(splatq_lane_v),
553 NEONMAP0(splatq_laneq_v),
554 NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
555 NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
556 NEONMAP1(vabs_v, arm_neon_vabs, 0),
557 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
558 NEONMAP0(vadd_v),
559 NEONMAP0(vaddhn_v),
560 NEONMAP0(vaddq_v),
561 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
562 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
563 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
564 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
565 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
566 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
567 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
568 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
569 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
570 NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType),
571 NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType),
572 NEONMAP1(vcadd_rot270_f16, arm_neon_vcadd_rot270, Add1ArgType),
573 NEONMAP1(vcadd_rot270_f32, arm_neon_vcadd_rot270, Add1ArgType),
574 NEONMAP1(vcadd_rot90_f16, arm_neon_vcadd_rot90, Add1ArgType),
575 NEONMAP1(vcadd_rot90_f32, arm_neon_vcadd_rot90, Add1ArgType),
576 NEONMAP1(vcaddq_rot270_f16, arm_neon_vcadd_rot270, Add1ArgType),
577 NEONMAP1(vcaddq_rot270_f32, arm_neon_vcadd_rot270, Add1ArgType),
578 NEONMAP1(vcaddq_rot270_f64, arm_neon_vcadd_rot270, Add1ArgType),
579 NEONMAP1(vcaddq_rot90_f16, arm_neon_vcadd_rot90, Add1ArgType),
580 NEONMAP1(vcaddq_rot90_f32, arm_neon_vcadd_rot90, Add1ArgType),
581 NEONMAP1(vcaddq_rot90_f64, arm_neon_vcadd_rot90, Add1ArgType),
582 NEONMAP1(vcage_v, arm_neon_vacge, 0),
583 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
584 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
585 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
586 NEONMAP1(vcale_v, arm_neon_vacge, 0),
587 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
588 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
589 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
590 NEONMAP0(vceqz_v),
591 NEONMAP0(vceqzq_v),
592 NEONMAP0(vcgez_v),
593 NEONMAP0(vcgezq_v),
594 NEONMAP0(vcgtz_v),
595 NEONMAP0(vcgtzq_v),
596 NEONMAP0(vclez_v),
597 NEONMAP0(vclezq_v),
598 NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType),
599 NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType),
600 NEONMAP0(vcltz_v),
601 NEONMAP0(vcltzq_v),
602 NEONMAP1(vclz_v, ctlz, Add1ArgType),
603 NEONMAP1(vclzq_v, ctlz, Add1ArgType),
604 NEONMAP1(vcnt_v, ctpop, Add1ArgType),
605 NEONMAP1(vcntq_v, ctpop, Add1ArgType),
606 NEONMAP0(vcvt_f16_s16),
607 NEONMAP0(vcvt_f16_u16),
608 NEONMAP0(vcvt_f32_v),
609 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
610 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
611 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
612 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
613 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
614 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
615 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
616 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
617 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
618 NEONMAP0(vcvt_s16_f16),
619 NEONMAP0(vcvt_s32_v),
620 NEONMAP0(vcvt_s64_v),
621 NEONMAP0(vcvt_u16_f16),
622 NEONMAP0(vcvt_u32_v),
623 NEONMAP0(vcvt_u64_v),
624 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
625 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
626 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
627 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
628 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
629 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
630 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
631 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
632 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
633 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
634 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
635 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
636 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
637 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
638 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
639 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
640 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
641 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
642 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
643 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
644 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
645 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
646 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
647 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
648 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
649 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
650 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
651 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
652 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
653 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
654 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
655 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
656 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
657 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
658 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
659 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
660 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
661 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
662 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
663 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
664 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
665 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
666 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
667 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
668 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
669 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
670 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
671 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
672 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
673 NEONMAP0(vcvtq_f16_s16),
674 NEONMAP0(vcvtq_f16_u16),
675 NEONMAP0(vcvtq_f32_v),
676 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
677 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
678 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
679 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
680 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
681 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
682 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
683 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
684 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
685 NEONMAP0(vcvtq_s16_f16),
686 NEONMAP0(vcvtq_s32_v),
687 NEONMAP0(vcvtq_s64_v),
688 NEONMAP0(vcvtq_u16_f16),
689 NEONMAP0(vcvtq_u32_v),
690 NEONMAP0(vcvtq_u64_v),
691 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
692 NEONMAP1(vdot_u32, arm_neon_udot, 0),
693 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
694 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
695 NEONMAP0(vext_v),
696 NEONMAP0(vextq_v),
697 NEONMAP0(vfma_v),
698 NEONMAP0(vfmaq_v),
699 NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
700 NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
701 NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
702 NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
703 NEONMAP0(vld1_dup_v),
704 NEONMAP1(vld1_v, arm_neon_vld1, 0),
705 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
706 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
707 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
708 NEONMAP0(vld1q_dup_v),
709 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
710 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
711 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
712 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
713 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
714 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
715 NEONMAP1(vld2_v, arm_neon_vld2, 0),
716 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
717 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
718 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
719 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
720 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
721 NEONMAP1(vld3_v, arm_neon_vld3, 0),
722 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
723 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
724 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
725 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
726 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
727 NEONMAP1(vld4_v, arm_neon_vld4, 0),
728 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
729 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
730 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
731 NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
732 NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType),
733 NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType),
734 NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
735 NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
736 NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType),
737 NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType),
738 NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
739 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
740 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
741 NEONMAP0(vmovl_v),
742 NEONMAP0(vmovn_v),
743 NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType),
744 NEONMAP0(vmull_v),
745 NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType),
746 NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
747 NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
748 NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType),
749 NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
750 NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
751 NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType),
752 NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts),
753 NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts),
754 NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType),
755 NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType),
756 NEONMAP2(vqadd_v, uadd_sat, sadd_sat, Add1ArgType | UnsignedAlts),
757 NEONMAP2(vqaddq_v, uadd_sat, sadd_sat, Add1ArgType | UnsignedAlts),
758 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
759 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
760 NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType),
761 NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType),
762 NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType),
763 NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts),
764 NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType),
765 NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType),
766 NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType),
767 NEONMAP1(vqrdmlah_s16, arm_neon_vqrdmlah, Add1ArgType),
768 NEONMAP1(vqrdmlah_s32, arm_neon_vqrdmlah, Add1ArgType),
769 NEONMAP1(vqrdmlahq_s16, arm_neon_vqrdmlah, Add1ArgType),
770 NEONMAP1(vqrdmlahq_s32, arm_neon_vqrdmlah, Add1ArgType),
771 NEONMAP1(vqrdmlsh_s16, arm_neon_vqrdmlsh, Add1ArgType),
772 NEONMAP1(vqrdmlsh_s32, arm_neon_vqrdmlsh, Add1ArgType),
773 NEONMAP1(vqrdmlshq_s16, arm_neon_vqrdmlsh, Add1ArgType),
774 NEONMAP1(vqrdmlshq_s32, arm_neon_vqrdmlsh, Add1ArgType),
775 NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType),
776 NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType),
777 NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
778 NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
779 NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
780 NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
781 NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
782 NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
783 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
784 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
785 NEONMAP2(vqsub_v, usub_sat, ssub_sat, Add1ArgType | UnsignedAlts),
786 NEONMAP2(vqsubq_v, usub_sat, ssub_sat, Add1ArgType | UnsignedAlts),
787 NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType),
788 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
789 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
790 NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType),
791 NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType),
792 NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
793 NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
794 NEONMAP1(vrnd_v, trunc, Add1ArgType),
795 NEONMAP1(vrnda_v, round, Add1ArgType),
796 NEONMAP1(vrndaq_v, round, Add1ArgType),
797 NEONMAP0(vrndi_v),
798 NEONMAP0(vrndiq_v),
799 NEONMAP1(vrndm_v, floor, Add1ArgType),
800 NEONMAP1(vrndmq_v, floor, Add1ArgType),
801 NEONMAP1(vrndn_v, roundeven, Add1ArgType),
802 NEONMAP1(vrndnq_v, roundeven, Add1ArgType),
803 NEONMAP1(vrndp_v, ceil, Add1ArgType),
804 NEONMAP1(vrndpq_v, ceil, Add1ArgType),
805 NEONMAP1(vrndq_v, trunc, Add1ArgType),
806 NEONMAP1(vrndx_v, rint, Add1ArgType),
807 NEONMAP1(vrndxq_v, rint, Add1ArgType),
808 NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
809 NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
810 NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
811 NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
812 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
813 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
814 NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType),
815 NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType),
816 NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType),
817 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
818 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
819 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
820 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
821 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
822 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
823 NEONMAP0(vshl_n_v),
824 NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
825 NEONMAP0(vshll_n_v),
826 NEONMAP0(vshlq_n_v),
827 NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
828 NEONMAP0(vshr_n_v),
829 NEONMAP0(vshrn_n_v),
830 NEONMAP0(vshrq_n_v),
831 NEONMAP1(vst1_v, arm_neon_vst1, 0),
832 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
833 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
834 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
835 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
836 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
837 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
838 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
839 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
840 NEONMAP1(vst2_v, arm_neon_vst2, 0),
841 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
842 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
843 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
844 NEONMAP1(vst3_v, arm_neon_vst3, 0),
845 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
846 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
847 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
848 NEONMAP1(vst4_v, arm_neon_vst4, 0),
849 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
850 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
851 NEONMAP0(vsubhn_v),
852 NEONMAP0(vtrn_v),
853 NEONMAP0(vtrnq_v),
854 NEONMAP0(vtst_v),
855 NEONMAP0(vtstq_v),
856 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
857 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
858 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
859 NEONMAP0(vuzp_v),
860 NEONMAP0(vuzpq_v),
861 NEONMAP0(vzip_v),
862 NEONMAP0(vzipq_v)
863};
864
865// clang-format on
866
867// Some intrinsics are equivalent for codegen.
868static const std::pair<unsigned, unsigned> NEONEquivalentIntrinsicMap[] = {
869 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
870 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
871 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
872 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
873 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
874 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
875 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
876 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
877 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
878 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
879 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
880 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
881 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
882 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
883 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
884 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
885 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
886 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
887 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
888 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
889 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
890 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
891 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
892 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
893 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
894 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
895 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
896 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
897 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
898 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
899 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
900 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
901 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
902 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
903 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
904 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
905 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
906 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
907 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
908 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
909 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
910 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
911 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
912 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
913 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
914 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
915 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
916 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
917 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
918 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
919 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
920 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
921 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
922 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
923 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
924 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
925 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
926 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
927 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
928 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
929 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
930 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
931 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
932 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
933 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
934 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
935 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
936 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
937 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
938 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
939 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
940 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
941 // The mangling rules cause us to have one ID for each type for vldap1(q)_lane
942 // and vstl1(q)_lane, but codegen is equivalent for all of them. Choose an
943 // arbitrary one to be handled as tha canonical variation.
944 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
945 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
946 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
947 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
948 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
949 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
950 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
951 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
952 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
953 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
954 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
955 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
956};
957
958#undef NEONMAP0
959#undef NEONMAP1
960#undef NEONMAP2
961
962#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
963 {SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, TypeModifier}
964
965#define SVEMAP2(NameBase, TypeModifier) \
966 {SVE::BI__builtin_sve_##NameBase, 0, TypeModifier}
968#define GET_SVE_LLVM_INTRINSIC_MAP
969#include "clang/Basic/arm_sve_builtin_cg.inc"
970#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
971#undef GET_SVE_LLVM_INTRINSIC_MAP
972};
973
974#undef SVEMAP1
975#undef SVEMAP2
976
977#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
978 {SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, TypeModifier}
979
980#define SMEMAP2(NameBase, TypeModifier) \
981 {SME::BI__builtin_sme_##NameBase, 0, TypeModifier}
983#define GET_SME_LLVM_INTRINSIC_MAP
984#include "clang/Basic/arm_sme_builtin_cg.inc"
985#undef GET_SME_LLVM_INTRINSIC_MAP
986};
987
988#undef SMEMAP1
989#undef SMEMAP2
990
992
997
998// Check if Builtin `BuiltinId` is present in `IntrinsicMap`. If yes, returns
999// the corresponding info struct.
1000template <typename IntrinsicInfo>
1001static const IntrinsicInfo *
1003 unsigned BuiltinID, bool &MapProvenSorted) {
1004
1005#ifndef NDEBUG
1006 if (!MapProvenSorted) {
1007 assert(llvm::is_sorted(IntrinsicMap));
1008 MapProvenSorted = true;
1009 }
1010#endif
1011
1012 const IntrinsicInfo *Builtin = llvm::lower_bound(IntrinsicMap, BuiltinID);
1013
1014 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
1015 return Builtin;
1016
1017 return nullptr;
1018}
1019
1021 unsigned Modifier,
1022 llvm::Type *ArgType,
1023 const CallExpr *E) {
1024 int VectorSize = 0;
1025 if (Modifier & Use64BitVectors)
1026 VectorSize = 64;
1027 else if (Modifier & Use128BitVectors)
1028 VectorSize = 128;
1029
1030 // Return type.
1032 if (Modifier & AddRetType) {
1033 llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
1034 if (Modifier & VectorizeRetType)
1035 Ty = llvm::FixedVectorType::get(
1036 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
1037
1038 Tys.push_back(Ty);
1039 }
1040
1041 // Arguments.
1042 if (Modifier & VectorizeArgTypes) {
1043 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
1044 ArgType = llvm::FixedVectorType::get(ArgType, Elts);
1045 }
1046
1047 if (Modifier & (Add1ArgType | Add2ArgTypes))
1048 Tys.push_back(ArgType);
1049
1050 if (Modifier & Add2ArgTypes)
1051 Tys.push_back(ArgType);
1052
1053 if (Modifier & InventFloatType)
1054 Tys.push_back(FloatTy);
1055
1056 return CGM.getIntrinsic(IntrinsicID, Tys);
1057}
1058
1059//===----------------------------------------------------------------------===//
1060// Emit-helpers
1061//===----------------------------------------------------------------------===//
1063 CodeGenFunction &CGF, const ARMNeonVectorIntrinsicInfo &SISDInfo,
1064 SmallVectorImpl<Value *> &Ops, const CallExpr *E) {
1065 assert(SISDInfo.LLVMIntrinsic && "Generic code assumes a valid intrinsic");
1066
1067 switch (SISDInfo.BuiltinID) {
1068 case NEON::BI__builtin_neon_vcled_s64:
1069 case NEON::BI__builtin_neon_vcled_u64:
1070 case NEON::BI__builtin_neon_vcles_f32:
1071 case NEON::BI__builtin_neon_vcled_f64:
1072 case NEON::BI__builtin_neon_vcltd_s64:
1073 case NEON::BI__builtin_neon_vcltd_u64:
1074 case NEON::BI__builtin_neon_vclts_f32:
1075 case NEON::BI__builtin_neon_vcltd_f64:
1076 case NEON::BI__builtin_neon_vcales_f32:
1077 case NEON::BI__builtin_neon_vcaled_f64:
1078 case NEON::BI__builtin_neon_vcalts_f32:
1079 case NEON::BI__builtin_neon_vcaltd_f64:
1080 // Only one direction of comparisons actually exist, cmle is actually a cmge
1081 // with swapped operands. The table gives us the right intrinsic but we
1082 // still need to do the swap.
1083 std::swap(Ops[0], Ops[1]);
1084 break;
1085 }
1086
1087 // Use fptosi.sat/fptoui.sat unless under strict FP.
1088 unsigned LLVMIntrinsic = SISDInfo.LLVMIntrinsic;
1089 if (!CGF.Builder.getIsFPConstrained()) {
1090 if (LLVMIntrinsic == Intrinsic::aarch64_neon_fcvtzs)
1091 LLVMIntrinsic = Intrinsic::fptosi_sat;
1092 else if (LLVMIntrinsic == Intrinsic::aarch64_neon_fcvtzu)
1093 LLVMIntrinsic = Intrinsic::fptoui_sat;
1094 }
1095 llvm::Type *ArgTy = CGF.ConvertType(E->getArg(0)->getType());
1096 Function *F = CGF.LookupNeonLLVMIntrinsic(LLVMIntrinsic,
1097 SISDInfo.TypeModifier, ArgTy, E);
1098
1099 int j = 0;
1100 ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0);
1101 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
1102 ai != ae; ++ai, ++j) {
1103 llvm::Type *ArgTy = ai->getType();
1104 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
1105 ArgTy->getPrimitiveSizeInBits())
1106 continue;
1107 assert(
1108 ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy() &&
1109 "Expecting vector LLVM intrinsic type and scalar Clang builtin type!");
1110
1111 // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate
1112 // it before inserting.
1113 Ops[j] = CGF.Builder.CreateTruncOrBitCast(
1114 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType());
1115 Ops[j] =
1116 CGF.Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0);
1117 }
1118
1119 Value *Result = CGF.EmitNeonCall(F, Ops, SISDInfo.NameHint);
1120 llvm::Type *ResultType = CGF.ConvertType(E->getType());
1121 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
1122 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
1123 return CGF.Builder.CreateExtractElement(Result, C0);
1124
1125 return CGF.Builder.CreateBitCast(Result, ResultType, SISDInfo.NameHint);
1126}
1127
1129 unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
1130 const char *NameHint, unsigned Modifier, const CallExpr *E,
1131 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1,
1132 llvm::Triple::ArchType Arch) {
1133
1134 // Extract the trailing immediate argument that encodes the type discriminator
1135 // for this overloaded intrinsic.
1136 // TODO: Move to the parent code that takes care of argument processing.
1137 const Expr *Arg = E->getArg(E->getNumArgs() - 1);
1138 std::optional<llvm::APSInt> NeonTypeConst =
1140 if (!NeonTypeConst)
1141 return nullptr;
1142
1143 // Determine the type of this overloaded NEON intrinsic.
1144 NeonTypeFlags Type(NeonTypeConst->getZExtValue());
1145 const bool Usgn = Type.isUnsigned();
1146 const bool Quad = Type.isQuad();
1147 const bool Floating = Type.isFloatingPoint();
1148 const bool HasFastHalfType = getTarget().hasFastHalfType();
1149 const bool AllowBFloatArgsAndRet =
1150 getTargetHooks().getABIInfo().allowBFloatArgsAndRet();
1151
1152 llvm::FixedVectorType *VTy =
1153 GetNeonType(this, Type, HasFastHalfType, false, AllowBFloatArgsAndRet);
1154 llvm::Type *Ty = VTy;
1155 if (!Ty)
1156 return nullptr;
1157
1158 auto getAlignmentValue32 = [&](Address addr) -> Value* {
1159 return Builder.getInt32(addr.getAlignment().getQuantity());
1160 };
1161
1162 unsigned Int = LLVMIntrinsic;
1163 if ((Modifier & UnsignedAlts) && !Usgn)
1164 Int = AltLLVMIntrinsic;
1165
1166 switch (BuiltinID) {
1167 default: break;
1168 case NEON::BI__builtin_neon_splat_lane_v:
1169 case NEON::BI__builtin_neon_splat_laneq_v:
1170 case NEON::BI__builtin_neon_splatq_lane_v:
1171 case NEON::BI__builtin_neon_splatq_laneq_v: {
1172 auto NumElements = VTy->getElementCount();
1173 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
1174 NumElements = NumElements * 2;
1175 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
1176 NumElements = NumElements.divideCoefficientBy(2);
1177
1178 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
1179 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements);
1180 }
1181 case NEON::BI__builtin_neon_vpadd_v:
1182 case NEON::BI__builtin_neon_vpaddq_v:
1183 // We don't allow fp/int overloading of intrinsics.
1184 if (VTy->getElementType()->isFloatingPointTy() &&
1185 Int == Intrinsic::aarch64_neon_addp)
1186 Int = Intrinsic::aarch64_neon_faddp;
1187 break;
1188 case NEON::BI__builtin_neon_vabs_v:
1189 case NEON::BI__builtin_neon_vabsq_v:
1190 if (VTy->getElementType()->isFloatingPointTy())
1191 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs");
1192 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs");
1193 case NEON::BI__builtin_neon_vadd_v:
1194 case NEON::BI__builtin_neon_vaddq_v: {
1195 llvm::Type *VTy = llvm::FixedVectorType::get(Int8Ty, Quad ? 16 : 8);
1196 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
1197 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
1198 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]);
1199 return Builder.CreateBitCast(Ops[0], Ty);
1200 }
1201 case NEON::BI__builtin_neon_vaddhn_v: {
1202 llvm::FixedVectorType *SrcTy =
1203 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1204
1205 // %sum = add <4 x i32> %lhs, %rhs
1206 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
1207 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
1208 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn");
1209
1210 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
1211 Constant *ShiftAmt =
1212 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
1213 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn");
1214
1215 // %res = trunc <4 x i32> %high to <4 x i16>
1216 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn");
1217 }
1218 case NEON::BI__builtin_neon_vcale_v:
1219 case NEON::BI__builtin_neon_vcaleq_v:
1220 case NEON::BI__builtin_neon_vcalt_v:
1221 case NEON::BI__builtin_neon_vcaltq_v:
1222 std::swap(Ops[0], Ops[1]);
1223 [[fallthrough]];
1224 case NEON::BI__builtin_neon_vcage_v:
1225 case NEON::BI__builtin_neon_vcageq_v:
1226 case NEON::BI__builtin_neon_vcagt_v:
1227 case NEON::BI__builtin_neon_vcagtq_v: {
1228 llvm::Type *Ty;
1229 switch (VTy->getScalarSizeInBits()) {
1230 default: llvm_unreachable("unexpected type");
1231 case 32:
1232 Ty = FloatTy;
1233 break;
1234 case 64:
1235 Ty = DoubleTy;
1236 break;
1237 case 16:
1238 Ty = HalfTy;
1239 break;
1240 }
1241 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
1242 llvm::Type *Tys[] = { VTy, VecFlt };
1243 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
1244 return EmitNeonCall(F, Ops, NameHint);
1245 }
1246 case NEON::BI__builtin_neon_vceqz_v:
1247 case NEON::BI__builtin_neon_vceqzq_v:
1249 Ops[0], Ty, Floating ? ICmpInst::FCMP_OEQ : ICmpInst::ICMP_EQ, "vceqz");
1250 case NEON::BI__builtin_neon_vcgez_v:
1251 case NEON::BI__builtin_neon_vcgezq_v:
1253 Ops[0], Ty, Floating ? ICmpInst::FCMP_OGE : ICmpInst::ICMP_SGE,
1254 "vcgez");
1255 case NEON::BI__builtin_neon_vclez_v:
1256 case NEON::BI__builtin_neon_vclezq_v:
1258 Ops[0], Ty, Floating ? ICmpInst::FCMP_OLE : ICmpInst::ICMP_SLE,
1259 "vclez");
1260 case NEON::BI__builtin_neon_vcgtz_v:
1261 case NEON::BI__builtin_neon_vcgtzq_v:
1263 Ops[0], Ty, Floating ? ICmpInst::FCMP_OGT : ICmpInst::ICMP_SGT,
1264 "vcgtz");
1265 case NEON::BI__builtin_neon_vcltz_v:
1266 case NEON::BI__builtin_neon_vcltzq_v:
1268 Ops[0], Ty, Floating ? ICmpInst::FCMP_OLT : ICmpInst::ICMP_SLT,
1269 "vcltz");
1270 case NEON::BI__builtin_neon_vclz_v:
1271 case NEON::BI__builtin_neon_vclzq_v:
1272 // We generate target-independent intrinsic, which needs a second argument
1273 // for whether or not clz of zero is undefined; on ARM it isn't.
1274 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
1275 break;
1276 case NEON::BI__builtin_neon_vcvt_f32_v:
1277 case NEON::BI__builtin_neon_vcvtq_f32_v:
1278 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1279 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad),
1280 HasFastHalfType);
1281 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
1282 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
1283 case NEON::BI__builtin_neon_vcvt_f16_s16:
1284 case NEON::BI__builtin_neon_vcvt_f16_u16:
1285 case NEON::BI__builtin_neon_vcvtq_f16_s16:
1286 case NEON::BI__builtin_neon_vcvtq_f16_u16:
1287 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1288 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float16, false, Quad),
1289 HasFastHalfType);
1290 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
1291 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
1292 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
1293 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
1294 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
1295 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
1296 llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty };
1297 Function *F = CGM.getIntrinsic(Int, Tys);
1298 return EmitNeonCall(F, Ops, "vcvt_n");
1299 }
1300 case NEON::BI__builtin_neon_vcvt_n_f32_v:
1301 case NEON::BI__builtin_neon_vcvt_n_f64_v:
1302 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
1303 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
1304 llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty };
1305 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
1306 Function *F = CGM.getIntrinsic(Int, Tys);
1307 return EmitNeonCall(F, Ops, "vcvt_n");
1308 }
1309 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
1310 case NEON::BI__builtin_neon_vcvt_n_s32_v:
1311 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
1312 case NEON::BI__builtin_neon_vcvt_n_u32_v:
1313 case NEON::BI__builtin_neon_vcvt_n_s64_v:
1314 case NEON::BI__builtin_neon_vcvt_n_u64_v:
1315 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
1316 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
1317 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
1318 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
1319 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
1320 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
1321 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
1322 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
1323 return EmitNeonCall(F, Ops, "vcvt_n");
1324 }
1325 case NEON::BI__builtin_neon_vcvt_s32_v:
1326 case NEON::BI__builtin_neon_vcvt_u32_v:
1327 case NEON::BI__builtin_neon_vcvt_s64_v:
1328 case NEON::BI__builtin_neon_vcvt_u64_v:
1329 case NEON::BI__builtin_neon_vcvt_s16_f16:
1330 case NEON::BI__builtin_neon_vcvt_u16_f16:
1331 case NEON::BI__builtin_neon_vcvtq_s32_v:
1332 case NEON::BI__builtin_neon_vcvtq_u32_v:
1333 case NEON::BI__builtin_neon_vcvtq_s64_v:
1334 case NEON::BI__builtin_neon_vcvtq_u64_v:
1335 case NEON::BI__builtin_neon_vcvtq_s16_f16:
1336 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
1337 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
1338 if (Int) {
1339 // AArch64: use fptosi.sat/fptoui.sat unless under strict FP.
1340 if (!Builder.getIsFPConstrained())
1341 Int = Usgn ? Intrinsic::fptoui_sat : Intrinsic::fptosi_sat;
1342 llvm::Type *Tys[2] = {Ty, Ops[0]->getType()};
1343 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtz");
1344 }
1345 // FIXME: ARM uses plain fptoui/fptosi which have UB on out-of-range
1346 // values. These should also use saturating intrinsics.
1347 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
1348 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
1349 }
1350 case NEON::BI__builtin_neon_vcvta_s16_f16:
1351 case NEON::BI__builtin_neon_vcvta_s32_v:
1352 case NEON::BI__builtin_neon_vcvta_s64_v:
1353 case NEON::BI__builtin_neon_vcvta_u16_f16:
1354 case NEON::BI__builtin_neon_vcvta_u32_v:
1355 case NEON::BI__builtin_neon_vcvta_u64_v:
1356 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
1357 case NEON::BI__builtin_neon_vcvtaq_s32_v:
1358 case NEON::BI__builtin_neon_vcvtaq_s64_v:
1359 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
1360 case NEON::BI__builtin_neon_vcvtaq_u32_v:
1361 case NEON::BI__builtin_neon_vcvtaq_u64_v:
1362 case NEON::BI__builtin_neon_vcvtn_s16_f16:
1363 case NEON::BI__builtin_neon_vcvtn_s32_v:
1364 case NEON::BI__builtin_neon_vcvtn_s64_v:
1365 case NEON::BI__builtin_neon_vcvtn_u16_f16:
1366 case NEON::BI__builtin_neon_vcvtn_u32_v:
1367 case NEON::BI__builtin_neon_vcvtn_u64_v:
1368 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
1369 case NEON::BI__builtin_neon_vcvtnq_s32_v:
1370 case NEON::BI__builtin_neon_vcvtnq_s64_v:
1371 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
1372 case NEON::BI__builtin_neon_vcvtnq_u32_v:
1373 case NEON::BI__builtin_neon_vcvtnq_u64_v:
1374 case NEON::BI__builtin_neon_vcvtp_s16_f16:
1375 case NEON::BI__builtin_neon_vcvtp_s32_v:
1376 case NEON::BI__builtin_neon_vcvtp_s64_v:
1377 case NEON::BI__builtin_neon_vcvtp_u16_f16:
1378 case NEON::BI__builtin_neon_vcvtp_u32_v:
1379 case NEON::BI__builtin_neon_vcvtp_u64_v:
1380 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
1381 case NEON::BI__builtin_neon_vcvtpq_s32_v:
1382 case NEON::BI__builtin_neon_vcvtpq_s64_v:
1383 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
1384 case NEON::BI__builtin_neon_vcvtpq_u32_v:
1385 case NEON::BI__builtin_neon_vcvtpq_u64_v:
1386 case NEON::BI__builtin_neon_vcvtm_s16_f16:
1387 case NEON::BI__builtin_neon_vcvtm_s32_v:
1388 case NEON::BI__builtin_neon_vcvtm_s64_v:
1389 case NEON::BI__builtin_neon_vcvtm_u16_f16:
1390 case NEON::BI__builtin_neon_vcvtm_u32_v:
1391 case NEON::BI__builtin_neon_vcvtm_u64_v:
1392 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
1393 case NEON::BI__builtin_neon_vcvtmq_s32_v:
1394 case NEON::BI__builtin_neon_vcvtmq_s64_v:
1395 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
1396 case NEON::BI__builtin_neon_vcvtmq_u32_v:
1397 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
1398 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
1399 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
1400 }
1401 case NEON::BI__builtin_neon_vcvtx_f32_v: {
1402 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
1403 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
1404
1405 }
1406 case NEON::BI__builtin_neon_vext_v:
1407 case NEON::BI__builtin_neon_vextq_v: {
1408 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
1409 SmallVector<int, 16> Indices;
1410 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
1411 Indices.push_back(i+CV);
1412
1413 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1414 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
1415 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext");
1416 }
1417 case NEON::BI__builtin_neon_vfma_v:
1418 case NEON::BI__builtin_neon_vfmaq_v: {
1419 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1420 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
1421 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
1422
1423 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
1425 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
1426 {Ops[1], Ops[2], Ops[0]});
1427 }
1428 case NEON::BI__builtin_neon_vld1_x2_v:
1429 case NEON::BI__builtin_neon_vld1q_x2_v:
1430 case NEON::BI__builtin_neon_vld1_x3_v:
1431 case NEON::BI__builtin_neon_vld1q_x3_v:
1432 case NEON::BI__builtin_neon_vld1_x4_v:
1433 case NEON::BI__builtin_neon_vld1q_x4_v: {
1434 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
1435 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
1436 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN");
1437 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
1438 }
1439 case NEON::BI__builtin_neon_vld1_v:
1440 case NEON::BI__builtin_neon_vld1q_v: {
1441 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1442 Ops.push_back(getAlignmentValue32(PtrOp0));
1443 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1");
1444 }
1445 case NEON::BI__builtin_neon_vld2_v:
1446 case NEON::BI__builtin_neon_vld2q_v:
1447 case NEON::BI__builtin_neon_vld3_v:
1448 case NEON::BI__builtin_neon_vld3q_v:
1449 case NEON::BI__builtin_neon_vld4_v:
1450 case NEON::BI__builtin_neon_vld4q_v:
1451 case NEON::BI__builtin_neon_vld2_dup_v:
1452 case NEON::BI__builtin_neon_vld2q_dup_v:
1453 case NEON::BI__builtin_neon_vld3_dup_v:
1454 case NEON::BI__builtin_neon_vld3q_dup_v:
1455 case NEON::BI__builtin_neon_vld4_dup_v:
1456 case NEON::BI__builtin_neon_vld4q_dup_v: {
1457 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1458 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
1459 Value *Align = getAlignmentValue32(PtrOp1);
1460 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
1461 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
1462 }
1463 case NEON::BI__builtin_neon_vld1_dup_v:
1464 case NEON::BI__builtin_neon_vld1q_dup_v: {
1465 Value *V = PoisonValue::get(Ty);
1466 PtrOp0 = PtrOp0.withElementType(VTy->getElementType());
1467 LoadInst *Ld = Builder.CreateLoad(PtrOp0);
1468 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
1469 Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
1470 return EmitNeonSplat(Ops[0], CI);
1471 }
1472 case NEON::BI__builtin_neon_vld2_lane_v:
1473 case NEON::BI__builtin_neon_vld2q_lane_v:
1474 case NEON::BI__builtin_neon_vld3_lane_v:
1475 case NEON::BI__builtin_neon_vld3q_lane_v:
1476 case NEON::BI__builtin_neon_vld4_lane_v:
1477 case NEON::BI__builtin_neon_vld4q_lane_v: {
1478 llvm::Type *Tys[] = {Ty, Int8PtrTy};
1479 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
1480 for (unsigned I = 2; I < Ops.size() - 1; ++I)
1481 Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
1482 Ops.push_back(getAlignmentValue32(PtrOp1));
1483 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), NameHint);
1484 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
1485 }
1486 case NEON::BI__builtin_neon_vmovl_v: {
1487 llvm::FixedVectorType *DTy =
1488 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
1489 Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
1490 if (Usgn)
1491 return Builder.CreateZExt(Ops[0], Ty, "vmovl");
1492 return Builder.CreateSExt(Ops[0], Ty, "vmovl");
1493 }
1494 case NEON::BI__builtin_neon_vmovn_v: {
1495 llvm::FixedVectorType *QTy =
1496 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1497 Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
1498 return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
1499 }
1500 case NEON::BI__builtin_neon_vmull_v:
1501 // FIXME: the integer vmull operations could be emitted in terms of pure
1502 // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of
1503 // hoisting the exts outside loops. Until global ISel comes along that can
1504 // see through such movement this leads to bad CodeGen. So we need an
1505 // intrinsic for now.
1506 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
1507 Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
1508 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
1509 case NEON::BI__builtin_neon_vpadal_v:
1510 case NEON::BI__builtin_neon_vpadalq_v: {
1511 // The source operand type has twice as many elements of half the size.
1512 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
1513 llvm::Type *EltTy =
1514 llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
1515 auto *NarrowTy =
1516 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
1517 llvm::Type *Tys[2] = { Ty, NarrowTy };
1518 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
1519 }
1520 case NEON::BI__builtin_neon_vpaddl_v:
1521 case NEON::BI__builtin_neon_vpaddlq_v: {
1522 // The source operand type has twice as many elements of half the size.
1523 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
1524 llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
1525 auto *NarrowTy =
1526 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
1527 llvm::Type *Tys[2] = { Ty, NarrowTy };
1528 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl");
1529 }
1530 case NEON::BI__builtin_neon_vqdmlal_v:
1531 case NEON::BI__builtin_neon_vqdmlsl_v: {
1532 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
1533 Ops[1] =
1534 EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps, "vqdmlal");
1535 Ops.resize(2);
1536 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint);
1537 }
1538 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
1539 case NEON::BI__builtin_neon_vqdmulh_lane_v:
1540 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
1541 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
1542 auto *RTy = cast<llvm::FixedVectorType>(Ty);
1543 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
1544 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
1545 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
1546 RTy->getNumElements() * 2);
1547 llvm::Type *Tys[2] = {
1548 RTy, GetNeonType(this, NeonTypeFlags(Type.getEltType(), false,
1549 /*isQuad*/ false))};
1550 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
1551 }
1552 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
1553 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
1554 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
1555 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
1556 llvm::Type *Tys[2] = {
1557 Ty, GetNeonType(this, NeonTypeFlags(Type.getEltType(), false,
1558 /*isQuad*/ true))};
1559 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
1560 }
1561 case NEON::BI__builtin_neon_vqshl_n_v:
1562 case NEON::BI__builtin_neon_vqshlq_n_v:
1563 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n",
1564 1, false);
1565 case NEON::BI__builtin_neon_vqshlu_n_v:
1566 case NEON::BI__builtin_neon_vqshluq_n_v:
1567 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n",
1568 1, false);
1569 case NEON::BI__builtin_neon_vrecpe_v:
1570 case NEON::BI__builtin_neon_vrecpeq_v:
1571 case NEON::BI__builtin_neon_vrsqrte_v:
1572 case NEON::BI__builtin_neon_vrsqrteq_v:
1573 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
1574 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
1575 case NEON::BI__builtin_neon_vrndi_v:
1576 case NEON::BI__builtin_neon_vrndiq_v:
1577 Int = Builder.getIsFPConstrained()
1578 ? Intrinsic::experimental_constrained_nearbyint
1579 : Intrinsic::nearbyint;
1580 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
1581 case NEON::BI__builtin_neon_vrshr_n_v:
1582 case NEON::BI__builtin_neon_vrshrq_n_v:
1583 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n",
1584 1, true);
1585 case NEON::BI__builtin_neon_vsha512hq_u64:
1586 case NEON::BI__builtin_neon_vsha512h2q_u64:
1587 case NEON::BI__builtin_neon_vsha512su0q_u64:
1588 case NEON::BI__builtin_neon_vsha512su1q_u64: {
1589 Function *F = CGM.getIntrinsic(Int);
1590 return EmitNeonCall(F, Ops, "");
1591 }
1592 case NEON::BI__builtin_neon_vshl_n_v:
1593 case NEON::BI__builtin_neon_vshlq_n_v:
1594 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false);
1595 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
1596 "vshl_n");
1597 case NEON::BI__builtin_neon_vshll_n_v: {
1598 llvm::FixedVectorType *SrcTy =
1599 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
1600 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
1601 if (Usgn)
1602 Ops[0] = Builder.CreateZExt(Ops[0], VTy);
1603 else
1604 Ops[0] = Builder.CreateSExt(Ops[0], VTy);
1605 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false);
1606 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n");
1607 }
1608 case NEON::BI__builtin_neon_vshrn_n_v: {
1609 llvm::FixedVectorType *SrcTy =
1610 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1611 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
1612 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false);
1613 if (Usgn)
1614 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
1615 else
1616 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
1617 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n");
1618 }
1619 case NEON::BI__builtin_neon_vshr_n_v:
1620 case NEON::BI__builtin_neon_vshrq_n_v:
1621 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n");
1622 case NEON::BI__builtin_neon_vst1_v:
1623 case NEON::BI__builtin_neon_vst1q_v:
1624 case NEON::BI__builtin_neon_vst2_v:
1625 case NEON::BI__builtin_neon_vst2q_v:
1626 case NEON::BI__builtin_neon_vst3_v:
1627 case NEON::BI__builtin_neon_vst3q_v:
1628 case NEON::BI__builtin_neon_vst4_v:
1629 case NEON::BI__builtin_neon_vst4q_v:
1630 case NEON::BI__builtin_neon_vst2_lane_v:
1631 case NEON::BI__builtin_neon_vst2q_lane_v:
1632 case NEON::BI__builtin_neon_vst3_lane_v:
1633 case NEON::BI__builtin_neon_vst3q_lane_v:
1634 case NEON::BI__builtin_neon_vst4_lane_v:
1635 case NEON::BI__builtin_neon_vst4q_lane_v: {
1636 llvm::Type *Tys[] = {Int8PtrTy, Ty};
1637 Ops.push_back(getAlignmentValue32(PtrOp0));
1638 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "");
1639 }
1640 case NEON::BI__builtin_neon_vsm3partw1q_u32:
1641 case NEON::BI__builtin_neon_vsm3partw2q_u32:
1642 case NEON::BI__builtin_neon_vsm3ss1q_u32:
1643 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
1644 case NEON::BI__builtin_neon_vsm4eq_u32: {
1645 Function *F = CGM.getIntrinsic(Int);
1646 return EmitNeonCall(F, Ops, "");
1647 }
1648 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
1649 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
1650 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
1651 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
1652 Function *F = CGM.getIntrinsic(Int);
1653 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
1654 return EmitNeonCall(F, Ops, "");
1655 }
1656 case NEON::BI__builtin_neon_vst1_x2_v:
1657 case NEON::BI__builtin_neon_vst1q_x2_v:
1658 case NEON::BI__builtin_neon_vst1_x3_v:
1659 case NEON::BI__builtin_neon_vst1q_x3_v:
1660 case NEON::BI__builtin_neon_vst1_x4_v:
1661 case NEON::BI__builtin_neon_vst1q_x4_v: {
1662 // TODO: Currently in AArch32 mode the pointer operand comes first, whereas
1663 // in AArch64 it comes last. We may want to stick to one or another.
1664 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
1665 Arch == llvm::Triple::aarch64_32) {
1666 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
1667 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
1668 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "");
1669 }
1670 llvm::Type *Tys[2] = {DefaultPtrTy, VTy};
1671 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "");
1672 }
1673 case NEON::BI__builtin_neon_vsubhn_v: {
1674 llvm::FixedVectorType *SrcTy =
1675 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1676
1677 // %sum = add <4 x i32> %lhs, %rhs
1678 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
1679 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
1680 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn");
1681
1682 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
1683 Constant *ShiftAmt =
1684 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
1685 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn");
1686
1687 // %res = trunc <4 x i32> %high to <4 x i16>
1688 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn");
1689 }
1690 case NEON::BI__builtin_neon_vtrn_v:
1691 case NEON::BI__builtin_neon_vtrnq_v: {
1692 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
1693 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
1694 Value *SV = nullptr;
1695
1696 for (unsigned vi = 0; vi != 2; ++vi) {
1697 SmallVector<int, 16> Indices;
1698 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
1699 Indices.push_back(i+vi);
1700 Indices.push_back(i+e+vi);
1701 }
1702 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
1703 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
1704 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
1705 }
1706 return SV;
1707 }
1708 case NEON::BI__builtin_neon_vtst_v:
1709 case NEON::BI__builtin_neon_vtstq_v: {
1710 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1711 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
1712 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
1713 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
1714 ConstantAggregateZero::get(Ty));
1715 return Builder.CreateSExt(Ops[0], Ty, "vtst");
1716 }
1717 case NEON::BI__builtin_neon_vuzp_v:
1718 case NEON::BI__builtin_neon_vuzpq_v: {
1719 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
1720 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
1721 Value *SV = nullptr;
1722
1723 for (unsigned vi = 0; vi != 2; ++vi) {
1724 SmallVector<int, 16> Indices;
1725 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
1726 Indices.push_back(2*i+vi);
1727
1728 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
1729 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
1730 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
1731 }
1732 return SV;
1733 }
1734 case NEON::BI__builtin_neon_vxarq_u64: {
1735 Function *F = CGM.getIntrinsic(Int);
1736 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
1737 return EmitNeonCall(F, Ops, "");
1738 }
1739 case NEON::BI__builtin_neon_vzip_v:
1740 case NEON::BI__builtin_neon_vzipq_v: {
1741 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
1742 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
1743 Value *SV = nullptr;
1744
1745 for (unsigned vi = 0; vi != 2; ++vi) {
1746 SmallVector<int, 16> Indices;
1747 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
1748 Indices.push_back((i + vi*e) >> 1);
1749 Indices.push_back(((i + vi*e) >> 1)+e);
1750 }
1751 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
1752 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
1753 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
1754 }
1755 return SV;
1756 }
1757 case NEON::BI__builtin_neon_vdot_s32:
1758 case NEON::BI__builtin_neon_vdot_u32:
1759 case NEON::BI__builtin_neon_vdotq_s32:
1760 case NEON::BI__builtin_neon_vdotq_u32: {
1761 auto *InputTy =
1762 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
1763 llvm::Type *Tys[2] = { Ty, InputTy };
1764 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vdot");
1765 }
1766 case NEON::BI__builtin_neon_vfmlal_low_f16:
1767 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
1768 auto *InputTy =
1769 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1770 llvm::Type *Tys[2] = { Ty, InputTy };
1771 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_low");
1772 }
1773 case NEON::BI__builtin_neon_vfmlsl_low_f16:
1774 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
1775 auto *InputTy =
1776 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1777 llvm::Type *Tys[2] = { Ty, InputTy };
1778 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_low");
1779 }
1780 case NEON::BI__builtin_neon_vfmlal_high_f16:
1781 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
1782 auto *InputTy =
1783 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1784 llvm::Type *Tys[2] = { Ty, InputTy };
1785 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_high");
1786 }
1787 case NEON::BI__builtin_neon_vfmlsl_high_f16:
1788 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
1789 auto *InputTy =
1790 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1791 llvm::Type *Tys[2] = { Ty, InputTy };
1792 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_high");
1793 }
1794 case NEON::BI__builtin_neon_vmmlaq_s32:
1795 case NEON::BI__builtin_neon_vmmlaq_u32: {
1796 auto *InputTy =
1797 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
1798 llvm::Type *Tys[2] = { Ty, InputTy };
1799 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vmmla");
1800 }
1801 case NEON::BI__builtin_neon_vmmlaq_f16:
1802 case NEON::BI__builtin_neon_vmmlaq_f32_f16: {
1803 auto *InputTy =
1804 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1805 llvm::Type *Tys[2] = {Ty, InputTy};
1806 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fmmla");
1807 }
1808 case NEON::BI__builtin_neon_vusmmlaq_s32: {
1809 auto *InputTy =
1810 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
1811 llvm::Type *Tys[2] = { Ty, InputTy };
1812 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusmmla");
1813 }
1814 case NEON::BI__builtin_neon_vusdot_s32:
1815 case NEON::BI__builtin_neon_vusdotq_s32: {
1816 auto *InputTy =
1817 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
1818 llvm::Type *Tys[2] = { Ty, InputTy };
1819 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusdot");
1820 }
1821 case NEON::BI__builtin_neon_vbfdot_f32:
1822 case NEON::BI__builtin_neon_vbfdotq_f32: {
1823 llvm::Type *InputTy =
1824 llvm::FixedVectorType::get(BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
1825 llvm::Type *Tys[2] = { Ty, InputTy };
1826 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vbfdot");
1827 }
1828 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
1829 llvm::Type *Tys[1] = { Ty };
1830 Function *F = CGM.getIntrinsic(Int, Tys);
1831 return EmitNeonCall(F, Ops, "vcvtfp2bf");
1832 }
1833
1834 }
1835
1836 assert(Int && "Expected valid intrinsic number");
1837
1838 // Determine the type(s) of this overloaded AArch64 intrinsic.
1839 Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E);
1840
1841 Value *Result = EmitNeonCall(F, Ops, NameHint);
1842 llvm::Type *ResultType = ConvertType(E->getType());
1843 // AArch64 intrinsic one-element vector type cast to
1844 // scalar type expected by the builtin
1845 return Builder.CreateBitCast(Result, ResultType, NameHint);
1846}
1847
1848Value *
1850 const CmpInst::Predicate Pred,
1851 const Twine &Name) {
1852
1853 if (isa<FixedVectorType>(Ty)) {
1854 // Vector types are cast to i8 vectors. Recover original type.
1855 Op = Builder.CreateBitCast(Op, Ty);
1856 }
1857
1858 Constant *zero = Constant::getNullValue(Op->getType());
1859
1860 if (CmpInst::isFPPredicate(Pred)) {
1861 if (Pred == CmpInst::FCMP_OEQ)
1862 Op = Builder.CreateFCmp(Pred, Op, zero);
1863 else
1864 Op = Builder.CreateFCmpS(Pred, Op, zero);
1865 } else {
1866 Op = Builder.CreateICmp(Pred, Op, zero);
1867 }
1868
1869 llvm::Type *ResTy = Ty;
1870 if (auto *VTy = dyn_cast<FixedVectorType>(Ty))
1871 ResTy = FixedVectorType::get(
1872 IntegerType::get(getLLVMContext(), VTy->getScalarSizeInBits()),
1873 VTy->getNumElements());
1874
1875 return Builder.CreateSExt(Op, ResTy, Name);
1876}
1877
1879 Value *ExtOp, Value *IndexOp,
1880 llvm::Type *ResTy, unsigned IntID,
1881 const char *Name) {
1883 if (ExtOp)
1884 TblOps.push_back(ExtOp);
1885
1886 // Build a vector containing sequential number like (0, 1, 2, ..., 15)
1887 SmallVector<int, 16> Indices;
1888 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
1889 for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
1890 Indices.push_back(2*i);
1891 Indices.push_back(2*i+1);
1892 }
1893
1894 int PairPos = 0, End = Ops.size() - 1;
1895 while (PairPos < End) {
1896 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
1897 Ops[PairPos+1], Indices,
1898 Name));
1899 PairPos += 2;
1900 }
1901
1902 // If there's an odd number of 64-bit lookup table, fill the high 64-bit
1903 // of the 128-bit lookup table with zero.
1904 if (PairPos == End) {
1905 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
1906 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
1907 ZeroTbl, Indices, Name));
1908 }
1909
1910 Function *TblF;
1911 TblOps.push_back(IndexOp);
1912 TblF = CGF.CGM.getIntrinsic(IntID, ResTy);
1913
1914 return CGF.EmitNeonCall(TblF, TblOps, Name);
1915}
1916
1917Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) {
1918 unsigned Value;
1919 switch (BuiltinID) {
1920 default:
1921 return nullptr;
1922 case clang::ARM::BI__builtin_arm_nop:
1923 Value = 0;
1924 break;
1925 case clang::ARM::BI__builtin_arm_yield:
1926 case clang::ARM::BI__yield:
1927 Value = 1;
1928 break;
1929 case clang::ARM::BI__builtin_arm_wfe:
1930 case clang::ARM::BI__wfe:
1931 Value = 2;
1932 break;
1933 case clang::ARM::BI__builtin_arm_wfi:
1934 case clang::ARM::BI__wfi:
1935 Value = 3;
1936 break;
1937 case clang::ARM::BI__builtin_arm_sev:
1938 case clang::ARM::BI__sev:
1939 Value = 4;
1940 break;
1941 case clang::ARM::BI__builtin_arm_sevl:
1942 case clang::ARM::BI__sevl:
1943 Value = 5;
1944 break;
1945 }
1946
1947 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
1948 llvm::ConstantInt::get(Int32Ty, Value));
1949}
1950
1956
1957// Generates the IR for the read/write special register builtin,
1958// ValueType is the type of the value that is to be written or read,
1959// RegisterType is the type of the register being written to or read from.
1961 const CallExpr *E,
1962 llvm::Type *RegisterType,
1963 llvm::Type *ValueType,
1964 SpecialRegisterAccessKind AccessKind,
1965 StringRef SysReg = "") {
1966 // write and register intrinsics only support 32, 64 and 128 bit operations.
1967 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64) ||
1968 RegisterType->isIntegerTy(128)) &&
1969 "Unsupported size for register.");
1970
1971 CodeGen::CGBuilderTy &Builder = CGF.Builder;
1972 CodeGen::CodeGenModule &CGM = CGF.CGM;
1973 LLVMContext &Context = CGM.getLLVMContext();
1974
1975 if (SysReg.empty()) {
1976 const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts();
1977 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
1978 }
1979
1980 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
1981 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
1982 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
1983
1984 llvm::Type *Types[] = { RegisterType };
1985
1986 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
1987 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
1988 && "Can't fit 64-bit value in 32-bit register");
1989
1990 if (AccessKind != Write) {
1991 assert(AccessKind == NormalRead || AccessKind == VolatileRead);
1992 llvm::Function *F = CGM.getIntrinsic(
1993 AccessKind == VolatileRead ? Intrinsic::read_volatile_register
1994 : Intrinsic::read_register,
1995 Types);
1996 llvm::Value *Call = Builder.CreateCall(F, Metadata);
1997
1998 if (MixedTypes)
1999 // Read into 64 bit register and then truncate result to 32 bit.
2000 return Builder.CreateTrunc(Call, ValueType);
2001
2002 if (ValueType->isPointerTy())
2003 // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*).
2004 return Builder.CreateIntToPtr(Call, ValueType);
2005
2006 return Call;
2007 }
2008
2009 llvm::Function *F = CGM.getIntrinsic(Intrinsic::write_register, Types);
2010 llvm::Value *ArgValue = CGF.EmitScalarExpr(E->getArg(1));
2011 if (MixedTypes) {
2012 // Extend 32 bit write value to 64 bit to pass to write.
2013 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
2014 return Builder.CreateCall(F, { Metadata, ArgValue });
2015 }
2016
2017 if (ValueType->isPointerTy()) {
2018 // Have VoidPtrTy ArgValue but want to return an i32/i64.
2019 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
2020 return Builder.CreateCall(F, { Metadata, ArgValue });
2021 }
2022
2023 return Builder.CreateCall(F, { Metadata, ArgValue });
2024}
2025
2026static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID,
2027 const CallExpr *E) {
2028 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2029 CodeGen::CodeGenModule &CGM = CGF.CGM;
2031
2032 auto getIntArg = [&](unsigned ArgNo) {
2034 if (!E->getArg(ArgNo)->EvaluateAsInt(Result, CGM.getContext()))
2035 llvm_unreachable("Expected constant argument to range prefetch.");
2036 return Result.Val.getInt().getExtValue();
2037 };
2038
2039 Ops.push_back(CGF.EmitScalarExpr(E->getArg(0))); /*Addr*/
2040 Ops.push_back(CGF.EmitScalarExpr(E->getArg(1))); /*Access Kind*/
2041 Ops.push_back(CGF.EmitScalarExpr(E->getArg(2))); /*Policy*/
2042
2043 if (BuiltinID == clang::AArch64::BI__builtin_arm_range_prefetch_x) {
2044 auto Length = getIntArg(3);
2045 auto Count = getIntArg(4) - 1;
2046 auto Stride = getIntArg(5);
2047 auto Distance = getIntArg(6);
2048
2049 // Map ReuseDistance given in bytes to four bits representing decreasing
2050 // powers of two in the range 512MiB (0b0001) to 32KiB (0b1111). Values
2051 // are rounded up to the nearest power of 2, starting at 32KiB. Any value
2052 // over the maximum is represented by 0 (distance not known).
2053 if (Distance > 0) {
2054 Distance = llvm::Log2_32_Ceil(Distance);
2055 if (Distance < 15)
2056 Distance = 15;
2057 else if (Distance > 29)
2058 Distance = 0;
2059 else
2060 Distance = 30 - Distance;
2061 }
2062
2063 uint64_t Mask22 = (1ULL << 22) - 1;
2064 uint64_t Mask16 = (1ULL << 16) - 1;
2065 uint64_t Metadata = (Distance << 60) | ((Stride & Mask22) << 38) |
2066 ((Count & Mask16) << 22) | (Length & Mask22);
2067
2068 Ops.push_back(llvm::ConstantInt::get(Builder.getInt64Ty(), Metadata));
2069 } else
2070 Ops.push_back(CGF.EmitScalarExpr(E->getArg(3)));
2071
2072 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_range_prefetch),
2073 Ops);
2074}
2075
2077 Instruction *Atomic, unsigned PtrOperand) {
2078 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2079 LLVMContext &Ctx = CGF.CGM.getLLVMContext();
2081 if (!E->getArg(3)->EvaluateAsInt(Result, CGF.getContext()))
2082 llvm_unreachable(
2083 "Expected integer hint argument to atomic operation with hint.");
2084 unsigned HintArg = Result.Val.getInt().getExtValue();
2085
2086 auto MemoryHint = toAArch64MemoryHint(HintArg);
2087 if (MemoryHint == AArch64MemoryHint::NONE ||
2088 (isa<AtomicRMWInst>(Atomic) && MemoryHint != AArch64MemoryHint::SHUH &&
2089 MemoryHint != AArch64MemoryHint::SHUH_PH))
2090 return;
2091
2092 MDNode *MemHint = MDNode::get(
2093 Ctx, {MDString::get(Ctx, "aarch64.mem_hint"),
2094 llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))});
2095 MDNode *HintNode = MDNode::get(
2096 Ctx,
2097 {llvm::ConstantAsMetadata::get(Builder.getInt32(PtrOperand)), MemHint});
2098 Atomic->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode);
2099}
2100
2102 unsigned BuiltinID,
2103 const CallExpr *E) {
2104 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2105 CodeGen::CodeGenModule &CGM = CGF.CGM;
2107 if (!E->getArg(2)->EvaluateAsInt(Result, CGM.getContext()))
2108 llvm_unreachable(
2109 "Expected integer policy argument to atomic store with hint.");
2110
2111 const Expr *Ptr = E->getArg(0);
2113 Addr = Addr.withElementType(
2114 CGF.ConvertTypeForMem(Ptr->getType()->getPointeeType()));
2115
2116 const Expr *Data = E->getArg(1);
2117 Value *DataVal = CGF.EmitToMemory(CGF.EmitScalarExpr(Data), Data->getType());
2118
2119 StoreInst *Store = Builder.CreateStore(DataVal, Addr);
2120 Store->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified());
2121
2122 AtomicOrdering Ordering;
2123 unsigned OrderingArg = Result.Val.getInt().getExtValue();
2124 assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering");
2125
2126 switch (static_cast<AtomicOrderingCABI>(OrderingArg)) {
2127 default:
2128 llvm_unreachable("Unsupported atomic ordering found.");
2129 case AtomicOrderingCABI::relaxed:
2130 Ordering = AtomicOrdering::Monotonic;
2131 break;
2132 case AtomicOrderingCABI::release:
2133 Ordering = AtomicOrdering::Release;
2134 break;
2135 case AtomicOrderingCABI::seq_cst:
2136 Ordering = AtomicOrdering::SequentiallyConsistent;
2137 break;
2138 }
2139 Store->setAtomic(Ordering);
2140 AttachAtomicHintMetadata(CGF, E, Store, 1);
2141 return Store;
2142}
2143
2145 const CallExpr *E,
2146 AtomicRMWInst::BinOp Op) {
2147 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2149 if (!E->getArg(2)->EvaluateAsInt(Result, CGF.getContext()))
2150 llvm_unreachable(
2151 "Expected integer policy argument to atomic fetch with hint.");
2152
2153 AtomicOrdering Ordering;
2154 unsigned OrderingArg = Result.Val.getInt().getExtValue();
2155 assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering");
2156
2157 switch (static_cast<AtomicOrderingCABI>(OrderingArg)) {
2158 case AtomicOrderingCABI::relaxed:
2159 Ordering = AtomicOrdering::Monotonic;
2160 break;
2161 case AtomicOrderingCABI::consume:
2162 case AtomicOrderingCABI::acquire:
2163 Ordering = AtomicOrdering::Acquire;
2164 break;
2165 case AtomicOrderingCABI::release:
2166 Ordering = AtomicOrdering::Release;
2167 break;
2168 case AtomicOrderingCABI::acq_rel:
2169 Ordering = AtomicOrdering::AcquireRelease;
2170 break;
2171 case AtomicOrderingCABI::seq_cst:
2172 Ordering = AtomicOrdering::SequentiallyConsistent;
2173 break;
2174 }
2175
2176 const Expr *Ptr = E->getArg(0);
2178 Addr = Addr.withElementType(
2179 CGF.ConvertTypeForMem(Ptr->getType()->getPointeeType()));
2180 const Expr *Data = E->getArg(1);
2181 Value *DataVal = CGF.EmitToMemory(CGF.EmitScalarExpr(Data), Data->getType());
2182
2183 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(Op, Addr, DataVal, Ordering);
2184 RMW->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified());
2185 AttachAtomicHintMetadata(CGF, E, RMW, 0);
2186 return CGF.EmitFromMemory(RMW, E->getType());
2187}
2188
2190 const CallExpr *E,
2192 llvm::Triple::ArchType Arch) {
2193 if (auto Hint = GetValueForARMHint(BuiltinID))
2194 return Hint;
2195
2196 if (BuiltinID == clang::ARM::BI__emit) {
2197 bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
2198 llvm::FunctionType *FTy =
2199 llvm::FunctionType::get(VoidTy, /*Variadic=*/false);
2200
2202 if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
2203 llvm_unreachable("Sema will ensure that the parameter is constant");
2204
2205 llvm::APSInt Value = Result.Val.getInt();
2206 uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
2207
2208 llvm::InlineAsm *Emit =
2209 IsThumb ? InlineAsm::get(FTy, ".inst.n 0x" + utohexstr(ZExtValue), "",
2210 /*hasSideEffects=*/true)
2211 : InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "",
2212 /*hasSideEffects=*/true);
2213
2214 return Builder.CreateCall(Emit);
2215 }
2216
2217 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
2218 Value *Option = EmitScalarExpr(E->getArg(0));
2219 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
2220 }
2221
2222 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
2224 Value *RW = EmitScalarExpr(E->getArg(1));
2225 Value *IsData = EmitScalarExpr(E->getArg(2));
2226
2227 // Locality is not supported on ARM target
2228 Value *Locality = llvm::ConstantInt::get(Int32Ty, 3);
2229
2230 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
2231 return Builder.CreateCall(F, {Address, RW, Locality, IsData});
2232 }
2233
2234 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
2235 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
2236 return Builder.CreateCall(
2237 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
2238 }
2239
2240 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
2241 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
2242 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
2243 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Arg->getType());
2244 Value *Res = Builder.CreateCall(F, {Arg, Builder.getInt1(false)});
2245 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
2246 Res = Builder.CreateTrunc(Res, Builder.getInt32Ty());
2247 return Res;
2248 }
2249
2250
2251 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
2252 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
2253 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls), Arg, "cls");
2254 }
2255 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
2256 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
2257 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls64), Arg,
2258 "cls");
2259 }
2260
2261 if (BuiltinID == clang::ARM::BI__clear_cache) {
2262 Value *Begin = EmitScalarExpr(E->getArg(0));
2263 Value *End = EmitScalarExpr(E->getArg(1));
2264 Function *F = CGM.getIntrinsic(Intrinsic::clear_cache, {CGM.DefaultPtrTy});
2265 return Builder.CreateCall(F, {Begin, End});
2266 }
2267
2268 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
2269 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
2270 Function *F;
2271
2272 switch (BuiltinID) {
2273 default: llvm_unreachable("unexpected builtin");
2274 case clang::ARM::BI__builtin_arm_mcrr:
2275 F = CGM.getIntrinsic(Intrinsic::arm_mcrr);
2276 break;
2277 case clang::ARM::BI__builtin_arm_mcrr2:
2278 F = CGM.getIntrinsic(Intrinsic::arm_mcrr2);
2279 break;
2280 }
2281
2282 // MCRR{2} instruction has 5 operands but
2283 // the intrinsic has 4 because Rt and Rt2
2284 // are represented as a single unsigned 64
2285 // bit integer in the intrinsic definition
2286 // but internally it's represented as 2 32
2287 // bit integers.
2288
2289 Value *Coproc = EmitScalarExpr(E->getArg(0));
2290 Value *Opc1 = EmitScalarExpr(E->getArg(1));
2291 Value *RtAndRt2 = EmitScalarExpr(E->getArg(2));
2292 Value *CRm = EmitScalarExpr(E->getArg(3));
2293
2294 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
2295 Value *Rt = Builder.CreateTruncOrBitCast(RtAndRt2, Int32Ty);
2296 Value *Rt2 = Builder.CreateLShr(RtAndRt2, C1);
2297 Rt2 = Builder.CreateTruncOrBitCast(Rt2, Int32Ty);
2298
2299 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
2300 }
2301
2302 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
2303 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
2304 Function *F;
2305
2306 switch (BuiltinID) {
2307 default: llvm_unreachable("unexpected builtin");
2308 case clang::ARM::BI__builtin_arm_mrrc:
2309 F = CGM.getIntrinsic(Intrinsic::arm_mrrc);
2310 break;
2311 case clang::ARM::BI__builtin_arm_mrrc2:
2312 F = CGM.getIntrinsic(Intrinsic::arm_mrrc2);
2313 break;
2314 }
2315
2316 Value *Coproc = EmitScalarExpr(E->getArg(0));
2317 Value *Opc1 = EmitScalarExpr(E->getArg(1));
2318 Value *CRm = EmitScalarExpr(E->getArg(2));
2319 Value *RtAndRt2 = Builder.CreateCall(F, {Coproc, Opc1, CRm});
2320
2321 // Returns an unsigned 64 bit integer, represented
2322 // as two 32 bit integers.
2323
2324 Value *Rt = Builder.CreateExtractValue(RtAndRt2, 1);
2325 Value *Rt1 = Builder.CreateExtractValue(RtAndRt2, 0);
2326 Rt = Builder.CreateZExt(Rt, Int64Ty);
2327 Rt1 = Builder.CreateZExt(Rt1, Int64Ty);
2328
2329 Value *ShiftCast = llvm::ConstantInt::get(Int64Ty, 32);
2330 RtAndRt2 = Builder.CreateShl(Rt, ShiftCast, "shl", true);
2331 RtAndRt2 = Builder.CreateOr(RtAndRt2, Rt1);
2332
2333 return Builder.CreateBitCast(RtAndRt2, ConvertType(E->getType()));
2334 }
2335
2336 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
2337 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
2338 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
2339 getContext().getTypeSize(E->getType()) == 64) ||
2340 BuiltinID == clang::ARM::BI__ldrexd) {
2341 Function *F;
2342
2343 switch (BuiltinID) {
2344 default: llvm_unreachable("unexpected builtin");
2345 case clang::ARM::BI__builtin_arm_ldaex:
2346 F = CGM.getIntrinsic(Intrinsic::arm_ldaexd);
2347 break;
2348 case clang::ARM::BI__builtin_arm_ldrexd:
2349 case clang::ARM::BI__builtin_arm_ldrex:
2350 case clang::ARM::BI__ldrexd:
2351 F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
2352 break;
2353 }
2354
2355 Value *LdPtr = EmitScalarExpr(E->getArg(0));
2356 Value *Val = Builder.CreateCall(F, LdPtr, "ldrexd");
2357
2358 Value *Val0 = Builder.CreateExtractValue(Val, 1);
2359 Value *Val1 = Builder.CreateExtractValue(Val, 0);
2360 Val0 = Builder.CreateZExt(Val0, Int64Ty);
2361 Val1 = Builder.CreateZExt(Val1, Int64Ty);
2362
2363 Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32);
2364 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
2365 Val = Builder.CreateOr(Val, Val1);
2366 return Builder.CreateBitCast(Val, ConvertType(E->getType()));
2367 }
2368
2369 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
2370 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
2371 Value *LoadAddr = EmitScalarExpr(E->getArg(0));
2372
2373 QualType Ty = E->getType();
2374 llvm::Type *RealResTy = ConvertType(Ty);
2375 llvm::Type *IntTy =
2376 llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
2377
2378 Function *F = CGM.getIntrinsic(
2379 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
2380 : Intrinsic::arm_ldrex,
2381 DefaultPtrTy);
2382 CallInst *Val = Builder.CreateCall(F, LoadAddr, "ldrex");
2383 Val->addParamAttr(
2384 0, Attribute::get(getLLVMContext(), Attribute::ElementType, IntTy));
2385
2386 if (RealResTy->isPointerTy())
2387 return Builder.CreateIntToPtr(Val, RealResTy);
2388 else {
2389 llvm::Type *IntResTy = llvm::IntegerType::get(
2390 getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
2391 return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
2392 RealResTy);
2393 }
2394 }
2395
2396 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
2397 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
2398 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
2399 getContext().getTypeSize(E->getArg(0)->getType()) == 64)) {
2400 Function *F = CGM.getIntrinsic(
2401 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
2402 : Intrinsic::arm_strexd);
2403 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty);
2404
2406 Value *Val = EmitScalarExpr(E->getArg(0));
2407 Builder.CreateStore(Val, Tmp);
2408
2409 Address LdPtr = Tmp.withElementType(STy);
2410 Val = Builder.CreateLoad(LdPtr);
2411
2412 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
2413 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
2414 Value *StPtr = EmitScalarExpr(E->getArg(1));
2415 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd");
2416 }
2417
2418 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
2419 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
2420 Value *StoreVal = EmitScalarExpr(E->getArg(0));
2421 Value *StoreAddr = EmitScalarExpr(E->getArg(1));
2422
2423 QualType Ty = E->getArg(0)->getType();
2424 llvm::Type *StoreTy =
2425 llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
2426
2427 if (StoreVal->getType()->isPointerTy())
2428 StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty);
2429 else {
2430 llvm::Type *IntTy = llvm::IntegerType::get(
2432 CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
2433 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
2434 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty);
2435 }
2436
2437 Function *F = CGM.getIntrinsic(
2438 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
2439 : Intrinsic::arm_strex,
2440 StoreAddr->getType());
2441
2442 CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex");
2443 CI->addParamAttr(
2444 1, Attribute::get(getLLVMContext(), Attribute::ElementType, StoreTy));
2445 return CI;
2446 }
2447
2448 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
2449 Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
2450 return Builder.CreateCall(F);
2451 }
2452
2453 // CRC32
2454 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
2455 switch (BuiltinID) {
2456 case clang::ARM::BI__builtin_arm_crc32b:
2457 CRCIntrinsicID = Intrinsic::arm_crc32b; break;
2458 case clang::ARM::BI__builtin_arm_crc32cb:
2459 CRCIntrinsicID = Intrinsic::arm_crc32cb; break;
2460 case clang::ARM::BI__builtin_arm_crc32h:
2461 CRCIntrinsicID = Intrinsic::arm_crc32h; break;
2462 case clang::ARM::BI__builtin_arm_crc32ch:
2463 CRCIntrinsicID = Intrinsic::arm_crc32ch; break;
2464 case clang::ARM::BI__builtin_arm_crc32w:
2465 case clang::ARM::BI__builtin_arm_crc32d:
2466 CRCIntrinsicID = Intrinsic::arm_crc32w; break;
2467 case clang::ARM::BI__builtin_arm_crc32cw:
2468 case clang::ARM::BI__builtin_arm_crc32cd:
2469 CRCIntrinsicID = Intrinsic::arm_crc32cw; break;
2470 }
2471
2472 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
2473 Value *Arg0 = EmitScalarExpr(E->getArg(0));
2474 Value *Arg1 = EmitScalarExpr(E->getArg(1));
2475
2476 // crc32{c,}d intrinsics are implemented as two calls to crc32{c,}w
2477 // intrinsics, hence we need different codegen for these cases.
2478 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
2479 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
2480 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
2481 Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty);
2482 Value *Arg1b = Builder.CreateLShr(Arg1, C1);
2483 Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty);
2484
2485 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
2486 Value *Res = Builder.CreateCall(F, {Arg0, Arg1a});
2487 return Builder.CreateCall(F, {Res, Arg1b});
2488 } else {
2489 Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty);
2490
2491 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
2492 return Builder.CreateCall(F, {Arg0, Arg1});
2493 }
2494 }
2495
2496 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
2497 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2498 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
2499 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
2500 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
2501 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
2502
2503 SpecialRegisterAccessKind AccessKind = Write;
2504 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
2505 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2506 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
2507 AccessKind = VolatileRead;
2508
2509 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
2510 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
2511
2512 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2513 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
2514
2515 llvm::Type *ValueType;
2516 llvm::Type *RegisterType;
2517 if (IsPointerBuiltin) {
2518 ValueType = VoidPtrTy;
2520 } else if (Is64Bit) {
2521 ValueType = RegisterType = Int64Ty;
2522 } else {
2523 ValueType = RegisterType = Int32Ty;
2524 }
2525
2526 return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType,
2527 AccessKind);
2528 }
2529
2530 if (BuiltinID == ARM::BI__builtin_sponentry) {
2531 llvm::Function *F = CGM.getIntrinsic(Intrinsic::sponentry, AllocaInt8PtrTy);
2532 return Builder.CreateCall(F);
2533 }
2534
2535 // Handle MSVC intrinsics before argument evaluation to prevent double
2536 // evaluation.
2537 if (std::optional<MSVCIntrin> MsvcIntId = translateArmToMsvcIntrin(BuiltinID))
2538 return EmitMSVCBuiltinExpr(*MsvcIntId, E);
2539
2540 // Deal with MVE builtins
2541 if (Value *Result = EmitARMMVEBuiltinExpr(BuiltinID, E, ReturnValue, Arch))
2542 return Result;
2543 // Handle CDE builtins
2544 if (Value *Result = EmitARMCDEBuiltinExpr(BuiltinID, E, ReturnValue, Arch))
2545 return Result;
2546
2547 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
2548 auto It = llvm::find_if(NEONEquivalentIntrinsicMap, [BuiltinID](auto &P) {
2549 return P.first == BuiltinID;
2550 });
2551 if (It != end(NEONEquivalentIntrinsicMap))
2552 BuiltinID = It->second;
2553
2554 // Find out if any arguments are required to be integer constant
2555 // expressions.
2556 unsigned ICEArguments = 0;
2558 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
2559 assert(Error == ASTContext::GE_None && "Should not codegen an error");
2560
2561 auto getAlignmentValue32 = [&](Address addr) -> Value* {
2562 return Builder.getInt32(addr.getAlignment().getQuantity());
2563 };
2564
2565 Address PtrOp0 = Address::invalid();
2566 Address PtrOp1 = Address::invalid();
2568 bool HasExtraArg = CodeGenUtils::hasExtraNeonArgument(BuiltinID);
2569 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
2570 for (unsigned i = 0, e = NumArgs; i != e; i++) {
2571 if (i == 0) {
2572 switch (BuiltinID) {
2573 case NEON::BI__builtin_neon_vld1_v:
2574 case NEON::BI__builtin_neon_vld1q_v:
2575 case NEON::BI__builtin_neon_vld1q_lane_v:
2576 case NEON::BI__builtin_neon_vld1_lane_v:
2577 case NEON::BI__builtin_neon_vld1_dup_v:
2578 case NEON::BI__builtin_neon_vld1q_dup_v:
2579 case NEON::BI__builtin_neon_vst1_v:
2580 case NEON::BI__builtin_neon_vst1q_v:
2581 case NEON::BI__builtin_neon_vst1q_lane_v:
2582 case NEON::BI__builtin_neon_vst1_lane_v:
2583 case NEON::BI__builtin_neon_vst2_v:
2584 case NEON::BI__builtin_neon_vst2q_v:
2585 case NEON::BI__builtin_neon_vst2_lane_v:
2586 case NEON::BI__builtin_neon_vst2q_lane_v:
2587 case NEON::BI__builtin_neon_vst3_v:
2588 case NEON::BI__builtin_neon_vst3q_v:
2589 case NEON::BI__builtin_neon_vst3_lane_v:
2590 case NEON::BI__builtin_neon_vst3q_lane_v:
2591 case NEON::BI__builtin_neon_vst4_v:
2592 case NEON::BI__builtin_neon_vst4q_v:
2593 case NEON::BI__builtin_neon_vst4_lane_v:
2594 case NEON::BI__builtin_neon_vst4q_lane_v:
2595 // Get the alignment for the argument in addition to the value;
2596 // we'll use it later.
2597 PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
2598 Ops.push_back(PtrOp0.emitRawPointer(*this));
2599 continue;
2600 }
2601 }
2602 if (i == 1) {
2603 switch (BuiltinID) {
2604 case NEON::BI__builtin_neon_vld2_v:
2605 case NEON::BI__builtin_neon_vld2q_v:
2606 case NEON::BI__builtin_neon_vld3_v:
2607 case NEON::BI__builtin_neon_vld3q_v:
2608 case NEON::BI__builtin_neon_vld4_v:
2609 case NEON::BI__builtin_neon_vld4q_v:
2610 case NEON::BI__builtin_neon_vld2_lane_v:
2611 case NEON::BI__builtin_neon_vld2q_lane_v:
2612 case NEON::BI__builtin_neon_vld3_lane_v:
2613 case NEON::BI__builtin_neon_vld3q_lane_v:
2614 case NEON::BI__builtin_neon_vld4_lane_v:
2615 case NEON::BI__builtin_neon_vld4q_lane_v:
2616 case NEON::BI__builtin_neon_vld2_dup_v:
2617 case NEON::BI__builtin_neon_vld2q_dup_v:
2618 case NEON::BI__builtin_neon_vld3_dup_v:
2619 case NEON::BI__builtin_neon_vld3q_dup_v:
2620 case NEON::BI__builtin_neon_vld4_dup_v:
2621 case NEON::BI__builtin_neon_vld4q_dup_v:
2622 // Get the alignment for the argument in addition to the value;
2623 // we'll use it later.
2624 PtrOp1 = EmitPointerWithAlignment(E->getArg(1));
2625 Ops.push_back(PtrOp1.emitRawPointer(*this));
2626 continue;
2627 }
2628 }
2629
2630 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E));
2631 }
2632
2633 switch (BuiltinID) {
2634 default: break;
2635
2636 case NEON::BI__builtin_neon_vget_lane_i8:
2637 case NEON::BI__builtin_neon_vget_lane_i16:
2638 case NEON::BI__builtin_neon_vget_lane_i32:
2639 case NEON::BI__builtin_neon_vget_lane_i64:
2640 case NEON::BI__builtin_neon_vget_lane_bf16:
2641 case NEON::BI__builtin_neon_vget_lane_f32:
2642 case NEON::BI__builtin_neon_vgetq_lane_i8:
2643 case NEON::BI__builtin_neon_vgetq_lane_i16:
2644 case NEON::BI__builtin_neon_vgetq_lane_i32:
2645 case NEON::BI__builtin_neon_vgetq_lane_i64:
2646 case NEON::BI__builtin_neon_vgetq_lane_bf16:
2647 case NEON::BI__builtin_neon_vgetq_lane_f32:
2648 case NEON::BI__builtin_neon_vduph_lane_bf16:
2649 case NEON::BI__builtin_neon_vduph_laneq_bf16:
2650 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
2651
2652 case NEON::BI__builtin_neon_vrndns_f32: {
2653 Value *Arg = EmitScalarExpr(E->getArg(0));
2654 llvm::Type *Tys[] = {Arg->getType()};
2655 Function *F = CGM.getIntrinsic(Intrinsic::roundeven, Tys);
2656 return Builder.CreateCall(F, {Arg}, "vrndn"); }
2657
2658 case NEON::BI__builtin_neon_vset_lane_i8:
2659 case NEON::BI__builtin_neon_vset_lane_i16:
2660 case NEON::BI__builtin_neon_vset_lane_i32:
2661 case NEON::BI__builtin_neon_vset_lane_i64:
2662 case NEON::BI__builtin_neon_vset_lane_bf16:
2663 case NEON::BI__builtin_neon_vset_lane_f32:
2664 case NEON::BI__builtin_neon_vsetq_lane_i8:
2665 case NEON::BI__builtin_neon_vsetq_lane_i16:
2666 case NEON::BI__builtin_neon_vsetq_lane_i32:
2667 case NEON::BI__builtin_neon_vsetq_lane_i64:
2668 case NEON::BI__builtin_neon_vsetq_lane_bf16:
2669 case NEON::BI__builtin_neon_vsetq_lane_f32:
2670 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
2671
2672 case NEON::BI__builtin_neon_vsha1h_u32:
2673 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops,
2674 "vsha1h");
2675 case NEON::BI__builtin_neon_vsha1cq_u32:
2676 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops,
2677 "vsha1h");
2678 case NEON::BI__builtin_neon_vsha1pq_u32:
2679 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops,
2680 "vsha1h");
2681 case NEON::BI__builtin_neon_vsha1mq_u32:
2682 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
2683 "vsha1h");
2684
2685 case NEON::BI__builtin_neon_vcvth_bf16_f32:
2686 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvtbfp2bf), Ops,
2687 "vcvtbfp2bf");
2688 case NEON::BI__builtin_neon_vcvt_f16_f32:
2689 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvtfp2hf), Ops,
2690 "vcvtfp2hf");
2691 case NEON::BI__builtin_neon_vcvt_f32_f16:
2692 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvthf2fp), Ops,
2693 "vcvthf2fp");
2694
2695 // The ARM _MoveToCoprocessor builtins put the input register value as
2696 // the first argument, but the LLVM intrinsic expects it as the third one.
2697 case clang::ARM::BI_MoveToCoprocessor:
2698 case clang::ARM::BI_MoveToCoprocessor2: {
2699 Function *F = CGM.getIntrinsic(BuiltinID == clang::ARM::BI_MoveToCoprocessor
2700 ? Intrinsic::arm_mcr
2701 : Intrinsic::arm_mcr2);
2702 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
2703 Ops[3], Ops[4], Ops[5]});
2704 }
2705 }
2706
2707 // Get the last argument, which specifies the vector type.
2708 assert(HasExtraArg);
2709 const Expr *Arg = E->getArg(E->getNumArgs()-1);
2710 std::optional<llvm::APSInt> Result =
2712 if (!Result)
2713 return nullptr;
2714
2715 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
2716 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
2717 // Determine the overloaded type of this builtin.
2718 llvm::Type *Ty;
2719 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
2720 Ty = FloatTy;
2721 else
2722 Ty = DoubleTy;
2723
2724 // Determine whether this is an unsigned conversion or not.
2725 bool usgn = Result->getZExtValue() == 1;
2726 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
2727
2728 // Call the appropriate intrinsic.
2729 Function *F = CGM.getIntrinsic(Int, Ty);
2730 return Builder.CreateCall(F, Ops, "vcvtr");
2731 }
2732
2733 // Determine the type of this overloaded NEON intrinsic.
2734 NeonTypeFlags Type = Result->getZExtValue();
2735 bool usgn = Type.isUnsigned();
2736 bool rightShift = false;
2737
2738 llvm::FixedVectorType *VTy =
2739 GetNeonType(this, Type, getTarget().hasFastHalfType(), false,
2740 getTarget().hasBFloat16Type());
2741 llvm::Type *Ty = VTy;
2742 if (!Ty)
2743 return nullptr;
2744
2745 // Many NEON builtins have identical semantics and uses in ARM and
2746 // AArch64. Emit these in a single function.
2747 auto IntrinsicMap = ArrayRef(ARMSIMDIntrinsicMap);
2749 IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted);
2750 if (Builtin)
2752 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
2753 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
2754
2755 unsigned Int;
2756 switch (BuiltinID) {
2757 default: return nullptr;
2758 case NEON::BI__builtin_neon_vld1q_lane_v:
2759 // Handle 64-bit integer elements as a special case. Use shuffles of
2760 // one-element vectors to avoid poor code for i64 in the backend.
2761 if (VTy->getElementType()->isIntegerTy(64)) {
2762 // Extract the other lane.
2763 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2764 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
2765 Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
2766 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
2767 // Load the value as a one-element vector.
2768 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
2769 llvm::Type *Tys[] = {Ty, Int8PtrTy};
2770 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys);
2771 Value *Align = getAlignmentValue32(PtrOp0);
2772 Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
2773 // Combine them.
2774 int Indices[] = {1 - Lane, Lane};
2775 return Builder.CreateShuffleVector(Ops[1], Ld, Indices, "vld1q_lane");
2776 }
2777 [[fallthrough]];
2778 case NEON::BI__builtin_neon_vld1_lane_v: {
2779 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2780 PtrOp0 = PtrOp0.withElementType(VTy->getElementType());
2781 Value *Ld = Builder.CreateLoad(PtrOp0);
2782 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
2783 }
2784 case NEON::BI__builtin_neon_vqrshrn_n_v:
2785 Int =
2786 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
2787 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n",
2788 1, true);
2789 case NEON::BI__builtin_neon_vqrshrun_n_v:
2790 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
2791 Ops, "vqrshrun_n", 1, true);
2792 case NEON::BI__builtin_neon_vqshrn_n_v:
2793 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
2794 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n",
2795 1, true);
2796 case NEON::BI__builtin_neon_vqshrun_n_v:
2797 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
2798 Ops, "vqshrun_n", 1, true);
2799 case NEON::BI__builtin_neon_vrecpe_v:
2800 case NEON::BI__builtin_neon_vrecpeq_v:
2801 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty),
2802 Ops, "vrecpe");
2803 case NEON::BI__builtin_neon_vrshrn_n_v:
2804 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
2805 Ops, "vrshrn_n", 1, true);
2806 case NEON::BI__builtin_neon_vrsra_n_v:
2807 case NEON::BI__builtin_neon_vrsraq_n_v:
2808 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2809 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2810 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true);
2811 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
2812 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]});
2813 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
2814 case NEON::BI__builtin_neon_vsri_n_v:
2815 case NEON::BI__builtin_neon_vsriq_n_v:
2816 rightShift = true;
2817 [[fallthrough]];
2818 case NEON::BI__builtin_neon_vsli_n_v:
2819 case NEON::BI__builtin_neon_vsliq_n_v:
2820 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift);
2821 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
2822 Ops, "vsli_n");
2823 case NEON::BI__builtin_neon_vsra_n_v:
2824 case NEON::BI__builtin_neon_vsraq_n_v:
2825 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2826 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
2827 return Builder.CreateAdd(Ops[0], Ops[1]);
2828 case NEON::BI__builtin_neon_vst1q_lane_v:
2829 // Handle 64-bit integer elements as a special case. Use a shuffle to get
2830 // a one-element vector and avoid poor code for i64 in the backend.
2831 if (VTy->getElementType()->isIntegerTy(64)) {
2832 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2833 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
2834 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
2835 Ops[2] = getAlignmentValue32(PtrOp0);
2836 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
2837 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
2838 Tys), Ops);
2839 }
2840 [[fallthrough]];
2841 case NEON::BI__builtin_neon_vst1_lane_v: {
2842 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2843 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
2844 return Builder.CreateStore(Ops[1],
2845 PtrOp0.withElementType(Ops[1]->getType()));
2846 }
2847 case NEON::BI__builtin_neon_vtbl1_v:
2848 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
2849 Ops, "vtbl1");
2850 case NEON::BI__builtin_neon_vtbl2_v:
2851 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2),
2852 Ops, "vtbl2");
2853 case NEON::BI__builtin_neon_vtbl3_v:
2854 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3),
2855 Ops, "vtbl3");
2856 case NEON::BI__builtin_neon_vtbl4_v:
2857 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4),
2858 Ops, "vtbl4");
2859 case NEON::BI__builtin_neon_vtbx1_v:
2860 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1),
2861 Ops, "vtbx1");
2862 case NEON::BI__builtin_neon_vtbx2_v:
2863 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2),
2864 Ops, "vtbx2");
2865 case NEON::BI__builtin_neon_vtbx3_v:
2866 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3),
2867 Ops, "vtbx3");
2868 case NEON::BI__builtin_neon_vtbx4_v:
2869 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4),
2870 Ops, "vtbx4");
2871 }
2872}
2873
2874template<typename Integer>
2876 return E->getIntegerConstantExpr(Context)->getExtValue();
2877}
2878
2879static llvm::Value *SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V,
2880 llvm::Type *T, bool Unsigned) {
2881 // Helper function called by Tablegen-constructed ARM MVE builtin codegen,
2882 // which finds it convenient to specify signed/unsigned as a boolean flag.
2883 return Unsigned ? Builder.CreateZExt(V, T) : Builder.CreateSExt(V, T);
2884}
2885
2886static llvm::Value *MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V,
2887 uint32_t Shift, bool Unsigned) {
2888 // MVE helper function for integer shift right. This must handle signed vs
2889 // unsigned, and also deal specially with the case where the shift count is
2890 // equal to the lane size. In LLVM IR, an LShr with that parameter would be
2891 // undefined behavior, but in MVE it's legal, so we must convert it to code
2892 // that is not undefined in IR.
2893 unsigned LaneBits = cast<llvm::VectorType>(V->getType())
2894 ->getElementType()
2895 ->getPrimitiveSizeInBits();
2896 if (Shift == LaneBits) {
2897 // An unsigned shift of the full lane size always generates zero, so we can
2898 // simply emit a zero vector. A signed shift of the full lane size does the
2899 // same thing as shifting by one bit fewer.
2900 if (Unsigned)
2901 return llvm::Constant::getNullValue(V->getType());
2902 else
2903 --Shift;
2904 }
2905 return Unsigned ? Builder.CreateLShr(V, Shift) : Builder.CreateAShr(V, Shift);
2906}
2907
2908static llvm::Value *ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V) {
2909 // MVE-specific helper function for a vector splat, which infers the element
2910 // count of the output vector by knowing that MVE vectors are all 128 bits
2911 // wide.
2912 unsigned Elements = 128 / V->getType()->getPrimitiveSizeInBits();
2913 return Builder.CreateVectorSplat(Elements, V);
2914}
2915
2916static llvm::Value *ARMMVEVectorReinterpret(CGBuilderTy &Builder,
2917 CodeGenFunction *CGF,
2918 llvm::Value *V,
2919 llvm::Type *DestType) {
2920 // Convert one MVE vector type into another by reinterpreting its in-register
2921 // format.
2922 //
2923 // Little-endian, this is identical to a bitcast (which reinterprets the
2924 // memory format). But big-endian, they're not necessarily the same, because
2925 // the register and memory formats map to each other differently depending on
2926 // the lane size.
2927 //
2928 // We generate a bitcast whenever we can (if we're little-endian, or if the
2929 // lane sizes are the same anyway). Otherwise we fall back to an IR intrinsic
2930 // that performs the different kind of reinterpretation.
2931 if (CGF->getTarget().isBigEndian() &&
2932 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
2933 return Builder.CreateCall(
2934 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vreinterpretq,
2935 {DestType, V->getType()}),
2936 V);
2937 } else {
2938 return Builder.CreateBitCast(V, DestType);
2939 }
2940}
2941
2942static llvm::Value *VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd) {
2943 // Make a shufflevector that extracts every other element of a vector (evens
2944 // or odds, as desired).
2945 SmallVector<int, 16> Indices;
2946 unsigned InputElements =
2947 cast<llvm::FixedVectorType>(V->getType())->getNumElements();
2948 for (unsigned i = 0; i < InputElements; i += 2)
2949 Indices.push_back(i + Odd);
2950 return Builder.CreateShuffleVector(V, Indices);
2951}
2952
2953static llvm::Value *VectorZip(CGBuilderTy &Builder, llvm::Value *V0,
2954 llvm::Value *V1) {
2955 // Make a shufflevector that interleaves two vectors element by element.
2956 assert(V0->getType() == V1->getType() && "Can't zip different vector types");
2957 SmallVector<int, 16> Indices;
2958 unsigned InputElements =
2959 cast<llvm::FixedVectorType>(V0->getType())->getNumElements();
2960 for (unsigned i = 0; i < InputElements; i++) {
2961 Indices.push_back(i);
2962 Indices.push_back(i + InputElements);
2963 }
2964 return Builder.CreateShuffleVector(V0, V1, Indices);
2965}
2966
2967template<unsigned HighBit, unsigned OtherBits>
2968static llvm::Value *ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT) {
2969 // MVE-specific helper function to make a vector splat of a constant such as
2970 // UINT_MAX or INT_MIN, in which all bits below the highest one are equal.
2971 llvm::Type *T = cast<llvm::VectorType>(VT)->getElementType();
2972 unsigned LaneBits = T->getPrimitiveSizeInBits();
2973 uint32_t Value = HighBit << (LaneBits - 1);
2974 if (OtherBits)
2975 Value |= (1UL << (LaneBits - 1)) - 1;
2976 llvm::Value *Lane = llvm::ConstantInt::get(T, Value);
2977 return ARMMVEVectorSplat(Builder, Lane);
2978}
2979
2980static llvm::Value *ARMMVEVectorElementReverse(CGBuilderTy &Builder,
2981 llvm::Value *V,
2982 unsigned ReverseWidth) {
2983 // MVE-specific helper function which reverses the elements of a
2984 // vector within every (ReverseWidth)-bit collection of lanes.
2985 SmallVector<int, 16> Indices;
2986 unsigned LaneSize = V->getType()->getScalarSizeInBits();
2987 unsigned Elements = 128 / LaneSize;
2988 unsigned Mask = ReverseWidth / LaneSize - 1;
2989 for (unsigned i = 0; i < Elements; i++)
2990 Indices.push_back(i ^ Mask);
2991 return Builder.CreateShuffleVector(V, Indices);
2992}
2993
2994static llvm::Value *ARMMVECreateSIToFP(CGBuilderTy &Builder,
2995 CodeGenFunction *CGF, llvm::Value *V,
2996 llvm::Type *Ty) {
2997 return Builder.CreateCall(
2998 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vcvt_fp_int, {Ty, V->getType()}),
2999 {V, llvm::ConstantInt::get(Builder.getInt32Ty(), 0)});
3000}
3001
3002static llvm::Value *ARMMVECreateUIToFP(CGBuilderTy &Builder,
3003 CodeGenFunction *CGF, llvm::Value *V,
3004 llvm::Type *Ty) {
3005 return Builder.CreateCall(
3006 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vcvt_fp_int, {Ty, V->getType()}),
3007 {V, llvm::ConstantInt::get(Builder.getInt32Ty(), 1)});
3008}
3009
3010static llvm::Value *ARMMVECreateFPToSI(CGBuilderTy &Builder,
3011 CodeGenFunction *CGF, llvm::Value *V,
3012 llvm::Type *Ty) {
3013 return Builder.CreateCall(
3014 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vcvt_int_fp, {Ty, V->getType()}),
3015 {V, llvm::ConstantInt::get(Builder.getInt32Ty(), 0)});
3016}
3017
3018static llvm::Value *ARMMVECreateFPToUI(CGBuilderTy &Builder,
3019 CodeGenFunction *CGF, llvm::Value *V,
3020 llvm::Type *Ty) {
3021 return Builder.CreateCall(
3022 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vcvt_int_fp, {Ty, V->getType()}),
3023 {V, llvm::ConstantInt::get(Builder.getInt32Ty(), 1)});
3024}
3025
3027 const CallExpr *E,
3029 llvm::Triple::ArchType Arch) {
3030 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
3031 Intrinsic::ID IRIntr;
3032 unsigned NumVectors;
3033
3034 // Code autogenerated by Tablegen will handle all the simple builtins.
3035 switch (BuiltinID) {
3036 #include "clang/Basic/arm_mve_builtin_cg.inc"
3037
3038 // If we didn't match an MVE builtin id at all, go back to the
3039 // main EmitARMBuiltinExpr.
3040 default:
3041 return nullptr;
3042 }
3043
3044 // Anything that breaks from that switch is an MVE builtin that
3045 // needs handwritten code to generate.
3046
3047 switch (CustomCodeGenType) {
3048
3049 case CustomCodeGen::VLD24: {
3052
3053 auto MvecCType = E->getType();
3054 auto MvecLType = ConvertType(MvecCType);
3055 assert(MvecLType->isStructTy() &&
3056 "Return type for vld[24]q should be a struct");
3057 assert(MvecLType->getStructNumElements() == 1 &&
3058 "Return-type struct for vld[24]q should have one element");
3059 auto MvecLTypeInner = MvecLType->getStructElementType(0);
3060 assert(MvecLTypeInner->isArrayTy() &&
3061 "Return-type struct for vld[24]q should contain an array");
3062 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
3063 "Array member of return-type struct vld[24]q has wrong length");
3064 auto VecLType = MvecLTypeInner->getArrayElementType();
3065
3066 Tys.push_back(VecLType);
3067
3068 auto Addr = E->getArg(0);
3069 Ops.push_back(EmitScalarExpr(Addr));
3070 Tys.push_back(ConvertType(Addr->getType()));
3071
3072 Function *F = CGM.getIntrinsic(IRIntr, ArrayRef(Tys));
3073 Value *LoadResult = Builder.CreateCall(F, Ops);
3074 Value *MvecOut = PoisonValue::get(MvecLType);
3075 for (unsigned i = 0; i < NumVectors; ++i) {
3076 Value *Vec = Builder.CreateExtractValue(LoadResult, i);
3077 MvecOut = Builder.CreateInsertValue(MvecOut, Vec, {0, i});
3078 }
3079
3080 if (ReturnValue.isNull())
3081 return MvecOut;
3082 else
3083 return Builder.CreateStore(MvecOut, ReturnValue.getAddress());
3084 }
3085
3086 case CustomCodeGen::VST24: {
3089
3090 auto Addr = E->getArg(0);
3091 Ops.push_back(EmitScalarExpr(Addr));
3092 Tys.push_back(ConvertType(Addr->getType()));
3093
3094 auto MvecCType = E->getArg(1)->getType();
3095 auto MvecLType = ConvertType(MvecCType);
3096 assert(MvecLType->isStructTy() && "Data type for vst2q should be a struct");
3097 assert(MvecLType->getStructNumElements() == 1 &&
3098 "Data-type struct for vst2q should have one element");
3099 auto MvecLTypeInner = MvecLType->getStructElementType(0);
3100 assert(MvecLTypeInner->isArrayTy() &&
3101 "Data-type struct for vst2q should contain an array");
3102 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
3103 "Array member of return-type struct vld[24]q has wrong length");
3104 auto VecLType = MvecLTypeInner->getArrayElementType();
3105
3106 Tys.push_back(VecLType);
3107
3108 AggValueSlot MvecSlot = CreateAggTemp(MvecCType);
3109 EmitAggExpr(E->getArg(1), MvecSlot);
3110 auto Mvec = Builder.CreateLoad(MvecSlot.getAddress());
3111 for (unsigned i = 0; i < NumVectors; i++)
3112 Ops.push_back(Builder.CreateExtractValue(Mvec, {0, i}));
3113
3114 Function *F = CGM.getIntrinsic(IRIntr, ArrayRef(Tys));
3115 Value *ToReturn = nullptr;
3116 for (unsigned i = 0; i < NumVectors; i++) {
3117 Ops.push_back(llvm::ConstantInt::get(Int32Ty, i));
3118 ToReturn = Builder.CreateCall(F, Ops);
3119 Ops.pop_back();
3120 }
3121 return ToReturn;
3122 }
3123 }
3124 llvm_unreachable("unknown custom codegen type.");
3125}
3126
3128 const CallExpr *E,
3130 llvm::Triple::ArchType Arch) {
3131 switch (BuiltinID) {
3132 default:
3133 return nullptr;
3134#include "clang/Basic/arm_cde_builtin_cg.inc"
3135 }
3136}
3137
3138static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID,
3139 const CallExpr *E,
3141 llvm::Triple::ArchType Arch) {
3142 unsigned int Int = 0;
3143 const char *s = nullptr;
3144
3145 switch (BuiltinID) {
3146 default:
3147 return nullptr;
3148 case NEON::BI__builtin_neon_vtbl1_v:
3149 case NEON::BI__builtin_neon_vqtbl1_v:
3150 case NEON::BI__builtin_neon_vqtbl1q_v:
3151 case NEON::BI__builtin_neon_vtbl2_v:
3152 case NEON::BI__builtin_neon_vqtbl2_v:
3153 case NEON::BI__builtin_neon_vqtbl2q_v:
3154 case NEON::BI__builtin_neon_vtbl3_v:
3155 case NEON::BI__builtin_neon_vqtbl3_v:
3156 case NEON::BI__builtin_neon_vqtbl3q_v:
3157 case NEON::BI__builtin_neon_vtbl4_v:
3158 case NEON::BI__builtin_neon_vqtbl4_v:
3159 case NEON::BI__builtin_neon_vqtbl4q_v:
3160 break;
3161 case NEON::BI__builtin_neon_vtbx1_v:
3162 case NEON::BI__builtin_neon_vqtbx1_v:
3163 case NEON::BI__builtin_neon_vqtbx1q_v:
3164 case NEON::BI__builtin_neon_vtbx2_v:
3165 case NEON::BI__builtin_neon_vqtbx2_v:
3166 case NEON::BI__builtin_neon_vqtbx2q_v:
3167 case NEON::BI__builtin_neon_vtbx3_v:
3168 case NEON::BI__builtin_neon_vqtbx3_v:
3169 case NEON::BI__builtin_neon_vqtbx3q_v:
3170 case NEON::BI__builtin_neon_vtbx4_v:
3171 case NEON::BI__builtin_neon_vqtbx4_v:
3172 case NEON::BI__builtin_neon_vqtbx4q_v:
3173 break;
3174 }
3175
3176 assert(E->getNumArgs() >= 3);
3177
3178 // Get the last argument, which specifies the vector type.
3179 const Expr *Arg = E->getArg(E->getNumArgs() - 1);
3180 std::optional<llvm::APSInt> Result =
3182 if (!Result)
3183 return nullptr;
3184
3185 // Determine the type of this overloaded NEON intrinsic.
3186 NeonTypeFlags Type = Result->getZExtValue();
3187 llvm::FixedVectorType *Ty = GetNeonType(&CGF, Type);
3188 if (!Ty)
3189 return nullptr;
3190
3191 CodeGen::CGBuilderTy &Builder = CGF.Builder;
3192
3193 // AArch64 scalar builtins are not overloaded, they do not have an extra
3194 // argument that specifies the vector type, need to handle each case.
3195 switch (BuiltinID) {
3196 case NEON::BI__builtin_neon_vtbl1_v: {
3197 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 1), nullptr, Ops[1],
3198 Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1");
3199 }
3200 case NEON::BI__builtin_neon_vtbl2_v: {
3201 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 2), nullptr, Ops[2],
3202 Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1");
3203 }
3204 case NEON::BI__builtin_neon_vtbl3_v: {
3205 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 3), nullptr, Ops[3],
3206 Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2");
3207 }
3208 case NEON::BI__builtin_neon_vtbl4_v: {
3209 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 4), nullptr, Ops[4],
3210 Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2");
3211 }
3212 case NEON::BI__builtin_neon_vtbx1_v: {
3213 Value *TblRes =
3214 packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 1), nullptr, Ops[2], Ty,
3215 Intrinsic::aarch64_neon_tbl1, "vtbl1");
3216
3217 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
3218 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
3219 CmpRes = Builder.CreateSExt(CmpRes, Ty);
3220
3221 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
3222 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
3223 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
3224 }
3225 case NEON::BI__builtin_neon_vtbx2_v: {
3226 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 2), Ops[0], Ops[3],
3227 Ty, Intrinsic::aarch64_neon_tbx1, "vtbx1");
3228 }
3229 case NEON::BI__builtin_neon_vtbx3_v: {
3230 Value *TblRes =
3231 packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 3), nullptr, Ops[4], Ty,
3232 Intrinsic::aarch64_neon_tbl2, "vtbl2");
3233
3234 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
3235 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
3236 TwentyFourV);
3237 CmpRes = Builder.CreateSExt(CmpRes, Ty);
3238
3239 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
3240 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
3241 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
3242 }
3243 case NEON::BI__builtin_neon_vtbx4_v: {
3244 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 4), Ops[0], Ops[5],
3245 Ty, Intrinsic::aarch64_neon_tbx2, "vtbx2");
3246 }
3247 case NEON::BI__builtin_neon_vqtbl1_v:
3248 case NEON::BI__builtin_neon_vqtbl1q_v:
3249 Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break;
3250 case NEON::BI__builtin_neon_vqtbl2_v:
3251 case NEON::BI__builtin_neon_vqtbl2q_v: {
3252 Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break;
3253 case NEON::BI__builtin_neon_vqtbl3_v:
3254 case NEON::BI__builtin_neon_vqtbl3q_v:
3255 Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break;
3256 case NEON::BI__builtin_neon_vqtbl4_v:
3257 case NEON::BI__builtin_neon_vqtbl4q_v:
3258 Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break;
3259 case NEON::BI__builtin_neon_vqtbx1_v:
3260 case NEON::BI__builtin_neon_vqtbx1q_v:
3261 Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break;
3262 case NEON::BI__builtin_neon_vqtbx2_v:
3263 case NEON::BI__builtin_neon_vqtbx2q_v:
3264 Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break;
3265 case NEON::BI__builtin_neon_vqtbx3_v:
3266 case NEON::BI__builtin_neon_vqtbx3q_v:
3267 Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break;
3268 case NEON::BI__builtin_neon_vqtbx4_v:
3269 case NEON::BI__builtin_neon_vqtbx4q_v:
3270 Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break;
3271 }
3272 }
3273
3274 if (!Int)
3275 return nullptr;
3276
3277 Function *F = CGF.CGM.getIntrinsic(Int, Ty);
3278 return CGF.EmitNeonCall(F, Ops, s);
3279}
3280
3282 auto *VTy = llvm::FixedVectorType::get(Int16Ty, 4);
3283 Op = Builder.CreateBitCast(Op, Int16Ty);
3284 Value *V = PoisonValue::get(VTy);
3285 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
3286 Op = Builder.CreateInsertElement(V, Op, CI);
3287 return Op;
3288}
3289
3290/// SVEBuiltinMemEltTy - Returns the memory element type for this memory
3291/// access builtin. Only required if it can't be inferred from the base pointer
3292/// operand.
3294 switch (TypeFlags.getMemEltType()) {
3295 case SVETypeFlags::MemEltTyDefault:
3296 return getEltType(TypeFlags);
3297 case SVETypeFlags::MemEltTyInt8:
3298 return Builder.getInt8Ty();
3299 case SVETypeFlags::MemEltTyInt16:
3300 return Builder.getInt16Ty();
3301 case SVETypeFlags::MemEltTyInt32:
3302 return Builder.getInt32Ty();
3303 case SVETypeFlags::MemEltTyInt64:
3304 return Builder.getInt64Ty();
3305 }
3306 llvm_unreachable("Unknown MemEltType");
3307}
3308
3309llvm::Type *CodeGenFunction::getEltType(const SVETypeFlags &TypeFlags) {
3310 switch (TypeFlags.getEltType()) {
3311 default:
3312 llvm_unreachable("Invalid SVETypeFlag!");
3313
3314 case SVETypeFlags::EltTyMFloat8:
3315 case SVETypeFlags::EltTyInt8:
3316 return Builder.getInt8Ty();
3317 case SVETypeFlags::EltTyInt16:
3318 return Builder.getInt16Ty();
3319 case SVETypeFlags::EltTyInt32:
3320 return Builder.getInt32Ty();
3321 case SVETypeFlags::EltTyInt64:
3322 return Builder.getInt64Ty();
3323 case SVETypeFlags::EltTyInt128:
3324 return Builder.getInt128Ty();
3325
3326 case SVETypeFlags::EltTyFloat16:
3327 return Builder.getHalfTy();
3328 case SVETypeFlags::EltTyFloat32:
3329 return Builder.getFloatTy();
3330 case SVETypeFlags::EltTyFloat64:
3331 return Builder.getDoubleTy();
3332
3333 case SVETypeFlags::EltTyBFloat16:
3334 return Builder.getBFloatTy();
3335
3336 case SVETypeFlags::EltTyBool8:
3337 case SVETypeFlags::EltTyBool16:
3338 case SVETypeFlags::EltTyBool32:
3339 case SVETypeFlags::EltTyBool64:
3340 return Builder.getInt1Ty();
3341 }
3342}
3343
3344// Return the llvm predicate vector type corresponding to the specified element
3345// TypeFlags.
3346llvm::ScalableVectorType *
3348 switch (TypeFlags.getEltType()) {
3349 default: llvm_unreachable("Unhandled SVETypeFlag!");
3350
3351 case SVETypeFlags::EltTyInt8:
3352 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
3353 case SVETypeFlags::EltTyInt16:
3354 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3355 case SVETypeFlags::EltTyInt32:
3356 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
3357 case SVETypeFlags::EltTyInt64:
3358 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
3359
3360 case SVETypeFlags::EltTyBFloat16:
3361 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3362 case SVETypeFlags::EltTyFloat16:
3363 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3364 case SVETypeFlags::EltTyFloat32:
3365 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
3366 case SVETypeFlags::EltTyFloat64:
3367 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
3368
3369 case SVETypeFlags::EltTyBool8:
3370 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
3371 case SVETypeFlags::EltTyBool16:
3372 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3373 case SVETypeFlags::EltTyBool32:
3374 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
3375 case SVETypeFlags::EltTyBool64:
3376 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
3377 }
3378}
3379
3380// Return the llvm vector type corresponding to the specified element TypeFlags.
3381llvm::ScalableVectorType *
3383 switch (TypeFlags.getEltType()) {
3384 default:
3385 llvm_unreachable("Invalid SVETypeFlag!");
3386
3387 case SVETypeFlags::EltTyInt8:
3388 return llvm::ScalableVectorType::get(Builder.getInt8Ty(), 16);
3389 case SVETypeFlags::EltTyInt16:
3390 return llvm::ScalableVectorType::get(Builder.getInt16Ty(), 8);
3391 case SVETypeFlags::EltTyInt32:
3392 return llvm::ScalableVectorType::get(Builder.getInt32Ty(), 4);
3393 case SVETypeFlags::EltTyInt64:
3394 return llvm::ScalableVectorType::get(Builder.getInt64Ty(), 2);
3395
3396 case SVETypeFlags::EltTyMFloat8:
3397 return llvm::ScalableVectorType::get(Builder.getInt8Ty(), 16);
3398 case SVETypeFlags::EltTyFloat16:
3399 return llvm::ScalableVectorType::get(Builder.getHalfTy(), 8);
3400 case SVETypeFlags::EltTyBFloat16:
3401 return llvm::ScalableVectorType::get(Builder.getBFloatTy(), 8);
3402 case SVETypeFlags::EltTyFloat32:
3403 return llvm::ScalableVectorType::get(Builder.getFloatTy(), 4);
3404 case SVETypeFlags::EltTyFloat64:
3405 return llvm::ScalableVectorType::get(Builder.getDoubleTy(), 2);
3406
3407 case SVETypeFlags::EltTyBool8:
3408 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
3409 case SVETypeFlags::EltTyBool16:
3410 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3411 case SVETypeFlags::EltTyBool32:
3412 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
3413 case SVETypeFlags::EltTyBool64:
3414 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
3415 }
3416}
3417
3418constexpr unsigned SVEBitsPerBlock = 128;
3419
3420static llvm::ScalableVectorType *getSVEVectorForElementType(llvm::Type *EltTy) {
3421 unsigned NumElts = SVEBitsPerBlock / EltTy->getScalarSizeInBits();
3422 return llvm::ScalableVectorType::get(EltTy, NumElts);
3423}
3424
3425// Reinterpret the input predicate so that it can be used to correctly isolate
3426// the elements of the specified datatype.
3428 llvm::ScalableVectorType *VTy) {
3429
3430 if (isa<TargetExtType>(Pred->getType()) &&
3431 cast<TargetExtType>(Pred->getType())->getName() == "aarch64.svcount")
3432 return Pred;
3433
3434 auto *RTy = llvm::VectorType::get(IntegerType::get(getLLVMContext(), 1), VTy);
3435 if (Pred->getType() == RTy)
3436 return Pred;
3437
3438 unsigned IntID;
3439 llvm::Type *IntrinsicTy;
3440 switch (VTy->getMinNumElements()) {
3441 default:
3442 llvm_unreachable("unsupported element count!");
3443 case 1:
3444 case 2:
3445 case 4:
3446 case 8:
3447 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
3448 IntrinsicTy = RTy;
3449 break;
3450 case 16:
3451 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
3452 IntrinsicTy = Pred->getType();
3453 break;
3454 }
3455
3456 Function *F = CGM.getIntrinsic(IntID, IntrinsicTy);
3457 Value *C = Builder.CreateCall(F, Pred);
3458 assert(C->getType() == RTy && "Unexpected return type!");
3459 return C;
3460}
3461
3463 llvm::StructType *Ty) {
3464 if (PredTuple->getType() == Ty)
3465 return PredTuple;
3466
3467 Value *Ret = llvm::PoisonValue::get(Ty);
3468 for (unsigned I = 0; I < Ty->getNumElements(); ++I) {
3469 Value *Pred = Builder.CreateExtractValue(PredTuple, I);
3470 Pred = EmitSVEPredicateCast(
3471 Pred, cast<llvm::ScalableVectorType>(Ty->getTypeAtIndex(I)));
3472 Ret = Builder.CreateInsertValue(Ret, Pred, I);
3473 }
3474
3475 return Ret;
3476}
3477
3480 unsigned IntID) {
3481 auto *ResultTy = getSVEType(TypeFlags);
3482 auto *OverloadedTy =
3483 llvm::ScalableVectorType::get(SVEBuiltinMemEltTy(TypeFlags), ResultTy);
3484 Function *F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[1]->getType()});
3485
3486 // At the ACLE level there's only one predicate type, svbool_t, which is
3487 // mapped to <n x 16 x i1>. However, this might be incompatible with the
3488 // actual type being loaded. For example, when loading doubles (i64) the
3489 // predicate should be <n x 2 x i1> instead. At the IR level the type of
3490 // the predicate and the data being loaded must match. Cast to the type
3491 // expected by the intrinsic. The intrinsic itself should be defined in
3492 // a way than enforces relations between parameter types.
3493 Ops[0] = EmitSVEPredicateCast(
3494 Ops[0], cast<llvm::ScalableVectorType>(F->getArg(0)->getType()));
3495
3496 // Pass 0 when the offset is missing. This can only be applied when using
3497 // the "vector base" addressing mode for which ACLE allows no offset. The
3498 // corresponding LLVM IR always requires an offset.
3499 if (Ops.size() == 2) {
3500 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
3501 Ops.push_back(ConstantInt::get(Int64Ty, 0));
3502 }
3503
3504 // For "vector base, scalar index" scale the index so that it becomes a
3505 // scalar offset.
3506 if (!TypeFlags.isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
3507 unsigned BytesPerElt =
3508 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
3509 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
3510 }
3511
3512 Value *Call = Builder.CreateCall(F, Ops);
3513
3514 // The following sext/zext is only needed when ResultTy != OverloadedTy. In
3515 // other cases it's folded into a nop.
3516 return TypeFlags.isZExtReturn() ? Builder.CreateZExt(Call, ResultTy)
3517 : Builder.CreateSExt(Call, ResultTy);
3518}
3519
3522 unsigned IntID) {
3523 auto *SrcDataTy = getSVEType(TypeFlags);
3524 auto *OverloadedTy =
3525 llvm::ScalableVectorType::get(SVEBuiltinMemEltTy(TypeFlags), SrcDataTy);
3526
3527 // In ACLE the source data is passed in the last argument, whereas in LLVM IR
3528 // it's the first argument. Move it accordingly.
3529 Ops.insert(Ops.begin(), Ops.pop_back_val());
3530
3531 Function *F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[2]->getType()});
3532
3533 // Pass 0 when the offset is missing. This can only be applied when using
3534 // the "vector base" addressing mode for which ACLE allows no offset. The
3535 // corresponding LLVM IR always requires an offset.
3536 if (Ops.size() == 3) {
3537 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
3538 Ops.push_back(ConstantInt::get(Int64Ty, 0));
3539 }
3540
3541 // Truncation is needed when SrcDataTy != OverloadedTy. In other cases it's
3542 // folded into a nop.
3543 Ops[0] = Builder.CreateTrunc(Ops[0], OverloadedTy);
3544
3545 // At the ACLE level there's only one predicate type, svbool_t, which is
3546 // mapped to <n x 16 x i1>. However, this might be incompatible with the
3547 // actual type being stored. For example, when storing doubles (i64) the
3548 // predicated should be <n x 2 x i1> instead. At the IR level the type of
3549 // the predicate and the data being stored must match. Cast to the type
3550 // expected by the intrinsic. The intrinsic itself should be defined in
3551 // a way that enforces relations between parameter types.
3552 Ops[1] = EmitSVEPredicateCast(
3553 Ops[1], cast<llvm::ScalableVectorType>(F->getArg(1)->getType()));
3554
3555 // For "vector base, scalar index" scale the index so that it becomes a
3556 // scalar offset.
3557 if (!TypeFlags.isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
3558 unsigned BytesPerElt =
3559 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
3560 Ops[3] = Builder.CreateShl(Ops[3], Log2_32(BytesPerElt));
3561 }
3562
3563 return Builder.CreateCall(F, Ops);
3564}
3565
3568 unsigned IntID) {
3569 // The gather prefetches are overloaded on the vector input - this can either
3570 // be the vector of base addresses or vector of offsets.
3571 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType());
3572 if (!OverloadedTy)
3573 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType());
3574
3575 // Cast the predicate from svbool_t to the right number of elements.
3576 Ops[0] = EmitSVEPredicateCast(Ops[0], OverloadedTy);
3577
3578 // vector + imm addressing modes
3579 if (Ops[1]->getType()->isVectorTy()) {
3580 if (Ops.size() == 3) {
3581 // Pass 0 for 'vector+imm' when the index is omitted.
3582 Ops.push_back(ConstantInt::get(Int64Ty, 0));
3583
3584 // The sv_prfop is the last operand in the builtin and IR intrinsic.
3585 std::swap(Ops[2], Ops[3]);
3586 } else {
3587 // Index needs to be passed as scaled offset.
3588 llvm::Type *MemEltTy = SVEBuiltinMemEltTy(TypeFlags);
3589 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
3590 if (BytesPerElt > 1)
3591 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
3592 }
3593
3594 Function *F = CGM.getIntrinsic(IntID, OverloadedTy);
3595 return Builder.CreateCall(F, Ops);
3596 }
3597
3598 Function *F = CGM.getIntrinsic(IntID, {Ops[1]->getType(), OverloadedTy});
3599 return Builder.CreateCall(F, Ops);
3600}
3601
3604 unsigned IntID) {
3605 llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
3606 Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy);
3607 Value *BasePtr = Ops[1];
3608
3609 // Does the load have an offset?
3610 if (Ops.size() > 2)
3611 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]);
3612
3613 Function *F = CGM.getIntrinsic(IntID, {VTy, BasePtr->getType()});
3614 return Builder.CreateCall(F, {Predicate, BasePtr});
3615}
3616
3619 unsigned IntID) {
3620 llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
3621
3622 unsigned N;
3623 switch (IntID) {
3624 case Intrinsic::aarch64_sve_st2:
3625 case Intrinsic::aarch64_sve_st1_pn_x2:
3626 case Intrinsic::aarch64_sve_stnt1_pn_x2:
3627 case Intrinsic::aarch64_sve_st2q:
3628 N = 2;
3629 break;
3630 case Intrinsic::aarch64_sve_st3:
3631 case Intrinsic::aarch64_sve_st3q:
3632 N = 3;
3633 break;
3634 case Intrinsic::aarch64_sve_st4:
3635 case Intrinsic::aarch64_sve_st1_pn_x4:
3636 case Intrinsic::aarch64_sve_stnt1_pn_x4:
3637 case Intrinsic::aarch64_sve_st4q:
3638 N = 4;
3639 break;
3640 default:
3641 llvm_unreachable("unknown intrinsic!");
3642 }
3643
3644 Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy);
3645 Value *BasePtr = Ops[1];
3646
3647 // Does the store have an offset?
3648 if (Ops.size() > (2 + N))
3649 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]);
3650
3651 // The llvm.aarch64.sve.st2/3/4 intrinsics take legal part vectors, so we
3652 // need to break up the tuple vector.
3654 for (unsigned I = Ops.size() - N; I < Ops.size(); ++I)
3655 Operands.push_back(Ops[I]);
3656 Operands.append({Predicate, BasePtr});
3657 Function *F = CGM.getIntrinsic(IntID, {VTy, BasePtr->getType()});
3658
3659 return Builder.CreateCall(F, Operands);
3660}
3661
3662// SVE2's svpmullb and svpmullt builtins are similar to the svpmullb_pair and
3663// svpmullt_pair intrinsics, with the exception that their results are bitcast
3664// to a wider type.
3667 unsigned BuiltinID) {
3668 // Splat scalar operand to vector (intrinsics with _n infix)
3669 if (TypeFlags.hasSplatOperand()) {
3670 unsigned OpNo = TypeFlags.getSplatOperand();
3671 Ops[OpNo] = EmitSVEDupX(Ops[OpNo]);
3672 }
3673
3674 // The pair-wise function has a narrower overloaded type.
3675 Function *F = CGM.getIntrinsic(BuiltinID, Ops[0]->getType());
3676 Value *Call = Builder.CreateCall(F, {Ops[0], Ops[1]});
3677
3678 // Now bitcast to the wider result type.
3679 llvm::ScalableVectorType *Ty = getSVEType(TypeFlags);
3680 return EmitSVEReinterpret(Call, Ty);
3681}
3682
3684 ArrayRef<Value *> Ops, unsigned BuiltinID) {
3685 llvm::Type *OverloadedTy = getSVEType(TypeFlags);
3686 Function *F = CGM.getIntrinsic(BuiltinID, OverloadedTy);
3687 return Builder.CreateCall(F, {Ops[0], Builder.getInt32(0)});
3688}
3689
3692 unsigned BuiltinID) {
3693 auto *MemEltTy = SVEBuiltinMemEltTy(TypeFlags);
3694 auto *VectorTy = getSVEVectorForElementType(MemEltTy);
3695 auto *MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
3696
3697 Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy);
3698 Value *BasePtr = Ops[1];
3699
3700 // Implement the index operand if not omitted.
3701 if (Ops.size() > 3)
3702 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
3703
3704 Value *PrfOp = Ops.back();
3705
3706 llvm::Type *Tys[2] = {Predicate->getType(), BasePtr->getType()};
3707 Function *F = CGM.getIntrinsic(BuiltinID, Tys);
3708 return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
3709}
3710
3712 llvm::Type *ReturnTy,
3714 unsigned IntrinsicID,
3715 bool IsZExtReturn) {
3716 QualType LangPTy = E->getArg(1)->getType();
3717 llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
3718 LangPTy->castAs<PointerType>()->getPointeeType());
3719
3720 // Mfloat8 types is stored as a vector, so extra work
3721 // to extract sclar element type is necessary.
3722 if (MemEltTy->isVectorTy()) {
3723 assert(MemEltTy == FixedVectorType::get(Int8Ty, 1) &&
3724 "Only <1 x i8> expected");
3725 MemEltTy = cast<llvm::VectorType>(MemEltTy)->getElementType();
3726 }
3727
3728 // The vector type that is returned may be different from the
3729 // eventual type loaded from memory.
3730 auto VectorTy = cast<llvm::ScalableVectorType>(ReturnTy);
3731 llvm::ScalableVectorType *MemoryTy = nullptr;
3732 llvm::ScalableVectorType *PredTy = nullptr;
3733 bool IsQuadLoad = false;
3734 switch (IntrinsicID) {
3735 case Intrinsic::aarch64_sve_ld1uwq:
3736 case Intrinsic::aarch64_sve_ld1udq:
3737 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
3738 PredTy = llvm::ScalableVectorType::get(
3739 llvm::Type::getInt1Ty(getLLVMContext()), 1);
3740 IsQuadLoad = true;
3741 break;
3742 default:
3743 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
3744 PredTy = MemoryTy;
3745 break;
3746 }
3747
3748 Value *Predicate = EmitSVEPredicateCast(Ops[0], PredTy);
3749 Value *BasePtr = Ops[1];
3750
3751 // Does the load have an offset?
3752 if (Ops.size() > 2)
3753 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
3754
3755 llvm::Type *Tys[2] = {IsQuadLoad ? VectorTy : MemoryTy, BasePtr->getType()};
3756 Function *F = CGM.getIntrinsic(IntrinsicID, Tys);
3757 auto *Load = Builder.CreateCall(F, {Predicate, BasePtr});
3758 auto TBAAInfo = CGM.getTBAAAccessInfo(LangPTy->getPointeeType());
3759 CGM.DecorateInstructionWithTBAA(Load, TBAAInfo);
3760
3761 if (IsQuadLoad)
3762 return Load;
3763
3764 return IsZExtReturn ? Builder.CreateZExt(Load, VectorTy)
3765 : Builder.CreateSExt(Load, VectorTy);
3766}
3767
3770 unsigned IntrinsicID) {
3771 QualType LangPTy = E->getArg(1)->getType();
3772 llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
3773 LangPTy->castAs<PointerType>()->getPointeeType());
3774
3775 // Mfloat8 types is stored as a vector, so extra work
3776 // to extract sclar element type is necessary.
3777 if (MemEltTy->isVectorTy()) {
3778 assert(MemEltTy == FixedVectorType::get(Int8Ty, 1) &&
3779 "Only <1 x i8> expected");
3780 MemEltTy = cast<llvm::VectorType>(MemEltTy)->getElementType();
3781 }
3782
3783 // The vector type that is stored may be different from the
3784 // eventual type stored to memory.
3785 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType());
3786 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
3787
3788 auto PredTy = MemoryTy;
3789 auto AddrMemoryTy = MemoryTy;
3790 bool IsQuadStore = false;
3791
3792 switch (IntrinsicID) {
3793 case Intrinsic::aarch64_sve_st1wq:
3794 case Intrinsic::aarch64_sve_st1dq:
3795 AddrMemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
3796 PredTy =
3797 llvm::ScalableVectorType::get(IntegerType::get(getLLVMContext(), 1), 1);
3798 IsQuadStore = true;
3799 break;
3800 default:
3801 break;
3802 }
3803 Value *Predicate = EmitSVEPredicateCast(Ops[0], PredTy);
3804 Value *BasePtr = Ops[1];
3805
3806 // Does the store have an offset?
3807 if (Ops.size() == 4)
3808 BasePtr = Builder.CreateGEP(AddrMemoryTy, BasePtr, Ops[2]);
3809
3810 // Last value is always the data
3811 Value *Val =
3812 IsQuadStore ? Ops.back() : Builder.CreateTrunc(Ops.back(), MemoryTy);
3813
3814 llvm::Type *Tys[2] = {IsQuadStore ? VectorTy : MemoryTy, BasePtr->getType()};
3815 Function *F = CGM.getIntrinsic(IntrinsicID, Tys);
3816 auto *Store = Builder.CreateCall(F, {Val, Predicate, BasePtr});
3817 auto TBAAInfo = CGM.getTBAAAccessInfo(LangPTy->getPointeeType());
3818 CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
3819 return Store;
3820}
3821
3824 unsigned IntID) {
3825 Ops[2] = EmitSVEPredicateCast(
3827
3828 SmallVector<Value *> NewOps;
3829 NewOps.push_back(Ops[2]);
3830
3831 llvm::Value *BasePtr = Ops[3];
3832 llvm::Value *RealSlice = Ops[1];
3833 // If the intrinsic contains the vnum parameter, multiply it with the vector
3834 // size in bytes.
3835 if (Ops.size() == 5) {
3836 Function *StreamingVectorLength =
3837 CGM.getIntrinsic(Intrinsic::aarch64_sme_cntsd);
3838 llvm::Value *StreamingVectorLengthCall =
3839 Builder.CreateMul(Builder.CreateCall(StreamingVectorLength),
3840 llvm::ConstantInt::get(Int64Ty, 8), "svl",
3841 /* HasNUW */ true, /* HasNSW */ true);
3842 llvm::Value *Mulvl =
3843 Builder.CreateMul(StreamingVectorLengthCall, Ops[4], "mulvl");
3844 // The type of the ptr parameter is void *, so use Int8Ty here.
3845 BasePtr = Builder.CreateGEP(Int8Ty, Ops[3], Mulvl);
3846 RealSlice = Builder.CreateZExt(RealSlice, Int64Ty);
3847 RealSlice = Builder.CreateAdd(RealSlice, Ops[4]);
3848 RealSlice = Builder.CreateTrunc(RealSlice, Int32Ty);
3849 }
3850 NewOps.push_back(BasePtr);
3851 NewOps.push_back(Ops[0]);
3852 NewOps.push_back(RealSlice);
3853 Function *F = CGM.getIntrinsic(IntID, BasePtr->getType());
3854 return Builder.CreateCall(F, NewOps);
3855}
3856
3859 unsigned IntID) {
3860 auto *VecTy = getSVEType(TypeFlags);
3861 Function *F = CGM.getIntrinsic(IntID, VecTy);
3862 if (TypeFlags.isReadZA())
3863 Ops[1] = EmitSVEPredicateCast(Ops[1], VecTy);
3864 else if (TypeFlags.isWriteZA())
3865 Ops[2] = EmitSVEPredicateCast(Ops[2], VecTy);
3866 return Builder.CreateCall(F, Ops);
3867}
3868
3871 unsigned IntID) {
3872 // svzero_za() intrinsic zeros the entire za tile and has no paramters.
3873 if (Ops.size() == 0)
3874 Ops.push_back(llvm::ConstantInt::get(Int32Ty, 255));
3875 Function *F = CGM.getIntrinsic(IntID, {});
3876 return Builder.CreateCall(F, Ops);
3877}
3878
3881 unsigned IntID) {
3882 if (Ops.size() == 2)
3883 Ops.push_back(Builder.getInt32(0));
3884 else
3885 Ops[2] = Builder.CreateIntCast(Ops[2], Int32Ty, true);
3886 Function *F = CGM.getIntrinsic(IntID, Ops[1]->getType());
3887 return Builder.CreateCall(F, Ops);
3888}
3889
3890// Limit the usage of scalable llvm IR generated by the ACLE by using the
3891// sve dup.x intrinsic instead of IRBuilder::CreateVectorSplat.
3892Value *CodeGenFunction::EmitSVEDupX(Value *Scalar, llvm::Type *Ty) {
3893 return Builder.CreateVectorSplat(
3894 cast<llvm::VectorType>(Ty)->getElementCount(), Scalar);
3895}
3896
3898 if (auto *Ty = Scalar->getType(); Ty->isVectorTy()) {
3899#ifndef NDEBUG
3900 auto *VecTy = cast<llvm::VectorType>(Ty);
3901 ElementCount EC = VecTy->getElementCount();
3902 assert(EC.isScalar() && VecTy->getElementType() == Int8Ty &&
3903 "Only <1 x i8> expected");
3904#endif
3905 Scalar = Builder.CreateExtractElement(Scalar, uint64_t(0));
3906 }
3907 return EmitSVEDupX(Scalar, getSVEVectorForElementType(Scalar->getType()));
3908}
3909
3911 // FIXME: For big endian this needs an additional REV, or needs a separate
3912 // intrinsic that is code-generated as a no-op, because the LLVM bitcast
3913 // instruction is defined as 'bitwise' equivalent from memory point of
3914 // view (when storing/reloading), whereas the svreinterpret builtin
3915 // implements bitwise equivalent cast from register point of view.
3916 // LLVM CodeGen for a bitcast must add an explicit REV for big-endian.
3917
3918 if (auto *StructTy = dyn_cast<StructType>(Ty)) {
3919 Value *Tuple = llvm::PoisonValue::get(Ty);
3920
3921 for (unsigned I = 0; I < StructTy->getNumElements(); ++I) {
3922 Value *In = Builder.CreateExtractValue(Val, I);
3923 Value *Out = Builder.CreateBitCast(In, StructTy->getTypeAtIndex(I));
3924 Tuple = Builder.CreateInsertValue(Tuple, Out, I);
3925 }
3926
3927 return Tuple;
3928 }
3929
3930 return Builder.CreateBitCast(Val, Ty);
3931}
3932
3933static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty,
3935 auto *SplatZero = Constant::getNullValue(Ty);
3936 Ops.insert(Ops.begin(), SplatZero);
3937}
3938
3939static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty,
3941 auto *SplatUndef = UndefValue::get(Ty);
3942 Ops.insert(Ops.begin(), SplatUndef);
3943}
3944
3945SmallVector<llvm::Type *, 2>
3947 llvm::Type *ResultType,
3948 ArrayRef<Value *> Ops) {
3949 if (TypeFlags.isOverloadNone())
3950 return {};
3951
3952 llvm::Type *DefaultType = getSVEType(TypeFlags);
3953
3954 if (TypeFlags.isOverloadWhileOrMultiVecCvt())
3955 return {DefaultType, Ops[1]->getType()};
3956
3957 if (TypeFlags.isOverloadWhileRW())
3958 return {getSVEPredType(TypeFlags), Ops[0]->getType()};
3959
3960 if (TypeFlags.isOverloadDefaultAndOp0())
3961 return {DefaultType, Ops[0]->getType()};
3962
3963 if (TypeFlags.isOverloadFirstandLast())
3964 return {Ops[0]->getType(), Ops.back()->getType()};
3965
3966 if (TypeFlags.isReductionQV())
3967 return {ResultType, Ops[1]->getType()};
3968
3969 assert(TypeFlags.isOverloadDefault() && "Unexpected value for overloads");
3970 return {DefaultType};
3971}
3972
3974 ArrayRef<Value *> Ops) {
3975 assert((TypeFlags.isTupleSet() || TypeFlags.isTupleGet()) &&
3976 "Expects TypleFlags.isTupleSet() or TypeFlags.isTupleGet()");
3977 unsigned Idx = cast<ConstantInt>(Ops[1])->getZExtValue();
3978
3979 if (TypeFlags.isTupleSet())
3980 return Builder.CreateInsertValue(Ops[0], Ops[2], Idx);
3981 return Builder.CreateExtractValue(Ops[0], Idx);
3982}
3983
3985 llvm::Type *Ty,
3986 ArrayRef<Value *> Ops) {
3987 assert(TypeFlags.isTupleCreate() && "Expects TypleFlag isTupleCreate");
3988
3989 Value *Tuple = llvm::PoisonValue::get(Ty);
3990 for (unsigned Idx = 0; Idx < Ops.size(); Idx++)
3991 Tuple = Builder.CreateInsertValue(Tuple, Ops[Idx], Idx);
3992
3993 return Tuple;
3994}
3995
3997 unsigned BuiltinID, const CallExpr *E, SmallVectorImpl<Value *> &Ops,
3998 SVETypeFlags TypeFlags) {
3999 // Find out if any arguments are required to be integer constant expressions.
4000 unsigned ICEArguments = 0;
4002 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
4003 assert(Error == ASTContext::GE_None && "Should not codegen an error");
4004
4005 // Tuple set/get only requires one insert/extract vector, which is
4006 // created by EmitSVETupleSetOrGet.
4007 bool IsTupleGetOrSet = TypeFlags.isTupleSet() || TypeFlags.isTupleGet();
4008
4009 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
4010 bool IsICE = ICEArguments & (1 << i);
4011 Value *Arg = EmitScalarExpr(E->getArg(i));
4012
4013 if (IsICE) {
4014 // If this is required to be a constant, constant fold it so that we know
4015 // that the generated intrinsic gets a ConstantInt.
4016 std::optional<llvm::APSInt> Result =
4018 assert(Result && "Expected argument to be a constant");
4019
4020 // Immediates for SVE llvm intrinsics are always 32bit. We can safely
4021 // truncate because the immediate has been range checked and no valid
4022 // immediate requires more than a handful of bits.
4023 *Result = Result->extOrTrunc(32);
4024 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), *Result));
4025 continue;
4026 }
4027
4028 if (isa<StructType>(Arg->getType()) && !IsTupleGetOrSet) {
4029 for (unsigned I = 0; I < Arg->getType()->getStructNumElements(); ++I)
4030 Ops.push_back(Builder.CreateExtractValue(Arg, I));
4031
4032 continue;
4033 }
4034
4035 Ops.push_back(Arg);
4036 }
4037}
4038
4040 const CallExpr *E) {
4041 llvm::Type *Ty = ConvertType(E->getType());
4042 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
4043 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
4044 Value *Val = EmitScalarExpr(E->getArg(0));
4045 return EmitSVEReinterpret(Val, Ty);
4046 }
4047
4048 auto *Builtin =
4051
4053 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4054 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags);
4055
4056 if (TypeFlags.isLoad())
4057 return EmitSVEMaskedLoad(E, Ty, Ops, Builtin->LLVMIntrinsic,
4058 TypeFlags.isZExtReturn());
4059 if (TypeFlags.isStore())
4060 return EmitSVEMaskedStore(E, Ops, Builtin->LLVMIntrinsic);
4061 if (TypeFlags.isGatherLoad())
4062 return EmitSVEGatherLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4063 if (TypeFlags.isScatterStore())
4064 return EmitSVEScatterStore(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4065 if (TypeFlags.isPrefetch())
4066 return EmitSVEPrefetchLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4067 if (TypeFlags.isGatherPrefetch())
4068 return EmitSVEGatherPrefetch(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4069 if (TypeFlags.isStructLoad())
4070 return EmitSVEStructLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4071 if (TypeFlags.isStructStore())
4072 return EmitSVEStructStore(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4073 if (TypeFlags.isTupleSet() || TypeFlags.isTupleGet())
4074 return EmitSVETupleSetOrGet(TypeFlags, Ops);
4075 if (TypeFlags.isTupleCreate())
4076 return EmitSVETupleCreate(TypeFlags, Ty, Ops);
4077 if (TypeFlags.isUndef())
4078 return UndefValue::get(Ty);
4079
4080 // Handle built-ins for which there is a corresponding LLVM Intrinsic.
4081 // -------------------------------------------------------------------
4082 if (Builtin->LLVMIntrinsic != 0) {
4083 // Emit set FPMR for intrinsics that require it
4084 if (TypeFlags.setsFPMR())
4085 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_set_fpmr),
4086 Ops.pop_back_val());
4087 if (TypeFlags.getMergeType() == SVETypeFlags::MergeZeroExp)
4089
4090 if (TypeFlags.getMergeType() == SVETypeFlags::MergeAnyExp)
4092
4093 // Some ACLE builtins leave out the argument to specify the predicate
4094 // pattern, which is expected to be expanded to an SV_ALL pattern.
4095 if (TypeFlags.isAppendSVALL())
4096 Ops.push_back(Builder.getInt32(/*SV_ALL*/ 31));
4097 if (TypeFlags.isInsertOp1SVALL())
4098 Ops.insert(&Ops[1], Builder.getInt32(/*SV_ALL*/ 31));
4099
4100 // Predicates must match the main datatype.
4101 for (Value *&Op : Ops)
4102 if (auto PredTy = dyn_cast<llvm::VectorType>(Op->getType()))
4103 if (PredTy->getElementType()->isIntegerTy(1))
4104 Op = EmitSVEPredicateCast(Op, getSVEType(TypeFlags));
4105
4106 // Splat scalar operand to vector (intrinsics with _n infix)
4107 if (TypeFlags.hasSplatOperand()) {
4108 unsigned OpNo = TypeFlags.getSplatOperand();
4109 Ops[OpNo] = EmitSVEDupX(Ops[OpNo]);
4110 }
4111
4112 if (TypeFlags.isReverseCompare())
4113 std::swap(Ops[1], Ops[2]);
4114 else if (TypeFlags.isReverseUSDOT())
4115 std::swap(Ops[1], Ops[2]);
4116 else if (TypeFlags.isReverseMergeAnyBinOp() &&
4117 TypeFlags.getMergeType() == SVETypeFlags::MergeAny)
4118 std::swap(Ops[1], Ops[2]);
4119 else if (TypeFlags.isReverseMergeAnyAccOp() &&
4120 TypeFlags.getMergeType() == SVETypeFlags::MergeAny)
4121 std::swap(Ops[1], Ops[3]);
4122
4123 // Predicated intrinsics with _z suffix need a select w/ zeroinitializer.
4124 if (TypeFlags.getMergeType() == SVETypeFlags::MergeZero) {
4125 llvm::Type *OpndTy = Ops[1]->getType();
4126 auto *SplatZero = Constant::getNullValue(OpndTy);
4127 Ops[1] = Builder.CreateSelect(Ops[0], Ops[1], SplatZero);
4128 }
4129
4130 Function *F = CGM.getIntrinsic(Builtin->LLVMIntrinsic,
4131 getSVEOverloadTypes(TypeFlags, Ty, Ops));
4132 Value *Call = Builder.CreateCall(F, Ops);
4133
4134 if (Call->getType() == Ty)
4135 return Call;
4136
4137 // Predicate results must be converted to svbool_t.
4138 if (auto PredTy = dyn_cast<llvm::ScalableVectorType>(Ty))
4139 return EmitSVEPredicateCast(Call, PredTy);
4140 if (auto PredTupleTy = dyn_cast<llvm::StructType>(Ty))
4141 return EmitSVEPredicateTupleCast(Call, PredTupleTy);
4142
4143 llvm_unreachable("unsupported element count!");
4144 }
4145
4146 switch (BuiltinID) {
4147 default:
4148 return nullptr;
4149
4150 case SVE::BI__builtin_sve_svreinterpret_b: {
4151 Function *CastFromSVCountF =
4152 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svcount);
4153 return Builder.CreateCall(CastFromSVCountF, Ops[0]);
4154 }
4155 case SVE::BI__builtin_sve_svreinterpret_c: {
4156 Function *CastToSVCountF =
4157 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_to_svcount);
4158 return Builder.CreateCall(CastToSVCountF, Ops[0]);
4159 }
4160
4161 case SVE::BI__builtin_sve_svpsel_lane_b8:
4162 case SVE::BI__builtin_sve_svpsel_lane_b16:
4163 case SVE::BI__builtin_sve_svpsel_lane_b32:
4164 case SVE::BI__builtin_sve_svpsel_lane_b64:
4165 case SVE::BI__builtin_sve_svpsel_lane_c8:
4166 case SVE::BI__builtin_sve_svpsel_lane_c16:
4167 case SVE::BI__builtin_sve_svpsel_lane_c32:
4168 case SVE::BI__builtin_sve_svpsel_lane_c64: {
4169 auto OverloadedTy = getSVEType(SVETypeFlags(Builtin->TypeModifier));
4170 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_psel,
4171 {Ops[0]->getType(), OverloadedTy});
4172 llvm::Value *Ops1 = EmitSVEPredicateCast(Ops[1], OverloadedTy);
4173 return Builder.CreateCall(F, {Ops[0], Ops1, Ops[2]});
4174 }
4175 case SVE::BI__builtin_sve_svmov_b_z: {
4176 // svmov_b_z(pg, op) <=> svand_b_z(pg, op, op)
4177 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4178 llvm::Type* OverloadedTy = getSVEType(TypeFlags);
4179 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_and_z, OverloadedTy);
4180 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
4181 }
4182
4183 case SVE::BI__builtin_sve_svnot_b_z: {
4184 // svnot_b_z(pg, op) <=> sveor_b_z(pg, op, pg)
4185 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4186 llvm::Type* OverloadedTy = getSVEType(TypeFlags);
4187 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_eor_z, OverloadedTy);
4188 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
4189 }
4190
4191 case SVE::BI__builtin_sve_svmovlb_u16:
4192 case SVE::BI__builtin_sve_svmovlb_u32:
4193 case SVE::BI__builtin_sve_svmovlb_u64:
4194 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
4195
4196 case SVE::BI__builtin_sve_svmovlb_s16:
4197 case SVE::BI__builtin_sve_svmovlb_s32:
4198 case SVE::BI__builtin_sve_svmovlb_s64:
4199 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
4200
4201 case SVE::BI__builtin_sve_svmovlt_u16:
4202 case SVE::BI__builtin_sve_svmovlt_u32:
4203 case SVE::BI__builtin_sve_svmovlt_u64:
4204 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
4205
4206 case SVE::BI__builtin_sve_svmovlt_s16:
4207 case SVE::BI__builtin_sve_svmovlt_s32:
4208 case SVE::BI__builtin_sve_svmovlt_s64:
4209 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
4210
4211 case SVE::BI__builtin_sve_svpmullt_u16:
4212 case SVE::BI__builtin_sve_svpmullt_u64:
4213 case SVE::BI__builtin_sve_svpmullt_n_u16:
4214 case SVE::BI__builtin_sve_svpmullt_n_u64:
4215 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
4216
4217 case SVE::BI__builtin_sve_svpmullb_u16:
4218 case SVE::BI__builtin_sve_svpmullb_u64:
4219 case SVE::BI__builtin_sve_svpmullb_n_u16:
4220 case SVE::BI__builtin_sve_svpmullb_n_u64:
4221 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
4222
4223 case SVE::BI__builtin_sve_svdup_n_b8:
4224 case SVE::BI__builtin_sve_svdup_n_b16:
4225 case SVE::BI__builtin_sve_svdup_n_b32:
4226 case SVE::BI__builtin_sve_svdup_n_b64: {
4227 llvm::ScalableVectorType *OverloadedTy = getSVEType(TypeFlags);
4228 Value *Dup = EmitSVEDupX(Ops[0], OverloadedTy);
4230 }
4231
4232 case SVE::BI__builtin_sve_svdupq_n_b8:
4233 case SVE::BI__builtin_sve_svdupq_n_b16:
4234 case SVE::BI__builtin_sve_svdupq_n_b32:
4235 case SVE::BI__builtin_sve_svdupq_n_b64:
4236 case SVE::BI__builtin_sve_svdupq_n_u8:
4237 case SVE::BI__builtin_sve_svdupq_n_s8:
4238 case SVE::BI__builtin_sve_svdupq_n_u64:
4239 case SVE::BI__builtin_sve_svdupq_n_f64:
4240 case SVE::BI__builtin_sve_svdupq_n_s64:
4241 case SVE::BI__builtin_sve_svdupq_n_u16:
4242 case SVE::BI__builtin_sve_svdupq_n_f16:
4243 case SVE::BI__builtin_sve_svdupq_n_bf16:
4244 case SVE::BI__builtin_sve_svdupq_n_s16:
4245 case SVE::BI__builtin_sve_svdupq_n_u32:
4246 case SVE::BI__builtin_sve_svdupq_n_f32:
4247 case SVE::BI__builtin_sve_svdupq_n_s32: {
4248 // These builtins are implemented by storing each element to an array and using
4249 // ld1rq to materialize a vector.
4250 unsigned NumOpnds = Ops.size();
4251
4252 bool IsBoolTy =
4253 cast<llvm::VectorType>(Ty)->getElementType()->isIntegerTy(1);
4254
4255 // For svdupq_n_b* the element type of is an integer of type 128/numelts,
4256 // so that the compare can use the width that is natural for the expected
4257 // number of predicate lanes.
4258 llvm::Type *EltTy = Ops[0]->getType();
4259 if (IsBoolTy)
4260 EltTy = IntegerType::get(getLLVMContext(), SVEBitsPerBlock / NumOpnds);
4261
4263 for (unsigned I = 0; I < NumOpnds; ++I)
4264 VecOps.push_back(Builder.CreateZExt(Ops[I], EltTy));
4265 Value *Vec = BuildVector(VecOps);
4266
4267 llvm::Type *OverloadedTy = getSVEVectorForElementType(EltTy);
4268 Value *InsertSubVec = Builder.CreateInsertVector(
4269 OverloadedTy, PoisonValue::get(OverloadedTy), Vec, uint64_t(0));
4270
4271 Function *F =
4272 CGM.getIntrinsic(Intrinsic::aarch64_sve_dupq_lane, OverloadedTy);
4273 Value *DupQLane =
4274 Builder.CreateCall(F, {InsertSubVec, Builder.getInt64(0)});
4275
4276 if (!IsBoolTy)
4277 return DupQLane;
4278
4279 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4280 Constant *Pred = ConstantInt::getTrue(getSVEPredType(TypeFlags));
4281
4282 // For svdupq_n_b* we need to add an additional 'cmpne' with '0'.
4283 F = CGM.getIntrinsic(NumOpnds == 2 ? Intrinsic::aarch64_sve_cmpne
4284 : Intrinsic::aarch64_sve_cmpne_wide,
4285 OverloadedTy);
4286 Value *Call = Builder.CreateCall(
4287 F, {Pred, DupQLane, EmitSVEDupX(Builder.getInt64(0))});
4289 }
4290
4291 case SVE::BI__builtin_sve_svpfalse_b:
4292 return ConstantInt::getFalse(Ty);
4293
4294 case SVE::BI__builtin_sve_svpfalse_c:
4295 return Constant::getNullValue(Ty);
4296
4297 case SVE::BI__builtin_sve_svlen_bf16:
4298 case SVE::BI__builtin_sve_svlen_f16:
4299 case SVE::BI__builtin_sve_svlen_f32:
4300 case SVE::BI__builtin_sve_svlen_f64:
4301 case SVE::BI__builtin_sve_svlen_s8:
4302 case SVE::BI__builtin_sve_svlen_s16:
4303 case SVE::BI__builtin_sve_svlen_s32:
4304 case SVE::BI__builtin_sve_svlen_s64:
4305 case SVE::BI__builtin_sve_svlen_u8:
4306 case SVE::BI__builtin_sve_svlen_u16:
4307 case SVE::BI__builtin_sve_svlen_u32:
4308 case SVE::BI__builtin_sve_svlen_u64: {
4309 SVETypeFlags TF(Builtin->TypeModifier);
4310 return Builder.CreateElementCount(Ty, getSVEType(TF)->getElementCount());
4311 }
4312
4313 case SVE::BI__builtin_sve_svtbl2_u8:
4314 case SVE::BI__builtin_sve_svtbl2_s8:
4315 case SVE::BI__builtin_sve_svtbl2_u16:
4316 case SVE::BI__builtin_sve_svtbl2_s16:
4317 case SVE::BI__builtin_sve_svtbl2_u32:
4318 case SVE::BI__builtin_sve_svtbl2_s32:
4319 case SVE::BI__builtin_sve_svtbl2_u64:
4320 case SVE::BI__builtin_sve_svtbl2_s64:
4321 case SVE::BI__builtin_sve_svtbl2_f16:
4322 case SVE::BI__builtin_sve_svtbl2_bf16:
4323 case SVE::BI__builtin_sve_svtbl2_f32:
4324 case SVE::BI__builtin_sve_svtbl2_f64: {
4325 SVETypeFlags TF(Builtin->TypeModifier);
4326 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_tbl2, getSVEType(TF));
4327 return Builder.CreateCall(F, Ops);
4328 }
4329
4330 case SVE::BI__builtin_sve_svset_neonq_s8:
4331 case SVE::BI__builtin_sve_svset_neonq_s16:
4332 case SVE::BI__builtin_sve_svset_neonq_s32:
4333 case SVE::BI__builtin_sve_svset_neonq_s64:
4334 case SVE::BI__builtin_sve_svset_neonq_u8:
4335 case SVE::BI__builtin_sve_svset_neonq_u16:
4336 case SVE::BI__builtin_sve_svset_neonq_u32:
4337 case SVE::BI__builtin_sve_svset_neonq_u64:
4338 case SVE::BI__builtin_sve_svset_neonq_f16:
4339 case SVE::BI__builtin_sve_svset_neonq_f32:
4340 case SVE::BI__builtin_sve_svset_neonq_f64:
4341 case SVE::BI__builtin_sve_svset_neonq_bf16:
4342 case SVE::BI__builtin_sve_svset_neonq_mf8: {
4343 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], uint64_t(0));
4344 }
4345
4346 case SVE::BI__builtin_sve_svget_neonq_s8:
4347 case SVE::BI__builtin_sve_svget_neonq_s16:
4348 case SVE::BI__builtin_sve_svget_neonq_s32:
4349 case SVE::BI__builtin_sve_svget_neonq_s64:
4350 case SVE::BI__builtin_sve_svget_neonq_u8:
4351 case SVE::BI__builtin_sve_svget_neonq_u16:
4352 case SVE::BI__builtin_sve_svget_neonq_u32:
4353 case SVE::BI__builtin_sve_svget_neonq_u64:
4354 case SVE::BI__builtin_sve_svget_neonq_f16:
4355 case SVE::BI__builtin_sve_svget_neonq_f32:
4356 case SVE::BI__builtin_sve_svget_neonq_f64:
4357 case SVE::BI__builtin_sve_svget_neonq_bf16:
4358 case SVE::BI__builtin_sve_svget_neonq_mf8: {
4359 return Builder.CreateExtractVector(Ty, Ops[0], uint64_t(0));
4360 }
4361
4362 case SVE::BI__builtin_sve_svdup_neonq_s8:
4363 case SVE::BI__builtin_sve_svdup_neonq_s16:
4364 case SVE::BI__builtin_sve_svdup_neonq_s32:
4365 case SVE::BI__builtin_sve_svdup_neonq_s64:
4366 case SVE::BI__builtin_sve_svdup_neonq_u8:
4367 case SVE::BI__builtin_sve_svdup_neonq_u16:
4368 case SVE::BI__builtin_sve_svdup_neonq_u32:
4369 case SVE::BI__builtin_sve_svdup_neonq_u64:
4370 case SVE::BI__builtin_sve_svdup_neonq_f16:
4371 case SVE::BI__builtin_sve_svdup_neonq_f32:
4372 case SVE::BI__builtin_sve_svdup_neonq_f64:
4373 case SVE::BI__builtin_sve_svdup_neonq_bf16:
4374 case SVE::BI__builtin_sve_svdup_neonq_mf8: {
4375 Value *Insert = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0],
4376 uint64_t(0));
4377 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
4378 {Insert, Builder.getInt64(0)});
4379 }
4380 }
4381
4382 /// Should not happen
4383 return nullptr;
4384}
4385
4386static void swapCommutativeSMEOperands(unsigned BuiltinID,
4388 unsigned MultiVec;
4389 switch (BuiltinID) {
4390 default:
4391 return;
4392 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1:
4393 MultiVec = 1;
4394 break;
4395 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2:
4396 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2:
4397 MultiVec = 2;
4398 break;
4399 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4:
4400 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4:
4401 MultiVec = 4;
4402 break;
4403 }
4404
4405 if (MultiVec > 0)
4406 for (unsigned I = 0; I < MultiVec; ++I)
4407 std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]);
4408}
4409
4411 const CallExpr *E) {
4412 auto *Builtin =
4415
4417 SVETypeFlags TypeFlags(Builtin->TypeModifier);
4418 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags);
4419
4420 if (TypeFlags.isLoad() || TypeFlags.isStore())
4421 return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4422 if (TypeFlags.isReadZA() || TypeFlags.isWriteZA())
4423 return EmitSMEReadWrite(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4424 if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
4425 BuiltinID == SME::BI__builtin_sme_svzero_za)
4426 return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4427 if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
4428 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za ||
4429 BuiltinID == SME::BI__builtin_sme_svldr_za ||
4430 BuiltinID == SME::BI__builtin_sme_svstr_za)
4431 return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic);
4432
4433 // Emit set FPMR for intrinsics that require it
4434 if (TypeFlags.setsFPMR())
4435 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_set_fpmr),
4436 Ops.pop_back_val());
4437 // Handle builtins which require their multi-vector operands to be swapped
4438 swapCommutativeSMEOperands(BuiltinID, Ops);
4439
4440 auto isCntsBuiltin = [&]() {
4441 switch (BuiltinID) {
4442 default:
4443 return 0;
4444 case SME::BI__builtin_sme_svcntsb:
4445 return 8;
4446 case SME::BI__builtin_sme_svcntsh:
4447 return 4;
4448 case SME::BI__builtin_sme_svcntsw:
4449 return 2;
4450 }
4451 };
4452
4453 if (auto Mul = isCntsBuiltin()) {
4454 llvm::Value *Cntd =
4455 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_sme_cntsd));
4456 return Builder.CreateMul(Cntd, llvm::ConstantInt::get(Int64Ty, Mul),
4457 "mulsvl", /* HasNUW */ true, /* HasNSW */ true);
4458 }
4459
4460 // Should not happen!
4461 if (Builtin->LLVMIntrinsic == 0)
4462 return nullptr;
4463
4464 // Predicates must match the main datatype.
4465 for (Value *&Op : Ops)
4466 if (auto PredTy = dyn_cast<llvm::VectorType>(Op->getType()))
4467 if (PredTy->getElementType()->isIntegerTy(1))
4468 Op = EmitSVEPredicateCast(Op, getSVEType(TypeFlags));
4469
4470 if (BuiltinID == SME::BI__builtin_sme_svldr_zt ||
4471 BuiltinID == SME::BI__builtin_sme_svstr_zt) {
4472 Function *F = CGM.getIntrinsic(Builtin->LLVMIntrinsic, Ops[1]->getType());
4473 return Builder.CreateCall(F, Ops);
4474 }
4475
4476 Function *F =
4477 TypeFlags.isOverloadNone()
4478 ? CGM.getIntrinsic(Builtin->LLVMIntrinsic)
4479 : CGM.getIntrinsic(Builtin->LLVMIntrinsic, {getSVEType(TypeFlags)});
4480
4481 return Builder.CreateCall(F, Ops);
4482}
4483
4484/// Helper for the read/write/add/inc X18 builtins: read the X18 register and
4485/// return it as an i8 pointer.
4487 LLVMContext &Context = CGF.CGM.getLLVMContext();
4488 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")};
4489 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
4490 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
4491 llvm::Function *F =
4492 CGF.CGM.getIntrinsic(Intrinsic::read_register, {CGF.Int64Ty});
4493 llvm::Value *X18 = CGF.Builder.CreateCall(F, Metadata);
4494 return CGF.Builder.CreateIntToPtr(X18, CGF.Int8PtrTy);
4495}
4496
4498 const CallExpr *E,
4499 llvm::Triple::ArchType Arch) {
4500 if (BuiltinID >= clang::AArch64::FirstSVEBuiltin &&
4501 BuiltinID <= clang::AArch64::LastSVEBuiltin)
4502 return EmitAArch64SVEBuiltinExpr(BuiltinID, E);
4503
4504 if (BuiltinID >= clang::AArch64::FirstSMEBuiltin &&
4505 BuiltinID <= clang::AArch64::LastSMEBuiltin)
4506 return EmitAArch64SMEBuiltinExpr(BuiltinID, E);
4507
4508 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
4509 return EmitAArch64CpuSupports(E);
4510
4511 unsigned HintID = static_cast<unsigned>(-1);
4512 switch (BuiltinID) {
4513 default: break;
4514 case clang::AArch64::BI__builtin_arm_nop:
4515 HintID = 0;
4516 break;
4517 case clang::AArch64::BI__builtin_arm_yield:
4518 case clang::AArch64::BI__yield:
4519 HintID = 1;
4520 break;
4521 case clang::AArch64::BI__builtin_arm_wfe:
4522 case clang::AArch64::BI__wfe:
4523 HintID = 2;
4524 break;
4525 case clang::AArch64::BI__builtin_arm_wfi:
4526 case clang::AArch64::BI__wfi:
4527 HintID = 3;
4528 break;
4529 case clang::AArch64::BI__builtin_arm_sev:
4530 case clang::AArch64::BI__sev:
4531 HintID = 4;
4532 break;
4533 case clang::AArch64::BI__builtin_arm_sevl:
4534 case clang::AArch64::BI__sevl:
4535 HintID = 5;
4536 break;
4537 }
4538
4539 if (HintID != static_cast<unsigned>(-1)) {
4540 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint);
4541 return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
4542 }
4543
4544 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
4545 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_break);
4546 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4547 return Builder.CreateCall(F, Builder.CreateZExt(Arg, CGM.Int32Ty));
4548 }
4549
4550 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
4551 // Create call to __arm_sme_state and store the results to the two pointers.
4552 CallInst *CI = EmitRuntimeCall(CGM.CreateRuntimeFunction(
4553 llvm::FunctionType::get(StructType::get(CGM.Int64Ty, CGM.Int64Ty), {},
4554 false),
4555 "__arm_sme_state"));
4556 auto Attrs = AttributeList().addFnAttribute(getLLVMContext(),
4557 "aarch64_pstate_sm_compatible");
4558 CI->setAttributes(Attrs);
4559 CI->setCallingConv(
4560 llvm::CallingConv::
4561 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
4562 Builder.CreateStore(Builder.CreateExtractValue(CI, 0),
4564 return Builder.CreateStore(Builder.CreateExtractValue(CI, 1),
4566 }
4567
4568 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
4569 assert((getContext().getTypeSize(E->getType()) == 32) &&
4570 "rbit of unusual size!");
4571 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4572 return Builder.CreateCall(
4573 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
4574 }
4575 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
4576 assert((getContext().getTypeSize(E->getType()) == 64) &&
4577 "rbit of unusual size!");
4578 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4579 return Builder.CreateCall(
4580 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
4581 }
4582
4583 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
4584 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
4585 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4586 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Arg->getType());
4587 Value *Res = Builder.CreateCall(F, {Arg, Builder.getInt1(false)});
4588 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
4589 Res = Builder.CreateTrunc(Res, Builder.getInt32Ty());
4590 return Res;
4591 }
4592
4593 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
4594 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4595 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls), Arg,
4596 "cls");
4597 }
4598 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
4599 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4600 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls64), Arg,
4601 "cls");
4602 }
4603
4604 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
4605 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
4606 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4607 llvm::Type *Ty = Arg->getType();
4608 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint32z, Ty),
4609 Arg, "frint32z");
4610 }
4611
4612 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
4613 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
4614 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4615 llvm::Type *Ty = Arg->getType();
4616 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint64z, Ty),
4617 Arg, "frint64z");
4618 }
4619
4620 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
4621 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
4622 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4623 llvm::Type *Ty = Arg->getType();
4624 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint32x, Ty),
4625 Arg, "frint32x");
4626 }
4627
4628 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
4629 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
4630 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4631 llvm::Type *Ty = Arg->getType();
4632 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint64x, Ty),
4633 Arg, "frint64x");
4634 }
4635
4636 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
4637 assert((getContext().getTypeSize(E->getType()) == 32) &&
4638 "__jcvt of unusual size!");
4639 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
4640 return Builder.CreateCall(
4641 CGM.getIntrinsic(Intrinsic::aarch64_fjcvtzs), Arg);
4642 }
4643
4644 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
4645 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
4646 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
4647 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
4648 llvm::Value *MemAddr = EmitScalarExpr(E->getArg(0));
4649 llvm::Value *ValPtr = EmitScalarExpr(E->getArg(1));
4650
4651 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
4652 // Load from the address via an LLVM intrinsic, receiving a
4653 // tuple of 8 i64 words, and store each one to ValPtr.
4654 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_ld64b);
4655 llvm::Value *Val = Builder.CreateCall(F, MemAddr);
4656 llvm::Value *ToRet;
4657 for (size_t i = 0; i < 8; i++) {
4658 llvm::Value *ValOffsetPtr =
4659 Builder.CreateGEP(Int64Ty, ValPtr, Builder.getInt32(i));
4660 Address Addr =
4661 Address(ValOffsetPtr, Int64Ty, CharUnits::fromQuantity(8));
4662 ToRet = Builder.CreateStore(Builder.CreateExtractValue(Val, i), Addr);
4663 }
4664 return ToRet;
4665 }
4666
4667 // Load 8 i64 words from ValPtr, and store them to the address
4668 // via an LLVM intrinsic.
4670 Args.push_back(MemAddr);
4671 for (size_t i = 0; i < 8; i++) {
4672 llvm::Value *ValOffsetPtr =
4673 Builder.CreateGEP(Int64Ty, ValPtr, Builder.getInt32(i));
4674 Address Addr = Address(ValOffsetPtr, Int64Ty, CharUnits::fromQuantity(8));
4675 Args.push_back(Builder.CreateLoad(Addr));
4676 }
4677
4678 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
4679 ? Intrinsic::aarch64_st64b
4680 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
4681 ? Intrinsic::aarch64_st64bv
4682 : Intrinsic::aarch64_st64bv0);
4683 Function *F = CGM.getIntrinsic(Intr);
4684 return Builder.CreateCall(F, Args);
4685 }
4686
4687 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
4688 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
4689
4690 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
4691 ? Intrinsic::aarch64_rndr
4692 : Intrinsic::aarch64_rndrrs);
4693 Function *F = CGM.getIntrinsic(Intr);
4694 llvm::Value *Val = Builder.CreateCall(F);
4695 Value *RandomValue = Builder.CreateExtractValue(Val, 0);
4696 Value *Status = Builder.CreateExtractValue(Val, 1);
4697
4698 Address MemAddress = EmitPointerWithAlignment(E->getArg(0));
4699 Builder.CreateStore(RandomValue, MemAddress);
4700 Status = Builder.CreateZExt(Status, Int32Ty);
4701 return Status;
4702 }
4703
4704 if (BuiltinID == clang::AArch64::BI__clear_cache) {
4705 Value *Begin = EmitScalarExpr(E->getArg(0));
4706 Value *End = EmitScalarExpr(E->getArg(1));
4707 Function *F = CGM.getIntrinsic(Intrinsic::clear_cache, {CGM.DefaultPtrTy});
4708 return Builder.CreateCall(F, {Begin, End});
4709 }
4710
4711 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
4712 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
4713 getContext().getTypeSize(E->getType()) == 128) {
4714 Function *F =
4715 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
4716 ? Intrinsic::aarch64_ldaxp
4717 : Intrinsic::aarch64_ldxp);
4718
4719 Value *LdPtr = EmitScalarExpr(E->getArg(0));
4720 Value *Val = Builder.CreateCall(F, LdPtr, "ldxp");
4721
4722 Value *Val0 = Builder.CreateExtractValue(Val, 1);
4723 Value *Val1 = Builder.CreateExtractValue(Val, 0);
4724 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
4725 Val0 = Builder.CreateZExt(Val0, Int128Ty);
4726 Val1 = Builder.CreateZExt(Val1, Int128Ty);
4727
4728 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
4729 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
4730 Val = Builder.CreateOr(Val, Val1);
4731 return Builder.CreateBitCast(Val, ConvertType(E->getType()));
4732 } else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
4733 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
4734 Value *LoadAddr = EmitScalarExpr(E->getArg(0));
4735
4736 QualType Ty = E->getType();
4737 llvm::Type *RealResTy = ConvertType(Ty);
4738 llvm::Type *IntTy =
4739 llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
4740
4741 Function *F =
4742 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
4743 ? Intrinsic::aarch64_ldaxr
4744 : Intrinsic::aarch64_ldxr,
4745 DefaultPtrTy);
4746 CallInst *Val = Builder.CreateCall(F, LoadAddr, "ldxr");
4747 Val->addParamAttr(
4748 0, Attribute::get(getLLVMContext(), Attribute::ElementType, IntTy));
4749
4750 if (RealResTy->isPointerTy())
4751 return Builder.CreateIntToPtr(Val, RealResTy);
4752
4753 llvm::Type *IntResTy = llvm::IntegerType::get(
4754 getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
4755 return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
4756 RealResTy);
4757 }
4758
4759 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
4760 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
4761 getContext().getTypeSize(E->getArg(0)->getType()) == 128) {
4762 Function *F =
4763 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_stlex
4764 ? Intrinsic::aarch64_stlxp
4765 : Intrinsic::aarch64_stxp);
4766 llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty);
4767
4769 EmitAnyExprToMem(E->getArg(0), Tmp, Qualifiers(), /*init*/ true);
4770
4771 Tmp = Tmp.withElementType(STy);
4772 llvm::Value *Val = Builder.CreateLoad(Tmp);
4773
4774 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
4775 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
4776 Value *StPtr = EmitScalarExpr(E->getArg(1));
4777 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp");
4778 }
4779
4780 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
4781 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
4782 Value *StoreVal = EmitScalarExpr(E->getArg(0));
4783 Value *StoreAddr = EmitScalarExpr(E->getArg(1));
4784
4785 QualType Ty = E->getArg(0)->getType();
4786 llvm::Type *StoreTy =
4787 llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
4788
4789 if (StoreVal->getType()->isPointerTy())
4790 StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty);
4791 else {
4792 llvm::Type *IntTy = llvm::IntegerType::get(
4794 CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
4795 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
4796 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
4797 }
4798
4799 Function *F =
4800 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_stlex
4801 ? Intrinsic::aarch64_stlxr
4802 : Intrinsic::aarch64_stxr,
4803 StoreAddr->getType());
4804 CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
4805 CI->addParamAttr(
4806 1, Attribute::get(getLLVMContext(), Attribute::ElementType, StoreTy));
4807 return CI;
4808 }
4809
4810 if (BuiltinID == clang::AArch64::BI__getReg ||
4811 BuiltinID == clang::AArch64::BI__setReg) {
4813 if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
4814 llvm_unreachable("Sema will ensure that the parameter is constant");
4815
4816 llvm::APSInt Value = Result.Val.getInt();
4817 LLVMContext &Context = CGM.getLLVMContext();
4818 std::string Reg = Value == 31 ? "sp" : "x" + toString(Value, 10);
4819
4820 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
4821 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
4822 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
4823
4824 CallInst *CI;
4825 if (BuiltinID == clang::AArch64::BI__getReg) {
4826 llvm::Function *F =
4827 CGM.getIntrinsic(Intrinsic::read_volatile_register, {Int64Ty});
4828 CI = Builder.CreateCall(F, Metadata);
4829 } else {
4830 llvm::Function *F =
4831 CGM.getIntrinsic(Intrinsic::write_volatile_register, {Int64Ty});
4832 CI = Builder.CreateCall(F, {Metadata, EmitScalarExpr(E->getArg(1))});
4833 }
4834 return CI;
4835 }
4836
4837 if (BuiltinID == clang::AArch64::BI__getRegFp ||
4838 BuiltinID == clang::AArch64::BI__setRegFp) {
4840 if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
4841 llvm_unreachable("Sema will ensure that the parameter is constant");
4842
4843 llvm::APSInt Value = Result.Val.getInt();
4844 LLVMContext &Context = CGM.getLLVMContext();
4845 std::string Reg = "d" + toString(Value, 10);
4846
4847 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
4848 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
4849 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
4850
4851 llvm::Value *Ret;
4852 if (BuiltinID == clang::AArch64::BI__getRegFp) {
4853 llvm::Function *F =
4854 CGM.getIntrinsic(Intrinsic::read_volatile_register, {Int64Ty});
4855 llvm::Value *Bits = Builder.CreateCall(F, Metadata);
4856 Ret = Builder.CreateBitCast(Bits, llvm::Type::getDoubleTy(Context));
4857 } else {
4858 llvm::Value *Val = EmitScalarExpr(E->getArg(1));
4859 llvm::Value *Bits = Builder.CreateBitCast(Val, Int64Ty);
4860 llvm::Function *F =
4861 CGM.getIntrinsic(Intrinsic::write_volatile_register, {Int64Ty});
4862 Ret = Builder.CreateCall(F, {Metadata, Bits});
4863 }
4864 return Ret;
4865 }
4866
4867 if (BuiltinID == clang::AArch64::BI__break) {
4869 if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
4870 llvm_unreachable("Sema will ensure that the parameter is constant");
4871
4872 llvm::Function *F = CGM.getIntrinsic(Intrinsic::aarch64_break);
4873 return Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0))});
4874 }
4875
4876 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
4877 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
4878 return Builder.CreateCall(F);
4879 }
4880
4881 // CRC32
4882 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
4883 switch (BuiltinID) {
4884 case clang::AArch64::BI__builtin_arm_crc32b:
4885 CRCIntrinsicID = Intrinsic::aarch64_crc32b; break;
4886 case clang::AArch64::BI__builtin_arm_crc32cb:
4887 CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break;
4888 case clang::AArch64::BI__builtin_arm_crc32h:
4889 CRCIntrinsicID = Intrinsic::aarch64_crc32h; break;
4890 case clang::AArch64::BI__builtin_arm_crc32ch:
4891 CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break;
4892 case clang::AArch64::BI__builtin_arm_crc32w:
4893 CRCIntrinsicID = Intrinsic::aarch64_crc32w; break;
4894 case clang::AArch64::BI__builtin_arm_crc32cw:
4895 CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break;
4896 case clang::AArch64::BI__builtin_arm_crc32d:
4897 CRCIntrinsicID = Intrinsic::aarch64_crc32x; break;
4898 case clang::AArch64::BI__builtin_arm_crc32cd:
4899 CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break;
4900 }
4901
4902 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
4903 Value *Arg0 = EmitScalarExpr(E->getArg(0));
4904 Value *Arg1 = EmitScalarExpr(E->getArg(1));
4905 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
4906
4907 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
4908 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
4909
4910 return Builder.CreateCall(F, {Arg0, Arg1});
4911 }
4912
4913 // Memory Operations (MOPS)
4914 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
4915 Value *Dst = EmitScalarExpr(E->getArg(0));
4916 Value *Val = EmitScalarExpr(E->getArg(1));
4917 Value *Size = EmitScalarExpr(E->getArg(2));
4918 Val = Builder.CreateTrunc(Val, Int8Ty);
4919 Size = Builder.CreateIntCast(Size, Int64Ty, false);
4920 return Builder.CreateCall(
4921 CGM.getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
4922 }
4923
4924 if (BuiltinID == AArch64::BI__builtin_arm_range_prefetch ||
4925 BuiltinID == AArch64::BI__builtin_arm_range_prefetch_x)
4926 return EmitRangePrefetchBuiltin(*this, BuiltinID, E);
4927
4928 switch (BuiltinID) {
4929 default:
4930 break;
4931 case AArch64::BI__builtin_arm_atomic_store_with_hint:
4932 return EmitAtomicStoreWithHintBuiltin(*this, BuiltinID, E);
4933 case AArch64::BI__builtin_arm_atomic_fetch_add_with_hint:
4934 return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Add);
4935 case AArch64::BI__builtin_arm_atomic_fetch_sub_with_hint:
4936 return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Sub);
4937 case AArch64::BI__builtin_arm_atomic_fetch_and_with_hint:
4938 return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::And);
4939 case AArch64::BI__builtin_arm_atomic_fetch_xor_with_hint:
4940 return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Xor);
4941 case AArch64::BI__builtin_arm_atomic_fetch_or_with_hint:
4942 return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Or);
4943 }
4944
4945 // Memory Tagging Extensions (MTE) Intrinsics
4946 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
4947 switch (BuiltinID) {
4948 case clang::AArch64::BI__builtin_arm_irg:
4949 MTEIntrinsicID = Intrinsic::aarch64_irg; break;
4950 case clang::AArch64::BI__builtin_arm_addg:
4951 MTEIntrinsicID = Intrinsic::aarch64_addg; break;
4952 case clang::AArch64::BI__builtin_arm_gmi:
4953 MTEIntrinsicID = Intrinsic::aarch64_gmi; break;
4954 case clang::AArch64::BI__builtin_arm_ldg:
4955 MTEIntrinsicID = Intrinsic::aarch64_ldg; break;
4956 case clang::AArch64::BI__builtin_arm_stg:
4957 MTEIntrinsicID = Intrinsic::aarch64_stg; break;
4958 case clang::AArch64::BI__builtin_arm_subp:
4959 MTEIntrinsicID = Intrinsic::aarch64_subp; break;
4960 }
4961
4962 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_lscale) {
4963 Value *FPM = EmitScalarExpr(E->getArg(0));
4964 Value *Scale = EmitScalarExpr(E->getArg(1));
4965 Scale = Builder.CreateAnd(Scale, Builder.getInt64(0x7f));
4966
4967 Value *MaskedFPM = Builder.CreateAnd(FPM, Builder.getInt64(~0x7f0000ULL));
4968 Value *ShiftedScale = Builder.CreateShl(Scale, Builder.getInt64(16));
4969
4970 return Builder.CreateOr(MaskedFPM, ShiftedScale);
4971 }
4972 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_nscale) {
4973 Value *FPM = EmitScalarExpr(E->getArg(0));
4974 Value *Scale = EmitScalarExpr(E->getArg(1));
4975 Scale = Builder.CreateAnd(Scale, Builder.getInt64(0xff));
4976
4977 Value *MaskedFPM = Builder.CreateAnd(FPM, Builder.getInt64(~0xff000000ULL));
4978 Value *ShiftedScale = Builder.CreateShl(Scale, Builder.getInt64(24));
4979
4980 return Builder.CreateOr(MaskedFPM, ShiftedScale);
4981 }
4982 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_lscale2) {
4983 Value *FPM = EmitScalarExpr(E->getArg(0));
4984 Value *Scale = EmitScalarExpr(E->getArg(1));
4985 Scale = Builder.CreateAnd(Scale, Builder.getInt64(0x3f));
4986
4987 Value *LowFPM = Builder.CreateAnd(FPM, Builder.getInt64(~0x3f00000000ULL));
4988 Value *ShiftedScale = Builder.CreateShl(Scale, Builder.getInt64(32));
4989
4990 return Builder.CreateOr(LowFPM, ShiftedScale);
4991 }
4992
4993 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
4994 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
4996 Value *Mask = EmitScalarExpr(E->getArg(1));
4997 assert(Mask->getType()->getScalarSizeInBits() == 64 &&
4998 "SemaARM::BuiltinARMMemoryTaggingCall() enforces this");
4999 return Builder.CreateCall(CGM.getIntrinsic(MTEIntrinsicID),
5000 {Pointer, Mask});
5001 }
5002 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
5004 Value *TagOffset = EmitScalarExpr(E->getArg(1));
5005
5006 TagOffset = Builder.CreateZExt(TagOffset, Int64Ty);
5007 return Builder.CreateCall(CGM.getIntrinsic(MTEIntrinsicID),
5008 {Pointer, TagOffset});
5009 }
5010 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
5012 Value *ExcludedMask = EmitScalarExpr(E->getArg(1));
5013 assert(ExcludedMask->getType()->getScalarSizeInBits() == 64 &&
5014 "SemaARM::BuiltinARMMemoryTaggingCall() enforces this");
5015 return Builder.CreateCall(CGM.getIntrinsic(MTEIntrinsicID),
5016 {Pointer, ExcludedMask});
5017 }
5018 // Although it is possible to supply a different return
5019 // address (first arg) to this intrinsic, for now we set
5020 // return address same as input address.
5021 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
5022 Value *TagAddress = EmitScalarExpr(E->getArg(0));
5023 return Builder.CreateCall(CGM.getIntrinsic(MTEIntrinsicID),
5024 {TagAddress, TagAddress});
5025 }
5026 // Although it is possible to supply a different tag (to set)
5027 // to this intrinsic (as first arg), for now we supply
5028 // the tag that is in input address arg (common use case).
5029 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
5030 Value *TagAddress = EmitScalarExpr(E->getArg(0));
5031 return Builder.CreateCall(CGM.getIntrinsic(MTEIntrinsicID),
5032 {TagAddress, TagAddress});
5033 }
5034 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
5035 Value *PointerA = EmitScalarExpr(E->getArg(0));
5036 Value *PointerB = EmitScalarExpr(E->getArg(1));
5037 return Builder.CreateCall(
5038 CGM.getIntrinsic(MTEIntrinsicID), {PointerA, PointerB});
5039 }
5040 }
5041
5042 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5043 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
5044 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5045 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
5046 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
5047 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
5048 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
5049 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
5050
5051 SpecialRegisterAccessKind AccessKind = Write;
5052 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5053 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
5054 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5055 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
5056 AccessKind = VolatileRead;
5057
5058 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
5059 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
5060
5061 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5062 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
5063
5064 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5065 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
5066
5067 llvm::Type *ValueType;
5068 llvm::Type *RegisterType = Int64Ty;
5069 if (Is32Bit) {
5070 ValueType = Int32Ty;
5071 } else if (Is128Bit) {
5072 llvm::Type *Int128Ty =
5073 llvm::IntegerType::getInt128Ty(CGM.getLLVMContext());
5074 ValueType = Int128Ty;
5075 RegisterType = Int128Ty;
5076 } else if (IsPointerBuiltin) {
5077 ValueType = VoidPtrTy;
5078 } else {
5079 ValueType = Int64Ty;
5080 };
5081
5082 return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType,
5083 AccessKind);
5084 }
5085
5086 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
5087 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
5088 LLVMContext &Context = CGM.getLLVMContext();
5089
5090 unsigned SysReg =
5091 E->getArg(0)->EvaluateKnownConstInt(getContext()).getZExtValue();
5092
5093 std::string SysRegStr;
5094 llvm::raw_string_ostream(SysRegStr)
5095 << (0b10 | SysReg >> 14) << ":" << ((SysReg >> 11) & 7) << ":"
5096 << ((SysReg >> 7) & 15) << ":" << ((SysReg >> 3) & 15) << ":"
5097 << (SysReg & 7);
5098
5099 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
5100 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
5101 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
5102
5103 llvm::Type *RegisterType = Int64Ty;
5104 llvm::Type *Types[] = { RegisterType };
5105
5106 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
5107 llvm::Function *F = CGM.getIntrinsic(Intrinsic::read_register, Types);
5108
5109 return Builder.CreateCall(F, Metadata);
5110 }
5111
5112 llvm::Function *F = CGM.getIntrinsic(Intrinsic::write_register, Types);
5113 llvm::Value *ArgValue = EmitScalarExpr(E->getArg(1));
5114 llvm::Value *Result = Builder.CreateCall(F, {Metadata, ArgValue});
5115
5116 return Result;
5117 }
5118
5119 if (BuiltinID == clang::AArch64::BI__sys) {
5120 unsigned SysReg =
5121 E->getArg(0)->EvaluateKnownConstInt(getContext()).getZExtValue();
5122 const unsigned Op1 = SysReg >> 11;
5123 const unsigned CRn = (SysReg >> 7) & 0xf;
5124 const unsigned CRm = (SysReg >> 3) & 0xf;
5125 const unsigned Op2 = SysReg & 0x7;
5126
5127 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_sys),
5128 {Builder.getInt32(Op1), Builder.getInt32(CRn),
5129 Builder.getInt32(CRm), Builder.getInt32(Op2),
5130 EmitScalarExpr(E->getArg(1))});
5131
5132 // Return 0 for convenience, even though MSVC returns some other undefined
5133 // value.
5134 return ConstantInt::get(Builder.getInt32Ty(), 0);
5135 }
5136
5137 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
5138 llvm::Function *F =
5139 CGM.getIntrinsic(Intrinsic::addressofreturnaddress, AllocaInt8PtrTy);
5140 return Builder.CreateCall(F);
5141 }
5142
5143 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
5144 llvm::Function *F = CGM.getIntrinsic(Intrinsic::sponentry, AllocaInt8PtrTy);
5145 return Builder.CreateCall(F);
5146 }
5147
5148 if (BuiltinID == clang::AArch64::BI__mulh ||
5149 BuiltinID == clang::AArch64::BI__umulh) {
5150 llvm::Type *ResType = ConvertType(E->getType());
5151 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
5152
5153 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
5154 Value *LHS =
5155 Builder.CreateIntCast(EmitScalarExpr(E->getArg(0)), Int128Ty, IsSigned);
5156 Value *RHS =
5157 Builder.CreateIntCast(EmitScalarExpr(E->getArg(1)), Int128Ty, IsSigned);
5158
5159 Value *MulResult, *HigherBits;
5160 if (IsSigned) {
5161 MulResult = Builder.CreateNSWMul(LHS, RHS);
5162 HigherBits = Builder.CreateAShr(MulResult, 64);
5163 } else {
5164 MulResult = Builder.CreateNUWMul(LHS, RHS);
5165 HigherBits = Builder.CreateLShr(MulResult, 64);
5166 }
5167 HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
5168
5169 return HigherBits;
5170 }
5171
5172 if (BuiltinID == AArch64::BI__writex18byte ||
5173 BuiltinID == AArch64::BI__writex18word ||
5174 BuiltinID == AArch64::BI__writex18dword ||
5175 BuiltinID == AArch64::BI__writex18qword) {
5176 // Process the args first
5177 Value *OffsetArg = EmitScalarExpr(E->getArg(0));
5178 Value *DataArg = EmitScalarExpr(E->getArg(1));
5179
5180 // Read x18 as i8*
5181 llvm::Value *X18 = readX18AsPtr(*this);
5182
5183 // Store val at x18 + offset
5184 Value *Offset = Builder.CreateZExt(OffsetArg, Int64Ty);
5185 Value *Ptr = Builder.CreateGEP(Int8Ty, X18, Offset);
5186 StoreInst *Store =
5187 Builder.CreateAlignedStore(DataArg, Ptr, CharUnits::One());
5188 return Store;
5189 }
5190
5191 if (BuiltinID == AArch64::BI__readx18byte ||
5192 BuiltinID == AArch64::BI__readx18word ||
5193 BuiltinID == AArch64::BI__readx18dword ||
5194 BuiltinID == AArch64::BI__readx18qword) {
5195 // Process the args first
5196 Value *OffsetArg = EmitScalarExpr(E->getArg(0));
5197
5198 // Read x18 as i8*
5199 llvm::Value *X18 = readX18AsPtr(*this);
5200
5201 // Load x18 + offset
5202 Value *Offset = Builder.CreateZExt(OffsetArg, Int64Ty);
5203 Value *Ptr = Builder.CreateGEP(Int8Ty, X18, Offset);
5204 llvm::Type *IntTy = ConvertType(E->getType());
5205 LoadInst *Load = Builder.CreateAlignedLoad(IntTy, Ptr, CharUnits::One());
5206 return Load;
5207 }
5208
5209 if (BuiltinID == AArch64::BI__addx18byte ||
5210 BuiltinID == AArch64::BI__addx18word ||
5211 BuiltinID == AArch64::BI__addx18dword ||
5212 BuiltinID == AArch64::BI__addx18qword ||
5213 BuiltinID == AArch64::BI__incx18byte ||
5214 BuiltinID == AArch64::BI__incx18word ||
5215 BuiltinID == AArch64::BI__incx18dword ||
5216 BuiltinID == AArch64::BI__incx18qword) {
5217 llvm::Type *IntTy;
5218 bool isIncrement;
5219 switch (BuiltinID) {
5220 case AArch64::BI__incx18byte:
5221 IntTy = Int8Ty;
5222 isIncrement = true;
5223 break;
5224 case AArch64::BI__incx18word:
5225 IntTy = Int16Ty;
5226 isIncrement = true;
5227 break;
5228 case AArch64::BI__incx18dword:
5229 IntTy = Int32Ty;
5230 isIncrement = true;
5231 break;
5232 case AArch64::BI__incx18qword:
5233 IntTy = Int64Ty;
5234 isIncrement = true;
5235 break;
5236 default:
5237 IntTy = ConvertType(E->getArg(1)->getType());
5238 isIncrement = false;
5239 break;
5240 }
5241 // Process the args first
5242 Value *OffsetArg = EmitScalarExpr(E->getArg(0));
5243 Value *ValToAdd =
5244 isIncrement ? ConstantInt::get(IntTy, 1) : EmitScalarExpr(E->getArg(1));
5245
5246 // Read x18 as i8*
5247 llvm::Value *X18 = readX18AsPtr(*this);
5248
5249 // Load x18 + offset
5250 Value *Offset = Builder.CreateZExt(OffsetArg, Int64Ty);
5251 Value *Ptr = Builder.CreateGEP(Int8Ty, X18, Offset);
5252 LoadInst *Load = Builder.CreateAlignedLoad(IntTy, Ptr, CharUnits::One());
5253
5254 // Add values
5255 Value *AddResult = Builder.CreateAdd(Load, ValToAdd);
5256
5257 // Store val at x18 + offset
5258 StoreInst *Store =
5259 Builder.CreateAlignedStore(AddResult, Ptr, CharUnits::One());
5260 return Store;
5261 }
5262
5263 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
5264 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
5265 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
5266 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
5267 Value *Arg = EmitScalarExpr(E->getArg(0));
5268 llvm::Type *RetTy = ConvertType(E->getType());
5269 return Builder.CreateBitCast(Arg, RetTy);
5270 }
5271
5272 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
5273 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
5274 BuiltinID == AArch64::BI_CountLeadingZeros ||
5275 BuiltinID == AArch64::BI_CountLeadingZeros64) {
5276 Value *Arg = EmitScalarExpr(E->getArg(0));
5277 llvm::Type *ArgType = Arg->getType();
5278
5279 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
5280 BuiltinID == AArch64::BI_CountLeadingOnes64)
5281 Arg = Builder.CreateXor(Arg, Constant::getAllOnesValue(ArgType));
5282
5283 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
5284 Value *Result = Builder.CreateCall(F, {Arg, Builder.getInt1(false)});
5285
5286 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
5287 BuiltinID == AArch64::BI_CountLeadingZeros64)
5288 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
5289 return Result;
5290 }
5291
5292 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
5293 BuiltinID == AArch64::BI_CountLeadingSigns64) {
5294 Value *Arg = EmitScalarExpr(E->getArg(0));
5295
5296 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
5297 ? CGM.getIntrinsic(Intrinsic::aarch64_cls)
5298 : CGM.getIntrinsic(Intrinsic::aarch64_cls64);
5299
5300 Value *Result = Builder.CreateCall(F, Arg, "cls");
5301 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
5302 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
5303 return Result;
5304 }
5305
5306 if (BuiltinID == AArch64::BI_CountOneBits ||
5307 BuiltinID == AArch64::BI_CountOneBits64) {
5308 Value *ArgValue = EmitScalarExpr(E->getArg(0));
5309 llvm::Type *ArgType = ArgValue->getType();
5310 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
5311
5312 Value *Result = Builder.CreateCall(F, ArgValue);
5313 if (BuiltinID == AArch64::BI_CountOneBits64)
5314 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
5315 return Result;
5316 }
5317
5318 if (BuiltinID == AArch64::BI_CountTrailingZeros ||
5319 BuiltinID == AArch64::BI_CountTrailingZeros64) {
5320 Value *ArgValue = EmitScalarExpr(E->getArg(0));
5321 llvm::Type *ArgType = ArgValue->getType();
5322 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
5323
5324 // MSVC leaves 0 undefined; use false for predictable codegen
5325 Value *Result = Builder.CreateCall(F, {ArgValue, Builder.getInt1(false)});
5326 if (BuiltinID == AArch64::BI_CountTrailingZeros64)
5327 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
5328 return Result;
5329 }
5330
5331 if (BuiltinID == AArch64::BI__prefetch) {
5333 Value *RW = llvm::ConstantInt::get(Int32Ty, 0);
5334 Value *Locality = ConstantInt::get(Int32Ty, 3);
5335 Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
5336 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
5337 return Builder.CreateCall(F, {Address, RW, Locality, Data});
5338 }
5339
5340 if (BuiltinID == AArch64::BI__prefetch2) {
5342 llvm::APSInt PrfOp = E->getArg(1)->EvaluateKnownConstInt(CGM.getContext());
5343 // Decode 5-bit PRFM encoding: bits[4:3]=type, bits[2:1]=target,
5344 // bit[0]=policy
5345 // type: PLD=0(load), PLI=1(instr), PST=2(store)
5346 // target: L1=0, L2=1, L3=2
5347 // policy: KEEP=0, STRM=1
5348 uint64_t Op = PrfOp.getZExtValue();
5349 uint64_t Type = (Op >> 3) & 0x3;
5350 uint64_t Target = (Op >> 1) & 0x3;
5351 uint64_t Policy = Op & 0x1;
5352 Value *RW = Builder.getInt32(Type == 2 ? 1 : 0);
5353 Value *Local = Builder.getInt32(Target);
5354 Value *IsStream = Builder.getInt32(Policy);
5355 Value *IsData = Builder.getInt32(Type == 1 ? 0 : 1);
5356 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_prefetch);
5357 return Builder.CreateCall(F, {Address, RW, Local, IsStream, IsData});
5358 }
5359
5360 if (BuiltinID == AArch64::BI__hlt) {
5361 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hlt);
5362 Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0))});
5363
5364 // FIXME: MSVC documents __hlt as taking further arguments in X0-X3 and
5365 // returning the value in X0, like __hvc/__svc below. This ignores the
5366 // extra arguments and returns 0.
5367 return ConstantInt::get(Builder.getInt32Ty(), 0);
5368 }
5369
5370 if (BuiltinID == AArch64::BI__hvc || BuiltinID == AArch64::BI__svc) {
5371 unsigned IID = BuiltinID == AArch64::BI__svc ? Intrinsic::aarch64_svc
5372 : Intrinsic::aarch64_hvc;
5373 // The first argument is the instruction immediate; it must be a constant
5374 // (ImmArg on the intrinsic, encoded in the instruction). The remaining
5375 // arguments (at most four, enforced by Sema) are passed in X0-X3, widened
5376 // to 64 bits. The intrinsic takes exactly four register operands, so any
5377 // unused trailing ones are passed as poison and dropped during lowering.
5378 SmallVector<Value *, 5> Args{Builder.getInt32(
5380 for (unsigned I = 1, N = E->getNumArgs(); I < N; ++I) {
5381 Value *Arg = EmitScalarExpr(E->getArg(I));
5382 llvm::Type *ArgTy = Arg->getType();
5383 if (ArgTy->isPointerTy())
5384 Arg = Builder.CreatePtrToInt(Arg, Int64Ty);
5385 else if (ArgTy->isFloatingPointTy())
5386 // Reinterpret the bits into the integer register, matching MSVC (e.g.
5387 // "fmov x0, d0" for a double).
5388 Arg = Builder.CreateZExtOrTrunc(
5389 Builder.CreateBitCast(
5390 Arg, Builder.getIntNTy(ArgTy->getPrimitiveSizeInBits())),
5391 Int64Ty);
5392 else
5393 Arg = Builder.CreateIntCast(
5394 Arg, Int64Ty, E->getArg(I)->getType()->isSignedIntegerType());
5395 Args.push_back(Arg);
5396 }
5397 while (Args.size() < 5)
5398 Args.push_back(llvm::PoisonValue::get(Int64Ty));
5399 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Args);
5400 // MSVC returns unsigned int, i.e. the low 32 bits of the X0 result.
5401 return Builder.CreateTrunc(Call, Int32Ty);
5402 }
5403
5404 if (BuiltinID == NEON::BI__builtin_neon_vcvth_bf16_f32)
5405 return Builder.CreateFPTrunc(
5406 Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)),
5407 Builder.getFloatTy()),
5408 Builder.getBFloatTy());
5409
5410 // Handle MSVC intrinsics before argument evaluation to prevent double
5411 // evaluation.
5412 if (std::optional<MSVCIntrin> MsvcIntId =
5414 return EmitMSVCBuiltinExpr(*MsvcIntId, E);
5415
5416 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
5417 auto It = llvm::find_if(NEONEquivalentIntrinsicMap, [BuiltinID](auto &P) {
5418 return P.first == BuiltinID;
5419 });
5420 if (It != end(NEONEquivalentIntrinsicMap))
5421 BuiltinID = It->second;
5422
5423 // Check whether this is an SISD builtin.
5424 auto SISDMap = ArrayRef(AArch64SISDIntrinsicMap);
5426 SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted);
5427 bool IsSISD = (Builtin != nullptr);
5428
5429 // Find out if any arguments are required to be integer constant
5430 // expressions.
5431 unsigned ICEArguments = 0;
5433 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
5434 assert(Error == ASTContext::GE_None && "Should not codegen an error");
5435
5437 Address PtrOp0 = Address::invalid();
5438 // Note the assumption that SISD intrinsics do not contain extra arguments.
5439 // TODO: Fold this into a single function call instead of, effectively, two
5440 // separate checks.
5441 bool HasExtraArg = !IsSISD && CodeGenUtils::hasExtraNeonArgument(BuiltinID);
5442 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
5443 for (unsigned i = 0, e = NumArgs; i != e; i++) {
5444 if (i == 0) {
5445 switch (BuiltinID) {
5446 case NEON::BI__builtin_neon_vld1_v:
5447 case NEON::BI__builtin_neon_vld1q_v:
5448 case NEON::BI__builtin_neon_vld1_dup_v:
5449 case NEON::BI__builtin_neon_vld1q_dup_v:
5450 case NEON::BI__builtin_neon_vld1_lane_v:
5451 case NEON::BI__builtin_neon_vld1q_lane_v:
5452 case NEON::BI__builtin_neon_vst1_v:
5453 case NEON::BI__builtin_neon_vst1q_v:
5454 case NEON::BI__builtin_neon_vst1_lane_v:
5455 case NEON::BI__builtin_neon_vst1q_lane_v:
5456 case NEON::BI__builtin_neon_vldap1_lane_s64:
5457 case NEON::BI__builtin_neon_vldap1q_lane_s64:
5458 case NEON::BI__builtin_neon_vstl1_lane_s64:
5459 case NEON::BI__builtin_neon_vstl1q_lane_s64:
5460 // Get the alignment for the argument in addition to the value;
5461 // we'll use it later.
5462 PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
5463 Ops.push_back(PtrOp0.emitRawPointer(*this));
5464 continue;
5465 }
5466 }
5467 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E));
5468 }
5469
5470 if (Builtin) {
5472 assert(Result && "SISD intrinsic should have been handled");
5473 return Result;
5474 }
5475
5476 const Expr *Arg = E->getArg(E->getNumArgs()-1);
5478 if (std::optional<llvm::APSInt> Result =
5480 // Determine the type of this overloaded NEON intrinsic.
5481 Type = NeonTypeFlags(Result->getZExtValue());
5482
5483 bool usgn = Type.isUnsigned();
5484 bool quad = Type.isQuad();
5485 unsigned Int;
5486
5487 // Not all intrinsics handled by the common case work for AArch64 yet, so only
5488 // defer to common code if it's been added to our special map.
5489 Builtin =
5492
5493 if (Builtin)
5495 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
5496 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
5497 /*never use addresses*/ Address::invalid(), Address::invalid(), Arch);
5498
5499 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops, Arch))
5500 return V;
5501
5502 // Handle non-overloaded intrinsics first.
5503 switch (BuiltinID) {
5504 default: break;
5505 case NEON::BI__builtin_neon_vabsh_f16:
5506 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, HalfTy), Ops, "vabs");
5507 case NEON::BI__builtin_neon_vaddq_p128: {
5508 llvm::Type *Ty = GetNeonType(this, NeonTypeFlags::Poly128);
5509 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5510 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5511 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]);
5512 llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
5513 return Builder.CreateBitCast(Ops[0], Int128Ty);
5514 }
5515 case NEON::BI__builtin_neon_vldrq_p128: {
5516 llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
5517 return Builder.CreateAlignedLoad(Int128Ty, Ops[0],
5519 }
5520 case NEON::BI__builtin_neon_vstrq_p128: {
5521 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5522 }
5523 case NEON::BI__builtin_neon_vcvts_f32_u32:
5524 case NEON::BI__builtin_neon_vcvtd_f64_u64:
5525 usgn = true;
5526 [[fallthrough]];
5527 case NEON::BI__builtin_neon_vcvts_f32_s32:
5528 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
5529 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5530 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
5531 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
5532 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
5533 if (usgn)
5534 return Builder.CreateUIToFP(Ops[0], FTy);
5535 return Builder.CreateSIToFP(Ops[0], FTy);
5536 }
5537 case NEON::BI__builtin_neon_vcvth_f16_u16:
5538 case NEON::BI__builtin_neon_vcvth_f16_u32:
5539 case NEON::BI__builtin_neon_vcvth_f16_u64:
5540 usgn = true;
5541 [[fallthrough]];
5542 case NEON::BI__builtin_neon_vcvth_f16_s16:
5543 case NEON::BI__builtin_neon_vcvth_f16_s32:
5544 case NEON::BI__builtin_neon_vcvth_f16_s64: {
5545 llvm::Type *FTy = HalfTy;
5546 llvm::Type *InTy;
5547 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
5548 InTy = Int64Ty;
5549 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
5550 InTy = Int32Ty;
5551 else
5552 InTy = Int16Ty;
5553 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
5554 if (usgn)
5555 return Builder.CreateUIToFP(Ops[0], FTy);
5556 return Builder.CreateSIToFP(Ops[0], FTy);
5557 }
5558 case NEON::BI__builtin_neon_vcvtah_u16_f16:
5559 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
5560 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
5561 case NEON::BI__builtin_neon_vcvtph_u16_f16:
5562 case NEON::BI__builtin_neon_vcvtah_s16_f16:
5563 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
5564 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
5565 case NEON::BI__builtin_neon_vcvtph_s16_f16: {
5566 llvm::Type *InTy = Int16Ty;
5567 llvm::Type* FTy = HalfTy;
5568 llvm::Type *Tys[2] = {InTy, FTy};
5569 switch (BuiltinID) {
5570 default: llvm_unreachable("missing builtin ID in switch!");
5571 case NEON::BI__builtin_neon_vcvtah_u16_f16:
5572 Int = Intrinsic::aarch64_neon_fcvtau; break;
5573 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
5574 Int = Intrinsic::aarch64_neon_fcvtmu; break;
5575 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
5576 Int = Intrinsic::aarch64_neon_fcvtnu; break;
5577 case NEON::BI__builtin_neon_vcvtph_u16_f16:
5578 Int = Intrinsic::aarch64_neon_fcvtpu; break;
5579 case NEON::BI__builtin_neon_vcvtah_s16_f16:
5580 Int = Intrinsic::aarch64_neon_fcvtas; break;
5581 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
5582 Int = Intrinsic::aarch64_neon_fcvtms; break;
5583 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
5584 Int = Intrinsic::aarch64_neon_fcvtns; break;
5585 case NEON::BI__builtin_neon_vcvtph_s16_f16:
5586 Int = Intrinsic::aarch64_neon_fcvtps; break;
5587 }
5588 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvt");
5589 }
5590 case NEON::BI__builtin_neon_vcaleh_f16:
5591 case NEON::BI__builtin_neon_vcalth_f16:
5592 case NEON::BI__builtin_neon_vcageh_f16:
5593 case NEON::BI__builtin_neon_vcagth_f16: {
5594 llvm::Type* InTy = Int32Ty;
5595 llvm::Type* FTy = HalfTy;
5596 llvm::Type *Tys[2] = {InTy, FTy};
5597 switch (BuiltinID) {
5598 default: llvm_unreachable("missing builtin ID in switch!");
5599 case NEON::BI__builtin_neon_vcageh_f16:
5600 Int = Intrinsic::aarch64_neon_facge; break;
5601 case NEON::BI__builtin_neon_vcagth_f16:
5602 Int = Intrinsic::aarch64_neon_facgt; break;
5603 case NEON::BI__builtin_neon_vcaleh_f16:
5604 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]); break;
5605 case NEON::BI__builtin_neon_vcalth_f16:
5606 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]); break;
5607 }
5608 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "facg");
5609 return Builder.CreateTrunc(Ops[0], Int16Ty);
5610 }
5611 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
5612 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
5613 llvm::Type* InTy = Int32Ty;
5614 llvm::Type* FTy = HalfTy;
5615 llvm::Type *Tys[2] = {InTy, FTy};
5616 switch (BuiltinID) {
5617 default: llvm_unreachable("missing builtin ID in switch!");
5618 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
5619 Int = Intrinsic::aarch64_neon_vcvtfp2fxs; break;
5620 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
5621 Int = Intrinsic::aarch64_neon_vcvtfp2fxu; break;
5622 }
5623 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n");
5624 return Builder.CreateTrunc(Ops[0], Int16Ty);
5625 }
5626 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
5627 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
5628 llvm::Type* FTy = HalfTy;
5629 llvm::Type* InTy = Int32Ty;
5630 llvm::Type *Tys[2] = {FTy, InTy};
5631 switch (BuiltinID) {
5632 default: llvm_unreachable("missing builtin ID in switch!");
5633 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
5634 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
5635 Ops[0] = Builder.CreateSExt(Ops[0], InTy, "sext");
5636 break;
5637 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
5638 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
5639 Ops[0] = Builder.CreateZExt(Ops[0], InTy);
5640 break;
5641 }
5642 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n");
5643 }
5644 case NEON::BI__builtin_neon_vpaddd_s64: {
5645 // TODO: Isn't this handled by
5646 // EmitCommonNeonSISDBuiltinExpr?
5647 auto *Ty = llvm::FixedVectorType::get(Int64Ty, 2);
5648 // The vector is v2f64, so make sure it's bitcast to that.
5649 Ops[0] = Builder.CreateBitCast(Ops[0], Ty, "v2i64");
5650 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
5651 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
5652 Value *Op0 = Builder.CreateExtractElement(Ops[0], Idx0, "lane0");
5653 Value *Op1 = Builder.CreateExtractElement(Ops[0], Idx1, "lane1");
5654 // Pairwise addition of a v2f64 into a scalar f64.
5655 return Builder.CreateAdd(Op0, Op1, "vpaddd");
5656 }
5657 case NEON::BI__builtin_neon_vpaddd_f64: {
5658 auto *Ty = llvm::FixedVectorType::get(DoubleTy, 2);
5659 // The vector is v2f64, so make sure it's bitcast to that.
5660 Ops[0] = Builder.CreateBitCast(Ops[0], Ty, "v2f64");
5661 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
5662 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
5663 Value *Op0 = Builder.CreateExtractElement(Ops[0], Idx0, "lane0");
5664 Value *Op1 = Builder.CreateExtractElement(Ops[0], Idx1, "lane1");
5665 // Pairwise addition of a v2f64 into a scalar f64.
5666 return Builder.CreateFAdd(Op0, Op1, "vpaddd");
5667 }
5668 case NEON::BI__builtin_neon_vpadds_f32: {
5669 auto *Ty = llvm::FixedVectorType::get(FloatTy, 2);
5670 // The vector is v2f32, so make sure it's bitcast to that.
5671 Ops[0] = Builder.CreateBitCast(Ops[0], Ty, "v2f32");
5672 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
5673 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
5674 Value *Op0 = Builder.CreateExtractElement(Ops[0], Idx0, "lane0");
5675 Value *Op1 = Builder.CreateExtractElement(Ops[0], Idx1, "lane1");
5676 // Pairwise addition of a v2f32 into a scalar f32.
5677 return Builder.CreateFAdd(Op0, Op1, "vpaddd");
5678 }
5679 case NEON::BI__builtin_neon_vceqzd_s64:
5682 ICmpInst::ICMP_EQ, "vceqz");
5683 case NEON::BI__builtin_neon_vceqzd_f64:
5684 case NEON::BI__builtin_neon_vceqzs_f32:
5685 case NEON::BI__builtin_neon_vceqzh_f16:
5688 ICmpInst::FCMP_OEQ, "vceqz");
5689 case NEON::BI__builtin_neon_vcgezd_s64:
5692 ICmpInst::ICMP_SGE, "vcgez");
5693 case NEON::BI__builtin_neon_vcgezd_f64:
5694 case NEON::BI__builtin_neon_vcgezs_f32:
5695 case NEON::BI__builtin_neon_vcgezh_f16:
5698 ICmpInst::FCMP_OGE, "vcgez");
5699 case NEON::BI__builtin_neon_vclezd_s64:
5702 ICmpInst::ICMP_SLE, "vclez");
5703 case NEON::BI__builtin_neon_vclezd_f64:
5704 case NEON::BI__builtin_neon_vclezs_f32:
5705 case NEON::BI__builtin_neon_vclezh_f16:
5708 ICmpInst::FCMP_OLE, "vclez");
5709 case NEON::BI__builtin_neon_vcgtzd_s64:
5712 ICmpInst::ICMP_SGT, "vcgtz");
5713 case NEON::BI__builtin_neon_vcgtzd_f64:
5714 case NEON::BI__builtin_neon_vcgtzs_f32:
5715 case NEON::BI__builtin_neon_vcgtzh_f16:
5718 ICmpInst::FCMP_OGT, "vcgtz");
5719 case NEON::BI__builtin_neon_vcltzd_s64:
5722 ICmpInst::ICMP_SLT, "vcltz");
5723
5724 case NEON::BI__builtin_neon_vcltzd_f64:
5725 case NEON::BI__builtin_neon_vcltzs_f32:
5726 case NEON::BI__builtin_neon_vcltzh_f16:
5729 ICmpInst::FCMP_OLT, "vcltz");
5730
5731 case NEON::BI__builtin_neon_vceqzd_u64: {
5734 ICmpInst::ICMP_EQ, "vceqzd");
5735 }
5736 case NEON::BI__builtin_neon_vceqd_f64:
5737 case NEON::BI__builtin_neon_vcled_f64:
5738 case NEON::BI__builtin_neon_vcltd_f64:
5739 case NEON::BI__builtin_neon_vcged_f64:
5740 case NEON::BI__builtin_neon_vcgtd_f64: {
5741 llvm::CmpInst::Predicate P;
5742 switch (BuiltinID) {
5743 default: llvm_unreachable("missing builtin ID in switch!");
5744 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break;
5745 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break;
5746 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break;
5747 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break;
5748 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break;
5749 }
5750 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
5751 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
5752 if (P == llvm::FCmpInst::FCMP_OEQ)
5753 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
5754 else
5755 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
5756 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd");
5757 }
5758 case NEON::BI__builtin_neon_vceqs_f32:
5759 case NEON::BI__builtin_neon_vcles_f32:
5760 case NEON::BI__builtin_neon_vclts_f32:
5761 case NEON::BI__builtin_neon_vcges_f32:
5762 case NEON::BI__builtin_neon_vcgts_f32: {
5763 llvm::CmpInst::Predicate P;
5764 switch (BuiltinID) {
5765 default: llvm_unreachable("missing builtin ID in switch!");
5766 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break;
5767 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break;
5768 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break;
5769 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break;
5770 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break;
5771 }
5772 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
5773 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
5774 if (P == llvm::FCmpInst::FCMP_OEQ)
5775 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
5776 else
5777 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
5778 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd");
5779 }
5780 case NEON::BI__builtin_neon_vceqh_f16:
5781 case NEON::BI__builtin_neon_vcleh_f16:
5782 case NEON::BI__builtin_neon_vclth_f16:
5783 case NEON::BI__builtin_neon_vcgeh_f16:
5784 case NEON::BI__builtin_neon_vcgth_f16: {
5785 llvm::CmpInst::Predicate P;
5786 switch (BuiltinID) {
5787 default: llvm_unreachable("missing builtin ID in switch!");
5788 case NEON::BI__builtin_neon_vceqh_f16: P = llvm::FCmpInst::FCMP_OEQ; break;
5789 case NEON::BI__builtin_neon_vcleh_f16: P = llvm::FCmpInst::FCMP_OLE; break;
5790 case NEON::BI__builtin_neon_vclth_f16: P = llvm::FCmpInst::FCMP_OLT; break;
5791 case NEON::BI__builtin_neon_vcgeh_f16: P = llvm::FCmpInst::FCMP_OGE; break;
5792 case NEON::BI__builtin_neon_vcgth_f16: P = llvm::FCmpInst::FCMP_OGT; break;
5793 }
5794 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
5795 Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy);
5796 if (P == llvm::FCmpInst::FCMP_OEQ)
5797 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
5798 else
5799 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
5800 return Builder.CreateSExt(Ops[0], Int16Ty, "vcmpd");
5801 }
5802 case NEON::BI__builtin_neon_vceqd_s64:
5803 case NEON::BI__builtin_neon_vceqd_u64:
5804 case NEON::BI__builtin_neon_vcgtd_s64:
5805 case NEON::BI__builtin_neon_vcgtd_u64:
5806 case NEON::BI__builtin_neon_vcltd_s64:
5807 case NEON::BI__builtin_neon_vcltd_u64:
5808 case NEON::BI__builtin_neon_vcged_u64:
5809 case NEON::BI__builtin_neon_vcged_s64:
5810 case NEON::BI__builtin_neon_vcled_u64:
5811 case NEON::BI__builtin_neon_vcled_s64: {
5812 llvm::CmpInst::Predicate P;
5813 switch (BuiltinID) {
5814 default: llvm_unreachable("missing builtin ID in switch!");
5815 case NEON::BI__builtin_neon_vceqd_s64:
5816 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break;
5817 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break;
5818 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break;
5819 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break;
5820 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break;
5821 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break;
5822 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break;
5823 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break;
5824 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break;
5825 }
5826 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
5827 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
5828 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
5829 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd");
5830 }
5831 case NEON::BI__builtin_neon_vnegd_s64:
5832 return Builder.CreateNeg(Ops[0], "vnegd");
5833 case NEON::BI__builtin_neon_vnegh_f16:
5834 return Builder.CreateFNeg(Ops[0], "vnegh");
5835 case NEON::BI__builtin_neon_vtstd_s64:
5836 case NEON::BI__builtin_neon_vtstd_u64: {
5837 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
5838 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
5839 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
5840 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
5841 llvm::Constant::getNullValue(Int64Ty));
5842 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd");
5843 }
5844 case NEON::BI__builtin_neon_vset_lane_i8:
5845 case NEON::BI__builtin_neon_vset_lane_i16:
5846 case NEON::BI__builtin_neon_vset_lane_i32:
5847 case NEON::BI__builtin_neon_vset_lane_i64:
5848 case NEON::BI__builtin_neon_vset_lane_bf16:
5849 case NEON::BI__builtin_neon_vset_lane_f32:
5850 case NEON::BI__builtin_neon_vsetq_lane_i8:
5851 case NEON::BI__builtin_neon_vsetq_lane_i16:
5852 case NEON::BI__builtin_neon_vsetq_lane_i32:
5853 case NEON::BI__builtin_neon_vsetq_lane_i64:
5854 case NEON::BI__builtin_neon_vsetq_lane_bf16:
5855 case NEON::BI__builtin_neon_vsetq_lane_f32:
5856 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
5857 case NEON::BI__builtin_neon_vset_lane_f64:
5858 // The vector type needs a cast for the v1f64 variant.
5859 Ops[1] =
5860 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 1));
5861 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
5862 case NEON::BI__builtin_neon_vset_lane_mf8:
5863 case NEON::BI__builtin_neon_vsetq_lane_mf8:
5864 // The input vector type needs a cast to scalar type.
5865 Ops[0] =
5866 Builder.CreateBitCast(Ops[0], llvm::Type::getInt8Ty(getLLVMContext()));
5867 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
5868 case NEON::BI__builtin_neon_vsetq_lane_f64:
5869 // The vector type needs a cast for the v2f64 variant.
5870 Ops[1] =
5871 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 2));
5872 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
5873
5874 case NEON::BI__builtin_neon_vget_lane_i8:
5875 case NEON::BI__builtin_neon_vdupb_lane_i8:
5876 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5877 case NEON::BI__builtin_neon_vgetq_lane_i8:
5878 case NEON::BI__builtin_neon_vdupb_laneq_i8:
5879 return Builder.CreateExtractElement(Ops[0], Ops[1], "vgetq_lane");
5880 case NEON::BI__builtin_neon_vget_lane_mf8:
5881 case NEON::BI__builtin_neon_vdupb_lane_mf8:
5882 case NEON::BI__builtin_neon_vgetq_lane_mf8:
5883 case NEON::BI__builtin_neon_vdupb_laneq_mf8:
5884 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5885 case NEON::BI__builtin_neon_vget_lane_i16:
5886 case NEON::BI__builtin_neon_vduph_lane_i16:
5887 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5888 case NEON::BI__builtin_neon_vgetq_lane_i16:
5889 case NEON::BI__builtin_neon_vduph_laneq_i16:
5890 return Builder.CreateExtractElement(Ops[0], Ops[1], "vgetq_lane");
5891 case NEON::BI__builtin_neon_vget_lane_i32:
5892 case NEON::BI__builtin_neon_vdups_lane_i32:
5893 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5894 case NEON::BI__builtin_neon_vdups_lane_f32:
5895 return Builder.CreateExtractElement(Ops[0], Ops[1], "vdups_lane");
5896 case NEON::BI__builtin_neon_vgetq_lane_i32:
5897 case NEON::BI__builtin_neon_vdups_laneq_i32:
5898 return Builder.CreateExtractElement(Ops[0], Ops[1], "vgetq_lane");
5899 case NEON::BI__builtin_neon_vget_lane_i64:
5900 case NEON::BI__builtin_neon_vdupd_lane_i64:
5901 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5902 case NEON::BI__builtin_neon_vdupd_lane_f64:
5903 return Builder.CreateExtractElement(Ops[0], Ops[1], "vdupd_lane");
5904 case NEON::BI__builtin_neon_vgetq_lane_i64:
5905 case NEON::BI__builtin_neon_vdupd_laneq_i64:
5906 return Builder.CreateExtractElement(Ops[0], Ops[1], "vgetq_lane");
5907 case NEON::BI__builtin_neon_vget_lane_f32:
5908 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5909 case NEON::BI__builtin_neon_vget_lane_f64:
5910 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5911 case NEON::BI__builtin_neon_vgetq_lane_f32:
5912 case NEON::BI__builtin_neon_vdups_laneq_f32:
5913 return Builder.CreateExtractElement(Ops[0], Ops[1], "vgetq_lane");
5914 case NEON::BI__builtin_neon_vgetq_lane_f64:
5915 case NEON::BI__builtin_neon_vdupd_laneq_f64:
5916 return Builder.CreateExtractElement(Ops[0], Ops[1], "vgetq_lane");
5917 case NEON::BI__builtin_neon_vaddh_f16:
5918 return Builder.CreateFAdd(Ops[0], Ops[1], "vaddh");
5919 case NEON::BI__builtin_neon_vsubh_f16:
5920 return Builder.CreateFSub(Ops[0], Ops[1], "vsubh");
5921 case NEON::BI__builtin_neon_vmulh_f16:
5922 return Builder.CreateFMul(Ops[0], Ops[1], "vmulh");
5923 case NEON::BI__builtin_neon_vdivh_f16:
5924 return Builder.CreateFDiv(Ops[0], Ops[1], "vdivh");
5925 case NEON::BI__builtin_neon_vfmah_f16:
5926 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
5928 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, HalfTy,
5929 {Ops[1], Ops[2], Ops[0]});
5930 case NEON::BI__builtin_neon_vfmsh_f16: {
5931 Value *Neg = Builder.CreateFNeg(Ops[1], "vsubh");
5932
5933 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
5935 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, HalfTy,
5936 {Neg, Ops[2], Ops[0]});
5937 }
5938 case NEON::BI__builtin_neon_vaddd_s64:
5939 case NEON::BI__builtin_neon_vaddd_u64:
5940 return Builder.CreateAdd(Ops[0], Ops[1], "vaddd");
5941 case NEON::BI__builtin_neon_vsubd_s64:
5942 case NEON::BI__builtin_neon_vsubd_u64:
5943 return Builder.CreateSub(Ops[0], Ops[1], "vsubd");
5944 case NEON::BI__builtin_neon_vqdmlalh_s16:
5945 case NEON::BI__builtin_neon_vqdmlslh_s16: {
5946 SmallVector<Value *, 2> ProductOps;
5947 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
5948 ProductOps.push_back(vectorWrapScalar16(Ops[2]));
5949 auto *VTy = llvm::FixedVectorType::get(Int32Ty, 4);
5950 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
5951 ProductOps, "vqdmlXl");
5952 Constant *CI = ConstantInt::get(SizeTy, 0);
5953 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
5954
5955 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
5956 ? Intrinsic::aarch64_neon_sqadd
5957 : Intrinsic::aarch64_neon_sqsub;
5958 // Drop the 2nd multiplication argument before the accumulation
5959 Ops.pop_back();
5960 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl");
5961 }
5962 case NEON::BI__builtin_neon_vqshlud_n_s64: {
5963 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
5964 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty),
5965 Ops, "vqshlu_n");
5966 }
5967 case NEON::BI__builtin_neon_vqshld_n_u64:
5968 case NEON::BI__builtin_neon_vqshld_n_s64: {
5969 Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
5970 ? Intrinsic::aarch64_neon_uqshl
5971 : Intrinsic::aarch64_neon_sqshl;
5972 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
5973 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n");
5974 }
5975 case NEON::BI__builtin_neon_vrshrd_n_u64:
5976 case NEON::BI__builtin_neon_vrshrd_n_s64: {
5977 Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
5978 ? Intrinsic::aarch64_neon_urshl
5979 : Intrinsic::aarch64_neon_srshl;
5980 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
5981 Ops[1] = ConstantInt::get(Int64Ty, -SV);
5982 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n");
5983 }
5984 case NEON::BI__builtin_neon_vrsrad_n_u64:
5985 case NEON::BI__builtin_neon_vrsrad_n_s64: {
5986 Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
5987 ? Intrinsic::aarch64_neon_urshl
5988 : Intrinsic::aarch64_neon_srshl;
5989 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
5990 Ops[2] = Builder.CreateNeg(Ops[2]);
5991 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty),
5992 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
5993 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
5994 }
5995 case NEON::BI__builtin_neon_vshld_n_s64:
5996 case NEON::BI__builtin_neon_vshld_n_u64: {
5997 llvm::ConstantInt *Amt = cast<ConstantInt>(Ops[1]);
5998 return Builder.CreateShl(
5999 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n");
6000 }
6001 case NEON::BI__builtin_neon_vshrd_n_s64: {
6002 llvm::ConstantInt *Amt = cast<ConstantInt>(Ops[1]);
6003 return Builder.CreateAShr(
6004 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
6005 Amt->getZExtValue())),
6006 "shrd_n");
6007 }
6008 case NEON::BI__builtin_neon_vshrd_n_u64: {
6009 llvm::ConstantInt *Amt = cast<ConstantInt>(Ops[1]);
6010 uint64_t ShiftAmt = Amt->getZExtValue();
6011 // Right-shifting an unsigned value by its size yields 0.
6012 if (ShiftAmt == 64)
6013 return ConstantInt::get(Int64Ty, 0);
6014 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt),
6015 "shrd_n");
6016 }
6017 case NEON::BI__builtin_neon_vsrad_n_s64: {
6018 llvm::ConstantInt *Amt = cast<ConstantInt>(Ops[2]);
6019 Ops[1] = Builder.CreateAShr(
6020 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
6021 Amt->getZExtValue())),
6022 "shrd_n");
6023 return Builder.CreateAdd(Ops[0], Ops[1]);
6024 }
6025 case NEON::BI__builtin_neon_vsrad_n_u64: {
6026 llvm::ConstantInt *Amt = cast<ConstantInt>(Ops[2]);
6027 uint64_t ShiftAmt = Amt->getZExtValue();
6028 // Right-shifting an unsigned value by its size yields 0.
6029 // As Op + 0 = Op, return Ops[0] directly.
6030 if (ShiftAmt == 64)
6031 return Ops[0];
6032 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
6033 "shrd_n");
6034 return Builder.CreateAdd(Ops[0], Ops[1]);
6035 }
6036 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
6037 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
6038 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
6039 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
6040 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "lane");
6041 SmallVector<Value *, 2> ProductOps;
6042 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
6043 ProductOps.push_back(vectorWrapScalar16(Ops[2]));
6044 auto *VTy = llvm::FixedVectorType::get(Int32Ty, 4);
6045 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
6046 ProductOps, "vqdmlXl");
6047 Constant *CI = ConstantInt::get(SizeTy, 0);
6048 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
6049 // Drop lane-selection and the corresponding vector argument (these have
6050 // already been used)
6051 Ops.pop_back_n(2);
6052
6053 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
6054 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
6055 ? Intrinsic::aarch64_neon_sqadd
6056 : Intrinsic::aarch64_neon_sqsub;
6057 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl");
6058 }
6059 case NEON::BI__builtin_neon_vqdmlals_s32:
6060 case NEON::BI__builtin_neon_vqdmlsls_s32: {
6061 SmallVector<Value *, 2> ProductOps;
6062 ProductOps.push_back(Ops[1]);
6063 ProductOps.push_back(Ops[2]);
6064 Ops[1] =
6065 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
6066 ProductOps, "vqdmlXl");
6067
6068 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
6069 ? Intrinsic::aarch64_neon_sqadd
6070 : Intrinsic::aarch64_neon_sqsub;
6071 // Drop the 2nd multiplication argument before the accumulation
6072 Ops.pop_back();
6073 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl");
6074 }
6075 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
6076 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
6077 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
6078 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
6079 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "lane");
6080 SmallVector<Value *, 2> ProductOps;
6081 ProductOps.push_back(Ops[1]);
6082 ProductOps.push_back(Ops[2]);
6083 Ops[1] =
6084 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
6085 ProductOps, "vqdmlXl");
6086 // Drop lane-selection and the corresponding vector argument (these have
6087 // already been used)
6088 Ops.pop_back_n(2);
6089
6090 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
6091 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
6092 ? Intrinsic::aarch64_neon_sqadd
6093 : Intrinsic::aarch64_neon_sqsub;
6094 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl");
6095 }
6096 case NEON::BI__builtin_neon_vget_lane_bf16:
6097 case NEON::BI__builtin_neon_vduph_lane_bf16:
6098 case NEON::BI__builtin_neon_vduph_lane_f16: {
6099 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
6100 }
6101 case NEON::BI__builtin_neon_vgetq_lane_bf16:
6102 case NEON::BI__builtin_neon_vduph_laneq_bf16:
6103 case NEON::BI__builtin_neon_vduph_laneq_f16: {
6104 return Builder.CreateExtractElement(Ops[0], Ops[1], "vgetq_lane");
6105 }
6106 case NEON::BI__builtin_neon_vcvt_bf16_f32: {
6107 llvm::Type *V4F32 = FixedVectorType::get(Builder.getFloatTy(), 4);
6108 llvm::Type *V4BF16 = FixedVectorType::get(Builder.getBFloatTy(), 4);
6109 return Builder.CreateFPTrunc(Builder.CreateBitCast(Ops[0], V4F32), V4BF16);
6110 }
6111 case NEON::BI__builtin_neon_vcvtq_low_bf16_f32: {
6112 SmallVector<int, 16> ConcatMask(8);
6113 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
6114 llvm::Type *V4F32 = FixedVectorType::get(Builder.getFloatTy(), 4);
6115 llvm::Type *V4BF16 = FixedVectorType::get(Builder.getBFloatTy(), 4);
6116 llvm::Value *Trunc =
6117 Builder.CreateFPTrunc(Builder.CreateBitCast(Ops[0], V4F32), V4BF16);
6118 return Builder.CreateShuffleVector(
6119 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
6120 }
6121 case NEON::BI__builtin_neon_vcvtq_high_bf16_f32: {
6122 SmallVector<int, 16> ConcatMask(8);
6123 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
6124 SmallVector<int, 16> LoMask(4);
6125 std::iota(LoMask.begin(), LoMask.end(), 0);
6126 llvm::Type *V4F32 = FixedVectorType::get(Builder.getFloatTy(), 4);
6127 llvm::Type *V4BF16 = FixedVectorType::get(Builder.getBFloatTy(), 4);
6128 llvm::Type *V8BF16 = FixedVectorType::get(Builder.getBFloatTy(), 8);
6129 llvm::Value *Inactive = Builder.CreateShuffleVector(
6130 Builder.CreateBitCast(Ops[0], V8BF16), LoMask);
6131 llvm::Value *Trunc =
6132 Builder.CreateFPTrunc(Builder.CreateBitCast(Ops[1], V4F32), V4BF16);
6133 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
6134 }
6135 case NEON::BI__builtin_neon_vcvt_f16_f32: {
6136 llvm::Type *V4F32 = FixedVectorType::get(Builder.getFloatTy(), 4);
6137 llvm::Type *V4F16 = FixedVectorType::get(Builder.getHalfTy(), 4);
6138 return Builder.CreateFPTrunc(Builder.CreateBitCast(Ops[0], V4F32), V4F16);
6139 }
6140 case NEON::BI__builtin_neon_vcvt_f32_f16: {
6141 llvm::Type *V4F32 = FixedVectorType::get(Builder.getFloatTy(), 4);
6142 llvm::Type *V4F16 = FixedVectorType::get(Builder.getHalfTy(), 4);
6143 return Builder.CreateFPExt(Builder.CreateBitCast(Ops[0], V4F16), V4F32);
6144 }
6145
6146 case clang::AArch64::BI_InterlockedAdd:
6147 case clang::AArch64::BI_InterlockedAdd_acq:
6148 case clang::AArch64::BI_InterlockedAdd_rel:
6149 case clang::AArch64::BI_InterlockedAdd_nf:
6150 case clang::AArch64::BI_InterlockedAdd64:
6151 case clang::AArch64::BI_InterlockedAdd64_acq:
6152 case clang::AArch64::BI_InterlockedAdd64_rel:
6153 case clang::AArch64::BI_InterlockedAdd64_nf: {
6154 Address DestAddr = CheckAtomicAlignment(*this, E);
6155 Value *Val = Ops[1];
6156 llvm::AtomicOrdering Ordering;
6157 switch (BuiltinID) {
6158 case clang::AArch64::BI_InterlockedAdd:
6159 case clang::AArch64::BI_InterlockedAdd64:
6160 Ordering = llvm::AtomicOrdering::SequentiallyConsistent;
6161 break;
6162 case clang::AArch64::BI_InterlockedAdd_acq:
6163 case clang::AArch64::BI_InterlockedAdd64_acq:
6164 Ordering = llvm::AtomicOrdering::Acquire;
6165 break;
6166 case clang::AArch64::BI_InterlockedAdd_rel:
6167 case clang::AArch64::BI_InterlockedAdd64_rel:
6168 Ordering = llvm::AtomicOrdering::Release;
6169 break;
6170 case clang::AArch64::BI_InterlockedAdd_nf:
6171 case clang::AArch64::BI_InterlockedAdd64_nf:
6172 Ordering = llvm::AtomicOrdering::Monotonic;
6173 break;
6174 default:
6175 llvm_unreachable("missing builtin ID in switch!");
6176 }
6177 AtomicRMWInst *RMWI =
6178 Builder.CreateAtomicRMW(AtomicRMWInst::Add, DestAddr, Val, Ordering);
6179 return Builder.CreateAdd(RMWI, Val);
6180 }
6181 }
6182
6183 llvm::FixedVectorType *VTy = GetNeonType(this, Type);
6184 llvm::Type *Ty = VTy;
6185 if (!Ty)
6186 return nullptr;
6187
6188 bool ExtractLow = false;
6189 bool ExtendLaneArg = false;
6190 switch (BuiltinID) {
6191 default: return nullptr;
6192 case NEON::BI__builtin_neon_vbsl_v:
6193 case NEON::BI__builtin_neon_vbslq_v: {
6194 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
6195 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl");
6196 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl");
6197 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl");
6198
6199 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl");
6200 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl");
6201 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl");
6202 return Builder.CreateBitCast(Ops[0], Ty);
6203 }
6204 case NEON::BI__builtin_neon_vfma_lane_v:
6205 case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types
6206 // The ARM builtins (and instructions) have the addend as the first
6207 // operand, but the 'fma' intrinsics have it last. Swap it around here.
6208 Value *Addend = Ops[0];
6209 Value *Multiplicand = Ops[1];
6210 Value *LaneSource = Ops[2];
6211 Ops[0] = Multiplicand;
6212 Ops[1] = LaneSource;
6213 Ops[2] = Addend;
6214
6215 // Now adjust things to handle the lane access.
6216 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
6217 ? llvm::FixedVectorType::get(VTy->getElementType(),
6218 VTy->getNumElements() / 2)
6219 : VTy;
6220 llvm::Constant *cst = cast<Constant>(Ops[3]);
6221 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
6222 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
6223 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
6224
6225 Ops.pop_back();
6226 Int = Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
6227 : Intrinsic::fma;
6228 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla");
6229 }
6230 case NEON::BI__builtin_neon_vfma_laneq_v: {
6231 auto *VTy = cast<llvm::FixedVectorType>(Ty);
6232 // v1f64 fma should be mapped to Neon scalar f64 fma
6233 if (VTy && VTy->getElementType() == DoubleTy) {
6234 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
6235 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
6236 llvm::FixedVectorType *VTy =
6238 Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
6239 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
6240 Value *Result;
6242 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
6243 DoubleTy, {Ops[1], Ops[2], Ops[0]});
6244 return Builder.CreateBitCast(Result, Ty);
6245 }
6246 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6247 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6248
6249 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
6250 VTy->getNumElements() * 2);
6251 Ops[2] = Builder.CreateBitCast(Ops[2], STy);
6252 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
6253 cast<ConstantInt>(Ops[3]));
6254 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
6255
6257 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
6258 {Ops[2], Ops[1], Ops[0]});
6259 }
6260 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
6261 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6262 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6263
6264 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6265 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
6267 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
6268 {Ops[2], Ops[1], Ops[0]});
6269 }
6270 case NEON::BI__builtin_neon_vfmah_lane_f16:
6271 case NEON::BI__builtin_neon_vfmas_lane_f32:
6272 case NEON::BI__builtin_neon_vfmah_laneq_f16:
6273 case NEON::BI__builtin_neon_vfmas_laneq_f32:
6274 case NEON::BI__builtin_neon_vfmad_lane_f64:
6275 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
6276 llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
6277 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
6279 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
6280 {Ops[1], Ops[2], Ops[0]});
6281 }
6282 case NEON::BI__builtin_neon_vmull_v:
6283 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6284 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
6285 if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
6286 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
6287 case NEON::BI__builtin_neon_vmax_v:
6288 case NEON::BI__builtin_neon_vmaxq_v:
6289 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6290 Int = usgn ? Intrinsic::umax : Intrinsic::smax;
6291 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
6292 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax");
6293 case NEON::BI__builtin_neon_vmaxh_f16: {
6294 Int = Intrinsic::aarch64_neon_fmax;
6295 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmax");
6296 }
6297 case NEON::BI__builtin_neon_vmin_v:
6298 case NEON::BI__builtin_neon_vminq_v:
6299 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6300 Int = usgn ? Intrinsic::umin : Intrinsic::smin;
6301 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
6302 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin");
6303 case NEON::BI__builtin_neon_vminh_f16: {
6304 Int = Intrinsic::aarch64_neon_fmin;
6305 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmin");
6306 }
6307 case NEON::BI__builtin_neon_vabd_v:
6308 case NEON::BI__builtin_neon_vabdq_v:
6309 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6310 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
6311 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
6312 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd");
6313 case NEON::BI__builtin_neon_vpadal_v:
6314 case NEON::BI__builtin_neon_vpadalq_v: {
6315 unsigned ArgElts = VTy->getNumElements();
6316 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
6317 unsigned BitWidth = EltTy->getBitWidth();
6318 auto *ArgTy = llvm::FixedVectorType::get(
6319 llvm::IntegerType::get(getLLVMContext(), BitWidth / 2), 2 * ArgElts);
6320 llvm::Type* Tys[2] = { VTy, ArgTy };
6321 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
6323 TmpOps.push_back(Ops[1]);
6324 Function *F = CGM.getIntrinsic(Int, Tys);
6325 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal");
6326 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
6327 return Builder.CreateAdd(tmp, addend);
6328 }
6329 case NEON::BI__builtin_neon_vpmin_v:
6330 case NEON::BI__builtin_neon_vpminq_v:
6331 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6332 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
6333 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
6334 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin");
6335 case NEON::BI__builtin_neon_vpmax_v:
6336 case NEON::BI__builtin_neon_vpmaxq_v:
6337 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6338 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
6339 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
6340 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax");
6341 case NEON::BI__builtin_neon_vminnm_v:
6342 case NEON::BI__builtin_neon_vminnmq_v:
6343 Int = Intrinsic::aarch64_neon_fminnm;
6344 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm");
6345 case NEON::BI__builtin_neon_vminnmh_f16:
6346 Int = Intrinsic::aarch64_neon_fminnm;
6347 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vminnm");
6348 case NEON::BI__builtin_neon_vmaxnm_v:
6349 case NEON::BI__builtin_neon_vmaxnmq_v:
6350 Int = Intrinsic::aarch64_neon_fmaxnm;
6351 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm");
6352 case NEON::BI__builtin_neon_vmaxnmh_f16:
6353 Int = Intrinsic::aarch64_neon_fmaxnm;
6354 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmaxnm");
6355 case NEON::BI__builtin_neon_vrecpss_f32: {
6356 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy),
6357 Ops, "vrecps");
6358 }
6359 case NEON::BI__builtin_neon_vrecpsd_f64:
6360 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy),
6361 Ops, "vrecps");
6362 case NEON::BI__builtin_neon_vrecpsh_f16:
6363 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, HalfTy),
6364 Ops, "vrecps");
6365 case NEON::BI__builtin_neon_vqshrun_n_v:
6366 Int = Intrinsic::aarch64_neon_sqshrun;
6367 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n");
6368 case NEON::BI__builtin_neon_vqrshrun_n_v:
6369 Int = Intrinsic::aarch64_neon_sqrshrun;
6370 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n");
6371 case NEON::BI__builtin_neon_vqshrn_n_v:
6372 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
6373 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n");
6374 case NEON::BI__builtin_neon_vrshrn_n_v:
6375 Int = Intrinsic::aarch64_neon_rshrn;
6376 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n");
6377 case NEON::BI__builtin_neon_vqrshrn_n_v:
6378 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
6379 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n");
6380 case NEON::BI__builtin_neon_vrndah_f16: {
6381 Int = Builder.getIsFPConstrained()
6382 ? Intrinsic::experimental_constrained_round
6383 : Intrinsic::round;
6384 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrnda");
6385 }
6386 case NEON::BI__builtin_neon_vrnda_v:
6387 case NEON::BI__builtin_neon_vrndaq_v: {
6388 Int = Builder.getIsFPConstrained()
6389 ? Intrinsic::experimental_constrained_round
6390 : Intrinsic::round;
6391 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda");
6392 }
6393 case NEON::BI__builtin_neon_vrndih_f16: {
6394 Int = Builder.getIsFPConstrained()
6395 ? Intrinsic::experimental_constrained_nearbyint
6396 : Intrinsic::nearbyint;
6397 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndi");
6398 }
6399 case NEON::BI__builtin_neon_vrndmh_f16: {
6400 Int = Builder.getIsFPConstrained()
6401 ? Intrinsic::experimental_constrained_floor
6402 : Intrinsic::floor;
6403 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndm");
6404 }
6405 case NEON::BI__builtin_neon_vrndm_v:
6406 case NEON::BI__builtin_neon_vrndmq_v: {
6407 Int = Builder.getIsFPConstrained()
6408 ? Intrinsic::experimental_constrained_floor
6409 : Intrinsic::floor;
6410 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm");
6411 }
6412 case NEON::BI__builtin_neon_vrndnh_f16: {
6413 Int = Builder.getIsFPConstrained()
6414 ? Intrinsic::experimental_constrained_roundeven
6415 : Intrinsic::roundeven;
6416 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndn");
6417 }
6418 case NEON::BI__builtin_neon_vrndn_v:
6419 case NEON::BI__builtin_neon_vrndnq_v: {
6420 Int = Builder.getIsFPConstrained()
6421 ? Intrinsic::experimental_constrained_roundeven
6422 : Intrinsic::roundeven;
6423 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn");
6424 }
6425 case NEON::BI__builtin_neon_vrndns_f32: {
6426 Int = Builder.getIsFPConstrained()
6427 ? Intrinsic::experimental_constrained_roundeven
6428 : Intrinsic::roundeven;
6429 return EmitNeonCall(CGM.getIntrinsic(Int, FloatTy), Ops, "vrndn");
6430 }
6431 case NEON::BI__builtin_neon_vrndph_f16: {
6432 Int = Builder.getIsFPConstrained()
6433 ? Intrinsic::experimental_constrained_ceil
6434 : Intrinsic::ceil;
6435 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndp");
6436 }
6437 case NEON::BI__builtin_neon_vrndp_v:
6438 case NEON::BI__builtin_neon_vrndpq_v: {
6439 Int = Builder.getIsFPConstrained()
6440 ? Intrinsic::experimental_constrained_ceil
6441 : Intrinsic::ceil;
6442 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp");
6443 }
6444 case NEON::BI__builtin_neon_vrndxh_f16: {
6445 Int = Builder.getIsFPConstrained()
6446 ? Intrinsic::experimental_constrained_rint
6447 : Intrinsic::rint;
6448 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndx");
6449 }
6450 case NEON::BI__builtin_neon_vrndx_v:
6451 case NEON::BI__builtin_neon_vrndxq_v: {
6452 Int = Builder.getIsFPConstrained()
6453 ? Intrinsic::experimental_constrained_rint
6454 : Intrinsic::rint;
6455 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx");
6456 }
6457 case NEON::BI__builtin_neon_vrndh_f16: {
6458 Int = Builder.getIsFPConstrained()
6459 ? Intrinsic::experimental_constrained_trunc
6460 : Intrinsic::trunc;
6461 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndz");
6462 }
6463 case NEON::BI__builtin_neon_vrnd_v:
6464 case NEON::BI__builtin_neon_vrndq_v: {
6465 Int = Builder.getIsFPConstrained()
6466 ? Intrinsic::experimental_constrained_trunc
6467 : Intrinsic::trunc;
6468 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz");
6469 }
6470 case NEON::BI__builtin_neon_vcvt_f64_v:
6471 case NEON::BI__builtin_neon_vcvtq_f64_v:
6472 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6473 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
6474 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
6475 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
6476 case NEON::BI__builtin_neon_vcvt_f64_f32: {
6477 assert(Type.getEltType() == NeonTypeFlags::Float64 && quad &&
6478 "unexpected vcvt_f64_f32 builtin");
6479 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false);
6480 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
6481
6482 return Builder.CreateFPExt(Ops[0], Ty, "vcvt");
6483 }
6484 case NEON::BI__builtin_neon_vcvt_f32_f64: {
6485 assert(Type.getEltType() == NeonTypeFlags::Float32 &&
6486 "unexpected vcvt_f32_f64 builtin");
6487 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true);
6488 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
6489
6490 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt");
6491 }
6492 case NEON::BI__builtin_neon_vcvta_s16_f16:
6493 case NEON::BI__builtin_neon_vcvta_u16_f16:
6494 case NEON::BI__builtin_neon_vcvta_s32_v:
6495 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
6496 case NEON::BI__builtin_neon_vcvtaq_s32_v:
6497 case NEON::BI__builtin_neon_vcvta_u32_v:
6498 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
6499 case NEON::BI__builtin_neon_vcvtaq_u32_v:
6500 case NEON::BI__builtin_neon_vcvta_s64_v:
6501 case NEON::BI__builtin_neon_vcvtaq_s64_v:
6502 case NEON::BI__builtin_neon_vcvta_u64_v:
6503 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
6504 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
6505 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6506 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta");
6507 }
6508 case NEON::BI__builtin_neon_vcvtm_s16_f16:
6509 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
6510 case NEON::BI__builtin_neon_vcvtm_u16_f16:
6511 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
6512 case NEON::BI__builtin_neon_vcvtm_s32_v:
6513 case NEON::BI__builtin_neon_vcvtmq_s32_v:
6514 case NEON::BI__builtin_neon_vcvtm_u32_v:
6515 case NEON::BI__builtin_neon_vcvtmq_u32_v:
6516 case NEON::BI__builtin_neon_vcvtm_s64_v:
6517 case NEON::BI__builtin_neon_vcvtmq_s64_v:
6518 case NEON::BI__builtin_neon_vcvtm_u64_v:
6519 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
6520 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
6521 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6522 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm");
6523 }
6524 case NEON::BI__builtin_neon_vcvtn_s16_f16:
6525 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
6526 case NEON::BI__builtin_neon_vcvtn_u16_f16:
6527 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
6528 case NEON::BI__builtin_neon_vcvtn_s32_v:
6529 case NEON::BI__builtin_neon_vcvtnq_s32_v:
6530 case NEON::BI__builtin_neon_vcvtn_u32_v:
6531 case NEON::BI__builtin_neon_vcvtnq_u32_v:
6532 case NEON::BI__builtin_neon_vcvtn_s64_v:
6533 case NEON::BI__builtin_neon_vcvtnq_s64_v:
6534 case NEON::BI__builtin_neon_vcvtn_u64_v:
6535 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
6536 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
6537 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6538 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn");
6539 }
6540 case NEON::BI__builtin_neon_vcvtp_s16_f16:
6541 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
6542 case NEON::BI__builtin_neon_vcvtp_u16_f16:
6543 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
6544 case NEON::BI__builtin_neon_vcvtp_s32_v:
6545 case NEON::BI__builtin_neon_vcvtpq_s32_v:
6546 case NEON::BI__builtin_neon_vcvtp_u32_v:
6547 case NEON::BI__builtin_neon_vcvtpq_u32_v:
6548 case NEON::BI__builtin_neon_vcvtp_s64_v:
6549 case NEON::BI__builtin_neon_vcvtpq_s64_v:
6550 case NEON::BI__builtin_neon_vcvtp_u64_v:
6551 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
6552 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
6553 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6554 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp");
6555 }
6556 case NEON::BI__builtin_neon_vmulx_v:
6557 case NEON::BI__builtin_neon_vmulxq_v: {
6558 Int = Intrinsic::aarch64_neon_fmulx;
6559 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx");
6560 }
6561 case NEON::BI__builtin_neon_vmulxh_lane_f16:
6562 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
6563 // vmulx_lane should be mapped to Neon scalar mulx after
6564 // extracting the scalar element
6565 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
6566 Ops.pop_back();
6567 Int = Intrinsic::aarch64_neon_fmulx;
6568 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmulx");
6569 }
6570 case NEON::BI__builtin_neon_vmul_lane_v:
6571 case NEON::BI__builtin_neon_vmul_laneq_v: {
6572 // v1f64 vmul_lane should be mapped to Neon scalar mul lane
6573 bool Quad = false;
6574 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
6575 Quad = true;
6576 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
6577 llvm::FixedVectorType *VTy =
6579 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
6580 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
6581 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
6582 return Builder.CreateBitCast(Result, Ty);
6583 }
6584 case NEON::BI__builtin_neon_vpmaxnm_v:
6585 case NEON::BI__builtin_neon_vpmaxnmq_v: {
6586 Int = Intrinsic::aarch64_neon_fmaxnmp;
6587 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm");
6588 }
6589 case NEON::BI__builtin_neon_vpminnm_v:
6590 case NEON::BI__builtin_neon_vpminnmq_v: {
6591 Int = Intrinsic::aarch64_neon_fminnmp;
6592 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm");
6593 }
6594 case NEON::BI__builtin_neon_vsqrth_f16: {
6595 Int = Builder.getIsFPConstrained()
6596 ? Intrinsic::experimental_constrained_sqrt
6597 : Intrinsic::sqrt;
6598 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vsqrt");
6599 }
6600 case NEON::BI__builtin_neon_vsqrt_v:
6601 case NEON::BI__builtin_neon_vsqrtq_v: {
6602 Int = Builder.getIsFPConstrained()
6603 ? Intrinsic::experimental_constrained_sqrt
6604 : Intrinsic::sqrt;
6605 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6606 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt");
6607 }
6608 case NEON::BI__builtin_neon_vrbit_v:
6609 case NEON::BI__builtin_neon_vrbitq_v: {
6610 Int = Intrinsic::bitreverse;
6611 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit");
6612 }
6613 case NEON::BI__builtin_neon_vmaxv_f16: {
6614 Int = Intrinsic::aarch64_neon_fmaxv;
6615 Ty = HalfTy;
6616 VTy = llvm::FixedVectorType::get(HalfTy, 4);
6617 llvm::Type *Tys[2] = {Ty, VTy};
6618 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6619 }
6620 case NEON::BI__builtin_neon_vmaxvq_f16: {
6621 Int = Intrinsic::aarch64_neon_fmaxv;
6622 Ty = HalfTy;
6623 VTy = llvm::FixedVectorType::get(HalfTy, 8);
6624 llvm::Type *Tys[2] = {Ty, VTy};
6625 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6626 }
6627 case NEON::BI__builtin_neon_vminv_f16: {
6628 Int = Intrinsic::aarch64_neon_fminv;
6629 Ty = HalfTy;
6630 VTy = llvm::FixedVectorType::get(HalfTy, 4);
6631 llvm::Type *Tys[2] = {Ty, VTy};
6632 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6633 }
6634 case NEON::BI__builtin_neon_vminvq_f16: {
6635 Int = Intrinsic::aarch64_neon_fminv;
6636 Ty = HalfTy;
6637 VTy = llvm::FixedVectorType::get(HalfTy, 8);
6638 llvm::Type *Tys[2] = {Ty, VTy};
6639 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6640 }
6641 case NEON::BI__builtin_neon_vmaxnmv_f16: {
6642 Int = Intrinsic::aarch64_neon_fmaxnmv;
6643 Ty = HalfTy;
6644 VTy = llvm::FixedVectorType::get(HalfTy, 4);
6645 llvm::Type *Tys[2] = {Ty, VTy};
6646 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv");
6647 }
6648 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
6649 Int = Intrinsic::aarch64_neon_fmaxnmv;
6650 Ty = HalfTy;
6651 VTy = llvm::FixedVectorType::get(HalfTy, 8);
6652 llvm::Type *Tys[2] = {Ty, VTy};
6653 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv");
6654 }
6655 case NEON::BI__builtin_neon_vminnmv_f16: {
6656 Int = Intrinsic::aarch64_neon_fminnmv;
6657 Ty = HalfTy;
6658 VTy = llvm::FixedVectorType::get(HalfTy, 4);
6659 llvm::Type *Tys[2] = {Ty, VTy};
6660 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv");
6661 }
6662 case NEON::BI__builtin_neon_vminnmvq_f16: {
6663 Int = Intrinsic::aarch64_neon_fminnmv;
6664 Ty = HalfTy;
6665 VTy = llvm::FixedVectorType::get(HalfTy, 8);
6666 llvm::Type *Tys[2] = {Ty, VTy};
6667 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv");
6668 }
6669 case NEON::BI__builtin_neon_vmul_n_f64: {
6670 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
6671 Value *RHS = Builder.CreateBitCast(Ops[1], DoubleTy);
6672 return Builder.CreateFMul(Ops[0], RHS);
6673 }
6674 case NEON::BI__builtin_neon_vaddlv_u8:
6675 case NEON::BI__builtin_neon_vaddlvq_u8:
6676 case NEON::BI__builtin_neon_vaddlv_u16:
6677 case NEON::BI__builtin_neon_vaddlvq_u16: {
6678 Int = Intrinsic::aarch64_neon_uaddlv;
6679 Ty = Int32Ty;
6680 VTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
6681 llvm::Type *Tys[2] = {Ty, VTy};
6682 Value *Result = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6683 if (VTy->getElementType()->getPrimitiveSizeInBits() == 8)
6684 return Builder.CreateTrunc(Result, Int16Ty);
6685 return Result;
6686 }
6687 case NEON::BI__builtin_neon_vaddlv_s8:
6688 case NEON::BI__builtin_neon_vaddlvq_s8:
6689 case NEON::BI__builtin_neon_vaddlv_s16:
6690 case NEON::BI__builtin_neon_vaddlvq_s16: {
6691 Int = Intrinsic::aarch64_neon_saddlv;
6692 Ty = Int32Ty;
6693 VTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
6694 llvm::Type *Tys[2] = {Ty, VTy};
6695 Value *Result = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6696 if (VTy->getElementType()->getPrimitiveSizeInBits() == 8)
6697 return Builder.CreateTrunc(Result, Int16Ty);
6698 return Result;
6699 }
6700 case NEON::BI__builtin_neon_vsri_n_v:
6701 case NEON::BI__builtin_neon_vsriq_n_v: {
6702 Int = Intrinsic::aarch64_neon_vsri;
6703 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
6704 return EmitNeonCall(Intrin, Ops, "vsri_n");
6705 }
6706 case NEON::BI__builtin_neon_vsli_n_v:
6707 case NEON::BI__builtin_neon_vsliq_n_v: {
6708 Int = Intrinsic::aarch64_neon_vsli;
6709 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
6710 return EmitNeonCall(Intrin, Ops, "vsli_n");
6711 }
6712 case NEON::BI__builtin_neon_vsra_n_v:
6713 case NEON::BI__builtin_neon_vsraq_n_v:
6714 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6715 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
6716 return Builder.CreateAdd(Ops[0], Ops[1]);
6717 case NEON::BI__builtin_neon_vrsra_n_v:
6718 case NEON::BI__builtin_neon_vrsraq_n_v: {
6719 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
6721 TmpOps.push_back(Ops[1]);
6722 TmpOps.push_back(Ops[2]);
6723 Function* F = CGM.getIntrinsic(Int, Ty);
6724 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true);
6725 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
6726 return Builder.CreateAdd(Ops[0], tmp);
6727 }
6728 case NEON::BI__builtin_neon_vld1_v:
6729 case NEON::BI__builtin_neon_vld1q_v: {
6730 return Builder.CreateAlignedLoad(VTy, Ops[0], PtrOp0.getAlignment());
6731 }
6732 case NEON::BI__builtin_neon_vst1_v:
6733 case NEON::BI__builtin_neon_vst1q_v:
6734 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
6735 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment());
6736 case NEON::BI__builtin_neon_vld1_lane_v:
6737 case NEON::BI__builtin_neon_vld1q_lane_v: {
6738 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6739 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
6740 PtrOp0.getAlignment());
6741 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane");
6742 }
6743 case NEON::BI__builtin_neon_vldap1_lane_s64:
6744 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
6745 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6746 llvm::LoadInst *LI = Builder.CreateAlignedLoad(
6747 VTy->getElementType(), Ops[0], PtrOp0.getAlignment());
6748 LI->setAtomic(llvm::AtomicOrdering::Acquire);
6749 Ops[0] = LI;
6750 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vldap1_lane");
6751 }
6752 case NEON::BI__builtin_neon_vld1_dup_v:
6753 case NEON::BI__builtin_neon_vld1q_dup_v: {
6754 Value *V = PoisonValue::get(Ty);
6755 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
6756 PtrOp0.getAlignment());
6757 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
6758 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
6759 return EmitNeonSplat(Ops[0], CI);
6760 }
6761 case NEON::BI__builtin_neon_vst1_lane_v:
6762 case NEON::BI__builtin_neon_vst1q_lane_v:
6763 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6764 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
6765 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment());
6766 case NEON::BI__builtin_neon_vstl1_lane_s64:
6767 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
6768 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6769 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
6770 llvm::StoreInst *SI =
6771 Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment());
6772 SI->setAtomic(llvm::AtomicOrdering::Release);
6773 return SI;
6774 }
6775 case NEON::BI__builtin_neon_vld2_v:
6776 case NEON::BI__builtin_neon_vld2q_v: {
6777 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6778 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys);
6779 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
6780 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6781 }
6782 case NEON::BI__builtin_neon_vld3_v:
6783 case NEON::BI__builtin_neon_vld3q_v: {
6784 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6785 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys);
6786 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
6787 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6788 }
6789 case NEON::BI__builtin_neon_vld4_v:
6790 case NEON::BI__builtin_neon_vld4q_v: {
6791 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6792 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys);
6793 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
6794 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6795 }
6796 case NEON::BI__builtin_neon_vld2_dup_v:
6797 case NEON::BI__builtin_neon_vld2q_dup_v: {
6798 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6799 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys);
6800 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
6801 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6802 }
6803 case NEON::BI__builtin_neon_vld3_dup_v:
6804 case NEON::BI__builtin_neon_vld3q_dup_v: {
6805 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6806 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys);
6807 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
6808 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6809 }
6810 case NEON::BI__builtin_neon_vld4_dup_v:
6811 case NEON::BI__builtin_neon_vld4q_dup_v: {
6812 llvm::Type *Tys[2] = {VTy, DefaultPtrTy};
6813 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys);
6814 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
6815 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6816 }
6817 case NEON::BI__builtin_neon_vld2_lane_v:
6818 case NEON::BI__builtin_neon_vld2q_lane_v: {
6819 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6820 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
6821 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
6822 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6823 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6824 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
6825 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld2_lane");
6826 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6827 }
6828 case NEON::BI__builtin_neon_vld3_lane_v:
6829 case NEON::BI__builtin_neon_vld3q_lane_v: {
6830 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6831 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
6832 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
6833 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6834 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6835 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
6836 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
6837 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld3_lane");
6838 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6839 }
6840 case NEON::BI__builtin_neon_vld4_lane_v:
6841 case NEON::BI__builtin_neon_vld4q_lane_v: {
6842 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6843 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
6844 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
6845 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6846 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6847 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
6848 Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
6849 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty);
6850 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld4_lane");
6851 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6852 }
6853 case NEON::BI__builtin_neon_vst2_v:
6854 case NEON::BI__builtin_neon_vst2q_v: {
6855 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6856 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
6857 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys),
6858 Ops, "");
6859 }
6860 case NEON::BI__builtin_neon_vst2_lane_v:
6861 case NEON::BI__builtin_neon_vst2q_lane_v: {
6862 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6863 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
6864 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6865 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys),
6866 Ops, "");
6867 }
6868 case NEON::BI__builtin_neon_vst3_v:
6869 case NEON::BI__builtin_neon_vst3q_v: {
6870 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6871 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6872 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys),
6873 Ops, "");
6874 }
6875 case NEON::BI__builtin_neon_vst3_lane_v:
6876 case NEON::BI__builtin_neon_vst3q_lane_v: {
6877 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6878 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
6879 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6880 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys),
6881 Ops, "");
6882 }
6883 case NEON::BI__builtin_neon_vst4_v:
6884 case NEON::BI__builtin_neon_vst4q_v: {
6885 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6886 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6887 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys),
6888 Ops, "");
6889 }
6890 case NEON::BI__builtin_neon_vst4_lane_v:
6891 case NEON::BI__builtin_neon_vst4q_lane_v: {
6892 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6893 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
6894 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
6895 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys),
6896 Ops, "");
6897 }
6898 case NEON::BI__builtin_neon_vtrn_v:
6899 case NEON::BI__builtin_neon_vtrnq_v: {
6900 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6901 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6902 Value *SV = nullptr;
6903
6904 for (unsigned vi = 0; vi != 2; ++vi) {
6905 SmallVector<int, 16> Indices;
6906 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6907 Indices.push_back(i+vi);
6908 Indices.push_back(i+e+vi);
6909 }
6910 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6911 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
6912 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
6913 }
6914 return SV;
6915 }
6916 case NEON::BI__builtin_neon_vuzp_v:
6917 case NEON::BI__builtin_neon_vuzpq_v: {
6918 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6919 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6920 Value *SV = nullptr;
6921
6922 for (unsigned vi = 0; vi != 2; ++vi) {
6923 SmallVector<int, 16> Indices;
6924 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
6925 Indices.push_back(2*i+vi);
6926
6927 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6928 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
6929 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
6930 }
6931 return SV;
6932 }
6933 case NEON::BI__builtin_neon_vzip_v:
6934 case NEON::BI__builtin_neon_vzipq_v: {
6935 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6936 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6937 Value *SV = nullptr;
6938
6939 for (unsigned vi = 0; vi != 2; ++vi) {
6940 SmallVector<int, 16> Indices;
6941 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6942 Indices.push_back((i + vi*e) >> 1);
6943 Indices.push_back(((i + vi*e) >> 1)+e);
6944 }
6945 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6946 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
6947 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
6948 }
6949 return SV;
6950 }
6951 case NEON::BI__builtin_neon_vqtbl1q_v: {
6952 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty),
6953 Ops, "vtbl1");
6954 }
6955 case NEON::BI__builtin_neon_vqtbl2q_v: {
6956 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty),
6957 Ops, "vtbl2");
6958 }
6959 case NEON::BI__builtin_neon_vqtbl3q_v: {
6960 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty),
6961 Ops, "vtbl3");
6962 }
6963 case NEON::BI__builtin_neon_vqtbl4q_v: {
6964 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty),
6965 Ops, "vtbl4");
6966 }
6967 case NEON::BI__builtin_neon_vqtbx1q_v: {
6968 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty),
6969 Ops, "vtbx1");
6970 }
6971 case NEON::BI__builtin_neon_vqtbx2q_v: {
6972 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty),
6973 Ops, "vtbx2");
6974 }
6975 case NEON::BI__builtin_neon_vqtbx3q_v: {
6976 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty),
6977 Ops, "vtbx3");
6978 }
6979 case NEON::BI__builtin_neon_vqtbx4q_v: {
6980 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty),
6981 Ops, "vtbx4");
6982 }
6983 case NEON::BI__builtin_neon_vsqadd_v:
6984 case NEON::BI__builtin_neon_vsqaddq_v: {
6985 Int = Intrinsic::aarch64_neon_usqadd;
6986 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd");
6987 }
6988 case NEON::BI__builtin_neon_vuqadd_v:
6989 case NEON::BI__builtin_neon_vuqaddq_v: {
6990 Int = Intrinsic::aarch64_neon_suqadd;
6991 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd");
6992 }
6993
6994 case NEON::BI__builtin_neon_vluti2_laneq_mf8:
6995 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
6996 case NEON::BI__builtin_neon_vluti2_laneq_f16:
6997 case NEON::BI__builtin_neon_vluti2_laneq_p16:
6998 case NEON::BI__builtin_neon_vluti2_laneq_p8:
6999 case NEON::BI__builtin_neon_vluti2_laneq_s16:
7000 case NEON::BI__builtin_neon_vluti2_laneq_s8:
7001 case NEON::BI__builtin_neon_vluti2_laneq_u16:
7002 case NEON::BI__builtin_neon_vluti2_laneq_u8: {
7003 Int = Intrinsic::aarch64_neon_vluti2_laneq;
7004 llvm::Type *Tys[2];
7005 Tys[0] = Ty;
7006 Tys[1] = GetNeonType(this, NeonTypeFlags(Type.getEltType(), false,
7007 /*isQuad*/ false));
7008 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vluti2_laneq");
7009 }
7010 case NEON::BI__builtin_neon_vluti2q_laneq_mf8:
7011 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
7012 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
7013 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
7014 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
7015 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
7016 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
7017 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
7018 case NEON::BI__builtin_neon_vluti2q_laneq_u8: {
7019 Int = Intrinsic::aarch64_neon_vluti2_laneq;
7020 llvm::Type *Tys[2];
7021 Tys[0] = Ty;
7022 Tys[1] = GetNeonType(this, NeonTypeFlags(Type.getEltType(), false,
7023 /*isQuad*/ true));
7024 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vluti2_laneq");
7025 }
7026 case NEON::BI__builtin_neon_vluti2_lane_mf8:
7027 case NEON::BI__builtin_neon_vluti2_lane_bf16:
7028 case NEON::BI__builtin_neon_vluti2_lane_f16:
7029 case NEON::BI__builtin_neon_vluti2_lane_p16:
7030 case NEON::BI__builtin_neon_vluti2_lane_p8:
7031 case NEON::BI__builtin_neon_vluti2_lane_s16:
7032 case NEON::BI__builtin_neon_vluti2_lane_s8:
7033 case NEON::BI__builtin_neon_vluti2_lane_u16:
7034 case NEON::BI__builtin_neon_vluti2_lane_u8: {
7035 Int = Intrinsic::aarch64_neon_vluti2_lane;
7036 llvm::Type *Tys[2];
7037 Tys[0] = Ty;
7038 Tys[1] = GetNeonType(this, NeonTypeFlags(Type.getEltType(), false,
7039 /*isQuad*/ false));
7040 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vluti2_lane");
7041 }
7042 case NEON::BI__builtin_neon_vluti2q_lane_mf8:
7043 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
7044 case NEON::BI__builtin_neon_vluti2q_lane_f16:
7045 case NEON::BI__builtin_neon_vluti2q_lane_p16:
7046 case NEON::BI__builtin_neon_vluti2q_lane_p8:
7047 case NEON::BI__builtin_neon_vluti2q_lane_s16:
7048 case NEON::BI__builtin_neon_vluti2q_lane_s8:
7049 case NEON::BI__builtin_neon_vluti2q_lane_u16:
7050 case NEON::BI__builtin_neon_vluti2q_lane_u8: {
7051 Int = Intrinsic::aarch64_neon_vluti2_lane;
7052 llvm::Type *Tys[2];
7053 Tys[0] = Ty;
7054 Tys[1] = GetNeonType(this, NeonTypeFlags(Type.getEltType(), false,
7055 /*isQuad*/ true));
7056 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vluti2_lane");
7057 }
7058 case NEON::BI__builtin_neon_vluti4q_lane_mf8:
7059 case NEON::BI__builtin_neon_vluti4q_lane_p8:
7060 case NEON::BI__builtin_neon_vluti4q_lane_s8:
7061 case NEON::BI__builtin_neon_vluti4q_lane_u8: {
7062 Int = Intrinsic::aarch64_neon_vluti4q_lane;
7063 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vluti4q_lane");
7064 }
7065 case NEON::BI__builtin_neon_vluti4q_laneq_mf8:
7066 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
7067 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
7068 case NEON::BI__builtin_neon_vluti4q_laneq_u8: {
7069 Int = Intrinsic::aarch64_neon_vluti4q_laneq;
7070 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vluti4q_laneq");
7071 }
7072 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
7073 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
7074 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
7075 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
7076 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2: {
7077 Int = Intrinsic::aarch64_neon_vluti4q_lane_x2;
7078 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vluti4q_lane_x2");
7079 }
7080 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
7081 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
7082 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
7083 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
7084 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2: {
7085 Int = Intrinsic::aarch64_neon_vluti4q_laneq_x2;
7086 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vluti4q_laneq_x2");
7087 }
7088 case NEON::BI__builtin_neon_vmmlaq_f16_mf8_fpm:
7089 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fmmla,
7090 {llvm::FixedVectorType::get(HalfTy, 8),
7091 llvm::FixedVectorType::get(Int8Ty, 16)},
7092 Ops, E, "fmmla");
7093 case NEON::BI__builtin_neon_vmmlaq_f32_mf8_fpm:
7094 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fmmla,
7095 {llvm::FixedVectorType::get(FloatTy, 4),
7096 llvm::FixedVectorType::get(Int8Ty, 16)},
7097 Ops, E, "fmmla");
7098 case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
7099 ExtractLow = true;
7100 [[fallthrough]];
7101 case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
7102 case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
7103 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_cvtl1,
7104 llvm::FixedVectorType::get(BFloatTy, 8),
7105 Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt1");
7106 case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
7107 ExtractLow = true;
7108 [[fallthrough]];
7109 case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
7110 case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
7111 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_cvtl2,
7112 llvm::FixedVectorType::get(BFloatTy, 8),
7113 Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt2");
7114 case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
7115 ExtractLow = true;
7116 [[fallthrough]];
7117 case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
7118 case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
7119 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_cvtl1,
7120 llvm::FixedVectorType::get(HalfTy, 8),
7121 Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt1");
7122 case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
7123 ExtractLow = true;
7124 [[fallthrough]];
7125 case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
7126 case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
7127 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_cvtl2,
7128 llvm::FixedVectorType::get(HalfTy, 8),
7129 Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt2");
7130 case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
7131 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_fcvtn,
7132 llvm::FixedVectorType::get(Int8Ty, 8),
7133 Ops[0]->getType(), false, Ops, E, "vfcvtn");
7134 case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
7135 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_fcvtn,
7136 llvm::FixedVectorType::get(Int8Ty, 8),
7137 llvm::FixedVectorType::get(HalfTy, 4), false, Ops,
7138 E, "vfcvtn");
7139 case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
7140 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_fcvtn,
7141 llvm::FixedVectorType::get(Int8Ty, 16),
7142 llvm::FixedVectorType::get(HalfTy, 8), false, Ops,
7143 E, "vfcvtn");
7144 case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm: {
7145 llvm::Type *Ty = llvm::FixedVectorType::get(Int8Ty, 16);
7146 Ops[0] = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0],
7147 uint64_t(0));
7148 return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_fcvtn2, Ty,
7149 Ops[1]->getType(), false, Ops, E, "vfcvtn2");
7150 }
7151
7152 case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
7153 case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
7154 return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot2, false, HalfTy,
7155 Ops, E, "fdot2");
7156 case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
7157 case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
7158 ExtendLaneArg = true;
7159 [[fallthrough]];
7160 case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
7161 case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
7162 return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot2_lane,
7163 ExtendLaneArg, HalfTy, Ops, E, "fdot2_lane");
7164 case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
7165 case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
7166 return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot4, false,
7167 FloatTy, Ops, E, "fdot4");
7168 case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
7169 case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
7170 ExtendLaneArg = true;
7171 [[fallthrough]];
7172 case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
7173 case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
7174 return EmitFP8NeonFDOTCall(Intrinsic::aarch64_neon_fp8_fdot4_lane,
7175 ExtendLaneArg, FloatTy, Ops, E, "fdot4_lane");
7176
7177 case NEON::BI__builtin_neon_vdot_f32_f16:
7178 case NEON::BI__builtin_neon_vdotq_f32_f16: {
7179 llvm::Type *InputTy =
7180 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
7181 llvm::Type *Tys[2] = {Ty, InputTy};
7182 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_fdot, Tys),
7183 Ops, "vdot");
7184 }
7185
7186 case NEON::BI__builtin_neon_vdot_lane_f32_f16:
7187 case NEON::BI__builtin_neon_vdot_laneq_f32_f16:
7188 case NEON::BI__builtin_neon_vdotq_lane_f32_f16:
7189 case NEON::BI__builtin_neon_vdotq_laneq_f32_f16: {
7190 llvm::FixedVectorType *InputTy =
7191 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
7192 llvm::FixedVectorType *LaneTy = llvm::FixedVectorType::get(
7193 HalfTy, Ops[2]->getType()->getPrimitiveSizeInBits() / 16);
7194 // Treat the lane argument as a splat and use non-lane version of the
7195 // intrinsic.
7196 Ops[2] = Builder.CreateBitCast(Ops[2], LaneTy);
7197 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]),
7198 InputTy->getElementCount());
7199 llvm::Type *Tys[2] = {Ty, InputTy};
7200 Ops.pop_back();
7201 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_fdot, Tys),
7202 Ops, "vdot");
7203 }
7204
7205 case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
7206 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalb,
7207 {llvm::FixedVectorType::get(HalfTy, 8)}, Ops, E,
7208 "vmlal");
7209 case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
7210 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalt,
7211 {llvm::FixedVectorType::get(HalfTy, 8)}, Ops, E,
7212 "vmlal");
7213 case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
7214 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlallbb,
7215 {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
7216 "vmlall");
7217 case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
7218 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlallbt,
7219 {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
7220 "vmlall");
7221 case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
7222 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalltb,
7223 {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
7224 "vmlall");
7225 case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
7226 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalltt,
7227 {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
7228 "vmlall");
7229 case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
7230 ExtendLaneArg = true;
7231 [[fallthrough]];
7232 case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
7233 return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlalb_lane,
7234 ExtendLaneArg, HalfTy, Ops, E, "vmlal_lane");
7235 case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
7236 ExtendLaneArg = true;
7237 [[fallthrough]];
7238 case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
7239 return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlalt_lane,
7240 ExtendLaneArg, HalfTy, Ops, E, "vmlal_lane");
7241 case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
7242 ExtendLaneArg = true;
7243 [[fallthrough]];
7244 case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
7245 return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlallbb_lane,
7246 ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
7247 case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
7248 ExtendLaneArg = true;
7249 [[fallthrough]];
7250 case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
7251 return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlallbt_lane,
7252 ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
7253 case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
7254 ExtendLaneArg = true;
7255 [[fallthrough]];
7256 case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
7257 return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlalltb_lane,
7258 ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
7259 case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
7260 ExtendLaneArg = true;
7261 [[fallthrough]];
7262 case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
7263 return EmitFP8NeonFMLACall(Intrinsic::aarch64_neon_fp8_fmlalltt_lane,
7264 ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
7265 case NEON::BI__builtin_neon_vamin_f16:
7266 case NEON::BI__builtin_neon_vaminq_f16:
7267 case NEON::BI__builtin_neon_vamin_f32:
7268 case NEON::BI__builtin_neon_vaminq_f32:
7269 case NEON::BI__builtin_neon_vaminq_f64: {
7270 Int = Intrinsic::aarch64_neon_famin;
7271 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "famin");
7272 }
7273 case NEON::BI__builtin_neon_vamax_f16:
7274 case NEON::BI__builtin_neon_vamaxq_f16:
7275 case NEON::BI__builtin_neon_vamax_f32:
7276 case NEON::BI__builtin_neon_vamaxq_f32:
7277 case NEON::BI__builtin_neon_vamaxq_f64: {
7278 Int = Intrinsic::aarch64_neon_famax;
7279 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "famax");
7280 }
7281 case NEON::BI__builtin_neon_vscale_f16:
7282 case NEON::BI__builtin_neon_vscaleq_f16:
7283 case NEON::BI__builtin_neon_vscale_f32:
7284 case NEON::BI__builtin_neon_vscaleq_f32:
7285 case NEON::BI__builtin_neon_vscaleq_f64: {
7286 Int = Intrinsic::aarch64_neon_fp8_fscale;
7287 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fscale");
7288 }
7289 }
7290}
7291
7293 const CallExpr *E) {
7294 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
7295 BuiltinID == BPF::BI__builtin_btf_type_id ||
7296 BuiltinID == BPF::BI__builtin_preserve_type_info ||
7297 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
7298 "unexpected BPF builtin");
7299
7300 // A sequence number, injected into IR builtin functions, to
7301 // prevent CSE given the only difference of the function
7302 // may just be the debuginfo metadata.
7303 static uint32_t BuiltinSeqNum;
7304
7305 switch (BuiltinID) {
7306 default:
7307 llvm_unreachable("Unexpected BPF builtin");
7308 case BPF::BI__builtin_preserve_field_info: {
7309 const Expr *Arg = E->getArg(0);
7310 bool IsBitField = Arg->IgnoreParens()->getObjectKind() == OK_BitField;
7311
7312 if (!getDebugInfo()) {
7313 CGM.Error(E->getExprLoc(),
7314 "using __builtin_preserve_field_info() without -g");
7315 return IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this)
7316 : EmitLValue(Arg).emitRawPointer(*this);
7317 }
7318
7319 // Enable underlying preserve_*_access_index() generation. Field info
7320 // never dereferences the field, so skip preserve_static_offset markers:
7321 // the backend would replace the access indices under them with plain GEPs.
7322 bool OldIsInPreservedAIRegion = IsInPreservedAIRegion;
7323 bool OldIsInPreserveFieldInfo = IsInPreserveFieldInfo;
7324 IsInPreservedAIRegion = true;
7325 IsInPreserveFieldInfo = true;
7326 Value *FieldAddr = IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this)
7327 : EmitLValue(Arg).emitRawPointer(*this);
7328 IsInPreservedAIRegion = OldIsInPreservedAIRegion;
7329 IsInPreserveFieldInfo = OldIsInPreserveFieldInfo;
7330
7331 ConstantInt *C = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
7332 Value *InfoKind = ConstantInt::get(Int64Ty, C->getSExtValue());
7333
7334 // Built the IR for the preserve_field_info intrinsic.
7335 llvm::Function *FnGetFieldInfo = Intrinsic::getOrInsertDeclaration(
7336 &CGM.getModule(), Intrinsic::bpf_preserve_field_info,
7337 {FieldAddr->getType()});
7338 return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
7339 }
7340 case BPF::BI__builtin_btf_type_id:
7341 case BPF::BI__builtin_preserve_type_info: {
7342 if (!getDebugInfo()) {
7343 CGM.Error(E->getExprLoc(), "using builtin function without -g");
7344 return nullptr;
7345 }
7346
7347 const Expr *Arg0 = E->getArg(0);
7348 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(
7349 Arg0->getType(), Arg0->getExprLoc());
7350
7351 ConstantInt *Flag = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
7352 Value *FlagValue = ConstantInt::get(Int64Ty, Flag->getSExtValue());
7353 Value *SeqNumVal = ConstantInt::get(Int32Ty, BuiltinSeqNum++);
7354
7355 llvm::Function *FnDecl;
7356 if (BuiltinID == BPF::BI__builtin_btf_type_id)
7357 FnDecl = Intrinsic::getOrInsertDeclaration(
7358 &CGM.getModule(), Intrinsic::bpf_btf_type_id, {});
7359 else
7360 FnDecl = Intrinsic::getOrInsertDeclaration(
7361 &CGM.getModule(), Intrinsic::bpf_preserve_type_info, {});
7362 CallInst *Fn = Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
7363 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
7364 return Fn;
7365 }
7366 case BPF::BI__builtin_preserve_enum_value: {
7367 if (!getDebugInfo()) {
7368 CGM.Error(E->getExprLoc(), "using builtin function without -g");
7369 return nullptr;
7370 }
7371
7372 const Expr *Arg0 = E->getArg(0);
7373 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(
7374 Arg0->getType(), Arg0->getExprLoc());
7375
7376 // Find enumerator
7377 const auto *UO = cast<UnaryOperator>(Arg0->IgnoreParens());
7378 const auto *CE = cast<CStyleCastExpr>(UO->getSubExpr());
7379 const auto *DR = cast<DeclRefExpr>(CE->getSubExpr());
7380 const auto *Enumerator = cast<EnumConstantDecl>(DR->getDecl());
7381
7382 auto InitVal = Enumerator->getInitVal();
7383 std::string InitValStr;
7384 if (InitVal.isNegative() || InitVal > uint64_t(INT64_MAX))
7385 InitValStr = std::to_string(InitVal.getSExtValue());
7386 else
7387 InitValStr = std::to_string(InitVal.getZExtValue());
7388 std::string EnumStr = Enumerator->getNameAsString() + ":" + InitValStr;
7389 Value *EnumStrVal = Builder.CreateGlobalString(EnumStr);
7390
7391 ConstantInt *Flag = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
7392 Value *FlagValue = ConstantInt::get(Int64Ty, Flag->getSExtValue());
7393 Value *SeqNumVal = ConstantInt::get(Int32Ty, BuiltinSeqNum++);
7394
7395 llvm::Function *IntrinsicFn = Intrinsic::getOrInsertDeclaration(
7396 &CGM.getModule(), Intrinsic::bpf_preserve_enum_value, {});
7397 CallInst *Fn =
7398 Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
7399 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
7400 return Fn;
7401 }
7402 }
7403}
7404
7407 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
7408 "Not a power-of-two sized vector!");
7409 bool AllConstants = true;
7410 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
7411 AllConstants &= isa<Constant>(Ops[i]);
7412
7413 // If this is a constant vector, create a ConstantVector.
7414 if (AllConstants) {
7416 for (llvm::Value *Op : Ops)
7417 CstOps.push_back(cast<Constant>(Op));
7418 return llvm::ConstantVector::get(CstOps);
7419 }
7420
7421 // Otherwise, insertelement the values to build the vector.
7422 Value *Result = llvm::PoisonValue::get(
7423 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size()));
7424
7425 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
7426 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt64(i));
7427
7428 return Result;
7429}
7430
7431Value *CodeGenFunction::EmitAArch64CpuInit() {
7432 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
7433 llvm::FunctionCallee Func =
7434 CGM.CreateRuntimeFunction(FTy, "__init_cpu_features_resolver");
7435 cast<llvm::GlobalValue>(Func.getCallee())->setDSOLocal(true);
7436 cast<llvm::GlobalValue>(Func.getCallee())
7437 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
7438 return Builder.CreateCall(Func);
7439}
7440
7441Value *CodeGenFunction::EmitAArch64CpuSupports(const CallExpr *E) {
7442 const Expr *ArgExpr = E->getArg(0)->IgnoreParenCasts();
7443 StringRef ArgStr = cast<StringLiteral>(ArgExpr)->getString();
7445 ArgStr.split(OrigFeatures, "+");
7447 for (StringRef Feature : OrigFeatures) {
7448 Feature = Feature.trim();
7449 if (!llvm::AArch64::parseFMVExtension(Feature))
7450 return Builder.getFalse();
7451 if (Feature != "default")
7452 Features.push_back(Feature);
7453 }
7454 return EmitAArch64CpuSupports(Features);
7455}
7456
7457llvm::Value *
7458CodeGenFunction::EmitAArch64CpuSupports(ArrayRef<StringRef> FeaturesStrs) {
7459 llvm::APInt FeaturesMask = llvm::AArch64::getCpuSupportsMask(FeaturesStrs);
7460 Value *Result = Builder.getTrue();
7461 if (FeaturesMask != 0) {
7462 // Get features from structure in runtime library
7463 // struct {
7464 // unsigned long long features;
7465 // } __aarch64_cpu_features;
7466 llvm::Type *STy = llvm::StructType::get(Int64Ty);
7467 llvm::Constant *AArch64CPUFeatures =
7468 CGM.CreateRuntimeVariable(STy, "__aarch64_cpu_features");
7469 cast<llvm::GlobalValue>(AArch64CPUFeatures)->setDSOLocal(true);
7470 llvm::Value *CpuFeatures = Builder.CreateGEP(
7471 STy, AArch64CPUFeatures,
7472 {ConstantInt::get(Int32Ty, 0), ConstantInt::get(Int32Ty, 0)});
7473 Value *Features = Builder.CreateAlignedLoad(Int64Ty, CpuFeatures,
7475 Value *Mask = Builder.getInt(FeaturesMask.trunc(64));
7476 Value *Bitset = Builder.CreateAnd(Features, Mask);
7477 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
7478 Result = Builder.CreateAnd(Result, Cmp);
7479 }
7480 return Result;
7481}
Utilities used for generating code for AArch64 that are shared between the classic and ClangIR code-g...
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
#define NEONMAP0(NameBase)
#define V(N, I)
Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E)
static cir::VectorType getSVEVectorForElementType(CIRGenModule &cgm, mlir::Type eltTy)
static const IntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< IntrinsicInfo > intrinsicMap, unsigned builtinID, bool &mapProvenSorted)
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, SpecialRegisterAccessKind AccessKind, StringRef SysReg="")
Definition ARM.cpp:1960
static llvm::Value * ARMMVEVectorReinterpret(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *DestType)
Definition ARM.cpp:2916
static Value * EmitAtomicFetchWithHintBuiltin(CodeGenFunction &CGF, const CallExpr *E, AtomicRMWInst::BinOp Op)
Definition ARM.cpp:2144
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
Definition ARM.cpp:409
static llvm::Value * MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V, uint32_t Shift, bool Unsigned)
Definition ARM.cpp:2886
static llvm::Value * SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V, llvm::Type *T, bool Unsigned)
Definition ARM.cpp:2879
static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
Definition ARM.cpp:3933
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops, llvm::Triple::ArchType Arch)
Definition ARM.cpp:3138
static void swapCommutativeSMEOperands(unsigned BuiltinID, SmallVectorImpl< Value * > &Ops)
Definition ARM.cpp:4386
static bool AArch64SISDIntrinsicsProvenSorted
Definition ARM.cpp:994
static llvm::Value * ARMMVECreateFPToSI(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *Ty)
Definition ARM.cpp:3010
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const ARMNeonVectorIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
Definition ARM.cpp:1062
static llvm::Value * ARMMVECreateFPToUI(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *Ty)
Definition ARM.cpp:3018
static llvm::Value * ARMMVECreateSIToFP(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *Ty)
Definition ARM.cpp:2994
static bool AArch64SVEIntrinsicsProvenSorted
Definition ARM.cpp:995
static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
Definition ARM.cpp:3939
static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context)
Definition ARM.cpp:2875
static bool AArch64SMEIntrinsicsProvenSorted
Definition ARM.cpp:996
static llvm::Value * VectorZip(CGBuilderTy &Builder, llvm::Value *V0, llvm::Value *V1)
Definition ARM.cpp:2953
constexpr unsigned SVEBitsPerBlock
Definition ARM.cpp:3418
static const std::pair< unsigned, unsigned > NEONEquivalentIntrinsicMap[]
Definition ARM.cpp:868
static void AttachAtomicHintMetadata(CodeGenFunction &CGF, const CallExpr *E, Instruction *Atomic, unsigned PtrOperand)
Definition ARM.cpp:2076
static llvm::FixedVectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasFastHalfType=true, bool V1Ty=false, bool AllowBFloatArgsAndRet=true)
Definition ARM.cpp:369
Value * readX18AsPtr(CodeGenFunction &CGF)
Helper for the read/write/add/inc X18 builtins: read the X18 register and return it as an i8 pointer.
Definition ARM.cpp:4486
static llvm::Value * ARMMVEVectorElementReverse(CGBuilderTy &Builder, llvm::Value *V, unsigned ReverseWidth)
Definition ARM.cpp:2980
static std::optional< CodeGenFunction::MSVCIntrin > translateAarch64ToMsvcIntrin(unsigned BuiltinID)
Definition ARM.cpp:35
static std::optional< CodeGenFunction::MSVCIntrin > translateArmToMsvcIntrin(unsigned BuiltinID)
Definition ARM.cpp:196
static llvm::Value * ARMMVECreateUIToFP(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *Ty)
Definition ARM.cpp:3002
static llvm::Value * VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd)
Definition ARM.cpp:2942
static Value * EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Definition ARM.cpp:2101
static llvm::Value * ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT)
Definition ARM.cpp:2968
SpecialRegisterAccessKind
Definition ARM.cpp:1951
@ VolatileRead
Definition ARM.cpp:1953
@ NormalRead
Definition ARM.cpp:1952
@ Write
Definition ARM.cpp:1954
static const AArch64SVEAndSMEVectorIntrinsicInfo AArch64SMEIntrinsicMap[]
Definition ARM.cpp:982
static llvm::Value * ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V)
Definition ARM.cpp:2908
static bool NEONSIMDIntrinsicsProvenSorted
Definition ARM.cpp:991
static Value * emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, llvm::Type *Ty, ArrayRef< Value * > Args)
Definition ARM.cpp:352
static Value * EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Definition ARM.cpp:2026
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
Definition ARM.cpp:1878
static bool AArch64SIMDIntrinsicsProvenSorted
Definition ARM.cpp:993
static const ARMNeonVectorIntrinsicInfo ARMSIMDIntrinsicMap[]
Definition ARM.cpp:548
static const AArch64SVEAndSMEVectorIntrinsicInfo AArch64SVEIntrinsicMap[]
Definition ARM.cpp:967
TokenType getType() const
Returns the token's type, e.g.
Result
Implement __builtin_bit_cast and related operations.
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
HLSLResourceBindingAttr::RegisterType RegisterType
Definition SemaHLSL.cpp:64
Enumerates target-specific builtins in their own namespaces within namespace clang.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition ASTContext.h:245
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
@ GE_None
No error.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Definition Expr.h:3012
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
Definition Expr.h:3216
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
Definition Expr.h:3203
QualType getCallReturnType(const ASTContext &Ctx) const
getCallReturnType - Get the return type of the call expr.
Definition Expr.cpp:1635
static CharUnits One()
Construct a CharUnits quantity of one.
Definition CharUnits.h:55
static CharUnits fromQuantity(QuantityType Quantity)
Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:58
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
Definition Address.h:128
static Address invalid()
Definition Address.h:176
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
Definition Address.h:253
CharUnits getAlignment() const
Definition Address.h:194
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Definition Address.h:276
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition Address.h:204
An aggregate value slot.
Definition CGValue.h:551
Address getAddress() const
Definition CGValue.h:689
llvm::DIType * getOrCreateStandaloneType(QualType Ty, SourceLocation Loc)
Emit standalone debug info for a type.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Value * EmitSVEPredicateCast(llvm::Value *Pred, llvm::ScalableVectorType *VTy)
Definition ARM.cpp:3427
llvm::Value * EmitFP8NeonFMLACall(unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy, SmallVectorImpl< llvm::Value * > &Ops, const CallExpr *E, const char *name)
Definition ARM.cpp:480
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
Definition ARM.cpp:7406
llvm::Value * EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx, const CallExpr *E)
llvm::Value * EmitSVEStructLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3602
llvm::Value * EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID, bool IsZExtReturn)
Definition ARM.cpp:3711
llvm::Value * EmitFP8NeonCall(unsigned IID, ArrayRef< llvm::Type * > Tys, SmallVectorImpl< llvm::Value * > &O, const CallExpr *E, const char *name)
Definition ARM.cpp:455
llvm::Type * ConvertType(QualType T)
llvm::Value * EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3566
llvm::Value * EmitSMEReadWrite(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3857
llvm::Type * SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags)
SVEBuiltinMemEltTy - Returns the memory element type for this memory access builtin.
Definition ARM.cpp:3293
llvm::Value * EmitSVEScatterStore(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3520
llvm::Value * EmitSVEMaskedStore(const CallExpr *, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
Definition ARM.cpp:3768
llvm::Value * EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Definition ARM.cpp:4410
void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, SVETypeFlags TypeFlags)
Definition ARM.cpp:3996
llvm::Value * EmitSVEGatherLoad(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3478
bool IsInPreserveFieldInfo
True if CodeGen currently emits the argument of __builtin_preserve_field_info.
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
Definition ARM.cpp:1020
llvm::Type * getEltType(const SVETypeFlags &TypeFlags)
Definition ARM.cpp:3309
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
Definition ARM.cpp:1128
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx, const llvm::ElementCount &Count)
llvm::Value * EmitSVEDupX(llvm::Value *Scalar)
const TargetInfo & getTarget() const
llvm::Value * EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Definition ARM.cpp:4039
llvm::Value * EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0, llvm::Type *Ty1, bool Extract, SmallVectorImpl< llvm::Value * > &Ops, const CallExpr *E, const char *name)
Definition ARM.cpp:501
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
Definition ARM.cpp:2189
llvm::ScalableVectorType * getSVEType(const SVETypeFlags &TypeFlags)
Definition ARM.cpp:3382
llvm::Value * EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Definition ARM.cpp:7292
llvm::Value * EmitSMELdrStr(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3879
llvm::Value * EmitSVETupleCreate(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
Definition ARM.cpp:3984
llvm::Value * EmitSVEPMull(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
Definition ARM.cpp:3665
llvm::Value * EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
Definition ARM.cpp:3026
AggValueSlot CreateAggTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateAggTemp - Create a temporary memory object for the given aggregate type.
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
Definition ARM.cpp:517
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
Definition CGExpr.cpp:2261
SmallVector< llvm::Type *, 2 > getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
Definition ARM.cpp:3946
const TargetCodeGenInfo & getTargetHooks() const
RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen without...
Definition CGExpr.cpp:236
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
Definition ARM.cpp:495
bool IsInPreservedAIRegion
True if CodeGen currently emits code inside presereved access index region.
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
Definition ARM.cpp:4497
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
llvm::Value * EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy, SmallVectorImpl< llvm::Value * > &Ops, const CallExpr *E, const char *name)
Definition ARM.cpp:464
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
Definition ARM.cpp:3281
llvm::Value * EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
Definition ARM.cpp:3127
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Pred, const llvm::Twine &Name="")
Definition ARM.cpp:1849
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
Definition CGExpr.cpp:314
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertTypeForMem(QualType T)
llvm::Value * EmitSVEMovl(const SVETypeFlags &TypeFlags, llvm::ArrayRef< llvm::Value * > Ops, unsigned BuiltinID)
Definition ARM.cpp:3683
llvm::Value * EmitSVEPredicateTupleCast(llvm::Value *PredTuple, llvm::StructType *Ty)
Definition ARM.cpp:3462
CodeGenUtils::MSVCIntrin MSVCIntrin
llvm::Value * EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
Definition ARM.cpp:3690
llvm::Value * EmitSMEZero(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3869
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
Definition CGExpr.cpp:1661
llvm::Value * EmitSVEStructStore(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3617
llvm::Value * EmitSMELd1St1(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:3822
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
Definition CGExpr.cpp:2295
llvm::Value * EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty)
Definition ARM.cpp:3910
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
Definition CGExpr.cpp:1777
llvm::LLVMContext & getLLVMContext()
llvm::ScalableVectorType * getSVEPredType(const SVETypeFlags &TypeFlags)
Definition ARM.cpp:3347
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
Definition ARM.cpp:435
llvm::Value * EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags, ArrayRef< llvm::Value * > Ops)
Definition ARM.cpp:3973
This class organizes the cross-function state that is used while generating LLVM code.
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
ASTContext & getContext() const
llvm::LLVMContext & getLLVMContext()
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys={})
llvm::Value * getRawBitFieldPointer(CodeGenFunction &CGF) const
Definition CGValue.h:441
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
Definition CGCall.h:384
This represents one expression.
Definition Expr.h:114
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
Definition Expr.cpp:3134
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
Definition Expr.cpp:3125
ExprObjectKind getObjectKind() const
getObjectKind - The object kind that this expression produces.
Definition Expr.h:456
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, bool AllowRelaxedEval=false) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition Expr.cpp:283
QualType getType() const
Definition Expr.h:146
Flags to identify the types for overloaded Neon builtins.
EltType getEltType() const
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition TypeBase.h:3403
QualType getPointeeType() const
Definition TypeBase.h:3413
A (possibly-)qualified type.
Definition TypeBase.h:938
The collection of all-type qualifiers we support.
Definition TypeBase.h:332
Flags to identify the types for overloaded SVE builtins.
bool isZExtReturn() const
bool isReverseUSDOT() const
bool isOverloadNone() const
MemEltType getMemEltType() const
bool isGatherLoad() const
EltType getEltType() const
bool isOverloadFirstandLast() const
bool isOverloadDefault() const
bool isPrefetch() const
bool isOverloadWhileRW() const
bool isTupleSet() const
bool isReverseMergeAnyAccOp() const
bool isReductionQV() const
bool isTupleGet() const
bool isInsertOp1SVALL() const
bool isAppendSVALL() const
bool isReverseMergeAnyBinOp() const
bool isStructStore() const
bool isOverloadDefaultAndOp0() const
bool isTupleCreate() const
bool isGatherPrefetch() const
bool hasSplatOperand() const
MergeType getMergeType() const
bool isByteIndexed() const
bool isStructLoad() const
bool isOverloadWhileOrMultiVecCvt() const
unsigned getSplatOperand() const
bool isScatterStore() const
bool isReverseCompare() const
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
virtual bool hasFastHalfType() const
Determine whether the target has fast native support for operations on half types.
Definition TargetInfo.h:707
bool isBigEndian() const
The base class of the type hierarchy.
Definition TypeBase.h:1879
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
Definition Type.cpp:2392
const T * castAs() const
Member-template castAs<specific type>.
Definition TypeBase.h:9498
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
Definition Type.cpp:885
QualType getType() const
Definition Value.cpp:238
bool hasExtraNeonArgument(unsigned BuiltinID)
Returns whether the Neon builtin BuiltinID takes a trailing argument that discriminates the operand t...
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
Definition CGValue.h:155
const ARMNeonVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
const ARMNeonVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
Top level wrappers for InstallAPI frontend operations.
bool isa(CodeGen::Address addr)
Definition Address.h:330
@ OK_BitField
A bitfield object is a bitfield on a C or C++ record.
Definition Specifiers.h:158
@ Result
The result type of a method or function.
Definition TypeBase.h:906
const FunctionProtoType * T
U cast(CodeGen::Address addr)
Definition Address.h:327
@ Enumerator
Enumerator value with fixed underlying type.
Definition Sema.h:834
Diagnostic wrappers for TextAPI types for error reporting.
Definition Dominators.h:30
__builtin_elementwise_add_sat __builtin_elementwise_sub_sat uint64_t
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::Type * HalfTy
half, bfloat, float, double
EvalResult is a struct with detailed info about an evaluated expression.
Definition Expr.h:667
Describes an AArch64 SVE or SME intrinsic.
Describes an ARM or AArch64 NEON intrinsic, or an AArch64 SISD intrinsic.
#define trunc(__x)
Definition tgmath.h:1216
#define round(__x)
Definition tgmath.h:1148
#define rint(__x)
Definition tgmath.h:1131
#define floor(__x)
Definition tgmath.h:722
#define ceil(__x)
Definition tgmath.h:601