clang 20.0.0git
AArch64.cpp
Go to the documentation of this file.
1//===- AArch64.cpp --------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "TargetInfo.h"
11#include "clang/AST/Decl.h"
13#include "llvm/TargetParser/AArch64TargetParser.h"
14
15using namespace clang;
16using namespace clang::CodeGen;
17
18//===----------------------------------------------------------------------===//
19// AArch64 ABI Implementation
20//===----------------------------------------------------------------------===//
21
22namespace {
23
24class AArch64ABIInfo : public ABIInfo {
26
27public:
28 AArch64ABIInfo(CodeGenTypes &CGT, AArch64ABIKind Kind)
29 : ABIInfo(CGT), Kind(Kind) {}
30
31 bool isSoftFloat() const { return Kind == AArch64ABIKind::AAPCSSoft; }
32
33private:
34 AArch64ABIKind getABIKind() const { return Kind; }
35 bool isDarwinPCS() const { return Kind == AArch64ABIKind::DarwinPCS; }
36
37 ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadicFn) const;
38 ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadicFn,
39 bool IsNamedArg, unsigned CallingConvention,
40 unsigned &NSRN, unsigned &NPRN) const;
41 llvm::Type *convertFixedToScalableVectorType(const VectorType *VT) const;
42 ABIArgInfo coerceIllegalVector(QualType Ty, unsigned &NSRN,
43 unsigned &NPRN) const;
44 ABIArgInfo coerceAndExpandPureScalableAggregate(
45 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
46 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
47 unsigned &NPRN) const;
48 bool isHomogeneousAggregateBaseType(QualType Ty) const override;
50 uint64_t Members) const override;
52
53 bool isIllegalVectorType(QualType Ty) const;
54
55 bool passAsAggregateType(QualType Ty) const;
56 bool passAsPureScalableType(QualType Ty, unsigned &NV, unsigned &NP,
57 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const;
58
59 void flattenType(llvm::Type *Ty,
60 SmallVectorImpl<llvm::Type *> &Flattened) const;
61
62 void computeInfo(CGFunctionInfo &FI) const override {
63 if (!::classifyReturnType(getCXXABI(), FI, *this))
64 FI.getReturnInfo() =
66
67 unsigned ArgNo = 0;
68 unsigned NSRN = 0, NPRN = 0;
69 for (auto &it : FI.arguments()) {
70 const bool IsNamedArg =
71 !FI.isVariadic() || ArgNo < FI.getRequiredArgs().getNumRequiredArgs();
72 ++ArgNo;
73 it.info = classifyArgumentType(it.type, FI.isVariadic(), IsNamedArg,
74 FI.getCallingConvention(), NSRN, NPRN);
75 }
76 }
77
78 RValue EmitDarwinVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
79 AggValueSlot Slot) const;
80
81 RValue EmitAAPCSVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
82 AArch64ABIKind Kind, AggValueSlot Slot) const;
83
84 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
85 AggValueSlot Slot) const override {
86 llvm::Type *BaseTy = CGF.ConvertType(Ty);
87 if (isa<llvm::ScalableVectorType>(BaseTy))
88 llvm::report_fatal_error("Passing SVE types to variadic functions is "
89 "currently not supported");
90
91 return Kind == AArch64ABIKind::Win64
92 ? EmitMSVAArg(CGF, VAListAddr, Ty, Slot)
93 : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF, Slot)
94 : EmitAAPCSVAArg(VAListAddr, Ty, CGF, Kind, Slot);
95 }
96
98 AggValueSlot Slot) const override;
99
100 bool allowBFloatArgsAndRet() const override {
101 return getTarget().hasBFloat16Type();
102 }
103
105 void appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index,
106 raw_ostream &Out) const override;
107 void appendAttributeMangling(StringRef AttrStr,
108 raw_ostream &Out) const override;
109};
110
111class AArch64SwiftABIInfo : public SwiftABIInfo {
112public:
113 explicit AArch64SwiftABIInfo(CodeGenTypes &CGT)
114 : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/true) {}
115
116 bool isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,
117 unsigned NumElts) const override;
118};
119
120class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
121public:
122 AArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIKind Kind)
123 : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(CGT, Kind)) {
124 SwiftInfo = std::make_unique<AArch64SwiftABIInfo>(CGT);
125 }
126
127 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
128 return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue";
129 }
130
131 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
132 return 31;
133 }
134
135 bool doesReturnSlotInterfereWithArgs() const override { return false; }
136
137 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
138 CodeGen::CodeGenModule &CGM) const override {
139 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
140 if (!FD)
141 return;
142
144
145 if (const auto *TA = FD->getAttr<TargetAttr>()) {
147 CGM.getTarget().parseTargetAttr(TA->getFeaturesStr());
148 if (!Attr.BranchProtection.empty()) {
149 StringRef Error;
150 (void)CGM.getTarget().validateBranchProtection(Attr.BranchProtection,
151 Attr.CPU, BPI, Error);
152 assert(Error.empty());
153 }
154 }
155 auto *Fn = cast<llvm::Function>(GV);
157 }
158
160 llvm::Type *Ty) const override {
161 if (CGF.getTarget().hasFeature("ls64")) {
162 auto *ST = dyn_cast<llvm::StructType>(Ty);
163 if (ST && ST->getNumElements() == 1) {
164 auto *AT = dyn_cast<llvm::ArrayType>(ST->getElementType(0));
165 if (AT && AT->getNumElements() == 8 &&
166 AT->getElementType()->isIntegerTy(64))
167 return true;
168 }
169 }
171 }
172
174 const FunctionDecl *Decl) const override;
175
177 const FunctionDecl *Caller,
178 const FunctionDecl *Callee, const CallArgList &Args,
179 QualType ReturnType) const override;
180
182 const FunctionDecl *Caller, const FunctionDecl *Callee) const override;
183
184private:
185 // Diagnose calls between functions with incompatible Streaming SVE
186 // attributes.
187 void checkFunctionCallABIStreaming(CodeGenModule &CGM, SourceLocation CallLoc,
188 const FunctionDecl *Caller,
189 const FunctionDecl *Callee) const;
190 // Diagnose calls which must pass arguments in floating-point registers when
191 // the selected target does not have floating-point registers.
192 void checkFunctionCallABISoftFloat(CodeGenModule &CGM, SourceLocation CallLoc,
193 const FunctionDecl *Caller,
194 const FunctionDecl *Callee,
195 const CallArgList &Args,
196 QualType ReturnType) const;
197};
198
199class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo {
200public:
201 WindowsAArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIKind K)
202 : AArch64TargetCodeGenInfo(CGT, K) {}
203
204 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
205 CodeGen::CodeGenModule &CGM) const override;
206
207 void getDependentLibraryOption(llvm::StringRef Lib,
208 llvm::SmallString<24> &Opt) const override {
209 Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);
210 }
211
212 void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,
213 llvm::SmallString<32> &Opt) const override {
214 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
215 }
216};
217
218void WindowsAArch64TargetCodeGenInfo::setTargetAttributes(
219 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
220 AArch64TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
221 if (GV->isDeclaration())
222 return;
223 addStackProbeTargetAttributes(D, GV, CGM);
224}
225}
226
227llvm::Type *
228AArch64ABIInfo::convertFixedToScalableVectorType(const VectorType *VT) const {
229 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
230
231 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
232 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
233 BuiltinType::UChar &&
234 "unexpected builtin type for SVE predicate!");
235 return llvm::ScalableVectorType::get(llvm::Type::getInt1Ty(getVMContext()),
236 16);
237 }
238
239 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
240 const auto *BT = VT->getElementType()->castAs<BuiltinType>();
241 switch (BT->getKind()) {
242 default:
243 llvm_unreachable("unexpected builtin type for SVE vector!");
244
245 case BuiltinType::SChar:
246 case BuiltinType::UChar:
247 case BuiltinType::MFloat8:
248 return llvm::ScalableVectorType::get(
249 llvm::Type::getInt8Ty(getVMContext()), 16);
250
251 case BuiltinType::Short:
252 case BuiltinType::UShort:
253 return llvm::ScalableVectorType::get(
254 llvm::Type::getInt16Ty(getVMContext()), 8);
255
256 case BuiltinType::Int:
257 case BuiltinType::UInt:
258 return llvm::ScalableVectorType::get(
259 llvm::Type::getInt32Ty(getVMContext()), 4);
260
261 case BuiltinType::Long:
262 case BuiltinType::ULong:
263 return llvm::ScalableVectorType::get(
264 llvm::Type::getInt64Ty(getVMContext()), 2);
265
266 case BuiltinType::Half:
267 return llvm::ScalableVectorType::get(
268 llvm::Type::getHalfTy(getVMContext()), 8);
269
270 case BuiltinType::Float:
271 return llvm::ScalableVectorType::get(
272 llvm::Type::getFloatTy(getVMContext()), 4);
273
274 case BuiltinType::Double:
275 return llvm::ScalableVectorType::get(
276 llvm::Type::getDoubleTy(getVMContext()), 2);
277
278 case BuiltinType::BFloat16:
279 return llvm::ScalableVectorType::get(
280 llvm::Type::getBFloatTy(getVMContext()), 8);
281 }
282 }
283
284 llvm_unreachable("expected fixed-length SVE vector");
285}
286
287ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty, unsigned &NSRN,
288 unsigned &NPRN) const {
289 assert(Ty->isVectorType() && "expected vector type!");
290
291 const auto *VT = Ty->castAs<VectorType>();
292 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
293 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
294 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
295 BuiltinType::UChar &&
296 "unexpected builtin type for SVE predicate!");
297 NPRN = std::min(NPRN + 1, 4u);
298 return ABIArgInfo::getDirect(llvm::ScalableVectorType::get(
299 llvm::Type::getInt1Ty(getVMContext()), 16));
300 }
301
302 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
303 NSRN = std::min(NSRN + 1, 8u);
304 return ABIArgInfo::getDirect(convertFixedToScalableVectorType(VT));
305 }
306
307 uint64_t Size = getContext().getTypeSize(Ty);
308 // Android promotes <2 x i8> to i16, not i32
309 if ((isAndroid() || isOHOSFamily()) && (Size <= 16)) {
310 llvm::Type *ResType = llvm::Type::getInt16Ty(getVMContext());
311 return ABIArgInfo::getDirect(ResType);
312 }
313 if (Size <= 32) {
314 llvm::Type *ResType = llvm::Type::getInt32Ty(getVMContext());
315 return ABIArgInfo::getDirect(ResType);
316 }
317 if (Size == 64) {
318 NSRN = std::min(NSRN + 1, 8u);
319 auto *ResType =
320 llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 2);
321 return ABIArgInfo::getDirect(ResType);
322 }
323 if (Size == 128) {
324 NSRN = std::min(NSRN + 1, 8u);
325 auto *ResType =
326 llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 4);
327 return ABIArgInfo::getDirect(ResType);
328 }
329
330 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
331}
332
333ABIArgInfo AArch64ABIInfo::coerceAndExpandPureScalableAggregate(
334 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
335 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
336 unsigned &NPRN) const {
337 if (!IsNamedArg || NSRN + NVec > 8 || NPRN + NPred > 4)
338 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
339 NSRN += NVec;
340 NPRN += NPred;
341
342 // Handle SVE vector tuples.
343 if (Ty->isSVESizelessBuiltinType())
344 return ABIArgInfo::getDirect();
345
346 llvm::Type *UnpaddedCoerceToType =
347 UnpaddedCoerceToSeq.size() == 1
348 ? UnpaddedCoerceToSeq[0]
349 : llvm::StructType::get(CGT.getLLVMContext(), UnpaddedCoerceToSeq,
350 true);
351
352 SmallVector<llvm::Type *> CoerceToSeq;
353 flattenType(CGT.ConvertType(Ty), CoerceToSeq);
354 auto *CoerceToType =
355 llvm::StructType::get(CGT.getLLVMContext(), CoerceToSeq, false);
356
357 return ABIArgInfo::getCoerceAndExpand(CoerceToType, UnpaddedCoerceToType);
358}
359
360ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,
361 bool IsNamedArg,
362 unsigned CallingConvention,
363 unsigned &NSRN,
364 unsigned &NPRN) const {
366
367 // Handle illegal vector types here.
368 if (isIllegalVectorType(Ty))
369 return coerceIllegalVector(Ty, NSRN, NPRN);
370
371 if (!passAsAggregateType(Ty)) {
372 // Treat an enum type as its underlying type.
373 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
374 Ty = EnumTy->getDecl()->getIntegerType();
375
376 if (const auto *EIT = Ty->getAs<BitIntType>())
377 if (EIT->getNumBits() > 128)
378 return getNaturalAlignIndirect(Ty, false);
379
380 if (Ty->isVectorType())
381 NSRN = std::min(NSRN + 1, 8u);
382 else if (const auto *BT = Ty->getAs<BuiltinType>()) {
383 if (BT->isFloatingPoint())
384 NSRN = std::min(NSRN + 1, 8u);
385 else {
386 switch (BT->getKind()) {
387 case BuiltinType::SveBool:
388 case BuiltinType::SveCount:
389 NPRN = std::min(NPRN + 1, 4u);
390 break;
391 case BuiltinType::SveBoolx2:
392 NPRN = std::min(NPRN + 2, 4u);
393 break;
394 case BuiltinType::SveBoolx4:
395 NPRN = std::min(NPRN + 4, 4u);
396 break;
397 default:
398 if (BT->isSVESizelessBuiltinType())
399 NSRN = std::min(
400 NSRN + getContext().getBuiltinVectorTypeInfo(BT).NumVectors,
401 8u);
402 }
403 }
404 }
405
406 return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS()
407 ? ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty))
409 }
410
411 // Structures with either a non-trivial destructor or a non-trivial
412 // copy constructor are always indirect.
413 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
414 return getNaturalAlignIndirect(Ty, /*ByVal=*/RAA ==
416 }
417
418 // Empty records are always ignored on Darwin, but actually passed in C++ mode
419 // elsewhere for GNU compatibility.
420 uint64_t Size = getContext().getTypeSize(Ty);
421 bool IsEmpty = isEmptyRecord(getContext(), Ty, true);
422 if (!Ty->isSVESizelessBuiltinType() && (IsEmpty || Size == 0)) {
423 if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS())
424 return ABIArgInfo::getIgnore();
425
426 // GNU C mode. The only argument that gets ignored is an empty one with size
427 // 0.
428 if (IsEmpty && Size == 0)
429 return ABIArgInfo::getIgnore();
430 return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));
431 }
432
433 // Homogeneous Floating-point Aggregates (HFAs) need to be expanded.
434 const Type *Base = nullptr;
435 uint64_t Members = 0;
436 bool IsWin64 = Kind == AArch64ABIKind::Win64 ||
437 CallingConvention == llvm::CallingConv::Win64;
438 bool IsWinVariadic = IsWin64 && IsVariadicFn;
439 // In variadic functions on Windows, all composite types are treated alike,
440 // no special handling of HFAs/HVAs.
441 if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) {
442 NSRN = std::min(NSRN + Members, uint64_t(8));
443 if (Kind != AArch64ABIKind::AAPCS)
445 llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members));
446
447 // For HFAs/HVAs, cap the argument alignment to 16, otherwise
448 // set it to 8 according to the AAPCS64 document.
449 unsigned Align =
450 getContext().getTypeUnadjustedAlignInChars(Ty).getQuantity();
451 Align = (Align >= 16) ? 16 : 8;
453 llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members), 0,
454 nullptr, true, Align);
455 }
456
457 // In AAPCS named arguments of a Pure Scalable Type are passed expanded in
458 // registers, or indirectly if there are not enough registers.
459 if (Kind == AArch64ABIKind::AAPCS) {
460 unsigned NVec = 0, NPred = 0;
461 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
462 if (passAsPureScalableType(Ty, NVec, NPred, UnpaddedCoerceToSeq) &&
463 (NVec + NPred) > 0)
464 return coerceAndExpandPureScalableAggregate(
465 Ty, IsNamedArg, NVec, NPred, UnpaddedCoerceToSeq, NSRN, NPRN);
466 }
467
468 // Aggregates <= 16 bytes are passed directly in registers or on the stack.
469 if (Size <= 128) {
470 unsigned Alignment;
471 if (Kind == AArch64ABIKind::AAPCS) {
472 Alignment = getContext().getTypeUnadjustedAlign(Ty);
473 Alignment = Alignment < 128 ? 64 : 128;
474 } else {
475 Alignment =
476 std::max(getContext().getTypeAlign(Ty),
477 (unsigned)getTarget().getPointerWidth(LangAS::Default));
478 }
479 Size = llvm::alignTo(Size, Alignment);
480
481 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
482 // For aggregates with 16-byte alignment, we use i128.
483 llvm::Type *BaseTy = llvm::Type::getIntNTy(getVMContext(), Alignment);
485 Size == Alignment ? BaseTy
486 : llvm::ArrayType::get(BaseTy, Size / Alignment));
487 }
488
489 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
490}
491
492ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
493 bool IsVariadicFn) const {
494 if (RetTy->isVoidType())
495 return ABIArgInfo::getIgnore();
496
497 if (const auto *VT = RetTy->getAs<VectorType>()) {
498 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
499 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
500 unsigned NSRN = 0, NPRN = 0;
501 return coerceIllegalVector(RetTy, NSRN, NPRN);
502 }
503 }
504
505 // Large vector types should be returned via memory.
506 if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 128)
507 return getNaturalAlignIndirect(RetTy);
508
509 if (!passAsAggregateType(RetTy)) {
510 // Treat an enum type as its underlying type.
511 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
512 RetTy = EnumTy->getDecl()->getIntegerType();
513
514 if (const auto *EIT = RetTy->getAs<BitIntType>())
515 if (EIT->getNumBits() > 128)
516 return getNaturalAlignIndirect(RetTy);
517
518 return (isPromotableIntegerTypeForABI(RetTy) && isDarwinPCS()
519 ? ABIArgInfo::getExtend(RetTy)
521 }
522
523 uint64_t Size = getContext().getTypeSize(RetTy);
524 if (!RetTy->isSVESizelessBuiltinType() &&
525 (isEmptyRecord(getContext(), RetTy, true) || Size == 0))
526 return ABIArgInfo::getIgnore();
527
528 const Type *Base = nullptr;
529 uint64_t Members = 0;
530 if (isHomogeneousAggregate(RetTy, Base, Members) &&
531 !(getTarget().getTriple().getArch() == llvm::Triple::aarch64_32 &&
532 IsVariadicFn))
533 // Homogeneous Floating-point Aggregates (HFAs) are returned directly.
534 return ABIArgInfo::getDirect();
535
536 // In AAPCS return values of a Pure Scalable type are treated as a single
537 // named argument and passed expanded in registers, or indirectly if there are
538 // not enough registers.
539 if (Kind == AArch64ABIKind::AAPCS) {
540 unsigned NSRN = 0, NPRN = 0;
541 unsigned NVec = 0, NPred = 0;
542 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
543 if (passAsPureScalableType(RetTy, NVec, NPred, UnpaddedCoerceToSeq) &&
544 (NVec + NPred) > 0)
545 return coerceAndExpandPureScalableAggregate(
546 RetTy, /* IsNamedArg */ true, NVec, NPred, UnpaddedCoerceToSeq, NSRN,
547 NPRN);
548 }
549
550 // Aggregates <= 16 bytes are returned directly in registers or on the stack.
551 if (Size <= 128) {
552 if (Size <= 64 && getDataLayout().isLittleEndian()) {
553 // Composite types are returned in lower bits of a 64-bit register for LE,
554 // and in higher bits for BE. However, integer types are always returned
555 // in lower bits for both LE and BE, and they are not rounded up to
556 // 64-bits. We can skip rounding up of composite types for LE, but not for
557 // BE, otherwise composite types will be indistinguishable from integer
558 // types.
560 llvm::IntegerType::get(getVMContext(), Size));
561 }
562
563 unsigned Alignment = getContext().getTypeAlign(RetTy);
564 Size = llvm::alignTo(Size, 64); // round up to multiple of 8 bytes
565
566 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
567 // For aggregates with 16-byte alignment, we use i128.
568 if (Alignment < 128 && Size == 128) {
569 llvm::Type *BaseTy = llvm::Type::getInt64Ty(getVMContext());
570 return ABIArgInfo::getDirect(llvm::ArrayType::get(BaseTy, Size / 64));
571 }
572 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size));
573 }
574
575 return getNaturalAlignIndirect(RetTy);
576}
577
578/// isIllegalVectorType - check whether the vector type is legal for AArch64.
579bool AArch64ABIInfo::isIllegalVectorType(QualType Ty) const {
580 if (const VectorType *VT = Ty->getAs<VectorType>()) {
581 // Check whether VT is a fixed-length SVE vector. These types are
582 // represented as scalable vectors in function args/return and must be
583 // coerced from fixed vectors.
584 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
585 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)
586 return true;
587
588 // Check whether VT is legal.
589 unsigned NumElements = VT->getNumElements();
590 uint64_t Size = getContext().getTypeSize(VT);
591 // NumElements should be power of 2.
592 if (!llvm::isPowerOf2_32(NumElements))
593 return true;
594
595 // arm64_32 has to be compatible with the ARM logic here, which allows huge
596 // vectors for some reason.
597 llvm::Triple Triple = getTarget().getTriple();
598 if (Triple.getArch() == llvm::Triple::aarch64_32 &&
599 Triple.isOSBinFormatMachO())
600 return Size <= 32;
601
602 return Size != 64 && (Size != 128 || NumElements == 1);
603 }
604 return false;
605}
606
607bool AArch64SwiftABIInfo::isLegalVectorType(CharUnits VectorSize,
608 llvm::Type *EltTy,
609 unsigned NumElts) const {
610 if (!llvm::isPowerOf2_32(NumElts))
611 return false;
612 if (VectorSize.getQuantity() != 8 &&
613 (VectorSize.getQuantity() != 16 || NumElts == 1))
614 return false;
615 return true;
616}
617
618bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
619 // For the soft-float ABI variant, no types are considered to be homogeneous
620 // aggregates.
621 if (isSoftFloat())
622 return false;
623
624 // Homogeneous aggregates for AAPCS64 must have base types of a floating
625 // point type or a short-vector type. This is the same as the 32-bit ABI,
626 // but with the difference that any floating-point type is allowed,
627 // including __fp16.
628 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
629 if (BT->isFloatingPoint())
630 return true;
631 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
632 if (auto Kind = VT->getVectorKind();
633 Kind == VectorKind::SveFixedLengthData ||
634 Kind == VectorKind::SveFixedLengthPredicate)
635 return false;
636
637 unsigned VecSize = getContext().getTypeSize(VT);
638 if (VecSize == 64 || VecSize == 128)
639 return true;
640 }
641 return false;
642}
643
644bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
645 uint64_t Members) const {
646 return Members <= 4;
647}
648
649bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate()
650 const {
651 // AAPCS64 says that the rule for whether something is a homogeneous
652 // aggregate is applied to the output of the data layout decision. So
653 // anything that doesn't affect the data layout also does not affect
654 // homogeneity. In particular, zero-length bitfields don't stop a struct
655 // being homogeneous.
656 return true;
657}
658
659bool AArch64ABIInfo::passAsAggregateType(QualType Ty) const {
660 if (Kind == AArch64ABIKind::AAPCS && Ty->isSVESizelessBuiltinType()) {
661 const auto *BT = Ty->castAs<BuiltinType>();
662 return !BT->isSVECount() &&
663 getContext().getBuiltinVectorTypeInfo(BT).NumVectors > 1;
664 }
665 return isAggregateTypeForABI(Ty);
666}
667
668// Check if a type needs to be passed in registers as a Pure Scalable Type (as
669// defined by AAPCS64). Return the number of data vectors and the number of
670// predicate vectors in the type, into `NVec` and `NPred`, respectively. Upon
671// return `CoerceToSeq` contains an expanded sequence of LLVM IR types, one
672// element for each non-composite member. For practical purposes, limit the
673// length of `CoerceToSeq` to about 12 (the maximum that could possibly fit
674// in registers) and return false, the effect of which will be to pass the
675// argument under the rules for a large (> 128 bytes) composite.
676bool AArch64ABIInfo::passAsPureScalableType(
677 QualType Ty, unsigned &NVec, unsigned &NPred,
678 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const {
679 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
680 uint64_t NElt = AT->getZExtSize();
681 if (NElt == 0)
682 return false;
683
684 unsigned NV = 0, NP = 0;
685 SmallVector<llvm::Type *> EltCoerceToSeq;
686 if (!passAsPureScalableType(AT->getElementType(), NV, NP, EltCoerceToSeq))
687 return false;
688
689 if (CoerceToSeq.size() + NElt * EltCoerceToSeq.size() > 12)
690 return false;
691
692 for (uint64_t I = 0; I < NElt; ++I)
693 llvm::copy(EltCoerceToSeq, std::back_inserter(CoerceToSeq));
694
695 NVec += NElt * NV;
696 NPred += NElt * NP;
697 return true;
698 }
699
700 if (const RecordType *RT = Ty->getAs<RecordType>()) {
701 // If the record cannot be passed in registers, then it's not a PST.
702 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
704 return false;
705
706 // Pure scalable types are never unions and never contain unions.
707 const RecordDecl *RD = RT->getDecl();
708 if (RD->isUnion())
709 return false;
710
711 // If this is a C++ record, check the bases.
712 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
713 for (const auto &I : CXXRD->bases()) {
714 if (isEmptyRecord(getContext(), I.getType(), true))
715 continue;
716 if (!passAsPureScalableType(I.getType(), NVec, NPred, CoerceToSeq))
717 return false;
718 }
719 }
720
721 // Check members.
722 for (const auto *FD : RD->fields()) {
723 QualType FT = FD->getType();
724 if (isEmptyField(getContext(), FD, /* AllowArrays */ true))
725 continue;
726 if (!passAsPureScalableType(FT, NVec, NPred, CoerceToSeq))
727 return false;
728 }
729
730 return true;
731 }
732
733 if (const auto *VT = Ty->getAs<VectorType>()) {
734 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
735 ++NPred;
736 if (CoerceToSeq.size() + 1 > 12)
737 return false;
738 CoerceToSeq.push_back(convertFixedToScalableVectorType(VT));
739 return true;
740 }
741
742 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
743 ++NVec;
744 if (CoerceToSeq.size() + 1 > 12)
745 return false;
746 CoerceToSeq.push_back(convertFixedToScalableVectorType(VT));
747 return true;
748 }
749
750 return false;
751 }
752
753 if (!Ty->isBuiltinType())
754 return false;
755
756 bool isPredicate;
757 switch (Ty->getAs<BuiltinType>()->getKind()) {
758#define SVE_VECTOR_TYPE(Name, MangledName, Id, SingletonId) \
759 case BuiltinType::Id: \
760 isPredicate = false; \
761 break;
762#define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId) \
763 case BuiltinType::Id: \
764 isPredicate = true; \
765 break;
766#define SVE_TYPE(Name, Id, SingletonId)
767#include "clang/Basic/AArch64SVEACLETypes.def"
768 default:
769 return false;
770 }
771
773 getContext().getBuiltinVectorTypeInfo(cast<BuiltinType>(Ty));
774 assert(Info.NumVectors > 0 && Info.NumVectors <= 4 &&
775 "Expected 1, 2, 3 or 4 vectors!");
776 if (isPredicate)
777 NPred += Info.NumVectors;
778 else
779 NVec += Info.NumVectors;
780 llvm::Type *EltTy = Info.ElementType->isMFloat8Type()
781 ? llvm::Type::getInt8Ty(getVMContext())
782 : CGT.ConvertType(Info.ElementType);
783 auto *VTy = llvm::ScalableVectorType::get(EltTy, Info.EC.getKnownMinValue());
784
785 if (CoerceToSeq.size() + Info.NumVectors > 12)
786 return false;
787 std::fill_n(std::back_inserter(CoerceToSeq), Info.NumVectors, VTy);
788
789 return true;
790}
791
792// Expand an LLVM IR type into a sequence with a element for each non-struct,
793// non-array member of the type, with the exception of the padding types, which
794// are retained.
795void AArch64ABIInfo::flattenType(
796 llvm::Type *Ty, SmallVectorImpl<llvm::Type *> &Flattened) const {
797
799 Flattened.push_back(Ty);
800 return;
801 }
802
803 if (const auto *AT = dyn_cast<llvm::ArrayType>(Ty)) {
804 uint64_t NElt = AT->getNumElements();
805 if (NElt == 0)
806 return;
807
808 SmallVector<llvm::Type *> EltFlattened;
809 flattenType(AT->getElementType(), EltFlattened);
810
811 for (uint64_t I = 0; I < NElt; ++I)
812 llvm::copy(EltFlattened, std::back_inserter(Flattened));
813 return;
814 }
815
816 if (const auto *ST = dyn_cast<llvm::StructType>(Ty)) {
817 for (auto *ET : ST->elements())
818 flattenType(ET, Flattened);
819 return;
820 }
821
822 Flattened.push_back(Ty);
823}
824
825RValue AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
827 AggValueSlot Slot) const {
828 // These numbers are not used for variadic arguments, hence it doesn't matter
829 // they don't retain their values across multiple calls to
830 // `classifyArgumentType` here.
831 unsigned NSRN = 0, NPRN = 0;
832 ABIArgInfo AI =
833 classifyArgumentType(Ty, /*IsVariadicFn=*/true, /* IsNamedArg */ false,
834 CGF.CurFnInfo->getCallingConvention(), NSRN, NPRN);
835 // Empty records are ignored for parameter passing purposes.
836 if (AI.isIgnore())
837 return Slot.asRValue();
838
839 bool IsIndirect = AI.isIndirect();
840
841 llvm::Type *BaseTy = CGF.ConvertType(Ty);
842 if (IsIndirect)
843 BaseTy = llvm::PointerType::getUnqual(BaseTy);
844 else if (AI.getCoerceToType())
845 BaseTy = AI.getCoerceToType();
846
847 unsigned NumRegs = 1;
848 if (llvm::ArrayType *ArrTy = dyn_cast<llvm::ArrayType>(BaseTy)) {
849 BaseTy = ArrTy->getElementType();
850 NumRegs = ArrTy->getNumElements();
851 }
852 bool IsFPR =
853 !isSoftFloat() && (BaseTy->isFloatingPointTy() || BaseTy->isVectorTy());
854
855 // The AArch64 va_list type and handling is specified in the Procedure Call
856 // Standard, section B.4:
857 //
858 // struct {
859 // void *__stack;
860 // void *__gr_top;
861 // void *__vr_top;
862 // int __gr_offs;
863 // int __vr_offs;
864 // };
865
866 llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock("vaarg.maybe_reg");
867 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
868 llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock("vaarg.on_stack");
869 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
870
871 CharUnits TySize = getContext().getTypeSizeInChars(Ty);
872 CharUnits TyAlign = getContext().getTypeUnadjustedAlignInChars(Ty);
873
874 Address reg_offs_p = Address::invalid();
875 llvm::Value *reg_offs = nullptr;
876 int reg_top_index;
877 int RegSize = IsIndirect ? 8 : TySize.getQuantity();
878 if (!IsFPR) {
879 // 3 is the field number of __gr_offs
880 reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 3, "gr_offs_p");
881 reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "gr_offs");
882 reg_top_index = 1; // field number for __gr_top
883 RegSize = llvm::alignTo(RegSize, 8);
884 } else {
885 // 4 is the field number of __vr_offs.
886 reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 4, "vr_offs_p");
887 reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "vr_offs");
888 reg_top_index = 2; // field number for __vr_top
889 RegSize = 16 * NumRegs;
890 }
891
892 //=======================================
893 // Find out where argument was passed
894 //=======================================
895
896 // If reg_offs >= 0 we're already using the stack for this type of
897 // argument. We don't want to keep updating reg_offs (in case it overflows,
898 // though anyone passing 2GB of arguments, each at most 16 bytes, deserves
899 // whatever they get).
900 llvm::Value *UsingStack = nullptr;
901 UsingStack = CGF.Builder.CreateICmpSGE(
902 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, 0));
903
904 CGF.Builder.CreateCondBr(UsingStack, OnStackBlock, MaybeRegBlock);
905
906 // Otherwise, at least some kind of argument could go in these registers, the
907 // question is whether this particular type is too big.
908 CGF.EmitBlock(MaybeRegBlock);
909
910 // Integer arguments may need to correct register alignment (for example a
911 // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we
912 // align __gr_offs to calculate the potential address.
913 if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) {
914 int Align = TyAlign.getQuantity();
915
916 reg_offs = CGF.Builder.CreateAdd(
917 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, Align - 1),
918 "align_regoffs");
919 reg_offs = CGF.Builder.CreateAnd(
920 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, -Align),
921 "aligned_regoffs");
922 }
923
924 // Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list.
925 // The fact that this is done unconditionally reflects the fact that
926 // allocating an argument to the stack also uses up all the remaining
927 // registers of the appropriate kind.
928 llvm::Value *NewOffset = nullptr;
929 NewOffset = CGF.Builder.CreateAdd(
930 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, RegSize), "new_reg_offs");
931 CGF.Builder.CreateStore(NewOffset, reg_offs_p);
932
933 // Now we're in a position to decide whether this argument really was in
934 // registers or not.
935 llvm::Value *InRegs = nullptr;
936 InRegs = CGF.Builder.CreateICmpSLE(
937 NewOffset, llvm::ConstantInt::get(CGF.Int32Ty, 0), "inreg");
938
939 CGF.Builder.CreateCondBr(InRegs, InRegBlock, OnStackBlock);
940
941 //=======================================
942 // Argument was in registers
943 //=======================================
944
945 // Now we emit the code for if the argument was originally passed in
946 // registers. First start the appropriate block:
947 CGF.EmitBlock(InRegBlock);
948
949 llvm::Value *reg_top = nullptr;
950 Address reg_top_p =
951 CGF.Builder.CreateStructGEP(VAListAddr, reg_top_index, "reg_top_p");
952 reg_top = CGF.Builder.CreateLoad(reg_top_p, "reg_top");
953 Address BaseAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, reg_top, reg_offs),
954 CGF.Int8Ty, CharUnits::fromQuantity(IsFPR ? 16 : 8));
955 Address RegAddr = Address::invalid();
956 llvm::Type *MemTy = CGF.ConvertTypeForMem(Ty), *ElementTy = MemTy;
957
958 if (IsIndirect) {
959 // If it's been passed indirectly (actually a struct), whatever we find from
960 // stored registers or on the stack will actually be a struct **.
961 MemTy = llvm::PointerType::getUnqual(MemTy);
962 }
963
964 const Type *Base = nullptr;
965 uint64_t NumMembers = 0;
966 bool IsHFA = isHomogeneousAggregate(Ty, Base, NumMembers);
967 if (IsHFA && NumMembers > 1) {
968 // Homogeneous aggregates passed in registers will have their elements split
969 // and stored 16-bytes apart regardless of size (they're notionally in qN,
970 // qN+1, ...). We reload and store into a temporary local variable
971 // contiguously.
972 assert(!IsIndirect && "Homogeneous aggregates should be passed directly");
973 auto BaseTyInfo = getContext().getTypeInfoInChars(QualType(Base, 0));
974 llvm::Type *BaseTy = CGF.ConvertType(QualType(Base, 0));
975 llvm::Type *HFATy = llvm::ArrayType::get(BaseTy, NumMembers);
976 Address Tmp = CGF.CreateTempAlloca(HFATy,
977 std::max(TyAlign, BaseTyInfo.Align));
978
979 // On big-endian platforms, the value will be right-aligned in its slot.
980 int Offset = 0;
981 if (CGF.CGM.getDataLayout().isBigEndian() &&
982 BaseTyInfo.Width.getQuantity() < 16)
983 Offset = 16 - BaseTyInfo.Width.getQuantity();
984
985 for (unsigned i = 0; i < NumMembers; ++i) {
986 CharUnits BaseOffset = CharUnits::fromQuantity(16 * i + Offset);
987 Address LoadAddr =
988 CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, BaseOffset);
989 LoadAddr = LoadAddr.withElementType(BaseTy);
990
991 Address StoreAddr = CGF.Builder.CreateConstArrayGEP(Tmp, i);
992
993 llvm::Value *Elem = CGF.Builder.CreateLoad(LoadAddr);
994 CGF.Builder.CreateStore(Elem, StoreAddr);
995 }
996
997 RegAddr = Tmp.withElementType(MemTy);
998 } else {
999 // Otherwise the object is contiguous in memory.
1000
1001 // It might be right-aligned in its slot.
1002 CharUnits SlotSize = BaseAddr.getAlignment();
1003 if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect &&
1004 (IsHFA || !isAggregateTypeForABI(Ty)) &&
1005 TySize < SlotSize) {
1006 CharUnits Offset = SlotSize - TySize;
1007 BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, Offset);
1008 }
1009
1010 RegAddr = BaseAddr.withElementType(MemTy);
1011 }
1012
1013 CGF.EmitBranch(ContBlock);
1014
1015 //=======================================
1016 // Argument was on the stack
1017 //=======================================
1018 CGF.EmitBlock(OnStackBlock);
1019
1020 Address stack_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "stack_p");
1021 llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(stack_p, "stack");
1022
1023 // Again, stack arguments may need realignment. In this case both integer and
1024 // floating-point ones might be affected.
1025 if (!IsIndirect && TyAlign.getQuantity() > 8) {
1026 OnStackPtr = emitRoundPointerUpToAlignment(CGF, OnStackPtr, TyAlign);
1027 }
1028 Address OnStackAddr = Address(OnStackPtr, CGF.Int8Ty,
1029 std::max(CharUnits::fromQuantity(8), TyAlign));
1030
1031 // All stack slots are multiples of 8 bytes.
1032 CharUnits StackSlotSize = CharUnits::fromQuantity(8);
1033 CharUnits StackSize;
1034 if (IsIndirect)
1035 StackSize = StackSlotSize;
1036 else
1037 StackSize = TySize.alignTo(StackSlotSize);
1038
1039 llvm::Value *StackSizeC = CGF.Builder.getSize(StackSize);
1040 llvm::Value *NewStack = CGF.Builder.CreateInBoundsGEP(
1041 CGF.Int8Ty, OnStackPtr, StackSizeC, "new_stack");
1042
1043 // Write the new value of __stack for the next call to va_arg
1044 CGF.Builder.CreateStore(NewStack, stack_p);
1045
1046 if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(Ty) &&
1047 TySize < StackSlotSize) {
1048 CharUnits Offset = StackSlotSize - TySize;
1049 OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(OnStackAddr, Offset);
1050 }
1051
1052 OnStackAddr = OnStackAddr.withElementType(MemTy);
1053
1054 CGF.EmitBranch(ContBlock);
1055
1056 //=======================================
1057 // Tidy up
1058 //=======================================
1059 CGF.EmitBlock(ContBlock);
1060
1061 Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, OnStackAddr,
1062 OnStackBlock, "vaargs.addr");
1063
1064 if (IsIndirect)
1065 return CGF.EmitLoadOfAnyValue(
1066 CGF.MakeAddrLValue(
1067 Address(CGF.Builder.CreateLoad(ResAddr, "vaarg.addr"), ElementTy,
1068 TyAlign),
1069 Ty),
1070 Slot);
1071
1072 return CGF.EmitLoadOfAnyValue(CGF.MakeAddrLValue(ResAddr, Ty), Slot);
1073}
1074
1075RValue AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty,
1076 CodeGenFunction &CGF,
1077 AggValueSlot Slot) const {
1078 // The backend's lowering doesn't support va_arg for aggregates or
1079 // illegal vector types. Lower VAArg here for these cases and use
1080 // the LLVM va_arg instruction for everything else.
1081 if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty))
1082 return CGF.EmitLoadOfAnyValue(
1083 CGF.MakeAddrLValue(
1084 EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect()), Ty),
1085 Slot);
1086
1087 uint64_t PointerSize = getTarget().getPointerWidth(LangAS::Default) / 8;
1088 CharUnits SlotSize = CharUnits::fromQuantity(PointerSize);
1089
1090 // Empty records are ignored for parameter passing purposes.
1091 if (isEmptyRecord(getContext(), Ty, true))
1092 return Slot.asRValue();
1093
1094 // The size of the actual thing passed, which might end up just
1095 // being a pointer for indirect types.
1096 auto TyInfo = getContext().getTypeInfoInChars(Ty);
1097
1098 // Arguments bigger than 16 bytes which aren't homogeneous
1099 // aggregates should be passed indirectly.
1100 bool IsIndirect = false;
1101 if (TyInfo.Width.getQuantity() > 16) {
1102 const Type *Base = nullptr;
1103 uint64_t Members = 0;
1104 IsIndirect = !isHomogeneousAggregate(Ty, Base, Members);
1105 }
1106
1107 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TyInfo, SlotSize,
1108 /*AllowHigherAlign*/ true, Slot);
1109}
1110
1111RValue AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
1112 QualType Ty, AggValueSlot Slot) const {
1113 bool IsIndirect = false;
1114
1115 // Composites larger than 16 bytes are passed by reference.
1116 if (isAggregateTypeForABI(Ty) && getContext().getTypeSize(Ty) > 128)
1117 IsIndirect = true;
1118
1119 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
1122 /*allowHigherAlign*/ false, Slot);
1123}
1124
1126 if (const auto *T = F->getType()->getAs<FunctionProtoType>())
1127 return T->getAArch64SMEAttributes() &
1129 return false;
1130}
1131
1132// Report an error if an argument or return value of type Ty would need to be
1133// passed in a floating-point register.
1135 const StringRef ABIName,
1136 const AArch64ABIInfo &ABIInfo,
1137 const QualType &Ty, const NamedDecl *D,
1138 SourceLocation loc) {
1139 const Type *HABase = nullptr;
1140 uint64_t HAMembers = 0;
1141 if (Ty->isFloatingType() || Ty->isVectorType() ||
1142 ABIInfo.isHomogeneousAggregate(Ty, HABase, HAMembers)) {
1143 Diags.Report(loc, diag::err_target_unsupported_type_for_abi)
1144 << D->getDeclName() << Ty << ABIName;
1145 }
1146}
1147
1148// If we are using a hard-float ABI, but do not have floating point registers,
1149// then report an error for any function arguments or returns which would be
1150// passed in floating-pint registers.
1151void AArch64TargetCodeGenInfo::checkFunctionABI(
1152 CodeGenModule &CGM, const FunctionDecl *FuncDecl) const {
1153 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
1155
1156 if (!TI.hasFeature("fp") && !ABIInfo.isSoftFloat()) {
1158 FuncDecl->getReturnType(), FuncDecl,
1159 FuncDecl->getLocation());
1160 for (ParmVarDecl *PVD : FuncDecl->parameters()) {
1161 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, PVD->getType(),
1162 PVD, FuncDecl->getLocation());
1163 }
1164 }
1165}
1166
1167enum class ArmSMEInlinability : uint8_t {
1168 Ok = 0,
1169 ErrorCalleeRequiresNewZA = 1 << 0,
1173
1176
1178};
1179
1180/// Determines if there are any Arm SME ABI issues with inlining \p Callee into
1181/// \p Caller. Returns the issue (if any) in the ArmSMEInlinability bit enum.
1183 const FunctionDecl *Callee) {
1184 bool CallerIsStreaming =
1185 IsArmStreamingFunction(Caller, /*IncludeLocallyStreaming=*/true);
1186 bool CalleeIsStreaming =
1187 IsArmStreamingFunction(Callee, /*IncludeLocallyStreaming=*/true);
1188 bool CallerIsStreamingCompatible = isStreamingCompatible(Caller);
1189 bool CalleeIsStreamingCompatible = isStreamingCompatible(Callee);
1190
1192
1193 if (!CalleeIsStreamingCompatible &&
1194 (CallerIsStreaming != CalleeIsStreaming || CallerIsStreamingCompatible)) {
1195 if (CalleeIsStreaming)
1197 else
1199 }
1200 if (auto *NewAttr = Callee->getAttr<ArmNewAttr>()) {
1201 if (NewAttr->isNewZA())
1203 if (NewAttr->isNewZT0())
1205 }
1206
1207 return Inlinability;
1208}
1209
1210void AArch64TargetCodeGenInfo::checkFunctionCallABIStreaming(
1211 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1212 const FunctionDecl *Callee) const {
1213 if (!Caller || !Callee || !Callee->hasAttr<AlwaysInlineAttr>())
1214 return;
1215
1216 ArmSMEInlinability Inlinability = GetArmSMEInlinability(Caller, Callee);
1217
1220 CGM.getDiags().Report(
1221 CallLoc,
1224 ? diag::err_function_always_inline_attribute_mismatch
1225 : diag::warn_function_always_inline_attribute_mismatch)
1226 << Caller->getDeclName() << Callee->getDeclName() << "streaming";
1227
1228 if ((Inlinability & ArmSMEInlinability::ErrorCalleeRequiresNewZA) ==
1230 CGM.getDiags().Report(CallLoc, diag::err_function_always_inline_new_za)
1231 << Callee->getDeclName();
1232
1233 if ((Inlinability & ArmSMEInlinability::ErrorCalleeRequiresNewZT0) ==
1235 CGM.getDiags().Report(CallLoc, diag::err_function_always_inline_new_zt0)
1236 << Callee->getDeclName();
1237}
1238
1239// If the target does not have floating-point registers, but we are using a
1240// hard-float ABI, there is no way to pass floating-point, vector or HFA values
1241// to functions, so we report an error.
1242void AArch64TargetCodeGenInfo::checkFunctionCallABISoftFloat(
1243 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1244 const FunctionDecl *Callee, const CallArgList &Args,
1245 QualType ReturnType) const {
1246 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
1248
1249 if (!Caller || TI.hasFeature("fp") || ABIInfo.isSoftFloat())
1250 return;
1251
1252 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, ReturnType,
1253 Callee ? Callee : Caller, CallLoc);
1254
1255 for (const CallArg &Arg : Args)
1256 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, Arg.getType(),
1257 Callee ? Callee : Caller, CallLoc);
1258}
1259
1260void AArch64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,
1261 SourceLocation CallLoc,
1262 const FunctionDecl *Caller,
1263 const FunctionDecl *Callee,
1264 const CallArgList &Args,
1265 QualType ReturnType) const {
1266 checkFunctionCallABIStreaming(CGM, CallLoc, Caller, Callee);
1267 checkFunctionCallABISoftFloat(CGM, CallLoc, Caller, Callee, Args, ReturnType);
1268}
1269
1270bool AArch64TargetCodeGenInfo::wouldInliningViolateFunctionCallABI(
1271 const FunctionDecl *Caller, const FunctionDecl *Callee) const {
1272 return Caller && Callee &&
1274}
1275
1276void AArch64ABIInfo::appendAttributeMangling(TargetClonesAttr *Attr,
1277 unsigned Index,
1278 raw_ostream &Out) const {
1279 appendAttributeMangling(Attr->getFeatureStr(Index), Out);
1280}
1281
1282void AArch64ABIInfo::appendAttributeMangling(StringRef AttrStr,
1283 raw_ostream &Out) const {
1284 if (AttrStr == "default") {
1285 Out << ".default";
1286 return;
1287 }
1288
1289 Out << "._";
1291 AttrStr.split(Features, "+");
1292 for (auto &Feat : Features)
1293 Feat = Feat.trim();
1294
1295 llvm::sort(Features, [](const StringRef LHS, const StringRef RHS) {
1296 return LHS.compare(RHS) < 0;
1297 });
1298
1299 llvm::SmallDenseSet<StringRef, 8> UniqueFeats;
1300 for (auto &Feat : Features)
1301 if (auto Ext = llvm::AArch64::parseFMVExtension(Feat))
1302 if (UniqueFeats.insert(Ext->Name).second)
1303 Out << 'M' << Ext->Name;
1304}
1305
1306std::unique_ptr<TargetCodeGenInfo>
1308 AArch64ABIKind Kind) {
1309 return std::make_unique<AArch64TargetCodeGenInfo>(CGM.getTypes(), Kind);
1310}
1311
1312std::unique_ptr<TargetCodeGenInfo>
1314 AArch64ABIKind K) {
1315 return std::make_unique<WindowsAArch64TargetCodeGenInfo>(CGM.getTypes(), K);
1316}
const Decl * D
static bool isStreamingCompatible(const FunctionDecl *F)
Definition: AArch64.cpp:1125
ArmSMEInlinability
Definition: AArch64.cpp:1167
static ArmSMEInlinability GetArmSMEInlinability(const FunctionDecl *Caller, const FunctionDecl *Callee)
Determines if there are any Arm SME ABI issues with inlining Callee into Caller.
Definition: AArch64.cpp:1182
static void diagnoseIfNeedsFPReg(DiagnosticsEngine &Diags, const StringRef ABIName, const AArch64ABIInfo &ABIInfo, const QualType &Ty, const NamedDecl *D, SourceLocation loc)
Definition: AArch64.cpp:1134
TypeInfoChars getTypeInfoInChars(const Type *T) const
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:799
Attr - This represents one attribute.
Definition: Attr.h:43
A fixed int type of a specified bitwidth.
Definition: Type.h:7820
This class is used for builtin types like 'int'.
Definition: Type.h:3035
bool isSVECount() const
Definition: Type.h:3114
Kind getKind() const
Definition: Type.h:3083
Represents a C++ struct/union/class.
Definition: DeclCXX.h:258
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:185
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
Definition: CharUnits.h:63
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
Definition: CharUnits.h:201
ABIArgInfo - Helper class to encapsulate information about how a specific C type should be passed to ...
static ABIArgInfo getIgnore()
static bool isPaddingForCoerceAndExpand(llvm::Type *eltType)
static ABIArgInfo getDirect(llvm::Type *T=nullptr, unsigned Offset=0, llvm::Type *Padding=nullptr, bool CanBeFlattened=true, unsigned Align=0)
static ABIArgInfo getExtend(QualType Ty, llvm::Type *T=nullptr)
static ABIArgInfo getCoerceAndExpand(llvm::StructType *coerceToType, llvm::Type *unpaddedCoerceToType)
llvm::Type * getCoerceToType() const
ABIInfo - Target specific hooks for defining how a type should be passed or returned from functions.
Definition: ABIInfo.h:48
virtual bool allowBFloatArgsAndRet() const
Definition: ABIInfo.h:59
bool isHomogeneousAggregate(QualType Ty, const Type *&Base, uint64_t &Members) const
isHomogeneousAggregate - Return true if a type is an ELFv2 homogeneous aggregate.
Definition: ABIInfo.cpp:61
CodeGen::CGCXXABI & getCXXABI() const
Definition: ABIInfo.cpp:18
ASTContext & getContext() const
Definition: ABIInfo.cpp:20
virtual bool isHomogeneousAggregateBaseType(QualType Ty) const
Definition: ABIInfo.cpp:47
virtual void appendAttributeMangling(TargetAttr *Attr, raw_ostream &Out) const
Definition: ABIInfo.cpp:187
virtual RValue EmitMSVAArg(CodeGen::CodeGenFunction &CGF, CodeGen::Address VAListAddr, QualType Ty, AggValueSlot Slot) const
Emit the target dependent code to load a value of.
Definition: ABIInfo.cpp:42
virtual bool isHomogeneousAggregateSmallEnough(const Type *Base, uint64_t Members) const
Definition: ABIInfo.cpp:51
const TargetInfo & getTarget() const
Definition: ABIInfo.cpp:30
virtual RValue EmitVAArg(CodeGen::CodeGenFunction &CGF, CodeGen::Address VAListAddr, QualType Ty, AggValueSlot Slot) const =0
EmitVAArg - Emit the target dependent code to load a value of.
virtual bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const
Definition: ABIInfo.cpp:56
virtual void computeInfo(CodeGen::CGFunctionInfo &FI) const =0
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
Definition: Address.h:128
static Address invalid()
Definition: Address.h:176
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Definition: Address.h:274
An aggregate value slot.
Definition: CGValue.h:504
RValue asRValue() const
Definition: CGValue.h:666
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Definition: CGBuilder.h:136
Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name="")
Given a pointer to i8, adjust it by a given constant offset.
Definition: CGBuilder.h:305
Address CreateConstArrayGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = [n x T]* ... produce name = getelementptr inbounds addr, i64 0, i64 index where i64 is a...
Definition: CGBuilder.h:241
Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name="")
Definition: CGBuilder.h:219
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:108
llvm::ConstantInt * getSize(CharUnits N)
Definition: CGBuilder.h:99
Address CreateInBoundsGEP(Address Addr, ArrayRef< llvm::Value * > IdxList, llvm::Type *ElementType, CharUnits Align, const Twine &Name="")
Definition: CGBuilder.h:346
RecordArgABI
Specify how one should pass an argument of a record type.
Definition: CGCXXABI.h:150
@ RAA_Default
Pass it using the normal C aggregate rules for the ABI, potentially introducing extra copies and pass...
Definition: CGCXXABI.h:153
@ RAA_DirectInMemory
Pass it on the stack using its defined layout.
Definition: CGCXXABI.h:158
CGFunctionInfo - Class to encapsulate the information about a function definition.
unsigned getCallingConvention() const
getCallingConvention - Return the user specified calling convention, which has been translated into a...
CanQualType getReturnType() const
MutableArrayRef< ArgInfo > arguments()
RequiredArgs getRequiredArgs() const
CallArgList - Type for representing both the value and type of arguments in a call.
Definition: CGCall.h:274
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
llvm::AllocaInst * CreateTempAlloca(llvm::Type *Ty, const Twine &Name="tmp", llvm::Value *ArraySize=nullptr)
CreateTempAlloca - This creates an alloca and inserts it into the entry block if ArraySize is nullptr...
llvm::Type * ConvertTypeForMem(QualType T)
const TargetInfo & getTarget() const
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
llvm::Type * ConvertType(QualType T)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
const CGFunctionInfo * CurFnInfo
RValue EmitLoadOfAnyValue(LValue V, AggValueSlot Slot=AggValueSlot::ignored(), SourceLocation Loc={})
Like EmitLoadOfLValue but also handles complex and aggregate types.
This class organizes the cross-function state that is used while generating LLVM code.
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
const TargetInfo & getTarget() const
const llvm::DataLayout & getDataLayout() const
This class organizes the cross-module state that is used while lowering AST types to LLVM types.
Definition: CodeGenTypes.h:54
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
Definition: CGValue.h:42
unsigned getNumRequiredArgs() const
Target specific hooks for defining how a type should be passed or returned from functions with one of...
Definition: ABIInfo.h:138
virtual bool isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy, unsigned NumElts) const
Returns true if the given vector type is legal from Swift's calling convention perspective.
Definition: ABIInfo.cpp:286
TargetCodeGenInfo - This class organizes various target-specific codegeneration issues,...
Definition: TargetInfo.h:47
virtual bool doesReturnSlotInterfereWithArgs() const
doesReturnSlotInterfereWithArgs - Return true if the target uses an argument slot for an 'sret' type.
Definition: TargetInfo.h:213
virtual bool wouldInliningViolateFunctionCallABI(const FunctionDecl *Caller, const FunctionDecl *Callee) const
Returns true if inlining the function call would produce incorrect code for the current target and sh...
Definition: TargetInfo.h:114
virtual StringRef getARCRetainAutoreleasedReturnValueMarker() const
Retrieve the address of a function to call immediately before calling objc_retainAutoreleasedReturnVa...
Definition: TargetInfo.h:225
static void setBranchProtectionFnAttributes(const TargetInfo::BranchProtectionInfo &BPI, llvm::Function &F)
Definition: TargetInfo.cpp:210
virtual void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller, const FunctionDecl *Callee, const CallArgList &Args, QualType ReturnType) const
Any further codegen related checks that need to be done on a function call in a target specific manne...
Definition: TargetInfo.h:95
virtual void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const
setTargetAttributes - Provides a convenient hook to handle extra target-specific attributes for the g...
Definition: TargetInfo.h:76
virtual void checkFunctionABI(CodeGenModule &CGM, const FunctionDecl *Decl) const
Any further codegen related checks that need to be done on a function signature in a target specific ...
Definition: TargetInfo.h:90
virtual bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF, llvm::Type *Ty) const
Target hook to decide whether an inline asm operand can be passed by value.
Definition: TargetInfo.h:198
virtual int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const
Determines the DWARF register number for the stack pointer, for exception-handling purposes.
Definition: TargetInfo.h:142
Represents the canonical version of C arrays with a specified constant size.
Definition: Type.h:3616
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:86
T * getAttr() const
Definition: DeclBase.h:576
SourceLocation getLocation() const
Definition: DeclBase.h:442
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:231
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1493
A helper class that allows the use of isa/cast/dyncast to detect TagType objects of enums.
Definition: Type.h:6104
Represents a function declaration or definition.
Definition: Decl.h:1935
QualType getReturnType() const
Definition: Decl.h:2720
ArrayRef< ParmVarDecl * > parameters() const
Definition: Decl.h:2649
Represents a prototype with parameter type info, e.g.
Definition: Type.h:5108
unsigned getAArch64SMEAttributes() const
Return a bitmask describing the SME attributes on the function type, see AArch64SMETypeAttributes for...
Definition: Type.h:5567
@ SME_PStateSMCompatibleMask
Definition: Type.h:4589
This represents a decl that may have a name.
Definition: Decl.h:253
DeclarationName getDeclName() const
Get the actual, stored name of the declaration, which may be a special name.
Definition: Decl.h:319
Represents a parameter to a function.
Definition: Decl.h:1725
A (possibly-)qualified type.
Definition: Type.h:929
Represents a struct/union/class.
Definition: Decl.h:4162
field_range fields() const
Definition: Decl.h:4376
A helper class that allows the use of isa/cast/dyncast to detect TagType objects of structs/unions/cl...
Definition: Type.h:6078
Encodes a location in the source.
bool isUnion() const
Definition: Decl.h:3784
Exposes information about the current target.
Definition: TargetInfo.h:220
virtual StringRef getABI() const
Get the ABI currently in use.
Definition: TargetInfo.h:1330
virtual ParsedTargetAttr parseTargetAttr(StringRef Str) const
Definition: TargetInfo.cpp:566
virtual bool hasBFloat16Type() const
Determine whether the _BFloat16 type is supported on this target.
Definition: TargetInfo.h:709
virtual bool hasFeature(StringRef Feature) const
Determine whether the given target has the given feature.
Definition: TargetInfo.h:1493
virtual bool validateBranchProtection(StringRef Spec, StringRef Arch, BranchProtectionInfo &BPI, StringRef &Err) const
Determine if this TargetInfo supports the given branch protection specification.
Definition: TargetInfo.h:1469
The base class of the type hierarchy.
Definition: Type.h:1828
bool isVoidType() const
Definition: Type.h:8516
bool isMFloat8Type() const
Definition: Type.h:8541
bool isSVESizelessBuiltinType() const
Returns true for SVE scalable vector types.
Definition: Type.cpp:2517
const T * castAs() const
Member-template castAs<specific type>.
Definition: Type.h:8810
bool isBuiltinType() const
Helper methods to distinguish type categories.
Definition: Type.h:8288
bool isVectorType() const
Definition: Type.h:8304
bool isFloatingType() const
Definition: Type.cpp:2283
const T * getAs() const
Member-template getAs<specific type>'.
Definition: Type.h:8741
QualType getType() const
Definition: Decl.h:682
Represents a GCC generic vector type.
Definition: Type.h:4035
unsigned getNumElements() const
Definition: Type.h:4050
VectorKind getVectorKind() const
Definition: Type.h:4055
QualType getElementType() const
Definition: Type.h:4049
ABIArgInfo classifyArgumentType(CodeGenModule &CGM, CanQualType type)
Classify the rules for how to pass a particular type.
CGCXXABI::RecordArgABI getRecordArgABI(const RecordType *RT, CGCXXABI &CXXABI)
bool classifyReturnType(const CGCXXABI &CXXABI, CGFunctionInfo &FI, const ABIInfo &Info)
Address EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, const ABIArgInfo &AI)
RValue emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType ValueTy, bool IsIndirect, TypeInfoChars ValueInfo, CharUnits SlotSizeAndAlign, bool AllowHigherAlign, AggValueSlot Slot, bool ForceRightAdjust=false)
Emit va_arg for a platform using the common void* representation, where arguments are simply emitted ...
Address emitMergePHI(CodeGenFunction &CGF, Address Addr1, llvm::BasicBlock *Block1, Address Addr2, llvm::BasicBlock *Block2, const llvm::Twine &Name="")
bool isEmptyField(ASTContext &Context, const FieldDecl *FD, bool AllowArrays, bool AsIfNoUniqueAddr=false)
isEmptyField - Return true iff a the field is "empty", that is it is an unnamed bit-field or an (arra...
llvm::Value * emitRoundPointerUpToAlignment(CodeGenFunction &CGF, llvm::Value *Ptr, CharUnits Align)
bool isAggregateTypeForABI(QualType T)
Definition: ABIInfoImpl.cpp:94
std::unique_ptr< TargetCodeGenInfo > createAArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind Kind)
Definition: AArch64.cpp:1307
QualType useFirstFieldIfTransparentUnion(QualType Ty)
Pass transparent unions as if they were the type of the first element.
std::unique_ptr< TargetCodeGenInfo > createWindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind K)
Definition: AArch64.cpp:1313
bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays, bool AsIfNoUniqueAddr=false)
isEmptyRecord - Return true iff a structure contains only empty fields.
The JSON file list parser is used to communicate input to InstallAPI.
@ CPlusPlus
Definition: LangStandard.h:55
const FunctionProtoType * T
bool IsArmStreamingFunction(const FunctionDecl *FD, bool IncludeLocallyStreaming)
Returns whether the given FunctionDecl has an __arm[_locally]_streaming attribute.
Definition: Decl.cpp:5834
unsigned long uint64_t
#define true
Definition: stdbool.h:25
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
Contains information gathered from parsing the contents of TargetAttr.
Definition: TargetInfo.h:58