clang 22.0.0git
AArch64.cpp
Go to the documentation of this file.
1//===- AArch64.cpp --------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "TargetInfo.h"
11#include "clang/AST/Decl.h"
13#include "llvm/TargetParser/AArch64TargetParser.h"
14
15using namespace clang;
16using namespace clang::CodeGen;
17
18//===----------------------------------------------------------------------===//
19// AArch64 ABI Implementation
20//===----------------------------------------------------------------------===//
21
22namespace {
23
24class AArch64ABIInfo : public ABIInfo {
25 AArch64ABIKind Kind;
26
27public:
28 AArch64ABIInfo(CodeGenTypes &CGT, AArch64ABIKind Kind)
29 : ABIInfo(CGT), Kind(Kind) {}
30
31 bool isSoftFloat() const { return Kind == AArch64ABIKind::AAPCSSoft; }
32
33private:
34 AArch64ABIKind getABIKind() const { return Kind; }
35 bool isDarwinPCS() const { return Kind == AArch64ABIKind::DarwinPCS; }
36
37 ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadicFn) const;
38 ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadicFn,
39 bool IsNamedArg, unsigned CallingConvention,
40 unsigned &NSRN, unsigned &NPRN) const;
41 llvm::Type *convertFixedToScalableVectorType(const VectorType *VT) const;
42 ABIArgInfo coerceIllegalVector(QualType Ty, unsigned &NSRN,
43 unsigned &NPRN) const;
44 ABIArgInfo coerceAndExpandPureScalableAggregate(
45 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
46 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
47 unsigned &NPRN) const;
48 bool isHomogeneousAggregateBaseType(QualType Ty) const override;
49 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
50 uint64_t Members) const override;
51 bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override;
52
53 bool isIllegalVectorType(QualType Ty) const;
54
55 bool passAsAggregateType(QualType Ty) const;
56 bool passAsPureScalableType(QualType Ty, unsigned &NV, unsigned &NP,
57 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const;
58
59 void flattenType(llvm::Type *Ty,
60 SmallVectorImpl<llvm::Type *> &Flattened) const;
61
62 void computeInfo(CGFunctionInfo &FI) const override {
63 if (!::classifyReturnType(getCXXABI(), FI, *this))
64 FI.getReturnInfo() =
66
67 unsigned ArgNo = 0;
68 unsigned NSRN = 0, NPRN = 0;
69 for (auto &it : FI.arguments()) {
70 const bool IsNamedArg =
71 !FI.isVariadic() || ArgNo < FI.getRequiredArgs().getNumRequiredArgs();
72 ++ArgNo;
73 it.info = classifyArgumentType(it.type, FI.isVariadic(), IsNamedArg,
74 FI.getCallingConvention(), NSRN, NPRN);
75 }
76 }
77
78 RValue EmitDarwinVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
79 AggValueSlot Slot) const;
80
81 RValue EmitAAPCSVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
82 AArch64ABIKind Kind, AggValueSlot Slot) const;
83
84 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
85 AggValueSlot Slot) const override {
86 llvm::Type *BaseTy = CGF.ConvertType(Ty);
88 llvm::report_fatal_error("Passing SVE types to variadic functions is "
89 "currently not supported");
90
91 return Kind == AArch64ABIKind::Win64
92 ? EmitMSVAArg(CGF, VAListAddr, Ty, Slot)
93 : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF, Slot)
94 : EmitAAPCSVAArg(VAListAddr, Ty, CGF, Kind, Slot);
95 }
96
97 RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
98 AggValueSlot Slot) const override;
99
100 bool allowBFloatArgsAndRet() const override {
101 return getTarget().hasBFloat16Type();
102 }
103
105 void appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index,
106 raw_ostream &Out) const override;
107 void appendAttributeMangling(StringRef AttrStr,
108 raw_ostream &Out) const override;
109};
110
111class AArch64SwiftABIInfo : public SwiftABIInfo {
112public:
113 explicit AArch64SwiftABIInfo(CodeGenTypes &CGT)
114 : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/true) {}
115
116 bool isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,
117 unsigned NumElts) const override;
118};
119
120class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
121public:
122 AArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIKind Kind)
123 : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(CGT, Kind)) {
124 SwiftInfo = std::make_unique<AArch64SwiftABIInfo>(CGT);
125 }
126
127 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
128 return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue";
129 }
130
131 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
132 return 31;
133 }
134
135 bool doesReturnSlotInterfereWithArgs() const override { return false; }
136
137 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
138 CodeGen::CodeGenModule &CGM) const override {
139 auto *Fn = dyn_cast<llvm::Function>(GV);
140 if (!Fn)
141 return;
142
143 const auto *FD = dyn_cast_or_null<FunctionDecl>(D);
144 TargetInfo::BranchProtectionInfo BPI(CGM.getLangOpts());
145
146 if (FD && FD->hasAttr<TargetAttr>()) {
147 const auto *TA = FD->getAttr<TargetAttr>();
148 ParsedTargetAttr Attr =
149 CGM.getTarget().parseTargetAttr(TA->getFeaturesStr());
150 if (!Attr.BranchProtection.empty()) {
151 StringRef Error;
153 Attr.BranchProtection, Attr.CPU, BPI, CGM.getLangOpts(), Error);
154 assert(Error.empty());
155 }
156 }
157 setBranchProtectionFnAttributes(BPI, *Fn);
158 setPointerAuthFnAttributes(CGM.getCodeGenOpts().PointerAuth, *Fn);
159 }
160
161 bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF,
162 llvm::Type *Ty) const override {
163 if (CGF.getTarget().hasFeature("ls64")) {
164 auto *ST = dyn_cast<llvm::StructType>(Ty);
165 if (ST && ST->getNumElements() == 1) {
166 auto *AT = dyn_cast<llvm::ArrayType>(ST->getElementType(0));
167 if (AT && AT->getNumElements() == 8 &&
168 AT->getElementType()->isIntegerTy(64))
169 return true;
170 }
171 }
173 }
174
175 void checkFunctionABI(CodeGenModule &CGM,
176 const FunctionDecl *Decl) const override;
177
178 void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
179 const FunctionDecl *Caller,
180 const FunctionDecl *Callee, const CallArgList &Args,
181 QualType ReturnType) const override;
182
183 bool wouldInliningViolateFunctionCallABI(
184 const FunctionDecl *Caller, const FunctionDecl *Callee) const override;
185
186private:
187 // Diagnose calls between functions with incompatible Streaming SVE
188 // attributes.
189 void checkFunctionCallABIStreaming(CodeGenModule &CGM, SourceLocation CallLoc,
190 const FunctionDecl *Caller,
191 const FunctionDecl *Callee) const;
192 // Diagnose calls which must pass arguments in floating-point registers when
193 // the selected target does not have floating-point registers.
194 void checkFunctionCallABISoftFloat(CodeGenModule &CGM, SourceLocation CallLoc,
195 const FunctionDecl *Caller,
196 const FunctionDecl *Callee,
197 const CallArgList &Args,
198 QualType ReturnType) const;
199};
200
201class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo {
202public:
203 WindowsAArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIKind K)
204 : AArch64TargetCodeGenInfo(CGT, K) {}
205
206 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
207 CodeGen::CodeGenModule &CGM) const override;
208
209 void getDependentLibraryOption(llvm::StringRef Lib,
210 llvm::SmallString<24> &Opt) const override {
211 Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);
212 }
213
214 void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,
215 llvm::SmallString<32> &Opt) const override {
216 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
217 }
218};
219
220void WindowsAArch64TargetCodeGenInfo::setTargetAttributes(
221 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
222 AArch64TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
223 if (GV->isDeclaration())
224 return;
225 addStackProbeTargetAttributes(D, GV, CGM);
226}
227}
228
229llvm::Type *
230AArch64ABIInfo::convertFixedToScalableVectorType(const VectorType *VT) const {
231 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
232
233 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
234 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
235 BuiltinType::UChar &&
236 "unexpected builtin type for SVE predicate!");
237 return llvm::ScalableVectorType::get(llvm::Type::getInt1Ty(getVMContext()),
238 16);
239 }
240
241 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
242 const auto *BT = VT->getElementType()->castAs<BuiltinType>();
243 switch (BT->getKind()) {
244 default:
245 llvm_unreachable("unexpected builtin type for SVE vector!");
246
247 case BuiltinType::SChar:
248 case BuiltinType::UChar:
249 case BuiltinType::MFloat8:
250 return llvm::ScalableVectorType::get(
251 llvm::Type::getInt8Ty(getVMContext()), 16);
252
253 case BuiltinType::Short:
254 case BuiltinType::UShort:
255 return llvm::ScalableVectorType::get(
256 llvm::Type::getInt16Ty(getVMContext()), 8);
257
258 case BuiltinType::Int:
259 case BuiltinType::UInt:
260 return llvm::ScalableVectorType::get(
261 llvm::Type::getInt32Ty(getVMContext()), 4);
262
263 case BuiltinType::Long:
264 case BuiltinType::ULong:
265 return llvm::ScalableVectorType::get(
266 llvm::Type::getInt64Ty(getVMContext()), 2);
267
268 case BuiltinType::Half:
269 return llvm::ScalableVectorType::get(
270 llvm::Type::getHalfTy(getVMContext()), 8);
271
272 case BuiltinType::Float:
273 return llvm::ScalableVectorType::get(
274 llvm::Type::getFloatTy(getVMContext()), 4);
275
276 case BuiltinType::Double:
277 return llvm::ScalableVectorType::get(
278 llvm::Type::getDoubleTy(getVMContext()), 2);
279
280 case BuiltinType::BFloat16:
281 return llvm::ScalableVectorType::get(
282 llvm::Type::getBFloatTy(getVMContext()), 8);
283 }
284 }
285
286 llvm_unreachable("expected fixed-length SVE vector");
287}
288
289ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty, unsigned &NSRN,
290 unsigned &NPRN) const {
291 assert(Ty->isVectorType() && "expected vector type!");
292
293 const auto *VT = Ty->castAs<VectorType>();
294 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
295 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
296 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
297 BuiltinType::UChar &&
298 "unexpected builtin type for SVE predicate!");
299 NPRN = std::min(NPRN + 1, 4u);
300 return ABIArgInfo::getDirect(llvm::ScalableVectorType::get(
301 llvm::Type::getInt1Ty(getVMContext()), 16));
302 }
303
304 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
305 NSRN = std::min(NSRN + 1, 8u);
306 return ABIArgInfo::getDirect(convertFixedToScalableVectorType(VT));
307 }
308
309 uint64_t Size = getContext().getTypeSize(Ty);
310 // Android promotes <2 x i8> to i16, not i32
311 if ((isAndroid() || isOHOSFamily()) && (Size <= 16)) {
312 llvm::Type *ResType = llvm::Type::getInt16Ty(getVMContext());
313 return ABIArgInfo::getDirect(ResType);
314 }
315 if (Size <= 32) {
316 llvm::Type *ResType = llvm::Type::getInt32Ty(getVMContext());
317 return ABIArgInfo::getDirect(ResType);
318 }
319 if (Size == 64) {
320 NSRN = std::min(NSRN + 1, 8u);
321 auto *ResType =
322 llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 2);
323 return ABIArgInfo::getDirect(ResType);
324 }
325 if (Size == 128) {
326 NSRN = std::min(NSRN + 1, 8u);
327 auto *ResType =
328 llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 4);
329 return ABIArgInfo::getDirect(ResType);
330 }
331
332 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),
333 /*ByVal=*/false);
334}
335
336ABIArgInfo AArch64ABIInfo::coerceAndExpandPureScalableAggregate(
337 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
338 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
339 unsigned &NPRN) const {
340 if (!IsNamedArg || NSRN + NVec > 8 || NPRN + NPred > 4)
341 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),
342 /*ByVal=*/false);
343 NSRN += NVec;
344 NPRN += NPred;
345
346 // Handle SVE vector tuples.
347 if (Ty->isSVESizelessBuiltinType())
348 return ABIArgInfo::getDirect();
349
350 llvm::Type *UnpaddedCoerceToType =
351 UnpaddedCoerceToSeq.size() == 1
352 ? UnpaddedCoerceToSeq[0]
353 : llvm::StructType::get(CGT.getLLVMContext(), UnpaddedCoerceToSeq,
354 true);
355
356 SmallVector<llvm::Type *> CoerceToSeq;
357 flattenType(CGT.ConvertType(Ty), CoerceToSeq);
358 auto *CoerceToType =
359 llvm::StructType::get(CGT.getLLVMContext(), CoerceToSeq, false);
360
361 return ABIArgInfo::getCoerceAndExpand(CoerceToType, UnpaddedCoerceToType);
362}
363
364ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,
365 bool IsNamedArg,
366 unsigned CallingConvention,
367 unsigned &NSRN,
368 unsigned &NPRN) const {
370
371 // Handle illegal vector types here.
372 if (isIllegalVectorType(Ty))
373 return coerceIllegalVector(Ty, NSRN, NPRN);
374
375 if (!passAsAggregateType(Ty)) {
376 // Treat an enum type as its underlying type.
377 if (const auto *ED = Ty->getAsEnumDecl())
378 Ty = ED->getIntegerType();
379
380 if (const auto *EIT = Ty->getAs<BitIntType>())
381 if (EIT->getNumBits() > 128)
382 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),
383 false);
384
385 if (Ty->isVectorType())
386 NSRN = std::min(NSRN + 1, 8u);
387 else if (const auto *BT = Ty->getAs<BuiltinType>()) {
388 if (BT->isFloatingPoint())
389 NSRN = std::min(NSRN + 1, 8u);
390 else {
391 switch (BT->getKind()) {
392 case BuiltinType::SveBool:
393 case BuiltinType::SveCount:
394 NPRN = std::min(NPRN + 1, 4u);
395 break;
396 case BuiltinType::SveBoolx2:
397 NPRN = std::min(NPRN + 2, 4u);
398 break;
399 case BuiltinType::SveBoolx4:
400 NPRN = std::min(NPRN + 4, 4u);
401 break;
402 default:
403 if (BT->isSVESizelessBuiltinType())
404 NSRN = std::min(
405 NSRN + getContext().getBuiltinVectorTypeInfo(BT).NumVectors,
406 8u);
407 }
408 }
409 }
410
411 return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS()
412 ? ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty))
414 }
415
416 // Structures with either a non-trivial destructor or a non-trivial
417 // copy constructor are always indirect.
418 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
419 return getNaturalAlignIndirect(
420 Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
421 /*ByVal=*/RAA == CGCXXABI::RAA_DirectInMemory);
422 }
423
424 // Empty records:
425 uint64_t Size = getContext().getTypeSize(Ty);
426 bool IsEmpty = isEmptyRecord(getContext(), Ty, true);
427 if (!Ty->isSVESizelessBuiltinType() && (IsEmpty || Size == 0)) {
428 // Empty records are ignored in C mode, and in C++ on Darwin.
429 if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS())
430 return ABIArgInfo::getIgnore();
431
432 // In C++ mode, arguments which have sizeof() == 0 (which are non-standard
433 // C++) are ignored. This isn't defined by any standard, so we copy GCC's
434 // behaviour here.
435 if (Size == 0)
436 return ABIArgInfo::getIgnore();
437
438 // Otherwise, they are passed as if they have a size of 1 byte.
439 return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));
440 }
441
442 // Homogeneous Floating-point Aggregates (HFAs) need to be expanded.
443 const Type *Base = nullptr;
444 uint64_t Members = 0;
445 bool IsWin64 = Kind == AArch64ABIKind::Win64 ||
446 CallingConvention == llvm::CallingConv::Win64;
447 bool IsWinVariadic = IsWin64 && IsVariadicFn;
448 // In variadic functions on Windows, all composite types are treated alike,
449 // no special handling of HFAs/HVAs.
450 if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) {
451 NSRN = std::min(NSRN + Members, uint64_t(8));
452 if (Kind != AArch64ABIKind::AAPCS)
454 llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members));
455
456 // For HFAs/HVAs, cap the argument alignment to 16, otherwise
457 // set it to 8 according to the AAPCS64 document.
458 unsigned Align =
459 getContext().getTypeUnadjustedAlignInChars(Ty).getQuantity();
460 Align = (Align >= 16) ? 16 : 8;
462 llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members), 0,
463 nullptr, true, Align);
464 }
465
466 // In AAPCS named arguments of a Pure Scalable Type are passed expanded in
467 // registers, or indirectly if there are not enough registers.
468 if (Kind == AArch64ABIKind::AAPCS) {
469 unsigned NVec = 0, NPred = 0;
470 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
471 if (passAsPureScalableType(Ty, NVec, NPred, UnpaddedCoerceToSeq) &&
472 (NVec + NPred) > 0)
473 return coerceAndExpandPureScalableAggregate(
474 Ty, IsNamedArg, NVec, NPred, UnpaddedCoerceToSeq, NSRN, NPRN);
475 }
476
477 // Aggregates <= 16 bytes are passed directly in registers or on the stack.
478 if (Size <= 128) {
479 unsigned Alignment;
480 if (Kind == AArch64ABIKind::AAPCS) {
481 Alignment = getContext().getTypeUnadjustedAlign(Ty);
482 Alignment = Alignment < 128 ? 64 : 128;
483 } else {
484 Alignment =
485 std::max(getContext().getTypeAlign(Ty),
486 (unsigned)getTarget().getPointerWidth(LangAS::Default));
487 }
488 Size = llvm::alignTo(Size, Alignment);
489
490 // If the Aggregate is made up of pointers, use an array of pointers for the
491 // coerced type. This prevents having to convert ptr2int->int2ptr through
492 // the call, allowing alias analysis to produce better code.
493 auto ContainsOnlyPointers = [&](const auto &Self, QualType Ty) {
494 if (isEmptyRecord(getContext(), Ty, true))
495 return false;
496 const auto *RD = Ty->getAsRecordDecl();
497 if (!RD)
498 return false;
499 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
500 for (const auto &I : CXXRD->bases())
501 if (!Self(Self, I.getType()))
502 return false;
503 }
504 return all_of(RD->fields(), [&](FieldDecl *FD) {
505 QualType FDTy = FD->getType();
506 if (FDTy->isArrayType())
507 FDTy = getContext().getBaseElementType(FDTy);
508 return (FDTy->isPointerOrReferenceType() &&
509 getContext().getTypeSize(FDTy) == 64 &&
510 !FDTy->getPointeeType().hasAddressSpace()) ||
511 Self(Self, FDTy);
512 });
513 };
514
515 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
516 // For aggregates with 16-byte alignment, we use i128.
517 llvm::Type *BaseTy = llvm::Type::getIntNTy(getVMContext(), Alignment);
518 if ((Size == 64 || Size == 128) && Alignment == 64 &&
519 ContainsOnlyPointers(ContainsOnlyPointers, Ty))
520 BaseTy = llvm::PointerType::getUnqual(getVMContext());
522 Size == Alignment ? BaseTy
523 : llvm::ArrayType::get(BaseTy, Size / Alignment));
524 }
525
526 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),
527 /*ByVal=*/false);
528}
529
530ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
531 bool IsVariadicFn) const {
532 if (RetTy->isVoidType())
533 return ABIArgInfo::getIgnore();
534
535 if (const auto *VT = RetTy->getAs<VectorType>()) {
536 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
537 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
538 unsigned NSRN = 0, NPRN = 0;
539 return coerceIllegalVector(RetTy, NSRN, NPRN);
540 }
541 }
542
543 // Large vector types should be returned via memory.
544 if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 128)
545 return getNaturalAlignIndirect(RetTy, getDataLayout().getAllocaAddrSpace());
546
547 if (!passAsAggregateType(RetTy)) {
548 // Treat an enum type as its underlying type.
549 if (const auto *ED = RetTy->getAsEnumDecl())
550 RetTy = ED->getIntegerType();
551
552 if (const auto *EIT = RetTy->getAs<BitIntType>())
553 if (EIT->getNumBits() > 128)
554 return getNaturalAlignIndirect(RetTy,
555 getDataLayout().getAllocaAddrSpace());
556
557 return (isPromotableIntegerTypeForABI(RetTy) && isDarwinPCS()
558 ? ABIArgInfo::getExtend(RetTy)
560 }
561
562 uint64_t Size = getContext().getTypeSize(RetTy);
563 if (!RetTy->isSVESizelessBuiltinType() &&
564 (isEmptyRecord(getContext(), RetTy, true) || Size == 0))
565 return ABIArgInfo::getIgnore();
566
567 const Type *Base = nullptr;
568 uint64_t Members = 0;
569 if (isHomogeneousAggregate(RetTy, Base, Members) &&
570 !(getTarget().getTriple().getArch() == llvm::Triple::aarch64_32 &&
571 IsVariadicFn))
572 // Homogeneous Floating-point Aggregates (HFAs) are returned directly.
573 return ABIArgInfo::getDirect();
574
575 // In AAPCS return values of a Pure Scalable type are treated as a single
576 // named argument and passed expanded in registers, or indirectly if there are
577 // not enough registers.
578 if (Kind == AArch64ABIKind::AAPCS) {
579 unsigned NSRN = 0, NPRN = 0;
580 unsigned NVec = 0, NPred = 0;
581 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
582 if (passAsPureScalableType(RetTy, NVec, NPred, UnpaddedCoerceToSeq) &&
583 (NVec + NPred) > 0)
584 return coerceAndExpandPureScalableAggregate(
585 RetTy, /* IsNamedArg */ true, NVec, NPred, UnpaddedCoerceToSeq, NSRN,
586 NPRN);
587 }
588
589 // Aggregates <= 16 bytes are returned directly in registers or on the stack.
590 if (Size <= 128) {
591 if (Size <= 64 && getDataLayout().isLittleEndian()) {
592 // Composite types are returned in lower bits of a 64-bit register for LE,
593 // and in higher bits for BE. However, integer types are always returned
594 // in lower bits for both LE and BE, and they are not rounded up to
595 // 64-bits. We can skip rounding up of composite types for LE, but not for
596 // BE, otherwise composite types will be indistinguishable from integer
597 // types.
599 llvm::IntegerType::get(getVMContext(), Size));
600 }
601
602 unsigned Alignment = getContext().getTypeAlign(RetTy);
603 Size = llvm::alignTo(Size, 64); // round up to multiple of 8 bytes
604
605 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
606 // For aggregates with 16-byte alignment, we use i128.
607 if (Alignment < 128 && Size == 128) {
608 llvm::Type *BaseTy = llvm::Type::getInt64Ty(getVMContext());
609 return ABIArgInfo::getDirect(llvm::ArrayType::get(BaseTy, Size / 64));
610 }
611 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size));
612 }
613
614 return getNaturalAlignIndirect(RetTy, getDataLayout().getAllocaAddrSpace());
615}
616
617/// isIllegalVectorType - check whether the vector type is legal for AArch64.
618bool AArch64ABIInfo::isIllegalVectorType(QualType Ty) const {
619 if (const VectorType *VT = Ty->getAs<VectorType>()) {
620 // Check whether VT is a fixed-length SVE vector. These types are
621 // represented as scalable vectors in function args/return and must be
622 // coerced from fixed vectors.
623 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
624 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)
625 return true;
626
627 // Check whether VT is legal.
628 unsigned NumElements = VT->getNumElements();
629 uint64_t Size = getContext().getTypeSize(VT);
630 // NumElements should be power of 2.
631 if (!llvm::isPowerOf2_32(NumElements))
632 return true;
633
634 // arm64_32 has to be compatible with the ARM logic here, which allows huge
635 // vectors for some reason.
636 llvm::Triple Triple = getTarget().getTriple();
637 if (Triple.getArch() == llvm::Triple::aarch64_32 &&
638 Triple.isOSBinFormatMachO())
639 return Size <= 32;
640
641 return Size != 64 && (Size != 128 || NumElements == 1);
642 }
643 return false;
644}
645
646bool AArch64SwiftABIInfo::isLegalVectorType(CharUnits VectorSize,
647 llvm::Type *EltTy,
648 unsigned NumElts) const {
649 if (!llvm::isPowerOf2_32(NumElts))
650 return false;
651 if (VectorSize.getQuantity() != 8 &&
652 (VectorSize.getQuantity() != 16 || NumElts == 1))
653 return false;
654 return true;
655}
656
657bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
658 // For the soft-float ABI variant, no types are considered to be homogeneous
659 // aggregates.
660 if (isSoftFloat())
661 return false;
662
663 // Homogeneous aggregates for AAPCS64 must have base types of a floating
664 // point type or a short-vector type. This is the same as the 32-bit ABI,
665 // but with the difference that any floating-point type is allowed,
666 // including __fp16.
667 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
668 if (BT->isFloatingPoint())
669 return true;
670 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
671 if (auto Kind = VT->getVectorKind();
672 Kind == VectorKind::SveFixedLengthData ||
673 Kind == VectorKind::SveFixedLengthPredicate)
674 return false;
675
676 unsigned VecSize = getContext().getTypeSize(VT);
677 if (VecSize == 64 || VecSize == 128)
678 return true;
679 }
680 return false;
681}
682
683bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
684 uint64_t Members) const {
685 return Members <= 4;
686}
687
688bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate()
689 const {
690 // AAPCS64 says that the rule for whether something is a homogeneous
691 // aggregate is applied to the output of the data layout decision. So
692 // anything that doesn't affect the data layout also does not affect
693 // homogeneity. In particular, zero-length bitfields don't stop a struct
694 // being homogeneous.
695 return true;
696}
697
698bool AArch64ABIInfo::passAsAggregateType(QualType Ty) const {
699 if (Kind == AArch64ABIKind::AAPCS && Ty->isSVESizelessBuiltinType()) {
700 const auto *BT = Ty->castAs<BuiltinType>();
701 return !BT->isSVECount() &&
702 getContext().getBuiltinVectorTypeInfo(BT).NumVectors > 1;
703 }
704 return isAggregateTypeForABI(Ty);
705}
706
707// Check if a type needs to be passed in registers as a Pure Scalable Type (as
708// defined by AAPCS64). Return the number of data vectors and the number of
709// predicate vectors in the type, into `NVec` and `NPred`, respectively. Upon
710// return `CoerceToSeq` contains an expanded sequence of LLVM IR types, one
711// element for each non-composite member. For practical purposes, limit the
712// length of `CoerceToSeq` to about 12 (the maximum that could possibly fit
713// in registers) and return false, the effect of which will be to pass the
714// argument under the rules for a large (> 128 bytes) composite.
715bool AArch64ABIInfo::passAsPureScalableType(
716 QualType Ty, unsigned &NVec, unsigned &NPred,
717 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const {
718 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
719 uint64_t NElt = AT->getZExtSize();
720 if (NElt == 0)
721 return false;
722
723 unsigned NV = 0, NP = 0;
724 SmallVector<llvm::Type *> EltCoerceToSeq;
725 if (!passAsPureScalableType(AT->getElementType(), NV, NP, EltCoerceToSeq))
726 return false;
727
728 if (CoerceToSeq.size() + NElt * EltCoerceToSeq.size() > 12)
729 return false;
730
731 for (uint64_t I = 0; I < NElt; ++I)
732 llvm::append_range(CoerceToSeq, EltCoerceToSeq);
733
734 NVec += NElt * NV;
735 NPred += NElt * NP;
736 return true;
737 }
738
739 if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {
740 // If the record cannot be passed in registers, then it's not a PST.
741 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
743 return false;
744
745 // Pure scalable types are never unions and never contain unions.
746 const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
747 if (RD->isUnion())
748 return false;
749
750 // If this is a C++ record, check the bases.
751 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
752 for (const auto &I : CXXRD->bases()) {
753 if (isEmptyRecord(getContext(), I.getType(), true))
754 continue;
755 if (!passAsPureScalableType(I.getType(), NVec, NPred, CoerceToSeq))
756 return false;
757 }
758 }
759
760 // Check members.
761 for (const auto *FD : RD->fields()) {
762 QualType FT = FD->getType();
763 if (isEmptyField(getContext(), FD, /* AllowArrays */ true))
764 continue;
765 if (!passAsPureScalableType(FT, NVec, NPred, CoerceToSeq))
766 return false;
767 }
768
769 return true;
770 }
771
772 if (const auto *VT = Ty->getAs<VectorType>()) {
773 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
774 ++NPred;
775 if (CoerceToSeq.size() + 1 > 12)
776 return false;
777 CoerceToSeq.push_back(convertFixedToScalableVectorType(VT));
778 return true;
779 }
780
781 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
782 ++NVec;
783 if (CoerceToSeq.size() + 1 > 12)
784 return false;
785 CoerceToSeq.push_back(convertFixedToScalableVectorType(VT));
786 return true;
787 }
788
789 return false;
790 }
791
792 if (!Ty->isBuiltinType())
793 return false;
794
795 bool isPredicate;
796 switch (Ty->castAs<BuiltinType>()->getKind()) {
797#define SVE_VECTOR_TYPE(Name, MangledName, Id, SingletonId) \
798 case BuiltinType::Id: \
799 isPredicate = false; \
800 break;
801#define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId) \
802 case BuiltinType::Id: \
803 isPredicate = true; \
804 break;
805#include "clang/Basic/AArch64ACLETypes.def"
806 default:
807 return false;
808 }
809
810 ASTContext::BuiltinVectorTypeInfo Info =
811 getContext().getBuiltinVectorTypeInfo(cast<BuiltinType>(Ty));
812 assert(Info.NumVectors > 0 && Info.NumVectors <= 4 &&
813 "Expected 1, 2, 3 or 4 vectors!");
814 if (isPredicate)
815 NPred += Info.NumVectors;
816 else
817 NVec += Info.NumVectors;
818 llvm::Type *EltTy = Info.ElementType->isMFloat8Type()
819 ? llvm::Type::getInt8Ty(getVMContext())
820 : CGT.ConvertType(Info.ElementType);
821 auto *VTy = llvm::ScalableVectorType::get(EltTy, Info.EC.getKnownMinValue());
822
823 if (CoerceToSeq.size() + Info.NumVectors > 12)
824 return false;
825 std::fill_n(std::back_inserter(CoerceToSeq), Info.NumVectors, VTy);
826
827 return true;
828}
829
830// Expand an LLVM IR type into a sequence with a element for each non-struct,
831// non-array member of the type, with the exception of the padding types, which
832// are retained.
833void AArch64ABIInfo::flattenType(
834 llvm::Type *Ty, SmallVectorImpl<llvm::Type *> &Flattened) const {
835
837 Flattened.push_back(Ty);
838 return;
839 }
840
841 if (const auto *AT = dyn_cast<llvm::ArrayType>(Ty)) {
842 uint64_t NElt = AT->getNumElements();
843 if (NElt == 0)
844 return;
845
846 SmallVector<llvm::Type *> EltFlattened;
847 flattenType(AT->getElementType(), EltFlattened);
848
849 for (uint64_t I = 0; I < NElt; ++I)
850 llvm::append_range(Flattened, EltFlattened);
851 return;
852 }
853
854 if (const auto *ST = dyn_cast<llvm::StructType>(Ty)) {
855 for (auto *ET : ST->elements())
856 flattenType(ET, Flattened);
857 return;
858 }
859
860 Flattened.push_back(Ty);
861}
862
863RValue AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
864 CodeGenFunction &CGF, AArch64ABIKind Kind,
865 AggValueSlot Slot) const {
866 // These numbers are not used for variadic arguments, hence it doesn't matter
867 // they don't retain their values across multiple calls to
868 // `classifyArgumentType` here.
869 unsigned NSRN = 0, NPRN = 0;
870 ABIArgInfo AI =
871 classifyArgumentType(Ty, /*IsVariadicFn=*/true, /* IsNamedArg */ false,
872 CGF.CurFnInfo->getCallingConvention(), NSRN, NPRN);
873 // Empty records are ignored for parameter passing purposes.
874 if (AI.isIgnore())
875 return Slot.asRValue();
876
877 bool IsIndirect = AI.isIndirect();
878
879 llvm::Type *BaseTy = CGF.ConvertType(Ty);
880 if (IsIndirect)
881 BaseTy = llvm::PointerType::getUnqual(BaseTy->getContext());
882 else if (AI.getCoerceToType())
883 BaseTy = AI.getCoerceToType();
884
885 unsigned NumRegs = 1;
886 if (llvm::ArrayType *ArrTy = dyn_cast<llvm::ArrayType>(BaseTy)) {
887 BaseTy = ArrTy->getElementType();
888 NumRegs = ArrTy->getNumElements();
889 }
890 bool IsFPR =
891 !isSoftFloat() && (BaseTy->isFloatingPointTy() || BaseTy->isVectorTy());
892
893 // The AArch64 va_list type and handling is specified in the Procedure Call
894 // Standard, section B.4:
895 //
896 // struct {
897 // void *__stack;
898 // void *__gr_top;
899 // void *__vr_top;
900 // int __gr_offs;
901 // int __vr_offs;
902 // };
903
904 llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock("vaarg.maybe_reg");
905 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
906 llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock("vaarg.on_stack");
907 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
908
909 CharUnits TySize = getContext().getTypeSizeInChars(Ty);
910 CharUnits TyAlign = getContext().getTypeUnadjustedAlignInChars(Ty);
911
912 Address reg_offs_p = Address::invalid();
913 llvm::Value *reg_offs = nullptr;
914 int reg_top_index;
915 int RegSize = IsIndirect ? 8 : TySize.getQuantity();
916 if (!IsFPR) {
917 // 3 is the field number of __gr_offs
918 reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 3, "gr_offs_p");
919 reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "gr_offs");
920 reg_top_index = 1; // field number for __gr_top
921 RegSize = llvm::alignTo(RegSize, 8);
922 } else {
923 // 4 is the field number of __vr_offs.
924 reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 4, "vr_offs_p");
925 reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "vr_offs");
926 reg_top_index = 2; // field number for __vr_top
927 RegSize = 16 * NumRegs;
928 }
929
930 //=======================================
931 // Find out where argument was passed
932 //=======================================
933
934 // If reg_offs >= 0 we're already using the stack for this type of
935 // argument. We don't want to keep updating reg_offs (in case it overflows,
936 // though anyone passing 2GB of arguments, each at most 16 bytes, deserves
937 // whatever they get).
938 llvm::Value *UsingStack = nullptr;
939 UsingStack = CGF.Builder.CreateICmpSGE(
940 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, 0));
941
942 CGF.Builder.CreateCondBr(UsingStack, OnStackBlock, MaybeRegBlock);
943
944 // Otherwise, at least some kind of argument could go in these registers, the
945 // question is whether this particular type is too big.
946 CGF.EmitBlock(MaybeRegBlock);
947
948 // Integer arguments may need to correct register alignment (for example a
949 // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we
950 // align __gr_offs to calculate the potential address.
951 if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) {
952 int Align = TyAlign.getQuantity();
953
954 reg_offs = CGF.Builder.CreateAdd(
955 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, Align - 1),
956 "align_regoffs");
957 reg_offs = CGF.Builder.CreateAnd(
958 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, -Align),
959 "aligned_regoffs");
960 }
961
962 // Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list.
963 // The fact that this is done unconditionally reflects the fact that
964 // allocating an argument to the stack also uses up all the remaining
965 // registers of the appropriate kind.
966 llvm::Value *NewOffset = nullptr;
967 NewOffset = CGF.Builder.CreateAdd(
968 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, RegSize), "new_reg_offs");
969 CGF.Builder.CreateStore(NewOffset, reg_offs_p);
970
971 // Now we're in a position to decide whether this argument really was in
972 // registers or not.
973 llvm::Value *InRegs = nullptr;
974 InRegs = CGF.Builder.CreateICmpSLE(
975 NewOffset, llvm::ConstantInt::get(CGF.Int32Ty, 0), "inreg");
976
977 CGF.Builder.CreateCondBr(InRegs, InRegBlock, OnStackBlock);
978
979 //=======================================
980 // Argument was in registers
981 //=======================================
982
983 // Now we emit the code for if the argument was originally passed in
984 // registers. First start the appropriate block:
985 CGF.EmitBlock(InRegBlock);
986
987 llvm::Value *reg_top = nullptr;
988 Address reg_top_p =
989 CGF.Builder.CreateStructGEP(VAListAddr, reg_top_index, "reg_top_p");
990 reg_top = CGF.Builder.CreateLoad(reg_top_p, "reg_top");
991 Address BaseAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, reg_top, reg_offs),
992 CGF.Int8Ty, CharUnits::fromQuantity(IsFPR ? 16 : 8));
993 Address RegAddr = Address::invalid();
994 llvm::Type *MemTy = CGF.ConvertTypeForMem(Ty), *ElementTy = MemTy;
995
996 if (IsIndirect) {
997 // If it's been passed indirectly (actually a struct), whatever we find from
998 // stored registers or on the stack will actually be a struct **.
999 MemTy = llvm::PointerType::getUnqual(MemTy->getContext());
1000 }
1001
1002 const Type *Base = nullptr;
1003 uint64_t NumMembers = 0;
1004 bool IsHFA = isHomogeneousAggregate(Ty, Base, NumMembers);
1005 if (IsHFA && NumMembers > 1) {
1006 // Homogeneous aggregates passed in registers will have their elements split
1007 // and stored 16-bytes apart regardless of size (they're notionally in qN,
1008 // qN+1, ...). We reload and store into a temporary local variable
1009 // contiguously.
1010 assert(!IsIndirect && "Homogeneous aggregates should be passed directly");
1011 auto BaseTyInfo = getContext().getTypeInfoInChars(QualType(Base, 0));
1012 llvm::Type *BaseTy = CGF.ConvertType(QualType(Base, 0));
1013 llvm::Type *HFATy = llvm::ArrayType::get(BaseTy, NumMembers);
1014 Address Tmp = CGF.CreateTempAlloca(HFATy,
1015 std::max(TyAlign, BaseTyInfo.Align));
1016
1017 // On big-endian platforms, the value will be right-aligned in its slot.
1018 int Offset = 0;
1019 if (CGF.CGM.getDataLayout().isBigEndian() &&
1020 BaseTyInfo.Width.getQuantity() < 16)
1021 Offset = 16 - BaseTyInfo.Width.getQuantity();
1022
1023 for (unsigned i = 0; i < NumMembers; ++i) {
1024 CharUnits BaseOffset = CharUnits::fromQuantity(16 * i + Offset);
1025 Address LoadAddr =
1026 CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, BaseOffset);
1027 LoadAddr = LoadAddr.withElementType(BaseTy);
1028
1029 Address StoreAddr = CGF.Builder.CreateConstArrayGEP(Tmp, i);
1030
1031 llvm::Value *Elem = CGF.Builder.CreateLoad(LoadAddr);
1032 CGF.Builder.CreateStore(Elem, StoreAddr);
1033 }
1034
1035 RegAddr = Tmp.withElementType(MemTy);
1036 } else {
1037 // Otherwise the object is contiguous in memory.
1038
1039 // It might be right-aligned in its slot.
1040 CharUnits SlotSize = BaseAddr.getAlignment();
1041 if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect &&
1042 (IsHFA || !isAggregateTypeForABI(Ty)) &&
1043 TySize < SlotSize) {
1044 CharUnits Offset = SlotSize - TySize;
1045 BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, Offset);
1046 }
1047
1048 RegAddr = BaseAddr.withElementType(MemTy);
1049 }
1050
1051 CGF.EmitBranch(ContBlock);
1052
1053 //=======================================
1054 // Argument was on the stack
1055 //=======================================
1056 CGF.EmitBlock(OnStackBlock);
1057
1058 Address stack_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "stack_p");
1059 llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(stack_p, "stack");
1060
1061 // Again, stack arguments may need realignment. In this case both integer and
1062 // floating-point ones might be affected.
1063 if (!IsIndirect && TyAlign.getQuantity() > 8) {
1064 OnStackPtr = emitRoundPointerUpToAlignment(CGF, OnStackPtr, TyAlign);
1065 }
1066 Address OnStackAddr = Address(OnStackPtr, CGF.Int8Ty,
1067 std::max(CharUnits::fromQuantity(8), TyAlign));
1068
1069 // All stack slots are multiples of 8 bytes.
1070 CharUnits StackSlotSize = CharUnits::fromQuantity(8);
1071 CharUnits StackSize;
1072 if (IsIndirect)
1073 StackSize = StackSlotSize;
1074 else
1075 StackSize = TySize.alignTo(StackSlotSize);
1076
1077 llvm::Value *StackSizeC = CGF.Builder.getSize(StackSize);
1078 llvm::Value *NewStack = CGF.Builder.CreateInBoundsGEP(
1079 CGF.Int8Ty, OnStackPtr, StackSizeC, "new_stack");
1080
1081 // Write the new value of __stack for the next call to va_arg
1082 CGF.Builder.CreateStore(NewStack, stack_p);
1083
1084 if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(Ty) &&
1085 TySize < StackSlotSize) {
1086 CharUnits Offset = StackSlotSize - TySize;
1087 OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(OnStackAddr, Offset);
1088 }
1089
1090 OnStackAddr = OnStackAddr.withElementType(MemTy);
1091
1092 CGF.EmitBranch(ContBlock);
1093
1094 //=======================================
1095 // Tidy up
1096 //=======================================
1097 CGF.EmitBlock(ContBlock);
1098
1099 Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, OnStackAddr,
1100 OnStackBlock, "vaargs.addr");
1101
1102 if (IsIndirect)
1103 return CGF.EmitLoadOfAnyValue(
1104 CGF.MakeAddrLValue(
1105 Address(CGF.Builder.CreateLoad(ResAddr, "vaarg.addr"), ElementTy,
1106 TyAlign),
1107 Ty),
1108 Slot);
1109
1110 return CGF.EmitLoadOfAnyValue(CGF.MakeAddrLValue(ResAddr, Ty), Slot);
1111}
1112
1113RValue AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty,
1114 CodeGenFunction &CGF,
1115 AggValueSlot Slot) const {
1116 // The backend's lowering doesn't support va_arg for aggregates or
1117 // illegal vector types. Lower VAArg here for these cases and use
1118 // the LLVM va_arg instruction for everything else.
1119 if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty))
1120 return CGF.EmitLoadOfAnyValue(
1121 CGF.MakeAddrLValue(
1122 EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect()), Ty),
1123 Slot);
1124
1125 uint64_t PointerSize = getTarget().getPointerWidth(LangAS::Default) / 8;
1126 CharUnits SlotSize = CharUnits::fromQuantity(PointerSize);
1127
1128 // Empty records are ignored for parameter passing purposes.
1129 if (isEmptyRecord(getContext(), Ty, true))
1130 return Slot.asRValue();
1131
1132 // The size of the actual thing passed, which might end up just
1133 // being a pointer for indirect types.
1134 auto TyInfo = getContext().getTypeInfoInChars(Ty);
1135
1136 // Arguments bigger than 16 bytes which aren't homogeneous
1137 // aggregates should be passed indirectly.
1138 bool IsIndirect = false;
1139 if (TyInfo.Width.getQuantity() > 16) {
1140 const Type *Base = nullptr;
1141 uint64_t Members = 0;
1142 IsIndirect = !isHomogeneousAggregate(Ty, Base, Members);
1143 }
1144
1145 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TyInfo, SlotSize,
1146 /*AllowHigherAlign*/ true, Slot);
1147}
1148
1149RValue AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
1150 QualType Ty, AggValueSlot Slot) const {
1151 bool IsIndirect = false;
1152
1153 // Composites larger than 16 bytes are passed by reference.
1154 if (isAggregateTypeForABI(Ty) && getContext().getTypeSize(Ty) > 128)
1155 IsIndirect = true;
1156
1157 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
1160 /*allowHigherAlign*/ false, Slot);
1161}
1162
1164 if (const auto *T = F->getType()->getAs<FunctionProtoType>())
1165 return T->getAArch64SMEAttributes() &
1167 return false;
1168}
1169
1170// Report an error if an argument or return value of type Ty would need to be
1171// passed in a floating-point register.
1173 const StringRef ABIName,
1174 const AArch64ABIInfo &ABIInfo,
1175 const QualType &Ty, const NamedDecl *D,
1176 SourceLocation loc) {
1177 const Type *HABase = nullptr;
1178 uint64_t HAMembers = 0;
1179 if (Ty->isFloatingType() || Ty->isVectorType() ||
1180 ABIInfo.isHomogeneousAggregate(Ty, HABase, HAMembers)) {
1181 Diags.Report(loc, diag::err_target_unsupported_type_for_abi)
1182 << D->getDeclName() << Ty << ABIName;
1183 }
1184}
1185
1186// If we are using a hard-float ABI, but do not have floating point registers,
1187// then report an error for any function arguments or returns which would be
1188// passed in floating-pint registers.
1189void AArch64TargetCodeGenInfo::checkFunctionABI(
1190 CodeGenModule &CGM, const FunctionDecl *FuncDecl) const {
1191 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
1192 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
1193
1194 if (!TI.hasFeature("fp") && !ABIInfo.isSoftFloat()) {
1195 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo,
1196 FuncDecl->getReturnType(), FuncDecl,
1197 FuncDecl->getLocation());
1198 for (ParmVarDecl *PVD : FuncDecl->parameters()) {
1199 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, PVD->getType(),
1200 PVD, FuncDecl->getLocation());
1201 }
1202 }
1203}
1204
1217
1218/// Determines if there are any Arm SME ABI issues with inlining \p Callee into
1219/// \p Caller. Returns the issue (if any) in the ArmSMEInlinability bit enum.
1221 const FunctionDecl *Callee) {
1222 bool CallerIsStreaming =
1223 IsArmStreamingFunction(Caller, /*IncludeLocallyStreaming=*/true);
1224 bool CalleeIsStreaming =
1225 IsArmStreamingFunction(Callee, /*IncludeLocallyStreaming=*/true);
1226 bool CallerIsStreamingCompatible = isStreamingCompatible(Caller);
1227 bool CalleeIsStreamingCompatible = isStreamingCompatible(Callee);
1228
1230
1231 if (!CalleeIsStreamingCompatible &&
1232 (CallerIsStreaming != CalleeIsStreaming || CallerIsStreamingCompatible)) {
1233 if (CalleeIsStreaming)
1235 else
1237 }
1238 if (auto *NewAttr = Callee->getAttr<ArmNewAttr>()) {
1239 if (NewAttr->isNewZA())
1241 if (NewAttr->isNewZT0())
1243 }
1244
1245 return Inlinability;
1246}
1247
1248void AArch64TargetCodeGenInfo::checkFunctionCallABIStreaming(
1249 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1250 const FunctionDecl *Callee) const {
1251 if (!Caller || !Callee || !Callee->hasAttr<AlwaysInlineAttr>())
1252 return;
1253
1254 ArmSMEInlinability Inlinability = GetArmSMEInlinability(Caller, Callee);
1255
1258 CGM.getDiags().Report(
1259 CallLoc,
1262 ? diag::err_function_always_inline_attribute_mismatch
1263 : diag::warn_function_always_inline_attribute_mismatch)
1264 << Caller->getDeclName() << Callee->getDeclName() << "streaming";
1265
1266 if ((Inlinability & ArmSMEInlinability::ErrorCalleeRequiresNewZA) ==
1268 CGM.getDiags().Report(CallLoc, diag::err_function_always_inline_new_za)
1269 << Callee->getDeclName();
1270
1271 if ((Inlinability & ArmSMEInlinability::ErrorCalleeRequiresNewZT0) ==
1273 CGM.getDiags().Report(CallLoc, diag::err_function_always_inline_new_zt0)
1274 << Callee->getDeclName();
1275}
1276
1277// If the target does not have floating-point registers, but we are using a
1278// hard-float ABI, there is no way to pass floating-point, vector or HFA values
1279// to functions, so we report an error.
1280void AArch64TargetCodeGenInfo::checkFunctionCallABISoftFloat(
1281 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1282 const FunctionDecl *Callee, const CallArgList &Args,
1283 QualType ReturnType) const {
1284 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
1285 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
1286
1287 if (!Caller || TI.hasFeature("fp") || ABIInfo.isSoftFloat())
1288 return;
1289
1290 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, ReturnType,
1291 Callee ? Callee : Caller, CallLoc);
1292
1293 for (const CallArg &Arg : Args)
1294 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, Arg.getType(),
1295 Callee ? Callee : Caller, CallLoc);
1296}
1297
1298void AArch64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,
1299 SourceLocation CallLoc,
1300 const FunctionDecl *Caller,
1301 const FunctionDecl *Callee,
1302 const CallArgList &Args,
1303 QualType ReturnType) const {
1304 checkFunctionCallABIStreaming(CGM, CallLoc, Caller, Callee);
1305 checkFunctionCallABISoftFloat(CGM, CallLoc, Caller, Callee, Args, ReturnType);
1306}
1307
1308bool AArch64TargetCodeGenInfo::wouldInliningViolateFunctionCallABI(
1309 const FunctionDecl *Caller, const FunctionDecl *Callee) const {
1310 return Caller && Callee &&
1312}
1313
1314void AArch64ABIInfo::appendAttributeMangling(TargetClonesAttr *Attr,
1315 unsigned Index,
1316 raw_ostream &Out) const {
1317 appendAttributeMangling(Attr->getFeatureStr(Index), Out);
1318}
1319
1320void AArch64ABIInfo::appendAttributeMangling(StringRef AttrStr,
1321 raw_ostream &Out) const {
1322 if (AttrStr == "default") {
1323 Out << ".default";
1324 return;
1325 }
1326
1327 Out << "._";
1328 SmallVector<StringRef, 8> Features;
1329 AttrStr.split(Features, "+");
1330 for (auto &Feat : Features)
1331 Feat = Feat.trim();
1332
1333 llvm::sort(Features, [](const StringRef LHS, const StringRef RHS) {
1334 return LHS.compare(RHS) < 0;
1335 });
1336
1337 llvm::SmallDenseSet<StringRef, 8> UniqueFeats;
1338 for (auto &Feat : Features)
1339 if (auto Ext = llvm::AArch64::parseFMVExtension(Feat))
1340 if (UniqueFeats.insert(Ext->Name).second)
1341 Out << 'M' << Ext->Name;
1342}
1343
1344std::unique_ptr<TargetCodeGenInfo>
1346 AArch64ABIKind Kind) {
1347 return std::make_unique<AArch64TargetCodeGenInfo>(CGM.getTypes(), Kind);
1348}
1349
1350std::unique_ptr<TargetCodeGenInfo>
1352 AArch64ABIKind K) {
1353 return std::make_unique<WindowsAArch64TargetCodeGenInfo>(CGM.getTypes(), K);
1354}
static bool isStreamingCompatible(const FunctionDecl *F)
Definition AArch64.cpp:1163
ArmSMEInlinability
Definition AArch64.cpp:1205
static ArmSMEInlinability GetArmSMEInlinability(const FunctionDecl *Caller, const FunctionDecl *Callee)
Determines if there are any Arm SME ABI issues with inlining Callee into Caller.
Definition AArch64.cpp:1220
static void diagnoseIfNeedsFPReg(DiagnosticsEngine &Diags, const StringRef ABIName, const AArch64ABIInfo &ABIInfo, const QualType &Ty, const NamedDecl *D, SourceLocation loc)
Definition AArch64.cpp:1172
TypeInfoChars getTypeInfoInChars(const Type *T) const
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition CharUnits.h:185
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:63
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
Definition CharUnits.h:201
PointerAuthOptions PointerAuth
Configuration for pointer-signing.
static ABIArgInfo getIgnore()
static bool isPaddingForCoerceAndExpand(llvm::Type *eltType)
static ABIArgInfo getDirect(llvm::Type *T=nullptr, unsigned Offset=0, llvm::Type *Padding=nullptr, bool CanBeFlattened=true, unsigned Align=0)
static ABIArgInfo getExtend(QualType Ty, llvm::Type *T=nullptr)
static ABIArgInfo getCoerceAndExpand(llvm::StructType *coerceToType, llvm::Type *unpaddedCoerceToType)
llvm::Type * getCoerceToType() const
ABIInfo - Target specific hooks for defining how a type should be passed or returned from functions.
Definition ABIInfo.h:48
bool isHomogeneousAggregate(QualType Ty, const Type *&Base, uint64_t &Members) const
isHomogeneousAggregate - Return true if a type is an ELFv2 homogeneous aggregate.
Definition ABIInfo.cpp:61
virtual void appendAttributeMangling(TargetAttr *Attr, raw_ostream &Out) const
Definition ABIInfo.cpp:186
static Address invalid()
Definition Address.h:176
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Definition Address.h:276
RValue asRValue() const
Definition CGValue.h:666
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Definition CGBuilder.h:140
Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name="")
Given a pointer to i8, adjust it by a given constant offset.
Definition CGBuilder.h:309
Address CreateConstArrayGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = [n x T]* ... produce name = getelementptr inbounds addr, i64 0, i64 index where i64 is a...
Definition CGBuilder.h:245
Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name="")
Definition CGBuilder.h:223
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition CGBuilder.h:112
llvm::ConstantInt * getSize(CharUnits N)
Definition CGBuilder.h:103
Address CreateInBoundsGEP(Address Addr, ArrayRef< llvm::Value * > IdxList, llvm::Type *ElementType, CharUnits Align, const Twine &Name="")
Definition CGBuilder.h:350
RecordArgABI
Specify how one should pass an argument of a record type.
Definition CGCXXABI.h:150
@ RAA_Default
Pass it using the normal C aggregate rules for the ABI, potentially introducing extra copies and pass...
Definition CGCXXABI.h:153
@ RAA_DirectInMemory
Pass it on the stack using its defined layout.
Definition CGCXXABI.h:158
unsigned getCallingConvention() const
getCallingConvention - Return the user specified calling convention, which has been translated into a...
CanQualType getReturnType() const
MutableArrayRef< ArgInfo > arguments()
RequiredArgs getRequiredArgs() const
llvm::Type * ConvertType(QualType T)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
RValue EmitLoadOfAnyValue(LValue V, AggValueSlot Slot=AggValueSlot::ignored(), SourceLocation Loc={})
Like EmitLoadOfLValue but also handles complex and aggregate types.
Definition CGExpr.cpp:2318
const TargetInfo & getTarget() const
llvm::AllocaInst * CreateTempAlloca(llvm::Type *Ty, const Twine &Name="tmp", llvm::Value *ArraySize=nullptr)
CreateTempAlloca - This creates an alloca and inserts it into the entry block if ArraySize is nullptr...
Definition CGExpr.cpp:151
llvm::Type * ConvertTypeForMem(QualType T)
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
Definition CGStmt.cpp:672
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
const CGFunctionInfo * CurFnInfo
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition CGStmt.cpp:652
This class organizes the cross-function state that is used while generating LLVM code.
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
const TargetInfo & getTarget() const
const llvm::DataLayout & getDataLayout() const
const CodeGenOptions & getCodeGenOpts() const
unsigned getNumRequiredArgs() const
Target specific hooks for defining how a type should be passed or returned from functions with one of...
Definition ABIInfo.h:145
TargetCodeGenInfo - This class organizes various target-specific codegeneration issues,...
Definition TargetInfo.h:47
virtual bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF, llvm::Type *Ty) const
Target hook to decide whether an inline asm operand can be passed by value.
Definition TargetInfo.h:202
Decl - This represents one declaration (or definition), e.g.
Definition DeclBase.h:86
SourceLocation getLocation() const
Definition DeclBase.h:439
Concrete class used by the front-end to report problems and issues.
Definition Diagnostic.h:231
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Represents a function declaration or definition.
Definition Decl.h:1999
QualType getReturnType() const
Definition Decl.h:2842
ArrayRef< ParmVarDecl * > parameters() const
Definition Decl.h:2771
Represents a prototype with parameter type info, e.g.
Definition TypeBase.h:5264
This represents a decl that may have a name.
Definition Decl.h:273
DeclarationName getDeclName() const
Get the actual, stored name of the declaration, which may be a special name.
Definition Decl.h:339
A (possibly-)qualified type.
Definition TypeBase.h:937
field_range fields() const
Definition Decl.h:4512
Encodes a location in the source.
bool isUnion() const
Definition Decl.h:3919
virtual bool validateBranchProtection(StringRef Spec, StringRef Arch, BranchProtectionInfo &BPI, const LangOptions &LO, StringRef &Err) const
Determine if this TargetInfo supports the given branch protection specification.
virtual StringRef getABI() const
Get the ABI currently in use.
virtual ParsedTargetAttr parseTargetAttr(StringRef Str) const
virtual bool hasFeature(StringRef Feature) const
Determine whether the given target has the given feature.
The base class of the type hierarchy.
Definition TypeBase.h:1833
bool isVoidType() const
Definition TypeBase.h:8878
bool isMFloat8Type() const
Definition TypeBase.h:8903
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Definition Type.h:41
bool isSVESizelessBuiltinType() const
Returns true for SVE scalable vector types.
Definition Type.cpp:2574
const T * castAs() const
Member-template castAs<specific type>.
Definition TypeBase.h:9165
bool isBuiltinType() const
Helper methods to distinguish type categories.
Definition TypeBase.h:8645
EnumDecl * getAsEnumDecl() const
Retrieves the EnumDecl this type refers to.
Definition Type.h:53
bool isVectorType() const
Definition TypeBase.h:8661
const T * getAsCanonical() const
If this type is canonically the specified type, return its canonical type cast to that specified type...
Definition TypeBase.h:2921
bool isFloatingType() const
Definition Type.cpp:2304
const T * getAs() const
Member-template getAs<specific type>'.
Definition TypeBase.h:9098
QualType getType() const
Definition Decl.h:722
unsigned getNumElements() const
Definition TypeBase.h:4188
VectorKind getVectorKind() const
Definition TypeBase.h:4193
QualType getElementType() const
Definition TypeBase.h:4187
ABIArgInfo classifyArgumentType(CodeGenModule &CGM, CanQualType type)
Classify the rules for how to pass a particular type.
bool isLegalVectorType(CodeGenModule &CGM, CharUnits vectorSize, llvm::VectorType *vectorTy)
Is the given vector type "legal" for Swift's perspective on the current platform?
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
Definition CGValue.h:145
CGCXXABI::RecordArgABI getRecordArgABI(const RecordType *RT, CGCXXABI &CXXABI)
bool classifyReturnType(const CGCXXABI &CXXABI, CGFunctionInfo &FI, const ABIInfo &Info)
Address EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, const ABIArgInfo &AI)
RValue emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType ValueTy, bool IsIndirect, TypeInfoChars ValueInfo, CharUnits SlotSizeAndAlign, bool AllowHigherAlign, AggValueSlot Slot, bool ForceRightAdjust=false)
Emit va_arg for a platform using the common void* representation, where arguments are simply emitted ...
Address emitMergePHI(CodeGenFunction &CGF, Address Addr1, llvm::BasicBlock *Block1, Address Addr2, llvm::BasicBlock *Block2, const llvm::Twine &Name="")
bool isEmptyField(ASTContext &Context, const FieldDecl *FD, bool AllowArrays, bool AsIfNoUniqueAddr=false)
isEmptyField - Return true iff a the field is "empty", that is it is an unnamed bit-field or an (arra...
llvm::Value * emitRoundPointerUpToAlignment(CodeGenFunction &CGF, llvm::Value *Ptr, CharUnits Align)
bool isAggregateTypeForABI(QualType T)
std::unique_ptr< TargetCodeGenInfo > createAArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind Kind)
Definition AArch64.cpp:1345
QualType useFirstFieldIfTransparentUnion(QualType Ty)
Pass transparent unions as if they were the type of the first element.
std::unique_ptr< TargetCodeGenInfo > createWindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind K)
Definition AArch64.cpp:1351
bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays, bool AsIfNoUniqueAddr=false)
isEmptyRecord - Return true iff a structure contains only empty fields.
The JSON file list parser is used to communicate input to InstallAPI.
bool isa(CodeGen::Address addr)
Definition Address.h:330
@ CPlusPlus
@ Self
'self' clause, allowed on Compute and Combined Constructs, plus 'update'.
const FunctionProtoType * T
@ Type
The name was classified as a type.
Definition Sema.h:562
U cast(CodeGen::Address addr)
Definition Address.h:327
bool IsArmStreamingFunction(const FunctionDecl *FD, bool IncludeLocallyStreaming)
Returns whether the given FunctionDecl has an __arm[_locally]_streaming attribute.
Definition Decl.cpp:5971
unsigned long uint64_t
#define true
Definition stdbool.h:25
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64