clang 17.0.0git
TargetInfo.cpp
Go to the documentation of this file.
1//===---- TargetInfo.cpp - Encapsulate target details -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// These classes wrap the information about a call or function
10// definition used to handle ABI compliancy.
11//
12//===----------------------------------------------------------------------===//
13
14#include "TargetInfo.h"
15#include "ABIInfo.h"
16#include "CGBlocks.h"
17#include "CGCXXABI.h"
18#include "CGValue.h"
19#include "CodeGenFunction.h"
20#include "clang/AST/Attr.h"
26#include "llvm/ADT/SmallBitVector.h"
27#include "llvm/ADT/StringExtras.h"
28#include "llvm/ADT/StringSwitch.h"
29#include "llvm/ADT/Twine.h"
30#include "llvm/IR/DataLayout.h"
31#include "llvm/IR/IntrinsicsNVPTX.h"
32#include "llvm/IR/IntrinsicsS390.h"
33#include "llvm/IR/Type.h"
34#include "llvm/Support/MathExtras.h"
35#include "llvm/Support/raw_ostream.h"
36#include "llvm/TargetParser/Triple.h"
37#include <algorithm>
38
39using namespace clang;
40using namespace CodeGen;
41
42// Helper for coercing an aggregate argument or return value into an integer
43// array of the same size (including padding) and alignment. This alternate
44// coercion happens only for the RenderScript ABI and can be removed after
45// runtimes that rely on it are no longer supported.
46//
47// RenderScript assumes that the size of the argument / return value in the IR
48// is the same as the size of the corresponding qualified type. This helper
49// coerces the aggregate type into an array of the same size (including
50// padding). This coercion is used in lieu of expansion of struct members or
51// other canonical coercions that return a coerced-type of larger size.
52//
53// Ty - The argument / return value type
54// Context - The associated ASTContext
55// LLVMContext - The associated LLVMContext
57 ASTContext &Context,
58 llvm::LLVMContext &LLVMContext) {
59 // Alignment and Size are measured in bits.
60 const uint64_t Size = Context.getTypeSize(Ty);
61 const uint64_t Alignment = Context.getTypeAlign(Ty);
62 llvm::Type *IntType = llvm::Type::getIntNTy(LLVMContext, Alignment);
63 const uint64_t NumElements = (Size + Alignment - 1) / Alignment;
64 return ABIArgInfo::getDirect(llvm::ArrayType::get(IntType, NumElements));
65}
66
68 llvm::Value *Array,
69 llvm::Value *Value,
70 unsigned FirstIndex,
71 unsigned LastIndex) {
72 // Alternatively, we could emit this as a loop in the source.
73 for (unsigned I = FirstIndex; I <= LastIndex; ++I) {
74 llvm::Value *Cell =
75 Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), Array, I);
76 Builder.CreateAlignedStore(Value, Cell, CharUnits::One());
77 }
78}
79
81 return !CodeGenFunction::hasScalarEvaluationKind(T) ||
83}
84
86 bool Realign,
87 llvm::Type *Padding) const {
88 return ABIArgInfo::getIndirect(getContext().getTypeAlignInChars(Ty), ByVal,
89 Realign, Padding);
90}
91
94 return ABIArgInfo::getIndirectInReg(getContext().getTypeAlignInChars(Ty),
95 /*ByVal*/ false, Realign);
96}
97
99 QualType Ty) const {
100 return Address::invalid();
101}
102
103static llvm::Type *getVAListElementType(CodeGenFunction &CGF) {
104 return CGF.ConvertTypeForMem(
106}
107
109 if (getContext().isPromotableIntegerType(Ty))
110 return true;
111
112 if (const auto *EIT = Ty->getAs<BitIntType>())
113 if (EIT->getNumBits() < getContext().getTypeSize(getContext().IntTy))
114 return true;
115
116 return false;
117}
118
119ABIInfo::~ABIInfo() = default;
120
122
123/// Does the given lowering require more than the given number of
124/// registers when expanded?
125///
126/// This is intended to be the basis of a reasonable basic implementation
127/// of should{Pass,Return}IndirectlyForSwift.
128///
129/// For most targets, a limit of four total registers is reasonable; this
130/// limits the amount of code required in order to move around the value
131/// in case it wasn't produced immediately prior to the call by the caller
132/// (or wasn't produced in exactly the right registers) or isn't used
133/// immediately within the callee. But some targets may need to further
134/// limit the register count due to an inability to support that many
135/// return registers.
137 ArrayRef<llvm::Type*> scalarTypes,
138 unsigned maxAllRegisters) {
139 unsigned intCount = 0, fpCount = 0;
140 for (llvm::Type *type : scalarTypes) {
141 if (type->isPointerTy()) {
142 intCount++;
143 } else if (auto intTy = dyn_cast<llvm::IntegerType>(type)) {
144 auto ptrWidth = cgt.getTarget().getPointerWidth(LangAS::Default);
145 intCount += (intTy->getBitWidth() + ptrWidth - 1) / ptrWidth;
146 } else {
147 assert(type->isVectorTy() || type->isFloatingPointTy());
148 fpCount++;
149 }
150 }
151
152 return (intCount + fpCount > maxAllRegisters);
153}
154
156 bool AsReturnValue) const {
157 return occupiesMoreThan(CGT, ComponentTys, /*total=*/4);
158}
159
160bool SwiftABIInfo::isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,
161 unsigned NumElts) const {
162 // The default implementation of this assumes that the target guarantees
163 // 128-bit SIMD support but nothing more.
164 return (VectorSize.getQuantity() > 8 && VectorSize.getQuantity() <= 16);
165}
166
168 CGCXXABI &CXXABI) {
169 const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
170 if (!RD) {
171 if (!RT->getDecl()->canPassInRegisters())
174 }
175 return CXXABI.getRecordArgABI(RD);
176}
177
179 CGCXXABI &CXXABI) {
180 const RecordType *RT = T->getAs<RecordType>();
181 if (!RT)
183 return getRecordArgABI(RT, CXXABI);
184}
185
187 const ABIInfo &Info) {
188 QualType Ty = FI.getReturnType();
189
190 if (const auto *RT = Ty->getAs<RecordType>())
191 if (!isa<CXXRecordDecl>(RT->getDecl()) &&
192 !RT->getDecl()->canPassInRegisters()) {
194 return true;
195 }
196
197 return CXXABI.classifyReturnType(FI);
198}
199
200/// Pass transparent unions as if they were the type of the first element. Sema
201/// should ensure that all elements of the union have the same "machine type".
203 if (const RecordType *UT = Ty->getAsUnionType()) {
204 const RecordDecl *UD = UT->getDecl();
205 if (UD->hasAttr<TransparentUnionAttr>()) {
206 assert(!UD->field_empty() && "sema created an empty transparent union");
207 return UD->field_begin()->getType();
208 }
209 }
210 return Ty;
211}
212
214 return CGT.getCXXABI();
215}
216
218 return CGT.getContext();
219}
220
221llvm::LLVMContext &ABIInfo::getVMContext() const {
222 return CGT.getLLVMContext();
223}
224
225const llvm::DataLayout &ABIInfo::getDataLayout() const {
226 return CGT.getDataLayout();
227}
228
230 return CGT.getTarget();
231}
232
234 return CGT.getCodeGenOpts();
235}
236
237bool ABIInfo::isAndroid() const { return getTarget().getTriple().isAndroid(); }
238
240 return getTarget().getTriple().isOHOSFamily();
241}
242
244 return false;
245}
246
248 uint64_t Members) const {
249 return false;
250}
251
253 // For compatibility with GCC, ignore empty bitfields in C++ mode.
254 return getContext().getLangOpts().CPlusPlus;
255}
256
257LLVM_DUMP_METHOD void ABIArgInfo::dump() const {
258 raw_ostream &OS = llvm::errs();
259 OS << "(ABIArgInfo Kind=";
260 switch (TheKind) {
261 case Direct:
262 OS << "Direct Type=";
263 if (llvm::Type *Ty = getCoerceToType())
264 Ty->print(OS);
265 else
266 OS << "null";
267 break;
268 case Extend:
269 OS << "Extend";
270 break;
271 case Ignore:
272 OS << "Ignore";
273 break;
274 case InAlloca:
275 OS << "InAlloca Offset=" << getInAllocaFieldIndex();
276 break;
277 case Indirect:
278 OS << "Indirect Align=" << getIndirectAlign().getQuantity()
279 << " ByVal=" << getIndirectByVal()
280 << " Realign=" << getIndirectRealign();
281 break;
282 case IndirectAliased:
283 OS << "Indirect Align=" << getIndirectAlign().getQuantity()
284 << " AadrSpace=" << getIndirectAddrSpace()
285 << " Realign=" << getIndirectRealign();
286 break;
287 case Expand:
288 OS << "Expand";
289 break;
290 case CoerceAndExpand:
291 OS << "CoerceAndExpand Type=";
292 getCoerceAndExpandType()->print(OS);
293 break;
294 }
295 OS << ")\n";
296}
297
298// Dynamically round a pointer up to a multiple of the given alignment.
300 llvm::Value *Ptr,
301 CharUnits Align) {
302 // OverflowArgArea = (OverflowArgArea + Align - 1) & -Align;
303 llvm::Value *RoundUp = CGF.Builder.CreateConstInBoundsGEP1_32(
304 CGF.Builder.getInt8Ty(), Ptr, Align.getQuantity() - 1);
305 return CGF.Builder.CreateIntrinsic(
306 llvm::Intrinsic::ptrmask, {CGF.AllocaInt8PtrTy, CGF.IntPtrTy},
307 {RoundUp, llvm::ConstantInt::get(CGF.IntPtrTy, -Align.getQuantity())},
308 nullptr, Ptr->getName() + ".aligned");
309}
310
311/// Emit va_arg for a platform using the common void* representation,
312/// where arguments are simply emitted in an array of slots on the stack.
313///
314/// This version implements the core direct-value passing rules.
315///
316/// \param SlotSize - The size and alignment of a stack slot.
317/// Each argument will be allocated to a multiple of this number of
318/// slots, and all the slots will be aligned to this value.
319/// \param AllowHigherAlign - The slot alignment is not a cap;
320/// an argument type with an alignment greater than the slot size
321/// will be emitted on a higher-alignment address, potentially
322/// leaving one or more empty slots behind as padding. If this
323/// is false, the returned address might be less-aligned than
324/// DirectAlign.
325/// \param ForceRightAdjust - Default is false. On big-endian platform and
326/// if the argument is smaller than a slot, set this flag will force
327/// right-adjust the argument in its slot irrespective of the type.
329 Address VAListAddr,
330 llvm::Type *DirectTy,
331 CharUnits DirectSize,
332 CharUnits DirectAlign,
333 CharUnits SlotSize,
334 bool AllowHigherAlign,
335 bool ForceRightAdjust = false) {
336 // Cast the element type to i8* if necessary. Some platforms define
337 // va_list as a struct containing an i8* instead of just an i8*.
338 if (VAListAddr.getElementType() != CGF.Int8PtrTy)
339 VAListAddr = CGF.Builder.CreateElementBitCast(VAListAddr, CGF.Int8PtrTy);
340
341 llvm::Value *Ptr = CGF.Builder.CreateLoad(VAListAddr, "argp.cur");
342
343 // If the CC aligns values higher than the slot size, do so if needed.
344 Address Addr = Address::invalid();
345 if (AllowHigherAlign && DirectAlign > SlotSize) {
346 Addr = Address(emitRoundPointerUpToAlignment(CGF, Ptr, DirectAlign),
347 CGF.Int8Ty, DirectAlign);
348 } else {
349 Addr = Address(Ptr, CGF.Int8Ty, SlotSize);
350 }
351
352 // Advance the pointer past the argument, then store that back.
353 CharUnits FullDirectSize = DirectSize.alignTo(SlotSize);
354 Address NextPtr =
355 CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next");
356 CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
357
358 // If the argument is smaller than a slot, and this is a big-endian
359 // target, the argument will be right-adjusted in its slot.
360 if (DirectSize < SlotSize && CGF.CGM.getDataLayout().isBigEndian() &&
361 (!DirectTy->isStructTy() || ForceRightAdjust)) {
362 Addr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, SlotSize - DirectSize);
363 }
364
365 Addr = CGF.Builder.CreateElementBitCast(Addr, DirectTy);
366 return Addr;
367}
368
369/// Emit va_arg for a platform using the common void* representation,
370/// where arguments are simply emitted in an array of slots on the stack.
371///
372/// \param IsIndirect - Values of this type are passed indirectly.
373/// \param ValueInfo - The size and alignment of this type, generally
374/// computed with getContext().getTypeInfoInChars(ValueTy).
375/// \param SlotSizeAndAlign - The size and alignment of a stack slot.
376/// Each argument will be allocated to a multiple of this number of
377/// slots, and all the slots will be aligned to this value.
378/// \param AllowHigherAlign - The slot alignment is not a cap;
379/// an argument type with an alignment greater than the slot size
380/// will be emitted on a higher-alignment address, potentially
381/// leaving one or more empty slots behind as padding.
382/// \param ForceRightAdjust - Default is false. On big-endian platform and
383/// if the argument is smaller than a slot, set this flag will force
384/// right-adjust the argument in its slot irrespective of the type.
386 QualType ValueTy, bool IsIndirect,
387 TypeInfoChars ValueInfo,
388 CharUnits SlotSizeAndAlign,
389 bool AllowHigherAlign,
390 bool ForceRightAdjust = false) {
391 // The size and alignment of the value that was passed directly.
392 CharUnits DirectSize, DirectAlign;
393 if (IsIndirect) {
394 DirectSize = CGF.getPointerSize();
395 DirectAlign = CGF.getPointerAlign();
396 } else {
397 DirectSize = ValueInfo.Width;
398 DirectAlign = ValueInfo.Align;
399 }
400
401 // Cast the address we've calculated to the right type.
402 llvm::Type *DirectTy = CGF.ConvertTypeForMem(ValueTy), *ElementTy = DirectTy;
403 if (IsIndirect)
404 DirectTy = DirectTy->getPointerTo(0);
405
406 Address Addr = emitVoidPtrDirectVAArg(CGF, VAListAddr, DirectTy, DirectSize,
407 DirectAlign, SlotSizeAndAlign,
408 AllowHigherAlign, ForceRightAdjust);
409
410 if (IsIndirect) {
411 Addr = Address(CGF.Builder.CreateLoad(Addr), ElementTy, ValueInfo.Align);
412 }
413
414 return Addr;
415}
416
418 QualType Ty, CharUnits SlotSize,
419 CharUnits EltSize, const ComplexType *CTy) {
420 Address Addr =
421 emitVoidPtrDirectVAArg(CGF, VAListAddr, CGF.Int8Ty, SlotSize * 2,
422 SlotSize, SlotSize, /*AllowHigher*/ true);
423
424 Address RealAddr = Addr;
425 Address ImagAddr = RealAddr;
426 if (CGF.CGM.getDataLayout().isBigEndian()) {
427 RealAddr =
428 CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize - EltSize);
429 ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(ImagAddr,
430 2 * SlotSize - EltSize);
431 } else {
432 ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize);
433 }
434
435 llvm::Type *EltTy = CGF.ConvertTypeForMem(CTy->getElementType());
436 RealAddr = CGF.Builder.CreateElementBitCast(RealAddr, EltTy);
437 ImagAddr = CGF.Builder.CreateElementBitCast(ImagAddr, EltTy);
438 llvm::Value *Real = CGF.Builder.CreateLoad(RealAddr, ".vareal");
439 llvm::Value *Imag = CGF.Builder.CreateLoad(ImagAddr, ".vaimag");
440
441 Address Temp = CGF.CreateMemTemp(Ty, "vacplx");
442 CGF.EmitStoreOfComplex({Real, Imag}, CGF.MakeAddrLValue(Temp, Ty),
443 /*init*/ true);
444 return Temp;
445}
446
448 Address Addr1, llvm::BasicBlock *Block1,
449 Address Addr2, llvm::BasicBlock *Block2,
450 const llvm::Twine &Name = "") {
451 assert(Addr1.getType() == Addr2.getType());
452 llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name);
453 PHI->addIncoming(Addr1.getPointer(), Block1);
454 PHI->addIncoming(Addr2.getPointer(), Block2);
455 CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment());
456 return Address(PHI, Addr1.getElementType(), Align);
457}
458
459TargetCodeGenInfo::TargetCodeGenInfo(std::unique_ptr<ABIInfo> Info)
460 : Info(std::move(Info)) {}
461
463
464// If someone can figure out a general rule for this, that would be great.
465// It's probably just doomed to be platform-dependent, though.
467 // Verified for:
468 // x86-64 FreeBSD, Linux, Darwin
469 // x86-32 FreeBSD, Linux, Darwin
470 // PowerPC Linux, Darwin
471 // ARM Darwin (*not* EABI)
472 // AArch64 Linux
473 return 32;
474}
475
477 const FunctionNoProtoType *fnType) const {
478 // The following conventions are known to require this to be false:
479 // x86_stdcall
480 // MIPS
481 // For everything else, we just prefer false unless we opt out.
482 return false;
483}
484
485void
487 llvm::SmallString<24> &Opt) const {
488 // This assumes the user is passing a library name like "rt" instead of a
489 // filename like "librt.a/so", and that they don't care whether it's static or
490 // dynamic.
491 Opt = "-l";
492 Opt += Lib;
493}
494
496 // OpenCL kernels are called via an explicit runtime API with arguments
497 // set with clSetKernelArg(), not as normal sub-functions.
498 // Return SPIR_KERNEL by default as the kernel calling convention to
499 // ensure the fingerprint is fixed such way that each OpenCL argument
500 // gets one matching argument in the produced kernel function argument
501 // list to enable feasible implementation of clSetKernelArg() with
502 // aggregates etc. In case we would use the default C calling conv here,
503 // clSetKernelArg() might break depending on the target-specific
504 // conventions; different targets might split structs passed as values
505 // to multiple function arguments etc.
506 return llvm::CallingConv::SPIR_KERNEL;
507}
508
510 llvm::PointerType *T, QualType QT) const {
511 return llvm::ConstantPointerNull::get(T);
512}
513
515 const VarDecl *D) const {
516 assert(!CGM.getLangOpts().OpenCL &&
517 !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
518 "Address space agnostic languages only");
519 return D ? D->getType().getAddressSpace() : LangAS::Default;
520}
521
523 CodeGen::CodeGenFunction &CGF, llvm::Value *Src, LangAS SrcAddr,
524 LangAS DestAddr, llvm::Type *DestTy, bool isNonNull) const {
525 // Since target may map different address spaces in AST to the same address
526 // space, an address space conversion may end up as a bitcast.
527 if (auto *C = dyn_cast<llvm::Constant>(Src))
528 return performAddrSpaceCast(CGF.CGM, C, SrcAddr, DestAddr, DestTy);
529 // Try to preserve the source's name to make IR more readable.
531 Src, DestTy, Src->hasName() ? Src->getName() + ".ascast" : "");
532}
533
534llvm::Constant *
536 LangAS SrcAddr, LangAS DestAddr,
537 llvm::Type *DestTy) const {
538 // Since target may map different address spaces in AST to the same address
539 // space, an address space conversion may end up as a bitcast.
540 return llvm::ConstantExpr::getPointerCast(Src, DestTy);
541}
542
543llvm::SyncScope::ID
546 llvm::AtomicOrdering Ordering,
547 llvm::LLVMContext &Ctx) const {
548 return Ctx.getOrInsertSyncScopeID(""); /* default sync scope */
549}
550
551static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays);
552
553/// isEmptyField - Return true iff a the field is "empty", that is it
554/// is an unnamed bit-field or an (array of) empty record(s).
555static bool isEmptyField(ASTContext &Context, const FieldDecl *FD,
556 bool AllowArrays) {
557 if (FD->isUnnamedBitfield())
558 return true;
559
560 QualType FT = FD->getType();
561
562 // Constant arrays of empty records count as empty, strip them off.
563 // Constant arrays of zero length always count as empty.
564 bool WasArray = false;
565 if (AllowArrays)
566 while (const ConstantArrayType *AT = Context.getAsConstantArrayType(FT)) {
567 if (AT->getSize() == 0)
568 return true;
569 FT = AT->getElementType();
570 // The [[no_unique_address]] special case below does not apply to
571 // arrays of C++ empty records, so we need to remember this fact.
572 WasArray = true;
573 }
574
575 const RecordType *RT = FT->getAs<RecordType>();
576 if (!RT)
577 return false;
578
579 // C++ record fields are never empty, at least in the Itanium ABI.
580 //
581 // FIXME: We should use a predicate for whether this behavior is true in the
582 // current ABI.
583 //
584 // The exception to the above rule are fields marked with the
585 // [[no_unique_address]] attribute (since C++20). Those do count as empty
586 // according to the Itanium ABI. The exception applies only to records,
587 // not arrays of records, so we must also check whether we stripped off an
588 // array type above.
589 if (isa<CXXRecordDecl>(RT->getDecl()) &&
590 (WasArray || !FD->hasAttr<NoUniqueAddressAttr>()))
591 return false;
592
593 return isEmptyRecord(Context, FT, AllowArrays);
594}
595
596/// isEmptyRecord - Return true iff a structure contains only empty
597/// fields. Note that a structure with a flexible array member is not
598/// considered empty.
599static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays) {
600 const RecordType *RT = T->getAs<RecordType>();
601 if (!RT)
602 return false;
603 const RecordDecl *RD = RT->getDecl();
604 if (RD->hasFlexibleArrayMember())
605 return false;
606
607 // If this is a C++ record, check the bases first.
608 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
609 for (const auto &I : CXXRD->bases())
610 if (!isEmptyRecord(Context, I.getType(), true))
611 return false;
612
613 for (const auto *I : RD->fields())
614 if (!isEmptyField(Context, I, AllowArrays))
615 return false;
616 return true;
617}
618
619/// isSingleElementStruct - Determine if a structure is a "single
620/// element struct", i.e. it has exactly one non-empty field or
621/// exactly one field which is itself a single element
622/// struct. Structures with flexible array members are never
623/// considered single element structs.
624///
625/// \return The field declaration for the single non-empty field, if
626/// it exists.
627static const Type *isSingleElementStruct(QualType T, ASTContext &Context) {
628 const RecordType *RT = T->getAs<RecordType>();
629 if (!RT)
630 return nullptr;
631
632 const RecordDecl *RD = RT->getDecl();
633 if (RD->hasFlexibleArrayMember())
634 return nullptr;
635
636 const Type *Found = nullptr;
637
638 // If this is a C++ record, check the bases first.
639 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
640 for (const auto &I : CXXRD->bases()) {
641 // Ignore empty records.
642 if (isEmptyRecord(Context, I.getType(), true))
643 continue;
644
645 // If we already found an element then this isn't a single-element struct.
646 if (Found)
647 return nullptr;
648
649 // If this is non-empty and not a single element struct, the composite
650 // cannot be a single element struct.
651 Found = isSingleElementStruct(I.getType(), Context);
652 if (!Found)
653 return nullptr;
654 }
655 }
656
657 // Check for single element.
658 for (const auto *FD : RD->fields()) {
659 QualType FT = FD->getType();
660
661 // Ignore empty fields.
662 if (isEmptyField(Context, FD, true))
663 continue;
664
665 // If we already found an element then this isn't a single-element
666 // struct.
667 if (Found)
668 return nullptr;
669
670 // Treat single element arrays as the element.
671 while (const ConstantArrayType *AT = Context.getAsConstantArrayType(FT)) {
672 if (AT->getSize().getZExtValue() != 1)
673 break;
674 FT = AT->getElementType();
675 }
676
677 if (!isAggregateTypeForABI(FT)) {
678 Found = FT.getTypePtr();
679 } else {
680 Found = isSingleElementStruct(FT, Context);
681 if (!Found)
682 return nullptr;
683 }
684 }
685
686 // We don't consider a struct a single-element struct if it has
687 // padding beyond the element type.
688 if (Found && Context.getTypeSize(Found) != Context.getTypeSize(T))
689 return nullptr;
690
691 return Found;
692}
693
694namespace {
695Address EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
696 const ABIArgInfo &AI) {
697 // This default implementation defers to the llvm backend's va_arg
698 // instruction. It can handle only passing arguments directly
699 // (typically only handled in the backend for primitive types), or
700 // aggregates passed indirectly by pointer (NOTE: if the "byval"
701 // flag has ABI impact in the callee, this implementation cannot
702 // work.)
703
704 // Only a few cases are covered here at the moment -- those needed
705 // by the default abi.
706 llvm::Value *Val;
707
708 if (AI.isIndirect()) {
709 assert(!AI.getPaddingType() &&
710 "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
711 assert(
712 !AI.getIndirectRealign() &&
713 "Unexpected IndirectRealign seen in arginfo in generic VAArg emitter!");
714
715 auto TyInfo = CGF.getContext().getTypeInfoInChars(Ty);
716 CharUnits TyAlignForABI = TyInfo.Align;
717
718 llvm::Type *ElementTy = CGF.ConvertTypeForMem(Ty);
719 llvm::Type *BaseTy = llvm::PointerType::getUnqual(ElementTy);
720 llvm::Value *Addr =
721 CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy);
722 return Address(Addr, ElementTy, TyAlignForABI);
723 } else {
724 assert((AI.isDirect() || AI.isExtend()) &&
725 "Unexpected ArgInfo Kind in generic VAArg emitter!");
726
727 assert(!AI.getInReg() &&
728 "Unexpected InReg seen in arginfo in generic VAArg emitter!");
729 assert(!AI.getPaddingType() &&
730 "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
731 assert(!AI.getDirectOffset() &&
732 "Unexpected DirectOffset seen in arginfo in generic VAArg emitter!");
733 assert(!AI.getCoerceToType() &&
734 "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!");
735
736 Address Temp = CGF.CreateMemTemp(Ty, "varet");
737 Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(),
738 CGF.ConvertTypeForMem(Ty));
739 CGF.Builder.CreateStore(Val, Temp);
740 return Temp;
741 }
742}
743
744/// DefaultABIInfo - The default implementation for ABI specific
745/// details. This implementation provides information which results in
746/// self-consistent and sensible LLVM IR generation, but does not
747/// conform to any particular ABI.
748class DefaultABIInfo : public ABIInfo {
749public:
750 DefaultABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
751
754
755 void computeInfo(CGFunctionInfo &FI) const override {
756 if (!getCXXABI().classifyReturnType(FI))
758 for (auto &I : FI.arguments())
759 I.info = classifyArgumentType(I.type);
760 }
761
762 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
763 QualType Ty) const override {
764 return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty));
765 }
766};
767
768class DefaultTargetCodeGenInfo : public TargetCodeGenInfo {
769public:
770 DefaultTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
771 : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {}
772};
773
774ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
776
777 if (isAggregateTypeForABI(Ty)) {
778 // Records with non-trivial destructors/copy-constructors should not be
779 // passed by value.
780 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
781 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
782
783 return getNaturalAlignIndirect(Ty);
784 }
785
786 // Treat an enum type as its underlying type.
787 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
788 Ty = EnumTy->getDecl()->getIntegerType();
789
790 ASTContext &Context = getContext();
791 if (const auto *EIT = Ty->getAs<BitIntType>())
792 if (EIT->getNumBits() >
793 Context.getTypeSize(Context.getTargetInfo().hasInt128Type()
794 ? Context.Int128Ty
795 : Context.LongLongTy))
796 return getNaturalAlignIndirect(Ty);
797
798 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
800}
801
802ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
803 if (RetTy->isVoidType())
804 return ABIArgInfo::getIgnore();
805
806 if (isAggregateTypeForABI(RetTy))
807 return getNaturalAlignIndirect(RetTy);
808
809 // Treat an enum type as its underlying type.
810 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
811 RetTy = EnumTy->getDecl()->getIntegerType();
812
813 if (const auto *EIT = RetTy->getAs<BitIntType>())
814 if (EIT->getNumBits() >
815 getContext().getTypeSize(getContext().getTargetInfo().hasInt128Type()
816 ? getContext().Int128Ty
817 : getContext().LongLongTy))
818 return getNaturalAlignIndirect(RetTy);
819
820 return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
822}
823
824//===----------------------------------------------------------------------===//
825// WebAssembly ABI Implementation
826//
827// This is a very simple ABI that relies a lot on DefaultABIInfo.
828//===----------------------------------------------------------------------===//
829
830class WebAssemblyABIInfo final : public ABIInfo {
831public:
832 enum ABIKind {
833 MVP = 0,
834 ExperimentalMV = 1,
835 };
836
837private:
838 DefaultABIInfo defaultInfo;
839 ABIKind Kind;
840
841public:
842 explicit WebAssemblyABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind)
843 : ABIInfo(CGT), defaultInfo(CGT), Kind(Kind) {}
844
845private:
848
849 // DefaultABIInfo's classifyReturnType and classifyArgumentType are
850 // non-virtual, but computeInfo and EmitVAArg are virtual, so we
851 // overload them.
852 void computeInfo(CGFunctionInfo &FI) const override {
853 if (!getCXXABI().classifyReturnType(FI))
855 for (auto &Arg : FI.arguments())
856 Arg.info = classifyArgumentType(Arg.type);
857 }
858
859 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
860 QualType Ty) const override;
861};
862
863class WebAssemblyTargetCodeGenInfo final : public TargetCodeGenInfo {
864public:
865 explicit WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
866 WebAssemblyABIInfo::ABIKind K)
867 : TargetCodeGenInfo(std::make_unique<WebAssemblyABIInfo>(CGT, K)) {
868 SwiftInfo =
869 std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/false);
870 }
871
872 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
873 CodeGen::CodeGenModule &CGM) const override {
875 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
876 if (const auto *Attr = FD->getAttr<WebAssemblyImportModuleAttr>()) {
877 llvm::Function *Fn = cast<llvm::Function>(GV);
878 llvm::AttrBuilder B(GV->getContext());
879 B.addAttribute("wasm-import-module", Attr->getImportModule());
880 Fn->addFnAttrs(B);
881 }
882 if (const auto *Attr = FD->getAttr<WebAssemblyImportNameAttr>()) {
883 llvm::Function *Fn = cast<llvm::Function>(GV);
884 llvm::AttrBuilder B(GV->getContext());
885 B.addAttribute("wasm-import-name", Attr->getImportName());
886 Fn->addFnAttrs(B);
887 }
888 if (const auto *Attr = FD->getAttr<WebAssemblyExportNameAttr>()) {
889 llvm::Function *Fn = cast<llvm::Function>(GV);
890 llvm::AttrBuilder B(GV->getContext());
891 B.addAttribute("wasm-export-name", Attr->getExportName());
892 Fn->addFnAttrs(B);
893 }
894 }
895
896 if (auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
897 llvm::Function *Fn = cast<llvm::Function>(GV);
898 if (!FD->doesThisDeclarationHaveABody() && !FD->hasPrototype())
899 Fn->addFnAttr("no-prototype");
900 }
901 }
902
903 /// Return the WebAssembly externref reference type.
904 virtual llvm::Type *getWasmExternrefReferenceType() const override {
905 return llvm::Type::getWasm_ExternrefTy(getABIInfo().getVMContext());
906 }
907 /// Return the WebAssembly funcref reference type.
908 virtual llvm::Type *getWasmFuncrefReferenceType() const override {
909 return llvm::Type::getWasm_FuncrefTy(getABIInfo().getVMContext());
910 }
911};
912
913/// Classify argument of given type \p Ty.
914ABIArgInfo WebAssemblyABIInfo::classifyArgumentType(QualType Ty) const {
916
917 if (isAggregateTypeForABI(Ty)) {
918 // Records with non-trivial destructors/copy-constructors should not be
919 // passed by value.
920 if (auto RAA = getRecordArgABI(Ty, getCXXABI()))
921 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
922 // Ignore empty structs/unions.
923 if (isEmptyRecord(getContext(), Ty, true))
924 return ABIArgInfo::getIgnore();
925 // Lower single-element structs to just pass a regular value. TODO: We
926 // could do reasonable-size multiple-element structs too, using getExpand(),
927 // though watch out for things like bitfields.
928 if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
929 return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
930 // For the experimental multivalue ABI, fully expand all other aggregates
931 if (Kind == ABIKind::ExperimentalMV) {
932 const RecordType *RT = Ty->getAs<RecordType>();
933 assert(RT);
934 bool HasBitField = false;
935 for (auto *Field : RT->getDecl()->fields()) {
936 if (Field->isBitField()) {
937 HasBitField = true;
938 break;
939 }
940 }
941 if (!HasBitField)
942 return ABIArgInfo::getExpand();
943 }
944 }
945
946 // Otherwise just do the default thing.
947 return defaultInfo.classifyArgumentType(Ty);
948}
949
950ABIArgInfo WebAssemblyABIInfo::classifyReturnType(QualType RetTy) const {
951 if (isAggregateTypeForABI(RetTy)) {
952 // Records with non-trivial destructors/copy-constructors should not be
953 // returned by value.
954 if (!getRecordArgABI(RetTy, getCXXABI())) {
955 // Ignore empty structs/unions.
956 if (isEmptyRecord(getContext(), RetTy, true))
957 return ABIArgInfo::getIgnore();
958 // Lower single-element structs to just return a regular value. TODO: We
959 // could do reasonable-size multiple-element structs too, using
960 // ABIArgInfo::getDirect().
961 if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
962 return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
963 // For the experimental multivalue ABI, return all other aggregates
964 if (Kind == ABIKind::ExperimentalMV)
965 return ABIArgInfo::getDirect();
966 }
967 }
968
969 // Otherwise just do the default thing.
970 return defaultInfo.classifyReturnType(RetTy);
971}
972
973Address WebAssemblyABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
974 QualType Ty) const {
975 bool IsIndirect = isAggregateTypeForABI(Ty) &&
976 !isEmptyRecord(getContext(), Ty, true) &&
977 !isSingleElementStruct(Ty, getContext());
978 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
979 getContext().getTypeInfoInChars(Ty),
981 /*AllowHigherAlign=*/true);
982}
983
984//===----------------------------------------------------------------------===//
985// le32/PNaCl bitcode ABI Implementation
986//
987// This is a simplified version of the x86_32 ABI. Arguments and return values
988// are always passed on the stack.
989//===----------------------------------------------------------------------===//
990
991class PNaClABIInfo : public ABIInfo {
992 public:
993 PNaClABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
994
997
998 void computeInfo(CGFunctionInfo &FI) const override;
999 Address EmitVAArg(CodeGenFunction &CGF,
1000 Address VAListAddr, QualType Ty) const override;
1001};
1002
1003class PNaClTargetCodeGenInfo : public TargetCodeGenInfo {
1004 public:
1005 PNaClTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
1006 : TargetCodeGenInfo(std::make_unique<PNaClABIInfo>(CGT)) {}
1007};
1008
1009void PNaClABIInfo::computeInfo(CGFunctionInfo &FI) const {
1010 if (!getCXXABI().classifyReturnType(FI))
1012
1013 for (auto &I : FI.arguments())
1014 I.info = classifyArgumentType(I.type);
1015}
1016
1017Address PNaClABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1018 QualType Ty) const {
1019 // The PNaCL ABI is a bit odd, in that varargs don't use normal
1020 // function classification. Structs get passed directly for varargs
1021 // functions, through a rewriting transform in
1022 // pnacl-llvm/lib/Transforms/NaCl/ExpandVarArgs.cpp, which allows
1023 // this target to actually support a va_arg instructions with an
1024 // aggregate type, unlike other targets.
1025 return EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect());
1026}
1027
1028/// Classify argument of given type \p Ty.
1029ABIArgInfo PNaClABIInfo::classifyArgumentType(QualType Ty) const {
1030 if (isAggregateTypeForABI(Ty)) {
1031 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
1032 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
1033 return getNaturalAlignIndirect(Ty);
1034 } else if (const EnumType *EnumTy = Ty->getAs<EnumType>()) {
1035 // Treat an enum type as its underlying type.
1036 Ty = EnumTy->getDecl()->getIntegerType();
1037 } else if (Ty->isFloatingType()) {
1038 // Floating-point types don't go inreg.
1039 return ABIArgInfo::getDirect();
1040 } else if (const auto *EIT = Ty->getAs<BitIntType>()) {
1041 // Treat bit-precise integers as integers if <= 64, otherwise pass
1042 // indirectly.
1043 if (EIT->getNumBits() > 64)
1044 return getNaturalAlignIndirect(Ty);
1045 return ABIArgInfo::getDirect();
1046 }
1047
1048 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
1050}
1051
1052ABIArgInfo PNaClABIInfo::classifyReturnType(QualType RetTy) const {
1053 if (RetTy->isVoidType())
1054 return ABIArgInfo::getIgnore();
1055
1056 // In the PNaCl ABI we always return records/structures on the stack.
1057 if (isAggregateTypeForABI(RetTy))
1058 return getNaturalAlignIndirect(RetTy);
1059
1060 // Treat bit-precise integers as integers if <= 64, otherwise pass indirectly.
1061 if (const auto *EIT = RetTy->getAs<BitIntType>()) {
1062 if (EIT->getNumBits() > 64)
1063 return getNaturalAlignIndirect(RetTy);
1064 return ABIArgInfo::getDirect();
1065 }
1066
1067 // Treat an enum type as its underlying type.
1068 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1069 RetTy = EnumTy->getDecl()->getIntegerType();
1070
1071 return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
1073}
1074
1075/// IsX86_MMXType - Return true if this is an MMX type.
1076bool IsX86_MMXType(llvm::Type *IRType) {
1077 // Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.
1078 return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
1079 cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy() &&
1080 IRType->getScalarSizeInBits() != 64;
1081}
1082
1083static llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1084 StringRef Constraint,
1085 llvm::Type* Ty) {
1086 bool IsMMXCons = llvm::StringSwitch<bool>(Constraint)
1087 .Cases("y", "&y", "^Ym", true)
1088 .Default(false);
1089 if (IsMMXCons && Ty->isVectorTy()) {
1090 if (cast<llvm::VectorType>(Ty)->getPrimitiveSizeInBits().getFixedValue() !=
1091 64) {
1092 // Invalid MMX constraint
1093 return nullptr;
1094 }
1095
1096 return llvm::Type::getX86_MMXTy(CGF.getLLVMContext());
1097 }
1098
1099 // No operation needed
1100 return Ty;
1101}
1102
1103/// Returns true if this type can be passed in SSE registers with the
1104/// X86_VectorCall calling convention. Shared between x86_32 and x86_64.
1105static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {
1106 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
1107 if (BT->isFloatingPoint() && BT->getKind() != BuiltinType::Half) {
1108 if (BT->getKind() == BuiltinType::LongDouble) {
1109 if (&Context.getTargetInfo().getLongDoubleFormat() ==
1110 &llvm::APFloat::x87DoubleExtended())
1111 return false;
1112 }
1113 return true;
1114 }
1115 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
1116 // vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX
1117 // registers specially.
1118 unsigned VecSize = Context.getTypeSize(VT);
1119 if (VecSize == 128 || VecSize == 256 || VecSize == 512)
1120 return true;
1121 }
1122 return false;
1123}
1124
1125/// Returns true if this aggregate is small enough to be passed in SSE registers
1126/// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.
1127static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {
1128 return NumMembers <= 4;
1129}
1130
1131/// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.
1132static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {
1133 auto AI = ABIArgInfo::getDirect(T);
1134 AI.setInReg(true);
1135 AI.setCanBeFlattened(false);
1136 return AI;
1137}
1138
1139//===----------------------------------------------------------------------===//
1140// X86-32 ABI Implementation
1141//===----------------------------------------------------------------------===//
1142
1143/// Similar to llvm::CCState, but for Clang.
1144struct CCState {
1145 CCState(CGFunctionInfo &FI)
1146 : IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()) {}
1147
1148 llvm::SmallBitVector IsPreassigned;
1149 unsigned CC = CallingConv::CC_C;
1150 unsigned FreeRegs = 0;
1151 unsigned FreeSSERegs = 0;
1152};
1153
1154/// X86_32ABIInfo - The X86-32 ABI information.
1155class X86_32ABIInfo : public ABIInfo {
1156 enum Class {
1157 Integer,
1158 Float
1159 };
1160
1161 static const unsigned MinABIStackAlignInBytes = 4;
1162
1163 bool IsDarwinVectorABI;
1164 bool IsRetSmallStructInRegABI;
1165 bool IsWin32StructABI;
1166 bool IsSoftFloatABI;
1167 bool IsMCUABI;
1168 bool IsLinuxABI;
1169 unsigned DefaultNumRegisterParameters;
1170
1171 static bool isRegisterSize(unsigned Size) {
1172 return (Size == 8 || Size == 16 || Size == 32 || Size == 64);
1173 }
1174
1175 bool isHomogeneousAggregateBaseType(QualType Ty) const override {
1176 // FIXME: Assumes vectorcall is in use.
1177 return isX86VectorTypeForVectorCall(getContext(), Ty);
1178 }
1179
1180 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
1181 uint64_t NumMembers) const override {
1182 // FIXME: Assumes vectorcall is in use.
1183 return isX86VectorCallAggregateSmallEnough(NumMembers);
1184 }
1185
1186 bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;
1187
1188 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1189 /// such that the argument will be passed in memory.
1190 ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
1191
1192 ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
1193
1194 /// Return the alignment to use for the given type on the stack.
1195 unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
1196
1197 Class classify(QualType Ty) const;
1198 ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;
1199 ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State) const;
1200
1201 /// Updates the number of available free registers, returns
1202 /// true if any registers were allocated.
1203 bool updateFreeRegs(QualType Ty, CCState &State) const;
1204
1205 bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,
1206 bool &NeedsPadding) const;
1207 bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;
1208
1209 bool canExpandIndirectArgument(QualType Ty) const;
1210
1211 /// Rewrite the function info so that all memory arguments use
1212 /// inalloca.
1213 void rewriteWithInAlloca(CGFunctionInfo &FI) const;
1214
1215 void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
1216 CharUnits &StackOffset, ABIArgInfo &Info,
1217 QualType Type) const;
1218 void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;
1219
1220public:
1221
1222 void computeInfo(CGFunctionInfo &FI) const override;
1223 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1224 QualType Ty) const override;
1225
1226 X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1227 bool RetSmallStructInRegABI, bool Win32StructABI,
1228 unsigned NumRegisterParameters, bool SoftFloatABI)
1229 : ABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
1230 IsRetSmallStructInRegABI(RetSmallStructInRegABI),
1231 IsWin32StructABI(Win32StructABI), IsSoftFloatABI(SoftFloatABI),
1232 IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
1233 IsLinuxABI(CGT.getTarget().getTriple().isOSLinux() ||
1234 CGT.getTarget().getTriple().isOSCygMing()),
1235 DefaultNumRegisterParameters(NumRegisterParameters) {}
1236};
1237
1238class X86_32SwiftABIInfo : public SwiftABIInfo {
1239public:
1240 explicit X86_32SwiftABIInfo(CodeGenTypes &CGT)
1241 : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/false) {}
1242
1244 bool AsReturnValue) const override {
1245 // LLVM's x86-32 lowering currently only assigns up to three
1246 // integer registers and three fp registers. Oddly, it'll use up to
1247 // four vector registers for vectors, but those can overlap with the
1248 // scalar registers.
1249 return occupiesMoreThan(CGT, ComponentTys, /*total=*/3);
1250 }
1251};
1252
1253class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
1254public:
1255 X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1256 bool RetSmallStructInRegABI, bool Win32StructABI,
1257 unsigned NumRegisterParameters, bool SoftFloatABI)
1258 : TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(
1259 CGT, DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI,
1260 NumRegisterParameters, SoftFloatABI)) {
1261 SwiftInfo = std::make_unique<X86_32SwiftABIInfo>(CGT);
1262 }
1263
1264 static bool isStructReturnInRegABI(
1265 const llvm::Triple &Triple, const CodeGenOptions &Opts);
1266
1267 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1268 CodeGen::CodeGenModule &CGM) const override;
1269
1270 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1271 // Darwin uses different dwarf register numbers for EH.
1272 if (CGM.getTarget().getTriple().isOSDarwin()) return 5;
1273 return 4;
1274 }
1275
1276 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1277 llvm::Value *Address) const override;
1278
1279 llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1280 StringRef Constraint,
1281 llvm::Type* Ty) const override {
1282 return X86AdjustInlineAsmType(CGF, Constraint, Ty);
1283 }
1284
1285 void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,
1286 std::string &Constraints,
1287 std::vector<llvm::Type *> &ResultRegTypes,
1288 std::vector<llvm::Type *> &ResultTruncRegTypes,
1289 std::vector<LValue> &ResultRegDests,
1290 std::string &AsmString,
1291 unsigned NumOutputs) const override;
1292
1293 llvm::Constant *
1294 getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
1295 unsigned Sig = (0xeb << 0) | // jmp rel8
1296 (0x06 << 8) | // .+0x08
1297 ('v' << 16) |
1298 ('2' << 24);
1299 return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
1300 }
1301
1302 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
1303 return "movl\t%ebp, %ebp"
1304 "\t\t// marker for objc_retainAutoreleaseReturnValue";
1305 }
1306};
1307
1308}
1309
1310/// Rewrite input constraint references after adding some output constraints.
1311/// In the case where there is one output and one input and we add one output,
1312/// we need to replace all operand references greater than or equal to 1:
1313/// mov $0, $1
1314/// mov eax, $1
1315/// The result will be:
1316/// mov $0, $2
1317/// mov eax, $2
1318static void rewriteInputConstraintReferences(unsigned FirstIn,
1319 unsigned NumNewOuts,
1320 std::string &AsmString) {
1321 std::string Buf;
1322 llvm::raw_string_ostream OS(Buf);
1323 size_t Pos = 0;
1324 while (Pos < AsmString.size()) {
1325 size_t DollarStart = AsmString.find('$', Pos);
1326 if (DollarStart == std::string::npos)
1327 DollarStart = AsmString.size();
1328 size_t DollarEnd = AsmString.find_first_not_of('$', DollarStart);
1329 if (DollarEnd == std::string::npos)
1330 DollarEnd = AsmString.size();
1331 OS << StringRef(&AsmString[Pos], DollarEnd - Pos);
1332 Pos = DollarEnd;
1333 size_t NumDollars = DollarEnd - DollarStart;
1334 if (NumDollars % 2 != 0 && Pos < AsmString.size()) {
1335 // We have an operand reference.
1336 size_t DigitStart = Pos;
1337 if (AsmString[DigitStart] == '{') {
1338 OS << '{';
1339 ++DigitStart;
1340 }
1341 size_t DigitEnd = AsmString.find_first_not_of("0123456789", DigitStart);
1342 if (DigitEnd == std::string::npos)
1343 DigitEnd = AsmString.size();
1344 StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);
1345 unsigned OperandIndex;
1346 if (!OperandStr.getAsInteger(10, OperandIndex)) {
1347 if (OperandIndex >= FirstIn)
1348 OperandIndex += NumNewOuts;
1349 OS << OperandIndex;
1350 } else {
1351 OS << OperandStr;
1352 }
1353 Pos = DigitEnd;
1354 }
1355 }
1356 AsmString = std::move(OS.str());
1357}
1358
1359/// Add output constraints for EAX:EDX because they are return registers.
1360void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
1361 CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,
1362 std::vector<llvm::Type *> &ResultRegTypes,
1363 std::vector<llvm::Type *> &ResultTruncRegTypes,
1364 std::vector<LValue> &ResultRegDests, std::string &AsmString,
1365 unsigned NumOutputs) const {
1366 uint64_t RetWidth = CGF.getContext().getTypeSize(ReturnSlot.getType());
1367
1368 // Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is
1369 // larger.
1370 if (!Constraints.empty())
1371 Constraints += ',';
1372 if (RetWidth <= 32) {
1373 Constraints += "={eax}";
1374 ResultRegTypes.push_back(CGF.Int32Ty);
1375 } else {
1376 // Use the 'A' constraint for EAX:EDX.
1377 Constraints += "=A";
1378 ResultRegTypes.push_back(CGF.Int64Ty);
1379 }
1380
1381 // Truncate EAX or EAX:EDX to an integer of the appropriate size.
1382 llvm::Type *CoerceTy = llvm::IntegerType::get(CGF.getLLVMContext(), RetWidth);
1383 ResultTruncRegTypes.push_back(CoerceTy);
1384
1385 // Coerce the integer by bitcasting the return slot pointer.
1386 ReturnSlot.setAddress(
1387 CGF.Builder.CreateElementBitCast(ReturnSlot.getAddress(CGF), CoerceTy));
1388 ResultRegDests.push_back(ReturnSlot);
1389
1390 rewriteInputConstraintReferences(NumOutputs, 1, AsmString);
1391}
1392
1393/// shouldReturnTypeInRegister - Determine if the given type should be
1394/// returned in a register (for the Darwin and MCU ABI).
1395bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
1396 ASTContext &Context) const {
1397 uint64_t Size = Context.getTypeSize(Ty);
1398
1399 // For i386, type must be register sized.
1400 // For the MCU ABI, it only needs to be <= 8-byte
1401 if ((IsMCUABI && Size > 64) || (!IsMCUABI && !isRegisterSize(Size)))
1402 return false;
1403
1404 if (Ty->isVectorType()) {
1405 // 64- and 128- bit vectors inside structures are not returned in
1406 // registers.
1407 if (Size == 64 || Size == 128)
1408 return false;
1409
1410 return true;
1411 }
1412
1413 // If this is a builtin, pointer, enum, complex type, member pointer, or
1414 // member function pointer it is ok.
1415 if (Ty->getAs<BuiltinType>() || Ty->hasPointerRepresentation() ||
1416 Ty->isAnyComplexType() || Ty->isEnumeralType() ||
1418 return true;
1419
1420 // Arrays are treated like records.
1421 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty))
1422 return shouldReturnTypeInRegister(AT->getElementType(), Context);
1423
1424 // Otherwise, it must be a record type.
1425 const RecordType *RT = Ty->getAs<RecordType>();
1426 if (!RT) return false;
1427
1428 // FIXME: Traverse bases here too.
1429
1430 // Structure types are passed in register if all fields would be
1431 // passed in a register.
1432 for (const auto *FD : RT->getDecl()->fields()) {
1433 // Empty fields are ignored.
1434 if (isEmptyField(Context, FD, true))
1435 continue;
1436
1437 // Check fields recursively.
1438 if (!shouldReturnTypeInRegister(FD->getType(), Context))
1439 return false;
1440 }
1441 return true;
1442}
1443
1444static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
1445 // Treat complex types as the element type.
1446 if (const ComplexType *CTy = Ty->getAs<ComplexType>())
1447 Ty = CTy->getElementType();
1448
1449 // Check for a type which we know has a simple scalar argument-passing
1450 // convention without any padding. (We're specifically looking for 32
1451 // and 64-bit integer and integer-equivalents, float, and double.)
1452 if (!Ty->getAs<BuiltinType>() && !Ty->hasPointerRepresentation() &&
1453 !Ty->isEnumeralType() && !Ty->isBlockPointerType())
1454 return false;
1455
1456 uint64_t Size = Context.getTypeSize(Ty);
1457 return Size == 32 || Size == 64;
1458}
1459
1460static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,
1461 uint64_t &Size) {
1462 for (const auto *FD : RD->fields()) {
1463 // Scalar arguments on the stack get 4 byte alignment on x86. If the
1464 // argument is smaller than 32-bits, expanding the struct will create
1465 // alignment padding.
1466 if (!is32Or64BitBasicType(FD->getType(), Context))
1467 return false;
1468
1469 // FIXME: Reject bit-fields wholesale; there are two problems, we don't know
1470 // how to expand them yet, and the predicate for telling if a bitfield still
1471 // counts as "basic" is more complicated than what we were doing previously.
1472 if (FD->isBitField())
1473 return false;
1474
1475 Size += Context.getTypeSize(FD->getType());
1476 }
1477 return true;
1478}
1479
1480static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,
1481 uint64_t &Size) {
1482 // Don't do this if there are any non-empty bases.
1483 for (const CXXBaseSpecifier &Base : RD->bases()) {
1484 if (!addBaseAndFieldSizes(Context, Base.getType()->getAsCXXRecordDecl(),
1485 Size))
1486 return false;
1487 }
1488 if (!addFieldSizes(Context, RD, Size))
1489 return false;
1490 return true;
1491}
1492
1493/// Test whether an argument type which is to be passed indirectly (on the
1494/// stack) would have the equivalent layout if it was expanded into separate
1495/// arguments. If so, we prefer to do the latter to avoid inhibiting
1496/// optimizations.
1497bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
1498 // We can only expand structure types.
1499 const RecordType *RT = Ty->getAs<RecordType>();
1500 if (!RT)
1501 return false;
1502 const RecordDecl *RD = RT->getDecl();
1503 uint64_t Size = 0;
1504 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
1505 if (!IsWin32StructABI) {
1506 // On non-Windows, we have to conservatively match our old bitcode
1507 // prototypes in order to be ABI-compatible at the bitcode level.
1508 if (!CXXRD->isCLike())
1509 return false;
1510 } else {
1511 // Don't do this for dynamic classes.
1512 if (CXXRD->isDynamicClass())
1513 return false;
1514 }
1515 if (!addBaseAndFieldSizes(getContext(), CXXRD, Size))
1516 return false;
1517 } else {
1518 if (!addFieldSizes(getContext(), RD, Size))
1519 return false;
1520 }
1521
1522 // We can do this if there was no alignment padding.
1523 return Size == getContext().getTypeSize(Ty);
1524}
1525
1526ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
1527 // If the return value is indirect, then the hidden argument is consuming one
1528 // integer register.
1529 if (State.FreeRegs) {
1530 --State.FreeRegs;
1531 if (!IsMCUABI)
1532 return getNaturalAlignIndirectInReg(RetTy);
1533 }
1534 return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
1535}
1536
1537ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
1538 CCState &State) const {
1539 if (RetTy->isVoidType())
1540 return ABIArgInfo::getIgnore();
1541
1542 const Type *Base = nullptr;
1543 uint64_t NumElts = 0;
1544 if ((State.CC == llvm::CallingConv::X86_VectorCall ||
1545 State.CC == llvm::CallingConv::X86_RegCall) &&
1546 isHomogeneousAggregate(RetTy, Base, NumElts)) {
1547 // The LLVM struct type for such an aggregate should lower properly.
1548 return ABIArgInfo::getDirect();
1549 }
1550
1551 if (const VectorType *VT = RetTy->getAs<VectorType>()) {
1552 // On Darwin, some vectors are returned in registers.
1553 if (IsDarwinVectorABI) {
1554 uint64_t Size = getContext().getTypeSize(RetTy);
1555
1556 // 128-bit vectors are a special case; they are returned in
1557 // registers and we need to make sure to pick a type the LLVM
1558 // backend will like.
1559 if (Size == 128)
1560 return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1561 llvm::Type::getInt64Ty(getVMContext()), 2));
1562
1563 // Always return in register if it fits in a general purpose
1564 // register, or if it is 64 bits and has a single element.
1565 if ((Size == 8 || Size == 16 || Size == 32) ||
1566 (Size == 64 && VT->getNumElements() == 1))
1567 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
1568 Size));
1569
1570 return getIndirectReturnResult(RetTy, State);
1571 }
1572
1573 return ABIArgInfo::getDirect();
1574 }
1575
1576 if (isAggregateTypeForABI(RetTy)) {
1577 if (const RecordType *RT = RetTy->getAs<RecordType>()) {
1578 // Structures with flexible arrays are always indirect.
1579 if (RT->getDecl()->hasFlexibleArrayMember())
1580 return getIndirectReturnResult(RetTy, State);
1581 }
1582
1583 // If specified, structs and unions are always indirect.
1584 if (!IsRetSmallStructInRegABI && !RetTy->isAnyComplexType())
1585 return getIndirectReturnResult(RetTy, State);
1586
1587 // Ignore empty structs/unions.
1588 if (isEmptyRecord(getContext(), RetTy, true))
1589 return ABIArgInfo::getIgnore();
1590
1591 // Return complex of _Float16 as <2 x half> so the backend will use xmm0.
1592 if (const ComplexType *CT = RetTy->getAs<ComplexType>()) {
1593 QualType ET = getContext().getCanonicalType(CT->getElementType());
1594 if (ET->isFloat16Type())
1595 return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1596 llvm::Type::getHalfTy(getVMContext()), 2));
1597 }
1598
1599 // Small structures which are register sized are generally returned
1600 // in a register.
1601 if (shouldReturnTypeInRegister(RetTy, getContext())) {
1602 uint64_t Size = getContext().getTypeSize(RetTy);
1603
1604 // As a special-case, if the struct is a "single-element" struct, and
1605 // the field is of type "float" or "double", return it in a
1606 // floating-point register. (MSVC does not apply this special case.)
1607 // We apply a similar transformation for pointer types to improve the
1608 // quality of the generated IR.
1609 if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
1610 if ((!IsWin32StructABI && SeltTy->isRealFloatingType())
1611 || SeltTy->hasPointerRepresentation())
1612 return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
1613
1614 // FIXME: We should be able to narrow this integer in cases with dead
1615 // padding.
1616 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),Size));
1617 }
1618
1619 return getIndirectReturnResult(RetTy, State);
1620 }
1621
1622 // Treat an enum type as its underlying type.
1623 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1624 RetTy = EnumTy->getDecl()->getIntegerType();
1625
1626 if (const auto *EIT = RetTy->getAs<BitIntType>())
1627 if (EIT->getNumBits() > 64)
1628 return getIndirectReturnResult(RetTy, State);
1629
1630 return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
1632}
1633
1634static bool isSIMDVectorType(ASTContext &Context, QualType Ty) {
1635 return Ty->getAs<VectorType>() && Context.getTypeSize(Ty) == 128;
1636}
1637
1639 const RecordType *RT = Ty->getAs<RecordType>();
1640 if (!RT)
1641 return false;
1642 const RecordDecl *RD = RT->getDecl();
1643
1644 // If this is a C++ record, check the bases first.
1645 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
1646 for (const auto &I : CXXRD->bases())
1647 if (!isRecordWithSIMDVectorType(Context, I.getType()))
1648 return false;
1649
1650 for (const auto *i : RD->fields()) {
1651 QualType FT = i->getType();
1652
1653 if (isSIMDVectorType(Context, FT))
1654 return true;
1655
1656 if (isRecordWithSIMDVectorType(Context, FT))
1657 return true;
1658 }
1659
1660 return false;
1661}
1662
1663unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
1664 unsigned Align) const {
1665 // Otherwise, if the alignment is less than or equal to the minimum ABI
1666 // alignment, just use the default; the backend will handle this.
1667 if (Align <= MinABIStackAlignInBytes)
1668 return 0; // Use default alignment.
1669
1670 if (IsLinuxABI) {
1671 // Exclude other System V OS (e.g Darwin, PS4 and FreeBSD) since we don't
1672 // want to spend any effort dealing with the ramifications of ABI breaks.
1673 //
1674 // If the vector type is __m128/__m256/__m512, return the default alignment.
1675 if (Ty->isVectorType() && (Align == 16 || Align == 32 || Align == 64))
1676 return Align;
1677 }
1678 // On non-Darwin, the stack type alignment is always 4.
1679 if (!IsDarwinVectorABI) {
1680 // Set explicit alignment, since we may need to realign the top.
1681 return MinABIStackAlignInBytes;
1682 }
1683
1684 // Otherwise, if the type contains an SSE vector type, the alignment is 16.
1685 if (Align >= 16 && (isSIMDVectorType(getContext(), Ty) ||
1686 isRecordWithSIMDVectorType(getContext(), Ty)))
1687 return 16;
1688
1689 return MinABIStackAlignInBytes;
1690}
1691
1692ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
1693 CCState &State) const {
1694 if (!ByVal) {
1695 if (State.FreeRegs) {
1696 --State.FreeRegs; // Non-byval indirects just use one pointer.
1697 if (!IsMCUABI)
1698 return getNaturalAlignIndirectInReg(Ty);
1699 }
1700 return getNaturalAlignIndirect(Ty, false);
1701 }
1702
1703 // Compute the byval alignment.
1704 unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
1705 unsigned StackAlign = getTypeStackAlignInBytes(Ty, TypeAlign);
1706 if (StackAlign == 0)
1707 return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true);
1708
1709 // If the stack alignment is less than the type alignment, realign the
1710 // argument.
1711 bool Realign = TypeAlign > StackAlign;
1713 /*ByVal=*/true, Realign);
1714}
1715
1716X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
1717 const Type *T = isSingleElementStruct(Ty, getContext());
1718 if (!T)
1719 T = Ty.getTypePtr();
1720
1721 if (const BuiltinType *BT = T->getAs<BuiltinType>()) {
1722 BuiltinType::Kind K = BT->getKind();
1723 if (K == BuiltinType::Float || K == BuiltinType::Double)
1724 return Float;
1725 }
1726 return Integer;
1727}
1728
1729bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {
1730 if (!IsSoftFloatABI) {
1731 Class C = classify(Ty);
1732 if (C == Float)
1733 return false;
1734 }
1735
1736 unsigned Size = getContext().getTypeSize(Ty);
1737 unsigned SizeInRegs = (Size + 31) / 32;
1738
1739 if (SizeInRegs == 0)
1740 return false;
1741
1742 if (!IsMCUABI) {
1743 if (SizeInRegs > State.FreeRegs) {
1744 State.FreeRegs = 0;
1745 return false;
1746 }
1747 } else {
1748 // The MCU psABI allows passing parameters in-reg even if there are
1749 // earlier parameters that are passed on the stack. Also,
1750 // it does not allow passing >8-byte structs in-register,
1751 // even if there are 3 free registers available.
1752 if (SizeInRegs > State.FreeRegs || SizeInRegs > 2)
1753 return false;
1754 }
1755
1756 State.FreeRegs -= SizeInRegs;
1757 return true;
1758}
1759
1760bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
1761 bool &InReg,
1762 bool &NeedsPadding) const {
1763 // On Windows, aggregates other than HFAs are never passed in registers, and
1764 // they do not consume register slots. Homogenous floating-point aggregates
1765 // (HFAs) have already been dealt with at this point.
1766 if (IsWin32StructABI && isAggregateTypeForABI(Ty))
1767 return false;
1768
1769 NeedsPadding = false;
1770 InReg = !IsMCUABI;
1771
1772 if (!updateFreeRegs(Ty, State))
1773 return false;
1774
1775 if (IsMCUABI)
1776 return true;
1777
1778 if (State.CC == llvm::CallingConv::X86_FastCall ||
1779 State.CC == llvm::CallingConv::X86_VectorCall ||
1780 State.CC == llvm::CallingConv::X86_RegCall) {
1781 if (getContext().getTypeSize(Ty) <= 32 && State.FreeRegs)
1782 NeedsPadding = true;
1783
1784 return false;
1785 }
1786
1787 return true;
1788}
1789
1790bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
1791 bool IsPtrOrInt = (getContext().getTypeSize(Ty) <= 32) &&
1793 Ty->isReferenceType());
1794
1795 if (!IsPtrOrInt && (State.CC == llvm::CallingConv::X86_FastCall ||
1796 State.CC == llvm::CallingConv::X86_VectorCall))
1797 return false;
1798
1799 if (!updateFreeRegs(Ty, State))
1800 return false;
1801
1802 if (!IsPtrOrInt && State.CC == llvm::CallingConv::X86_RegCall)
1803 return false;
1804
1805 // Return true to apply inreg to all legal parameters except for MCU targets.
1806 return !IsMCUABI;
1807}
1808
1809void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
1810 // Vectorcall x86 works subtly different than in x64, so the format is
1811 // a bit different than the x64 version. First, all vector types (not HVAs)
1812 // are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
1813 // This differs from the x64 implementation, where the first 6 by INDEX get
1814 // registers.
1815 // In the second pass over the arguments, HVAs are passed in the remaining
1816 // vector registers if possible, or indirectly by address. The address will be
1817 // passed in ECX/EDX if available. Any other arguments are passed according to
1818 // the usual fastcall rules.
1820 for (int I = 0, E = Args.size(); I < E; ++I) {
1821 const Type *Base = nullptr;
1822 uint64_t NumElts = 0;
1823 const QualType &Ty = Args[I].type;
1824 if ((Ty->isVectorType() || Ty->isBuiltinType()) &&
1825 isHomogeneousAggregate(Ty, Base, NumElts)) {
1826 if (State.FreeSSERegs >= NumElts) {
1827 State.FreeSSERegs -= NumElts;
1828 Args[I].info = ABIArgInfo::getDirectInReg();
1829 State.IsPreassigned.set(I);
1830 }
1831 }
1832 }
1833}
1834
1835ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
1836 CCState &State) const {
1837 // FIXME: Set alignment on indirect arguments.
1838 bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;
1839 bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;
1840 bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
1841
1843 TypeInfo TI = getContext().getTypeInfo(Ty);
1844
1845 // Check with the C++ ABI first.
1846 const RecordType *RT = Ty->getAs<RecordType>();
1847 if (RT) {
1848 CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
1849 if (RAA == CGCXXABI::RAA_Indirect) {
1850 return getIndirectResult(Ty, false, State);
1851 } else if (RAA == CGCXXABI::RAA_DirectInMemory) {
1852 // The field index doesn't matter, we'll fix it up later.
1853 return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);
1854 }
1855 }
1856
1857 // Regcall uses the concept of a homogenous vector aggregate, similar
1858 // to other targets.
1859 const Type *Base = nullptr;
1860 uint64_t NumElts = 0;
1861 if ((IsRegCall || IsVectorCall) &&
1862 isHomogeneousAggregate(Ty, Base, NumElts)) {
1863 if (State.FreeSSERegs >= NumElts) {
1864 State.FreeSSERegs -= NumElts;
1865
1866 // Vectorcall passes HVAs directly and does not flatten them, but regcall
1867 // does.
1868 if (IsVectorCall)
1869 return getDirectX86Hva();
1870
1871 if (Ty->isBuiltinType() || Ty->isVectorType())
1872 return ABIArgInfo::getDirect();
1873 return ABIArgInfo::getExpand();
1874 }
1875 return getIndirectResult(Ty, /*ByVal=*/false, State);
1876 }
1877
1878 if (isAggregateTypeForABI(Ty)) {
1879 // Structures with flexible arrays are always indirect.
1880 // FIXME: This should not be byval!
1881 if (RT && RT->getDecl()->hasFlexibleArrayMember())
1882 return getIndirectResult(Ty, true, State);
1883
1884 // Ignore empty structs/unions on non-Windows.
1885 if (!IsWin32StructABI && isEmptyRecord(getContext(), Ty, true))
1886 return ABIArgInfo::getIgnore();
1887
1888 llvm::LLVMContext &LLVMContext = getVMContext();
1889 llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);
1890 bool NeedsPadding = false;
1891 bool InReg;
1892 if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
1893 unsigned SizeInRegs = (TI.Width + 31) / 32;
1894 SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
1895 llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
1896 if (InReg)
1898 else
1900 }
1901 llvm::IntegerType *PaddingType = NeedsPadding ? Int32 : nullptr;
1902
1903 // Pass over-aligned aggregates on Windows indirectly. This behavior was
1904 // added in MSVC 2015.
1905 if (IsWin32StructABI && TI.isAlignRequired() && TI.Align > 32)
1906 return getIndirectResult(Ty, /*ByVal=*/false, State);
1907
1908 // Expand small (<= 128-bit) record types when we know that the stack layout
1909 // of those arguments will match the struct. This is important because the
1910 // LLVM backend isn't smart enough to remove byval, which inhibits many
1911 // optimizations.
1912 // Don't do this for the MCU if there are still free integer registers
1913 // (see X86_64 ABI for full explanation).
1914 if (TI.Width <= 4 * 32 && (!IsMCUABI || State.FreeRegs == 0) &&
1915 canExpandIndirectArgument(Ty))
1917 IsFastCall || IsVectorCall || IsRegCall, PaddingType);
1918
1919 return getIndirectResult(Ty, true, State);
1920 }
1921
1922 if (const VectorType *VT = Ty->getAs<VectorType>()) {
1923 // On Windows, vectors are passed directly if registers are available, or
1924 // indirectly if not. This avoids the need to align argument memory. Pass
1925 // user-defined vector types larger than 512 bits indirectly for simplicity.
1926 if (IsWin32StructABI) {
1927 if (TI.Width <= 512 && State.FreeSSERegs > 0) {
1928 --State.FreeSSERegs;
1930 }
1931 return getIndirectResult(Ty, /*ByVal=*/false, State);
1932 }
1933
1934 // On Darwin, some vectors are passed in memory, we handle this by passing
1935 // it as an i8/i16/i32/i64.
1936 if (IsDarwinVectorABI) {
1937 if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
1938 (TI.Width == 64 && VT->getNumElements() == 1))
1939 return ABIArgInfo::getDirect(
1940 llvm::IntegerType::get(getVMContext(), TI.Width));
1941 }
1942
1943 if (IsX86_MMXType(CGT.ConvertType(Ty)))
1944 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), 64));
1945
1946 return ABIArgInfo::getDirect();
1947 }
1948
1949
1950 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
1951 Ty = EnumTy->getDecl()->getIntegerType();
1952
1953 bool InReg = shouldPrimitiveUseInReg(Ty, State);
1954
1955 if (isPromotableIntegerTypeForABI(Ty)) {
1956 if (InReg)
1957 return ABIArgInfo::getExtendInReg(Ty);
1958 return ABIArgInfo::getExtend(Ty);
1959 }
1960
1961 if (const auto *EIT = Ty->getAs<BitIntType>()) {
1962 if (EIT->getNumBits() <= 64) {
1963 if (InReg)
1965 return ABIArgInfo::getDirect();
1966 }
1967 return getIndirectResult(Ty, /*ByVal=*/false, State);
1968 }
1969
1970 if (InReg)
1972 return ABIArgInfo::getDirect();
1973}
1974
1975void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
1976 CCState State(FI);
1977 if (IsMCUABI)
1978 State.FreeRegs = 3;
1979 else if (State.CC == llvm::CallingConv::X86_FastCall) {
1980 State.FreeRegs = 2;
1981 State.FreeSSERegs = 3;
1982 } else if (State.CC == llvm::CallingConv::X86_VectorCall) {
1983 State.FreeRegs = 2;
1984 State.FreeSSERegs = 6;
1985 } else if (FI.getHasRegParm())
1986 State.FreeRegs = FI.getRegParm();
1987 else if (State.CC == llvm::CallingConv::X86_RegCall) {
1988 State.FreeRegs = 5;
1989 State.FreeSSERegs = 8;
1990 } else if (IsWin32StructABI) {
1991 // Since MSVC 2015, the first three SSE vectors have been passed in
1992 // registers. The rest are passed indirectly.
1993 State.FreeRegs = DefaultNumRegisterParameters;
1994 State.FreeSSERegs = 3;
1995 } else
1996 State.FreeRegs = DefaultNumRegisterParameters;
1997
1998 if (!::classifyReturnType(getCXXABI(), FI, *this)) {
2000 } else if (FI.getReturnInfo().isIndirect()) {
2001 // The C++ ABI is not aware of register usage, so we have to check if the
2002 // return value was sret and put it in a register ourselves if appropriate.
2003 if (State.FreeRegs) {
2004 --State.FreeRegs; // The sret parameter consumes a register.
2005 if (!IsMCUABI)
2006 FI.getReturnInfo().setInReg(true);
2007 }
2008 }
2009
2010 // The chain argument effectively gives us another free register.
2011 if (FI.isChainCall())
2012 ++State.FreeRegs;
2013
2014 // For vectorcall, do a first pass over the arguments, assigning FP and vector
2015 // arguments to XMM registers as available.
2016 if (State.CC == llvm::CallingConv::X86_VectorCall)
2017 runVectorCallFirstPass(FI, State);
2018
2019 bool UsedInAlloca = false;
2021 for (int I = 0, E = Args.size(); I < E; ++I) {
2022 // Skip arguments that have already been assigned.
2023 if (State.IsPreassigned.test(I))
2024 continue;
2025
2026 Args[I].info = classifyArgumentType(Args[I].type, State);
2027 UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);
2028 }
2029
2030 // If we needed to use inalloca for any argument, do a second pass and rewrite
2031 // all the memory arguments to use inalloca.
2032 if (UsedInAlloca)
2033 rewriteWithInAlloca(FI);
2034}
2035
2036void
2037X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
2038 CharUnits &StackOffset, ABIArgInfo &Info,
2039 QualType Type) const {
2040 // Arguments are always 4-byte-aligned.
2041 CharUnits WordSize = CharUnits::fromQuantity(4);
2042 assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
2043
2044 // sret pointers and indirect things will require an extra pointer
2045 // indirection, unless they are byval. Most things are byval, and will not
2046 // require this indirection.
2047 bool IsIndirect = false;
2048 if (Info.isIndirect() && !Info.getIndirectByVal())
2049 IsIndirect = true;
2050 Info = ABIArgInfo::getInAlloca(FrameFields.size(), IsIndirect);
2051 llvm::Type *LLTy = CGT.ConvertTypeForMem(Type);
2052 if (IsIndirect)
2053 LLTy = LLTy->getPointerTo(0);
2054 FrameFields.push_back(LLTy);
2055 StackOffset += IsIndirect ? WordSize : getContext().getTypeSizeInChars(Type);
2056
2057 // Insert padding bytes to respect alignment.
2058 CharUnits FieldEnd = StackOffset;
2059 StackOffset = FieldEnd.alignTo(WordSize);
2060 if (StackOffset != FieldEnd) {
2061 CharUnits NumBytes = StackOffset - FieldEnd;
2062 llvm::Type *Ty = llvm::Type::getInt8Ty(getVMContext());
2063 Ty = llvm::ArrayType::get(Ty, NumBytes.getQuantity());
2064 FrameFields.push_back(Ty);
2065 }
2066}
2067
2068static bool isArgInAlloca(const ABIArgInfo &Info) {
2069 // Leave ignored and inreg arguments alone.
2070 switch (Info.getKind()) {
2072 return true;
2073 case ABIArgInfo::Ignore:
2075 return false;
2077 case ABIArgInfo::Direct:
2078 case ABIArgInfo::Extend:
2079 return !Info.getInReg();
2080 case ABIArgInfo::Expand:
2082 // These are aggregate types which are never passed in registers when
2083 // inalloca is involved.
2084 return true;
2085 }
2086 llvm_unreachable("invalid enum");
2087}
2088
2089void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
2090 assert(IsWin32StructABI && "inalloca only supported on win32");
2091
2092 // Build a packed struct type for all of the arguments in memory.
2093 SmallVector<llvm::Type *, 6> FrameFields;
2094
2095 // The stack alignment is always 4.
2096 CharUnits StackAlign = CharUnits::fromQuantity(4);
2097
2098 CharUnits StackOffset;
2100
2101 // Put 'this' into the struct before 'sret', if necessary.
2102 bool IsThisCall =
2103 FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;
2105 if (Ret.isIndirect() && Ret.isSRetAfterThis() && !IsThisCall &&
2106 isArgInAlloca(I->info)) {
2107 addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2108 ++I;
2109 }
2110
2111 // Put the sret parameter into the inalloca struct if it's in memory.
2112 if (Ret.isIndirect() && !Ret.getInReg()) {
2113 addFieldToArgStruct(FrameFields, StackOffset, Ret, FI.getReturnType());
2114 // On Windows, the hidden sret parameter is always returned in eax.
2115 Ret.setInAllocaSRet(IsWin32StructABI);
2116 }
2117
2118 // Skip the 'this' parameter in ecx.
2119 if (IsThisCall)
2120 ++I;
2121
2122 // Put arguments passed in memory into the struct.
2123 for (; I != E; ++I) {
2124 if (isArgInAlloca(I->info))
2125 addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2126 }
2127
2128 FI.setArgStruct(llvm::StructType::get(getVMContext(), FrameFields,
2129 /*isPacked=*/true),
2130 StackAlign);
2131}
2132
2133Address X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF,
2134 Address VAListAddr, QualType Ty) const {
2135
2136 auto TypeInfo = getContext().getTypeInfoInChars(Ty);
2137
2138 // x86-32 changes the alignment of certain arguments on the stack.
2139 //
2140 // Just messing with TypeInfo like this works because we never pass
2141 // anything indirectly.
2143 getTypeStackAlignInBytes(Ty, TypeInfo.Align.getQuantity()));
2144
2145 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false,
2147 /*AllowHigherAlign*/ true);
2148}
2149
2150bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
2151 const llvm::Triple &Triple, const CodeGenOptions &Opts) {
2152 assert(Triple.getArch() == llvm::Triple::x86);
2153
2154 switch (Opts.getStructReturnConvention()) {
2156 break;
2157 case CodeGenOptions::SRCK_OnStack: // -fpcc-struct-return
2158 return false;
2159 case CodeGenOptions::SRCK_InRegs: // -freg-struct-return
2160 return true;
2161 }
2162
2163 if (Triple.isOSDarwin() || Triple.isOSIAMCU())
2164 return true;
2165
2166 switch (Triple.getOS()) {
2167 case llvm::Triple::DragonFly:
2168 case llvm::Triple::FreeBSD:
2169 case llvm::Triple::OpenBSD:
2170 case llvm::Triple::Win32:
2171 return true;
2172 default:
2173 return false;
2174 }
2175}
2176
2177static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,
2179 if (!FD->hasAttr<AnyX86InterruptAttr>())
2180 return;
2181
2182 llvm::Function *Fn = cast<llvm::Function>(GV);
2183 Fn->setCallingConv(llvm::CallingConv::X86_INTR);
2184 if (FD->getNumParams() == 0)
2185 return;
2186
2187 auto PtrTy = cast<PointerType>(FD->getParamDecl(0)->getType());
2188 llvm::Type *ByValTy = CGM.getTypes().ConvertType(PtrTy->getPointeeType());
2189 llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(
2190 Fn->getContext(), ByValTy);
2191 Fn->addParamAttr(0, NewAttr);
2192}
2193
2194void X86_32TargetCodeGenInfo::setTargetAttributes(
2195 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2196 if (GV->isDeclaration())
2197 return;
2198 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2199 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2200 llvm::Function *Fn = cast<llvm::Function>(GV);
2201 Fn->addFnAttr("stackrealign");
2202 }
2203
2204 addX86InterruptAttrs(FD, GV, CGM);
2205 }
2206}
2207
2208bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
2210 llvm::Value *Address) const {
2211 CodeGen::CGBuilderTy &Builder = CGF.Builder;
2212
2213 llvm::Value *Four8 = llvm::ConstantInt::get(CGF.Int8Ty, 4);
2214
2215 // 0-7 are the eight integer registers; the order is different
2216 // on Darwin (for EH), but the range is the same.
2217 // 8 is %eip.
2218 AssignToArrayRange(Builder, Address, Four8, 0, 8);
2219
2220 if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {
2221 // 12-16 are st(0..4). Not sure why we stop at 4.
2222 // These have size 16, which is sizeof(long double) on
2223 // platforms with 8-byte alignment for that type.
2224 llvm::Value *Sixteen8 = llvm::ConstantInt::get(CGF.Int8Ty, 16);
2225 AssignToArrayRange(Builder, Address, Sixteen8, 12, 16);
2226
2227 } else {
2228 // 9 is %eflags, which doesn't get a size on Darwin for some
2229 // reason.
2230 Builder.CreateAlignedStore(
2231 Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9),
2232 CharUnits::One());
2233
2234 // 11-16 are st(0..5). Not sure why we stop at 5.
2235 // These have size 12, which is sizeof(long double) on
2236 // platforms with 4-byte alignment for that type.
2237 llvm::Value *Twelve8 = llvm::ConstantInt::get(CGF.Int8Ty, 12);
2238 AssignToArrayRange(Builder, Address, Twelve8, 11, 16);
2239 }
2240
2241 return false;
2242}
2243
2244//===----------------------------------------------------------------------===//
2245// X86-64 ABI Implementation
2246//===----------------------------------------------------------------------===//
2247
2248
2249namespace {
2250/// The AVX ABI level for X86 targets.
2251enum class X86AVXABILevel {
2252 None,
2253 AVX,
2254 AVX512
2255};
2256
2257/// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
2258static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
2259 switch (AVXLevel) {
2260 case X86AVXABILevel::AVX512:
2261 return 512;
2262 case X86AVXABILevel::AVX:
2263 return 256;
2264 case X86AVXABILevel::None:
2265 return 128;
2266 }
2267 llvm_unreachable("Unknown AVXLevel");
2268}
2269
2270/// X86_64ABIInfo - The X86_64 ABI information.
2271class X86_64ABIInfo : public ABIInfo {
2272 enum Class {
2273 Integer = 0,
2274 SSE,
2275 SSEUp,
2276 X87,
2277 X87Up,
2278 ComplexX87,
2279 NoClass,
2280 Memory
2281 };
2282
2283 /// merge - Implement the X86_64 ABI merging algorithm.
2284 ///
2285 /// Merge an accumulating classification \arg Accum with a field
2286 /// classification \arg Field.
2287 ///
2288 /// \param Accum - The accumulating classification. This should
2289 /// always be either NoClass or the result of a previous merge
2290 /// call. In addition, this should never be Memory (the caller
2291 /// should just return Memory for the aggregate).
2292 static Class merge(Class Accum, Class Field);
2293
2294 /// postMerge - Implement the X86_64 ABI post merging algorithm.
2295 ///
2296 /// Post merger cleanup, reduces a malformed Hi and Lo pair to
2297 /// final MEMORY or SSE classes when necessary.
2298 ///
2299 /// \param AggregateSize - The size of the current aggregate in
2300 /// the classification process.
2301 ///
2302 /// \param Lo - The classification for the parts of the type
2303 /// residing in the low word of the containing object.
2304 ///
2305 /// \param Hi - The classification for the parts of the type
2306 /// residing in the higher words of the containing object.
2307 ///
2308 void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
2309
2310 /// classify - Determine the x86_64 register classes in which the
2311 /// given type T should be passed.
2312 ///
2313 /// \param Lo - The classification for the parts of the type
2314 /// residing in the low word of the containing object.
2315 ///
2316 /// \param Hi - The classification for the parts of the type
2317 /// residing in the high word of the containing object.
2318 ///
2319 /// \param OffsetBase - The bit offset of this type in the
2320 /// containing object. Some parameters are classified different
2321 /// depending on whether they straddle an eightbyte boundary.
2322 ///
2323 /// \param isNamedArg - Whether the argument in question is a "named"
2324 /// argument, as used in AMD64-ABI 3.5.7.
2325 ///
2326 /// \param IsRegCall - Whether the calling conversion is regcall.
2327 ///
2328 /// If a word is unused its result will be NoClass; if a type should
2329 /// be passed in Memory then at least the classification of \arg Lo
2330 /// will be Memory.
2331 ///
2332 /// The \arg Lo class will be NoClass iff the argument is ignored.
2333 ///
2334 /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
2335 /// also be ComplexX87.
2336 void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
2337 bool isNamedArg, bool IsRegCall = false) const;
2338
2339 llvm::Type *GetByteVectorType(QualType Ty) const;
2340 llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
2341 unsigned IROffset, QualType SourceTy,
2342 unsigned SourceOffset) const;
2343 llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,
2344 unsigned IROffset, QualType SourceTy,
2345 unsigned SourceOffset) const;
2346
2347 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2348 /// such that the argument will be returned in memory.
2349 ABIArgInfo getIndirectReturnResult(QualType Ty) const;
2350
2351 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2352 /// such that the argument will be passed in memory.
2353 ///
2354 /// \param freeIntRegs - The number of free integer registers remaining
2355 /// available.
2356 ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;
2357
2359
2360 ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
2361 unsigned &neededInt, unsigned &neededSSE,
2362 bool isNamedArg,
2363 bool IsRegCall = false) const;
2364
2365 ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
2366 unsigned &NeededSSE,
2367 unsigned &MaxVectorWidth) const;
2368
2369 ABIArgInfo classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
2370 unsigned &NeededSSE,
2371 unsigned &MaxVectorWidth) const;
2372
2373 bool IsIllegalVectorType(QualType Ty) const;
2374
2375 /// The 0.98 ABI revision clarified a lot of ambiguities,
2376 /// unfortunately in ways that were not always consistent with
2377 /// certain previous compilers. In particular, platforms which
2378 /// required strict binary compatibility with older versions of GCC
2379 /// may need to exempt themselves.
2380 bool honorsRevision0_98() const {
2381 return !getTarget().getTriple().isOSDarwin();
2382 }
2383
2384 /// GCC classifies <1 x long long> as SSE but some platform ABIs choose to
2385 /// classify it as INTEGER (for compatibility with older clang compilers).
2386 bool classifyIntegerMMXAsSSE() const {
2387 // Clang <= 3.8 did not do this.
2388 if (getContext().getLangOpts().getClangABICompat() <=
2389 LangOptions::ClangABI::Ver3_8)
2390 return false;
2391
2392 const llvm::Triple &Triple = getTarget().getTriple();
2393 if (Triple.isOSDarwin() || Triple.isPS() || Triple.isOSFreeBSD())
2394 return false;
2395 return true;
2396 }
2397
2398 // GCC classifies vectors of __int128 as memory.
2399 bool passInt128VectorsInMem() const {
2400 // Clang <= 9.0 did not do this.
2401 if (getContext().getLangOpts().getClangABICompat() <=
2402 LangOptions::ClangABI::Ver9)
2403 return false;
2404
2405 const llvm::Triple &T = getTarget().getTriple();
2406 return T.isOSLinux() || T.isOSNetBSD();
2407 }
2408
2409 X86AVXABILevel AVXLevel;
2410 // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
2411 // 64-bit hardware.
2412 bool Has64BitPointers;
2413
2414public:
2415 X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2416 : ABIInfo(CGT), AVXLevel(AVXLevel),
2417 Has64BitPointers(CGT.getDataLayout().getPointerSize(0) == 8) {}
2418
2419 bool isPassedUsingAVXType(QualType type) const {
2420 unsigned neededInt, neededSSE;
2421 // The freeIntRegs argument doesn't matter here.
2422 ABIArgInfo info = classifyArgumentType(type, 0, neededInt, neededSSE,
2423 /*isNamedArg*/true);
2424 if (info.isDirect()) {
2425 llvm::Type *ty = info.getCoerceToType();
2426 if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(ty))
2427 return vectorTy->getPrimitiveSizeInBits().getFixedValue() > 128;
2428 }
2429 return false;
2430 }
2431
2432 void computeInfo(CGFunctionInfo &FI) const override;
2433
2434 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2435 QualType Ty) const override;
2436 Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
2437 QualType Ty) const override;
2438
2439 bool has64BitPointers() const {
2440 return Has64BitPointers;
2441 }
2442};
2443
2444/// WinX86_64ABIInfo - The Windows X86_64 ABI information.
2445class WinX86_64ABIInfo : public ABIInfo {
2446public:
2447 WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2448 : ABIInfo(CGT), AVXLevel(AVXLevel),
2449 IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
2450
2451 void computeInfo(CGFunctionInfo &FI) const override;
2452
2453 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2454 QualType Ty) const override;
2455
2456 bool isHomogeneousAggregateBaseType(QualType Ty) const override {
2457 // FIXME: Assumes vectorcall is in use.
2458 return isX86VectorTypeForVectorCall(getContext(), Ty);
2459 }
2460
2461 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
2462 uint64_t NumMembers) const override {
2463 // FIXME: Assumes vectorcall is in use.
2464 return isX86VectorCallAggregateSmallEnough(NumMembers);
2465 }
2466
2467private:
2468 ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType,
2469 bool IsVectorCall, bool IsRegCall) const;
2470 ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,
2471 const ABIArgInfo &current) const;
2472
2473 X86AVXABILevel AVXLevel;
2474
2475 bool IsMingw64;
2476};
2477
2478class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2479public:
2480 X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2481 : TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(CGT, AVXLevel)) {
2482 SwiftInfo =
2483 std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/true);
2484 }
2485
2486 const X86_64ABIInfo &getABIInfo() const {
2487 return static_cast<const X86_64ABIInfo&>(TargetCodeGenInfo::getABIInfo());
2488 }
2489
2490 /// Disable tail call on x86-64. The epilogue code before the tail jump blocks
2491 /// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
2492 bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
2493
2494 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2495 return 7;
2496 }
2497
2498 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2499 llvm::Value *Address) const override {
2500 llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2501
2502 // 0-15 are the 16 integer registers.
2503 // 16 is %rip.
2504 AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2505 return false;
2506 }
2507
2508 llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
2509 StringRef Constraint,
2510 llvm::Type* Ty) const override {
2511 return X86AdjustInlineAsmType(CGF, Constraint, Ty);
2512 }
2513
2514 bool isNoProtoCallVariadic(const CallArgList &args,
2515 const FunctionNoProtoType *fnType) const override {
2516 // The default CC on x86-64 sets %al to the number of SSA
2517 // registers used, and GCC sets this when calling an unprototyped
2518 // function, so we override the default behavior. However, don't do
2519 // that when AVX types are involved: the ABI explicitly states it is
2520 // undefined, and it doesn't work in practice because of how the ABI
2521 // defines varargs anyway.
2522 if (fnType->getCallConv() == CC_C) {
2523 bool HasAVXType = false;
2524 for (CallArgList::const_iterator
2525 it = args.begin(), ie = args.end(); it != ie; ++it) {
2526 if (getABIInfo().isPassedUsingAVXType(it->Ty)) {
2527 HasAVXType = true;
2528 break;
2529 }
2530 }
2531
2532 if (!HasAVXType)
2533 return true;
2534 }
2535
2536 return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);
2537 }
2538
2539 llvm::Constant *
2540 getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
2541 unsigned Sig = (0xeb << 0) | // jmp rel8
2542 (0x06 << 8) | // .+0x08
2543 ('v' << 16) |
2544 ('2' << 24);
2545 return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
2546 }
2547
2548 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2549 CodeGen::CodeGenModule &CGM) const override {
2550 if (GV->isDeclaration())
2551 return;
2552 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2553 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2554 llvm::Function *Fn = cast<llvm::Function>(GV);
2555 Fn->addFnAttr("stackrealign");
2556 }
2557
2558 addX86InterruptAttrs(FD, GV, CGM);
2559 }
2560 }
2561
2562 void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
2563 const FunctionDecl *Caller,
2564 const FunctionDecl *Callee,
2565 const CallArgList &Args) const override;
2566};
2567
2568static void initFeatureMaps(const ASTContext &Ctx,
2569 llvm::StringMap<bool> &CallerMap,
2570 const FunctionDecl *Caller,
2571 llvm::StringMap<bool> &CalleeMap,
2572 const FunctionDecl *Callee) {
2573 if (CalleeMap.empty() && CallerMap.empty()) {
2574 // The caller is potentially nullptr in the case where the call isn't in a
2575 // function. In this case, the getFunctionFeatureMap ensures we just get
2576 // the TU level setting (since it cannot be modified by 'target'..
2577 Ctx.getFunctionFeatureMap(CallerMap, Caller);
2578 Ctx.getFunctionFeatureMap(CalleeMap, Callee);
2579 }
2580}
2581
2582static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
2583 SourceLocation CallLoc,
2584 const llvm::StringMap<bool> &CallerMap,
2585 const llvm::StringMap<bool> &CalleeMap,
2586 QualType Ty, StringRef Feature,
2587 bool IsArgument) {
2588 bool CallerHasFeat = CallerMap.lookup(Feature);
2589 bool CalleeHasFeat = CalleeMap.lookup(Feature);
2590 if (!CallerHasFeat && !CalleeHasFeat)
2591 return Diag.Report(CallLoc, diag::warn_avx_calling_convention)
2592 << IsArgument << Ty << Feature;
2593
2594 // Mixing calling conventions here is very clearly an error.
2595 if (!CallerHasFeat || !CalleeHasFeat)
2596 return Diag.Report(CallLoc, diag::err_avx_calling_convention)
2597 << IsArgument << Ty << Feature;
2598
2599 // Else, both caller and callee have the required feature, so there is no need
2600 // to diagnose.
2601 return false;
2602}
2603
2604static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
2605 SourceLocation CallLoc,
2606 const llvm::StringMap<bool> &CallerMap,
2607 const llvm::StringMap<bool> &CalleeMap, QualType Ty,
2608 bool IsArgument) {
2609 uint64_t Size = Ctx.getTypeSize(Ty);
2610 if (Size > 256)
2611 return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty,
2612 "avx512f", IsArgument);
2613
2614 if (Size > 128)
2615 return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty, "avx",
2616 IsArgument);
2617
2618 return false;
2619}
2620
2621void X86_64TargetCodeGenInfo::checkFunctionCallABI(
2622 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
2623 const FunctionDecl *Callee, const CallArgList &Args) const {
2624 llvm::StringMap<bool> CallerMap;
2625 llvm::StringMap<bool> CalleeMap;
2626 unsigned ArgIndex = 0;
2627
2628 // We need to loop through the actual call arguments rather than the
2629 // function's parameters, in case this variadic.
2630 for (const CallArg &Arg : Args) {
2631 // The "avx" feature changes how vectors >128 in size are passed. "avx512f"
2632 // additionally changes how vectors >256 in size are passed. Like GCC, we
2633 // warn when a function is called with an argument where this will change.
2634 // Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
2635 // the caller and callee features are mismatched.
2636 // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
2637 // change its ABI with attribute-target after this call.
2638 if (Arg.getType()->isVectorType() &&
2639 CGM.getContext().getTypeSize(Arg.getType()) > 128) {
2640 initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2641 QualType Ty = Arg.getType();
2642 // The CallArg seems to have desugared the type already, so for clearer
2643 // diagnostics, replace it with the type in the FunctionDecl if possible.
2644 if (ArgIndex < Callee->getNumParams())
2645 Ty = Callee->getParamDecl(ArgIndex)->getType();
2646
2647 if (checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2648 CalleeMap, Ty, /*IsArgument*/ true))
2649 return;
2650 }
2651 ++ArgIndex;
2652 }
2653
2654 // Check return always, as we don't have a good way of knowing in codegen
2655 // whether this value is used, tail-called, etc.
2656 if (Callee->getReturnType()->isVectorType() &&
2657 CGM.getContext().getTypeSize(Callee->getReturnType()) > 128) {
2658 initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2659 checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2660 CalleeMap, Callee->getReturnType(),
2661 /*IsArgument*/ false);
2662 }
2663}
2664
2665static std::string qualifyWindowsLibrary(llvm::StringRef Lib) {
2666 // If the argument does not end in .lib, automatically add the suffix.
2667 // If the argument contains a space, enclose it in quotes.
2668 // This matches the behavior of MSVC.
2669 bool Quote = Lib.contains(' ');
2670 std::string ArgStr = Quote ? "\"" : "";
2671 ArgStr += Lib;
2672 if (!Lib.endswith_insensitive(".lib") && !Lib.endswith_insensitive(".a"))
2673 ArgStr += ".lib";
2674 ArgStr += Quote ? "\"" : "";
2675 return ArgStr;
2676}
2677
2678class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
2679public:
2680 WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2681 bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
2682 unsigned NumRegisterParameters)
2683 : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
2684 Win32StructABI, NumRegisterParameters, false) {}
2685
2686 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2687 CodeGen::CodeGenModule &CGM) const override;
2688
2689 void getDependentLibraryOption(llvm::StringRef Lib,
2690 llvm::SmallString<24> &Opt) const override {
2691 Opt = "/DEFAULTLIB:";
2692 Opt += qualifyWindowsLibrary(Lib);
2693 }
2694
2695 void getDetectMismatchOption(llvm::StringRef Name,
2696 llvm::StringRef Value,
2697 llvm::SmallString<32> &Opt) const override {
2698 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2699 }
2700};
2701
2702static void addStackProbeTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2704 if (llvm::Function *Fn = dyn_cast_or_null<llvm::Function>(GV)) {
2705
2706 if (CGM.getCodeGenOpts().StackProbeSize != 4096)
2707 Fn->addFnAttr("stack-probe-size",
2708 llvm::utostr(CGM.getCodeGenOpts().StackProbeSize));
2709 if (CGM.getCodeGenOpts().NoStackArgProbe)
2710 Fn->addFnAttr("no-stack-arg-probe");
2711 }
2712}
2713
2714void WinX86_32TargetCodeGenInfo::setTargetAttributes(
2715 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2716 X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2717 if (GV->isDeclaration())
2718 return;
2719 addStackProbeTargetAttributes(D, GV, CGM);
2720}
2721
2722class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2723public:
2724 WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2725 X86AVXABILevel AVXLevel)
2726 : TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(CGT, AVXLevel)) {
2727 SwiftInfo =
2728 std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/true);
2729 }
2730
2731 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2732 CodeGen::CodeGenModule &CGM) const override;
2733
2734 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2735 return 7;
2736 }
2737
2738 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2739 llvm::Value *Address) const override {
2740 llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2741
2742 // 0-15 are the 16 integer registers.
2743 // 16 is %rip.
2744 AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2745 return false;
2746 }
2747
2748 void getDependentLibraryOption(llvm::StringRef Lib,
2749 llvm::SmallString<24> &Opt) const override {
2750 Opt = "/DEFAULTLIB:";
2751 Opt += qualifyWindowsLibrary(Lib);
2752 }
2753
2754 void getDetectMismatchOption(llvm::StringRef Name,
2755 llvm::StringRef Value,
2756 llvm::SmallString<32> &Opt) const override {
2757 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2758 }
2759};
2760
2761void WinX86_64TargetCodeGenInfo::setTargetAttributes(
2762 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2764 if (GV->isDeclaration())
2765 return;
2766 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2767 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2768 llvm::Function *Fn = cast<llvm::Function>(GV);
2769 Fn->addFnAttr("stackrealign");
2770 }
2771
2772 addX86InterruptAttrs(FD, GV, CGM);
2773 }
2774
2775 addStackProbeTargetAttributes(D, GV, CGM);
2776}
2777}
2778
2779void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
2780 Class &Hi) const {
2781 // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
2782 //
2783 // (a) If one of the classes is Memory, the whole argument is passed in
2784 // memory.
2785 //
2786 // (b) If X87UP is not preceded by X87, the whole argument is passed in
2787 // memory.
2788 //
2789 // (c) If the size of the aggregate exceeds two eightbytes and the first
2790 // eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
2791 // argument is passed in memory. NOTE: This is necessary to keep the
2792 // ABI working for processors that don't support the __m256 type.
2793 //
2794 // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
2795 //
2796 // Some of these are enforced by the merging logic. Others can arise
2797 // only with unions; for example:
2798 // union { _Complex double; unsigned; }
2799 //
2800 // Note that clauses (b) and (c) were added in 0.98.
2801 //
2802 if (Hi == Memory)
2803 Lo = Memory;
2804 if (Hi == X87Up && Lo != X87 && honorsRevision0_98())
2805 Lo = Memory;
2806 if (AggregateSize > 128 && (Lo != SSE || Hi != SSEUp))
2807 Lo = Memory;
2808 if (Hi == SSEUp && Lo != SSE)
2809 Hi = SSE;
2810}
2811
2812X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
2813 // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
2814 // classified recursively so that always two fields are
2815 // considered. The resulting class is calculated according to
2816 // the classes of the fields in the eightbyte:
2817 //
2818 // (a) If both classes are equal, this is the resulting class.
2819 //
2820 // (b) If one of the classes is NO_CLASS, the resulting class is
2821 // the other class.
2822 //
2823 // (c) If one of the classes is MEMORY, the result is the MEMORY
2824 // class.
2825 //
2826 // (d) If one of the classes is INTEGER, the result is the
2827 // INTEGER.
2828 //
2829 // (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
2830 // MEMORY is used as class.
2831 //
2832 // (f) Otherwise class SSE is used.
2833
2834 // Accum should never be memory (we should have returned) or
2835 // ComplexX87 (because this cannot be passed in a structure).
2836 assert((Accum != Memory && Accum != ComplexX87) &&
2837 "Invalid accumulated classification during merge.");
2838 if (Accum == Field || Field == NoClass)
2839 return Accum;
2840 if (Field == Memory)
2841 return Memory;
2842 if (Accum == NoClass)
2843 return Field;
2844 if (Accum == Integer || Field == Integer)
2845 return Integer;
2846 if (Field == X87 || Field == X87Up || Field == ComplexX87 ||
2847 Accum == X87 || Accum == X87Up)
2848 return Memory;
2849 return SSE;
2850}
2851
2852void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
2853 Class &Hi, bool isNamedArg, bool IsRegCall) const {
2854 // FIXME: This code can be simplified by introducing a simple value class for
2855 // Class pairs with appropriate constructor methods for the various
2856 // situations.
2857
2858 // FIXME: Some of the split computations are wrong; unaligned vectors
2859 // shouldn't be passed in registers for example, so there is no chance they
2860 // can straddle an eightbyte. Verify & simplify.
2861
2862 Lo = Hi = NoClass;
2863
2864 Class &Current = OffsetBase < 64 ? Lo : Hi;
2865 Current = Memory;
2866
2867 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
2868 BuiltinType::Kind k = BT->getKind();
2869
2870 if (k == BuiltinType::Void) {
2871 Current = NoClass;
2872 } else if (k == BuiltinType::Int128 || k == BuiltinType::UInt128) {
2873 Lo = Integer;
2874 Hi = Integer;
2875 } else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) {
2876 Current = Integer;
2877 } else if (k == BuiltinType::Float || k == BuiltinType::Double ||
2878 k == BuiltinType::Float16 || k == BuiltinType::BFloat16) {
2879 Current = SSE;
2880 } else if (k == BuiltinType::LongDouble) {
2881 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2882 if (LDF == &llvm::APFloat::IEEEquad()) {
2883 Lo = SSE;
2884 Hi = SSEUp;
2885 } else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
2886 Lo = X87;
2887 Hi = X87Up;
2888 } else if (LDF == &llvm::APFloat::IEEEdouble()) {
2889 Current = SSE;
2890 } else
2891 llvm_unreachable("unexpected long double representation!");
2892 }
2893 // FIXME: _Decimal32 and _Decimal64 are SSE.
2894 // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
2895 return;
2896 }
2897
2898 if (const EnumType *ET = Ty->getAs<EnumType>()) {
2899 // Classify the underlying integer type.
2900 classify(ET->getDecl()->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
2901 return;
2902 }
2903
2904 if (Ty->hasPointerRepresentation()) {
2905 Current = Integer;
2906 return;
2907 }
2908
2909 if (Ty->isMemberPointerType()) {
2910 if (Ty->isMemberFunctionPointerType()) {
2911 if (Has64BitPointers) {
2912 // If Has64BitPointers, this is an {i64, i64}, so classify both
2913 // Lo and Hi now.
2914 Lo = Hi = Integer;
2915 } else {
2916 // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
2917 // straddles an eightbyte boundary, Hi should be classified as well.
2918 uint64_t EB_FuncPtr = (OffsetBase) / 64;
2919 uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
2920 if (EB_FuncPtr != EB_ThisAdj) {
2921 Lo = Hi = Integer;
2922 } else {
2923 Current = Integer;
2924 }
2925 }
2926 } else {
2927 Current = Integer;
2928 }
2929 return;
2930 }
2931
2932 if (const VectorType *VT = Ty->getAs<VectorType>()) {
2933 uint64_t Size = getContext().getTypeSize(VT);
2934 if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
2935 // gcc passes the following as integer:
2936 // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
2937 // 2 bytes - <2 x char>, <1 x short>
2938 // 1 byte - <1 x char>
2939 Current = Integer;
2940
2941 // If this type crosses an eightbyte boundary, it should be
2942 // split.
2943 uint64_t EB_Lo = (OffsetBase) / 64;
2944 uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
2945 if (EB_Lo != EB_Hi)
2946 Hi = Lo;
2947 } else if (Size == 64) {
2948 QualType ElementType = VT->getElementType();
2949
2950 // gcc passes <1 x double> in memory. :(
2951 if (ElementType->isSpecificBuiltinType(BuiltinType::Double))
2952 return;
2953
2954 // gcc passes <1 x long long> as SSE but clang used to unconditionally
2955 // pass them as integer. For platforms where clang is the de facto
2956 // platform compiler, we must continue to use integer.
2957 if (!classifyIntegerMMXAsSSE() &&
2958 (ElementType->isSpecificBuiltinType(BuiltinType::LongLong) ||
2959 ElementType->isSpecificBuiltinType(BuiltinType::ULongLong) ||
2960 ElementType->isSpecificBuiltinType(BuiltinType::Long) ||
2961 ElementType->isSpecificBuiltinType(BuiltinType::ULong)))
2962 Current = Integer;
2963 else
2964 Current = SSE;
2965
2966 // If this type crosses an eightbyte boundary, it should be
2967 // split.
2968 if (OffsetBase && OffsetBase != 64)
2969 Hi = Lo;
2970 } else if (Size == 128 ||
2971 (isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
2972 QualType ElementType = VT->getElementType();
2973
2974 // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
2975 if (passInt128VectorsInMem() && Size != 128 &&
2976 (ElementType->isSpecificBuiltinType(BuiltinType::Int128) ||
2977 ElementType->isSpecificBuiltinType(BuiltinType::UInt128)))
2978 return;
2979
2980 // Arguments of 256-bits are split into four eightbyte chunks. The
2981 // least significant one belongs to class SSE and all the others to class
2982 // SSEUP. The original Lo and Hi design considers that types can't be
2983 // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
2984 // This design isn't correct for 256-bits, but since there're no cases
2985 // where the upper parts would need to be inspected, avoid adding
2986 // complexity and just consider Hi to match the 64-256 part.
2987 //
2988 // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
2989 // registers if they are "named", i.e. not part of the "..." of a
2990 // variadic function.
2991 //
2992 // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
2993 // split into eight eightbyte chunks, one SSE and seven SSEUP.
2994 Lo = SSE;
2995 Hi = SSEUp;
2996 }
2997 return;
2998 }
2999
3000 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
3001 QualType ET = getContext().getCanonicalType(CT->getElementType());
3002
3003 uint64_t Size = getContext().getTypeSize(Ty);
3004 if (ET->isIntegralOrEnumerationType()) {
3005 if (Size <= 64)
3006 Current = Integer;
3007 else if (Size <= 128)
3008 Lo = Hi = Integer;
3009 } else if (ET->isFloat16Type() || ET == getContext().FloatTy ||
3010 ET->isBFloat16Type()) {
3011 Current = SSE;
3012 } else if (ET == getContext().DoubleTy) {
3013 Lo = Hi = SSE;
3014 } else if (ET == getContext().LongDoubleTy) {
3015 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
3016 if (LDF == &llvm::APFloat::IEEEquad())
3017 Current = Memory;
3018 else if (LDF == &llvm::APFloat::x87DoubleExtended())
3019 Current = ComplexX87;
3020 else if (LDF == &llvm::APFloat::IEEEdouble())
3021 Lo = Hi = SSE;
3022 else
3023 llvm_unreachable("unexpected long double representation!");
3024 }
3025
3026 // If this complex type crosses an eightbyte boundary then it
3027 // should be split.
3028 uint64_t EB_Real = (OffsetBase) / 64;
3029 uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
3030 if (Hi == NoClass && EB_Real != EB_Imag)
3031 Hi = Lo;
3032
3033 return;
3034 }
3035
3036 if (const auto *EITy = Ty->getAs<BitIntType>()) {
3037 if (EITy->getNumBits() <= 64)
3038 Current = Integer;
3039 else if (EITy->getNumBits() <= 128)
3040 Lo = Hi = Integer;
3041 // Larger values need to get passed in memory.
3042 return;
3043 }
3044
3045 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
3046 // Arrays are treated like structures.
3047
3048 uint64_t Size = getContext().getTypeSize(Ty);
3049
3050 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3051 // than eight eightbytes, ..., it has class MEMORY.
3052 // regcall ABI doesn't have limitation to an object. The only limitation
3053 // is the free registers, which will be checked in computeInfo.
3054 if (!IsRegCall && Size > 512)
3055 return;
3056
3057 // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
3058 // fields, it has class MEMORY.
3059 //
3060 // Only need to check alignment of array base.
3061 if (OffsetBase % getContext().getTypeAlign(AT->getElementType()))
3062 return;
3063
3064 // Otherwise implement simplified merge. We could be smarter about
3065 // this, but it isn't worth it and would be harder to verify.
3066 Current = NoClass;
3067 uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
3068 uint64_t ArraySize = AT->getSize().getZExtValue();
3069
3070 // The only case a 256-bit wide vector could be used is when the array
3071 // contains a single 256-bit element. Since Lo and Hi logic isn't extended
3072 // to work for sizes wider than 128, early check and fallback to memory.
3073 //
3074 if (Size > 128 &&
3075 (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
3076 return;
3077
3078 for (uint64_t i=0, Offset=OffsetBase; i<ArraySize; ++i, Offset += EltSize) {
3079 Class FieldLo, FieldHi;
3080 classify(AT->getElementType(), Offset, FieldLo, FieldHi, isNamedArg);
3081 Lo = merge(Lo, FieldLo);
3082 Hi = merge(Hi, FieldHi);
3083 if (Lo == Memory || Hi == Memory)
3084 break;
3085 }
3086
3087 postMerge(Size, Lo, Hi);
3088 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
3089 return;
3090 }
3091
3092 if (const RecordType *RT = Ty->getAs<RecordType>()) {
3093 uint64_t Size = getContext().getTypeSize(Ty);
3094
3095 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3096 // than eight eightbytes, ..., it has class MEMORY.
3097 if (Size > 512)
3098 return;
3099
3100 // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
3101 // copy constructor or a non-trivial destructor, it is passed by invisible
3102 // reference.
3103 if (getRecordArgABI(RT, getCXXABI()))
3104 return;
3105
3106 const RecordDecl *RD = RT->getDecl();
3107
3108 // Assume variable sized types are passed in memory.
3109 if (RD->hasFlexibleArrayMember())
3110 return;
3111
3112 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
3113
3114 // Reset Lo class, this will be recomputed.
3115 Current = NoClass;
3116
3117 // If this is a C++ record, classify the bases first.
3118 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3119 for (const auto &I : CXXRD->bases()) {
3120 assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3121 "Unexpected base class!");
3122 const auto *Base =
3123 cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3124
3125 // Classify this field.
3126 //
3127 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
3128 // single eightbyte, each is classified separately. Each eightbyte gets
3129 // initialized to class NO_CLASS.
3130 Class FieldLo, FieldHi;
3132 OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
3133 classify(I.getType(), Offset, FieldLo, FieldHi, isNamedArg);
3134 Lo = merge(Lo, FieldLo);
3135 Hi = merge(Hi, FieldHi);
3136 if (Lo == Memory || Hi == Memory) {
3137 postMerge(Size, Lo, Hi);
3138 return;
3139 }
3140 }
3141 }
3142
3143 // Classify the fields one at a time, merging the results.
3144 unsigned idx = 0;
3145 bool UseClang11Compat = getContext().getLangOpts().getClangABICompat() <=
3147 getContext().getTargetInfo().getTriple().isPS();
3148 bool IsUnion = RT->isUnionType() && !UseClang11Compat;
3149
3150 for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3151 i != e; ++i, ++idx) {
3152 uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3153 bool BitField = i->isBitField();
3154
3155 // Ignore padding bit-fields.
3156 if (BitField && i->isUnnamedBitfield())
3157 continue;
3158
3159 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
3160 // eight eightbytes, or it contains unaligned fields, it has class MEMORY.
3161 //
3162 // The only case a 256-bit or a 512-bit wide vector could be used is when
3163 // the struct contains a single 256-bit or 512-bit element. Early check
3164 // and fallback to memory.
3165 //
3166 // FIXME: Extended the Lo and Hi logic properly to work for size wider
3167 // than 128.
3168 if (Size > 128 &&
3169 ((!IsUnion && Size != getContext().getTypeSize(i->getType())) ||
3170 Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
3171 Lo = Memory;
3172 postMerge(Size, Lo, Hi);
3173 return;
3174 }
3175 // Note, skip this test for bit-fields, see below.
3176 if (!BitField && Offset % getContext().getTypeAlign(i->getType())) {
3177 Lo = Memory;
3178 postMerge(Size, Lo, Hi);
3179 return;
3180 }
3181
3182 // Classify this field.
3183 //
3184 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
3185 // exceeds a single eightbyte, each is classified
3186 // separately. Each eightbyte gets initialized to class
3187 // NO_CLASS.
3188 Class FieldLo, FieldHi;
3189
3190 // Bit-fields require special handling, they do not force the
3191 // structure to be passed in memory even if unaligned, and
3192 // therefore they can straddle an eightbyte.
3193 if (BitField) {
3194 assert(!i->isUnnamedBitfield());
3195 uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3196 uint64_t Size = i->getBitWidthValue(getContext());
3197
3198 uint64_t EB_Lo = Offset / 64;
3199 uint64_t EB_Hi = (Offset + Size - 1) / 64;
3200
3201 if (EB_Lo) {
3202 assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
3203 FieldLo = NoClass;
3204 FieldHi = Integer;
3205 } else {
3206 FieldLo = Integer;
3207 FieldHi = EB_Hi ? Integer : NoClass;
3208 }
3209 } else
3210 classify(i->getType(), Offset, FieldLo, FieldHi, isNamedArg);
3211 Lo = merge(Lo, FieldLo);
3212 Hi = merge(Hi, FieldHi);
3213 if (Lo == Memory || Hi == Memory)
3214 break;
3215 }
3216
3217 postMerge(Size, Lo, Hi);
3218 }
3219}
3220
3221ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
3222 // If this is a scalar LLVM value then assume LLVM will pass it in the right
3223 // place naturally.
3224 if (!isAggregateTypeForABI(Ty)) {
3225 // Treat an enum type as its underlying type.
3226 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3227 Ty = EnumTy->getDecl()->getIntegerType();
3228
3229 if (Ty->isBitIntType())
3230 return getNaturalAlignIndirect(Ty);
3231
3232 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
3234 }
3235
3236 return getNaturalAlignIndirect(Ty);
3237}
3238
3239bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
3240 if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
3241 uint64_t Size = getContext().getTypeSize(VecTy);
3242 unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
3243 if (Size <= 64 || Size > LargestVector)
3244 return true;
3245 QualType EltTy = VecTy->getElementType();
3246 if (passInt128VectorsInMem() &&
3247 (EltTy->isSpecificBuiltinType(BuiltinType::Int128) ||
3248 EltTy->isSpecificBuiltinType(BuiltinType::UInt128)))
3249 return true;
3250 }
3251
3252 return false;
3253}
3254
3255ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
3256 unsigned freeIntRegs) const {
3257 // If this is a scalar LLVM value then assume LLVM will pass it in the right
3258 // place naturally.
3259 //
3260 // This assumption is optimistic, as there could be free registers available
3261 // when we need to pass this argument in memory, and LLVM could try to pass
3262 // the argument in the free register. This does not seem to happen currently,
3263 // but this code would be much safer if we could mark the argument with
3264 // 'onstack'. See PR12193.
3265 if (!isAggregateTypeForABI(Ty) && !IsIllegalVectorType(Ty) &&
3266 !Ty->isBitIntType()) {
3267 // Treat an enum type as its underlying type.
3268 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3269 Ty = EnumTy->getDecl()->getIntegerType();
3270
3271 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
3273 }
3274
3275 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
3276 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
3277
3278 // Compute the byval alignment. We specify the alignment of the byval in all
3279 // cases so that the mid-level optimizer knows the alignment of the byval.
3280 unsigned Align = std::max(getContext().getTypeAlign(Ty) / 8, 8U);
3281
3282 // Attempt to avoid passing indirect results using byval when possible. This
3283 // is important for good codegen.
3284 //
3285 // We do this by coercing the value into a scalar type which the backend can
3286 // handle naturally (i.e., without using byval).
3287 //
3288 // For simplicity, we currently only do this when we have exhausted all of the
3289 // free integer registers. Doing this when there are free integer registers
3290 // would require more care, as we would have to ensure that the coerced value
3291 // did not claim the unused register. That would require either reording the
3292 // arguments to the function (so that any subsequent inreg values came first),
3293 // or only doing this optimization when there were no following arguments that
3294 // might be inreg.
3295 //
3296 // We currently expect it to be rare (particularly in well written code) for
3297 // arguments to be passed on the stack when there are still free integer
3298 // registers available (this would typically imply large structs being passed
3299 // by value), so this seems like a fair tradeoff for now.
3300 //
3301 // We can revisit this if the backend grows support for 'onstack' parameter
3302 // attributes. See PR12193.
3303 if (freeIntRegs == 0) {
3304 uint64_t Size = getContext().getTypeSize(Ty);
3305
3306 // If this type fits in an eightbyte, coerce it into the matching integral
3307 // type, which will end up on the stack (with alignment 8).
3308 if (Align == 8 && Size <= 64)
3309 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
3310 Size));
3311 }
3312
3314}
3315
3316/// The ABI specifies that a value should be passed in a full vector XMM/YMM
3317/// register. Pick an LLVM IR type that will be passed as a vector register.
3318llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
3319 // Wrapper structs/arrays that only contain vectors are passed just like
3320 // vectors; strip them off if present.
3321 if (const Type *InnerTy = isSingleElementStruct(Ty, getContext()))
3322 Ty = QualType(InnerTy, 0);
3323
3324 llvm::Type *IRType = CGT.ConvertType(Ty);
3325 if (isa<llvm::VectorType>(IRType)) {
3326 // Don't pass vXi128 vectors in their native type, the backend can't
3327 // legalize them.
3328 if (passInt128VectorsInMem() &&
3329 cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy(128)) {
3330 // Use a vXi64 vector.
3331 uint64_t Size = getContext().getTypeSize(Ty);
3332 return llvm::FixedVectorType::get(llvm::Type::getInt64Ty(getVMContext()),
3333 Size / 64);
3334 }
3335
3336 return IRType;
3337 }
3338
3339 if (IRType->getTypeID() == llvm::Type::FP128TyID)
3340 return IRType;
3341
3342 // We couldn't find the preferred IR vector type for 'Ty'.
3343 uint64_t Size = getContext().getTypeSize(Ty);
3344 assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
3345
3346
3347 // Return a LLVM IR vector type based on the size of 'Ty'.
3348 return llvm::FixedVectorType::get(llvm::Type::getDoubleTy(getVMContext()),
3349 Size / 64);
3350}
3351
3352/// BitsContainNoUserData - Return true if the specified [start,end) bit range
3353/// is known to either be off the end of the specified type or being in
3354/// alignment padding. The user type specified is known to be at most 128 bits
3355/// in size, and have passed through X86_64ABIInfo::classify with a successful
3356/// classification that put one of the two halves in the INTEGER class.
3357///
3358/// It is conservatively correct to return false.
3359static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
3360 unsigned EndBit, ASTContext &Context) {
3361 // If the bytes being queried are off the end of the type, there is no user
3362 // data hiding here. This handles analysis of builtins, vectors and other
3363 // types that don't contain interesting padding.
3364 unsigned TySize = (unsigned)Context.getTypeSize(Ty);
3365 if (TySize <= StartBit)
3366 return true;
3367
3368 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
3369 unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());
3370 unsigned NumElts = (unsigned)AT->getSize().getZExtValue();
3371
3372 // Check each element to see if the element overlaps with the queried range.
3373 for (unsigned i = 0; i != NumElts; ++i) {
3374 // If the element is after the span we care about, then we're done..
3375 unsigned EltOffset = i*EltSize;
3376 if (EltOffset >= EndBit) break;
3377
3378 unsigned EltStart = EltOffset < StartBit ? StartBit-EltOffset :0;
3379 if (!BitsContainNoUserData(AT->getElementType(), EltStart,
3380 EndBit-EltOffset, Context))
3381 return false;
3382 }
3383 // If it overlaps no elements, then it is safe to process as padding.
3384 return true;
3385 }
3386
3387 if (const RecordType *RT = Ty->getAs<RecordType>()) {
3388 const RecordDecl *RD = RT->getDecl();
3389 const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
3390
3391 // If this is a C++ record, check the bases first.
3392 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3393 for (const auto &I : CXXRD->bases()) {
3394 assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3395 "Unexpected base class!");
3396 const auto *Base =
3397 cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3398
3399 // If the base is after the span we care about, ignore it.
3400 unsigned BaseOffset = Context.toBits(Layout.getBaseClassOffset(Base));
3401 if (BaseOffset >= EndBit) continue;
3402
3403 unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :0;
3404 if (!BitsContainNoUserData(I.getType(), BaseStart,
3405 EndBit-BaseOffset, Context))
3406 return false;
3407 }
3408 }
3409
3410 // Verify that no field has data that overlaps the region of interest. Yes
3411 // this could be sped up a lot by being smarter about queried fields,
3412 // however we're only looking at structs up to 16 bytes, so we don't care
3413 // much.
3414 unsigned idx = 0;
3415 for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3416 i != e; ++i, ++idx) {
3417 unsigned FieldOffset = (unsigned)Layout.getFieldOffset(idx);
3418
3419 // If we found a field after the region we care about, then we're done.
3420 if (FieldOffset >= EndBit) break;
3421
3422 unsigned FieldStart = FieldOffset < StartBit ? StartBit-FieldOffset :0;
3423 if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit-FieldOffset,
3424 Context))
3425 return false;
3426 }
3427
3428 // If nothing in this record overlapped the area of interest, then we're
3429 // clean.
3430 return true;
3431 }
3432
3433 return false;
3434}
3435
3436/// getFPTypeAtOffset - Return a floating point type at the specified offset.
3437static llvm::Type *getFPTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3438 const llvm::DataLayout &TD) {
3439 if (IROffset == 0 && IRType->isFloatingPointTy())
3440 return IRType;
3441
3442 // If this is a struct, recurse into the field at the specified offset.
3443 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3444 if (!STy->getNumContainedTypes())
3445 return nullptr;
3446
3447 const llvm::StructLayout *SL = TD.getStructLayout(STy);
3448 unsigned Elt = SL->getElementContainingOffset(IROffset);
3449 IROffset -= SL->getElementOffset(Elt);
3450 return getFPTypeAtOffset(STy->getElementType(Elt), IROffset, TD);
3451 }
3452
3453 // If this is an array, recurse into the field at the specified offset.
3454 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3455 llvm::Type *EltTy = ATy->getElementType();
3456 unsigned EltSize = TD.getTypeAllocSize(EltTy);
3457 IROffset -= IROffset / EltSize * EltSize;
3458 return getFPTypeAtOffset(EltTy, IROffset, TD);
3459 }
3460
3461 return nullptr;
3462}
3463
3464/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
3465/// low 8 bytes of an XMM register, corresponding to the SSE class.
3466llvm::Type *X86_64ABIInfo::
3467GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3468 QualType SourceTy, unsigned SourceOffset) const {
3469 const llvm::DataLayout &TD = getDataLayout();
3470 unsigned SourceSize =
3471 (unsigned)getContext().getTypeSize(SourceTy) / 8 - SourceOffset;
3472 llvm::Type *T0 = getFPTypeAtOffset(IRType, IROffset, TD);
3473 if (!T0 || T0->isDoubleTy())
3474 return llvm::Type::getDoubleTy(getVMContext());
3475
3476 // Get the adjacent FP type.
3477 llvm::Type *T1 = nullptr;
3478 unsigned T0Size = TD.getTypeAllocSize(T0);
3479 if (SourceSize > T0Size)
3480 T1 = getFPTypeAtOffset(IRType, IROffset + T0Size, TD);
3481 if (T1 == nullptr) {
3482 // Check if IRType is a half/bfloat + float. float type will be in IROffset+4 due
3483 // to its alignment.
3484 if (T0->is16bitFPTy() && SourceSize > 4)
3485 T1 = getFPTypeAtOffset(IRType, IROffset + 4, TD);
3486 // If we can't get a second FP type, return a simple half or float.
3487 // avx512fp16-abi.c:pr51813_2 shows it works to return float for
3488 // {float, i8} too.
3489 if (T1 == nullptr)
3490 return T0;
3491 }
3492
3493 if (T0->isFloatTy() && T1->isFloatTy())
3494 return llvm::FixedVectorType::get(T0, 2);
3495
3496 if (T0->is16bitFPTy() && T1->is16bitFPTy()) {
3497 llvm::Type *T2 = nullptr;
3498 if (SourceSize > 4)
3499 T2 = getFPTypeAtOffset(IRType, IROffset + 4, TD);
3500 if (T2 == nullptr)
3501 return llvm::FixedVectorType::get(T0, 2);
3502 return llvm::FixedVectorType::get(T0, 4);
3503 }
3504
3505 if (T0->is16bitFPTy() || T1->is16bitFPTy())
3506 return llvm::FixedVectorType::get(llvm::Type::getHalfTy(getVMContext()), 4);
3507
3508 return llvm::Type::getDoubleTy(getVMContext());
3509}
3510
3511
3512/// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
3513/// an 8-byte GPR. This means that we either have a scalar or we are talking
3514/// about the high or low part of an up-to-16-byte struct. This routine picks
3515/// the best LLVM IR type to represent this, which may be i64 or may be anything
3516/// else that the backend will pass in a GPR that works better (e.g. i8, %foo*,
3517/// etc).
3518///
3519/// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
3520/// the source type. IROffset is an offset in bytes into the LLVM IR type that
3521/// the 8-byte value references. PrefType may be null.
3522///
3523/// SourceTy is the source-level type for the entire argument. SourceOffset is
3524/// an offset into this that we're processing (which is always either 0 or 8).
3525///
3526llvm::Type *X86_64ABIInfo::
3527GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3528 QualType SourceTy, unsigned SourceOffset) const {
3529 // If we're dealing with an un-offset LLVM IR type, then it means that we're
3530 // returning an 8-byte unit starting with it. See if we can safely use it.
3531 if (IROffset == 0) {
3532 // Pointers and int64's always fill the 8-byte unit.
3533 if ((isa<llvm::PointerType>(IRType) && Has64BitPointers) ||
3534 IRType->isIntegerTy(64))
3535 return IRType;
3536
3537 // If we have a 1/2/4-byte integer, we can use it only if the rest of the
3538 // goodness in the source type is just tail padding. This is allowed to
3539 // kick in for struct {double,int} on the int, but not on
3540 // struct{double,int,int} because we wouldn't return the second int. We
3541 // have to do this analysis on the source type because we can't depend on
3542 // unions being lowered a specific way etc.
3543 if (IRType->isIntegerTy(8) || IRType->isIntegerTy(16) ||
3544 IRType->isIntegerTy(32) ||
3545 (isa<llvm::PointerType>(IRType) && !Has64BitPointers)) {
3546 unsigned BitWidth = isa<llvm::PointerType>(IRType) ? 32 :
3547 cast<llvm::IntegerType>(IRType)->getBitWidth();
3548
3549 if (BitsContainNoUserData(SourceTy, SourceOffset*8+BitWidth,
3550 SourceOffset*8+64, getContext()))
3551 return IRType;
3552 }
3553 }
3554
3555 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3556 // If this is a struct, recurse into the field at the specified offset.
3557 const llvm::StructLayout *SL = getDataLayout().getStructLayout(STy);
3558 if (IROffset < SL->getSizeInBytes()) {
3559 unsigned FieldIdx = SL->getElementContainingOffset(IROffset);
3560 IROffset -= SL->getElementOffset(FieldIdx);
3561
3562 return GetINTEGERTypeAtOffset(STy->getElementType(FieldIdx), IROffset,
3563 SourceTy, SourceOffset);
3564 }
3565 }
3566
3567 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3568 llvm::Type *EltTy = ATy->getElementType();
3569 unsigned EltSize = getDataLayout().getTypeAllocSize(EltTy);
3570 unsigned EltOffset = IROffset/EltSize*EltSize;
3571 return GetINTEGERTypeAtOffset(EltTy, IROffset-EltOffset, SourceTy,
3572 SourceOffset);
3573 }
3574
3575 // Okay, we don't have any better idea of what to pass, so we pass this in an
3576 // integer register that isn't too big to fit the rest of the struct.
3577 unsigned TySizeInBytes =
3578 (unsigned)getContext().getTypeSizeInChars(SourceTy).getQuantity();
3579
3580 assert(TySizeInBytes != SourceOffset && "Empty field?");
3581
3582 // It is always safe to classify this as an integer type up to i64 that
3583 // isn't larger than the structure.
3584 return llvm::IntegerType::get(getVMContext(),
3585 std::min(TySizeInBytes-SourceOffset, 8U)*8);
3586}
3587
3588
3589/// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
3590/// be used as elements of a two register pair to pass or return, return a
3591/// first class aggregate to represent them. For example, if the low part of
3592/// a by-value argument should be passed as i32* and the high part as float,
3593/// return {i32*, float}.
3594static llvm::Type *
3595GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
3596 const llvm::DataLayout &TD) {
3597 // In order to correctly satisfy the ABI, we need to the high part to start
3598 // at offset 8. If the high and low parts we inferred are both 4-byte types
3599 // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
3600 // the second element at offset 8. Check for this:
3601 unsigned LoSize = (unsigned)TD.getTypeAllocSize(Lo);
3602 llvm::Align HiAlign = TD.getABITypeAlign(Hi);
3603 unsigned HiStart = llvm::alignTo(LoSize, HiAlign);
3604 assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
3605
3606 // To handle this, we have to increase the size of the low part so that the
3607 // second element will start at an 8 byte offset. We can't increase the size
3608 // of the second element because it might make us access off the end of the
3609 // struct.
3610 if (HiStart != 8) {
3611 // There are usually two sorts of types the ABI generation code can produce
3612 // for the low part of a pair that aren't 8 bytes in size: half, float or
3613 // i8/i16/i32. This can also include pointers when they are 32-bit (X32 and
3614 // NaCl).
3615 // Promote these to a larger type.
3616 if (Lo->isHalfTy() || Lo->isFloatTy())
3617 Lo = llvm::Type::getDoubleTy(Lo->getContext());
3618 else {
3619 assert((Lo->isIntegerTy() || Lo->isPointerTy())
3620 && "Invalid/unknown lo type");
3621 Lo = llvm::Type::getInt64Ty(Lo->getContext());
3622 }
3623 }
3624
3625 llvm::StructType *Result = llvm::StructType::get(Lo, Hi);
3626
3627 // Verify that the second element is at an 8-byte offset.
3628 assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
3629 "Invalid x86-64 argument pair!");
3630 return Result;
3631}
3632
3633ABIArgInfo X86_64ABIInfo::
3634classifyReturnType(QualType RetTy) const {
3635 // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
3636 // classification algorithm.
3637 X86_64ABIInfo::Class Lo, Hi;
3638 classify(RetTy, 0, Lo, Hi, /*isNamedArg*/ true);
3639
3640 // Check some invariants.
3641 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3642 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3643
3644 llvm::Type *ResType = nullptr;
3645 switch (Lo) {
3646 case NoClass:
3647 if (Hi == NoClass)
3648 return ABIArgInfo::getIgnore();
3649 // If the low part is just padding, it takes no register, leave ResType
3650 // null.
3651 assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3652 "Unknown missing lo part");
3653 break;
3654
3655 case SSEUp:
3656 case X87Up:
3657 llvm_unreachable("Invalid classification for lo word.");
3658
3659 // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
3660 // hidden argument.
3661 case Memory:
3662 return getIndirectReturnResult(RetTy);
3663
3664 // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
3665 // available register of the sequence %rax, %rdx is used.
3666 case Integer:
3667 ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3668
3669 // If we have a sign or zero extended integer, make sure to return Extend
3670 // so that the parameter gets the right LLVM IR attributes.
3671 if (Hi == NoClass && isa<llvm::IntegerType>(ResType)) {
3672 // Treat an enum type as its underlying type.
3673 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
3674 RetTy = EnumTy->getDecl()->getIntegerType();
3675
3676 if (RetTy->isIntegralOrEnumerationType() &&
3677 isPromotableIntegerTypeForABI(RetTy))
3678 return ABIArgInfo::getExtend(RetTy);
3679 }
3680 break;
3681
3682 // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
3683 // available SSE register of the sequence %xmm0, %xmm1 is used.
3684 case SSE:
3685 ResType = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3686 break;
3687
3688 // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
3689 // returned on the X87 stack in %st0 as 80-bit x87 number.
3690 case X87:
3691 ResType = llvm::Type::getX86_FP80Ty(getVMContext());
3692 break;
3693
3694 // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
3695 // part of the value is returned in %st0 and the imaginary part in
3696 // %st1.
3697 case ComplexX87:
3698 assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
3699 ResType = llvm::StructType::get(llvm::Type::getX86_FP80Ty(getVMContext()),
3700 llvm::Type::getX86_FP80Ty(getVMContext()));
3701 break;
3702 }
3703
3704 llvm::Type *HighPart = nullptr;
3705 switch (Hi) {
3706 // Memory was handled previously and X87 should
3707 // never occur as a hi class.
3708 case Memory:
3709 case X87:
3710 llvm_unreachable("Invalid classification for hi word.");
3711
3712 case ComplexX87: // Previously handled.
3713 case NoClass:
3714 break;
3715
3716 case Integer:
3717 HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3718 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
3719 return ABIArgInfo::getDirect(HighPart, 8);
3720 break;
3721 case SSE:
3722 HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3723 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
3724 return ABIArgInfo::getDirect(HighPart, 8);
3725 break;
3726
3727 // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
3728 // is passed in the next available eightbyte chunk if the last used
3729 // vector register.
3730 //
3731 // SSEUP should always be preceded by SSE, just widen.
3732 case SSEUp:
3733 assert(Lo == SSE && "Unexpected SSEUp classification.");
3734 ResType = GetByteVectorType(RetTy);
3735 break;
3736
3737 // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
3738 // returned together with the previous X87 value in %st0.
3739 case X87Up:
3740 // If X87Up is preceded by X87, we don't need to do
3741 // anything. However, in some cases with unions it may not be
3742 // preceded by X87. In such situations we follow gcc and pass the
3743 // extra bits in an SSE reg.
3744 if (Lo != X87) {
3745 HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3746 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
3747 return ABIArgInfo::getDirect(HighPart, 8);
3748 }
3749 break;
3750 }
3751
3752 // If a high part was specified, merge it together with the low part. It is
3753 // known to pass in the high eightbyte of the result. We do this by forming a
3754 // first class struct aggregate with the high and low part: {low, high}
3755 if (HighPart)
3756 ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3757
3758 return ABIArgInfo::getDirect(ResType);
3759}
3760
3762X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,
3763 unsigned &neededInt, unsigned &neededSSE,
3764 bool isNamedArg, bool IsRegCall) const {
3766
3767 X86_64ABIInfo::Class Lo, Hi;
3768 classify(Ty, 0, Lo, Hi, isNamedArg, IsRegCall);
3769
3770 // Check some invariants.
3771 // FIXME: Enforce these by construction.
3772 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3773 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3774
3775 neededInt = 0;
3776 neededSSE = 0;
3777 llvm::Type *ResType = nullptr;
3778 switch (Lo) {
3779 case NoClass:
3780 if (Hi == NoClass)
3781 return ABIArgInfo::getIgnore();
3782 // If the low part is just padding, it takes no register, leave ResType
3783 // null.
3784 assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3785 "Unknown missing lo part");
3786 break;
3787
3788 // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
3789 // on the stack.
3790 case Memory:
3791
3792 // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
3793 // COMPLEX_X87, it is passed in memory.
3794 case X87:
3795 case ComplexX87:
3796 if (getRecordArgABI(Ty, getCXXABI()) == CGCXXABI::RAA_Indirect)
3797 ++neededInt;
3798 return getIndirectResult(Ty, freeIntRegs);
3799
3800 case SSEUp:
3801 case X87Up:
3802 llvm_unreachable("Invalid classification for lo word.");
3803
3804 // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
3805 // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
3806 // and %r9 is used.
3807 case Integer:
3808 ++neededInt;
3809
3810 // Pick an 8-byte type based on the preferred type.
3811 ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 0, Ty, 0);
3812
3813 // If we have a sign or zero extended integer, make sure to return Extend
3814 // so that the parameter gets the right LLVM IR attributes.
3815 if (Hi == NoClass && isa<llvm::IntegerType>(ResType)) {
3816 // Treat an enum type as its underlying type.
3817 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3818 Ty = EnumTy->getDecl()->getIntegerType();
3819
3820 if (Ty->isIntegralOrEnumerationType() &&
3821 isPromotableIntegerTypeForABI(Ty))
3822 return ABIArgInfo::getExtend(Ty);
3823 }
3824
3825 break;
3826
3827 // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
3828 // available SSE register is used, the registers are taken in the
3829 // order from %xmm0 to %xmm7.
3830 case SSE: {
3831 llvm::Type *IRType = CGT.ConvertType(Ty);
3832 ResType = GetSSETypeAtOffset(IRType, 0, Ty, 0);
3833 ++neededSSE;
3834 break;
3835 }
3836 }
3837
3838 llvm::Type *HighPart = nullptr;
3839 switch (Hi) {
3840 // Memory was handled previously, ComplexX87 and X87 should
3841 // never occur as hi classes, and X87Up must be preceded by X87,
3842 // which is passed in memory.
3843 case Memory:
3844 case X87:
3845 case ComplexX87:
3846 llvm_unreachable("Invalid classification for hi word.");
3847
3848 case NoClass: break;
3849
3850 case Integer:
3851 ++neededInt;
3852 // Pick an 8-byte type based on the preferred type.
3853 HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3854
3855 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
3856 return ABIArgInfo::getDirect(HighPart, 8);
3857 break;
3858
3859 // X87Up generally doesn't occur here (long double is passed in
3860 // memory), except in situations involving unions.
3861 case X87Up:
3862 case SSE:
3863 HighPart = GetSSETypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3864
3865 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
3866 return ABIArgInfo::getDirect(HighPart, 8);
3867
3868 ++neededSSE;
3869 break;
3870
3871 // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
3872 // eightbyte is passed in the upper half of the last used SSE
3873 // register. This only happens when 128-bit vectors are passed.
3874 case SSEUp:
3875 assert(Lo == SSE && "Unexpected SSEUp classification");
3876 ResType = GetByteVectorType(Ty);
3877 break;
3878 }
3879
3880 // If a high part was specified, merge it together with the low part. It is
3881 // known to pass in the high eightbyte of the result. We do this by forming a
3882 // first class struct aggregate with the high and low part: {low, high}
3883 if (HighPart)
3884 ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3885
3886 return ABIArgInfo::getDirect(ResType);
3887}
3888
3890X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
3891 unsigned &NeededSSE,
3892 unsigned &MaxVectorWidth) const {
3893 auto RT = Ty->getAs<RecordType>();
3894 assert(RT && "classifyRegCallStructType only valid with struct types");
3895
3896 if (RT->getDecl()->hasFlexibleArrayMember())
3897 return getIndirectReturnResult(Ty);
3898
3899 // Sum up bases
3900 if (auto CXXRD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
3901 if (CXXRD->isDynamicClass()) {
3902 NeededInt = NeededSSE = 0;
3903 return getIndirectReturnResult(Ty);
3904 }
3905
3906 for (const auto &I : CXXRD->bases())
3907 if (classifyRegCallStructTypeImpl(I.getType(), NeededInt, NeededSSE,
3908 MaxVectorWidth)
3909 .isIndirect()) {
3910 NeededInt = NeededSSE = 0;
3911 return getIndirectReturnResult(Ty);
3912 }
3913 }
3914
3915 // Sum up members
3916 for (const auto *FD : RT->getDecl()->fields()) {
3917 QualType MTy = FD->getType();
3918 if (MTy->isRecordType() && !MTy->isUnionType()) {
3919 if (classifyRegCallStructTypeImpl(MTy, NeededInt, NeededSSE,
3920 MaxVectorWidth)
3921 .isIndirect()) {
3922 NeededInt = NeededSSE = 0;
3923 return getIndirectReturnResult(Ty);
3924 }
3925 } else {
3926 unsigned LocalNeededInt, LocalNeededSSE;
3927 if (classifyArgumentType(MTy, UINT_MAX, LocalNeededInt, LocalNeededSSE,
3928 true, true)
3929 .isIndirect()) {
3930 NeededInt = NeededSSE = 0;
3931 return getIndirectReturnResult(Ty);
3932 }
3933 if (const auto *AT = getContext().getAsConstantArrayType(MTy))
3934 MTy = AT->getElementType();
3935 if (const auto *VT = MTy->getAs<VectorType>())
3936 if (getContext().getTypeSize(VT) > MaxVectorWidth)
3937 MaxVectorWidth = getContext().getTypeSize(VT);
3938 NeededInt += LocalNeededInt;
3939 NeededSSE += LocalNeededSSE;
3940 }
3941 }
3942
3943 return ABIArgInfo::getDirect();
3944}
3945
3947X86_64ABIInfo::classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
3948 unsigned &NeededSSE,
3949 unsigned &MaxVectorWidth) const {
3950
3951 NeededInt = 0;
3952 NeededSSE = 0;
3953 MaxVectorWidth = 0;
3954
3955 return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE,
3956 MaxVectorWidth);
3957}
3958
3959void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3960
3961 const unsigned CallingConv = FI.getCallingConvention();
3962 // It is possible to force Win64 calling convention on any x86_64 target by
3963 // using __attribute__((ms_abi)). In such case to correctly emit Win64
3964 // compatible code delegate this call to WinX86_64ABIInfo::computeInfo.
3965 if (CallingConv == llvm::CallingConv::Win64) {
3966 WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);
3967 Win64ABIInfo.computeInfo(FI);
3968 return;
3969 }
3970
3971 bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;
3972
3973 // Keep track of the number of assigned registers.
3974 unsigned FreeIntRegs = IsRegCall ? 11 : 6;
3975 unsigned FreeSSERegs = IsRegCall ? 16 : 8;
3976 unsigned NeededInt = 0, NeededSSE = 0, MaxVectorWidth = 0;
3977
3978 if (!::classifyReturnType(getCXXABI(), FI, *this)) {
3979 if (IsRegCall && FI.getReturnType()->getTypePtr()->isRecordType() &&
3980 !FI.getReturnType()->getTypePtr()->isUnionType()) {
3981 FI.getReturnInfo() = classifyRegCallStructType(
3982 FI.getReturnType(), NeededInt, NeededSSE, MaxVectorWidth);
3983 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3984 FreeIntRegs -= NeededInt;
3985 FreeSSERegs -= NeededSSE;
3986 } else {
3987 FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3988 }
3989 } else if (IsRegCall && FI.getReturnType()->getAs<ComplexType>() &&
3990 getContext().getCanonicalType(FI.getReturnType()
3991 ->getAs<ComplexType>()
3992 ->getElementType()) ==
3993 getContext().LongDoubleTy)
3994 // Complex Long Double Type is passed in Memory when Regcall
3995 // calling convention is used.
3996 FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3997 else
3999 }
4000
4001 // If the return value is indirect, then the hidden argument is consuming one
4002 // integer register.
4003 if (FI.getReturnInfo().isIndirect())
4004 --FreeIntRegs;
4005 else if (NeededSSE && MaxVectorWidth > 0)
4006 FI.setMaxVectorWidth(MaxVectorWidth);
4007
4008 // The chain argument effectively gives us another free register.
4009 if (FI.isChainCall())
4010 ++FreeIntRegs;
4011
4012 unsigned NumRequiredArgs = FI.getNumRequiredArgs();
4013 // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
4014 // get assigned (in left-to-right order) for passing as follows...
4015 unsigned ArgNo = 0;
4016 for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
4017 it != ie; ++it, ++ArgNo) {
4018 bool IsNamedArg = ArgNo < NumRequiredArgs;
4019
4020 if (IsRegCall && it->type->isStructureOrClassType())
4021 it->info = classifyRegCallStructType(it->type, NeededInt, NeededSSE,
4022 MaxVectorWidth);
4023 else
4024 it->info = classifyArgumentType(it->type, FreeIntRegs, NeededInt,
4025 NeededSSE, IsNamedArg);
4026
4027 // AMD64-ABI 3.2.3p3: If there are no registers available for any
4028 // eightbyte of an argument, the whole argument is passed on the
4029 // stack. If registers have already been assigned for some
4030 // eightbytes of such an argument, the assignments get reverted.
4031 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
4032 FreeIntRegs -= NeededInt;
4033 FreeSSERegs -= NeededSSE;
4034 if (MaxVectorWidth > FI.getMaxVectorWidth())
4035 FI.setMaxVectorWidth(MaxVectorWidth);
4036 } else {
4037 it->info = getIndirectResult(it->type, FreeIntRegs);
4038 }
4039 }
4040}
4041
4043 Address VAListAddr, QualType Ty) {
4044 Address overflow_arg_area_p =
4045 CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_p");
4046 llvm::Value *overflow_arg_area =
4047 CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area");
4048
4049 // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
4050 // byte boundary if alignment needed by type exceeds 8 byte boundary.
4051 // It isn't stated explicitly in the standard, but in practice we use
4052 // alignment greater than 16 where necessary.
4053 CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4054 if (Align > CharUnits::fromQuantity(8)) {
4055 overflow_arg_area = emitRoundPointerUpToAlignment(CGF, overflow_arg_area,
4056 Align);
4057 }
4058
4059 // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
4060 llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4061 llvm::Value *Res =
4062 CGF.Builder.CreateBitCast(overflow_arg_area,
4063 llvm::PointerType::getUnqual(LTy));
4064
4065 // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
4066 // l->overflow_arg_area + sizeof(type).
4067 // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
4068 // an 8 byte boundary.
4069
4070 uint64_t SizeInBytes = (CGF.getContext().getTypeSize(Ty) + 7) / 8;
4071 llvm::Value *Offset =
4072 llvm::ConstantInt::get(CGF.Int32Ty, (SizeInBytes + 7) & ~7);
4073 overflow_arg_area = CGF.Builder.CreateGEP(CGF.Int8Ty, overflow_arg_area,
4074 Offset, "overflow_arg_area.next");
4075 CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p);
4076
4077 // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
4078 return Address(Res, LTy, Align);
4079}
4080
4081Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4082 QualType Ty) const {
4083 // Assume that va_list type is correct; should be pointer to LLVM type:
4084 // struct {
4085 // i32 gp_offset;
4086 // i32 fp_offset;
4087 // i8* overflow_arg_area;
4088 // i8* reg_save_area;
4089 // };
4090 unsigned neededInt, neededSSE;
4091
4092 Ty = getContext().getCanonicalType(Ty);
4093 ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE,
4094 /*isNamedArg*/false);
4095
4096 // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
4097 // in the registers. If not go to step 7.
4098 if (!neededInt && !neededSSE)
4099 return EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4100
4101 // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
4102 // general purpose registers needed to pass type and num_fp to hold
4103 // the number of floating point registers needed.
4104
4105 // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
4106 // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
4107 // l->fp_offset > 304 - num_fp * 16 go to step 7.
4108 //
4109 // NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
4110 // register save space).
4111
4112 llvm::Value *InRegs = nullptr;
4113 Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
4114 llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
4115 if (neededInt) {
4116 gp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "gp_offset_p");
4117 gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset");
4118 InRegs = llvm::ConstantInt::get(CGF.Int32Ty, 48 - neededInt * 8);
4119 InRegs = CGF.Builder.CreateICmpULE(gp_offset, InRegs, "fits_in_gp");
4120 }
4121
4122 if (neededSSE) {
4123 fp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 1, "fp_offset_p");
4124 fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset");
4125 llvm::Value *FitsInFP =
4126 llvm::ConstantInt::get(CGF.Int32Ty, 176 - neededSSE * 16);
4127 FitsInFP = CGF.Builder.CreateICmpULE(fp_offset, FitsInFP, "fits_in_fp");
4128 InRegs = InRegs ? CGF.Builder.CreateAnd(InRegs, FitsInFP) : FitsInFP;
4129 }
4130
4131 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
4132 llvm::BasicBlock *InMemBlock = CGF.createBasicBlock("vaarg.in_mem");
4133 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
4134 CGF.Builder.CreateCondBr(InRegs, InRegBlock, InMemBlock);
4135
4136 // Emit code to load the value if it was passed in registers.
4137
4138 CGF.EmitBlock(InRegBlock);
4139
4140 // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
4141 // an offset of l->gp_offset and/or l->fp_offset. This may require
4142 // copying to a temporary location in case the parameter is passed
4143 // in different register classes or requires an alignment greater
4144 // than 8 for general purpose registers and 16 for XMM registers.
4145 //
4146 // FIXME: This really results in shameful code when we end up needing to
4147 // collect arguments from different places; often what should result in a
4148 // simple assembling of a structure from scattered addresses has many more
4149 // loads than necessary. Can we clean this up?
4150 llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4151 llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
4152 CGF.Builder.CreateStructGEP(VAListAddr, 3), "reg_save_area");
4153
4154 Address RegAddr = Address::invalid();
4155 if (neededInt && neededSSE) {
4156 // FIXME: Cleanup.
4157 assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
4158 llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());
4159 Address Tmp = CGF.CreateMemTemp(Ty);
4160 Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4161 assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
4162 llvm::Type *TyLo = ST->getElementType(0);
4163 llvm::Type *TyHi = ST->getElementType(1);
4164 assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
4165 "Unexpected ABI info for mixed regs");
4166 llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
4167 llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
4168 llvm::Value *GPAddr =
4169 CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset);
4170 llvm::Value *FPAddr =
4171 CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset);
4172 llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? FPAddr : GPAddr;
4173 llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr;
4174
4175 // Copy the first element.
4176 // FIXME: Our choice of alignment here and below is probably pessimistic.
4177 llvm::Value *V = CGF.Builder.CreateAlignedLoad(
4178 TyLo, CGF.Builder.CreateBitCast(RegLoAddr, PTyLo),
4179 CharUnits::fromQuantity(getDataLayout().getABITypeAlign(TyLo)));
4180 CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4181
4182 // Copy the second element.
4184 TyHi, CGF.Builder.CreateBitCast(RegHiAddr, PTyHi),
4185 CharUnits::fromQuantity(getDataLayout().getABITypeAlign(TyHi)));
4186 CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4187
4188 RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4189 } else if (neededInt) {
4190 RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset),
4192 RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4193
4194 // Copy to a temporary if necessary to ensure the appropriate alignment.
4195 auto TInfo = getContext().getTypeInfoInChars(Ty);
4196 uint64_t TySize = TInfo.Width.getQuantity();
4197 CharUnits TyAlign = TInfo.Align;
4198
4199 // Copy into a temporary if the type is more aligned than the
4200 // register save area.
4201 if (TyAlign.getQuantity() > 8) {
4202 Address Tmp = CGF.CreateMemTemp(Ty);
4203 CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, false);
4204 RegAddr = Tmp;
4205 }
4206
4207 } else if (neededSSE == 1) {
4208 RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset),
4210 RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4211 } else {
4212 assert(neededSSE == 2 && "Invalid number of needed registers!");
4213 // SSE registers are spaced 16 bytes apart in the register save
4214 // area, we need to collect the two eightbytes together.
4215 // The ABI isn't explicit about this, but it seems reasonable
4216 // to assume that the slots are 16-byte aligned, since the stack is
4217 // naturally 16-byte aligned and the prologue is expected to store
4218 // all the SSE registers to the RSA.
4219 Address RegAddrLo = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea,
4220 fp_offset),
4222 Address RegAddrHi =
4223 CGF.Builder.CreateConstInBoundsByteGEP(RegAddrLo,
4225 llvm::Type *ST = AI.canHaveCoerceToType()
4226 ? AI.getCoerceToType()
4227 : llvm::StructType::get(CGF.DoubleTy, CGF.DoubleTy);
4228 llvm::Value *V;
4229 Address Tmp = CGF.CreateMemTemp(Ty);
4230 Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4232 RegAddrLo, ST->getStructElementType(0)));
4233 CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4235 RegAddrHi, ST->getStructElementType(1)));
4236 CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4237
4238 RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4239 }
4240
4241 // AMD64-ABI 3.5.7p5: Step 5. Set:
4242 // l->gp_offset = l->gp_offset + num_gp * 8
4243 // l->fp_offset = l->fp_offset + num_fp * 16.
4244 if (neededInt) {
4245 llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededInt * 8);
4246 CGF.Builder.CreateStore(CGF.Builder.CreateAdd(gp_offset, Offset),
4247 gp_offset_p);
4248 }
4249 if (neededSSE) {
4250 llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededSSE * 16);
4251 CGF.Builder.CreateStore(CGF.Builder.CreateAdd(fp_offset, Offset),
4252 fp_offset_p);
4253 }
4254 CGF.EmitBranch(ContBlock);
4255
4256 // Emit code to load the value if it was passed in memory.
4257
4258 CGF.EmitBlock(InMemBlock);
4259 Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4260
4261 // Return the appropriate result.
4262
4263 CGF.EmitBlock(ContBlock);
4264 Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock,
4265 "vaarg.addr");
4266 return ResAddr;
4267}
4268
4269Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
4270 QualType Ty) const {
4271 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4272 // not 1, 2, 4, or 8 bytes, must be passed by reference."
4273 uint64_t Width = getContext().getTypeSize(Ty);
4274 bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Width);
4275
4276 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4279 /*allowHigherAlign*/ false);
4280}
4281
4282ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(
4283 QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo &current) const {
4284 const Type *Base = nullptr;
4285 uint64_t NumElts = 0;
4286
4287 if (!Ty->isBuiltinType() && !Ty->isVectorType() &&
4288 isHomogeneousAggregate(Ty, Base, NumElts) && FreeSSERegs >= NumElts) {
4289 FreeSSERegs -= NumElts;
4290 return getDirectX86Hva();
4291 }
4292 return current;
4293}
4294
4295ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
4296 bool IsReturnType, bool IsVectorCall,
4297 bool IsRegCall) const {
4298
4299 if (Ty->isVoidType())
4300 return ABIArgInfo::getIgnore();
4301
4302 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4303 Ty = EnumTy->getDecl()->getIntegerType();
4304
4305 TypeInfo Info = getContext().getTypeInfo(Ty);
4306 uint64_t Width = Info.Width;
4307 CharUnits Align = getContext().toCharUnitsFromBits(Info.Align);
4308
4309 const RecordType *RT = Ty->getAs<RecordType>();
4310 if (RT) {
4311 if (!IsReturnType) {
4312 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()))
4313 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4314 }
4315
4316 if (RT->getDecl()->hasFlexibleArrayMember())
4317 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4318
4319 }
4320
4321 const Type *Base = nullptr;
4322 uint64_t NumElts = 0;
4323 // vectorcall adds the concept of a homogenous vector aggregate, similar to
4324 // other targets.
4325 if ((IsVectorCall || IsRegCall) &&
4326 isHomogeneousAggregate(Ty, Base, NumElts)) {
4327 if (IsRegCall) {
4328 if (FreeSSERegs >= NumElts) {
4329 FreeSSERegs -= NumElts;
4330 if (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())
4331 return ABIArgInfo::getDirect();
4332 return ABIArgInfo::getExpand();
4333 }
4334 return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4335 } else if (IsVectorCall) {
4336 if (FreeSSERegs >= NumElts &&
4337 (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())) {
4338 FreeSSERegs -= NumElts;
4339 return ABIArgInfo::getDirect();
4340 } else if (IsReturnType) {
4341 return ABIArgInfo::getExpand();
4342 } else if (!Ty->isBuiltinType() && !Ty->isVectorType()) {
4343 // HVAs are delayed and reclassified in the 2nd step.
4344 return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4345 }
4346 }
4347 }
4348
4349 if (Ty->isMemberPointerType()) {
4350 // If the member pointer is represented by an LLVM int or ptr, pass it
4351 // directly.
4352 llvm::Type *LLTy = CGT.ConvertType(Ty);
4353 if (LLTy->isPointerTy() || LLTy->isIntegerTy())
4354 return ABIArgInfo::getDirect();
4355 }
4356
4357 if (RT || Ty->isAnyComplexType() || Ty->isMemberPointerType()) {
4358 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4359 // not 1, 2, 4, or 8 bytes, must be passed by reference."
4360 if (Width > 64 || !llvm::isPowerOf2_64(Width))
4361 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4362
4363 // Otherwise, coerce it to a small integer.
4364 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Width));
4365 }
4366
4367 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
4368 switch (BT->getKind()) {
4369 case BuiltinType::Bool:
4370 // Bool type is always extended to the ABI, other builtin types are not
4371 // extended.
4372 return ABIArgInfo::getExtend(Ty);
4373
4374 case BuiltinType::LongDouble:
4375 // Mingw64 GCC uses the old 80 bit extended precision floating point
4376 // unit. It passes them indirectly through memory.
4377 if (IsMingw64) {
4378 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
4379 if (LDF == &llvm::APFloat::x87DoubleExtended())
4380 return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4381 }
4382 break;
4383
4384 case BuiltinType::Int128:
4385 case BuiltinType::UInt128:
4386 // If it's a parameter type, the normal ABI rule is that arguments larger
4387 // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
4388 // even though it isn't particularly efficient.
4389 if (!IsReturnType)
4390 return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4391
4392 // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
4393 // Clang matches them for compatibility.
4394 return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
4395 llvm::Type::getInt64Ty(getVMContext()), 2));
4396
4397 default:
4398 break;
4399 }
4400 }
4401
4402 if (Ty->isBitIntType()) {
4403 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4404 // not 1, 2, 4, or 8 bytes, must be passed by reference."
4405 // However, non-power-of-two bit-precise integers will be passed as 1, 2, 4,
4406 // or 8 bytes anyway as long is it fits in them, so we don't have to check
4407 // the power of 2.
4408 if (Width <= 64)
4409 return ABIArgInfo::getDirect();
4410 return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4411 }
4412
4413 return ABIArgInfo::getDirect();
4414}
4415
4416void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
4417 const unsigned CC = FI.getCallingConvention();
4418 bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
4419 bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
4420
4421 // If __attribute__((sysv_abi)) is in use, use the SysV argument
4422 // classification rules.
4423 if (CC == llvm::CallingConv::X86_64_SysV) {
4424 X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);
4425 SysVABIInfo.computeInfo(FI);
4426 return;
4427 }
4428
4429 unsigned FreeSSERegs = 0;
4430 if (IsVectorCall) {
4431 // We can use up to 4 SSE return registers with vectorcall.
4432 FreeSSERegs = 4;
4433 } else if (IsRegCall) {
4434 // RegCall gives us 16 SSE registers.
4435 FreeSSERegs = 16;
4436 }
4437
4438 if (!getCXXABI().classifyReturnType(FI))
4439 FI.getReturnInfo() = classify(FI.getReturnType(), FreeSSERegs, true,
4440 IsVectorCall, IsRegCall);
4441
4442 if (IsVectorCall) {
4443 // We can use up to 6 SSE register parameters with vectorcall.
4444 FreeSSERegs = 6;
4445 } else if (IsRegCall) {
4446 // RegCall gives us 16 SSE registers, we can reuse the return registers.
4447 FreeSSERegs = 16;
4448 }
4449
4450 unsigned ArgNum = 0;
4451 unsigned ZeroSSERegs = 0;
4452 for (auto &I : FI.arguments()) {
4453 // Vectorcall in x64 only permits the first 6 arguments to be passed as
4454 // XMM/YMM registers. After the sixth argument, pretend no vector
4455 // registers are left.
4456 unsigned *MaybeFreeSSERegs =
4457 (IsVectorCall && ArgNum >= 6) ? &ZeroSSERegs : &FreeSSERegs;
4458 I.info =
4459 classify(I.type, *MaybeFreeSSERegs, false, IsVectorCall, IsRegCall);
4460 ++ArgNum;
4461 }
4462
4463 if (IsVectorCall) {
4464 // For vectorcall, assign aggregate HVAs to any free vector registers in a
4465 // second pass.
4466 for (auto &I : FI.arguments())
4467 I.info = reclassifyHvaArgForVectorCall(I.type, FreeSSERegs, I.info);
4468 }
4469}
4470
4471Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4472 QualType Ty) const {
4473 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4474 // not 1, 2, 4, or 8 bytes, must be passed by reference."
4475 uint64_t Width = getContext().getTypeSize(Ty);
4476 bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Width);
4477
4478 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4481 /*allowHigherAlign*/ false);
4482}
4483
4485 llvm::Value *Address, bool Is64Bit,
4486 bool IsAIX) {
4487 // This is calculated from the LLVM and GCC tables and verified
4488 // against gcc output. AFAIK all PPC ABIs use the same encoding.
4489
4490 CodeGen::CGBuilderTy &Builder = CGF.Builder;
4491
4492 llvm::IntegerType *i8 = CGF.Int8Ty;
4493 llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
4494 llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
4495 llvm::Value *Sixteen8 = llvm::ConstantInt::get(i8, 16);
4496
4497 // 0-31: r0-31, the 4-byte or 8-byte general-purpose registers
4498 AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 0, 31);
4499
4500 // 32-63: fp0-31, the 8-byte floating-point registers
4501 AssignToArrayRange(Builder, Address, Eight8, 32, 63);
4502
4503 // 64-67 are various 4-byte or 8-byte special-purpose registers:
4504 // 64: mq
4505 // 65: lr
4506 // 66: ctr
4507 // 67: ap
4508 AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 64, 67);
4509
4510 // 68-76 are various 4-byte special-purpose registers:
4511 // 68-75 cr0-7
4512 // 76: xer
4513 AssignToArrayRange(Builder, Address, Four8, 68, 76);
4514
4515 // 77-108: v0-31, the 16-byte vector registers
4516 AssignToArrayRange(Builder, Address, Sixteen8, 77, 108);
4517
4518 // 109: vrsave
4519 // 110: vscr
4520 AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 109, 110);
4521
4522 // AIX does not utilize the rest of the registers.
4523 if (IsAIX)
4524 return false;
4525
4526 // 111: spe_acc
4527 // 112: spefscr
4528 // 113: sfp
4529 AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 111, 113);
4530
4531 if (!Is64Bit)
4532 return false;
4533
4534 // TODO: Need to verify if these registers are used on 64 bit AIX with Power8
4535 // or above CPU.
4536 // 64-bit only registers:
4537 // 114: tfhar
4538 // 115: tfiar
4539 // 116: texasr
4540 AssignToArrayRange(Builder, Address, Eight8, 114, 116);
4541
4542 return false;
4543}
4544
4545// AIX
4546namespace {
4547/// AIXABIInfo - The AIX XCOFF ABI information.
4548class AIXABIInfo : public ABIInfo {
4549 const bool Is64Bit;
4550 const unsigned PtrByteSize;
4551 CharUnits getParamTypeAlignment(QualType Ty) const;
4552
4553public:
4554 AIXABIInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4555 : ABIInfo(CGT), Is64Bit(Is64Bit), PtrByteSize(Is64Bit ? 8 : 4) {}
4556
4557 bool isPromotableTypeForABI(QualType Ty) const;
4558
4561
4562 void computeInfo(CGFunctionInfo &FI) const override {
4563 if (!getCXXABI().classifyReturnType(FI))
4565
4566 for (auto &I : FI.arguments())
4568 }
4569
4570 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4571 QualType Ty) const override;
4572};
4573
4574class AIXTargetCodeGenInfo : public TargetCodeGenInfo {
4575 const bool Is64Bit;
4576
4577public:
4578 AIXTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4579 : TargetCodeGenInfo(std::make_unique<AIXABIInfo>(CGT, Is64Bit)),
4580 Is64Bit(Is64Bit) {}
4581 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4582 return 1; // r1 is the dedicated stack pointer
4583 }
4584
4585 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4586 llvm::Value *Address) const override;
4587};
4588} // namespace
4589
4590// Return true if the ABI requires Ty to be passed sign- or zero-
4591// extended to 32/64 bits.
4592bool AIXABIInfo::isPromotableTypeForABI(QualType Ty) const {
4593 // Treat an enum type as its underlying type.
4594 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4595 Ty = EnumTy->getDecl()->getIntegerType();
4596
4597 // Promotable integer types are required to be promoted by the ABI.
4598 if (getContext().isPromotableIntegerType(Ty))
4599 return true;
4600
4601 if (!Is64Bit)
4602 return false;
4603
4604 // For 64 bit mode, in addition to the usual promotable integer types, we also
4605 // need to extend all 32-bit types, since the ABI requires promotion to 64
4606 // bits.
4607 if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
4608 switch (BT->getKind()) {
4609 case BuiltinType::Int:
4610 case BuiltinType::UInt:
4611 return true;
4612 default:
4613 break;
4614 }
4615
4616 return false;
4617}
4618
4619ABIArgInfo AIXABIInfo::classifyReturnType(QualType RetTy) const {
4620 if (RetTy->isAnyComplexType())
4621 return ABIArgInfo::getDirect();
4622
4623 if (RetTy->isVectorType())
4624 return ABIArgInfo::getDirect();
4625
4626 if (RetTy->isVoidType())
4627 return ABIArgInfo::getIgnore();
4628
4629 if (isAggregateTypeForABI(RetTy))
4630 return getNaturalAlignIndirect(RetTy);
4631
4632 return (isPromotableTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
4634}
4635
4636ABIArgInfo AIXABIInfo::classifyArgumentType(QualType Ty) const {
4638
4639 if (Ty->isAnyComplexType())
4640 return ABIArgInfo::getDirect();
4641
4642 if (Ty->isVectorType())
4643 return ABIArgInfo::getDirect();
4644
4645 if (isAggregateTypeForABI(Ty)) {
4646 // Records with non-trivial destructors/copy-constructors should not be
4647 // passed by value.
4648 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
4649 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4650
4651 CharUnits CCAlign = getParamTypeAlignment(Ty);
4652 CharUnits TyAlign = getContext().getTypeAlignInChars(Ty);
4653
4654 return ABIArgInfo::getIndirect(CCAlign, /*ByVal*/ true,
4655 /*Realign*/ TyAlign > CCAlign);
4656 }
4657
4658 return (isPromotableTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
4660}
4661
4662CharUnits AIXABIInfo::getParamTypeAlignment(QualType Ty) const {
4663 // Complex types are passed just like their elements.
4664 if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4665 Ty = CTy->getElementType();
4666
4667 if (Ty->isVectorType())
4668 return CharUnits::fromQuantity(16);
4669
4670 // If the structure contains a vector type, the alignment is 16.
4671 if (isRecordWithSIMDVectorType(getContext(), Ty))
4672 return CharUnits::fromQuantity(16);
4673
4674 return CharUnits::fromQuantity(PtrByteSize);
4675}
4676
4677Address AIXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4678 QualType Ty) const {
4679
4680 auto TypeInfo = getContext().getTypeInfoInChars(Ty);
4681 TypeInfo.Align = getParamTypeAlignment(Ty);
4682
4683 CharUnits SlotSize = CharUnits::fromQuantity(PtrByteSize);
4684
4685 // If we have a complex type and the base type is smaller than the register
4686 // size, the ABI calls for the real and imaginary parts to be right-adjusted
4687 // in separate words in 32bit mode or doublewords in 64bit mode. However,
4688 // Clang expects us to produce a pointer to a structure with the two parts
4689 // packed tightly. So generate loads of the real and imaginary parts relative
4690 // to the va_list pointer, and store them to a temporary structure. We do the
4691 // same as the PPC64ABI here.
4692 if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4693 CharUnits EltSize = TypeInfo.Width / 2;
4694 if (EltSize < SlotSize)
4695 return complexTempStructure(CGF, VAListAddr, Ty, SlotSize, EltSize, CTy);
4696 }
4697
4698 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo,
4699 SlotSize, /*AllowHigher*/ true);
4700}
4701
4702bool AIXTargetCodeGenInfo::initDwarfEHRegSizeTable(
4703 CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const {
4704 return PPC_initDwarfEHRegSizeTable(CGF, Address, Is64Bit, /*IsAIX*/ true);
4705}
4706
4707// PowerPC-32
4708namespace {
4709/// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information.
4710class PPC32_SVR4_ABIInfo : public DefaultABIInfo {
4711 bool IsSoftFloatABI;
4712 bool IsRetSmallStructInRegABI;
4713
4714 CharUnits getParamTypeAlignment(QualType Ty) const;
4715
4716public:
4717 PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, bool SoftFloatABI,
4718 bool RetSmallStructInRegABI)
4719 : DefaultABIInfo(CGT), IsSoftFloatABI(SoftFloatABI),
4720 IsRetSmallStructInRegABI(RetSmallStructInRegABI) {}
4721
4723
4724 void computeInfo(CGFunctionInfo &FI) const override {
4725 if (!getCXXABI().classifyReturnType(FI))
4727 for (auto &I : FI.arguments())
4729 }
4730
4731 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4732 QualType Ty) const override;
4733};
4734
4735class PPC32TargetCodeGenInfo : public TargetCodeGenInfo {
4736public:
4737 PPC32TargetCodeGenInfo(CodeGenTypes &CGT, bool SoftFloatABI,
4738 bool RetSmallStructInRegABI)
4739 : TargetCodeGenInfo(std::make_unique<PPC32_SVR4_ABIInfo>(
4740 CGT, SoftFloatABI, RetSmallStructInRegABI)) {}
4741
4742 static bool isStructReturnInRegABI(const llvm::Triple &Triple,
4743 const CodeGenOptions &Opts);
4744
4745 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4746 // This is recovered from gcc output.
4747 return 1; // r1 is the dedicated stack pointer
4748 }
4749
4750 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4751 llvm::Value *Address) const override;
4752};
4753}
4754
4755CharUnits PPC32_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
4756 // Complex types are passed just like their elements.
4757 if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4758 Ty = CTy->getElementType();
4759
4760 if (Ty->isVectorType())
4761 return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16
4762 : 4);
4763
4764 // For single-element float/vector structs, we consider the whole type
4765 // to have the same alignment requirements as its single element.
4766 const Type *AlignTy = nullptr;
4767 if (const Type *EltType = isSingleElementStruct(Ty, getContext())) {
4768 const BuiltinType *BT = EltType->getAs<BuiltinType>();
4769 if ((EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) ||
4770 (BT && BT->isFloatingPoint()))
4771 AlignTy = EltType;
4772 }
4773
4774 if (AlignTy)
4775 return CharUnits::fromQuantity(AlignTy->isVectorType() ? 16 : 4);
4776 return CharUnits::fromQuantity(4);
4777}
4778
4779ABIArgInfo PPC32_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const {
4780 uint64_t Size;
4781
4782 // -msvr4-struct-return puts small aggregates in GPR3 and GPR4.
4783 if (isAggregateTypeForABI(RetTy) && IsRetSmallStructInRegABI &&
4784 (Size = getContext().getTypeSize(RetTy)) <= 64) {
4785 // System V ABI (1995), page 3-22, specified:
4786 // > A structure or union whose size is less than or equal to 8 bytes
4787 // > shall be returned in r3 and r4, as if it were first stored in the
4788 // > 8-byte aligned memory area and then the low addressed word were
4789 // > loaded into r3 and the high-addressed word into r4. Bits beyond
4790 // > the last member of the structure or union are not defined.
4791 //
4792 // GCC for big-endian PPC32 inserts the pad before the first member,
4793 // not "beyond the last member" of the struct. To stay compatible
4794 // with GCC, we coerce the struct to an integer of the same size.
4795 // LLVM will extend it and return i32 in r3, or i64 in r3:r4.
4796 if (Size == 0)
4797 return ABIArgInfo::getIgnore();
4798 else {
4799 llvm::Type *CoerceTy = llvm::Type::getIntNTy(getVMContext(), Size);
4800 return ABIArgInfo::getDirect(CoerceTy);
4801 }
4802 }
4803
4804 return DefaultABIInfo::classifyReturnType(RetTy);
4805}
4806
4807// TODO: this implementation is now likely redundant with
4808// DefaultABIInfo::EmitVAArg.
4809Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
4810 QualType Ty) const {
4811 if (getTarget().getTriple().isOSDarwin()) {
4812 auto TI = getContext().getTypeInfoInChars(Ty);
4813 TI.Align = getParamTypeAlignment(Ty);
4814
4815 CharUnits SlotSize = CharUnits::fromQuantity(4);
4816 return emitVoidPtrVAArg(CGF, VAList, Ty,
4817 classifyArgumentType(Ty).isIndirect(), TI, SlotSize,
4818 /*AllowHigherAlign=*/true);
4819 }
4820
4821 const unsigned OverflowLimit = 8;
4822 if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4823 // TODO: Implement this. For now ignore.
4824 (void)CTy;
4825 return Address::invalid(); // FIXME?
4826 }
4827
4828 // struct __va_list_tag {
4829 // unsigned char gpr;
4830 // unsigned char fpr;
4831 // unsigned short reserved;
4832 // void *overflow_arg_area;
4833 // void *reg_save_area;
4834 // };
4835
4836 bool isI64 = Ty->isIntegerType() && getContext().getTypeSize(Ty) == 64;
4837 bool isInt = !Ty->isFloatingType();
4838 bool isF64 = Ty->isFloatingType() && getContext().getTypeSize(Ty) == 64;
4839
4840 // All aggregates are passed indirectly? That doesn't seem consistent
4841 // with the argument-lowering code.
4842 bool isIndirect = isAggregateTypeForABI(Ty);
4843
4844 CGBuilderTy &Builder = CGF.Builder;
4845
4846 // The calling convention either uses 1-2 GPRs or 1 FPR.
4847 Address NumRegsAddr = Address::invalid();
4848 if (isInt || IsSoftFloatABI) {
4849 NumRegsAddr = Builder.CreateStructGEP(VAList, 0, "gpr");
4850 } else {
4851 NumRegsAddr = Builder.CreateStructGEP(VAList, 1, "fpr");
4852 }
4853
4854 llvm::Value *NumRegs = Builder.CreateLoad(NumRegsAddr, "numUsedRegs");
4855
4856 // "Align" the register count when TY is i64.
4857 if (isI64 || (isF64 && IsSoftFloatABI)) {
4858 NumRegs = Builder.CreateAdd(NumRegs, Builder.getInt8(1));
4859 NumRegs = Builder.CreateAnd(NumRegs, Builder.getInt8((uint8_t) ~1U));
4860 }
4861
4862 llvm::Value *CC =
4863 Builder.CreateICmpULT(NumRegs, Builder.getInt8(OverflowLimit), "cond");
4864
4865 llvm::BasicBlock *UsingRegs = CGF.createBasicBlock("using_regs");
4866 llvm::BasicBlock *UsingOverflow = CGF.createBasicBlock("using_overflow");
4867 llvm::BasicBlock *Cont = CGF.createBasicBlock("cont");
4868
4869 Builder.CreateCondBr(CC, UsingRegs, UsingOverflow);
4870
4871 llvm::Type *DirectTy = CGF.ConvertType(Ty), *ElementTy = DirectTy;
4872 if (isIndirect) DirectTy = DirectTy->getPointerTo(0);
4873
4874 // Case 1: consume registers.
4875 Address RegAddr = Address::invalid();
4876 {
4877 CGF.EmitBlock(UsingRegs);
4878
4879 Address RegSaveAreaPtr = Builder.CreateStructGEP(VAList, 4);
4880 RegAddr = Address(Builder.CreateLoad(RegSaveAreaPtr), CGF.Int8Ty,
4882 assert(RegAddr.getElementType() == CGF.Int8Ty);
4883
4884 // Floating-point registers start after the general-purpose registers.
4885 if (!(isInt || IsSoftFloatABI)) {
4886 RegAddr = Builder.CreateConstInBoundsByteGEP(RegAddr,
4888 }
4889
4890 // Get the address of the saved value by scaling the number of
4891 // registers we've used by the number of
4892 CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 8);
4893 llvm::Value *RegOffset =
4894 Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity()));
4895 RegAddr = Address(
4896 Builder.CreateInBoundsGEP(CGF.Int8Ty, RegAddr.getPointer(), RegOffset),
4897 CGF.Int8Ty, RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
4898 RegAddr = Builder.CreateElementBitCast(RegAddr, DirectTy);
4899
4900 // Increase the used-register count.
4901 NumRegs =
4902 Builder.CreateAdd(NumRegs,
4903 Builder.getInt8((isI64 || (isF64 && IsSoftFloatABI)) ? 2 : 1));
4904 Builder.CreateStore(NumRegs, NumRegsAddr);
4905
4906 CGF.EmitBranch(Cont);
4907 }
4908
4909 // Case 2: consume space in the overflow area.
4910 Address MemAddr = Address::invalid();
4911 {
4912 CGF.EmitBlock(UsingOverflow);
4913
4914 Builder.CreateStore(Builder.getInt8(OverflowLimit), NumRegsAddr);
4915
4916 // Everything in the overflow area is rounded up to a size of at least 4.
4917 CharUnits OverflowAreaAlign = CharUnits::fromQuantity(4);
4918
4920 if (!isIndirect) {
4921 auto TypeInfo = CGF.getContext().getTypeInfoInChars(Ty);
4922 Size = TypeInfo.Width.alignTo(OverflowAreaAlign);
4923 } else {
4924 Size = CGF.getPointerSize();
4925 }
4926
4927 Address OverflowAreaAddr = Builder.CreateStructGEP(VAList, 3);
4928 Address OverflowArea =
4929 Address(Builder.CreateLoad(OverflowAreaAddr, "argp.cur"), CGF.Int8Ty,
4930 OverflowAreaAlign);
4931 // Round up address of argument to alignment
4932 CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4933 if (Align > OverflowAreaAlign) {
4934 llvm::Value *Ptr = OverflowArea.getPointer();
4935 OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align),
4936 OverflowArea.getElementType(), Align);
4937 }
4938
4939 MemAddr = Builder.CreateElementBitCast(OverflowArea, DirectTy);
4940
4941 // Increase the overflow area.
4942 OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size);
4943 Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr);
4944 CGF.EmitBranch(Cont);
4945 }
4946
4947 CGF.EmitBlock(Cont);
4948
4949 // Merge the cases with a phi.
4950 Address Result = emitMergePHI(CGF, RegAddr, UsingRegs, MemAddr, UsingOverflow,
4951 "vaarg.addr");
4952
4953 // Load the pointer if the argument was passed indirectly.
4954 if (isIndirect) {
4955 Result = Address(Builder.CreateLoad(Result, "aggr"), ElementTy,
4956 getContext().getTypeAlignInChars(Ty));
4957 }
4958
4959 return Result;
4960}
4961
4962bool PPC32TargetCodeGenInfo::isStructReturnInRegABI(
4963 const llvm::Triple &Triple, const CodeGenOptions &Opts) {
4964 assert(Triple.isPPC32());
4965
4966 switch (Opts.getStructReturnConvention()) {
4968 break;
4969 case CodeGenOptions::SRCK_OnStack: // -maix-struct-return
4970 return false;
4971 case CodeGenOptions::SRCK_InRegs: // -msvr4-struct-return
4972 return true;
4973 }
4974
4975 if (Triple.isOSBinFormatELF() && !Triple.isOSLinux())
4976 return true;
4977
4978 return false;
4979}
4980
4981bool
4982PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4983 llvm::Value *Address) const {
4984 return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ false,
4985 /*IsAIX*/ false);
4986}
4987
4988// PowerPC-64
4989
4990namespace {
4991/// PPC64_SVR4_ABIInfo - The 64-bit PowerPC ELF (SVR4) ABI information.
4992class PPC64_SVR4_ABIInfo : public ABIInfo {
4993public:
4994 enum ABIKind {
4995 ELFv1 = 0,
4996 ELFv2
4997 };
4998
4999private:
5000 static const unsigned GPRBits = 64;
5001 ABIKind Kind;
5002 bool IsSoftFloatABI;
5003
5004public:
5005 PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind,
5006 bool SoftFloatABI)
5007 : ABIInfo(CGT), Kind(Kind), IsSoftFloatABI(SoftFloatABI) {}
5008
5009 bool isPromotableTypeForABI(QualType Ty) const;
5010 CharUnits getParamTypeAlignment(QualType Ty) const;
5011
5014
5015 bool isHomogeneousAggregateBaseType(QualType Ty) const override;
5016 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
5017 uint64_t Members) const override;
5018
5019 // TODO: We can add more logic to computeInfo to improve performance.
5020 // Example: For aggregate arguments that fit in a register, we could
5021 // use getDirectInReg (as is done below for structs containing a single
5022 // floating-point value) to avoid pushing them to memory on function
5023 // entry. This would require changing the logic in PPCISelLowering
5024 // when lowering the parameters in the caller and args in the callee.
5025 void computeInfo(CGFunctionInfo &FI) const override {
5026 if (!getCXXABI().classifyReturnType(FI))
5028 for (auto &I : FI.arguments()) {
5029 // We rely on the default argument classification for the most part.
5030 // One exception: An aggregate containing a single floating-point
5031 // or vector item must be passed in a register if one is available.
5032 const Type *T = isSingleElementStruct(I.type, getContext());
5033 if (T) {
5034 const BuiltinType *BT = T->getAs<BuiltinType>();
5035 if ((T->isVectorType() && getContext().getTypeSize(T) == 128) ||
5036 (BT && BT->isFloatingPoint())) {
5037 QualType QT(T, 0);
5038 I.info = ABIArgInfo::getDirectInReg(CGT.ConvertType(QT));
5039 continue;
5040 }
5041 }
5043 }
5044 }
5045
5046 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
5047 QualType Ty) const override;
5048};
5049
5050class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo {
5051
5052public:
5053 PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT,
5054 PPC64_SVR4_ABIInfo::ABIKind Kind,
5055 bool SoftFloatABI)
5057 std::make_unique<PPC64_SVR4_ABIInfo>(CGT, Kind, SoftFloatABI)) {
5058 SwiftInfo =
5059 std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/false);
5060 }
5061
5062 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5063 // This is recovered from gcc output.
5064 return 1; // r1 is the dedicated stack pointer
5065 }
5066
5067 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5068 llvm::Value *Address) const override;
5069};
5070
5071class PPC64TargetCodeGenInfo : public DefaultTargetCodeGenInfo {
5072public:
5073 PPC64TargetCodeGenInfo(CodeGenTypes &CGT) : DefaultTargetCodeGenInfo(CGT) {}
5074
5075 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5076 // This is recovered from gcc output.
5077 return 1; // r1 is the dedicated stack pointer
5078 }
5079
5080 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5081 llvm::Value *Address) const override;
5082};
5083
5084}
5085
5086// Return true if the ABI requires Ty to be passed sign- or zero-
5087// extended to 64 bits.
5088bool
5089PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const {
5090 // Treat an enum type as its underlying type.
5091 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
5092 Ty = EnumTy->getDecl()->getIntegerType();
5093
5094 // Promotable integer types are required to be promoted by the ABI.
5095 if (isPromotableIntegerTypeForABI(Ty))
5096 return true;
5097
5098 // In addition to the usual promotable integer types, we also need to
5099 // extend all 32-bit types, since the ABI requires promotion to 64 bits.
5100 if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
5101 switch (BT->getKind()) {
5102 case BuiltinType::Int:
5103 case BuiltinType::UInt:
5104 return true;
5105 default:
5106 break;
5107 }
5108
5109 if (const auto *EIT = Ty->getAs<BitIntType>())
5110 if (EIT->getNumBits() < 64)
5111 return true;
5112
5113 return false;
5114}
5115
5116/// isAlignedParamType - Determine whether a type requires 16-byte or
5117/// higher alignment in the parameter area. Always returns at least 8.
5118CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
5119 // Complex types are passed just like their elements.
5120 if (const ComplexType *CTy = Ty->getAs<ComplexType>())
5121 Ty = CTy->getElementType();
5122
5123 auto FloatUsesVector = [this](QualType Ty){
5124 return Ty->isRealFloatingType() && &getContext().getFloatTypeSemantics(
5125 Ty) == &llvm::APFloat::IEEEquad();
5126 };
5127
5128 // Only vector types of size 16 bytes need alignment (larger types are
5129 // passed via reference, smaller types are not aligned).
5130 if (Ty->isVectorType()) {
5131 return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16 : 8);
5132 } else if (FloatUsesVector(Ty)) {
5133 // According to ABI document section 'Optional Save Areas': If extended
5134 // precision floating-point values in IEEE BINARY 128 QUADRUPLE PRECISION
5135 // format are supported, map them to a single quadword, quadword aligned.
5136 return CharUnits::fromQuantity(16);
5137 }
5138
5139 // For single-element float/vector structs, we consider the whole type
5140 // to have the same alignment requirements as its single element.
5141 const Type *AlignAsType = nullptr;
5142 const Type *EltType = isSingleElementStruct(Ty, getContext());
5143 if (EltType) {
5144 const BuiltinType *BT = EltType->getAs<BuiltinType>();
5145 if ((EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) ||
5146 (BT && BT->isFloatingPoint()))
5147 AlignAsType = EltType;
5148 }
5149
5150 // Likewise for ELFv2 homogeneous aggregates.
5151 const Type *Base = nullptr;
5152 uint64_t Members = 0;
5153 if (!AlignAsType && Kind == ELFv2 &&
5154 isAggregateTypeForABI(Ty) && isHomogeneousAggregate(Ty, Base, Members))
5155 AlignAsType = Base;
5156
5157 // With special case aggregates, only vector base types need alignment.
5158 if (AlignAsType) {
5159 bool UsesVector = AlignAsType->isVectorType() ||
5160 FloatUsesVector(QualType(AlignAsType, 0));
5161 return CharUnits::fromQuantity(UsesVector ? 16 : 8);
5162 }
5163
5164 // Otherwise, we only need alignment for any aggregate type that
5165 // has an alignment requirement of >= 16 bytes.
5166 if (isAggregateTypeForABI(Ty) && getContext().getTypeAlign(Ty) >= 128) {
5167 return CharUnits::fromQuantity(16);
5168 }
5169
5170 return CharUnits::fromQuantity(8);
5171}
5172
5173/// isHomogeneousAggregate - Return true if a type is an ELFv2 homogeneous
5174/// aggregate. Base is set to the base element type, and Members is set
5175/// to the number of base elements.
5177 uint64_t &Members) const {
5178 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
5179 uint64_t NElements = AT->getSize().getZExtValue();
5180 if (NElements == 0)
5181 return false;
5182 if (!isHomogeneousAggregate(AT->getElementType(), Base, Members))
5183 return false;
5184 Members *= NElements;
5185 } else if (const RecordType *RT = Ty->getAs<RecordType>()) {
5186 const RecordDecl *RD = RT->getDecl();
5187 if (RD->hasFlexibleArrayMember())
5188 return false;
5189
5190 Members = 0;
5191
5192 // If this is a C++ record, check the properties of the record such as
5193 // bases and ABI specific restrictions
5194 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
5195 if (!getCXXABI().isPermittedToBeHomogeneousAggregate(CXXRD))
5196 return false;
5197
5198 for (const auto &I : CXXRD->bases()) {
5199 // Ignore empty records.
5200 if (isEmptyRecord(getContext(), I.getType(), true))
5201 continue;
5202
5203 uint64_t FldMembers;
5204 if (!isHomogeneousAggregate(I.getType(), Base, FldMembers))
5205 return false;
5206
5207 Members += FldMembers;
5208 }
5209 }
5210
5211 for (const auto *FD : RD->fields()) {
5212 // Ignore (non-zero arrays of) empty records.
5213 QualType FT = FD->getType();
5214 while (const ConstantArrayType *AT =
5215 getContext().getAsConstantArrayType(FT)) {
5216 if (AT->getSize().getZExtValue() == 0)
5217 return false;
5218 FT = AT->getElementType();
5219 }
5220 if (isEmptyRecord(getContext(), FT, true))
5221 continue;
5222
5224 FD->isZeroLengthBitField(getContext()))
5225 continue;
5226
5227 uint64_t FldMembers;
5228 if (!isHomogeneousAggregate(FD->getType(), Base, FldMembers))
5229 return false;
5230
5231 Members = (RD->isUnion() ?
5232 std::max(Members, FldMembers) : Members + FldMembers);
5233 }
5234
5235 if (!Base)
5236 return false;
5237
5238 // Ensure there is no padding.
5239 if (getContext().getTypeSize(Base) * Members !=
5240 getContext().getTypeSize(Ty))
5241 return false;
5242 } else {
5243 Members = 1;
5244 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
5245 Members = 2;
5246 Ty = CT->getElementType();
5247 }
5248
5249 // Most ABIs only support float, double, and some vector type widths.
5251 return false;
5252
5253 // The base type must be the same for all members. Types that
5254 // agree in both total size and mode (float vs. vector) are
5255 // treated as being equivalent here.
5256 const Type *TyPtr = Ty.getTypePtr();
5257 if (!Base) {
5258 Base = TyPtr;
5259 // If it's a non-power-of-2 vector, its size is already a power-of-2,
5260 // so make sure to widen it explicitly.
5261 if (const VectorType *VT = Base->getAs<VectorType>()) {
5262 QualType EltTy = VT->getElementType();
5263 unsigned NumElements =
5265 Base = getContext()
5266 .getVectorType(EltTy, NumElements, VT->getVectorKind())
5267 .getTypePtr();
5268 }
5269 }
5270
5271 if (Base->isVectorType() != TyPtr->isVectorType() ||
5272 getContext().getTypeSize(Base) != getContext().getTypeSize(TyPtr))
5273 return false;
5274 }
5275 return Members > 0 && isHomogeneousAggregateSmallEnough(Base, Members);
5276}
5277
5278bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
5279 // Homogeneous aggregates for ELFv2 must have base types of float,
5280 // double, long double, or 128-bit vectors.
5281 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
5282 if (BT->getKind() == BuiltinType::Float ||
5283 BT->getKind() == BuiltinType::Double ||
5284 BT->getKind() == BuiltinType::LongDouble ||
5285 BT->getKind() == BuiltinType::Ibm128 ||
5286 (getContext().getTargetInfo().hasFloat128Type() &&
5287 (BT->getKind() == BuiltinType::Float128))) {
5288 if (IsSoftFloatABI)
5289 return false;
5290 return true;
5291 }
5292 }
5293 if (const VectorType *VT = Ty->getAs<VectorType>()) {
5294 if (getContext().getTypeSize(VT) == 128)
5295 return true;
5296 }
5297 return false;
5298}
5299
5300bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateSmallEnough(
5301 const Type *Base, uint64_t Members) const {
5302 // Vector and fp128 types require one register, other floating point types
5303 // require one or two registers depending on their size.
5304 uint32_t NumRegs =
5305 ((getContext().getTargetInfo().hasFloat128Type() &&
5306 Base->isFloat128Type()) ||
5307 Base->isVectorType()) ? 1
5308 : (getContext().getTypeSize(Base) + 63) / 64;
5309
5310 // Homogeneous Aggregates may occupy at most 8 registers.
5311 return Members * NumRegs <= 8;
5312}
5313
5315PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const {
5317
5318 if (Ty->isAnyComplexType())
5319 return ABIArgInfo::getDirect();
5320
5321 // Non-Altivec vector types are passed in GPRs (smaller than 16 bytes)
5322 // or via reference (larger than 16 bytes).
5323 if (Ty->isVectorType()) {
5324 uint64_t Size = getContext().getTypeSize(Ty);
5325 if (Size > 128)
5326 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
5327 else if (Size < 128) {
5328 llvm::Type *CoerceTy = llvm::IntegerType::get(getVMContext(), Size);
5329 return ABIArgInfo::getDirect(CoerceTy);
5330 }
5331 }
5332
5333 if (const auto *EIT = Ty->getAs<BitIntType>())
5334 if (EIT->getNumBits() > 128)
5335 return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
5336
5337 if (isAggregateTypeForABI(Ty)) {
5338 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
5339 return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
5340
5341 uint64_t ABIAlign = getParamTypeAlignment(Ty).getQuantity();
5342 uint64_t TyAlign = getContext().getTypeAlignInChars(Ty).getQuantity();
5343
5344 // ELFv2 homogeneous aggregates are passed as array types.
5345 const Type *Base = nullptr;
5346 uint64_t Members = 0;
5347 if (Kind == ELFv2 &&
5348 isHomogeneousAggregate(Ty, Base, Members)) {
5349 llvm::Type *BaseTy = CGT.ConvertType(QualType(Base, 0));
5350 llvm::Type *CoerceTy = llvm::ArrayType::get(BaseTy, Members);
5351 return ABIArgInfo::getDirect(CoerceTy);
5352 }
5353
5354 // If an aggregate may end up fully in registers, we do not
5355 // use the ByVal method, but pass the aggregate as array.
5356 // This is usually beneficial since we avoid forcing the
5357 // back-end to store the argument to memory.
5358 uint64_t Bits = getContext().getTypeSize(Ty);
5359 if (Bits > 0 && Bits <= 8 * GPRBits) {
5360 llvm::Type *CoerceTy;
5361
5362 // Types up to 8 bytes are passed as integer type (which will be
5363 // properly aligned in the argument save area doubleword).
5364 if (Bits <= GPRBits)
5365 CoerceTy =
5366 llvm::IntegerType::get(getVMContext(), llvm::alignTo(Bits, 8));
5367 // Larger types are passed as arrays, with the base type selected
5368 // according to the required alignment in the save area.
5369 else {
5370 uint64_t RegBits = ABIAlign * 8;
5371 uint64_t NumRegs = llvm::alignTo(Bits, RegBits) / RegBits;
5372 llvm::Type *RegTy = llvm::IntegerType::get(getVMContext(), RegBits);
5373 CoerceTy = llvm::ArrayType::get(RegTy, NumRegs);
5374 }
5375
5376 return ABIArgInfo::getDirect(CoerceTy);
5377 }
5378
5379 // All other aggregates are passed ByVal.
5381 /*ByVal=*/true,
5382 /*Realign=*/TyAlign > ABIAlign);
5383 }
5384
5385 return (isPromotableTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
5387}
5388
5390PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const {
5391 if (RetTy->isVoidType())
5392 return ABIArgInfo::getIgnore();
5393
5394 if (RetTy->isAnyComplexType())
5395 return ABIArgInfo::getDirect();
5396
5397 // Non-Altivec vector types are returned in GPRs (smaller than 16 bytes)
5398 // or via reference (larger than 16 bytes).
5399 if (RetTy->isVectorType()) {
5400 uint64_t Size = getContext().getTypeSize(RetTy);
5401 if (Size > 128)
5402 return getNaturalAlignIndirect(RetTy);
5403 else if (Size < 128) {
5404 llvm::Type *CoerceTy = llvm::IntegerType::get(getVMContext(), Size);
5405 return ABIArgInfo::getDirect(CoerceTy);
5406 }
5407 }
5408
5409 if (const auto *EIT = RetTy->getAs<BitIntType>())
5410 if (EIT->getNumBits() > 128)
5411 return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
5412
5413 if (isAggregateTypeForABI(RetTy)) {
5414 // ELFv2 homogeneous aggregates are returned as array types.
5415 const Type *Base = nullptr;
5416 uint64_t Members = 0;
5417 if (Kind == ELFv2 &&
5418 isHomogeneousAggregate(RetTy, Base, Members)) {
5419 llvm::Type *BaseTy = CGT.ConvertType(QualType(Base, 0));