clang  14.0.0git
TargetInfo.cpp
Go to the documentation of this file.
1 //===---- TargetInfo.cpp - Encapsulate target details -----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // These classes wrap the information about a call or function
10 // definition used to handle ABI compliancy.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "TargetInfo.h"
15 #include "ABIInfo.h"
16 #include "CGBlocks.h"
17 #include "CGCXXABI.h"
18 #include "CGValue.h"
19 #include "CodeGenFunction.h"
20 #include "clang/AST/Attr.h"
21 #include "clang/AST/RecordLayout.h"
24 #include "clang/Basic/Builtins.h"
27 #include "llvm/ADT/SmallBitVector.h"
28 #include "llvm/ADT/StringExtras.h"
29 #include "llvm/ADT/StringSwitch.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/IR/DataLayout.h"
33 #include "llvm/IR/IntrinsicsNVPTX.h"
34 #include "llvm/IR/IntrinsicsS390.h"
35 #include "llvm/IR/Type.h"
36 #include "llvm/Support/raw_ostream.h"
37 #include <algorithm> // std::sort
38 
39 using namespace clang;
40 using namespace CodeGen;
41 
42 // Helper for coercing an aggregate argument or return value into an integer
43 // array of the same size (including padding) and alignment. This alternate
44 // coercion happens only for the RenderScript ABI and can be removed after
45 // runtimes that rely on it are no longer supported.
46 //
47 // RenderScript assumes that the size of the argument / return value in the IR
48 // is the same as the size of the corresponding qualified type. This helper
49 // coerces the aggregate type into an array of the same size (including
50 // padding). This coercion is used in lieu of expansion of struct members or
51 // other canonical coercions that return a coerced-type of larger size.
52 //
53 // Ty - The argument / return value type
54 // Context - The associated ASTContext
55 // LLVMContext - The associated LLVMContext
57  ASTContext &Context,
58  llvm::LLVMContext &LLVMContext) {
59  // Alignment and Size are measured in bits.
60  const uint64_t Size = Context.getTypeSize(Ty);
61  const uint64_t Alignment = Context.getTypeAlign(Ty);
62  llvm::Type *IntType = llvm::Type::getIntNTy(LLVMContext, Alignment);
63  const uint64_t NumElements = (Size + Alignment - 1) / Alignment;
64  return ABIArgInfo::getDirect(llvm::ArrayType::get(IntType, NumElements));
65 }
66 
68  llvm::Value *Array,
70  unsigned FirstIndex,
71  unsigned LastIndex) {
72  // Alternatively, we could emit this as a loop in the source.
73  for (unsigned I = FirstIndex; I <= LastIndex; ++I) {
74  llvm::Value *Cell =
75  Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), Array, I);
76  Builder.CreateAlignedStore(Value, Cell, CharUnits::One());
77  }
78 }
79 
83 }
84 
86  bool Realign,
87  llvm::Type *Padding) const {
88  return ABIArgInfo::getIndirect(getContext().getTypeAlignInChars(Ty), ByVal,
89  Realign, Padding);
90 }
91 
94  return ABIArgInfo::getIndirectInReg(getContext().getTypeAlignInChars(Ty),
95  /*ByVal*/ false, Realign);
96 }
97 
99  QualType Ty) const {
100  return Address::invalid();
101 }
102 
104  if (Ty->isPromotableIntegerType())
105  return true;
106 
107  if (const auto *EIT = Ty->getAs<ExtIntType>())
108  if (EIT->getNumBits() < getContext().getTypeSize(getContext().IntTy))
109  return true;
110 
111  return false;
112 }
113 
115 
116 /// Does the given lowering require more than the given number of
117 /// registers when expanded?
118 ///
119 /// This is intended to be the basis of a reasonable basic implementation
120 /// of should{Pass,Return}IndirectlyForSwift.
121 ///
122 /// For most targets, a limit of four total registers is reasonable; this
123 /// limits the amount of code required in order to move around the value
124 /// in case it wasn't produced immediately prior to the call by the caller
125 /// (or wasn't produced in exactly the right registers) or isn't used
126 /// immediately within the callee. But some targets may need to further
127 /// limit the register count due to an inability to support that many
128 /// return registers.
130  ArrayRef<llvm::Type*> scalarTypes,
131  unsigned maxAllRegisters) {
132  unsigned intCount = 0, fpCount = 0;
133  for (llvm::Type *type : scalarTypes) {
134  if (type->isPointerTy()) {
135  intCount++;
136  } else if (auto intTy = dyn_cast<llvm::IntegerType>(type)) {
137  auto ptrWidth = cgt.getTarget().getPointerWidth(0);
138  intCount += (intTy->getBitWidth() + ptrWidth - 1) / ptrWidth;
139  } else {
140  assert(type->isVectorTy() || type->isFloatingPointTy());
141  fpCount++;
142  }
143  }
144 
145  return (intCount + fpCount > maxAllRegisters);
146 }
147 
149  llvm::Type *eltTy,
150  unsigned numElts) const {
151  // The default implementation of this assumes that the target guarantees
152  // 128-bit SIMD support but nothing more.
153  return (vectorSize.getQuantity() > 8 && vectorSize.getQuantity() <= 16);
154 }
155 
157  CGCXXABI &CXXABI) {
158  const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
159  if (!RD) {
160  if (!RT->getDecl()->canPassInRegisters())
161  return CGCXXABI::RAA_Indirect;
162  return CGCXXABI::RAA_Default;
163  }
164  return CXXABI.getRecordArgABI(RD);
165 }
166 
168  CGCXXABI &CXXABI) {
169  const RecordType *RT = T->getAs<RecordType>();
170  if (!RT)
171  return CGCXXABI::RAA_Default;
172  return getRecordArgABI(RT, CXXABI);
173 }
174 
176  const ABIInfo &Info) {
177  QualType Ty = FI.getReturnType();
178 
179  if (const auto *RT = Ty->getAs<RecordType>())
180  if (!isa<CXXRecordDecl>(RT->getDecl()) &&
181  !RT->getDecl()->canPassInRegisters()) {
182  FI.getReturnInfo() = Info.getNaturalAlignIndirect(Ty);
183  return true;
184  }
185 
186  return CXXABI.classifyReturnType(FI);
187 }
188 
189 /// Pass transparent unions as if they were the type of the first element. Sema
190 /// should ensure that all elements of the union have the same "machine type".
192  if (const RecordType *UT = Ty->getAsUnionType()) {
193  const RecordDecl *UD = UT->getDecl();
194  if (UD->hasAttr<TransparentUnionAttr>()) {
195  assert(!UD->field_empty() && "sema created an empty transparent union");
196  return UD->field_begin()->getType();
197  }
198  }
199  return Ty;
200 }
201 
203  return CGT.getCXXABI();
204 }
205 
207  return CGT.getContext();
208 }
209 
210 llvm::LLVMContext &ABIInfo::getVMContext() const {
211  return CGT.getLLVMContext();
212 }
213 
214 const llvm::DataLayout &ABIInfo::getDataLayout() const {
215  return CGT.getDataLayout();
216 }
217 
219  return CGT.getTarget();
220 }
221 
223  return CGT.getCodeGenOpts();
224 }
225 
226 bool ABIInfo::isAndroid() const { return getTarget().getTriple().isAndroid(); }
227 
229  return false;
230 }
231 
233  uint64_t Members) const {
234  return false;
235 }
236 
237 LLVM_DUMP_METHOD void ABIArgInfo::dump() const {
238  raw_ostream &OS = llvm::errs();
239  OS << "(ABIArgInfo Kind=";
240  switch (TheKind) {
241  case Direct:
242  OS << "Direct Type=";
243  if (llvm::Type *Ty = getCoerceToType())
244  Ty->print(OS);
245  else
246  OS << "null";
247  break;
248  case Extend:
249  OS << "Extend";
250  break;
251  case Ignore:
252  OS << "Ignore";
253  break;
254  case InAlloca:
255  OS << "InAlloca Offset=" << getInAllocaFieldIndex();
256  break;
257  case Indirect:
258  OS << "Indirect Align=" << getIndirectAlign().getQuantity()
259  << " ByVal=" << getIndirectByVal()
260  << " Realign=" << getIndirectRealign();
261  break;
262  case IndirectAliased:
263  OS << "Indirect Align=" << getIndirectAlign().getQuantity()
264  << " AadrSpace=" << getIndirectAddrSpace()
265  << " Realign=" << getIndirectRealign();
266  break;
267  case Expand:
268  OS << "Expand";
269  break;
270  case CoerceAndExpand:
271  OS << "CoerceAndExpand Type=";
272  getCoerceAndExpandType()->print(OS);
273  break;
274  }
275  OS << ")\n";
276 }
277 
278 // Dynamically round a pointer up to a multiple of the given alignment.
280  llvm::Value *Ptr,
281  CharUnits Align) {
282  llvm::Value *PtrAsInt = Ptr;
283  // OverflowArgArea = (OverflowArgArea + Align - 1) & -Align;
284  PtrAsInt = CGF.Builder.CreatePtrToInt(PtrAsInt, CGF.IntPtrTy);
285  PtrAsInt = CGF.Builder.CreateAdd(PtrAsInt,
286  llvm::ConstantInt::get(CGF.IntPtrTy, Align.getQuantity() - 1));
287  PtrAsInt = CGF.Builder.CreateAnd(PtrAsInt,
288  llvm::ConstantInt::get(CGF.IntPtrTy, -Align.getQuantity()));
289  PtrAsInt = CGF.Builder.CreateIntToPtr(PtrAsInt,
290  Ptr->getType(),
291  Ptr->getName() + ".aligned");
292  return PtrAsInt;
293 }
294 
295 /// Emit va_arg for a platform using the common void* representation,
296 /// where arguments are simply emitted in an array of slots on the stack.
297 ///
298 /// This version implements the core direct-value passing rules.
299 ///
300 /// \param SlotSize - The size and alignment of a stack slot.
301 /// Each argument will be allocated to a multiple of this number of
302 /// slots, and all the slots will be aligned to this value.
303 /// \param AllowHigherAlign - The slot alignment is not a cap;
304 /// an argument type with an alignment greater than the slot size
305 /// will be emitted on a higher-alignment address, potentially
306 /// leaving one or more empty slots behind as padding. If this
307 /// is false, the returned address might be less-aligned than
308 /// DirectAlign.
310  Address VAListAddr,
311  llvm::Type *DirectTy,
312  CharUnits DirectSize,
313  CharUnits DirectAlign,
314  CharUnits SlotSize,
315  bool AllowHigherAlign) {
316  // Cast the element type to i8* if necessary. Some platforms define
317  // va_list as a struct containing an i8* instead of just an i8*.
318  if (VAListAddr.getElementType() != CGF.Int8PtrTy)
319  VAListAddr = CGF.Builder.CreateElementBitCast(VAListAddr, CGF.Int8PtrTy);
320 
321  llvm::Value *Ptr = CGF.Builder.CreateLoad(VAListAddr, "argp.cur");
322 
323  // If the CC aligns values higher than the slot size, do so if needed.
324  Address Addr = Address::invalid();
325  if (AllowHigherAlign && DirectAlign > SlotSize) {
326  Addr = Address(emitRoundPointerUpToAlignment(CGF, Ptr, DirectAlign),
327  DirectAlign);
328  } else {
329  Addr = Address(Ptr, SlotSize);
330  }
331 
332  // Advance the pointer past the argument, then store that back.
333  CharUnits FullDirectSize = DirectSize.alignTo(SlotSize);
334  Address NextPtr =
335  CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next");
336  CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
337 
338  // If the argument is smaller than a slot, and this is a big-endian
339  // target, the argument will be right-adjusted in its slot.
340  if (DirectSize < SlotSize && CGF.CGM.getDataLayout().isBigEndian() &&
341  !DirectTy->isStructTy()) {
342  Addr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, SlotSize - DirectSize);
343  }
344 
345  Addr = CGF.Builder.CreateElementBitCast(Addr, DirectTy);
346  return Addr;
347 }
348 
349 /// Emit va_arg for a platform using the common void* representation,
350 /// where arguments are simply emitted in an array of slots on the stack.
351 ///
352 /// \param IsIndirect - Values of this type are passed indirectly.
353 /// \param ValueInfo - The size and alignment of this type, generally
354 /// computed with getContext().getTypeInfoInChars(ValueTy).
355 /// \param SlotSizeAndAlign - The size and alignment of a stack slot.
356 /// Each argument will be allocated to a multiple of this number of
357 /// slots, and all the slots will be aligned to this value.
358 /// \param AllowHigherAlign - The slot alignment is not a cap;
359 /// an argument type with an alignment greater than the slot size
360 /// will be emitted on a higher-alignment address, potentially
361 /// leaving one or more empty slots behind as padding.
363  QualType ValueTy, bool IsIndirect,
364  TypeInfoChars ValueInfo,
365  CharUnits SlotSizeAndAlign,
366  bool AllowHigherAlign) {
367  // The size and alignment of the value that was passed directly.
368  CharUnits DirectSize, DirectAlign;
369  if (IsIndirect) {
370  DirectSize = CGF.getPointerSize();
371  DirectAlign = CGF.getPointerAlign();
372  } else {
373  DirectSize = ValueInfo.Width;
374  DirectAlign = ValueInfo.Align;
375  }
376 
377  // Cast the address we've calculated to the right type.
378  llvm::Type *DirectTy = CGF.ConvertTypeForMem(ValueTy);
379  if (IsIndirect)
380  DirectTy = DirectTy->getPointerTo(0);
381 
382  Address Addr = emitVoidPtrDirectVAArg(CGF, VAListAddr, DirectTy,
383  DirectSize, DirectAlign,
384  SlotSizeAndAlign,
385  AllowHigherAlign);
386 
387  if (IsIndirect) {
388  Addr = Address(CGF.Builder.CreateLoad(Addr), ValueInfo.Align);
389  }
390 
391  return Addr;
392 
393 }
394 
396  QualType Ty, CharUnits SlotSize,
397  CharUnits EltSize, const ComplexType *CTy) {
398  Address Addr =
399  emitVoidPtrDirectVAArg(CGF, VAListAddr, CGF.Int8Ty, SlotSize * 2,
400  SlotSize, SlotSize, /*AllowHigher*/ true);
401 
402  Address RealAddr = Addr;
403  Address ImagAddr = RealAddr;
404  if (CGF.CGM.getDataLayout().isBigEndian()) {
405  RealAddr =
406  CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize - EltSize);
407  ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(ImagAddr,
408  2 * SlotSize - EltSize);
409  } else {
410  ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize);
411  }
412 
413  llvm::Type *EltTy = CGF.ConvertTypeForMem(CTy->getElementType());
414  RealAddr = CGF.Builder.CreateElementBitCast(RealAddr, EltTy);
415  ImagAddr = CGF.Builder.CreateElementBitCast(ImagAddr, EltTy);
416  llvm::Value *Real = CGF.Builder.CreateLoad(RealAddr, ".vareal");
417  llvm::Value *Imag = CGF.Builder.CreateLoad(ImagAddr, ".vaimag");
418 
419  Address Temp = CGF.CreateMemTemp(Ty, "vacplx");
420  CGF.EmitStoreOfComplex({Real, Imag}, CGF.MakeAddrLValue(Temp, Ty),
421  /*init*/ true);
422  return Temp;
423 }
424 
426  Address Addr1, llvm::BasicBlock *Block1,
427  Address Addr2, llvm::BasicBlock *Block2,
428  const llvm::Twine &Name = "") {
429  assert(Addr1.getType() == Addr2.getType());
430  llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name);
431  PHI->addIncoming(Addr1.getPointer(), Block1);
432  PHI->addIncoming(Addr2.getPointer(), Block2);
433  CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment());
434  return Address(PHI, Align);
435 }
436 
438 
439 // If someone can figure out a general rule for this, that would be great.
440 // It's probably just doomed to be platform-dependent, though.
442  // Verified for:
443  // x86-64 FreeBSD, Linux, Darwin
444  // x86-32 FreeBSD, Linux, Darwin
445  // PowerPC Linux, Darwin
446  // ARM Darwin (*not* EABI)
447  // AArch64 Linux
448  return 32;
449 }
450 
452  const FunctionNoProtoType *fnType) const {
453  // The following conventions are known to require this to be false:
454  // x86_stdcall
455  // MIPS
456  // For everything else, we just prefer false unless we opt out.
457  return false;
458 }
459 
460 void
462  llvm::SmallString<24> &Opt) const {
463  // This assumes the user is passing a library name like "rt" instead of a
464  // filename like "librt.a/so", and that they don't care whether it's static or
465  // dynamic.
466  Opt = "-l";
467  Opt += Lib;
468 }
469 
471  // OpenCL kernels are called via an explicit runtime API with arguments
472  // set with clSetKernelArg(), not as normal sub-functions.
473  // Return SPIR_KERNEL by default as the kernel calling convention to
474  // ensure the fingerprint is fixed such way that each OpenCL argument
475  // gets one matching argument in the produced kernel function argument
476  // list to enable feasible implementation of clSetKernelArg() with
477  // aggregates etc. In case we would use the default C calling conv here,
478  // clSetKernelArg() might break depending on the target-specific
479  // conventions; different targets might split structs passed as values
480  // to multiple function arguments etc.
481  return llvm::CallingConv::SPIR_KERNEL;
482 }
483 
485  llvm::PointerType *T, QualType QT) const {
486  return llvm::ConstantPointerNull::get(T);
487 }
488 
490  const VarDecl *D) const {
491  assert(!CGM.getLangOpts().OpenCL &&
492  !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
493  "Address space agnostic languages only");
494  return D ? D->getType().getAddressSpace() : LangAS::Default;
495 }
496 
498  CodeGen::CodeGenFunction &CGF, llvm::Value *Src, LangAS SrcAddr,
499  LangAS DestAddr, llvm::Type *DestTy, bool isNonNull) const {
500  // Since target may map different address spaces in AST to the same address
501  // space, an address space conversion may end up as a bitcast.
502  if (auto *C = dyn_cast<llvm::Constant>(Src))
503  return performAddrSpaceCast(CGF.CGM, C, SrcAddr, DestAddr, DestTy);
504  // Try to preserve the source's name to make IR more readable.
506  Src, DestTy, Src->hasName() ? Src->getName() + ".ascast" : "");
507 }
508 
509 llvm::Constant *
511  LangAS SrcAddr, LangAS DestAddr,
512  llvm::Type *DestTy) const {
513  // Since target may map different address spaces in AST to the same address
514  // space, an address space conversion may end up as a bitcast.
515  return llvm::ConstantExpr::getPointerCast(Src, DestTy);
516 }
517 
521  llvm::AtomicOrdering Ordering,
522  llvm::LLVMContext &Ctx) const {
523  return Ctx.getOrInsertSyncScopeID(""); /* default sync scope */
524 }
525 
526 static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays);
527 
528 /// isEmptyField - Return true iff a the field is "empty", that is it
529 /// is an unnamed bit-field or an (array of) empty record(s).
530 static bool isEmptyField(ASTContext &Context, const FieldDecl *FD,
531  bool AllowArrays) {
532  if (FD->isUnnamedBitfield())
533  return true;
534 
535  QualType FT = FD->getType();
536 
537  // Constant arrays of empty records count as empty, strip them off.
538  // Constant arrays of zero length always count as empty.
539  bool WasArray = false;
540  if (AllowArrays)
541  while (const ConstantArrayType *AT = Context.getAsConstantArrayType(FT)) {
542  if (AT->getSize() == 0)
543  return true;
544  FT = AT->getElementType();
545  // The [[no_unique_address]] special case below does not apply to
546  // arrays of C++ empty records, so we need to remember this fact.
547  WasArray = true;
548  }
549 
550  const RecordType *RT = FT->getAs<RecordType>();
551  if (!RT)
552  return false;
553 
554  // C++ record fields are never empty, at least in the Itanium ABI.
555  //
556  // FIXME: We should use a predicate for whether this behavior is true in the
557  // current ABI.
558  //
559  // The exception to the above rule are fields marked with the
560  // [[no_unique_address]] attribute (since C++20). Those do count as empty
561  // according to the Itanium ABI. The exception applies only to records,
562  // not arrays of records, so we must also check whether we stripped off an
563  // array type above.
564  if (isa<CXXRecordDecl>(RT->getDecl()) &&
565  (WasArray || !FD->hasAttr<NoUniqueAddressAttr>()))
566  return false;
567 
568  return isEmptyRecord(Context, FT, AllowArrays);
569 }
570 
571 /// isEmptyRecord - Return true iff a structure contains only empty
572 /// fields. Note that a structure with a flexible array member is not
573 /// considered empty.
574 static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays) {
575  const RecordType *RT = T->getAs<RecordType>();
576  if (!RT)
577  return false;
578  const RecordDecl *RD = RT->getDecl();
579  if (RD->hasFlexibleArrayMember())
580  return false;
581 
582  // If this is a C++ record, check the bases first.
583  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
584  for (const auto &I : CXXRD->bases())
585  if (!isEmptyRecord(Context, I.getType(), true))
586  return false;
587 
588  for (const auto *I : RD->fields())
589  if (!isEmptyField(Context, I, AllowArrays))
590  return false;
591  return true;
592 }
593 
594 /// isSingleElementStruct - Determine if a structure is a "single
595 /// element struct", i.e. it has exactly one non-empty field or
596 /// exactly one field which is itself a single element
597 /// struct. Structures with flexible array members are never
598 /// considered single element structs.
599 ///
600 /// \return The field declaration for the single non-empty field, if
601 /// it exists.
602 static const Type *isSingleElementStruct(QualType T, ASTContext &Context) {
603  const RecordType *RT = T->getAs<RecordType>();
604  if (!RT)
605  return nullptr;
606 
607  const RecordDecl *RD = RT->getDecl();
608  if (RD->hasFlexibleArrayMember())
609  return nullptr;
610 
611  const Type *Found = nullptr;
612 
613  // If this is a C++ record, check the bases first.
614  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
615  for (const auto &I : CXXRD->bases()) {
616  // Ignore empty records.
617  if (isEmptyRecord(Context, I.getType(), true))
618  continue;
619 
620  // If we already found an element then this isn't a single-element struct.
621  if (Found)
622  return nullptr;
623 
624  // If this is non-empty and not a single element struct, the composite
625  // cannot be a single element struct.
626  Found = isSingleElementStruct(I.getType(), Context);
627  if (!Found)
628  return nullptr;
629  }
630  }
631 
632  // Check for single element.
633  for (const auto *FD : RD->fields()) {
634  QualType FT = FD->getType();
635 
636  // Ignore empty fields.
637  if (isEmptyField(Context, FD, true))
638  continue;
639 
640  // If we already found an element then this isn't a single-element
641  // struct.
642  if (Found)
643  return nullptr;
644 
645  // Treat single element arrays as the element.
646  while (const ConstantArrayType *AT = Context.getAsConstantArrayType(FT)) {
647  if (AT->getSize().getZExtValue() != 1)
648  break;
649  FT = AT->getElementType();
650  }
651 
652  if (!isAggregateTypeForABI(FT)) {
653  Found = FT.getTypePtr();
654  } else {
655  Found = isSingleElementStruct(FT, Context);
656  if (!Found)
657  return nullptr;
658  }
659  }
660 
661  // We don't consider a struct a single-element struct if it has
662  // padding beyond the element type.
663  if (Found && Context.getTypeSize(Found) != Context.getTypeSize(T))
664  return nullptr;
665 
666  return Found;
667 }
668 
669 namespace {
670 Address EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
671  const ABIArgInfo &AI) {
672  // This default implementation defers to the llvm backend's va_arg
673  // instruction. It can handle only passing arguments directly
674  // (typically only handled in the backend for primitive types), or
675  // aggregates passed indirectly by pointer (NOTE: if the "byval"
676  // flag has ABI impact in the callee, this implementation cannot
677  // work.)
678 
679  // Only a few cases are covered here at the moment -- those needed
680  // by the default abi.
681  llvm::Value *Val;
682 
683  if (AI.isIndirect()) {
684  assert(!AI.getPaddingType() &&
685  "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
686  assert(
687  !AI.getIndirectRealign() &&
688  "Unexpected IndirectRealign seen in arginfo in generic VAArg emitter!");
689 
690  auto TyInfo = CGF.getContext().getTypeInfoInChars(Ty);
691  CharUnits TyAlignForABI = TyInfo.Align;
692 
693  llvm::Type *BaseTy =
694  llvm::PointerType::getUnqual(CGF.ConvertTypeForMem(Ty));
695  llvm::Value *Addr =
696  CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy);
697  return Address(Addr, TyAlignForABI);
698  } else {
699  assert((AI.isDirect() || AI.isExtend()) &&
700  "Unexpected ArgInfo Kind in generic VAArg emitter!");
701 
702  assert(!AI.getInReg() &&
703  "Unexpected InReg seen in arginfo in generic VAArg emitter!");
704  assert(!AI.getPaddingType() &&
705  "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
706  assert(!AI.getDirectOffset() &&
707  "Unexpected DirectOffset seen in arginfo in generic VAArg emitter!");
708  assert(!AI.getCoerceToType() &&
709  "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!");
710 
711  Address Temp = CGF.CreateMemTemp(Ty, "varet");
712  Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(), CGF.ConvertType(Ty));
713  CGF.Builder.CreateStore(Val, Temp);
714  return Temp;
715  }
716 }
717 
718 /// DefaultABIInfo - The default implementation for ABI specific
719 /// details. This implementation provides information which results in
720 /// self-consistent and sensible LLVM IR generation, but does not
721 /// conform to any particular ABI.
722 class DefaultABIInfo : public ABIInfo {
723 public:
724  DefaultABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
725 
728 
729  void computeInfo(CGFunctionInfo &FI) const override {
730  if (!getCXXABI().classifyReturnType(FI))
732  for (auto &I : FI.arguments())
733  I.info = classifyArgumentType(I.type);
734  }
735 
736  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
737  QualType Ty) const override {
738  return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty));
739  }
740 };
741 
742 class DefaultTargetCodeGenInfo : public TargetCodeGenInfo {
743 public:
744  DefaultTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
745  : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {}
746 };
747 
750 
751  if (isAggregateTypeForABI(Ty)) {
752  // Records with non-trivial destructors/copy-constructors should not be
753  // passed by value.
754  if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
755  return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
756 
757  return getNaturalAlignIndirect(Ty);
758  }
759 
760  // Treat an enum type as its underlying type.
761  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
762  Ty = EnumTy->getDecl()->getIntegerType();
763 
764  ASTContext &Context = getContext();
765  if (const auto *EIT = Ty->getAs<ExtIntType>())
766  if (EIT->getNumBits() >
767  Context.getTypeSize(Context.getTargetInfo().hasInt128Type()
768  ? Context.Int128Ty
769  : Context.LongLongTy))
770  return getNaturalAlignIndirect(Ty);
771 
772  return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
774 }
775 
777  if (RetTy->isVoidType())
778  return ABIArgInfo::getIgnore();
779 
780  if (isAggregateTypeForABI(RetTy))
781  return getNaturalAlignIndirect(RetTy);
782 
783  // Treat an enum type as its underlying type.
784  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
785  RetTy = EnumTy->getDecl()->getIntegerType();
786 
787  if (const auto *EIT = RetTy->getAs<ExtIntType>())
788  if (EIT->getNumBits() >
789  getContext().getTypeSize(getContext().getTargetInfo().hasInt128Type()
790  ? getContext().Int128Ty
791  : getContext().LongLongTy))
792  return getNaturalAlignIndirect(RetTy);
793 
794  return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
796 }
797 
798 //===----------------------------------------------------------------------===//
799 // WebAssembly ABI Implementation
800 //
801 // This is a very simple ABI that relies a lot on DefaultABIInfo.
802 //===----------------------------------------------------------------------===//
803 
804 class WebAssemblyABIInfo final : public SwiftABIInfo {
805 public:
806  enum ABIKind {
807  MVP = 0,
808  ExperimentalMV = 1,
809  };
810 
811 private:
812  DefaultABIInfo defaultInfo;
813  ABIKind Kind;
814 
815 public:
816  explicit WebAssemblyABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind)
817  : SwiftABIInfo(CGT), defaultInfo(CGT), Kind(Kind) {}
818 
819 private:
822 
823  // DefaultABIInfo's classifyReturnType and classifyArgumentType are
824  // non-virtual, but computeInfo and EmitVAArg are virtual, so we
825  // overload them.
826  void computeInfo(CGFunctionInfo &FI) const override {
827  if (!getCXXABI().classifyReturnType(FI))
829  for (auto &Arg : FI.arguments())
830  Arg.info = classifyArgumentType(Arg.type);
831  }
832 
833  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
834  QualType Ty) const override;
835 
836  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
837  bool asReturnValue) const override {
838  return occupiesMoreThan(CGT, scalars, /*total*/ 4);
839  }
840 
841  bool isSwiftErrorInRegister() const override {
842  return false;
843  }
844 };
845 
846 class WebAssemblyTargetCodeGenInfo final : public TargetCodeGenInfo {
847 public:
848  explicit WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
849  WebAssemblyABIInfo::ABIKind K)
850  : TargetCodeGenInfo(std::make_unique<WebAssemblyABIInfo>(CGT, K)) {}
851 
852  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
853  CodeGen::CodeGenModule &CGM) const override {
855  if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
856  if (const auto *Attr = FD->getAttr<WebAssemblyImportModuleAttr>()) {
857  llvm::Function *Fn = cast<llvm::Function>(GV);
858  llvm::AttrBuilder B;
859  B.addAttribute("wasm-import-module", Attr->getImportModule());
860  Fn->addFnAttrs(B);
861  }
862  if (const auto *Attr = FD->getAttr<WebAssemblyImportNameAttr>()) {
863  llvm::Function *Fn = cast<llvm::Function>(GV);
864  llvm::AttrBuilder B;
865  B.addAttribute("wasm-import-name", Attr->getImportName());
866  Fn->addFnAttrs(B);
867  }
868  if (const auto *Attr = FD->getAttr<WebAssemblyExportNameAttr>()) {
869  llvm::Function *Fn = cast<llvm::Function>(GV);
870  llvm::AttrBuilder B;
871  B.addAttribute("wasm-export-name", Attr->getExportName());
872  Fn->addFnAttrs(B);
873  }
874  }
875 
876  if (auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
877  llvm::Function *Fn = cast<llvm::Function>(GV);
878  if (!FD->doesThisDeclarationHaveABody() && !FD->hasPrototype())
879  Fn->addFnAttr("no-prototype");
880  }
881  }
882 };
883 
884 /// Classify argument of given type \p Ty.
887 
888  if (isAggregateTypeForABI(Ty)) {
889  // Records with non-trivial destructors/copy-constructors should not be
890  // passed by value.
891  if (auto RAA = getRecordArgABI(Ty, getCXXABI()))
892  return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
893  // Ignore empty structs/unions.
894  if (isEmptyRecord(getContext(), Ty, true))
895  return ABIArgInfo::getIgnore();
896  // Lower single-element structs to just pass a regular value. TODO: We
897  // could do reasonable-size multiple-element structs too, using getExpand(),
898  // though watch out for things like bitfields.
899  if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
900  return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
901  // For the experimental multivalue ABI, fully expand all other aggregates
902  if (Kind == ABIKind::ExperimentalMV) {
903  const RecordType *RT = Ty->getAs<RecordType>();
904  assert(RT);
905  bool HasBitField = false;
906  for (auto *Field : RT->getDecl()->fields()) {
907  if (Field->isBitField()) {
908  HasBitField = true;
909  break;
910  }
911  }
912  if (!HasBitField)
913  return ABIArgInfo::getExpand();
914  }
915  }
916 
917  // Otherwise just do the default thing.
918  return defaultInfo.classifyArgumentType(Ty);
919 }
920 
922  if (isAggregateTypeForABI(RetTy)) {
923  // Records with non-trivial destructors/copy-constructors should not be
924  // returned by value.
925  if (!getRecordArgABI(RetTy, getCXXABI())) {
926  // Ignore empty structs/unions.
927  if (isEmptyRecord(getContext(), RetTy, true))
928  return ABIArgInfo::getIgnore();
929  // Lower single-element structs to just return a regular value. TODO: We
930  // could do reasonable-size multiple-element structs too, using
931  // ABIArgInfo::getDirect().
932  if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
933  return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
934  // For the experimental multivalue ABI, return all other aggregates
935  if (Kind == ABIKind::ExperimentalMV)
936  return ABIArgInfo::getDirect();
937  }
938  }
939 
940  // Otherwise just do the default thing.
941  return defaultInfo.classifyReturnType(RetTy);
942 }
943 
944 Address WebAssemblyABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
945  QualType Ty) const {
946  bool IsIndirect = isAggregateTypeForABI(Ty) &&
947  !isEmptyRecord(getContext(), Ty, true) &&
948  !isSingleElementStruct(Ty, getContext());
949  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
950  getContext().getTypeInfoInChars(Ty),
952  /*AllowHigherAlign=*/true);
953 }
954 
955 //===----------------------------------------------------------------------===//
956 // le32/PNaCl bitcode ABI Implementation
957 //
958 // This is a simplified version of the x86_32 ABI. Arguments and return values
959 // are always passed on the stack.
960 //===----------------------------------------------------------------------===//
961 
962 class PNaClABIInfo : public ABIInfo {
963  public:
964  PNaClABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
965 
968 
969  void computeInfo(CGFunctionInfo &FI) const override;
970  Address EmitVAArg(CodeGenFunction &CGF,
971  Address VAListAddr, QualType Ty) const override;
972 };
973 
974 class PNaClTargetCodeGenInfo : public TargetCodeGenInfo {
975  public:
976  PNaClTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
977  : TargetCodeGenInfo(std::make_unique<PNaClABIInfo>(CGT)) {}
978 };
979 
980 void PNaClABIInfo::computeInfo(CGFunctionInfo &FI) const {
981  if (!getCXXABI().classifyReturnType(FI))
983 
984  for (auto &I : FI.arguments())
985  I.info = classifyArgumentType(I.type);
986 }
987 
988 Address PNaClABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
989  QualType Ty) const {
990  // The PNaCL ABI is a bit odd, in that varargs don't use normal
991  // function classification. Structs get passed directly for varargs
992  // functions, through a rewriting transform in
993  // pnacl-llvm/lib/Transforms/NaCl/ExpandVarArgs.cpp, which allows
994  // this target to actually support a va_arg instructions with an
995  // aggregate type, unlike other targets.
996  return EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect());
997 }
998 
999 /// Classify argument of given type \p Ty.
1001  if (isAggregateTypeForABI(Ty)) {
1002  if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
1003  return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
1004  return getNaturalAlignIndirect(Ty);
1005  } else if (const EnumType *EnumTy = Ty->getAs<EnumType>()) {
1006  // Treat an enum type as its underlying type.
1007  Ty = EnumTy->getDecl()->getIntegerType();
1008  } else if (Ty->isFloatingType()) {
1009  // Floating-point types don't go inreg.
1010  return ABIArgInfo::getDirect();
1011  } else if (const auto *EIT = Ty->getAs<ExtIntType>()) {
1012  // Treat extended integers as integers if <=64, otherwise pass indirectly.
1013  if (EIT->getNumBits() > 64)
1014  return getNaturalAlignIndirect(Ty);
1015  return ABIArgInfo::getDirect();
1016  }
1017 
1018  return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
1019  : ABIArgInfo::getDirect());
1020 }
1021 
1023  if (RetTy->isVoidType())
1024  return ABIArgInfo::getIgnore();
1025 
1026  // In the PNaCl ABI we always return records/structures on the stack.
1027  if (isAggregateTypeForABI(RetTy))
1028  return getNaturalAlignIndirect(RetTy);
1029 
1030  // Treat extended integers as integers if <=64, otherwise pass indirectly.
1031  if (const auto *EIT = RetTy->getAs<ExtIntType>()) {
1032  if (EIT->getNumBits() > 64)
1033  return getNaturalAlignIndirect(RetTy);
1034  return ABIArgInfo::getDirect();
1035  }
1036 
1037  // Treat an enum type as its underlying type.
1038  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1039  RetTy = EnumTy->getDecl()->getIntegerType();
1040 
1041  return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
1042  : ABIArgInfo::getDirect());
1043 }
1044 
1045 /// IsX86_MMXType - Return true if this is an MMX type.
1046 bool IsX86_MMXType(llvm::Type *IRType) {
1047  // Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.
1048  return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
1049  cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy() &&
1050  IRType->getScalarSizeInBits() != 64;
1051 }
1052 
1053 static llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1054  StringRef Constraint,
1055  llvm::Type* Ty) {
1056  bool IsMMXCons = llvm::StringSwitch<bool>(Constraint)
1057  .Cases("y", "&y", "^Ym", true)
1058  .Default(false);
1059  if (IsMMXCons && Ty->isVectorTy()) {
1060  if (cast<llvm::VectorType>(Ty)->getPrimitiveSizeInBits().getFixedSize() !=
1061  64) {
1062  // Invalid MMX constraint
1063  return nullptr;
1064  }
1065 
1066  return llvm::Type::getX86_MMXTy(CGF.getLLVMContext());
1067  }
1068 
1069  // No operation needed
1070  return Ty;
1071 }
1072 
1073 /// Returns true if this type can be passed in SSE registers with the
1074 /// X86_VectorCall calling convention. Shared between x86_32 and x86_64.
1075 static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {
1076  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
1077  if (BT->isFloatingPoint() && BT->getKind() != BuiltinType::Half) {
1078  if (BT->getKind() == BuiltinType::LongDouble) {
1079  if (&Context.getTargetInfo().getLongDoubleFormat() ==
1080  &llvm::APFloat::x87DoubleExtended())
1081  return false;
1082  }
1083  return true;
1084  }
1085  } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
1086  // vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX
1087  // registers specially.
1088  unsigned VecSize = Context.getTypeSize(VT);
1089  if (VecSize == 128 || VecSize == 256 || VecSize == 512)
1090  return true;
1091  }
1092  return false;
1093 }
1094 
1095 /// Returns true if this aggregate is small enough to be passed in SSE registers
1096 /// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.
1097 static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {
1098  return NumMembers <= 4;
1099 }
1100 
1101 /// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.
1102 static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {
1103  auto AI = ABIArgInfo::getDirect(T);
1104  AI.setInReg(true);
1105  AI.setCanBeFlattened(false);
1106  return AI;
1107 }
1108 
1109 //===----------------------------------------------------------------------===//
1110 // X86-32 ABI Implementation
1111 //===----------------------------------------------------------------------===//
1112 
1113 /// Similar to llvm::CCState, but for Clang.
1114 struct CCState {
1115  CCState(CGFunctionInfo &FI)
1116  : IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()) {}
1117 
1118  llvm::SmallBitVector IsPreassigned;
1119  unsigned CC = CallingConv::CC_C;
1120  unsigned FreeRegs = 0;
1121  unsigned FreeSSERegs = 0;
1122 };
1123 
1124 /// X86_32ABIInfo - The X86-32 ABI information.
1125 class X86_32ABIInfo : public SwiftABIInfo {
1126  enum Class {
1127  Integer,
1128  Float
1129  };
1130 
1131  static const unsigned MinABIStackAlignInBytes = 4;
1132 
1133  bool IsDarwinVectorABI;
1134  bool IsRetSmallStructInRegABI;
1135  bool IsWin32StructABI;
1136  bool IsSoftFloatABI;
1137  bool IsMCUABI;
1138  bool IsLinuxABI;
1139  unsigned DefaultNumRegisterParameters;
1140 
1141  static bool isRegisterSize(unsigned Size) {
1142  return (Size == 8 || Size == 16 || Size == 32 || Size == 64);
1143  }
1144 
1145  bool isHomogeneousAggregateBaseType(QualType Ty) const override {
1146  // FIXME: Assumes vectorcall is in use.
1147  return isX86VectorTypeForVectorCall(getContext(), Ty);
1148  }
1149 
1150  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
1151  uint64_t NumMembers) const override {
1152  // FIXME: Assumes vectorcall is in use.
1153  return isX86VectorCallAggregateSmallEnough(NumMembers);
1154  }
1155 
1156  bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;
1157 
1158  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1159  /// such that the argument will be passed in memory.
1160  ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
1161 
1162  ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
1163 
1164  /// Return the alignment to use for the given type on the stack.
1165  unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
1166 
1167  Class classify(QualType Ty) const;
1168  ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;
1169  ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State) const;
1170 
1171  /// Updates the number of available free registers, returns
1172  /// true if any registers were allocated.
1173  bool updateFreeRegs(QualType Ty, CCState &State) const;
1174 
1175  bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,
1176  bool &NeedsPadding) const;
1177  bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;
1178 
1179  bool canExpandIndirectArgument(QualType Ty) const;
1180 
1181  /// Rewrite the function info so that all memory arguments use
1182  /// inalloca.
1183  void rewriteWithInAlloca(CGFunctionInfo &FI) const;
1184 
1185  void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
1186  CharUnits &StackOffset, ABIArgInfo &Info,
1187  QualType Type) const;
1188  void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;
1189 
1190 public:
1191 
1192  void computeInfo(CGFunctionInfo &FI) const override;
1193  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1194  QualType Ty) const override;
1195 
1196  X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1197  bool RetSmallStructInRegABI, bool Win32StructABI,
1198  unsigned NumRegisterParameters, bool SoftFloatABI)
1199  : SwiftABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
1200  IsRetSmallStructInRegABI(RetSmallStructInRegABI),
1201  IsWin32StructABI(Win32StructABI), IsSoftFloatABI(SoftFloatABI),
1202  IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
1203  IsLinuxABI(CGT.getTarget().getTriple().isOSLinux() ||
1204  CGT.getTarget().getTriple().isOSCygMing()),
1205  DefaultNumRegisterParameters(NumRegisterParameters) {}
1206 
1207  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
1208  bool asReturnValue) const override {
1209  // LLVM's x86-32 lowering currently only assigns up to three
1210  // integer registers and three fp registers. Oddly, it'll use up to
1211  // four vector registers for vectors, but those can overlap with the
1212  // scalar registers.
1213  return occupiesMoreThan(CGT, scalars, /*total*/ 3);
1214  }
1215 
1216  bool isSwiftErrorInRegister() const override {
1217  // x86-32 lowering does not support passing swifterror in a register.
1218  return false;
1219  }
1220 };
1221 
1222 class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
1223 public:
1224  X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1225  bool RetSmallStructInRegABI, bool Win32StructABI,
1226  unsigned NumRegisterParameters, bool SoftFloatABI)
1227  : TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(
1228  CGT, DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI,
1229  NumRegisterParameters, SoftFloatABI)) {}
1230 
1231  static bool isStructReturnInRegABI(
1232  const llvm::Triple &Triple, const CodeGenOptions &Opts);
1233 
1234  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1235  CodeGen::CodeGenModule &CGM) const override;
1236 
1237  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1238  // Darwin uses different dwarf register numbers for EH.
1239  if (CGM.getTarget().getTriple().isOSDarwin()) return 5;
1240  return 4;
1241  }
1242 
1243  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1244  llvm::Value *Address) const override;
1245 
1246  llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1247  StringRef Constraint,
1248  llvm::Type* Ty) const override {
1249  return X86AdjustInlineAsmType(CGF, Constraint, Ty);
1250  }
1251 
1252  void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,
1253  std::string &Constraints,
1254  std::vector<llvm::Type *> &ResultRegTypes,
1255  std::vector<llvm::Type *> &ResultTruncRegTypes,
1256  std::vector<LValue> &ResultRegDests,
1257  std::string &AsmString,
1258  unsigned NumOutputs) const override;
1259 
1260  llvm::Constant *
1261  getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
1262  unsigned Sig = (0xeb << 0) | // jmp rel8
1263  (0x06 << 8) | // .+0x08
1264  ('v' << 16) |
1265  ('2' << 24);
1266  return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
1267  }
1268 
1269  StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
1270  return "movl\t%ebp, %ebp"
1271  "\t\t// marker for objc_retainAutoreleaseReturnValue";
1272  }
1273 };
1274 
1275 }
1276 
1277 /// Rewrite input constraint references after adding some output constraints.
1278 /// In the case where there is one output and one input and we add one output,
1279 /// we need to replace all operand references greater than or equal to 1:
1280 /// mov $0, $1
1281 /// mov eax, $1
1282 /// The result will be:
1283 /// mov $0, $2
1284 /// mov eax, $2
1285 static void rewriteInputConstraintReferences(unsigned FirstIn,
1286  unsigned NumNewOuts,
1287  std::string &AsmString) {
1288  std::string Buf;
1289  llvm::raw_string_ostream OS(Buf);
1290  size_t Pos = 0;
1291  while (Pos < AsmString.size()) {
1292  size_t DollarStart = AsmString.find('$', Pos);
1293  if (DollarStart == std::string::npos)
1294  DollarStart = AsmString.size();
1295  size_t DollarEnd = AsmString.find_first_not_of('$', DollarStart);
1296  if (DollarEnd == std::string::npos)
1297  DollarEnd = AsmString.size();
1298  OS << StringRef(&AsmString[Pos], DollarEnd - Pos);
1299  Pos = DollarEnd;
1300  size_t NumDollars = DollarEnd - DollarStart;
1301  if (NumDollars % 2 != 0 && Pos < AsmString.size()) {
1302  // We have an operand reference.
1303  size_t DigitStart = Pos;
1304  if (AsmString[DigitStart] == '{') {
1305  OS << '{';
1306  ++DigitStart;
1307  }
1308  size_t DigitEnd = AsmString.find_first_not_of("0123456789", DigitStart);
1309  if (DigitEnd == std::string::npos)
1310  DigitEnd = AsmString.size();
1311  StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);
1312  unsigned OperandIndex;
1313  if (!OperandStr.getAsInteger(10, OperandIndex)) {
1314  if (OperandIndex >= FirstIn)
1315  OperandIndex += NumNewOuts;
1316  OS << OperandIndex;
1317  } else {
1318  OS << OperandStr;
1319  }
1320  Pos = DigitEnd;
1321  }
1322  }
1323  AsmString = std::move(OS.str());
1324 }
1325 
1326 /// Add output constraints for EAX:EDX because they are return registers.
1327 void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
1328  CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,
1329  std::vector<llvm::Type *> &ResultRegTypes,
1330  std::vector<llvm::Type *> &ResultTruncRegTypes,
1331  std::vector<LValue> &ResultRegDests, std::string &AsmString,
1332  unsigned NumOutputs) const {
1333  uint64_t RetWidth = CGF.getContext().getTypeSize(ReturnSlot.getType());
1334 
1335  // Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is
1336  // larger.
1337  if (!Constraints.empty())
1338  Constraints += ',';
1339  if (RetWidth <= 32) {
1340  Constraints += "={eax}";
1341  ResultRegTypes.push_back(CGF.Int32Ty);
1342  } else {
1343  // Use the 'A' constraint for EAX:EDX.
1344  Constraints += "=A";
1345  ResultRegTypes.push_back(CGF.Int64Ty);
1346  }
1347 
1348  // Truncate EAX or EAX:EDX to an integer of the appropriate size.
1349  llvm::Type *CoerceTy = llvm::IntegerType::get(CGF.getLLVMContext(), RetWidth);
1350  ResultTruncRegTypes.push_back(CoerceTy);
1351 
1352  // Coerce the integer by bitcasting the return slot pointer.
1353  ReturnSlot.setAddress(CGF.Builder.CreateBitCast(ReturnSlot.getAddress(CGF),
1354  CoerceTy->getPointerTo()));
1355  ResultRegDests.push_back(ReturnSlot);
1356 
1357  rewriteInputConstraintReferences(NumOutputs, 1, AsmString);
1358 }
1359 
1360 /// shouldReturnTypeInRegister - Determine if the given type should be
1361 /// returned in a register (for the Darwin and MCU ABI).
1362 bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
1363  ASTContext &Context) const {
1364  uint64_t Size = Context.getTypeSize(Ty);
1365 
1366  // For i386, type must be register sized.
1367  // For the MCU ABI, it only needs to be <= 8-byte
1368  if ((IsMCUABI && Size > 64) || (!IsMCUABI && !isRegisterSize(Size)))
1369  return false;
1370 
1371  if (Ty->isVectorType()) {
1372  // 64- and 128- bit vectors inside structures are not returned in
1373  // registers.
1374  if (Size == 64 || Size == 128)
1375  return false;
1376 
1377  return true;
1378  }
1379 
1380  // If this is a builtin, pointer, enum, complex type, member pointer, or
1381  // member function pointer it is ok.
1382  if (Ty->getAs<BuiltinType>() || Ty->hasPointerRepresentation() ||
1383  Ty->isAnyComplexType() || Ty->isEnumeralType() ||
1384  Ty->isBlockPointerType() || Ty->isMemberPointerType())
1385  return true;
1386 
1387  // Arrays are treated like records.
1388  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty))
1389  return shouldReturnTypeInRegister(AT->getElementType(), Context);
1390 
1391  // Otherwise, it must be a record type.
1392  const RecordType *RT = Ty->getAs<RecordType>();
1393  if (!RT) return false;
1394 
1395  // FIXME: Traverse bases here too.
1396 
1397  // Structure types are passed in register if all fields would be
1398  // passed in a register.
1399  for (const auto *FD : RT->getDecl()->fields()) {
1400  // Empty fields are ignored.
1401  if (isEmptyField(Context, FD, true))
1402  continue;
1403 
1404  // Check fields recursively.
1405  if (!shouldReturnTypeInRegister(FD->getType(), Context))
1406  return false;
1407  }
1408  return true;
1409 }
1410 
1411 static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
1412  // Treat complex types as the element type.
1413  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
1414  Ty = CTy->getElementType();
1415 
1416  // Check for a type which we know has a simple scalar argument-passing
1417  // convention without any padding. (We're specifically looking for 32
1418  // and 64-bit integer and integer-equivalents, float, and double.)
1419  if (!Ty->getAs<BuiltinType>() && !Ty->hasPointerRepresentation() &&
1420  !Ty->isEnumeralType() && !Ty->isBlockPointerType())
1421  return false;
1422 
1423  uint64_t Size = Context.getTypeSize(Ty);
1424  return Size == 32 || Size == 64;
1425 }
1426 
1427 static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,
1428  uint64_t &Size) {
1429  for (const auto *FD : RD->fields()) {
1430  // Scalar arguments on the stack get 4 byte alignment on x86. If the
1431  // argument is smaller than 32-bits, expanding the struct will create
1432  // alignment padding.
1433  if (!is32Or64BitBasicType(FD->getType(), Context))
1434  return false;
1435 
1436  // FIXME: Reject bit-fields wholesale; there are two problems, we don't know
1437  // how to expand them yet, and the predicate for telling if a bitfield still
1438  // counts as "basic" is more complicated than what we were doing previously.
1439  if (FD->isBitField())
1440  return false;
1441 
1442  Size += Context.getTypeSize(FD->getType());
1443  }
1444  return true;
1445 }
1446 
1447 static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,
1448  uint64_t &Size) {
1449  // Don't do this if there are any non-empty bases.
1450  for (const CXXBaseSpecifier &Base : RD->bases()) {
1451  if (!addBaseAndFieldSizes(Context, Base.getType()->getAsCXXRecordDecl(),
1452  Size))
1453  return false;
1454  }
1455  if (!addFieldSizes(Context, RD, Size))
1456  return false;
1457  return true;
1458 }
1459 
1460 /// Test whether an argument type which is to be passed indirectly (on the
1461 /// stack) would have the equivalent layout if it was expanded into separate
1462 /// arguments. If so, we prefer to do the latter to avoid inhibiting
1463 /// optimizations.
1464 bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
1465  // We can only expand structure types.
1466  const RecordType *RT = Ty->getAs<RecordType>();
1467  if (!RT)
1468  return false;
1469  const RecordDecl *RD = RT->getDecl();
1470  uint64_t Size = 0;
1471  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
1472  if (!IsWin32StructABI) {
1473  // On non-Windows, we have to conservatively match our old bitcode
1474  // prototypes in order to be ABI-compatible at the bitcode level.
1475  if (!CXXRD->isCLike())
1476  return false;
1477  } else {
1478  // Don't do this for dynamic classes.
1479  if (CXXRD->isDynamicClass())
1480  return false;
1481  }
1482  if (!addBaseAndFieldSizes(getContext(), CXXRD, Size))
1483  return false;
1484  } else {
1485  if (!addFieldSizes(getContext(), RD, Size))
1486  return false;
1487  }
1488 
1489  // We can do this if there was no alignment padding.
1490  return Size == getContext().getTypeSize(Ty);
1491 }
1492 
1493 ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
1494  // If the return value is indirect, then the hidden argument is consuming one
1495  // integer register.
1496  if (State.FreeRegs) {
1497  --State.FreeRegs;
1498  if (!IsMCUABI)
1499  return getNaturalAlignIndirectInReg(RetTy);
1500  }
1501  return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
1502 }
1503 
1505  CCState &State) const {
1506  if (RetTy->isVoidType())
1507  return ABIArgInfo::getIgnore();
1508 
1509  const Type *Base = nullptr;
1510  uint64_t NumElts = 0;
1511  if ((State.CC == llvm::CallingConv::X86_VectorCall ||
1512  State.CC == llvm::CallingConv::X86_RegCall) &&
1513  isHomogeneousAggregate(RetTy, Base, NumElts)) {
1514  // The LLVM struct type for such an aggregate should lower properly.
1515  return ABIArgInfo::getDirect();
1516  }
1517 
1518  if (const VectorType *VT = RetTy->getAs<VectorType>()) {
1519  // On Darwin, some vectors are returned in registers.
1520  if (IsDarwinVectorABI) {
1521  uint64_t Size = getContext().getTypeSize(RetTy);
1522 
1523  // 128-bit vectors are a special case; they are returned in
1524  // registers and we need to make sure to pick a type the LLVM
1525  // backend will like.
1526  if (Size == 128)
1527  return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1528  llvm::Type::getInt64Ty(getVMContext()), 2));
1529 
1530  // Always return in register if it fits in a general purpose
1531  // register, or if it is 64 bits and has a single element.
1532  if ((Size == 8 || Size == 16 || Size == 32) ||
1533  (Size == 64 && VT->getNumElements() == 1))
1534  return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
1535  Size));
1536 
1537  return getIndirectReturnResult(RetTy, State);
1538  }
1539 
1540  return ABIArgInfo::getDirect();
1541  }
1542 
1543  if (isAggregateTypeForABI(RetTy)) {
1544  if (const RecordType *RT = RetTy->getAs<RecordType>()) {
1545  // Structures with flexible arrays are always indirect.
1546  if (RT->getDecl()->hasFlexibleArrayMember())
1547  return getIndirectReturnResult(RetTy, State);
1548  }
1549 
1550  // If specified, structs and unions are always indirect.
1551  if (!IsRetSmallStructInRegABI && !RetTy->isAnyComplexType())
1552  return getIndirectReturnResult(RetTy, State);
1553 
1554  // Ignore empty structs/unions.
1555  if (isEmptyRecord(getContext(), RetTy, true))
1556  return ABIArgInfo::getIgnore();
1557 
1558  // Return complex of _Float16 as <2 x half> so the backend will use xmm0.
1559  if (const ComplexType *CT = RetTy->getAs<ComplexType>()) {
1560  QualType ET = getContext().getCanonicalType(CT->getElementType());
1561  if (ET->isFloat16Type())
1562  return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1563  llvm::Type::getHalfTy(getVMContext()), 2));
1564  }
1565 
1566  // Small structures which are register sized are generally returned
1567  // in a register.
1568  if (shouldReturnTypeInRegister(RetTy, getContext())) {
1569  uint64_t Size = getContext().getTypeSize(RetTy);
1570 
1571  // As a special-case, if the struct is a "single-element" struct, and
1572  // the field is of type "float" or "double", return it in a
1573  // floating-point register. (MSVC does not apply this special case.)
1574  // We apply a similar transformation for pointer types to improve the
1575  // quality of the generated IR.
1576  if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
1577  if ((!IsWin32StructABI && SeltTy->isRealFloatingType())
1578  || SeltTy->hasPointerRepresentation())
1579  return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
1580 
1581  // FIXME: We should be able to narrow this integer in cases with dead
1582  // padding.
1583  return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),Size));
1584  }
1585 
1586  return getIndirectReturnResult(RetTy, State);
1587  }
1588 
1589  // Treat an enum type as its underlying type.
1590  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1591  RetTy = EnumTy->getDecl()->getIntegerType();
1592 
1593  if (const auto *EIT = RetTy->getAs<ExtIntType>())
1594  if (EIT->getNumBits() > 64)
1595  return getIndirectReturnResult(RetTy, State);
1596 
1597  return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
1598  : ABIArgInfo::getDirect());
1599 }
1600 
1601 static bool isSIMDVectorType(ASTContext &Context, QualType Ty) {
1602  return Ty->getAs<VectorType>() && Context.getTypeSize(Ty) == 128;
1603 }
1604 
1606  const RecordType *RT = Ty->getAs<RecordType>();
1607  if (!RT)
1608  return 0;
1609  const RecordDecl *RD = RT->getDecl();
1610 
1611  // If this is a C++ record, check the bases first.
1612  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
1613  for (const auto &I : CXXRD->bases())
1614  if (!isRecordWithSIMDVectorType(Context, I.getType()))
1615  return false;
1616 
1617  for (const auto *i : RD->fields()) {
1618  QualType FT = i->getType();
1619 
1620  if (isSIMDVectorType(Context, FT))
1621  return true;
1622 
1623  if (isRecordWithSIMDVectorType(Context, FT))
1624  return true;
1625  }
1626 
1627  return false;
1628 }
1629 
1630 unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
1631  unsigned Align) const {
1632  // Otherwise, if the alignment is less than or equal to the minimum ABI
1633  // alignment, just use the default; the backend will handle this.
1634  if (Align <= MinABIStackAlignInBytes)
1635  return 0; // Use default alignment.
1636 
1637  if (IsLinuxABI) {
1638  // Exclude other System V OS (e.g Darwin, PS4 and FreeBSD) since we don't
1639  // want to spend any effort dealing with the ramifications of ABI breaks.
1640  //
1641  // If the vector type is __m128/__m256/__m512, return the default alignment.
1642  if (Ty->isVectorType() && (Align == 16 || Align == 32 || Align == 64))
1643  return Align;
1644  }
1645  // On non-Darwin, the stack type alignment is always 4.
1646  if (!IsDarwinVectorABI) {
1647  // Set explicit alignment, since we may need to realign the top.
1648  return MinABIStackAlignInBytes;
1649  }
1650 
1651  // Otherwise, if the type contains an SSE vector type, the alignment is 16.
1652  if (Align >= 16 && (isSIMDVectorType(getContext(), Ty) ||
1653  isRecordWithSIMDVectorType(getContext(), Ty)))
1654  return 16;
1655 
1656  return MinABIStackAlignInBytes;
1657 }
1658 
1659 ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
1660  CCState &State) const {
1661  if (!ByVal) {
1662  if (State.FreeRegs) {
1663  --State.FreeRegs; // Non-byval indirects just use one pointer.
1664  if (!IsMCUABI)
1665  return getNaturalAlignIndirectInReg(Ty);
1666  }
1667  return getNaturalAlignIndirect(Ty, false);
1668  }
1669 
1670  // Compute the byval alignment.
1671  unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
1672  unsigned StackAlign = getTypeStackAlignInBytes(Ty, TypeAlign);
1673  if (StackAlign == 0)
1674  return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true);
1675 
1676  // If the stack alignment is less than the type alignment, realign the
1677  // argument.
1678  bool Realign = TypeAlign > StackAlign;
1680  /*ByVal=*/true, Realign);
1681 }
1682 
1683 X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
1684  const Type *T = isSingleElementStruct(Ty, getContext());
1685  if (!T)
1686  T = Ty.getTypePtr();
1687 
1688  if (const BuiltinType *BT = T->getAs<BuiltinType>()) {
1689  BuiltinType::Kind K = BT->getKind();
1690  if (K == BuiltinType::Float || K == BuiltinType::Double)
1691  return Float;
1692  }
1693  return Integer;
1694 }
1695 
1696 bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {
1697  if (!IsSoftFloatABI) {
1698  Class C = classify(Ty);
1699  if (C == Float)
1700  return false;
1701  }
1702 
1703  unsigned Size = getContext().getTypeSize(Ty);
1704  unsigned SizeInRegs = (Size + 31) / 32;
1705 
1706  if (SizeInRegs == 0)
1707  return false;
1708 
1709  if (!IsMCUABI) {
1710  if (SizeInRegs > State.FreeRegs) {
1711  State.FreeRegs = 0;
1712  return false;
1713  }
1714  } else {
1715  // The MCU psABI allows passing parameters in-reg even if there are
1716  // earlier parameters that are passed on the stack. Also,
1717  // it does not allow passing >8-byte structs in-register,
1718  // even if there are 3 free registers available.
1719  if (SizeInRegs > State.FreeRegs || SizeInRegs > 2)
1720  return false;
1721  }
1722 
1723  State.FreeRegs -= SizeInRegs;
1724  return true;
1725 }
1726 
1727 bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
1728  bool &InReg,
1729  bool &NeedsPadding) const {
1730  // On Windows, aggregates other than HFAs are never passed in registers, and
1731  // they do not consume register slots. Homogenous floating-point aggregates
1732  // (HFAs) have already been dealt with at this point.
1733  if (IsWin32StructABI && isAggregateTypeForABI(Ty))
1734  return false;
1735 
1736  NeedsPadding = false;
1737  InReg = !IsMCUABI;
1738 
1739  if (!updateFreeRegs(Ty, State))
1740  return false;
1741 
1742  if (IsMCUABI)
1743  return true;
1744 
1745  if (State.CC == llvm::CallingConv::X86_FastCall ||
1746  State.CC == llvm::CallingConv::X86_VectorCall ||
1747  State.CC == llvm::CallingConv::X86_RegCall) {
1748  if (getContext().getTypeSize(Ty) <= 32 && State.FreeRegs)
1749  NeedsPadding = true;
1750 
1751  return false;
1752  }
1753 
1754  return true;
1755 }
1756 
1757 bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
1758  if (!updateFreeRegs(Ty, State))
1759  return false;
1760 
1761  if (IsMCUABI)
1762  return false;
1763 
1764  if (State.CC == llvm::CallingConv::X86_FastCall ||
1765  State.CC == llvm::CallingConv::X86_VectorCall ||
1766  State.CC == llvm::CallingConv::X86_RegCall) {
1767  if (getContext().getTypeSize(Ty) > 32)
1768  return false;
1769 
1770  return (Ty->isIntegralOrEnumerationType() || Ty->isPointerType() ||
1771  Ty->isReferenceType());
1772  }
1773 
1774  return true;
1775 }
1776 
1777 void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
1778  // Vectorcall x86 works subtly different than in x64, so the format is
1779  // a bit different than the x64 version. First, all vector types (not HVAs)
1780  // are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
1781  // This differs from the x64 implementation, where the first 6 by INDEX get
1782  // registers.
1783  // In the second pass over the arguments, HVAs are passed in the remaining
1784  // vector registers if possible, or indirectly by address. The address will be
1785  // passed in ECX/EDX if available. Any other arguments are passed according to
1786  // the usual fastcall rules.
1788  for (int I = 0, E = Args.size(); I < E; ++I) {
1789  const Type *Base = nullptr;
1790  uint64_t NumElts = 0;
1791  const QualType &Ty = Args[I].type;
1792  if ((Ty->isVectorType() || Ty->isBuiltinType()) &&
1793  isHomogeneousAggregate(Ty, Base, NumElts)) {
1794  if (State.FreeSSERegs >= NumElts) {
1795  State.FreeSSERegs -= NumElts;
1796  Args[I].info = ABIArgInfo::getDirectInReg();
1797  State.IsPreassigned.set(I);
1798  }
1799  }
1800  }
1801 }
1802 
1804  CCState &State) const {
1805  // FIXME: Set alignment on indirect arguments.
1806  bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;
1807  bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;
1808  bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
1809 
1811  TypeInfo TI = getContext().getTypeInfo(Ty);
1812 
1813  // Check with the C++ ABI first.
1814  const RecordType *RT = Ty->getAs<RecordType>();
1815  if (RT) {
1816  CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
1817  if (RAA == CGCXXABI::RAA_Indirect) {
1818  return getIndirectResult(Ty, false, State);
1819  } else if (RAA == CGCXXABI::RAA_DirectInMemory) {
1820  // The field index doesn't matter, we'll fix it up later.
1821  return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);
1822  }
1823  }
1824 
1825  // Regcall uses the concept of a homogenous vector aggregate, similar
1826  // to other targets.
1827  const Type *Base = nullptr;
1828  uint64_t NumElts = 0;
1829  if ((IsRegCall || IsVectorCall) &&
1830  isHomogeneousAggregate(Ty, Base, NumElts)) {
1831  if (State.FreeSSERegs >= NumElts) {
1832  State.FreeSSERegs -= NumElts;
1833 
1834  // Vectorcall passes HVAs directly and does not flatten them, but regcall
1835  // does.
1836  if (IsVectorCall)
1837  return getDirectX86Hva();
1838 
1839  if (Ty->isBuiltinType() || Ty->isVectorType())
1840  return ABIArgInfo::getDirect();
1841  return ABIArgInfo::getExpand();
1842  }
1843  return getIndirectResult(Ty, /*ByVal=*/false, State);
1844  }
1845 
1846  if (isAggregateTypeForABI(Ty)) {
1847  // Structures with flexible arrays are always indirect.
1848  // FIXME: This should not be byval!
1849  if (RT && RT->getDecl()->hasFlexibleArrayMember())
1850  return getIndirectResult(Ty, true, State);
1851 
1852  // Ignore empty structs/unions on non-Windows.
1853  if (!IsWin32StructABI && isEmptyRecord(getContext(), Ty, true))
1854  return ABIArgInfo::getIgnore();
1855 
1856  llvm::LLVMContext &LLVMContext = getVMContext();
1857  llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);
1858  bool NeedsPadding = false;
1859  bool InReg;
1860  if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
1861  unsigned SizeInRegs = (TI.Width + 31) / 32;
1862  SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
1863  llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
1864  if (InReg)
1865  return ABIArgInfo::getDirectInReg(Result);
1866  else
1867  return ABIArgInfo::getDirect(Result);
1868  }
1869  llvm::IntegerType *PaddingType = NeedsPadding ? Int32 : nullptr;
1870 
1871  // Pass over-aligned aggregates on Windows indirectly. This behavior was
1872  // added in MSVC 2015.
1873  if (IsWin32StructABI && TI.isAlignRequired() && TI.Align > 32)
1874  return getIndirectResult(Ty, /*ByVal=*/false, State);
1875 
1876  // Expand small (<= 128-bit) record types when we know that the stack layout
1877  // of those arguments will match the struct. This is important because the
1878  // LLVM backend isn't smart enough to remove byval, which inhibits many
1879  // optimizations.
1880  // Don't do this for the MCU if there are still free integer registers
1881  // (see X86_64 ABI for full explanation).
1882  if (TI.Width <= 4 * 32 && (!IsMCUABI || State.FreeRegs == 0) &&
1883  canExpandIndirectArgument(Ty))
1885  IsFastCall || IsVectorCall || IsRegCall, PaddingType);
1886 
1887  return getIndirectResult(Ty, true, State);
1888  }
1889 
1890  if (const VectorType *VT = Ty->getAs<VectorType>()) {
1891  // On Windows, vectors are passed directly if registers are available, or
1892  // indirectly if not. This avoids the need to align argument memory. Pass
1893  // user-defined vector types larger than 512 bits indirectly for simplicity.
1894  if (IsWin32StructABI) {
1895  if (TI.Width <= 512 && State.FreeSSERegs > 0) {
1896  --State.FreeSSERegs;
1897  return ABIArgInfo::getDirectInReg();
1898  }
1899  return getIndirectResult(Ty, /*ByVal=*/false, State);
1900  }
1901 
1902  // On Darwin, some vectors are passed in memory, we handle this by passing
1903  // it as an i8/i16/i32/i64.
1904  if (IsDarwinVectorABI) {
1905  if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
1906  (TI.Width == 64 && VT->getNumElements() == 1))
1907  return ABIArgInfo::getDirect(
1908  llvm::IntegerType::get(getVMContext(), TI.Width));
1909  }
1910 
1911  if (IsX86_MMXType(CGT.ConvertType(Ty)))
1912  return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), 64));
1913 
1914  return ABIArgInfo::getDirect();
1915  }
1916 
1917 
1918  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
1919  Ty = EnumTy->getDecl()->getIntegerType();
1920 
1921  bool InReg = shouldPrimitiveUseInReg(Ty, State);
1922 
1923  if (isPromotableIntegerTypeForABI(Ty)) {
1924  if (InReg)
1925  return ABIArgInfo::getExtendInReg(Ty);
1926  return ABIArgInfo::getExtend(Ty);
1927  }
1928 
1929  if (const auto * EIT = Ty->getAs<ExtIntType>()) {
1930  if (EIT->getNumBits() <= 64) {
1931  if (InReg)
1932  return ABIArgInfo::getDirectInReg();
1933  return ABIArgInfo::getDirect();
1934  }
1935  return getIndirectResult(Ty, /*ByVal=*/false, State);
1936  }
1937 
1938  if (InReg)
1939  return ABIArgInfo::getDirectInReg();
1940  return ABIArgInfo::getDirect();
1941 }
1942 
1943 void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
1944  CCState State(FI);
1945  if (IsMCUABI)
1946  State.FreeRegs = 3;
1947  else if (State.CC == llvm::CallingConv::X86_FastCall) {
1948  State.FreeRegs = 2;
1949  State.FreeSSERegs = 3;
1950  } else if (State.CC == llvm::CallingConv::X86_VectorCall) {
1951  State.FreeRegs = 2;
1952  State.FreeSSERegs = 6;
1953  } else if (FI.getHasRegParm())
1954  State.FreeRegs = FI.getRegParm();
1955  else if (State.CC == llvm::CallingConv::X86_RegCall) {
1956  State.FreeRegs = 5;
1957  State.FreeSSERegs = 8;
1958  } else if (IsWin32StructABI) {
1959  // Since MSVC 2015, the first three SSE vectors have been passed in
1960  // registers. The rest are passed indirectly.
1961  State.FreeRegs = DefaultNumRegisterParameters;
1962  State.FreeSSERegs = 3;
1963  } else
1964  State.FreeRegs = DefaultNumRegisterParameters;
1965 
1966  if (!::classifyReturnType(getCXXABI(), FI, *this)) {
1968  } else if (FI.getReturnInfo().isIndirect()) {
1969  // The C++ ABI is not aware of register usage, so we have to check if the
1970  // return value was sret and put it in a register ourselves if appropriate.
1971  if (State.FreeRegs) {
1972  --State.FreeRegs; // The sret parameter consumes a register.
1973  if (!IsMCUABI)
1974  FI.getReturnInfo().setInReg(true);
1975  }
1976  }
1977 
1978  // The chain argument effectively gives us another free register.
1979  if (FI.isChainCall())
1980  ++State.FreeRegs;
1981 
1982  // For vectorcall, do a first pass over the arguments, assigning FP and vector
1983  // arguments to XMM registers as available.
1984  if (State.CC == llvm::CallingConv::X86_VectorCall)
1985  runVectorCallFirstPass(FI, State);
1986 
1987  bool UsedInAlloca = false;
1989  for (int I = 0, E = Args.size(); I < E; ++I) {
1990  // Skip arguments that have already been assigned.
1991  if (State.IsPreassigned.test(I))
1992  continue;
1993 
1994  Args[I].info = classifyArgumentType(Args[I].type, State);
1995  UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);
1996  }
1997 
1998  // If we needed to use inalloca for any argument, do a second pass and rewrite
1999  // all the memory arguments to use inalloca.
2000  if (UsedInAlloca)
2001  rewriteWithInAlloca(FI);
2002 }
2003 
2004 void
2005 X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
2006  CharUnits &StackOffset, ABIArgInfo &Info,
2007  QualType Type) const {
2008  // Arguments are always 4-byte-aligned.
2009  CharUnits WordSize = CharUnits::fromQuantity(4);
2010  assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
2011 
2012  // sret pointers and indirect things will require an extra pointer
2013  // indirection, unless they are byval. Most things are byval, and will not
2014  // require this indirection.
2015  bool IsIndirect = false;
2016  if (Info.isIndirect() && !Info.getIndirectByVal())
2017  IsIndirect = true;
2018  Info = ABIArgInfo::getInAlloca(FrameFields.size(), IsIndirect);
2019  llvm::Type *LLTy = CGT.ConvertTypeForMem(Type);
2020  if (IsIndirect)
2021  LLTy = LLTy->getPointerTo(0);
2022  FrameFields.push_back(LLTy);
2023  StackOffset += IsIndirect ? WordSize : getContext().getTypeSizeInChars(Type);
2024 
2025  // Insert padding bytes to respect alignment.
2026  CharUnits FieldEnd = StackOffset;
2027  StackOffset = FieldEnd.alignTo(WordSize);
2028  if (StackOffset != FieldEnd) {
2029  CharUnits NumBytes = StackOffset - FieldEnd;
2030  llvm::Type *Ty = llvm::Type::getInt8Ty(getVMContext());
2031  Ty = llvm::ArrayType::get(Ty, NumBytes.getQuantity());
2032  FrameFields.push_back(Ty);
2033  }
2034 }
2035 
2036 static bool isArgInAlloca(const ABIArgInfo &Info) {
2037  // Leave ignored and inreg arguments alone.
2038  switch (Info.getKind()) {
2039  case ABIArgInfo::InAlloca:
2040  return true;
2041  case ABIArgInfo::Ignore:
2043  return false;
2044  case ABIArgInfo::Indirect:
2045  case ABIArgInfo::Direct:
2046  case ABIArgInfo::Extend:
2047  return !Info.getInReg();
2048  case ABIArgInfo::Expand:
2050  // These are aggregate types which are never passed in registers when
2051  // inalloca is involved.
2052  return true;
2053  }
2054  llvm_unreachable("invalid enum");
2055 }
2056 
2057 void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
2058  assert(IsWin32StructABI && "inalloca only supported on win32");
2059 
2060  // Build a packed struct type for all of the arguments in memory.
2061  SmallVector<llvm::Type *, 6> FrameFields;
2062 
2063  // The stack alignment is always 4.
2064  CharUnits StackAlign = CharUnits::fromQuantity(4);
2065 
2066  CharUnits StackOffset;
2067  CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();
2068 
2069  // Put 'this' into the struct before 'sret', if necessary.
2070  bool IsThisCall =
2071  FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;
2072  ABIArgInfo &Ret = FI.getReturnInfo();
2073  if (Ret.isIndirect() && Ret.isSRetAfterThis() && !IsThisCall &&
2074  isArgInAlloca(I->info)) {
2075  addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2076  ++I;
2077  }
2078 
2079  // Put the sret parameter into the inalloca struct if it's in memory.
2080  if (Ret.isIndirect() && !Ret.getInReg()) {
2081  addFieldToArgStruct(FrameFields, StackOffset, Ret, FI.getReturnType());
2082  // On Windows, the hidden sret parameter is always returned in eax.
2083  Ret.setInAllocaSRet(IsWin32StructABI);
2084  }
2085 
2086  // Skip the 'this' parameter in ecx.
2087  if (IsThisCall)
2088  ++I;
2089 
2090  // Put arguments passed in memory into the struct.
2091  for (; I != E; ++I) {
2092  if (isArgInAlloca(I->info))
2093  addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2094  }
2095 
2096  FI.setArgStruct(llvm::StructType::get(getVMContext(), FrameFields,
2097  /*isPacked=*/true),
2098  StackAlign);
2099 }
2100 
2101 Address X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF,
2102  Address VAListAddr, QualType Ty) const {
2103 
2104  auto TypeInfo = getContext().getTypeInfoInChars(Ty);
2105 
2106  // x86-32 changes the alignment of certain arguments on the stack.
2107  //
2108  // Just messing with TypeInfo like this works because we never pass
2109  // anything indirectly.
2111  getTypeStackAlignInBytes(Ty, TypeInfo.Align.getQuantity()));
2112 
2113  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false,
2115  /*AllowHigherAlign*/ true);
2116 }
2117 
2118 bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
2119  const llvm::Triple &Triple, const CodeGenOptions &Opts) {
2120  assert(Triple.getArch() == llvm::Triple::x86);
2121 
2122  switch (Opts.getStructReturnConvention()) {
2124  break;
2125  case CodeGenOptions::SRCK_OnStack: // -fpcc-struct-return
2126  return false;
2127  case CodeGenOptions::SRCK_InRegs: // -freg-struct-return
2128  return true;
2129  }
2130 
2131  if (Triple.isOSDarwin() || Triple.isOSIAMCU())
2132  return true;
2133 
2134  switch (Triple.getOS()) {
2135  case llvm::Triple::DragonFly:
2136  case llvm::Triple::FreeBSD:
2137  case llvm::Triple::OpenBSD:
2138  case llvm::Triple::Win32:
2139  return true;
2140  default:
2141  return false;
2142  }
2143 }
2144 
2145 static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,
2146  CodeGen::CodeGenModule &CGM) {
2147  if (!FD->hasAttr<AnyX86InterruptAttr>())
2148  return;
2149 
2150  llvm::Function *Fn = cast<llvm::Function>(GV);
2151  Fn->setCallingConv(llvm::CallingConv::X86_INTR);
2152  if (FD->getNumParams() == 0)
2153  return;
2154 
2155  auto PtrTy = cast<PointerType>(FD->getParamDecl(0)->getType());
2156  llvm::Type *ByValTy = CGM.getTypes().ConvertType(PtrTy->getPointeeType());
2157  llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(
2158  Fn->getContext(), ByValTy);
2159  Fn->addParamAttr(0, NewAttr);
2160 }
2161 
2162 void X86_32TargetCodeGenInfo::setTargetAttributes(
2163  const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2164  if (GV->isDeclaration())
2165  return;
2166  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2167  if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2168  llvm::Function *Fn = cast<llvm::Function>(GV);
2169  Fn->addFnAttr("stackrealign");
2170  }
2171 
2172  addX86InterruptAttrs(FD, GV, CGM);
2173  }
2174 }
2175 
2176 bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
2178  llvm::Value *Address) const {
2179  CodeGen::CGBuilderTy &Builder = CGF.Builder;
2180 
2181  llvm::Value *Four8 = llvm::ConstantInt::get(CGF.Int8Ty, 4);
2182 
2183  // 0-7 are the eight integer registers; the order is different
2184  // on Darwin (for EH), but the range is the same.
2185  // 8 is %eip.
2186  AssignToArrayRange(Builder, Address, Four8, 0, 8);
2187 
2188  if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {
2189  // 12-16 are st(0..4). Not sure why we stop at 4.
2190  // These have size 16, which is sizeof(long double) on
2191  // platforms with 8-byte alignment for that type.
2192  llvm::Value *Sixteen8 = llvm::ConstantInt::get(CGF.Int8Ty, 16);
2193  AssignToArrayRange(Builder, Address, Sixteen8, 12, 16);
2194 
2195  } else {
2196  // 9 is %eflags, which doesn't get a size on Darwin for some
2197  // reason.
2198  Builder.CreateAlignedStore(
2199  Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9),
2200  CharUnits::One());
2201 
2202  // 11-16 are st(0..5). Not sure why we stop at 5.
2203  // These have size 12, which is sizeof(long double) on
2204  // platforms with 4-byte alignment for that type.
2205  llvm::Value *Twelve8 = llvm::ConstantInt::get(CGF.Int8Ty, 12);
2206  AssignToArrayRange(Builder, Address, Twelve8, 11, 16);
2207  }
2208 
2209  return false;
2210 }
2211 
2212 //===----------------------------------------------------------------------===//
2213 // X86-64 ABI Implementation
2214 //===----------------------------------------------------------------------===//
2215 
2216 
2217 namespace {
2218 /// The AVX ABI level for X86 targets.
2219 enum class X86AVXABILevel {
2220  None,
2221  AVX,
2222  AVX512
2223 };
2224 
2225 /// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
2226 static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
2227  switch (AVXLevel) {
2228  case X86AVXABILevel::AVX512:
2229  return 512;
2230  case X86AVXABILevel::AVX:
2231  return 256;
2232  case X86AVXABILevel::None:
2233  return 128;
2234  }
2235  llvm_unreachable("Unknown AVXLevel");
2236 }
2237 
2238 /// X86_64ABIInfo - The X86_64 ABI information.
2239 class X86_64ABIInfo : public SwiftABIInfo {
2240  enum Class {
2241  Integer = 0,
2242  SSE,
2243  SSEUp,
2244  X87,
2245  X87Up,
2246  ComplexX87,
2247  NoClass,
2248  Memory
2249  };
2250 
2251  /// merge - Implement the X86_64 ABI merging algorithm.
2252  ///
2253  /// Merge an accumulating classification \arg Accum with a field
2254  /// classification \arg Field.
2255  ///
2256  /// \param Accum - The accumulating classification. This should
2257  /// always be either NoClass or the result of a previous merge
2258  /// call. In addition, this should never be Memory (the caller
2259  /// should just return Memory for the aggregate).
2260  static Class merge(Class Accum, Class Field);
2261 
2262  /// postMerge - Implement the X86_64 ABI post merging algorithm.
2263  ///
2264  /// Post merger cleanup, reduces a malformed Hi and Lo pair to
2265  /// final MEMORY or SSE classes when necessary.
2266  ///
2267  /// \param AggregateSize - The size of the current aggregate in
2268  /// the classification process.
2269  ///
2270  /// \param Lo - The classification for the parts of the type
2271  /// residing in the low word of the containing object.
2272  ///
2273  /// \param Hi - The classification for the parts of the type
2274  /// residing in the higher words of the containing object.
2275  ///
2276  void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
2277 
2278  /// classify - Determine the x86_64 register classes in which the
2279  /// given type T should be passed.
2280  ///
2281  /// \param Lo - The classification for the parts of the type
2282  /// residing in the low word of the containing object.
2283  ///
2284  /// \param Hi - The classification for the parts of the type
2285  /// residing in the high word of the containing object.
2286  ///
2287  /// \param OffsetBase - The bit offset of this type in the
2288  /// containing object. Some parameters are classified different
2289  /// depending on whether they straddle an eightbyte boundary.
2290  ///
2291  /// \param isNamedArg - Whether the argument in question is a "named"
2292  /// argument, as used in AMD64-ABI 3.5.7.
2293  ///
2294  /// If a word is unused its result will be NoClass; if a type should
2295  /// be passed in Memory then at least the classification of \arg Lo
2296  /// will be Memory.
2297  ///
2298  /// The \arg Lo class will be NoClass iff the argument is ignored.
2299  ///
2300  /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
2301  /// also be ComplexX87.
2302  void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
2303  bool isNamedArg) const;
2304 
2305  llvm::Type *GetByteVectorType(QualType Ty) const;
2306  llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
2307  unsigned IROffset, QualType SourceTy,
2308  unsigned SourceOffset) const;
2309  llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,
2310  unsigned IROffset, QualType SourceTy,
2311  unsigned SourceOffset) const;
2312 
2313  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2314  /// such that the argument will be returned in memory.
2315  ABIArgInfo getIndirectReturnResult(QualType Ty) const;
2316 
2317  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2318  /// such that the argument will be passed in memory.
2319  ///
2320  /// \param freeIntRegs - The number of free integer registers remaining
2321  /// available.
2322  ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;
2323 
2324  ABIArgInfo classifyReturnType(QualType RetTy) const;
2325 
2326  ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
2327  unsigned &neededInt, unsigned &neededSSE,
2328  bool isNamedArg) const;
2329 
2330  ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
2331  unsigned &NeededSSE) const;
2332 
2333  ABIArgInfo classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
2334  unsigned &NeededSSE) const;
2335 
2336  bool IsIllegalVectorType(QualType Ty) const;
2337 
2338  /// The 0.98 ABI revision clarified a lot of ambiguities,
2339  /// unfortunately in ways that were not always consistent with
2340  /// certain previous compilers. In particular, platforms which
2341  /// required strict binary compatibility with older versions of GCC
2342  /// may need to exempt themselves.
2343  bool honorsRevision0_98() const {
2344  return !getTarget().getTriple().isOSDarwin();
2345  }
2346 
2347  /// GCC classifies <1 x long long> as SSE but some platform ABIs choose to
2348  /// classify it as INTEGER (for compatibility with older clang compilers).
2349  bool classifyIntegerMMXAsSSE() const {
2350  // Clang <= 3.8 did not do this.
2351  if (getContext().getLangOpts().getClangABICompat() <=
2353  return false;
2354 
2355  const llvm::Triple &Triple = getTarget().getTriple();
2356  if (Triple.isOSDarwin() || Triple.getOS() == llvm::Triple::PS4)
2357  return false;
2358  if (Triple.isOSFreeBSD() && Triple.getOSMajorVersion() >= 10)
2359  return false;
2360  return true;
2361  }
2362 
2363  // GCC classifies vectors of __int128 as memory.
2364  bool passInt128VectorsInMem() const {
2365  // Clang <= 9.0 did not do this.
2366  if (getContext().getLangOpts().getClangABICompat() <=
2368  return false;
2369 
2370  const llvm::Triple &T = getTarget().getTriple();
2371  return T.isOSLinux() || T.isOSNetBSD();
2372  }
2373 
2374  X86AVXABILevel AVXLevel;
2375  // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
2376  // 64-bit hardware.
2377  bool Has64BitPointers;
2378 
2379 public:
2380  X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel) :
2381  SwiftABIInfo(CGT), AVXLevel(AVXLevel),
2382  Has64BitPointers(CGT.getDataLayout().getPointerSize(0) == 8) {
2383  }
2384 
2385  bool isPassedUsingAVXType(QualType type) const {
2386  unsigned neededInt, neededSSE;
2387  // The freeIntRegs argument doesn't matter here.
2388  ABIArgInfo info = classifyArgumentType(type, 0, neededInt, neededSSE,
2389  /*isNamedArg*/true);
2390  if (info.isDirect()) {
2391  llvm::Type *ty = info.getCoerceToType();
2392  if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(ty))
2393  return vectorTy->getPrimitiveSizeInBits().getFixedSize() > 128;
2394  }
2395  return false;
2396  }
2397 
2398  void computeInfo(CGFunctionInfo &FI) const override;
2399 
2400  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2401  QualType Ty) const override;
2402  Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
2403  QualType Ty) const override;
2404 
2405  bool has64BitPointers() const {
2406  return Has64BitPointers;
2407  }
2408 
2409  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
2410  bool asReturnValue) const override {
2411  return occupiesMoreThan(CGT, scalars, /*total*/ 4);
2412  }
2413  bool isSwiftErrorInRegister() const override {
2414  return true;
2415  }
2416 };
2417 
2418 /// WinX86_64ABIInfo - The Windows X86_64 ABI information.
2419 class WinX86_64ABIInfo : public SwiftABIInfo {
2420 public:
2421  WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2422  : SwiftABIInfo(CGT), AVXLevel(AVXLevel),
2423  IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
2424 
2425  void computeInfo(CGFunctionInfo &FI) const override;
2426 
2427  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2428  QualType Ty) const override;
2429 
2430  bool isHomogeneousAggregateBaseType(QualType Ty) const override {
2431  // FIXME: Assumes vectorcall is in use.
2432  return isX86VectorTypeForVectorCall(getContext(), Ty);
2433  }
2434 
2435  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
2436  uint64_t NumMembers) const override {
2437  // FIXME: Assumes vectorcall is in use.
2438  return isX86VectorCallAggregateSmallEnough(NumMembers);
2439  }
2440 
2441  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type *> scalars,
2442  bool asReturnValue) const override {
2443  return occupiesMoreThan(CGT, scalars, /*total*/ 4);
2444  }
2445 
2446  bool isSwiftErrorInRegister() const override {
2447  return true;
2448  }
2449 
2450 private:
2451  ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType,
2452  bool IsVectorCall, bool IsRegCall) const;
2453  ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,
2454  const ABIArgInfo &current) const;
2455 
2456  X86AVXABILevel AVXLevel;
2457 
2458  bool IsMingw64;
2459 };
2460 
2461 class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2462 public:
2463  X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2464  : TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(CGT, AVXLevel)) {}
2465 
2466  const X86_64ABIInfo &getABIInfo() const {
2467  return static_cast<const X86_64ABIInfo&>(TargetCodeGenInfo::getABIInfo());
2468  }
2469 
2470  /// Disable tail call on x86-64. The epilogue code before the tail jump blocks
2471  /// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
2472  bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
2473 
2474  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2475  return 7;
2476  }
2477 
2478  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2479  llvm::Value *Address) const override {
2480  llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2481 
2482  // 0-15 are the 16 integer registers.
2483  // 16 is %rip.
2484  AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2485  return false;
2486  }
2487 
2488  llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
2489  StringRef Constraint,
2490  llvm::Type* Ty) const override {
2491  return X86AdjustInlineAsmType(CGF, Constraint, Ty);
2492  }
2493 
2494  bool isNoProtoCallVariadic(const CallArgList &args,
2495  const FunctionNoProtoType *fnType) const override {
2496  // The default CC on x86-64 sets %al to the number of SSA
2497  // registers used, and GCC sets this when calling an unprototyped
2498  // function, so we override the default behavior. However, don't do
2499  // that when AVX types are involved: the ABI explicitly states it is
2500  // undefined, and it doesn't work in practice because of how the ABI
2501  // defines varargs anyway.
2502  if (fnType->getCallConv() == CC_C) {
2503  bool HasAVXType = false;
2504  for (CallArgList::const_iterator
2505  it = args.begin(), ie = args.end(); it != ie; ++it) {
2506  if (getABIInfo().isPassedUsingAVXType(it->Ty)) {
2507  HasAVXType = true;
2508  break;
2509  }
2510  }
2511 
2512  if (!HasAVXType)
2513  return true;
2514  }
2515 
2516  return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);
2517  }
2518 
2519  llvm::Constant *
2520  getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
2521  unsigned Sig = (0xeb << 0) | // jmp rel8
2522  (0x06 << 8) | // .+0x08
2523  ('v' << 16) |
2524  ('2' << 24);
2525  return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
2526  }
2527 
2528  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2529  CodeGen::CodeGenModule &CGM) const override {
2530  if (GV->isDeclaration())
2531  return;
2532  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2533  if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2534  llvm::Function *Fn = cast<llvm::Function>(GV);
2535  Fn->addFnAttr("stackrealign");
2536  }
2537 
2538  addX86InterruptAttrs(FD, GV, CGM);
2539  }
2540  }
2541 
2542  void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
2543  const FunctionDecl *Caller,
2544  const FunctionDecl *Callee,
2545  const CallArgList &Args) const override;
2546 };
2547 
2548 static void initFeatureMaps(const ASTContext &Ctx,
2549  llvm::StringMap<bool> &CallerMap,
2550  const FunctionDecl *Caller,
2551  llvm::StringMap<bool> &CalleeMap,
2552  const FunctionDecl *Callee) {
2553  if (CalleeMap.empty() && CallerMap.empty()) {
2554  // The caller is potentially nullptr in the case where the call isn't in a
2555  // function. In this case, the getFunctionFeatureMap ensures we just get
2556  // the TU level setting (since it cannot be modified by 'target'..
2557  Ctx.getFunctionFeatureMap(CallerMap, Caller);
2558  Ctx.getFunctionFeatureMap(CalleeMap, Callee);
2559  }
2560 }
2561 
2562 static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
2563  SourceLocation CallLoc,
2564  const llvm::StringMap<bool> &CallerMap,
2565  const llvm::StringMap<bool> &CalleeMap,
2566  QualType Ty, StringRef Feature,
2567  bool IsArgument) {
2568  bool CallerHasFeat = CallerMap.lookup(Feature);
2569  bool CalleeHasFeat = CalleeMap.lookup(Feature);
2570  if (!CallerHasFeat && !CalleeHasFeat)
2571  return Diag.Report(CallLoc, diag::warn_avx_calling_convention)
2572  << IsArgument << Ty << Feature;
2573 
2574  // Mixing calling conventions here is very clearly an error.
2575  if (!CallerHasFeat || !CalleeHasFeat)
2576  return Diag.Report(CallLoc, diag::err_avx_calling_convention)
2577  << IsArgument << Ty << Feature;
2578 
2579  // Else, both caller and callee have the required feature, so there is no need
2580  // to diagnose.
2581  return false;
2582 }
2583 
2584 static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
2585  SourceLocation CallLoc,
2586  const llvm::StringMap<bool> &CallerMap,
2587  const llvm::StringMap<bool> &CalleeMap, QualType Ty,
2588  bool IsArgument) {
2589  uint64_t Size = Ctx.getTypeSize(Ty);
2590  if (Size > 256)
2591  return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty,
2592  "avx512f", IsArgument);
2593 
2594  if (Size > 128)
2595  return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty, "avx",
2596  IsArgument);
2597 
2598  return false;
2599 }
2600 
2601 void X86_64TargetCodeGenInfo::checkFunctionCallABI(
2602  CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
2603  const FunctionDecl *Callee, const CallArgList &Args) const {
2604  llvm::StringMap<bool> CallerMap;
2605  llvm::StringMap<bool> CalleeMap;
2606  unsigned ArgIndex = 0;
2607 
2608  // We need to loop through the actual call arguments rather than the the
2609  // function's parameters, in case this variadic.
2610  for (const CallArg &Arg : Args) {
2611  // The "avx" feature changes how vectors >128 in size are passed. "avx512f"
2612  // additionally changes how vectors >256 in size are passed. Like GCC, we
2613  // warn when a function is called with an argument where this will change.
2614  // Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
2615  // the caller and callee features are mismatched.
2616  // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
2617  // change its ABI with attribute-target after this call.
2618  if (Arg.getType()->isVectorType() &&
2619  CGM.getContext().getTypeSize(Arg.getType()) > 128) {
2620  initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2621  QualType Ty = Arg.getType();
2622  // The CallArg seems to have desugared the type already, so for clearer
2623  // diagnostics, replace it with the type in the FunctionDecl if possible.
2624  if (ArgIndex < Callee->getNumParams())
2625  Ty = Callee->getParamDecl(ArgIndex)->getType();
2626 
2627  if (checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2628  CalleeMap, Ty, /*IsArgument*/ true))
2629  return;
2630  }
2631  ++ArgIndex;
2632  }
2633 
2634  // Check return always, as we don't have a good way of knowing in codegen
2635  // whether this value is used, tail-called, etc.
2636  if (Callee->getReturnType()->isVectorType() &&
2637  CGM.getContext().getTypeSize(Callee->getReturnType()) > 128) {
2638  initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2639  checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2640  CalleeMap, Callee->getReturnType(),
2641  /*IsArgument*/ false);
2642  }
2643 }
2644 
2645 static std::string qualifyWindowsLibrary(llvm::StringRef Lib) {
2646  // If the argument does not end in .lib, automatically add the suffix.
2647  // If the argument contains a space, enclose it in quotes.
2648  // This matches the behavior of MSVC.
2649  bool Quote = Lib.contains(' ');
2650  std::string ArgStr = Quote ? "\"" : "";
2651  ArgStr += Lib;
2652  if (!Lib.endswith_insensitive(".lib") && !Lib.endswith_insensitive(".a"))
2653  ArgStr += ".lib";
2654  ArgStr += Quote ? "\"" : "";
2655  return ArgStr;
2656 }
2657 
2658 class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
2659 public:
2660  WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2661  bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
2662  unsigned NumRegisterParameters)
2663  : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
2664  Win32StructABI, NumRegisterParameters, false) {}
2665 
2666  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2667  CodeGen::CodeGenModule &CGM) const override;
2668 
2669  void getDependentLibraryOption(llvm::StringRef Lib,
2670  llvm::SmallString<24> &Opt) const override {
2671  Opt = "/DEFAULTLIB:";
2672  Opt += qualifyWindowsLibrary(Lib);
2673  }
2674 
2675  void getDetectMismatchOption(llvm::StringRef Name,
2676  llvm::StringRef Value,
2677  llvm::SmallString<32> &Opt) const override {
2678  Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2679  }
2680 };
2681 
2682 static void addStackProbeTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2683  CodeGen::CodeGenModule &CGM) {
2684  if (llvm::Function *Fn = dyn_cast_or_null<llvm::Function>(GV)) {
2685 
2686  if (CGM.getCodeGenOpts().StackProbeSize != 4096)
2687  Fn->addFnAttr("stack-probe-size",
2688  llvm::utostr(CGM.getCodeGenOpts().StackProbeSize));
2689  if (CGM.getCodeGenOpts().NoStackArgProbe)
2690  Fn->addFnAttr("no-stack-arg-probe");
2691  }
2692 }
2693 
2694 void WinX86_32TargetCodeGenInfo::setTargetAttributes(
2695  const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2696  X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2697  if (GV->isDeclaration())
2698  return;
2699  addStackProbeTargetAttributes(D, GV, CGM);
2700 }
2701 
2702 class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2703 public:
2704  WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2705  X86AVXABILevel AVXLevel)
2706  : TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(CGT, AVXLevel)) {}
2707 
2708  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2709  CodeGen::CodeGenModule &CGM) const override;
2710 
2711  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2712  return 7;
2713  }
2714 
2715  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2716  llvm::Value *Address) const override {
2717  llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2718 
2719  // 0-15 are the 16 integer registers.
2720  // 16 is %rip.
2721  AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2722  return false;
2723  }
2724 
2725  void getDependentLibraryOption(llvm::StringRef Lib,
2726  llvm::SmallString<24> &Opt) const override {
2727  Opt = "/DEFAULTLIB:";
2728  Opt += qualifyWindowsLibrary(Lib);
2729  }
2730 
2731  void getDetectMismatchOption(llvm::StringRef Name,
2732  llvm::StringRef Value,
2733  llvm::SmallString<32> &Opt) const override {
2734  Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2735  }
2736 };
2737 
2738 void WinX86_64TargetCodeGenInfo::setTargetAttributes(
2739  const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2741  if (GV->isDeclaration())
2742  return;
2743  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2744  if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2745  llvm::Function *Fn = cast<llvm::Function>(GV);
2746  Fn->addFnAttr("stackrealign");
2747  }
2748 
2749  addX86InterruptAttrs(FD, GV, CGM);
2750  }
2751 
2752  addStackProbeTargetAttributes(D, GV, CGM);
2753 }
2754 }
2755 
2756 void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
2757  Class &Hi) const {
2758  // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
2759  //
2760  // (a) If one of the classes is Memory, the whole argument is passed in
2761  // memory.
2762  //
2763  // (b) If X87UP is not preceded by X87, the whole argument is passed in
2764  // memory.
2765  //
2766  // (c) If the size of the aggregate exceeds two eightbytes and the first
2767  // eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
2768  // argument is passed in memory. NOTE: This is necessary to keep the
2769  // ABI working for processors that don't support the __m256 type.
2770  //
2771  // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
2772  //
2773  // Some of these are enforced by the merging logic. Others can arise
2774  // only with unions; for example:
2775  // union { _Complex double; unsigned; }
2776  //
2777  // Note that clauses (b) and (c) were added in 0.98.
2778  //
2779  if (Hi == Memory)
2780  Lo = Memory;
2781  if (Hi == X87Up && Lo != X87 && honorsRevision0_98())
2782  Lo = Memory;
2783  if (AggregateSize > 128 && (Lo != SSE || Hi != SSEUp))
2784  Lo = Memory;
2785  if (Hi == SSEUp && Lo != SSE)
2786  Hi = SSE;
2787 }
2788 
2789 X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
2790  // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
2791  // classified recursively so that always two fields are
2792  // considered. The resulting class is calculated according to
2793  // the classes of the fields in the eightbyte:
2794  //
2795  // (a) If both classes are equal, this is the resulting class.
2796  //
2797  // (b) If one of the classes is NO_CLASS, the resulting class is
2798  // the other class.
2799  //
2800  // (c) If one of the classes is MEMORY, the result is the MEMORY
2801  // class.
2802  //
2803  // (d) If one of the classes is INTEGER, the result is the
2804  // INTEGER.
2805  //
2806  // (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
2807  // MEMORY is used as class.
2808  //
2809  // (f) Otherwise class SSE is used.
2810 
2811  // Accum should never be memory (we should have returned) or
2812  // ComplexX87 (because this cannot be passed in a structure).
2813  assert((Accum != Memory && Accum != ComplexX87) &&
2814  "Invalid accumulated classification during merge.");
2815  if (Accum == Field || Field == NoClass)
2816  return Accum;
2817  if (Field == Memory)
2818  return Memory;
2819  if (Accum == NoClass)
2820  return Field;
2821  if (Accum == Integer || Field == Integer)
2822  return Integer;
2823  if (Field == X87 || Field == X87Up || Field == ComplexX87 ||
2824  Accum == X87 || Accum == X87Up)
2825  return Memory;
2826  return SSE;
2827 }
2828 
2829 void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
2830  Class &Lo, Class &Hi, bool isNamedArg) const {
2831  // FIXME: This code can be simplified by introducing a simple value class for
2832  // Class pairs with appropriate constructor methods for the various
2833  // situations.
2834 
2835  // FIXME: Some of the split computations are wrong; unaligned vectors
2836  // shouldn't be passed in registers for example, so there is no chance they
2837  // can straddle an eightbyte. Verify & simplify.
2838 
2839  Lo = Hi = NoClass;
2840 
2841  Class &Current = OffsetBase < 64 ? Lo : Hi;
2842  Current = Memory;
2843 
2844  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
2845  BuiltinType::Kind k = BT->getKind();
2846 
2847  if (k == BuiltinType::Void) {
2848  Current = NoClass;
2849  } else if (k == BuiltinType::Int128 || k == BuiltinType::UInt128) {
2850  Lo = Integer;
2851  Hi = Integer;
2852  } else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) {
2853  Current = Integer;
2854  } else if (k == BuiltinType::Float || k == BuiltinType::Double ||
2855  k == BuiltinType::Float16) {
2856  Current = SSE;
2857  } else if (k == BuiltinType::LongDouble) {
2858  const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2859  if (LDF == &llvm::APFloat::IEEEquad()) {
2860  Lo = SSE;
2861  Hi = SSEUp;
2862  } else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
2863  Lo = X87;
2864  Hi = X87Up;
2865  } else if (LDF == &llvm::APFloat::IEEEdouble()) {
2866  Current = SSE;
2867  } else
2868  llvm_unreachable("unexpected long double representation!");
2869  }
2870  // FIXME: _Decimal32 and _Decimal64 are SSE.
2871  // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
2872  return;
2873  }
2874 
2875  if (const EnumType *ET = Ty->getAs<EnumType>()) {
2876  // Classify the underlying integer type.
2877  classify(ET->getDecl()->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
2878  return;
2879  }
2880 
2881  if (Ty->hasPointerRepresentation()) {
2882  Current = Integer;
2883  return;
2884  }
2885 
2886  if (Ty->isMemberPointerType()) {
2887  if (Ty->isMemberFunctionPointerType()) {
2888  if (Has64BitPointers) {
2889  // If Has64BitPointers, this is an {i64, i64}, so classify both
2890  // Lo and Hi now.
2891  Lo = Hi = Integer;
2892  } else {
2893  // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
2894  // straddles an eightbyte boundary, Hi should be classified as well.
2895  uint64_t EB_FuncPtr = (OffsetBase) / 64;
2896  uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
2897  if (EB_FuncPtr != EB_ThisAdj) {
2898  Lo = Hi = Integer;
2899  } else {
2900  Current = Integer;
2901  }
2902  }
2903  } else {
2904  Current = Integer;
2905  }
2906  return;
2907  }
2908 
2909  if (const VectorType *VT = Ty->getAs<VectorType>()) {
2910  uint64_t Size = getContext().getTypeSize(VT);
2911  if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
2912  // gcc passes the following as integer:
2913  // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
2914  // 2 bytes - <2 x char>, <1 x short>
2915  // 1 byte - <1 x char>
2916  Current = Integer;
2917 
2918  // If this type crosses an eightbyte boundary, it should be
2919  // split.
2920  uint64_t EB_Lo = (OffsetBase) / 64;
2921  uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
2922  if (EB_Lo != EB_Hi)
2923  Hi = Lo;
2924  } else if (Size == 64) {
2925  QualType ElementType = VT->getElementType();
2926 
2927  // gcc passes <1 x double> in memory. :(
2928  if (ElementType->isSpecificBuiltinType(BuiltinType::Double))
2929  return;
2930 
2931  // gcc passes <1 x long long> as SSE but clang used to unconditionally
2932  // pass them as integer. For platforms where clang is the de facto
2933  // platform compiler, we must continue to use integer.
2934  if (!classifyIntegerMMXAsSSE() &&
2935  (ElementType->isSpecificBuiltinType(BuiltinType::LongLong) ||
2936  ElementType->isSpecificBuiltinType(BuiltinType::ULongLong) ||
2937  ElementType->isSpecificBuiltinType(BuiltinType::Long) ||
2938  ElementType->isSpecificBuiltinType(BuiltinType::ULong)))
2939  Current = Integer;
2940  else
2941  Current = SSE;
2942 
2943  // If this type crosses an eightbyte boundary, it should be
2944  // split.
2945  if (OffsetBase && OffsetBase != 64)
2946  Hi = Lo;
2947  } else if (Size == 128 ||
2948  (isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
2949  QualType ElementType = VT->getElementType();
2950 
2951  // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
2952  if (passInt128VectorsInMem() && Size != 128 &&
2953  (ElementType->isSpecificBuiltinType(BuiltinType::Int128) ||
2954  ElementType->isSpecificBuiltinType(BuiltinType::UInt128)))
2955  return;
2956 
2957  // Arguments of 256-bits are split into four eightbyte chunks. The
2958  // least significant one belongs to class SSE and all the others to class
2959  // SSEUP. The original Lo and Hi design considers that types can't be
2960  // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
2961  // This design isn't correct for 256-bits, but since there're no cases
2962  // where the upper parts would need to be inspected, avoid adding
2963  // complexity and just consider Hi to match the 64-256 part.
2964  //
2965  // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
2966  // registers if they are "named", i.e. not part of the "..." of a
2967  // variadic function.
2968  //
2969  // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
2970  // split into eight eightbyte chunks, one SSE and seven SSEUP.
2971  Lo = SSE;
2972  Hi = SSEUp;
2973  }
2974  return;
2975  }
2976 
2977  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
2978  QualType ET = getContext().getCanonicalType(CT->getElementType());
2979 
2980  uint64_t Size = getContext().getTypeSize(Ty);
2981  if (ET->isIntegralOrEnumerationType()) {
2982  if (Size <= 64)
2983  Current = Integer;
2984  else if (Size <= 128)
2985  Lo = Hi = Integer;
2986  } else if (ET->isFloat16Type() || ET == getContext().FloatTy) {
2987  Current = SSE;
2988  } else if (ET == getContext().DoubleTy) {
2989  Lo = Hi = SSE;
2990  } else if (ET == getContext().LongDoubleTy) {
2991  const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2992  if (LDF == &llvm::APFloat::IEEEquad())
2993  Current = Memory;
2994  else if (LDF == &llvm::APFloat::x87DoubleExtended())
2995  Current = ComplexX87;
2996  else if (LDF == &llvm::APFloat::IEEEdouble())
2997  Lo = Hi = SSE;
2998  else
2999  llvm_unreachable("unexpected long double representation!");
3000  }
3001 
3002  // If this complex type crosses an eightbyte boundary then it
3003  // should be split.
3004  uint64_t EB_Real = (OffsetBase) / 64;
3005  uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
3006  if (Hi == NoClass && EB_Real != EB_Imag)
3007  Hi = Lo;
3008 
3009  return;
3010  }
3011 
3012  if (const auto *EITy = Ty->getAs<ExtIntType>()) {
3013  if (EITy->getNumBits() <= 64)
3014  Current = Integer;
3015  else if (EITy->getNumBits() <= 128)
3016  Lo = Hi = Integer;
3017  // Larger values need to get passed in memory.
3018  return;
3019  }
3020 
3021  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
3022  // Arrays are treated like structures.
3023 
3024  uint64_t Size = getContext().getTypeSize(Ty);
3025 
3026  // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3027  // than eight eightbytes, ..., it has class MEMORY.
3028  if (Size > 512)
3029  return;
3030 
3031  // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
3032  // fields, it has class MEMORY.
3033  //
3034  // Only need to check alignment of array base.
3035  if (OffsetBase % getContext().getTypeAlign(AT->getElementType()))
3036  return;
3037 
3038  // Otherwise implement simplified merge. We could be smarter about
3039  // this, but it isn't worth it and would be harder to verify.
3040  Current = NoClass;
3041  uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
3042  uint64_t ArraySize = AT->getSize().getZExtValue();
3043 
3044  // The only case a 256-bit wide vector could be used is when the array
3045  // contains a single 256-bit element. Since Lo and Hi logic isn't extended
3046  // to work for sizes wider than 128, early check and fallback to memory.
3047  //
3048  if (Size > 128 &&
3049  (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
3050  return;
3051 
3052  for (uint64_t i=0, Offset=OffsetBase; i<ArraySize; ++i, Offset += EltSize) {
3053  Class FieldLo, FieldHi;
3054  classify(AT->getElementType(), Offset, FieldLo, FieldHi, isNamedArg);
3055  Lo = merge(Lo, FieldLo);
3056  Hi = merge(Hi, FieldHi);
3057  if (Lo == Memory || Hi == Memory)
3058  break;
3059  }
3060 
3061  postMerge(Size, Lo, Hi);
3062  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
3063  return;
3064  }
3065 
3066  if (const RecordType *RT = Ty->getAs<RecordType>()) {
3067  uint64_t Size = getContext().getTypeSize(Ty);
3068 
3069  // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3070  // than eight eightbytes, ..., it has class MEMORY.
3071  if (Size > 512)
3072  return;
3073 
3074  // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
3075  // copy constructor or a non-trivial destructor, it is passed by invisible
3076  // reference.
3077  if (getRecordArgABI(RT, getCXXABI()))
3078  return;
3079 
3080  const RecordDecl *RD = RT->getDecl();
3081 
3082  // Assume variable sized types are passed in memory.
3083  if (RD->hasFlexibleArrayMember())
3084  return;
3085 
3086  const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
3087 
3088  // Reset Lo class, this will be recomputed.
3089  Current = NoClass;
3090 
3091  // If this is a C++ record, classify the bases first.
3092  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3093  for (const auto &I : CXXRD->bases()) {
3094  assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3095  "Unexpected base class!");
3096  const auto *Base =
3097  cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3098 
3099  // Classify this field.
3100  //
3101  // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
3102  // single eightbyte, each is classified separately. Each eightbyte gets
3103  // initialized to class NO_CLASS.
3104  Class FieldLo, FieldHi;
3105  uint64_t Offset =
3106  OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
3107  classify(I.getType(), Offset, FieldLo, FieldHi, isNamedArg);
3108  Lo = merge(Lo, FieldLo);
3109  Hi = merge(Hi, FieldHi);
3110  if (Lo == Memory || Hi == Memory) {
3111  postMerge(Size, Lo, Hi);
3112  return;
3113  }
3114  }
3115  }
3116 
3117  // Classify the fields one at a time, merging the results.
3118  unsigned idx = 0;
3119  bool UseClang11Compat = getContext().getLangOpts().getClangABICompat() <=
3121  getContext().getTargetInfo().getTriple().isPS4();
3122  bool IsUnion = RT->isUnionType() && !UseClang11Compat;
3123 
3124  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3125  i != e; ++i, ++idx) {
3126  uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3127  bool BitField = i->isBitField();
3128 
3129  // Ignore padding bit-fields.
3130  if (BitField && i->isUnnamedBitfield())
3131  continue;
3132 
3133  // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
3134  // eight eightbytes, or it contains unaligned fields, it has class MEMORY.
3135  //
3136  // The only case a 256-bit or a 512-bit wide vector could be used is when
3137  // the struct contains a single 256-bit or 512-bit element. Early check
3138  // and fallback to memory.
3139  //
3140  // FIXME: Extended the Lo and Hi logic properly to work for size wider
3141  // than 128.
3142  if (Size > 128 &&
3143  ((!IsUnion && Size != getContext().getTypeSize(i->getType())) ||
3144  Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
3145  Lo = Memory;
3146  postMerge(Size, Lo, Hi);
3147  return;
3148  }
3149  // Note, skip this test for bit-fields, see below.
3150  if (!BitField && Offset % getContext().getTypeAlign(i->getType())) {
3151  Lo = Memory;
3152  postMerge(Size, Lo, Hi);
3153  return;
3154  }
3155 
3156  // Classify this field.
3157  //
3158  // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
3159  // exceeds a single eightbyte, each is classified
3160  // separately. Each eightbyte gets initialized to class
3161  // NO_CLASS.
3162  Class FieldLo, FieldHi;
3163 
3164  // Bit-fields require special handling, they do not force the
3165  // structure to be passed in memory even if unaligned, and
3166  // therefore they can straddle an eightbyte.
3167  if (BitField) {
3168  assert(!i->isUnnamedBitfield());
3169  uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3170  uint64_t Size = i->getBitWidthValue(getContext());
3171 
3172  uint64_t EB_Lo = Offset / 64;
3173  uint64_t EB_Hi = (Offset + Size - 1) / 64;
3174 
3175  if (EB_Lo) {
3176  assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
3177  FieldLo = NoClass;
3178  FieldHi = Integer;
3179  } else {
3180  FieldLo = Integer;
3181  FieldHi = EB_Hi ? Integer : NoClass;
3182  }
3183  } else
3184  classify(i->getType(), Offset, FieldLo, FieldHi, isNamedArg);
3185  Lo = merge(Lo, FieldLo);
3186  Hi = merge(Hi, FieldHi);
3187  if (Lo == Memory || Hi == Memory)
3188  break;
3189  }
3190 
3191  postMerge(Size, Lo, Hi);
3192  }
3193 }
3194 
3195 ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
3196  // If this is a scalar LLVM value then assume LLVM will pass it in the right
3197  // place naturally.
3198  if (!isAggregateTypeForABI(Ty)) {
3199  // Treat an enum type as its underlying type.
3200  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3201  Ty = EnumTy->getDecl()->getIntegerType();
3202 
3203  if (Ty->isExtIntType())
3204  return getNaturalAlignIndirect(Ty);
3205 
3206  return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
3207  : ABIArgInfo::getDirect());
3208  }
3209 
3210  return getNaturalAlignIndirect(Ty);
3211 }
3212 
3213 bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
3214  if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
3215  uint64_t Size = getContext().getTypeSize(VecTy);
3216  unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
3217  if (Size <= 64 || Size > LargestVector)
3218  return true;
3219  QualType EltTy = VecTy->getElementType();
3220  if (passInt128VectorsInMem() &&
3221  (EltTy->isSpecificBuiltinType(BuiltinType::Int128) ||
3222  EltTy->isSpecificBuiltinType(BuiltinType::UInt128)))
3223  return true;
3224  }
3225 
3226  return false;
3227 }
3228 
3229 ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
3230  unsigned freeIntRegs) const {
3231  // If this is a scalar LLVM value then assume LLVM will pass it in the right
3232  // place naturally.
3233  //
3234  // This assumption is optimistic, as there could be free registers available
3235  // when we need to pass this argument in memory, and LLVM could try to pass
3236  // the argument in the free register. This does not seem to happen currently,
3237  // but this code would be much safer if we could mark the argument with
3238  // 'onstack'. See PR12193.
3239  if (!isAggregateTypeForABI(Ty) && !IsIllegalVectorType(Ty) &&
3240  !Ty->isExtIntType()) {
3241  // Treat an enum type as its underlying type.
3242  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3243  Ty = EnumTy->getDecl()->getIntegerType();
3244 
3245  return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
3246  : ABIArgInfo::getDirect());
3247  }
3248 
3249  if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
3250  return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
3251 
3252  // Compute the byval alignment. We specify the alignment of the byval in all
3253  // cases so that the mid-level optimizer knows the alignment of the byval.
3254  unsigned Align = std::max(getContext().getTypeAlign(Ty) / 8, 8U);
3255 
3256  // Attempt to avoid passing indirect results using byval when possible. This
3257  // is important for good codegen.
3258  //
3259  // We do this by coercing the value into a scalar type which the backend can
3260  // handle naturally (i.e., without using byval).
3261  //
3262  // For simplicity, we currently only do this when we have exhausted all of the
3263  // free integer registers. Doing this when there are free integer registers
3264  // would require more care, as we would have to ensure that the coerced value
3265  // did not claim the unused register. That would require either reording the
3266  // arguments to the function (so that any subsequent inreg values came first),
3267  // or only doing this optimization when there were no following arguments that
3268  // might be inreg.
3269  //
3270  // We currently expect it to be rare (particularly in well written code) for
3271  // arguments to be passed on the stack when there are still free integer
3272  // registers available (this would typically imply large structs being passed
3273  // by value), so this seems like a fair tradeoff for now.
3274  //
3275  // We can revisit this if the backend grows support for 'onstack' parameter
3276  // attributes. See PR12193.
3277  if (freeIntRegs == 0) {
3278  uint64_t Size = getContext().getTypeSize(Ty);
3279 
3280  // If this type fits in an eightbyte, coerce it into the matching integral
3281  // type, which will end up on the stack (with alignment 8).
3282  if (Align == 8 && Size <= 64)
3283  return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
3284  Size));
3285  }
3286 
3288 }
3289 
3290 /// The ABI specifies that a value should be passed in a full vector XMM/YMM
3291 /// register. Pick an LLVM IR type that will be passed as a vector register.
3292 llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
3293  // Wrapper structs/arrays that only contain vectors are passed just like
3294  // vectors; strip them off if present.
3295  if (const Type *InnerTy = isSingleElementStruct(Ty, getContext()))
3296  Ty = QualType(InnerTy, 0);
3297 
3298  llvm::Type *IRType = CGT.ConvertType(Ty);
3299  if (isa<llvm::VectorType>(IRType)) {
3300  // Don't pass vXi128 vectors in their native type, the backend can't
3301  // legalize them.
3302  if (passInt128VectorsInMem() &&
3303  cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy(128)) {
3304  // Use a vXi64 vector.
3305  uint64_t Size = getContext().getTypeSize(Ty);
3306  return llvm::FixedVectorType::get(llvm::Type::getInt64Ty(getVMContext()),
3307  Size / 64);
3308  }
3309 
3310  return IRType;
3311  }
3312 
3313  if (IRType->getTypeID() == llvm::Type::FP128TyID)
3314  return IRType;
3315 
3316  // We couldn't find the preferred IR vector type for 'Ty'.
3317  uint64_t Size = getContext().getTypeSize(Ty);
3318  assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
3319 
3320 
3321  // Return a LLVM IR vector type based on the size of 'Ty'.
3322  return llvm::FixedVectorType::get(llvm::Type::getDoubleTy(getVMContext()),
3323  Size / 64);
3324 }
3325 
3326 /// BitsContainNoUserData - Return true if the specified [start,end) bit range
3327 /// is known to either be off the end of the specified type or being in
3328 /// alignment padding. The user type specified is known to be at most 128 bits
3329 /// in size, and have passed through X86_64ABIInfo::classify with a successful
3330 /// classification that put one of the two halves in the INTEGER class.
3331 ///
3332 /// It is conservatively correct to return false.
3333 static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
3334  unsigned EndBit, ASTContext &Context) {
3335  // If the bytes being queried are off the end of the type, there is no user
3336  // data hiding here. This handles analysis of builtins, vectors and other
3337  // types that don't contain interesting padding.
3338  unsigned TySize = (unsigned)Context.getTypeSize(Ty);
3339  if (TySize <= StartBit)
3340  return true;
3341 
3342  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
3343  unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());
3344  unsigned NumElts = (unsigned)AT->getSize().getZExtValue();
3345 
3346  // Check each element to see if the element overlaps with the queried range.
3347  for (unsigned i = 0; i != NumElts; ++i) {
3348  // If the element is after the span we care about, then we're done..
3349  unsigned EltOffset = i*EltSize;
3350  if (EltOffset >= EndBit) break;
3351 
3352  unsigned EltStart = EltOffset < StartBit ? StartBit-EltOffset :0;
3353  if (!BitsContainNoUserData(AT->getElementType(), EltStart,
3354  EndBit-EltOffset, Context))
3355  return false;
3356  }
3357  // If it overlaps no elements, then it is safe to process as padding.
3358  return true;
3359  }
3360 
3361  if (const RecordType *RT = Ty->getAs<RecordType>()) {
3362  const RecordDecl *RD = RT->getDecl();
3363  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
3364 
3365  // If this is a C++ record, check the bases first.
3366  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3367  for (const auto &I : CXXRD->bases()) {
3368  assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3369  "Unexpected base class!");
3370  const auto *Base =
3371  cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3372 
3373  // If the base is after the span we care about, ignore it.
3374  unsigned BaseOffset = Context.toBits(Layout.getBaseClassOffset(Base));
3375  if (BaseOffset >= EndBit) continue;
3376 
3377  unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :0;
3378  if (!BitsContainNoUserData(I.getType(), BaseStart,
3379  EndBit-BaseOffset, Context))
3380  return false;
3381  }
3382  }
3383 
3384  // Verify that no field has data that overlaps the region of interest. Yes
3385  // this could be sped up a lot by being smarter about queried fields,
3386  // however we're only looking at structs up to 16 bytes, so we don't care
3387  // much.
3388  unsigned idx = 0;
3389  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3390  i != e; ++i, ++idx) {
3391  unsigned FieldOffset = (unsigned)Layout.getFieldOffset(idx);
3392 
3393  // If we found a field after the region we care about, then we're done.
3394  if (FieldOffset >= EndBit) break;
3395 
3396  unsigned FieldStart = FieldOffset < StartBit ? StartBit-FieldOffset :0;
3397  if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit-FieldOffset,
3398  Context))
3399  return false;
3400  }
3401 
3402  // If nothing in this record overlapped the area of interest, then we're
3403  // clean.
3404  return true;
3405  }
3406 
3407  return false;
3408 }
3409 
3410 /// getFPTypeAtOffset - Return a floating point type at the specified offset.
3411 static llvm::Type *getFPTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3412  const llvm::DataLayout &TD) {
3413  if (IROffset == 0 && IRType->isFloatingPointTy())
3414  return IRType;
3415 
3416  // If this is a struct, recurse into the field at the specified offset.
3417  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3418  if (!STy->getNumContainedTypes())
3419  return nullptr;
3420 
3421  const llvm::StructLayout *SL = TD.getStructLayout(STy);
3422  unsigned Elt = SL->getElementContainingOffset(IROffset);
3423  IROffset -= SL->getElementOffset(Elt);
3424  return getFPTypeAtOffset(STy->getElementType(Elt), IROffset, TD);
3425  }
3426 
3427  // If this is an array, recurse into the field at the specified offset.
3428  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3429  llvm::Type *EltTy = ATy->getElementType();
3430  unsigned EltSize = TD.getTypeAllocSize(EltTy);
3431  IROffset -= IROffset / EltSize * EltSize;
3432  return getFPTypeAtOffset(EltTy, IROffset, TD);
3433  }
3434 
3435  return nullptr;
3436 }
3437 
3438 /// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
3439 /// low 8 bytes of an XMM register, corresponding to the SSE class.
3440 llvm::Type *X86_64ABIInfo::
3441 GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3442  QualType SourceTy, unsigned SourceOffset) const {
3443  const llvm::DataLayout &TD = getDataLayout();
3444  unsigned SourceSize =
3445  (unsigned)getContext().getTypeSize(SourceTy) / 8 - SourceOffset;
3446  llvm::Type *T0 = getFPTypeAtOffset(IRType, IROffset, TD);
3447  if (!T0 || T0->isDoubleTy())
3448  return llvm::Type::getDoubleTy(getVMContext());
3449 
3450  // Get the adjacent FP type.
3451  llvm::Type *T1 = nullptr;
3452  unsigned T0Size = TD.getTypeAllocSize(T0);
3453  if (SourceSize > T0Size)
3454  T1 = getFPTypeAtOffset(IRType, IROffset + T0Size, TD);
3455  if (T1 == nullptr) {
3456  // Check if IRType is a half + float. float type will be in IROffset+4 due
3457  // to its alignment.
3458  if (T0->isHalfTy() && SourceSize > 4)
3459  T1 = getFPTypeAtOffset(IRType, IROffset + 4, TD);
3460  // If we can't get a second FP type, return a simple half or float.
3461  // avx512fp16-abi.c:pr51813_2 shows it works to return float for
3462  // {float, i8} too.
3463  if (T1 == nullptr)
3464  return T0;
3465  }
3466 
3467  if (T0->isFloatTy() && T1->isFloatTy())
3468  return llvm::FixedVectorType::get(T0, 2);
3469 
3470  if (T0->isHalfTy() && T1->isHalfTy()) {
3471  llvm::Type *T2 = nullptr;
3472  if (SourceSize > 4)
3473  T2 = getFPTypeAtOffset(IRType, IROffset + 4, TD);
3474  if (T2 == nullptr)
3475  return llvm::FixedVectorType::get(T0, 2);
3476  return llvm::FixedVectorType::get(T0, 4);
3477  }
3478 
3479  if (T0->isHalfTy() || T1->isHalfTy())
3480  return llvm::FixedVectorType::get(llvm::Type::getHalfTy(getVMContext()), 4);
3481 
3482  return llvm::Type::getDoubleTy(getVMContext());
3483 }
3484 
3485 
3486 /// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
3487 /// an 8-byte GPR. This means that we either have a scalar or we are talking
3488 /// about the high or low part of an up-to-16-byte struct. This routine picks
3489 /// the best LLVM IR type to represent this, which may be i64 or may be anything
3490 /// else that the backend will pass in a GPR that works better (e.g. i8, %foo*,
3491 /// etc).
3492 ///
3493 /// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
3494 /// the source type. IROffset is an offset in bytes into the LLVM IR type that
3495 /// the 8-byte value references. PrefType may be null.
3496 ///
3497 /// SourceTy is the source-level type for the entire argument. SourceOffset is
3498 /// an offset into this that we're processing (which is always either 0 or 8).
3499 ///
3500 llvm::Type *X86_64ABIInfo::
3501 GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3502  QualType SourceTy, unsigned SourceOffset) const {
3503  // If we're dealing with an un-offset LLVM IR type, then it means that we're
3504  // returning an 8-byte unit starting with it. See if we can safely use it.
3505  if (IROffset == 0) {
3506  // Pointers and int64's always fill the 8-byte unit.
3507  if ((isa<llvm::PointerType>(IRType) && Has64BitPointers) ||
3508  IRType->isIntegerTy(64))
3509  return IRType;
3510 
3511  // If we have a 1/2/4-byte integer, we can use it only if the rest of the
3512  // goodness in the source type is just tail padding. This is allowed to
3513  // kick in for struct {double,int} on the int, but not on
3514  // struct{double,int,int} because we wouldn't return the second int. We
3515  // have to do this analysis on the source type because we can't depend on
3516  // unions being lowered a specific way etc.
3517  if (IRType->isIntegerTy(8) || IRType->isIntegerTy(16) ||
3518  IRType->isIntegerTy(32) ||
3519  (isa<llvm::PointerType>(IRType) && !Has64BitPointers)) {
3520  unsigned BitWidth = isa<llvm::PointerType>(IRType) ? 32 :
3521  cast<llvm::IntegerType>(IRType)->getBitWidth();
3522 
3523  if (BitsContainNoUserData(SourceTy, SourceOffset*8+BitWidth,
3524  SourceOffset*8+64, getContext()))
3525  return IRType;
3526  }
3527  }
3528 
3529  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3530  // If this is a struct, recurse into the field at the specified offset.
3531  const llvm::StructLayout *SL = getDataLayout().getStructLayout(STy);
3532  if (IROffset < SL->getSizeInBytes()) {
3533  unsigned FieldIdx = SL->getElementContainingOffset(IROffset);
3534  IROffset -= SL->getElementOffset(FieldIdx);
3535 
3536  return GetINTEGERTypeAtOffset(STy->getElementType(FieldIdx), IROffset,
3537  SourceTy, SourceOffset);
3538  }
3539  }
3540 
3541  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3542  llvm::Type *EltTy = ATy->getElementType();
3543  unsigned EltSize = getDataLayout().getTypeAllocSize(EltTy);
3544  unsigned EltOffset = IROffset/EltSize*EltSize;
3545  return GetINTEGERTypeAtOffset(EltTy, IROffset-EltOffset, SourceTy,
3546  SourceOffset);
3547  }
3548 
3549  // Okay, we don't have any better idea of what to pass, so we pass this in an
3550  // integer register that isn't too big to fit the rest of the struct.
3551  unsigned TySizeInBytes =
3552  (unsigned)getContext().getTypeSizeInChars(SourceTy).getQuantity();
3553 
3554  assert(TySizeInBytes != SourceOffset && "Empty field?");
3555 
3556  // It is always safe to classify this as an integer type up to i64 that
3557  // isn't larger than the structure.
3558  return llvm::IntegerType::get(getVMContext(),
3559  std::min(TySizeInBytes-SourceOffset, 8U)*8);
3560 }
3561 
3562 
3563 /// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
3564 /// be used as elements of a two register pair to pass or return, return a
3565 /// first class aggregate to represent them. For example, if the low part of
3566 /// a by-value argument should be passed as i32* and the high part as float,
3567 /// return {i32*, float}.
3568 static llvm::Type *
3569 GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
3570  const llvm::DataLayout &TD) {
3571  // In order to correctly satisfy the ABI, we need to the high part to start
3572  // at offset 8. If the high and low parts we inferred are both 4-byte types
3573  // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
3574  // the second element at offset 8. Check for this:
3575  unsigned LoSize = (unsigned)TD.getTypeAllocSize(Lo);
3576  unsigned HiAlign = TD.getABITypeAlignment(Hi);
3577  unsigned HiStart = llvm::alignTo(LoSize, HiAlign);
3578  assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
3579 
3580  // To handle this, we have to increase the size of the low part so that the
3581  // second element will start at an 8 byte offset. We can't increase the size
3582  // of the second element because it might make us access off the end of the
3583  // struct.
3584  if (HiStart != 8) {
3585  // There are usually two sorts of types the ABI generation code can produce
3586  // for the low part of a pair that aren't 8 bytes in size: half, float or
3587  // i8/i16/i32. This can also include pointers when they are 32-bit (X32 and
3588  // NaCl).
3589  // Promote these to a larger type.
3590  if (Lo->isHalfTy() || Lo->isFloatTy())
3591  Lo = llvm::Type::getDoubleTy(Lo->getContext());
3592  else {
3593  assert((Lo->isIntegerTy() || Lo->isPointerTy())
3594  && "Invalid/unknown lo type");
3595  Lo = llvm::Type::getInt64Ty(Lo->getContext());
3596  }
3597  }
3598 
3599  llvm::StructType *Result = llvm::StructType::get(Lo, Hi);
3600 
3601  // Verify that the second element is at an 8-byte offset.
3602  assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
3603  "Invalid x86-64 argument pair!");
3604  return Result;
3605 }
3606 
3608 classifyReturnType(QualType RetTy) const {
3609  // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
3610  // classification algorithm.
3611  X86_64ABIInfo::Class Lo, Hi;
3612  classify(RetTy, 0, Lo, Hi, /*isNamedArg*/ true);
3613 
3614  // Check some invariants.
3615  assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3616  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3617 
3618  llvm::Type *ResType = nullptr;
3619  switch (Lo) {
3620  case NoClass:
3621  if (Hi == NoClass)
3622  return ABIArgInfo::getIgnore();
3623  // If the low part is just padding, it takes no register, leave ResType
3624  // null.
3625  assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3626  "Unknown missing lo part");
3627  break;
3628 
3629  case SSEUp:
3630  case X87Up:
3631  llvm_unreachable("Invalid classification for lo word.");
3632 
3633  // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
3634  // hidden argument.
3635  case Memory:
3636  return getIndirectReturnResult(RetTy);
3637 
3638  // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
3639  // available register of the sequence %rax, %rdx is used.
3640  case Integer:
3641  ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3642 
3643  // If we have a sign or zero extended integer, make sure to return Extend
3644  // so that the parameter gets the right LLVM IR attributes.
3645  if (Hi == NoClass && isa<llvm::IntegerType>(ResType)) {
3646  // Treat an enum type as its underlying type.
3647  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
3648  RetTy = EnumTy->getDecl()->getIntegerType();
3649 
3650  if (RetTy->isIntegralOrEnumerationType() &&
3651  isPromotableIntegerTypeForABI(RetTy))
3652  return ABIArgInfo::getExtend(RetTy);
3653  }
3654  break;
3655 
3656  // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
3657  // available SSE register of the sequence %xmm0, %xmm1 is used.
3658  case SSE:
3659  ResType = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3660  break;
3661 
3662  // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
3663  // returned on the X87 stack in %st0 as 80-bit x87 number.
3664  case X87:
3665  ResType = llvm::Type::getX86_FP80Ty(getVMContext());
3666  break;
3667 
3668  // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
3669  // part of the value is returned in %st0 and the imaginary part in
3670  // %st1.
3671  case ComplexX87:
3672  assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
3673  ResType = llvm::StructType::get(llvm::Type::getX86_FP80Ty(getVMContext()),
3674  llvm::Type::getX86_FP80Ty(getVMContext()));
3675  break;
3676  }
3677 
3678  llvm::Type *HighPart = nullptr;
3679  switch (Hi) {
3680  // Memory was handled previously and X87 should
3681  // never occur as a hi class.
3682  case Memory:
3683  case X87:
3684  llvm_unreachable("Invalid classification for hi word.");
3685 
3686  case ComplexX87: // Previously handled.
3687  case NoClass:
3688  break;
3689 
3690  case Integer:
3691  HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3692  if (Lo == NoClass) // Return HighPart at offset 8 in memory.
3693  return ABIArgInfo::getDirect(HighPart, 8);
3694  break;
3695  case SSE:
3696  HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3697  if (Lo == NoClass) // Return HighPart at offset 8 in memory.
3698  return ABIArgInfo::getDirect(HighPart, 8);
3699  break;
3700 
3701  // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
3702  // is passed in the next available eightbyte chunk if the last used
3703  // vector register.
3704  //
3705  // SSEUP should always be preceded by SSE, just widen.
3706  case SSEUp:
3707  assert(Lo == SSE && "Unexpected SSEUp classification.");
3708  ResType = GetByteVectorType(RetTy);
3709  break;
3710 
3711  // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
3712  // returned together with the previous X87 value in %st0.
3713  case X87Up:
3714  // If X87Up is preceded by X87, we don't need to do
3715  // anything. However, in some cases with unions it may not be
3716  // preceded by X87. In such situations we follow gcc and pass the
3717  // extra bits in an SSE reg.
3718  if (Lo != X87) {
3719  HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3720  if (Lo == NoClass) // Return HighPart at offset 8 in memory.
3721  return ABIArgInfo::getDirect(HighPart, 8);
3722  }
3723  break;
3724  }
3725 
3726  // If a high part was specified, merge it together with the low part. It is
3727  // known to pass in the high eightbyte of the result. We do this by forming a
3728  // first class struct aggregate with the high and low part: {low, high}
3729  if (HighPart)
3730  ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3731 
3732  return ABIArgInfo::getDirect(ResType);
3733 }
3734 
3736  QualType Ty, unsigned freeIntRegs, unsigned &neededInt, unsigned &neededSSE,
3737  bool isNamedArg)
3738  const
3739 {
3741 
3742  X86_64ABIInfo::Class Lo, Hi;
3743  classify(Ty, 0, Lo, Hi, isNamedArg);
3744 
3745  // Check some invariants.
3746  // FIXME: Enforce these by construction.
3747  assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3748  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3749 
3750  neededInt = 0;
3751  neededSSE = 0;
3752  llvm::Type *ResType = nullptr;
3753  switch (Lo) {
3754  case NoClass:
3755  if (Hi == NoClass)
3756  return ABIArgInfo::getIgnore();
3757  // If the low part is just padding, it takes no register, leave ResType
3758  // null.
3759  assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3760  "Unknown missing lo part");
3761  break;
3762 
3763  // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
3764  // on the stack.
3765  case Memory:
3766 
3767  // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
3768  // COMPLEX_X87, it is passed in memory.
3769  case X87:
3770  case ComplexX87:
3771  if (getRecordArgABI(Ty, getCXXABI()) == CGCXXABI::RAA_Indirect)
3772  ++neededInt;
3773  return getIndirectResult(Ty, freeIntRegs);
3774 
3775  case SSEUp:
3776  case X87Up:
3777  llvm_unreachable("Invalid classification for lo word.");
3778 
3779  // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
3780  // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
3781  // and %r9 is used.
3782  case Integer:
3783  ++neededInt;
3784 
3785  // Pick an 8-byte type based on the preferred type.
3786  ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 0, Ty, 0);
3787 
3788  // If we have a sign or zero extended integer, make sure to return Extend
3789  // so that the parameter gets the right LLVM IR attributes.
3790  if (Hi == NoClass && isa<llvm::IntegerType>(ResType)) {
3791  // Treat an enum type as its underlying type.
3792  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3793  Ty = EnumTy->getDecl()->getIntegerType();
3794 
3795  if (Ty->isIntegralOrEnumerationType() &&
3796  isPromotableIntegerTypeForABI(Ty))
3797  return ABIArgInfo::getExtend(Ty);
3798  }
3799 
3800  break;
3801 
3802  // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
3803  // available SSE register is used, the registers are taken in the
3804  // order from %xmm0 to %xmm7.
3805  case SSE: {
3806  llvm::Type *IRType = CGT.ConvertType(Ty);
3807  ResType = GetSSETypeAtOffset(IRType, 0, Ty, 0);
3808  ++neededSSE;
3809  break;
3810  }
3811  }
3812 
3813  llvm::Type *HighPart = nullptr;
3814  switch (Hi) {
3815  // Memory was handled previously, ComplexX87 and X87 should
3816  // never occur as hi classes, and X87Up must be preceded by X87,
3817  // which is passed in memory.
3818  case Memory:
3819  case X87:
3820  case ComplexX87:
3821  llvm_unreachable("Invalid classification for hi word.");
3822 
3823  case NoClass: break;
3824 
3825  case Integer:
3826  ++neededInt;
3827  // Pick an 8-byte type based on the preferred type.
3828  HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3829 
3830  if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
3831  return ABIArgInfo::getDirect(HighPart, 8);
3832  break;
3833 
3834  // X87Up generally doesn't occur here (long double is passed in
3835  // memory), except in situations involving unions.
3836  case X87Up:
3837  case SSE:
3838  HighPart = GetSSETypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3839 
3840  if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
3841  return ABIArgInfo::getDirect(HighPart, 8);
3842 
3843  ++neededSSE;
3844  break;
3845 
3846  // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
3847  // eightbyte is passed in the upper half of the last used SSE
3848  // register. This only happens when 128-bit vectors are passed.
3849  case SSEUp:
3850  assert(Lo == SSE && "Unexpected SSEUp classification");
3851  ResType = GetByteVectorType(Ty);
3852  break;
3853  }
3854 
3855  // If a high part was specified, merge it together with the low part. It is
3856  // known to pass in the high eightbyte of the result. We do this by forming a
3857  // first class struct aggregate with the high and low part: {low, high}
3858  if (HighPart)
3859  ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3860 
3861  return ABIArgInfo::getDirect(ResType);
3862 }
3863 
3864 ABIArgInfo
3865 X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
3866  unsigned &NeededSSE) const {
3867  auto RT = Ty->getAs<RecordType>();
3868  assert(RT && "classifyRegCallStructType only valid with struct types");
3869 
3870  if (RT->getDecl()->hasFlexibleArrayMember())
3871  return getIndirectReturnResult(Ty);
3872 
3873  // Sum up bases
3874  if (auto CXXRD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
3875  if (CXXRD->isDynamicClass()) {
3876  NeededInt = NeededSSE = 0;
3877  return getIndirectReturnResult(Ty);
3878  }
3879 
3880  for (const auto &I : CXXRD->bases())
3881  if (classifyRegCallStructTypeImpl(I.getType(), NeededInt, NeededSSE)
3882  .isIndirect()) {
3883  NeededInt = NeededSSE = 0;
3884  return getIndirectReturnResult(Ty);
3885  }
3886  }
3887 
3888  // Sum up members
3889  for (const auto *FD : RT->getDecl()->fields()) {
3890  if (FD->getType()->isRecordType() && !FD->getType()->isUnionType()) {
3891  if (classifyRegCallStructTypeImpl(FD->getType(), NeededInt, NeededSSE)
3892  .isIndirect()) {
3893  NeededInt = NeededSSE = 0;
3894  return getIndirectReturnResult(Ty);
3895  }
3896  } else {
3897  unsigned LocalNeededInt, LocalNeededSSE;
3898  if (classifyArgumentType(FD->getType(), UINT_MAX, LocalNeededInt,
3899  LocalNeededSSE, true)
3900  .isIndirect()) {
3901  NeededInt = NeededSSE = 0;
3902  return getIndirectReturnResult(Ty);
3903  }
3904  NeededInt += LocalNeededInt;
3905  NeededSSE += LocalNeededSSE;
3906  }
3907  }
3908 
3909  return ABIArgInfo::getDirect();
3910 }
3911 
3912 ABIArgInfo X86_64ABIInfo::classifyRegCallStructType(QualType Ty,
3913  unsigned &NeededInt,
3914  unsigned &NeededSSE) const {
3915 
3916  NeededInt = 0;
3917  NeededSSE = 0;
3918 
3919  return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE);
3920 }
3921 
3922 void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3923 
3924  const unsigned CallingConv = FI.getCallingConvention();
3925  // It is possible to force Win64 calling convention on any x86_64 target by
3926  // using __attribute__((ms_abi)). In such case to correctly emit Win64
3927  // compatible code delegate this call to WinX86_64ABIInfo::computeInfo.
3928  if (CallingConv == llvm::CallingConv::Win64) {
3929  WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);
3930  Win64ABIInfo.computeInfo(FI);
3931  return;
3932  }
3933 
3934  bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;
3935 
3936  // Keep track of the number of assigned registers.
3937  unsigned FreeIntRegs = IsRegCall ? 11 : 6;
3938  unsigned FreeSSERegs = IsRegCall ? 16 : 8;
3939  unsigned NeededInt, NeededSSE;
3940 
3941  if (!::classifyReturnType(getCXXABI(), FI, *this)) {
3942  if (IsRegCall && FI.getReturnType()->getTypePtr()->isRecordType() &&
3943  !FI.getReturnType()->getTypePtr()->isUnionType()) {
3944  FI.getReturnInfo() =
3945  classifyRegCallStructType(FI.getReturnType(), NeededInt, NeededSSE);
3946  if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3947  FreeIntRegs -= NeededInt;
3948  FreeSSERegs -= NeededSSE;
3949  } else {
3950  FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3951  }
3952  } else if (IsRegCall && FI.getReturnType()->getAs<ComplexType>() &&
3953  getContext().getCanonicalType(FI.getReturnType()
3954  ->getAs<ComplexType>()
3955  ->getElementType()) ==
3956  getContext().LongDoubleTy)
3957  // Complex Long Double Type is passed in Memory when Regcall
3958  // calling convention is used.
3959  FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3960  else
3962  }
3963 
3964  // If the return value is indirect, then the hidden argument is consuming one
3965  // integer register.
3966  if (FI.getReturnInfo().isIndirect())
3967  --FreeIntRegs;
3968 
3969  // The chain argument effectively gives us another free register.
3970  if (FI.isChainCall())
3971  ++FreeIntRegs;
3972 
3973  unsigned NumRequiredArgs = FI.getNumRequiredArgs();
3974  // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
3975  // get assigned (in left-to-right order) for passing as follows...
3976  unsigned ArgNo = 0;
3977  for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
3978  it != ie; ++it, ++ArgNo) {
3979  bool IsNamedArg = ArgNo < NumRequiredArgs;
3980 
3981  if (IsRegCall && it->type->isStructureOrClassType())
3982  it->info = classifyRegCallStructType(it->type, NeededInt, NeededSSE);
3983  else
3984  it->info = classifyArgumentType(it->type, FreeIntRegs, NeededInt,
3985  NeededSSE, IsNamedArg);
3986 
3987  // AMD64-ABI 3.2.3p3: If there are no registers available for any
3988  // eightbyte of an argument, the whole argument is passed on the
3989  // stack. If registers have already been assigned for some
3990  // eightbytes of such an argument, the assignments get reverted.
3991  if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3992  FreeIntRegs -= NeededInt;
3993  FreeSSERegs -= NeededSSE;
3994  } else {
3995  it->info = getIndirectResult(it->type, FreeIntRegs);
3996  }
3997  }
3998 }
3999 
4001  Address VAListAddr, QualType Ty) {
4002  Address overflow_arg_area_p =
4003  CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_p");
4004  llvm::Value *overflow_arg_area =
4005  CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area");
4006 
4007  // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
4008  // byte boundary if alignment needed by type exceeds 8 byte boundary.
4009  // It isn't stated explicitly in the standard, but in practice we use
4010  // alignment greater than 16 where necessary.
4011  CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4012  if (Align > CharUnits::fromQuantity(8)) {
4013  overflow_arg_area = emitRoundPointerUpToAlignment(CGF, overflow_arg_area,
4014  Align);
4015  }
4016 
4017  // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
4018  llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4019  llvm::Value *Res =
4020  CGF.Builder.CreateBitCast(overflow_arg_area,
4021  llvm::PointerType::getUnqual(LTy));
4022 
4023  // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
4024  // l->overflow_arg_area + sizeof(type).
4025  // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
4026  // an 8 byte boundary.
4027 
4028  uint64_t SizeInBytes = (CGF.getContext().getTypeSize(Ty) + 7) / 8;
4029  llvm::Value *Offset =
4030  llvm::ConstantInt::get(CGF.Int32Ty, (SizeInBytes + 7) & ~7);
4031  overflow_arg_area = CGF.Builder.CreateGEP(CGF.Int8Ty, overflow_arg_area,
4032  Offset, "overflow_arg_area.next");
4033  CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p);
4034 
4035  // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
4036  return Address(Res, Align);
4037 }
4038 
4039 Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4040  QualType Ty) const {
4041  // Assume that va_list type is correct; should be pointer to LLVM type:
4042  // struct {
4043  // i32 gp_offset;
4044  // i32 fp_offset;
4045  // i8* overflow_arg_area;
4046  // i8* reg_save_area;
4047  // };
4048  unsigned neededInt, neededSSE;
4049 
4050  Ty = getContext().getCanonicalType(Ty);
4051  ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE,
4052  /*isNamedArg*/false);
4053 
4054  // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
4055  // in the registers. If not go to step 7.
4056  if (!neededInt && !neededSSE)
4057  return EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4058 
4059  // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
4060  // general purpose registers needed to pass type and num_fp to hold
4061  // the number of floating point registers needed.
4062 
4063  // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
4064  // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
4065  // l->fp_offset > 304 - num_fp * 16 go to step 7.
4066  //
4067  // NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
4068  // register save space).
4069 
4070  llvm::Value *InRegs = nullptr;
4071  Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
4072  llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
4073  if (neededInt) {
4074  gp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "gp_offset_p");
4075  gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset");
4076  InRegs = llvm::ConstantInt::get(CGF.Int32Ty, 48 - neededInt * 8);
4077  InRegs = CGF.Builder.CreateICmpULE(gp_offset, InRegs, "fits_in_gp");
4078  }
4079 
4080  if (neededSSE) {
4081  fp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 1, "fp_offset_p");
4082  fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset");
4083  llvm::Value *FitsInFP =
4084  llvm::ConstantInt::get(CGF.Int32Ty, 176 - neededSSE * 16);
4085  FitsInFP = CGF.Builder.CreateICmpULE(fp_offset, FitsInFP, "fits_in_fp");
4086  InRegs = InRegs ? CGF.Builder.CreateAnd(InRegs, FitsInFP) : FitsInFP;
4087  }
4088 
4089  llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
4090  llvm::BasicBlock *InMemBlock = CGF.createBasicBlock("vaarg.in_mem");
4091  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
4092  CGF.Builder.CreateCondBr(InRegs, InRegBlock, InMemBlock);
4093 
4094  // Emit code to load the value if it was passed in registers.
4095 
4096  CGF.EmitBlock(InRegBlock);
4097 
4098  // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
4099  // an offset of l->gp_offset and/or l->fp_offset. This may require
4100  // copying to a temporary location in case the parameter is passed
4101  // in different register classes or requires an alignment greater
4102  // than 8 for general purpose registers and 16 for XMM registers.
4103  //
4104  // FIXME: This really results in shameful code when we end up needing to
4105  // collect arguments from different places; often what should result in a
4106  // simple assembling of a structure from scattered addresses has many more
4107  // loads than necessary. Can we clean this up?
4108  llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4109  llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
4110  CGF.Builder.CreateStructGEP(VAListAddr, 3), "reg_save_area");
4111 
4112  Address RegAddr = Address::invalid();
4113  if (neededInt && neededSSE) {
4114  // FIXME: Cleanup.
4115  assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
4116  llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());
4117  Address Tmp = CGF.CreateMemTemp(Ty);
4118  Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4119  assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
4120  llvm::Type *TyLo = ST->getElementType(0);
4121  llvm::Type *TyHi = ST->getElementType(1);
4122  assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
4123  "Unexpected ABI info for mixed regs");
4124  llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
4125  llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
4126  llvm::Value *GPAddr =
4127  CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset);
4128  llvm::Value *FPAddr =
4129  CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset);
4130  llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? FPAddr : GPAddr;
4131  llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr;
4132 
4133  // Copy the first element.
4134  // FIXME: Our choice of alignment here and below is probably pessimistic.
4136  TyLo, CGF.Builder.CreateBitCast(RegLoAddr, PTyLo),
4137  CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(TyLo)));
4138  CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4139 
4140  // Copy the second element.
4141  V = CGF.Builder.CreateAlignedLoad(
4142  TyHi, CGF.Builder.CreateBitCast(RegHiAddr, PTyHi),
4143  CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(TyHi)));
4144  CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4145 
4146  RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4147  } else if (neededInt) {
4148  RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset),
4150  RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4151 
4152  // Copy to a temporary if necessary to ensure the appropriate alignment.
4153  auto TInfo = getContext().getTypeInfoInChars(Ty);
4154  uint64_t TySize = TInfo.Width.getQuantity();
4155  CharUnits TyAlign = TInfo.Align;
4156 
4157  // Copy into a temporary if the type is more aligned than the
4158  // register save area.
4159  if (TyAlign.getQuantity() > 8) {
4160  Address Tmp = CGF.CreateMemTemp(Ty);
4161  CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, false);
4162  RegAddr = Tmp;
4163  }
4164 
4165  } else if (neededSSE == 1) {
4166  RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset),
4168  RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4169  } else {
4170  assert(neededSSE == 2 && "Invalid number of needed registers!");
4171  // SSE registers are spaced 16 bytes apart in the register save
4172  // area, we need to collect the two eightbytes together.
4173  // The ABI isn't explicit about this, but it seems reasonable
4174  // to assume that the slots are 16-byte aligned, since the stack is
4175  // naturally 16-byte aligned and the prologue is expected to store
4176  // all the SSE registers to the RSA.
4177  Address RegAddrLo = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea,
4178  fp_offset),
4180  Address RegAddrHi =
4181  CGF.Builder.CreateConstInBoundsByteGEP(RegAddrLo,
4183  llvm::Type *ST = AI.canHaveCoerceToType()
4184  ? AI.getCoerceToType()
4185  : llvm::StructType::get(CGF.DoubleTy, CGF.DoubleTy);
4186  llvm::Value *V;
4187  Address Tmp = CGF.CreateMemTemp(Ty);
4188  Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4190  RegAddrLo, ST->getStructElementType(0)));
4191  CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4193  RegAddrHi, ST->getStructElementType(1)));
4194  CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4195 
4196  RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4197  }
4198 
4199  // AMD64-ABI 3.5.7p5: Step 5. Set:
4200  // l->gp_offset = l->gp_offset + num_gp * 8
4201  // l->fp_offset = l->fp_offset + num_fp * 16.
4202  if (neededInt) {
4203  llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededInt * 8);
4204  CGF.Builder.CreateStore(CGF.Builder.CreateAdd(gp_offset, Offset),
4205  gp_offset_p);
4206  }
4207  if (neededSSE) {
4208  llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededSSE * 16);
4209  CGF.Builder.CreateStore(CGF.Builder.CreateAdd(fp_offset, Offset),
4210  fp_offset_p);
4211  }
4212  CGF.EmitBranch(ContBlock);
4213 
4214  // Emit code to load the value if it was passed in memory.
4215 
4216  CGF.EmitBlock(InMemBlock);
4217  Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4218 
4219  // Return the appropriate result.
4220 
4221  CGF.EmitBlock(ContBlock);
4222  Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock,
4223  "vaarg.addr");
4224  return ResAddr;
4225 }
4226 
4227 Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
4228  QualType Ty) const {
4229  // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4230  // not 1, 2, 4, or 8 bytes, must be passed by reference."
4231  uint64_t Width = getContext().getTypeSize(Ty);
4232  bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Width);
4233 
4234  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4235  CGF.getContext().getTypeInfoInChars(Ty),
4237  /*allowHigherAlign*/ false);
4238 }
4239 
4240 ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(
4241  QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo &current) const {
4242  const Type *Base = nullptr;
4243  uint64_t NumElts = 0;
4244 
4245  if (!Ty->isBuiltinType() && !Ty->isVectorType() &&
4246  isHomogeneousAggregate(Ty, Base, NumElts) && FreeSSERegs >= NumElts) {
4247  FreeSSERegs -= NumElts;
4248  return getDirectX86Hva();
4249  }
4250  return current;
4251 }
4252 
4253 ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
4254  bool IsReturnType, bool IsVectorCall,
4255  bool IsRegCall) const {
4256 
4257  if (Ty->isVoidType())
4258  return ABIArgInfo::getIgnore();
4259 
4260  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4261  Ty = EnumTy->getDecl()->getIntegerType();
4262 
4263  TypeInfo Info = getContext().getTypeInfo(Ty);
4264  uint64_t Width = Info.Width;
4265  CharUnits Align = getContext().toCharUnitsFromBits(Info.Align);
4266 
4267  const RecordType *RT = Ty->getAs<RecordType>();
4268  if (RT) {
4269  if (!IsReturnType) {
4270  if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()))
4271  return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4272  }
4273 
4274  if (RT->getDecl()->hasFlexibleArrayMember())
4275  return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4276 
4277  }
4278 
4279  const Type *Base = nullptr;
4280  uint64_t NumElts = 0;
4281  // vectorcall adds the concept of a homogenous vector aggregate, similar to
4282  // other targets.
4283  if ((IsVectorCall || IsRegCall) &&
4284  isHomogeneousAggregate(Ty, Base, NumElts)) {
4285  if (IsRegCall) {
4286  if (FreeSSERegs >= NumElts) {
4287  FreeSSERegs -= NumElts;
4288  if (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())
4289  return ABIArgInfo::getDirect();
4290  return ABIArgInfo::getExpand();
4291  }
4292  return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4293  } else if (IsVectorCall) {
4294  if (FreeSSERegs >= NumElts &&
4295  (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())) {
4296  FreeSSERegs -= NumElts;
4297  return ABIArgInfo::getDirect();
4298  } else if (IsReturnType) {
4299  return ABIArgInfo::getExpand();
4300  } else if (!Ty->isBuiltinType() && !Ty->isVectorType()) {
4301  // HVAs are delayed and reclassified in the 2nd step.
4302  return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4303  }
4304  }
4305  }
4306 
4307  if (Ty->isMemberPointerType()) {
4308  // If the member pointer is represented by an LLVM int or ptr, pass it
4309  // directly.
4310  llvm::Type *LLTy = CGT.ConvertType(Ty);
4311  if (LLTy->isPointerTy() || LLTy->isIntegerTy())
4312  return ABIArgInfo::getDirect();
4313  }
4314 
4315  if (RT || Ty->isAnyComplexType() || Ty->isMemberPointerType()) {
4316  // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4317  // not 1, 2, 4, or 8 bytes, must be passed by reference."
4318  if (Width > 64 || !llvm::isPowerOf2_64(Width))
4319  return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4320 
4321  // Otherwise, coerce it to a small integer.
4322  return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Width));
4323  }
4324 
4325  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
4326  switch (BT->getKind()) {
4327  case BuiltinType::Bool:
4328  // Bool type is always extended to the ABI, other builtin types are not
4329  // extended.
4330  return ABIArgInfo::getExtend(Ty);
4331 
4332  case BuiltinType::LongDouble:
4333  // Mingw64 GCC uses the old 80 bit extended precision floating point
4334  // unit. It passes them indirectly through memory.
4335  if (IsMingw64) {
4336  const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
4337  if (LDF == &llvm::APFloat::x87DoubleExtended())
4338  return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4339  }
4340  break;
4341 
4342  case BuiltinType::Int128:
4343  case BuiltinType::UInt128:
4344  // If it's a parameter type, the normal ABI rule is that arguments larger
4345  // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
4346  // even though it isn't particularly efficient.
4347  if (!IsReturnType)
4348  return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4349 
4350  // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
4351  // Clang matches them for compatibility.
4352  return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
4353  llvm::Type::getInt64Ty(getVMContext()), 2));
4354 
4355  default:
4356  break;
4357  }
4358  }
4359 
4360  if (Ty->isExtIntType()) {
4361  // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4362  // not 1, 2, 4, or 8 bytes, must be passed by reference."
4363  // However, non-power-of-two _ExtInts will be passed as 1,2,4 or 8 bytes
4364  // anyway as long is it fits in them, so we don't have to check the power of
4365  // 2.
4366  if (Width <= 64)
4367  return ABIArgInfo::getDirect();
4368  return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4369  }
4370 
4371  return ABIArgInfo::getDirect();
4372 }
4373 
4374 void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
4375  const unsigned CC = FI.getCallingConvention();
4376  bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
4377  bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
4378 
4379  // If __attribute__((sysv_abi)) is in use, use the SysV argument
4380  // classification rules.
4381  if (CC == llvm::CallingConv::X86_64_SysV) {
4382  X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);
4383  SysVABIInfo.computeInfo(FI);
4384  return;
4385  }
4386 
4387  unsigned FreeSSERegs = 0;
4388  if (IsVectorCall) {
4389  // We can use up to 4 SSE return registers with vectorcall.
4390  FreeSSERegs = 4;
4391  } else if (IsRegCall) {
4392  // RegCall gives us 16 SSE registers.
4393  FreeSSERegs = 16;
4394  }
4395 
4396  if (!getCXXABI().classifyReturnType(FI))
4397  FI.getReturnInfo() = classify(FI.getReturnType(), FreeSSERegs, true,
4398  IsVectorCall, IsRegCall);
4399 
4400  if (IsVectorCall) {
4401  // We can use up to 6 SSE register parameters with vectorcall.
4402  FreeSSERegs = 6;
4403  } else if (IsRegCall) {
4404  // RegCall gives us 16 SSE registers, we can reuse the return registers.
4405  FreeSSERegs = 16;
4406  }
4407 
4408  unsigned ArgNum = 0;
4409  unsigned ZeroSSERegs = 0;
4410  for (auto &I : FI.arguments()) {
4411  // Vectorcall in x64 only permits the first 6 arguments to be passed as
4412  // XMM/YMM registers. After the sixth argument, pretend no vector
4413  // registers are left.
4414  unsigned *MaybeFreeSSERegs =
4415  (IsVectorCall && ArgNum >= 6) ? &ZeroSSERegs : &FreeSSERegs;
4416  I.info =
4417  classify(I.type, *MaybeFreeSSERegs, false, IsVectorCall, IsRegCall);
4418  ++ArgNum;
4419  }
4420 
4421  if (IsVectorCall) {
4422  // For vectorcall, assign aggregate HVAs to any free vector registers in a
4423  // second pass.
4424  for (auto &I : FI.arguments())
4425  I.info = reclassifyHvaArgForVectorCall(I.type, FreeSSERegs, I.info);
4426  }
4427 }
4428 
4429 Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4430  QualType Ty) const {
4431  // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4432  // not 1, 2, 4, or 8 bytes, must be passed by reference."
4433  uint64_t Width = getContext().getTypeSize(Ty);
4434  bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Width);
4435 
4436  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4437  CGF.getContext().getTypeInfoInChars(Ty),
4439  /*allowHigherAlign*/ false);
4440 }
4441 
4443  llvm::Value *Address, bool Is64Bit,
4444  bool IsAIX) {
4445  // This is calculated from the LLVM and GCC tables and verified
4446  // against gcc output. AFAIK all PPC ABIs use the same encoding.
4447 
4448  CodeGen::CGBuilderTy &Builder = CGF.Builder;
4449 
4450  llvm::IntegerType *i8 = CGF.Int8Ty;
4451  llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
4452  llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
4453  llvm::Value *Sixteen8 = llvm::ConstantInt::get(i8, 16);
4454 
4455  // 0-31: r0-31, the 4-byte or 8-byte general-purpose registers
4456  AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 0, 31);
4457 
4458  // 32-63: fp0-31, the 8-byte floating-point registers
4459  AssignToArrayRange(Builder, Address, Eight8, 32, 63);
4460 
4461  // 64-67 are various 4-byte or 8-byte special-purpose registers:
4462  // 64: mq
4463  // 65: lr
4464  // 66: ctr
4465  // 67: ap
4466  AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 64, 67);
4467 
4468  // 68-76 are various 4-byte special-purpose registers:
4469  // 68-75 cr0-7
4470  // 76: xer
4471  AssignToArrayRange(Builder, Address, Four8, 68, 76);
4472 
4473  // 77-108: v0-31, the 16-byte vector registers
4474  AssignToArrayRange(Builder, Address, Sixteen8, 77, 108);
4475 
4476  // 109: vrsave
4477  // 110: vscr
4478  AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 109, 110);
4479 
4480  // AIX does not utilize the rest of the registers.
4481  if (IsAIX)
4482  return false;
4483 
4484  // 111: spe_acc
4485  // 112: spefscr
4486  // 113: sfp
4487  AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 111, 113);
4488 
4489  if (!Is64Bit)
4490  return false;
4491 
4492  // TODO: Need to verify if these registers are used on 64 bit AIX with Power8
4493  // or above CPU.
4494  // 64-bit only registers:
4495  // 114: tfhar
4496  // 115: tfiar
4497  // 116: texasr
4498  AssignToArrayRange(Builder, Address, Eight8, 114, 116);
4499 
4500  return false;
4501 }
4502 
4503 // AIX
4504 namespace {
4505 /// AIXABIInfo - The AIX XCOFF ABI information.
4506 class AIXABIInfo : public ABIInfo {
4507  const bool Is64Bit;
4508  const unsigned PtrByteSize;
4509  CharUnits getParamTypeAlignment(QualType Ty) const;
4510 
4511 public:
4512  AIXABIInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4513  : ABIInfo(CGT), Is64Bit(Is64Bit), PtrByteSize(Is64Bit ? 8 : 4) {}
4514 
4515  bool isPromotableTypeForABI(QualType Ty) const;
4516 
4517  ABIArgInfo classifyReturnType(QualType RetTy) const;
4519 
4520  void computeInfo(CGFunctionInfo &FI) const override {
4521  if (!getCXXABI().classifyReturnType(FI))
4523 
4524  for (auto &I : FI.arguments())
4526  }
4527 
4528  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4529  QualType Ty) const override;
4530 };
4531 
4532 class AIXTargetCodeGenInfo : public TargetCodeGenInfo {
4533  const bool Is64Bit;
4534 
4535 public:
4536  AIXTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4537  : TargetCodeGenInfo(std::make_unique<AIXABIInfo>(CGT, Is64Bit)),
4538  Is64Bit(Is64Bit) {}
4539  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4540  return 1; // r1 is the dedicated stack pointer
4541  }
4542 
4543  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4544  llvm::Value *Address) const override;
4545 };
4546 } // namespace
4547 
4548 // Return true if the ABI requires Ty to be passed sign- or zero-
4549 // extended to 32/64 bits.
4550 bool AIXABIInfo::isPromotableTypeForABI(QualType Ty) const {
4551  // Treat an enum type as its underlying type.
4552  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4553  Ty = EnumTy->getDecl()->getIntegerType();
4554 
4555  // Promotable integer types are required to be promoted by the ABI.
4556  if (Ty->isPromotableIntegerType())
4557  return true;
4558 
4559  if (!Is64Bit)
4560  return false;
4561 
4562  // For 64 bit mode, in addition to the usual promotable integer types, we also
4563  // need to extend all 32-bit types, since the ABI requires promotion to 64
4564  // bits.
4565  if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
4566  switch (BT->getKind()) {
4567  case BuiltinType::Int:
4568  case BuiltinType::UInt:
4569  return true;
4570  default:
4571  break;
4572  }
4573 
4574  return false;
4575 }
4576 
4578  if (RetTy->isAnyComplexType())
4579  return ABIArgInfo::getDirect();
4580 
4581  if (RetTy->isVectorType())
4582  return ABIArgInfo::getDirect();
4583 
4584  if (RetTy->isVoidType())
4585  return ABIArgInfo::getIgnore();
4586 
4587  if (isAggregateTypeForABI(RetTy))
4588  return getNaturalAlignIndirect(RetTy);
4589 
4590  return (isPromotableTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
4591  : ABIArgInfo::getDirect());
4592 }
4593 
4596 
4597  if (Ty->isAnyComplexType())
4598  return ABIArgInfo::getDirect();
4599 
4600  if (Ty->isVectorType())
4601  return ABIArgInfo::getDirect();
4602 
4603  if (isAggregateTypeForABI(Ty)) {
4604  // Records with non-trivial destructors/copy-constructors should not be
4605  // passed by value.
4606  if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
4607  return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4608 
4609  CharUnits CCAlign = getParamTypeAlignment(Ty);
4610  CharUnits TyAlign = getContext().getTypeAlignInChars(Ty);
4611 
4612  return ABIArgInfo::getIndirect(CCAlign, /*ByVal*/ true,
4613  /*Realign*/ TyAlign > CCAlign);
4614  }
4615 
4616  return (isPromotableTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
4617  : ABIArgInfo::getDirect());
4618 }
4619 
4620 CharUnits AIXABIInfo::getParamTypeAlignment(QualType Ty) const {
4621  // Complex types are passed just like their elements.
4622  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4623  Ty = CTy->getElementType();
4624 
4625  if (Ty->isVectorType())
4626  return CharUnits::fromQuantity(16);
4627 
4628  // If the structure contains a vector type, the alignment is 16.
4629  if (isRecordWithSIMDVectorType(getContext(), Ty))
4630  return CharUnits::fromQuantity(16);
4631 
4632  return CharUnits::fromQuantity(PtrByteSize);
4633 }
4634 
4635 Address AIXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4636  QualType Ty) const {
4637 
4638  auto TypeInfo = getContext().getTypeInfoInChars(Ty);
4639  TypeInfo.Align = getParamTypeAlignment(Ty);
4640 
4641  CharUnits SlotSize = CharUnits::fromQuantity(PtrByteSize);
4642 
4643  // If we have a complex type and the base type is smaller than the register
4644  // size, the ABI calls for the real and imaginary parts to be right-adjusted
4645  // in separate words in 32bit mode or doublewords in 64bit mode. However,
4646  // Clang expects us to produce a pointer to a structure with the two parts
4647  // packed tightly. So generate loads of the real and imaginary parts relative
4648  // to the va_list pointer, and store them to a temporary structure. We do the
4649  // same as the PPC64ABI here.
4650  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4651  CharUnits EltSize = TypeInfo.Width / 2;
4652  if (EltSize < SlotSize)
4653  return complexTempStructure(CGF, VAListAddr, Ty, SlotSize, EltSize, CTy);
4654  }
4655 
4656  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo,
4657  SlotSize, /*AllowHigher*/ true);
4658 }
4659 
4660 bool AIXTargetCodeGenInfo::initDwarfEHRegSizeTable(
4662  return PPC_initDwarfEHRegSizeTable(CGF, Address, Is64Bit, /*IsAIX*/ true);
4663 }
4664 
4665 // PowerPC-32
4666 namespace {
4667 /// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information.
4668 class PPC32_SVR4_ABIInfo : public DefaultABIInfo {
4669  bool IsSoftFloatABI;
4670  bool IsRetSmallStructInRegABI;
4671 
4672  CharUnits getParamTypeAlignment(QualType Ty) const;
4673 
4674 public:
4675  PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, bool SoftFloatABI,
4676  bool RetSmallStructInRegABI)
4677  : DefaultABIInfo(CGT), IsSoftFloatABI(SoftFloatABI),
4678  IsRetSmallStructInRegABI(RetSmallStructInRegABI) {}
4679 
4680  ABIArgInfo classifyReturnType(QualType RetTy) const;
4681 
4682  void computeInfo(CGFunctionInfo &FI) const override {
4683  if (!getCXXABI().classifyReturnType(FI))
4685  for (auto &I : FI.arguments())
4687  }
4688 
4689  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4690  QualType Ty) const override;
4691 };
4692 
4693 class PPC32TargetCodeGenInfo : public TargetCodeGenInfo {
4694 public:
4695  PPC32TargetCodeGenInfo(CodeGenTypes &CGT, bool SoftFloatABI,
4696  bool RetSmallStructInRegABI)
4697  : TargetCodeGenInfo(std::make_unique<PPC32_SVR4_ABIInfo>(
4698  CGT, SoftFloatABI, RetSmallStructInRegABI)) {}
4699 
4700  static bool isStructReturnInRegABI(const llvm::Triple &Triple,
4701  const CodeGenOptions &Opts);
4702 
4703  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4704  // This is recovered from gcc output.
4705  return 1; // r1 is the dedicated stack pointer
4706  }
4707 
4708  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4709  llvm::Value *Address) const override;
4710 };
4711 }
4712 
4713 CharUnits PPC32_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
4714  // Complex types are passed just like their elements.
4715  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4716  Ty = CTy->getElementType();
4717 
4718  if (Ty->isVectorType())
4719  return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16
4720  : 4);
4721 
4722  // For single-element float/vector structs, we consider the whole type
4723  // to have the same alignment requirements as its single element.
4724  const Type *AlignTy = nullptr;
4725  if (const Type *EltType = isSingleElementStruct(Ty, getContext())) {
4726  const BuiltinType *BT = EltType->getAs<BuiltinType>();
4727  if ((EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) ||
4728  (BT && BT->isFloatingPoint()))
4729  AlignTy = EltType;
4730  }
4731 
4732  if (AlignTy)
4733  return CharUnits::fromQuantity(AlignTy->isVectorType() ? 16 : 4);
4734  return CharUnits::fromQuantity(4);
4735 }
4736 
4738  uint64_t Size;
4739 
4740  // -msvr4-struct-return puts small aggregates in GPR3 and GPR4.
4741  if (isAggregateTypeForABI(RetTy) && IsRetSmallStructInRegABI &&
4742  (Size = getContext().getTypeSize(RetTy)) <= 64) {
4743  // System V ABI (1995), page 3-22, specified:
4744  // > A structure or union whose size is less than or equal to 8 bytes
4745  // > shall be returned in r3 and r4, as if it were first stored in the
4746  // > 8-byte aligned memory area and then the low addressed word were
4747  // > loaded into r3 and the high-addressed word into r4. Bits beyond
4748  // > the last member of the structure or union are not defined.
4749  //
4750  // GCC for big-endian PPC32 inserts the pad before the first member,
4751  // not "beyond the last member" of the struct. To stay compatible
4752  // with GCC, we coerce the struct to an integer of the same size.
4753  // LLVM will extend it and return i32 in r3, or i64 in r3:r4.
4754  if (Size == 0)
4755  return ABIArgInfo::getIgnore();
4756  else {
4757  llvm::Type *CoerceTy = llvm::Type::getIntNTy(getVMContext(), Size);
4758  return ABIArgInfo::getDirect(CoerceTy);
4759  }
4760  }
4761 
4762  return DefaultABIInfo::classifyReturnType(RetTy);
4763 }
4764 
4765 // TODO: this implementation is now likely redundant with
4766 // DefaultABIInfo::EmitVAArg.
4767 Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
4768  QualType Ty) const {
4769  if (getTarget().getTriple().isOSDarwin()) {
4770  auto TI = getContext().getTypeInfoInChars(Ty);
4771  TI.Align = getParamTypeAlignment(Ty);
4772 
4773  CharUnits SlotSize = CharUnits::fromQuantity(4);
4774  return emitVoidPtrVAArg(CGF, VAList, Ty,
4775  classifyArgumentType(Ty).isIndirect(), TI, SlotSize,
4776  /*AllowHigherAlign=*/true);
4777  }
4778 
4779  const unsigned OverflowLimit = 8;
4780  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4781  // TODO: Implement this. For now ignore.
4782  (void)CTy;
4783  return Address::invalid(); // FIXME?
4784  }
4785 
4786  // struct __va_list_tag {
4787  // unsigned char gpr;
4788  // unsigned char fpr;
4789  // unsigned short reserved;
4790  // void *overflow_arg_area;
4791  // void *reg_save_area;
4792  // };
4793 
4794  bool isI64 = Ty->isIntegerType() && getContext().getTypeSize(Ty) == 64;
4795  bool isInt = !Ty->isFloatingType();
4796  bool isF64 = Ty->isFloatingType() && getContext().getTypeSize(Ty) == 64;
4797 
4798  // All aggregates are passed indirectly? That doesn't seem consistent
4799  // with the argument-lowering code.
4800  bool isIndirect = isAggregateTypeForABI(Ty);
4801 
4802  CGBuilderTy &Builder = CGF.Builder;
4803 
4804  // The calling convention either uses 1-2 GPRs or 1 FPR.
4805  Address NumRegsAddr = Address::invalid();
4806  if (isInt || IsSoftFloatABI) {
4807  NumRegsAddr = Builder.CreateStructGEP(VAList, 0, "gpr");
4808  } else {
4809  NumRegsAddr = Builder.CreateStructGEP(VAList, 1, "fpr");
4810  }
4811 
4812  llvm::Value *NumRegs = Builder.CreateLoad(NumRegsAddr, "numUsedRegs");
4813 
4814  // "Align" the register count when TY is i64.
4815  if (isI64 || (isF64 && IsSoftFloatABI)) {
4816  NumRegs = Builder.CreateAdd(NumRegs, Builder.getInt8(1));
4817  NumRegs = Builder.CreateAnd(NumRegs, Builder.getInt8((uint8_t) ~1U));
4818  }
4819 
4820  llvm::Value *CC =
4821  Builder.CreateICmpULT(NumRegs, Builder.getInt8(OverflowLimit), "cond");
4822 
4823  llvm::BasicBlock *UsingRegs = CGF.createBasicBlock("using_regs");
4824  llvm::BasicBlock *UsingOverflow = CGF.createBasicBlock("using_overflow");
4825  llvm::BasicBlock *Cont = CGF.createBasicBlock("cont");
4826 
4827  Builder.CreateCondBr(CC, UsingRegs, UsingOverflow);
4828 
4829  llvm::Type *DirectTy = CGF.ConvertType(Ty);
4830  if (isIndirect) DirectTy = DirectTy->getPointerTo(0);
4831 
4832  // Case 1: consume registers.
4833  Address RegAddr = Address::invalid();
4834  {
4835  CGF.EmitBlock(UsingRegs);
4836 
4837  Address RegSaveAreaPtr = Builder.CreateStructGEP(VAList, 4);
4838  RegAddr = Address(Builder.CreateLoad(RegSaveAreaPtr),
4840  assert(RegAddr.getElementType() == CGF.Int8Ty);
4841 
4842  // Floating-point registers start after the general-purpose registers.
4843  if (!(isInt || IsSoftFloatABI)) {
4844  RegAddr = Builder.CreateConstInBoundsByteGEP(RegAddr,
4846  }
4847 
4848  // Get the address of the saved value by scaling the number of
4849  // registers we've used by the number of
4850  CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 8);
4851  llvm::Value *RegOffset =
4852  Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity()));
4853  RegAddr = Address(Builder.CreateInBoundsGEP(CGF.Int8Ty,
4854  RegAddr.getPointer(), RegOffset),
4855  RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
4856  RegAddr = Builder.CreateElementBitCast(RegAddr, DirectTy);
4857 
4858  // Increase the used-register count.
4859  NumRegs =
4860  Builder.CreateAdd(NumRegs,
4861  Builder.getInt8((isI64 || (isF64 && IsSoftFloatABI)) ? 2 : 1));
4862  Builder.CreateStore(NumRegs, NumRegsAddr);
4863 
4864  CGF.EmitBranch(Cont);
4865  }
4866 
4867  // Case 2: consume space in the overflow area.
4868  Address MemAddr = Address::invalid();
4869  {
4870  CGF.EmitBlock(UsingOverflow);
4871 
4872  Builder.CreateStore(Builder.getInt8(OverflowLimit), NumRegsAddr);
4873 
4874  // Everything in the overflow area is rounded up to a size of at least 4.
4875  CharUnits OverflowAreaAlign = CharUnits::fromQuantity(4);
4876 
4877  CharUnits Size;
4878  if (!isIndirect) {
4879  auto TypeInfo = CGF.getContext().getTypeInfoInChars(Ty);
4880  Size = TypeInfo.Width.alignTo(OverflowAreaAlign);
4881  } else {
4882  Size = CGF.getPointerSize();
4883  }
4884 
4885  Address OverflowAreaAddr = Builder.CreateStructGEP(VAList, 3);
4886  Address OverflowArea(Builder.CreateLoad(OverflowAreaAddr, "argp.cur"),
4887  OverflowAreaAlign);
4888  // Round up address of argument to alignment
4889  CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4890  if (Align > OverflowAreaAlign) {
4891  llvm::Value *Ptr = OverflowArea.getPointer();
4892  OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align),
4893  Align);
4894  }
4895 
4896  MemAddr = Builder.CreateElementBitCast(OverflowArea, DirectTy);
4897 
4898  // Increase the overflow area.
4899  OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size);
4900  Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr);
4901  CGF.EmitBranch(Cont);
4902  }
4903 
4904  CGF.EmitBlock(Cont);
4905 
4906  // Merge the cases with a phi.
4907  Address Result = emitMergePHI(CGF, RegAddr, UsingRegs, MemAddr, UsingOverflow,
4908  "vaarg.addr");
4909 
4910  // Load the pointer if the argument was passed indirectly.
4911  if (isIndirect) {
4912  Result = Address(Builder.CreateLoad(Result, "aggr"),
4913  getContext().getTypeAlignInChars(Ty));
4914  }
4915 
4916  return Result;
4917 }
4918 
4919 bool PPC32TargetCodeGenInfo::isStructReturnInRegABI(
4920  const llvm::Triple &Triple, const CodeGenOptions &Opts) {
4921  assert(Triple.isPPC32());
4922 
4923  switch (Opts.getStructReturnConvention()) {
4925  break;
4926  case CodeGenOptions::SRCK_OnStack: // -maix-struct-return
4927  return false;
4928  case CodeGenOptions::SRCK_InRegs: // -msvr4-struct-return
4929  return true;
4930  }
4931 
4932  if (Triple.isOSBinFormatELF() && !Triple.isOSLinux())
4933  return true;
4934 
4935  return false;
4936 }
4937 
4938 bool
4939 PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4940  llvm::Value *Address) const {
4941  return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ false,
4942  /*IsAIX*/ false);
4943 }
4944 
4945 // PowerPC-64
4946 
4947 namespace {
4948 /// PPC64_SVR4_ABIInfo - The 64-bit PowerPC ELF (SVR4) ABI information.
4949 class PPC64_SVR4_ABIInfo : public SwiftABIInfo {
4950 public:
4951  enum ABIKind {
4952  ELFv1 = 0,
4953  ELFv2
4954  };
4955 
4956 private:
4957  static const unsigned GPRBits = 64;
4958  ABIKind Kind;
4959  bool IsSoftFloatABI;
4960 
4961 public:
4962  PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind,
4963  bool SoftFloatABI)
4964  : SwiftABIInfo(CGT), Kind(Kind), IsSoftFloatABI(SoftFloatABI) {}
4965 
4966  bool isPromotableTypeForABI(QualType Ty) const;
4967  CharUnits getParamTypeAlignment(QualType Ty) const;
4968 
4969  ABIArgInfo classifyReturnType(QualType RetTy) const;
4971 
4972  bool isHomogeneousAggregateBaseType(QualType Ty) const override;
4973  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
4974  uint64_t Members) const override;
4975 
4976  // TODO: We can add more logic to computeInfo to improve performance.
4977  // Example: For aggregate arguments that fit in a register, we could
4978  // use getDirectInReg (as is done below for structs containing a single
4979  // floating-point value) to avoid pushing them to memory on function
4980  // entry. This would require changing the logic in PPCISelLowering
4981  // when lowering the parameters in the caller and args in the callee.
4982  void computeInfo(CGFunctionInfo &FI) const override {
4983  if (!getCXXABI().classifyReturnType(FI))
4985  for (auto &I : FI.arguments()) {
4986  // We rely on the default argument classification for the most part.
4987  // One exception: An aggregate containing a single floating-point
4988  // or vector item must be passed in a register if one is available.
4989  const Type *T = isSingleElementStruct(I.type, getContext());
4990  if (T) {
4991  const BuiltinType *BT = T->getAs<BuiltinType>();
4992  if ((T->isVectorType() && getContext().getTypeSize(T) == 128) ||
4993  (BT && BT->isFloatingPoint())) {
4994  QualType QT(T, 0);
4995  I.info = ABIArgInfo::getDirectInReg(CGT.ConvertType(QT));
4996  continue;
4997  }
4998  }
5000  }
5001  }
5002 
5003  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
5004  QualType Ty) const override;
5005 
5006  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
5007  bool asReturnValue) const override {
5008  return occupiesMoreThan(CGT, scalars, /*total*/ 4);
5009  }
5010 
5011  bool isSwiftErrorInRegister() const override {
5012  return false;
5013  }
5014 };
5015 
5016 class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo {
5017 
5018 public:
5019  PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT,
5020  PPC64_SVR4_ABIInfo::ABIKind Kind,
5021  bool SoftFloatABI)
5023  std::make_unique<PPC64_SVR4_ABIInfo>(CGT, Kind, SoftFloatABI)) {}
5024 
5025  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5026  // This is recovered from gcc output.
5027  return 1; // r1 is the dedicated stack pointer
5028  }
5029 
5030  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5031  llvm::Value *Address) const override;
5032 };
5033 
5034 class PPC64TargetCodeGenInfo : public DefaultTargetCodeGenInfo {
5035 public:
5036  PPC64TargetCodeGenInfo(CodeGenTypes &CGT) : DefaultTargetCodeGenInfo(CGT) {}
5037 
5038  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5039  // This is recovered from gcc output.
5040  return 1; // r1 is the dedicated stack pointer
5041  }
5042 
5043  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5044  llvm::Value *Address) const override;
5045 };
5046 
5047 }
5048 
5049 // Return true if the ABI requires Ty to be passed sign- or zero-
5050 // extended to 64 bits.
5051 bool
5052 PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const {
5053  // Treat an enum type as its underlying type.
5054  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
5055  Ty = EnumTy->getDecl()->getIntegerType();
5056 
5057  // Promotable integer types are required to be promoted by the ABI.
5058  if (isPromotableIntegerTypeForABI(Ty))
5059  return true;
5060 
5061  // In addition to the usual promotable integer types, we also need to
5062  // extend all 32-bit types, since the ABI requires promotion to 64 bits.
5063  if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
5064  switch (BT->getKind()) {
5065  case BuiltinType::Int:
5066  case BuiltinType::UInt:
5067  return true;
5068  default:
5069  break;
5070  }
5071 
5072  if (const auto *EIT = Ty->getAs<ExtIntType>())
5073  if (EIT->getNumBits() < 64)
5074  return true;
5075 
5076  return false;
5077 }
5078 
5079 /// isAlignedParamType - Determine whether a type requires 16-byte or
5080 /// higher alignment in the parameter area. Always returns at least 8.
5081 CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
5082  // Complex types are passed just like their elements.
5083  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
5084  Ty = CTy->getElementType();
5085 
5086  // Only vector types of size 16 bytes need alignment (larger types are
5087  // passed via reference, smaller types are not aligned).
5088  if (Ty->isVectorType()) {
5089  return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16 : 8);
5090  } else if (Ty->isRealFloatingType() &&
5091  &getContext().getFloatTypeSemantics(Ty) ==
5092  &llvm::APFloat::IEEEquad()) {
5093  // According to ABI document section 'Optional Save Areas': If extended
5094  // precision floating-point values in IEEE BINARY 128 QUADRUPLE PRECISION
5095  // format are supported, map them to a single quadword, quadword aligned.
5096  return CharUnits::fromQuantity(16);
5097  }
5098 
5099  // For single-element float/vector structs, we consider the whole type
5100  // to have the same alignment requirements as its single element.
5101  const Type *AlignAsType = nullptr;
5102  const Type *EltType = isSingleElementStruct(Ty, getContext());
5103  if (EltType) {
5104  const BuiltinType *BT = EltType->getAs<BuiltinType>();
5105  if ((EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) ||
5106  (BT && BT->isFloatingPoint()))
5107  AlignAsType = EltType;
5108  }
5109 
5110  // Likewise for ELFv2 homogeneous aggregates.
5111  const Type *Base = nullptr;
5112  uint64_t Members = 0;
5113  if (!AlignAsType && Kind == ELFv2 &&
5114  isAggregateTypeForABI(Ty) && isHomogeneousAggregate(Ty, Base, Members))
5115  AlignAsType = Base;
5116 
5117  // With special case aggregates, only vector base types need alignment.
5118  if (AlignAsType) {
5119  return CharUnits::fromQuantity(AlignAsType->isVectorType() ? 16 : 8);
5120  }
5121 
5122  // Otherwise, we only need alignment for any aggregate type that
5123  // has an alignment requirement of >= 16 bytes.
5124  if (isAggregateTypeForABI(Ty) && getContext().getTypeAlign(Ty) >= 128) {
5125  return CharUnits::fromQuantity(16);
5126  }
5127 
5128  return CharUnits::fromQuantity(8);
5129 }
5130 
5131 /// isHomogeneousAggregate - Return true if a type is an ELFv2 homogeneous
5132 /// aggregate. Base is set to the base element type, and Members is set
5133 /// to the number of base elements.
5135  uint64_t &Members) const {
5136  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
5137  uint64_t NElements = AT->getSize().getZExtValue();
5138  if (NElements == 0)
5139  return false;
5140  if (!isHomogeneousAggregate(AT->getElementType(), Base, Members))
5141  return false;
5142  Members *= NElements;
5143  } else if (const RecordType *RT = Ty->getAs<RecordType>()) {
5144  const RecordDecl *RD = RT->getDecl();
5145  if (RD->hasFlexibleArrayMember())
5146  return false;
5147 
5148  Members = 0;
5149 
5150  // If this is a C++ record, check the properties of the record such as
5151  // bases and ABI specific restrictions
5152  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
5153  if (!getCXXABI().isPermittedToBeHomogeneousAggregate(CXXRD))
5154  return false;
5155 
5156  for (const auto &I : CXXRD->bases()) {
5157  // Ignore empty records.
5158  if (isEmptyRecord(getContext(), I.getType(), true))
5159  continue;
5160 
5161  uint64_t FldMembers;
5162  if (!isHomogeneousAggregate(I.getType(), Base, FldMembers))
5163  return false;
5164 
5165  Members += FldMembers;
5166  }
5167  }
5168 
5169  for (const auto *FD : RD->fields()) {
5170  // Ignore (non-zero arrays of) empty records.
5171  QualType FT = FD->getType();
5172  while (const ConstantArrayType *AT =
5173  getContext().getAsConstantArrayType(FT)) {
5174  if (AT->getSize().getZExtValue() == 0)
5175  return false;
5176  FT = AT->getElementType();
5177  }
5178  if (isEmptyRecord(getContext(), FT, true))
5179  continue;
5180 
5181  // For compatibility with GCC, ignore empty bitfields in C++ mode.
5182  if (getContext().getLangOpts().CPlusPlus &&
5183  FD->isZeroLengthBitField(getContext()))
5184  continue;
5185 
5186  uint64_t FldMembers;
5187  if (!isHomogeneousAggregate(FD->getType(), Base, FldMembers))
5188  return false;
5189 
5190  Members = (RD->isUnion() ?
5191  std::max(Members, FldMembers) : Members + FldMembers);
5192  }
5193 
5194  if (!Base)
5195  return false;
5196 
5197  // Ensure there is no padding.
5198  if (getContext().getTypeSize(Base) * Members !=
5199  getContext().getTypeSize(Ty))
5200  return false;
5201  } else {
5202  Members = 1;
5203  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
5204  Members = 2;
5205  Ty = CT->getElementType();
5206  }
5207 
5208  // Most ABIs only support float, double, and some vector type widths.
5210  return false;
5211 
5212  // The base type must be the same for all members. Types that
5213  // agree in both total size and mode (float vs. vector) are
5214  // treated as being equivalent here.
5215  const Type *TyPtr = Ty.getTypePtr();
5216  if (!Base) {
5217  Base = TyPtr;
5218  // If it's a non-power-of-2 vector, its size is already a power-of-2,
5219  // so make sure to widen it explicitly.
5220  if (const VectorType *VT = Base->getAs<VectorType>()) {
5221  QualType EltTy = VT->getElementType();
5222  unsigned NumElements =
5223  getContext().getTypeSize(VT) / getContext().getTypeSize(EltTy);
5224  Base = getContext()
5225  .getVectorType(EltTy, NumElements, VT->getVectorKind())
5226  .getTypePtr();
5227  }
5228  }
5229 
5230  if (Base->isVectorType() != TyPtr->isVectorType() ||
5231  getContext().getTypeSize(Base) != getContext().getTypeSize(TyPtr))
5232  return false;
5233  }
5234  return Members > 0 && isHomogeneousAggregateSmallEnough(Base, Members);
5235 }
5236 
5237 bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
5238  // Homogeneous aggregates for ELFv2 must have base types of float,
5239  // double, long double, or 128-bit vectors.
5240  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
5241  if (BT->getKind() == BuiltinType::Float ||
5242  BT->getKind() == BuiltinType::Double ||
5243  BT->getKind() == BuiltinType::LongDouble ||
5244  BT->getKind() == BuiltinType::Ibm128 ||
5245  (getContext().getTargetInfo().hasFloat128Type() &&
5246  (BT->getKind() == BuiltinType::Float128))) {
5247  if (IsSoftFloatABI)
5248  return false;
5249  return true;
5250  }
5251  }
5252  if (const VectorType *VT = Ty->getAs<VectorType>()) {
5253  if (getContext().getTypeSize(VT) == 128)
5254  return true;
5255  }
5256  return false;
5257 }
5258 
5259 bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateSmallEnough(
5260  const Type *Base, uint64_t Members) const {
5261  // Vector and fp128 types require one register, other floating point types
5262  // require one or two registers depending on their size.
5263  uint32_t NumRegs =
5264  ((getContext().getTargetInfo().hasFloat128Type() &&
5265  Base->isFloat128Type()) ||
5266  Base->isVectorType()) ? 1
5267  : (getContext().getTypeSize(Base) + 63) / 64;
5268 
5269  // Homogeneous Aggregates may occupy at most 8 registers.
5270  return Members * NumRegs <= 8;
5271 }
5272 
5273 ABIArgInfo
5276 
5277  if (Ty->isAnyComplexType())
5278  return ABIArgInfo::getDirect();
5279 
5280  // Non-Altivec vector types are passed in GPRs (smaller than 16 bytes)
5281  // or via reference (larger than 16 bytes).
5282  if (Ty->isVectorType()) {
5283  uint64_t Size = getContext().getTypeSize(Ty);
5284  if (Size > 128)
5285  return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
5286  else if (Size < 128) {
5287  llvm::Type *CoerceTy = llvm::IntegerType::get(getVMContext(), Size);
5288  return ABIArgInfo::getDirect(CoerceTy);
5289  }
5290  }
5291 
5292  if (const auto *EIT = Ty->getAs<ExtIntType>())
5293  if (EIT->getNumBits() > 128)
5294  return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
5295 
5296  if (isAggregateTypeForABI(Ty)) {
5297  if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
5298  return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
5299 
5300  uint64_t ABIAlign = getParamTypeAlignment(Ty).getQuantity();
5301  uint64_t TyAlign = getContext().getTypeAlignInChars(Ty).getQuantity();
5302 
5303  // ELFv2 homogeneous aggregates are passed as array types.
5304  const Type *Base = nullptr;
5305  uint64_t Members = 0;
5306  if (Kind == ELFv2 &&
5307  isHomogeneousAggregate(Ty, Base, Members)) {
5308  llvm::Type *BaseTy = CGT.ConvertType(QualType(Base, 0));
5309  llvm::Type *CoerceTy = llvm::ArrayType::get(BaseTy, Members);
5310  return ABIArgInfo::getDirect(CoerceTy);
5311  }
5312 
5313  // If an aggregate may end up fully in registers, we do not
5314  // use the ByVal method, but pass the aggregate as array.
5315  // This is usually beneficial since we avoid forcing the
5316  // back-end to store the argument to memory.
5317  uint64_t Bits = getContext().getTypeSize(Ty);
5318  if (Bits > 0 && Bits <= 8 * GPRBits) {
5319  llvm::Type *CoerceTy;
5320 
5321  // Types up to 8 bytes are passed as integer type (which will be
5322  // properly aligned in the argument save area doubleword).
5323  if (Bits <= GPRBits)
5324  CoerceTy =
5325  llvm::IntegerType::get(getVMContext(), llvm::alignTo(Bits, 8));
5326  // Larger types are passed as arrays, with the base type selected
5327  // according to the required alignment in the save area.
5328  else {
5329  uint64_t RegBits = ABIAlign * 8;
5330  uint64_t NumRegs = llvm::alignTo(Bits, RegBits) / RegBits;
5331  llvm::Type *RegTy = llvm::IntegerType::get(getVMContext(), RegBits);
5332  CoerceTy = llvm::ArrayType::get(RegTy, NumRegs);
5333  }
5334 
5335  return ABIArgInfo::getDirect(CoerceTy);
5336  }
5337 
5338  // All other aggregates are passed ByVal.
5340  /*ByVal=*/true,
5341  /*Realign=*/TyAlign > ABIAlign);
5342  }
5343 
5344  return (isPromotableTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
5345  : ABIArgInfo::getDirect());
5346 }
5347 
5348 ABIArgInfo
5350  if (RetTy->isVoidType())
5351  return ABIArgInfo::getIgnore();
5352 
5353  if (RetTy->isAnyComplexType())
5354  return ABIArgInfo::getDirect();
5355 
5356  // Non-Altivec vector types are returned in GPRs (smaller than 16 bytes)
5357  // or via reference (larger than 16 bytes).
5358  if (RetTy->isVectorType()) {
5359  uint64_t Size = getContext().getTypeSize(RetTy);
5360  if (Size > 128)
5361  return getNaturalAlignIndirect(RetTy);
5362  else if (Size < 128) {
5363  llvm::Type *CoerceTy = llvm::IntegerType::get(getVMContext(), Size);
5364  return ABIArgInfo::getDirect(CoerceTy);
5365  }
5366  }
5367 
5368  if (const auto *EIT = RetTy->getAs<ExtIntType>())
5369  if (EIT->getNumBits() > 128)
5370  return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
5371 
5372  if (isAggregateTypeForABI(RetTy)) {
5373  // ELFv2 homogeneous aggregates are returned as array types.
5374  const Type *Base = nullptr;
5375  uint64_t Members = 0;
5376  if (Kind == ELFv2 &&
5377  isHomogeneousAggregate(RetTy, Base, Members)) {
5378  llvm::Type *BaseTy = CGT.ConvertType(QualType(Base, 0));
5379  llvm::Type *CoerceTy = llvm::ArrayType::get(BaseTy, Members);
5380  return ABIArgInfo::getDirect(CoerceTy);
5381  }
5382 
5383  // ELFv2 small aggregates are returned in up to two registers.
5384  uint64_t Bits = getContext().getTypeSize(RetTy);
5385  if (Kind == ELFv2 && Bits <= 2 * GPRBits) {
5386  if (Bits == 0)
5387  return ABIArgInfo::getIgnore();
5388 
5389  llvm::Type *CoerceTy;
5390  if (Bits > GPRBits) {
5391  CoerceTy = llvm::IntegerType::get(getVMContext(), GPRBits);
5392  CoerceTy = llvm::StructType::get(CoerceTy, CoerceTy);
5393  } else
5394  CoerceTy =
5395  llvm::IntegerType::get(getVMContext(), llvm::alignTo(Bits, 8));
5396  return ABIArgInfo::getDirect(CoerceTy);
5397  }
5398 
5399  // All other aggregates are returned indirectly.
5400  return getNaturalAlignIndirect(RetTy);
5401  }
5402 
5403  return (isPromotableTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
5404  : ABIArgInfo::getDirect());
5405 }
5406 
5407 // Based on ARMABIInfo::EmitVAArg, adjusted for 64-bit machine.
5408 Address PPC64_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
5409  QualType Ty) const {
5410  auto TypeInfo = getContext().getTypeInfoInChars(Ty);
5411  TypeInfo.Align = getParamTypeAlignment(Ty);
5412 
5413  CharUnits SlotSize = CharUnits::fromQuantity(8);
5414 
5415  // If we have a complex type and the base type is smaller than 8 bytes,
5416  // the ABI calls for the real and imaginary parts to be right-adjusted
5417  // in separate doublewords. However, Clang expects us to produce a
5418  // pointer to a structure with the two parts packed tightly. So generate
5419  // loads of the real and imaginary parts relative to the va_list pointer,
5420  // and store them to a temporary structure.
5421  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
5422  CharUnits EltSize = TypeInfo.Width / 2;
5423  if (EltSize < SlotSize)
5424  return complexTempStructure(CGF, VAListAddr, Ty, SlotSize, EltSize, CTy);
5425  }
5426 
5427  // Otherwise, just use the general rule.
5428  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false,
5429  TypeInfo, SlotSize, /*AllowHigher*/ true);
5430 }
5431 
5432 bool
5433 PPC64_SVR4_TargetCodeGenInfo::initDwarfEHRegSizeTable(
5435  llvm::Value *Address) const {
5436  return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ true,
5437  /*IsAIX*/ false);
5438 }
5439 
5440 bool
5441 PPC64TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5442  llvm::Value *Address) const {
5443  return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ true,
5444  /*IsAIX*/ false);
5445 }
5446 
5447 //===----------------------------------------------------------------------===//
5448 // AArch64 ABI Implementation
5449 //===----------------------------------------------------------------------===//
5450 
5451 namespace {
5452 
5453 class AArch64ABIInfo : public SwiftABIInfo {
5454 public:
5455  enum ABIKind {
5456  AAPCS = 0,
5457  DarwinPCS,
5458  Win64
5459  };
5460 
5461 private:
5462  ABIKind Kind;
5463 
5464 public:
5465  AArch64ABIInfo(CodeGenTypes &CGT, ABIKind Kind)
5466  : SwiftABIInfo(CGT), Kind(Kind) {}
5467 
5468 private:
5469  ABIKind getABIKind() const { return Kind; }
5470  bool isDarwinPCS() const { return Kind == DarwinPCS; }
5471 
5472  ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadic) const;
5473  ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadic,
5474  unsigned CallingConvention) const;
5475  ABIArgInfo coerceIllegalVector(QualType Ty) const;
5476  bool isHomogeneousAggregateBaseType(QualType Ty) const override;
5477  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
5478  uint64_t Members) const override;
5479 
5480  bool isIllegalVectorType(QualType Ty) const;
5481 
5482  void computeInfo(CGFunctionInfo &FI) const override {
5483  if (!::classifyReturnType(getCXXABI(), FI, *this))
5484  FI.getReturnInfo() =
5486 
5487  for (auto &it : FI.arguments())
5488  it.info = classifyArgumentType(it.type, FI.isVariadic(),
5489  FI.getCallingConvention());
5490  }
5491 
5492  Address EmitDarwinVAArg(Address VAListAddr, QualType Ty,
5493  CodeGenFunction &CGF) const;
5494 
5495  Address EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
5496  CodeGenFunction &CGF) const;
5497 
5498  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
5499  QualType Ty) const override {
5500  llvm::Type *BaseTy = CGF.ConvertType(Ty);
5501  if (isa<llvm::ScalableVectorType>(BaseTy))
5502  llvm::report_fatal_error("Passing SVE types to variadic functions is "
5503  "currently not supported");
5504 
5505  return Kind == Win64 ? EmitMSVAArg(CGF, VAListAddr, Ty)
5506  : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF)
5507  : EmitAAPCSVAArg(VAListAddr, Ty, CGF);
5508  }
5509 
5510  Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
5511  QualType Ty) const override;
5512 
5513  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
5514  bool asReturnValue) const override {
5515  return occupiesMoreThan(CGT, scalars, /*total*/ 4);
5516  }
5517  bool isSwiftErrorInRegister() const override {
5518  return true;
5519  }
5520 
5521  bool isLegalVectorTypeForSwift(CharUnits totalSize, llvm::Type *eltTy,
5522  unsigned elts) const override;
5523 
5524  bool allowBFloatArgsAndRet() const override {
5525  return getTarget().hasBFloat16Type();
5526  }
5527 };
5528 
5529 class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
5530 public:
5531  AArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIInfo::ABIKind Kind)
5532  : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(CGT, Kind)) {}
5533 
5534  StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
5535  return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue";
5536  }
5537 
5538  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5539  return 31;
5540  }
5541 
5542  bool doesReturnSlotInterfereWithArgs() const override { return false; }
5543 
5544  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
5545  CodeGen::CodeGenModule &CGM) const override {
5546  const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
5547  if (!FD)
5548  return;
5549 
5550  const auto *TA = FD->getAttr<TargetAttr>();
5551  if (TA == nullptr)
5552  return;
5553 
5554  ParsedTargetAttr Attr = TA->parse();
5555  if (Attr.BranchProtection.empty())
5556  return;
5557 
5559  StringRef Error;
5560  (void)CGM.getTarget().validateBranchProtection(Attr.BranchProtection,
5561  BPI, Error);
5562  assert(Error.empty());
5563 
5564  auto *Fn = cast<llvm::Function>(GV);
5565  static const char *SignReturnAddrStr[] = {"none", "non-leaf", "all"};
5566  Fn->addFnAttr("sign-return-address", SignReturnAddrStr[static_cast<int>(BPI.SignReturnAddr)]);
5567 
5569  Fn->addFnAttr("sign-return-address-key",
5571  ? "a_key"
5572  : "b_key");
5573  }
5574 
5575  Fn->addFnAttr("branch-target-enforcement",
5576  BPI.BranchTargetEnforcement ? "true" : "false");
5577  }
5578 
5579  bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF,
5580  llvm::Type *Ty) const override {
5581  if (CGF.getTarget().hasFeature("ls64")) {
5582  auto *ST = dyn_cast<llvm::StructType>(Ty);
5583  if (ST && ST->getNumElements() == 1) {
5584  auto *AT = dyn_cast<llvm::ArrayType>(ST->getElementType(0));
5585  if (AT && AT->getNumElements() == 8 &&
5586  AT->getElementType()->isIntegerTy(64))
5587  return true;
5588  }
5589  }
5591  }
5592 };
5593 
5594 class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo {
5595 public:
5596  WindowsAArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIInfo::ABIKind K)
5597  : AArch64TargetCodeGenInfo(CGT, K) {}
5598 
5599  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
5600  CodeGen::CodeGenModule &CGM) const override;
5601 
5602  void getDependentLibraryOption(llvm::StringRef Lib,
5603  llvm::SmallString<24> &Opt) const override {
5604  Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);
5605  }
5606 
5607  void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,
5608  llvm::SmallString<32> &Opt) const override {
5609  Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
5610  }
5611 };
5612 
5613 void WindowsAArch64TargetCodeGenInfo::setTargetAttributes(
5614  const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
5615  AArch64TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
5616  if (GV->isDeclaration())
5617  return;
5618  addStackProbeTargetAttributes(D, GV, CGM);
5619 }
5620 }
5621 
5622 ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty) const {
5623  assert(Ty->isVectorType() && "expected vector type!");
5624 
5625  const auto *VT = Ty->castAs<VectorType>();
5626  if (VT->getVectorKind() == VectorType::SveFixedLengthPredicateVector) {
5627  assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
5628  assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
5629  BuiltinType::UChar &&
5630  "unexpected builtin type for SVE predicate!");
5631  return ABIArgInfo::getDirect(llvm::ScalableVectorType::get(
5632  llvm::Type::getInt1Ty(getVMContext()), 16));
5633  }
5634 
5635  if (VT->getVectorKind() == VectorType::SveFixedLengthDataVector) {
5636  assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
5637 
5638  const auto *BT = VT->getElementType()->castAs<BuiltinType>();
5639  llvm::ScalableVectorType *ResType = nullptr;
5640  switch (BT->getKind()) {
5641  default:
5642  llvm_unreachable("unexpected builtin type for SVE vector!");
5643  case BuiltinType::SChar:
5644  case BuiltinType::UChar:
5645  ResType = llvm::ScalableVectorType::get(
5646  llvm::Type::getInt8Ty(getVMContext()), 16);
5647  break;
5648  case BuiltinType::Short:
5649  case BuiltinType::UShort:
5650  ResType = llvm::ScalableVectorType::get(
5651  llvm::Type::getInt16Ty(getVMContext()), 8);
5652  break;
5653  case BuiltinType::Int:
5654  case BuiltinType::UInt:
5655  ResType = llvm::ScalableVectorType::get(
5656  llvm::Type::getInt32Ty(getVMContext()), 4);
5657  break;
5658  case BuiltinType::Long:
5659  case BuiltinType::ULong:
5660  ResType = llvm::ScalableVectorType::get(
5661  llvm::Type::getInt64Ty(getVMContext()), 2);
5662  break;
5663  case BuiltinType::Half:
5664  ResType = llvm::ScalableVectorType::get(
5665  llvm::Type::getHalfTy(getVMContext()), 8);
5666  break;
5667  case BuiltinType::Float:
5668  ResType = llvm::ScalableVectorType::get(
5669  llvm::Type::getFloatTy(getVMContext()), 4);
5670  break;
5671  case BuiltinType::Double:
5672  ResType = llvm::ScalableVectorType::get(
5673  llvm::Type::getDoubleTy(getVMContext()), 2);
5674  break;
5675  case BuiltinType::BFloat16:
5676  ResType = llvm::ScalableVectorType::get(
5677  llvm::Type::getBFloatTy(getVMContext()), 8);
5678  break;
5679  }
5680  return ABIArgInfo::getDirect(ResType);
5681  }
5682 
5683  uint64_t Size = getContext().getTypeSize(Ty);
5684  // Android promotes <2 x i8> to i16, not i32
5685  if (isAndroid() && (Size <= 16)) {
5686  llvm::Type *ResType = llvm::Type::getInt16Ty(getVMContext());
5687  return ABIArgInfo::getDirect(ResType);
5688  }
5689  if (Size <= 32) {
5690  llvm::Type *ResType = llvm::Type::getInt32Ty(getVMContext());
5691  return ABIArgInfo::getDirect(ResType);
5692  }
5693  if (Size == 64) {
5694  auto *ResType =
5695  llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 2);
5696  return ABIArgInfo::getDirect(ResType);
5697  }
5698  if (Size == 128) {
5699  auto *ResType =
5700  llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 4);
5701  return ABIArgInfo::getDirect(ResType);
5702  }
5703  return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
5704 }
5705 
5706 ABIArgInfo
5708  unsigned CallingConvention) const {
5710 
5711  // Handle illegal vector types here.
5712  if (isIllegalVectorType(Ty))
5713  return coerceIllegalVector(Ty);
5714