clang  15.0.0git
CGCall.cpp
Go to the documentation of this file.
1 //===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // These classes wrap the information about a call or function
10 // definition used to handle ABI compliancy.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGCall.h"
15 #include "ABIInfo.h"
16 #include "CGBlocks.h"
17 #include "CGCXXABI.h"
18 #include "CGCleanup.h"
19 #include "CGRecordLayout.h"
20 #include "CodeGenFunction.h"
21 #include "CodeGenModule.h"
22 #include "TargetInfo.h"
23 #include "clang/AST/Attr.h"
24 #include "clang/AST/Decl.h"
25 #include "clang/AST/DeclCXX.h"
26 #include "clang/AST/DeclObjC.h"
29 #include "clang/Basic/TargetInfo.h"
32 #include "llvm/ADT/StringExtras.h"
33 #include "llvm/Analysis/ValueTracking.h"
34 #include "llvm/IR/Assumptions.h"
35 #include "llvm/IR/Attributes.h"
36 #include "llvm/IR/CallingConv.h"
37 #include "llvm/IR/DataLayout.h"
38 #include "llvm/IR/InlineAsm.h"
39 #include "llvm/IR/IntrinsicInst.h"
40 #include "llvm/IR/Intrinsics.h"
41 #include "llvm/IR/Type.h"
42 #include "llvm/Transforms/Utils/Local.h"
43 using namespace clang;
44 using namespace CodeGen;
45 
46 /***/
47 
49  switch (CC) {
50  default: return llvm::CallingConv::C;
51  case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
52  case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
53  case CC_X86RegCall: return llvm::CallingConv::X86_RegCall;
54  case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
55  case CC_Win64: return llvm::CallingConv::Win64;
56  case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV;
57  case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
58  case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
59  case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI;
60  // TODO: Add support for __pascal to LLVM.
62  // TODO: Add support for __vectorcall to LLVM.
63  case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall;
64  case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall;
65  case CC_AArch64SVEPCS: return llvm::CallingConv::AArch64_SVE_VectorCall;
66  case CC_AMDGPUKernelCall: return llvm::CallingConv::AMDGPU_KERNEL;
67  case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC;
69  case CC_PreserveMost: return llvm::CallingConv::PreserveMost;
70  case CC_PreserveAll: return llvm::CallingConv::PreserveAll;
71  case CC_Swift: return llvm::CallingConv::Swift;
72  case CC_SwiftAsync: return llvm::CallingConv::SwiftTail;
73  }
74 }
75 
76 /// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
77 /// qualification. Either or both of RD and MD may be null. A null RD indicates
78 /// that there is no meaningful 'this' type, and a null MD can occur when
79 /// calling a method pointer.
81  const CXXMethodDecl *MD) {
82  QualType RecTy;
83  if (RD)
84  RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
85  else
86  RecTy = Context.VoidTy;
87 
88  if (MD)
89  RecTy = Context.getAddrSpaceQualType(RecTy, MD->getMethodQualifiers().getAddressSpace());
90  return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
91 }
92 
93 /// Returns the canonical formal type of the given C++ method.
95  return MD->getType()->getCanonicalTypeUnqualified()
97 }
98 
99 /// Returns the "extra-canonicalized" return type, which discards
100 /// qualifiers on the return type. Codegen doesn't care about them,
101 /// and it makes ABI code a little easier to be able to assume that
102 /// all parameter and return types are top-level unqualified.
105 }
106 
107 /// Arrange the argument and result information for a value of the given
108 /// unprototyped freestanding function type.
109 const CGFunctionInfo &
111  // When translating an unprototyped function type, always use a
112  // variadic type.
113  return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
114  /*instanceMethod=*/false,
115  /*chainCall=*/false, None,
116  FTNP->getExtInfo(), {}, RequiredArgs(0));
117 }
118 
121  const FunctionProtoType *proto,
122  unsigned prefixArgs,
123  unsigned totalArgs) {
124  assert(proto->hasExtParameterInfos());
125  assert(paramInfos.size() <= prefixArgs);
126  assert(proto->getNumParams() + prefixArgs <= totalArgs);
127 
128  paramInfos.reserve(totalArgs);
129 
130  // Add default infos for any prefix args that don't already have infos.
131  paramInfos.resize(prefixArgs);
132 
133  // Add infos for the prototype.
134  for (const auto &ParamInfo : proto->getExtParameterInfos()) {
135  paramInfos.push_back(ParamInfo);
136  // pass_object_size params have no parameter info.
137  if (ParamInfo.hasPassObjectSize())
138  paramInfos.emplace_back();
139  }
140 
141  assert(paramInfos.size() <= totalArgs &&
142  "Did we forget to insert pass_object_size args?");
143  // Add default infos for the variadic and/or suffix arguments.
144  paramInfos.resize(totalArgs);
145 }
146 
147 /// Adds the formal parameters in FPT to the given prefix. If any parameter in
148 /// FPT has pass_object_size attrs, then we'll add parameters for those, too.
149 static void appendParameterTypes(const CodeGenTypes &CGT,
150  SmallVectorImpl<CanQualType> &prefix,
151  SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
153  // Fast path: don't touch param info if we don't need to.
154  if (!FPT->hasExtParameterInfos()) {
155  assert(paramInfos.empty() &&
156  "We have paramInfos, but the prototype doesn't?");
157  prefix.append(FPT->param_type_begin(), FPT->param_type_end());
158  return;
159  }
160 
161  unsigned PrefixSize = prefix.size();
162  // In the vast majority of cases, we'll have precisely FPT->getNumParams()
163  // parameters; the only thing that can change this is the presence of
164  // pass_object_size. So, we preallocate for the common case.
165  prefix.reserve(prefix.size() + FPT->getNumParams());
166 
167  auto ExtInfos = FPT->getExtParameterInfos();
168  assert(ExtInfos.size() == FPT->getNumParams());
169  for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
170  prefix.push_back(FPT->getParamType(I));
171  if (ExtInfos[I].hasPassObjectSize())
172  prefix.push_back(CGT.getContext().getSizeType());
173  }
174 
175  addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize,
176  prefix.size());
177 }
178 
179 /// Arrange the LLVM function layout for a value of the given function
180 /// type, on top of any implicit parameters already stored.
181 static const CGFunctionInfo &
182 arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
183  SmallVectorImpl<CanQualType> &prefix,
186  RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
187  // FIXME: Kill copy.
188  appendParameterTypes(CGT, prefix, paramInfos, FTP);
189  CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
190 
191  return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod,
192  /*chainCall=*/false, prefix,
193  FTP->getExtInfo(), paramInfos,
194  Required);
195 }
196 
197 /// Arrange the argument and result information for a value of the
198 /// given freestanding function type.
199 const CGFunctionInfo &
202  return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,
203  FTP);
204 }
205 
207  bool IsWindows) {
208  // Set the appropriate calling convention for the Function.
209  if (D->hasAttr<StdCallAttr>())
210  return CC_X86StdCall;
211 
212  if (D->hasAttr<FastCallAttr>())
213  return CC_X86FastCall;
214 
215  if (D->hasAttr<RegCallAttr>())
216  return CC_X86RegCall;
217 
218  if (D->hasAttr<ThisCallAttr>())
219  return CC_X86ThisCall;
220 
221  if (D->hasAttr<VectorCallAttr>())
222  return CC_X86VectorCall;
223 
224  if (D->hasAttr<PascalAttr>())
225  return CC_X86Pascal;
226 
227  if (PcsAttr *PCS = D->getAttr<PcsAttr>())
228  return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
229 
230  if (D->hasAttr<AArch64VectorPcsAttr>())
231  return CC_AArch64VectorCall;
232 
233  if (D->hasAttr<AArch64SVEPcsAttr>())
234  return CC_AArch64SVEPCS;
235 
236  if (D->hasAttr<AMDGPUKernelCallAttr>())
237  return CC_AMDGPUKernelCall;
238 
239  if (D->hasAttr<IntelOclBiccAttr>())
240  return CC_IntelOclBicc;
241 
242  if (D->hasAttr<MSABIAttr>())
243  return IsWindows ? CC_C : CC_Win64;
244 
245  if (D->hasAttr<SysVABIAttr>())
246  return IsWindows ? CC_X86_64SysV : CC_C;
247 
248  if (D->hasAttr<PreserveMostAttr>())
249  return CC_PreserveMost;
250 
251  if (D->hasAttr<PreserveAllAttr>())
252  return CC_PreserveAll;
253 
254  return CC_C;
255 }
256 
257 /// Arrange the argument and result information for a call to an
258 /// unknown C++ non-static member function of the given abstract type.
259 /// (A null RD means we don't have any meaningful "this" argument type,
260 /// so fall back to a generic pointer type).
261 /// The member function must be an ordinary function, i.e. not a
262 /// constructor or destructor.
263 const CGFunctionInfo &
265  const FunctionProtoType *FTP,
266  const CXXMethodDecl *MD) {
268 
269  // Add the 'this' pointer.
270  argTypes.push_back(DeriveThisType(RD, MD));
271 
273  *this, true, argTypes,
275 }
276 
277 /// Set calling convention for CUDA/HIP kernel.
279  const FunctionDecl *FD) {
280  if (FD->hasAttr<CUDAGlobalAttr>()) {
281  const FunctionType *FT = FTy->getAs<FunctionType>();
283  FTy = FT->getCanonicalTypeUnqualified();
284  }
285 }
286 
287 /// Arrange the argument and result information for a declaration or
288 /// definition of the given C++ non-static member function. The
289 /// member function must be an ordinary function, i.e. not a
290 /// constructor or destructor.
291 const CGFunctionInfo &
293  assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
294  assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
295 
296  CanQualType FT = GetFormalType(MD).getAs<Type>();
297  setCUDAKernelCallingConvention(FT, CGM, MD);
298  auto prototype = FT.getAs<FunctionProtoType>();
299 
300  if (MD->isInstance()) {
301  // The abstract case is perfectly fine.
302  const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
303  return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD);
304  }
305 
306  return arrangeFreeFunctionType(prototype);
307 }
308 
310  const InheritedConstructor &Inherited, CXXCtorType Type) {
311  // Parameters are unnecessary if we're constructing a base class subobject
312  // and the inherited constructor lives in a virtual base.
313  return Type == Ctor_Complete ||
314  !Inherited.getShadowDecl()->constructsVirtualBase() ||
315  !Target.getCXXABI().hasConstructorVariants();
316 }
317 
318 const CGFunctionInfo &
320  auto *MD = cast<CXXMethodDecl>(GD.getDecl());
321 
324  argTypes.push_back(DeriveThisType(MD->getParent(), MD));
325 
326  bool PassParams = true;
327 
328  if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
329  // A base class inheriting constructor doesn't get forwarded arguments
330  // needed to construct a virtual base (or base class thereof).
331  if (auto Inherited = CD->getInheritedConstructor())
332  PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType());
333  }
334 
336 
337  // Add the formal parameters.
338  if (PassParams)
339  appendParameterTypes(*this, argTypes, paramInfos, FTP);
340 
342  TheCXXABI.buildStructorSignature(GD, argTypes);
343  if (!paramInfos.empty()) {
344  // Note: prefix implies after the first param.
345  if (AddedArgs.Prefix)
346  paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix,
348  if (AddedArgs.Suffix)
349  paramInfos.append(AddedArgs.Suffix,
351  }
352 
353  RequiredArgs required =
354  (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())
356 
357  FunctionType::ExtInfo extInfo = FTP->getExtInfo();
358  CanQualType resultType = TheCXXABI.HasThisReturn(GD)
359  ? argTypes.front()
360  : TheCXXABI.hasMostDerivedReturn(GD)
361  ? CGM.getContext().VoidPtrTy
362  : Context.VoidTy;
363  return arrangeLLVMFunctionInfo(resultType, /*instanceMethod=*/true,
364  /*chainCall=*/false, argTypes, extInfo,
365  paramInfos, required);
366 }
367 
371  for (auto &arg : args)
372  argTypes.push_back(ctx.getCanonicalParamType(arg.Ty));
373  return argTypes;
374 }
375 
379  for (auto &arg : args)
380  argTypes.push_back(ctx.getCanonicalParamType(arg->getType()));
381  return argTypes;
382 }
383 
386  unsigned prefixArgs, unsigned totalArgs) {
388  if (proto->hasExtParameterInfos()) {
389  addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs);
390  }
391  return result;
392 }
393 
394 /// Arrange a call to a C++ method, passing the given arguments.
395 ///
396 /// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
397 /// parameter.
398 /// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
399 /// args.
400 /// PassProtoArgs indicates whether `args` has args for the parameters in the
401 /// given CXXConstructorDecl.
402 const CGFunctionInfo &
404  const CXXConstructorDecl *D,
405  CXXCtorType CtorKind,
406  unsigned ExtraPrefixArgs,
407  unsigned ExtraSuffixArgs,
408  bool PassProtoArgs) {
409  // FIXME: Kill copy.
411  for (const auto &Arg : args)
412  ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
413 
414  // +1 for implicit this, which should always be args[0].
415  unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
416 
418  RequiredArgs Required = PassProtoArgs
420  FPT, TotalPrefixArgs + ExtraSuffixArgs)
422 
423  GlobalDecl GD(D, CtorKind);
424  CanQualType ResultType = TheCXXABI.HasThisReturn(GD)
425  ? ArgTypes.front()
426  : TheCXXABI.hasMostDerivedReturn(GD)
427  ? CGM.getContext().VoidPtrTy
428  : Context.VoidTy;
429 
430  FunctionType::ExtInfo Info = FPT->getExtInfo();
432  // If the prototype args are elided, we should only have ABI-specific args,
433  // which never have param info.
434  if (PassProtoArgs && FPT->hasExtParameterInfos()) {
435  // ABI-specific suffix arguments are treated the same as variadic arguments.
436  addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs,
437  ArgTypes.size());
438  }
439  return arrangeLLVMFunctionInfo(ResultType, /*instanceMethod=*/true,
440  /*chainCall=*/false, ArgTypes, Info,
441  ParamInfos, Required);
442 }
443 
444 /// Arrange the argument and result information for the declaration or
445 /// definition of the given function.
446 const CGFunctionInfo &
448  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
449  if (MD->isInstance())
450  return arrangeCXXMethodDeclaration(MD);
451 
453 
454  assert(isa<FunctionType>(FTy));
455  setCUDAKernelCallingConvention(FTy, CGM, FD);
456 
457  // When declaring a function without a prototype, always use a
458  // non-variadic type.
461  noProto->getReturnType(), /*instanceMethod=*/false,
462  /*chainCall=*/false, None, noProto->getExtInfo(), {},RequiredArgs::All);
463  }
464 
466 }
467 
468 /// Arrange the argument and result information for the declaration or
469 /// definition of an Objective-C method.
470 const CGFunctionInfo &
472  // It happens that this is the same as a call with no optional
473  // arguments, except also using the formal 'self' type.
475 }
476 
477 /// Arrange the argument and result information for the function type
478 /// through which to perform a send to the given Objective-C method,
479 /// using the given receiver type. The receiver type is not always
480 /// the 'self' type of the method or even an Objective-C pointer type.
481 /// This is *not* the right method for actually performing such a
482 /// message send, due to the possibility of optional arguments.
483 const CGFunctionInfo &
485  QualType receiverType) {
488  argTys.push_back(Context.getCanonicalParamType(receiverType));
489  argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
490  // FIXME: Kill copy?
491  for (const auto *I : MD->parameters()) {
492  argTys.push_back(Context.getCanonicalParamType(I->getType()));
494  I->hasAttr<NoEscapeAttr>());
495  extParamInfos.push_back(extParamInfo);
496  }
497 
498  FunctionType::ExtInfo einfo;
499  bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows();
500  einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows));
501 
502  if (getContext().getLangOpts().ObjCAutoRefCount &&
503  MD->hasAttr<NSReturnsRetainedAttr>())
504  einfo = einfo.withProducesResult(true);
505 
506  RequiredArgs required =
507  (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
508 
510  GetReturnType(MD->getReturnType()), /*instanceMethod=*/false,
511  /*chainCall=*/false, argTys, einfo, extParamInfos, required);
512 }
513 
514 const CGFunctionInfo &
516  const CallArgList &args) {
517  auto argTypes = getArgTypesForCall(Context, args);
518  FunctionType::ExtInfo einfo;
519 
521  GetReturnType(returnType), /*instanceMethod=*/false,
522  /*chainCall=*/false, argTypes, einfo, {}, RequiredArgs::All);
523 }
524 
525 const CGFunctionInfo &
527  // FIXME: Do we need to handle ObjCMethodDecl?
528  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
529 
530  if (isa<CXXConstructorDecl>(GD.getDecl()) ||
531  isa<CXXDestructorDecl>(GD.getDecl()))
533 
534  return arrangeFunctionDeclaration(FD);
535 }
536 
537 /// Arrange a thunk that takes 'this' as the first parameter followed by
538 /// varargs. Return a void pointer, regardless of the actual return type.
539 /// The body of the thunk will end in a musttail call to a function of the
540 /// correct type, and the caller will bitcast the function to the correct
541 /// prototype.
542 const CGFunctionInfo &
544  assert(MD->isVirtual() && "only methods have thunks");
546  CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)};
547  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/false,
548  /*chainCall=*/false, ArgTys,
549  FTP->getExtInfo(), {}, RequiredArgs(1));
550 }
551 
552 const CGFunctionInfo &
554  CXXCtorType CT) {
555  assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
556 
559  const CXXRecordDecl *RD = CD->getParent();
560  ArgTys.push_back(DeriveThisType(RD, CD));
561  if (CT == Ctor_CopyingClosure)
562  ArgTys.push_back(*FTP->param_type_begin());
563  if (RD->getNumVBases() > 0)
564  ArgTys.push_back(Context.IntTy);
566  /*IsVariadic=*/false, /*IsCXXMethod=*/true);
567  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true,
568  /*chainCall=*/false, ArgTys,
569  FunctionType::ExtInfo(CC), {},
571 }
572 
573 /// Arrange a call as unto a free function, except possibly with an
574 /// additional number of formal parameters considered required.
575 static const CGFunctionInfo &
577  CodeGenModule &CGM,
578  const CallArgList &args,
579  const FunctionType *fnType,
580  unsigned numExtraRequiredArgs,
581  bool chainCall) {
582  assert(args.size() >= numExtraRequiredArgs);
583 
585 
586  // In most cases, there are no optional arguments.
587  RequiredArgs required = RequiredArgs::All;
588 
589  // If we have a variadic prototype, the required arguments are the
590  // extra prefix plus the arguments in the prototype.
591  if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {
592  if (proto->isVariadic())
593  required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs);
594 
595  if (proto->hasExtParameterInfos())
596  addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs,
597  args.size());
598 
599  // If we don't have a prototype at all, but we're supposed to
600  // explicitly use the variadic convention for unprototyped calls,
601  // treat all of the arguments as required but preserve the nominal
602  // possibility of variadics.
603  } else if (CGM.getTargetCodeGenInfo()
604  .isNoProtoCallVariadic(args,
605  cast<FunctionNoProtoType>(fnType))) {
606  required = RequiredArgs(args.size());
607  }
608 
609  // FIXME: Kill copy.
611  for (const auto &arg : args)
612  argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty));
614  /*instanceMethod=*/false, chainCall,
615  argTypes, fnType->getExtInfo(), paramInfos,
616  required);
617 }
618 
619 /// Figure out the rules for calling a function with the given formal
620 /// type using the given arguments. The arguments are necessary
621 /// because the function might be unprototyped, in which case it's
622 /// target-dependent in crazy ways.
623 const CGFunctionInfo &
625  const FunctionType *fnType,
626  bool chainCall) {
627  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType,
628  chainCall ? 1 : 0, chainCall);
629 }
630 
631 /// A block function is essentially a free function with an
632 /// extra implicit argument.
633 const CGFunctionInfo &
635  const FunctionType *fnType) {
636  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1,
637  /*chainCall=*/false);
638 }
639 
640 const CGFunctionInfo &
642  const FunctionArgList &params) {
643  auto paramInfos = getExtParameterInfosForCall(proto, 1, params.size());
644  auto argTypes = getArgTypesForDeclaration(Context, params);
645 
647  /*instanceMethod*/ false, /*chainCall*/ false,
648  argTypes, proto->getExtInfo(), paramInfos,
650 }
651 
652 const CGFunctionInfo &
654  const CallArgList &args) {
655  // FIXME: Kill copy.
657  for (const auto &Arg : args)
658  argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
660  GetReturnType(resultType), /*instanceMethod=*/false,
661  /*chainCall=*/false, argTypes, FunctionType::ExtInfo(),
662  /*paramInfos=*/ {}, RequiredArgs::All);
663 }
664 
665 const CGFunctionInfo &
667  const FunctionArgList &args) {
668  auto argTypes = getArgTypesForDeclaration(Context, args);
669 
671  GetReturnType(resultType), /*instanceMethod=*/false, /*chainCall=*/false,
672  argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
673 }
674 
675 const CGFunctionInfo &
677  ArrayRef<CanQualType> argTypes) {
679  resultType, /*instanceMethod=*/false, /*chainCall=*/false,
680  argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
681 }
682 
683 /// Arrange a call to a C++ method, passing the given arguments.
684 ///
685 /// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
686 /// does not count `this`.
687 const CGFunctionInfo &
689  const FunctionProtoType *proto,
690  RequiredArgs required,
691  unsigned numPrefixArgs) {
692  assert(numPrefixArgs + 1 <= args.size() &&
693  "Emitting a call with less args than the required prefix?");
694  // Add one to account for `this`. It's a bit awkward here, but we don't count
695  // `this` in similar places elsewhere.
696  auto paramInfos =
697  getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());
698 
699  // FIXME: Kill copy.
700  auto argTypes = getArgTypesForCall(Context, args);
701 
702  FunctionType::ExtInfo info = proto->getExtInfo();
704  GetReturnType(proto->getReturnType()), /*instanceMethod=*/true,
705  /*chainCall=*/false, argTypes, info, paramInfos, required);
706 }
707 
710  getContext().VoidTy, /*instanceMethod=*/false, /*chainCall=*/false,
712 }
713 
714 const CGFunctionInfo &
716  const CallArgList &args) {
717  assert(signature.arg_size() <= args.size());
718  if (signature.arg_size() == args.size())
719  return signature;
720 
722  auto sigParamInfos = signature.getExtParameterInfos();
723  if (!sigParamInfos.empty()) {
724  paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
725  paramInfos.resize(args.size());
726  }
727 
728  auto argTypes = getArgTypesForCall(Context, args);
729 
730  assert(signature.getRequiredArgs().allowsOptionalArgs());
731  return arrangeLLVMFunctionInfo(signature.getReturnType(),
732  signature.isInstanceMethod(),
733  signature.isChainCall(),
734  argTypes,
735  signature.getExtInfo(),
736  paramInfos,
737  signature.getRequiredArgs());
738 }
739 
740 namespace clang {
741 namespace CodeGen {
743 }
744 }
745 
746 /// Arrange the argument and result information for an abstract value
747 /// of a given function type. This is the method which all of the
748 /// above functions ultimately defer to.
749 const CGFunctionInfo &
751  bool instanceMethod,
752  bool chainCall,
753  ArrayRef<CanQualType> argTypes,
756  RequiredArgs required) {
757  assert(llvm::all_of(argTypes,
758  [](CanQualType T) { return T.isCanonicalAsParam(); }));
759 
760  // Lookup or create unique function info.
761  llvm::FoldingSetNodeID ID;
762  CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos,
763  required, resultType, argTypes);
764 
765  void *insertPos = nullptr;
766  CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
767  if (FI)
768  return *FI;
769 
770  unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
771 
772  // Construct the function info. We co-allocate the ArgInfos.
773  FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info,
774  paramInfos, resultType, argTypes, required);
775  FunctionInfos.InsertNode(FI, insertPos);
776 
777  bool inserted = FunctionsBeingProcessed.insert(FI).second;
778  (void)inserted;
779  assert(inserted && "Recursively being processed?");
780 
781  // Compute ABI information.
782  if (CC == llvm::CallingConv::SPIR_KERNEL) {
783  // Force target independent argument handling for the host visible
784  // kernel functions.
785  computeSPIRKernelABIInfo(CGM, *FI);
786  } else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync) {
787  swiftcall::computeABIInfo(CGM, *FI);
788  } else {
789  getABIInfo().computeInfo(*FI);
790  }
791 
792  // Loop over all of the computed argument and return value info. If any of
793  // them are direct or extend without a specified coerce type, specify the
794  // default now.
795  ABIArgInfo &retInfo = FI->getReturnInfo();
796  if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
797  retInfo.setCoerceToType(ConvertType(FI->getReturnType()));
798 
799  for (auto &I : FI->arguments())
800  if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
801  I.info.setCoerceToType(ConvertType(I.type));
802 
803  bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
804  assert(erased && "Not in set?");
805 
806  return *FI;
807 }
808 
810  bool instanceMethod,
811  bool chainCall,
812  const FunctionType::ExtInfo &info,
813  ArrayRef<ExtParameterInfo> paramInfos,
814  CanQualType resultType,
815  ArrayRef<CanQualType> argTypes,
816  RequiredArgs required) {
817  assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
818  assert(!required.allowsOptionalArgs() ||
819  required.getNumRequiredArgs() <= argTypes.size());
820 
821  void *buffer =
822  operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
823  argTypes.size() + 1, paramInfos.size()));
824 
825  CGFunctionInfo *FI = new(buffer) CGFunctionInfo();
826  FI->CallingConvention = llvmCC;
827  FI->EffectiveCallingConvention = llvmCC;
828  FI->ASTCallingConvention = info.getCC();
829  FI->InstanceMethod = instanceMethod;
830  FI->ChainCall = chainCall;
831  FI->CmseNSCall = info.getCmseNSCall();
832  FI->NoReturn = info.getNoReturn();
833  FI->ReturnsRetained = info.getProducesResult();
834  FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
835  FI->NoCfCheck = info.getNoCfCheck();
836  FI->Required = required;
837  FI->HasRegParm = info.getHasRegParm();
838  FI->RegParm = info.getRegParm();
839  FI->ArgStruct = nullptr;
840  FI->ArgStructAlign = 0;
841  FI->NumArgs = argTypes.size();
842  FI->HasExtParameterInfos = !paramInfos.empty();
843  FI->getArgsBuffer()[0].type = resultType;
844  FI->MaxVectorWidth = 0;
845  for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
846  FI->getArgsBuffer()[i + 1].type = argTypes[i];
847  for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
848  FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
849  return FI;
850 }
851 
852 /***/
853 
854 namespace {
855 // ABIArgInfo::Expand implementation.
856 
857 // Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
858 struct TypeExpansion {
859  enum TypeExpansionKind {
860  // Elements of constant arrays are expanded recursively.
861  TEK_ConstantArray,
862  // Record fields are expanded recursively (but if record is a union, only
863  // the field with the largest size is expanded).
864  TEK_Record,
865  // For complex types, real and imaginary parts are expanded recursively.
866  TEK_Complex,
867  // All other types are not expandable.
868  TEK_None
869  };
870 
871  const TypeExpansionKind Kind;
872 
873  TypeExpansion(TypeExpansionKind K) : Kind(K) {}
874  virtual ~TypeExpansion() {}
875 };
876 
877 struct ConstantArrayExpansion : TypeExpansion {
878  QualType EltTy;
879  uint64_t NumElts;
880 
881  ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
882  : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
883  static bool classof(const TypeExpansion *TE) {
884  return TE->Kind == TEK_ConstantArray;
885  }
886 };
887 
888 struct RecordExpansion : TypeExpansion {
890 
892 
893  RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
895  : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
896  Fields(std::move(Fields)) {}
897  static bool classof(const TypeExpansion *TE) {
898  return TE->Kind == TEK_Record;
899  }
900 };
901 
902 struct ComplexExpansion : TypeExpansion {
903  QualType EltTy;
904 
905  ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
906  static bool classof(const TypeExpansion *TE) {
907  return TE->Kind == TEK_Complex;
908  }
909 };
910 
911 struct NoExpansion : TypeExpansion {
912  NoExpansion() : TypeExpansion(TEK_None) {}
913  static bool classof(const TypeExpansion *TE) {
914  return TE->Kind == TEK_None;
915  }
916 };
917 } // namespace
918 
919 static std::unique_ptr<TypeExpansion>
920 getTypeExpansion(QualType Ty, const ASTContext &Context) {
921  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
922  return std::make_unique<ConstantArrayExpansion>(
923  AT->getElementType(), AT->getSize().getZExtValue());
924  }
925  if (const RecordType *RT = Ty->getAs<RecordType>()) {
928  const RecordDecl *RD = RT->getDecl();
929  assert(!RD->hasFlexibleArrayMember() &&
930  "Cannot expand structure with flexible array.");
931  if (RD->isUnion()) {
932  // Unions can be here only in degenerative cases - all the fields are same
933  // after flattening. Thus we have to use the "largest" field.
934  const FieldDecl *LargestFD = nullptr;
935  CharUnits UnionSize = CharUnits::Zero();
936 
937  for (const auto *FD : RD->fields()) {
938  if (FD->isZeroLengthBitField(Context))
939  continue;
940  assert(!FD->isBitField() &&
941  "Cannot expand structure with bit-field members.");
942  CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
943  if (UnionSize < FieldSize) {
944  UnionSize = FieldSize;
945  LargestFD = FD;
946  }
947  }
948  if (LargestFD)
949  Fields.push_back(LargestFD);
950  } else {
951  if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
952  assert(!CXXRD->isDynamicClass() &&
953  "cannot expand vtable pointers in dynamic classes");
954  llvm::append_range(Bases, llvm::make_pointer_range(CXXRD->bases()));
955  }
956 
957  for (const auto *FD : RD->fields()) {
958  if (FD->isZeroLengthBitField(Context))
959  continue;
960  assert(!FD->isBitField() &&
961  "Cannot expand structure with bit-field members.");
962  Fields.push_back(FD);
963  }
964  }
965  return std::make_unique<RecordExpansion>(std::move(Bases),
966  std::move(Fields));
967  }
968  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
969  return std::make_unique<ComplexExpansion>(CT->getElementType());
970  }
971  return std::make_unique<NoExpansion>();
972 }
973 
974 static int getExpansionSize(QualType Ty, const ASTContext &Context) {
975  auto Exp = getTypeExpansion(Ty, Context);
976  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
977  return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
978  }
979  if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
980  int Res = 0;
981  for (auto BS : RExp->Bases)
982  Res += getExpansionSize(BS->getType(), Context);
983  for (auto FD : RExp->Fields)
984  Res += getExpansionSize(FD->getType(), Context);
985  return Res;
986  }
987  if (isa<ComplexExpansion>(Exp.get()))
988  return 2;
989  assert(isa<NoExpansion>(Exp.get()));
990  return 1;
991 }
992 
993 void
996  auto Exp = getTypeExpansion(Ty, Context);
997  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
998  for (int i = 0, n = CAExp->NumElts; i < n; i++) {
999  getExpandedTypes(CAExp->EltTy, TI);
1000  }
1001  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1002  for (auto BS : RExp->Bases)
1003  getExpandedTypes(BS->getType(), TI);
1004  for (auto FD : RExp->Fields)
1005  getExpandedTypes(FD->getType(), TI);
1006  } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
1007  llvm::Type *EltTy = ConvertType(CExp->EltTy);
1008  *TI++ = EltTy;
1009  *TI++ = EltTy;
1010  } else {
1011  assert(isa<NoExpansion>(Exp.get()));
1012  *TI++ = ConvertType(Ty);
1013  }
1014 }
1015 
1017  ConstantArrayExpansion *CAE,
1018  Address BaseAddr,
1019  llvm::function_ref<void(Address)> Fn) {
1020  CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
1021  CharUnits EltAlign =
1022  BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
1023  llvm::Type *EltTy = CGF.ConvertTypeForMem(CAE->EltTy);
1024 
1025  for (int i = 0, n = CAE->NumElts; i < n; i++) {
1026  llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32(
1027  BaseAddr.getElementType(), BaseAddr.getPointer(), 0, i);
1028  Fn(Address(EltAddr, EltTy, EltAlign));
1029  }
1030 }
1031 
1032 void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1033  llvm::Function::arg_iterator &AI) {
1034  assert(LV.isSimple() &&
1035  "Unexpected non-simple lvalue during struct expansion.");
1036 
1037  auto Exp = getTypeExpansion(Ty, getContext());
1038  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1040  *this, CAExp, LV.getAddress(*this), [&](Address EltAddr) {
1041  LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1042  ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
1043  });
1044  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1045  Address This = LV.getAddress(*this);
1046  for (const CXXBaseSpecifier *BS : RExp->Bases) {
1047  // Perform a single step derived-to-base conversion.
1048  Address Base =
1049  GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1050  /*NullCheckValue=*/false, SourceLocation());
1051  LValue SubLV = MakeAddrLValue(Base, BS->getType());
1052 
1053  // Recurse onto bases.
1054  ExpandTypeFromArgs(BS->getType(), SubLV, AI);
1055  }
1056  for (auto FD : RExp->Fields) {
1057  // FIXME: What are the right qualifiers here?
1058  LValue SubLV = EmitLValueForFieldInitialization(LV, FD);
1059  ExpandTypeFromArgs(FD->getType(), SubLV, AI);
1060  }
1061  } else if (isa<ComplexExpansion>(Exp.get())) {
1062  auto realValue = &*AI++;
1063  auto imagValue = &*AI++;
1064  EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);
1065  } else {
1066  // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1067  // primitive store.
1068  assert(isa<NoExpansion>(Exp.get()));
1069  llvm::Value *Arg = &*AI++;
1070  if (LV.isBitField()) {
1072  } else {
1073  // TODO: currently there are some places are inconsistent in what LLVM
1074  // pointer type they use (see D118744). Once clang uses opaque pointers
1075  // all LLVM pointer types will be the same and we can remove this check.
1076  if (Arg->getType()->isPointerTy()) {
1077  Address Addr = LV.getAddress(*this);
1078  Arg = Builder.CreateBitCast(Arg, Addr.getElementType());
1079  }
1080  EmitStoreOfScalar(Arg, LV);
1081  }
1082  }
1083 }
1084 
1085 void CodeGenFunction::ExpandTypeToArgs(
1086  QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1087  SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1088  auto Exp = getTypeExpansion(Ty, getContext());
1089  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1090  Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
1093  *this, CAExp, Addr, [&](Address EltAddr) {
1094  CallArg EltArg = CallArg(
1095  convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()),
1096  CAExp->EltTy);
1097  ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,
1098  IRCallArgPos);
1099  });
1100  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1101  Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
1103  for (const CXXBaseSpecifier *BS : RExp->Bases) {
1104  // Perform a single step derived-to-base conversion.
1105  Address Base =
1106  GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1107  /*NullCheckValue=*/false, SourceLocation());
1108  CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType());
1109 
1110  // Recurse onto bases.
1111  ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs,
1112  IRCallArgPos);
1113  }
1114 
1115  LValue LV = MakeAddrLValue(This, Ty);
1116  for (auto FD : RExp->Fields) {
1117  CallArg FldArg =
1118  CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType());
1119  ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,
1120  IRCallArgPos);
1121  }
1122  } else if (isa<ComplexExpansion>(Exp.get())) {
1124  IRCallArgs[IRCallArgPos++] = CV.first;
1125  IRCallArgs[IRCallArgPos++] = CV.second;
1126  } else {
1127  assert(isa<NoExpansion>(Exp.get()));
1128  auto RV = Arg.getKnownRValue();
1129  assert(RV.isScalar() &&
1130  "Unexpected non-scalar rvalue during struct expansion.");
1131 
1132  // Insert a bitcast as needed.
1133  llvm::Value *V = RV.getScalarVal();
1134  if (IRCallArgPos < IRFuncTy->getNumParams() &&
1135  V->getType() != IRFuncTy->getParamType(IRCallArgPos))
1136  V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));
1137 
1138  IRCallArgs[IRCallArgPos++] = V;
1139  }
1140 }
1141 
1142 /// Create a temporary allocation for the purposes of coercion.
1144  CharUnits MinAlign,
1145  const Twine &Name = "tmp") {
1146  // Don't use an alignment that's worse than what LLVM would prefer.
1147  auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(Ty);
1148  CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
1149 
1150  return CGF.CreateTempAlloca(Ty, Align, Name + ".coerce");
1151 }
1152 
1153 /// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1154 /// accessing some number of bytes out of it, try to gep into the struct to get
1155 /// at its inner goodness. Dive as deep as possible without entering an element
1156 /// with an in-memory size smaller than DstSize.
1157 static Address
1159  llvm::StructType *SrcSTy,
1160  uint64_t DstSize, CodeGenFunction &CGF) {
1161  // We can't dive into a zero-element struct.
1162  if (SrcSTy->getNumElements() == 0) return SrcPtr;
1163 
1164  llvm::Type *FirstElt = SrcSTy->getElementType(0);
1165 
1166  // If the first elt is at least as large as what we're looking for, or if the
1167  // first element is the same size as the whole struct, we can enter it. The
1168  // comparison must be made on the store size and not the alloca size. Using
1169  // the alloca size may overstate the size of the load.
1170  uint64_t FirstEltSize =
1171  CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt);
1172  if (FirstEltSize < DstSize &&
1173  FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy))
1174  return SrcPtr;
1175 
1176  // GEP into the first element.
1177  SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive");
1178 
1179  // If the first element is a struct, recurse.
1180  llvm::Type *SrcTy = SrcPtr.getElementType();
1181  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
1182  return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1183 
1184  return SrcPtr;
1185 }
1186 
1187 /// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1188 /// are either integers or pointers. This does a truncation of the value if it
1189 /// is too large or a zero extension if it is too small.
1190 ///
1191 /// This behaves as if the value were coerced through memory, so on big-endian
1192 /// targets the high bits are preserved in a truncation, while little-endian
1193 /// targets preserve the low bits.
1194 static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
1195  llvm::Type *Ty,
1196  CodeGenFunction &CGF) {
1197  if (Val->getType() == Ty)
1198  return Val;
1199 
1200  if (isa<llvm::PointerType>(Val->getType())) {
1201  // If this is Pointer->Pointer avoid conversion to and from int.
1202  if (isa<llvm::PointerType>(Ty))
1203  return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
1204 
1205  // Convert the pointer to an integer so we can play with its width.
1206  Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
1207  }
1208 
1209  llvm::Type *DestIntTy = Ty;
1210  if (isa<llvm::PointerType>(DestIntTy))
1211  DestIntTy = CGF.IntPtrTy;
1212 
1213  if (Val->getType() != DestIntTy) {
1214  const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1215  if (DL.isBigEndian()) {
1216  // Preserve the high bits on big-endian targets.
1217  // That is what memory coercion does.
1218  uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
1219  uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
1220 
1221  if (SrcSize > DstSize) {
1222  Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");
1223  Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");
1224  } else {
1225  Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");
1226  Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");
1227  }
1228  } else {
1229  // Little-endian targets preserve the low bits. No shifts required.
1230  Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
1231  }
1232  }
1233 
1234  if (isa<llvm::PointerType>(Ty))
1235  Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
1236  return Val;
1237 }
1238 
1239 
1240 
1241 /// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1242 /// a pointer to an object of type \arg Ty, known to be aligned to
1243 /// \arg SrcAlign bytes.
1244 ///
1245 /// This safely handles the case when the src type is smaller than the
1246 /// destination type; in this situation the values of bits which not
1247 /// present in the src are undefined.
1248 static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
1249  CodeGenFunction &CGF) {
1250  llvm::Type *SrcTy = Src.getElementType();
1251 
1252  // If SrcTy and Ty are the same, just do a load.
1253  if (SrcTy == Ty)
1254  return CGF.Builder.CreateLoad(Src);
1255 
1256  llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1257 
1258  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
1259  Src = EnterStructPointerForCoercedAccess(Src, SrcSTy,
1260  DstSize.getFixedSize(), CGF);
1261  SrcTy = Src.getElementType();
1262  }
1263 
1264  llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1265 
1266  // If the source and destination are integer or pointer types, just do an
1267  // extension or truncation to the desired type.
1268  if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
1269  (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
1270  llvm::Value *Load = CGF.Builder.CreateLoad(Src);
1271  return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
1272  }
1273 
1274  // If load is legal, just bitcast the src pointer.
1275  if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1276  SrcSize.getFixedSize() >= DstSize.getFixedSize()) {
1277  // Generally SrcSize is never greater than DstSize, since this means we are
1278  // losing bits. However, this can happen in cases where the structure has
1279  // additional padding, for example due to a user specified alignment.
1280  //
1281  // FIXME: Assert that we aren't truncating non-padding bits when have access
1282  // to that information.
1283  Src = CGF.Builder.CreateElementBitCast(Src, Ty);
1284  return CGF.Builder.CreateLoad(Src);
1285  }
1286 
1287  // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1288  // the types match, use the llvm.experimental.vector.insert intrinsic to
1289  // perform the conversion.
1290  if (auto *ScalableDst = dyn_cast<llvm::ScalableVectorType>(Ty)) {
1291  if (auto *FixedSrc = dyn_cast<llvm::FixedVectorType>(SrcTy)) {
1292  // If we are casting a fixed i8 vector to a scalable 16 x i1 predicate
1293  // vector, use a vector insert and bitcast the result.
1294  bool NeedsBitcast = false;
1295  auto PredType =
1296  llvm::ScalableVectorType::get(CGF.Builder.getInt1Ty(), 16);
1297  llvm::Type *OrigType = Ty;
1298  if (ScalableDst == PredType &&
1299  FixedSrc->getElementType() == CGF.Builder.getInt8Ty()) {
1300  ScalableDst = llvm::ScalableVectorType::get(CGF.Builder.getInt8Ty(), 2);
1301  NeedsBitcast = true;
1302  }
1303  if (ScalableDst->getElementType() == FixedSrc->getElementType()) {
1304  auto *Load = CGF.Builder.CreateLoad(Src);
1305  auto *UndefVec = llvm::UndefValue::get(ScalableDst);
1306  auto *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty);
1307  llvm::Value *Result = CGF.Builder.CreateInsertVector(
1308  ScalableDst, UndefVec, Load, Zero, "castScalableSve");
1309  if (NeedsBitcast)
1310  Result = CGF.Builder.CreateBitCast(Result, OrigType);
1311  return Result;
1312  }
1313  }
1314  }
1315 
1316  // Otherwise do coercion through memory. This is stupid, but simple.
1317  Address Tmp =
1318  CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName());
1319  CGF.Builder.CreateMemCpy(
1320  Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(),
1321  Src.getAlignment().getAsAlign(),
1322  llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinSize()));
1323  return CGF.Builder.CreateLoad(Tmp);
1324 }
1325 
1326 // Function to store a first-class aggregate into memory. We prefer to
1327 // store the elements rather than the aggregate to be more friendly to
1328 // fast-isel.
1329 // FIXME: Do we need to recurse here?
1330 void CodeGenFunction::EmitAggregateStore(llvm::Value *Val, Address Dest,
1331  bool DestIsVolatile) {
1332  // Prefer scalar stores to first-class aggregate stores.
1333  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) {
1334  for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1335  Address EltPtr = Builder.CreateStructGEP(Dest, i);
1336  llvm::Value *Elt = Builder.CreateExtractValue(Val, i);
1337  Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
1338  }
1339  } else {
1340  Builder.CreateStore(Val, Dest, DestIsVolatile);
1341  }
1342 }
1343 
1344 /// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
1345 /// where the source and destination may have different types. The
1346 /// destination is known to be aligned to \arg DstAlign bytes.
1347 ///
1348 /// This safely handles the case when the src type is larger than the
1349 /// destination type; the upper bits of the src will be lost.
1350 static void CreateCoercedStore(llvm::Value *Src,
1351  Address Dst,
1352  bool DstIsVolatile,
1353  CodeGenFunction &CGF) {
1354  llvm::Type *SrcTy = Src->getType();
1355  llvm::Type *DstTy = Dst.getElementType();
1356  if (SrcTy == DstTy) {
1357  CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1358  return;
1359  }
1360 
1361  llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1362 
1363  if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
1364  Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy,
1365  SrcSize.getFixedSize(), CGF);
1366  DstTy = Dst.getElementType();
1367  }
1368 
1369  llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy);
1370  llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy);
1371  if (SrcPtrTy && DstPtrTy &&
1372  SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()) {
1373  Src = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Src, DstTy);
1374  CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1375  return;
1376  }
1377 
1378  // If the source and destination are integer or pointer types, just do an
1379  // extension or truncation to the desired type.
1380  if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
1381  (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
1382  Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
1383  CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1384  return;
1385  }
1386 
1387  llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
1388 
1389  // If store is legal, just bitcast the src pointer.
1390  if (isa<llvm::ScalableVectorType>(SrcTy) ||
1391  isa<llvm::ScalableVectorType>(DstTy) ||
1392  SrcSize.getFixedSize() <= DstSize.getFixedSize()) {
1393  Dst = CGF.Builder.CreateElementBitCast(Dst, SrcTy);
1394  CGF.EmitAggregateStore(Src, Dst, DstIsVolatile);
1395  } else {
1396  // Otherwise do coercion through memory. This is stupid, but
1397  // simple.
1398 
1399  // Generally SrcSize is never greater than DstSize, since this means we are
1400  // losing bits. However, this can happen in cases where the structure has
1401  // additional padding, for example due to a user specified alignment.
1402  //
1403  // FIXME: Assert that we aren't truncating non-padding bits when have access
1404  // to that information.
1405  Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
1406  CGF.Builder.CreateStore(Src, Tmp);
1407  CGF.Builder.CreateMemCpy(
1408  Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(),
1409  Tmp.getAlignment().getAsAlign(),
1410  llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedSize()));
1411  }
1412 }
1413 
1415  const ABIArgInfo &info) {
1416  if (unsigned offset = info.getDirectOffset()) {
1417  addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8Ty);
1418  addr = CGF.Builder.CreateConstInBoundsByteGEP(addr,
1419  CharUnits::fromQuantity(offset));
1420  addr = CGF.Builder.CreateElementBitCast(addr, info.getCoerceToType());
1421  }
1422  return addr;
1423 }
1424 
1425 namespace {
1426 
1427 /// Encapsulates information about the way function arguments from
1428 /// CGFunctionInfo should be passed to actual LLVM IR function.
1429 class ClangToLLVMArgMapping {
1430  static const unsigned InvalidIndex = ~0U;
1431  unsigned InallocaArgNo;
1432  unsigned SRetArgNo;
1433  unsigned TotalIRArgs;
1434 
1435  /// Arguments of LLVM IR function corresponding to single Clang argument.
1436  struct IRArgs {
1437  unsigned PaddingArgIndex;
1438  // Argument is expanded to IR arguments at positions
1439  // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1440  unsigned FirstArgIndex;
1441  unsigned NumberOfArgs;
1442 
1443  IRArgs()
1444  : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1445  NumberOfArgs(0) {}
1446  };
1447 
1448  SmallVector<IRArgs, 8> ArgInfo;
1449 
1450 public:
1451  ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1452  bool OnlyRequiredArgs = false)
1453  : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1454  ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1455  construct(Context, FI, OnlyRequiredArgs);
1456  }
1457 
1458  bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1459  unsigned getInallocaArgNo() const {
1460  assert(hasInallocaArg());
1461  return InallocaArgNo;
1462  }
1463 
1464  bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1465  unsigned getSRetArgNo() const {
1466  assert(hasSRetArg());
1467  return SRetArgNo;
1468  }
1469 
1470  unsigned totalIRArgs() const { return TotalIRArgs; }
1471 
1472  bool hasPaddingArg(unsigned ArgNo) const {
1473  assert(ArgNo < ArgInfo.size());
1474  return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1475  }
1476  unsigned getPaddingArgNo(unsigned ArgNo) const {
1477  assert(hasPaddingArg(ArgNo));
1478  return ArgInfo[ArgNo].PaddingArgIndex;
1479  }
1480 
1481  /// Returns index of first IR argument corresponding to ArgNo, and their
1482  /// quantity.
1483  std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1484  assert(ArgNo < ArgInfo.size());
1485  return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1486  ArgInfo[ArgNo].NumberOfArgs);
1487  }
1488 
1489 private:
1490  void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1491  bool OnlyRequiredArgs);
1492 };
1493 
1494 void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1495  const CGFunctionInfo &FI,
1496  bool OnlyRequiredArgs) {
1497  unsigned IRArgNo = 0;
1498  bool SwapThisWithSRet = false;
1499  const ABIArgInfo &RetAI = FI.getReturnInfo();
1500 
1501  if (RetAI.getKind() == ABIArgInfo::Indirect) {
1502  SwapThisWithSRet = RetAI.isSRetAfterThis();
1503  SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1504  }
1505 
1506  unsigned ArgNo = 0;
1507  unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1508  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1509  ++I, ++ArgNo) {
1510  assert(I != FI.arg_end());
1511  QualType ArgType = I->type;
1512  const ABIArgInfo &AI = I->info;
1513  // Collect data about IR arguments corresponding to Clang argument ArgNo.
1514  auto &IRArgs = ArgInfo[ArgNo];
1515 
1516  if (AI.getPaddingType())
1517  IRArgs.PaddingArgIndex = IRArgNo++;
1518 
1519  switch (AI.getKind()) {
1520  case ABIArgInfo::Extend:
1521  case ABIArgInfo::Direct: {
1522  // FIXME: handle sseregparm someday...
1523  llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());
1524  if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1525  IRArgs.NumberOfArgs = STy->getNumElements();
1526  } else {
1527  IRArgs.NumberOfArgs = 1;
1528  }
1529  break;
1530  }
1531  case ABIArgInfo::Indirect:
1533  IRArgs.NumberOfArgs = 1;
1534  break;
1535  case ABIArgInfo::Ignore:
1536  case ABIArgInfo::InAlloca:
1537  // ignore and inalloca doesn't have matching LLVM parameters.
1538  IRArgs.NumberOfArgs = 0;
1539  break;
1541  IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1542  break;
1543  case ABIArgInfo::Expand:
1544  IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context);
1545  break;
1546  }
1547 
1548  if (IRArgs.NumberOfArgs > 0) {
1549  IRArgs.FirstArgIndex = IRArgNo;
1550  IRArgNo += IRArgs.NumberOfArgs;
1551  }
1552 
1553  // Skip over the sret parameter when it comes second. We already handled it
1554  // above.
1555  if (IRArgNo == 1 && SwapThisWithSRet)
1556  IRArgNo++;
1557  }
1558  assert(ArgNo == ArgInfo.size());
1559 
1560  if (FI.usesInAlloca())
1561  InallocaArgNo = IRArgNo++;
1562 
1563  TotalIRArgs = IRArgNo;
1564 }
1565 } // namespace
1566 
1567 /***/
1568 
1570  const auto &RI = FI.getReturnInfo();
1571  return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
1572 }
1573 
1575  return ReturnTypeUsesSRet(FI) &&
1577 }
1578 
1580  if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1581  switch (BT->getKind()) {
1582  default:
1583  return false;
1584  case BuiltinType::Float:
1586  case BuiltinType::Double:
1588  case BuiltinType::LongDouble:
1590  }
1591  }
1592 
1593  return false;
1594 }
1595 
1597  if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1598  if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1599  if (BT->getKind() == BuiltinType::LongDouble)
1601  }
1602  }
1603 
1604  return false;
1605 }
1606 
1608  const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1609  return GetFunctionType(FI);
1610 }
1611 
1612 llvm::FunctionType *
1614 
1615  bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1616  (void)Inserted;
1617  assert(Inserted && "Recursively being processed?");
1618 
1619  llvm::Type *resultType = nullptr;
1620  const ABIArgInfo &retAI = FI.getReturnInfo();
1621  switch (retAI.getKind()) {
1622  case ABIArgInfo::Expand:
1624  llvm_unreachable("Invalid ABI kind for return argument");
1625 
1626  case ABIArgInfo::Extend:
1627  case ABIArgInfo::Direct:
1628  resultType = retAI.getCoerceToType();
1629  break;
1630 
1631  case ABIArgInfo::InAlloca:
1632  if (retAI.getInAllocaSRet()) {
1633  // sret things on win32 aren't void, they return the sret pointer.
1634  QualType ret = FI.getReturnType();
1635  llvm::Type *ty = ConvertType(ret);
1636  unsigned addressSpace = Context.getTargetAddressSpace(ret);
1637  resultType = llvm::PointerType::get(ty, addressSpace);
1638  } else {
1639  resultType = llvm::Type::getVoidTy(getLLVMContext());
1640  }
1641  break;
1642 
1643  case ABIArgInfo::Indirect:
1644  case ABIArgInfo::Ignore:
1645  resultType = llvm::Type::getVoidTy(getLLVMContext());
1646  break;
1647 
1649  resultType = retAI.getUnpaddedCoerceAndExpandType();
1650  break;
1651  }
1652 
1653  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1654  SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1655 
1656  // Add type for sret argument.
1657  if (IRFunctionArgs.hasSRetArg()) {
1658  QualType Ret = FI.getReturnType();
1659  llvm::Type *Ty = ConvertType(Ret);
1660  unsigned AddressSpace = Context.getTargetAddressSpace(Ret);
1661  ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1662  llvm::PointerType::get(Ty, AddressSpace);
1663  }
1664 
1665  // Add type for inalloca argument.
1666  if (IRFunctionArgs.hasInallocaArg()) {
1667  auto ArgStruct = FI.getArgStruct();
1668  assert(ArgStruct);
1669  ArgTypes[IRFunctionArgs.getInallocaArgNo()] = ArgStruct->getPointerTo();
1670  }
1671 
1672  // Add in all of the required arguments.
1673  unsigned ArgNo = 0;
1675  ie = it + FI.getNumRequiredArgs();
1676  for (; it != ie; ++it, ++ArgNo) {
1677  const ABIArgInfo &ArgInfo = it->info;
1678 
1679  // Insert a padding type to ensure proper alignment.
1680  if (IRFunctionArgs.hasPaddingArg(ArgNo))
1681  ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1682  ArgInfo.getPaddingType();
1683 
1684  unsigned FirstIRArg, NumIRArgs;
1685  std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1686 
1687  switch (ArgInfo.getKind()) {
1688  case ABIArgInfo::Ignore:
1689  case ABIArgInfo::InAlloca:
1690  assert(NumIRArgs == 0);
1691  break;
1692 
1693  case ABIArgInfo::Indirect: {
1694  assert(NumIRArgs == 1);
1695  // indirect arguments are always on the stack, which is alloca addr space.
1696  llvm::Type *LTy = ConvertTypeForMem(it->type);
1697  ArgTypes[FirstIRArg] = LTy->getPointerTo(
1698  CGM.getDataLayout().getAllocaAddrSpace());
1699  break;
1700  }
1702  assert(NumIRArgs == 1);
1703  llvm::Type *LTy = ConvertTypeForMem(it->type);
1704  ArgTypes[FirstIRArg] = LTy->getPointerTo(ArgInfo.getIndirectAddrSpace());
1705  break;
1706  }
1707  case ABIArgInfo::Extend:
1708  case ABIArgInfo::Direct: {
1709  // Fast-isel and the optimizer generally like scalar values better than
1710  // FCAs, so we flatten them if this is safe to do for this argument.
1711  llvm::Type *argType = ArgInfo.getCoerceToType();
1712  llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
1713  if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
1714  assert(NumIRArgs == st->getNumElements());
1715  for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
1716  ArgTypes[FirstIRArg + i] = st->getElementType(i);
1717  } else {
1718  assert(NumIRArgs == 1);
1719  ArgTypes[FirstIRArg] = argType;
1720  }
1721  break;
1722  }
1723 
1725  auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1726  for (auto EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1727  *ArgTypesIter++ = EltTy;
1728  }
1729  assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1730  break;
1731  }
1732 
1733  case ABIArgInfo::Expand:
1734  auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1735  getExpandedTypes(it->type, ArgTypesIter);
1736  assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1737  break;
1738  }
1739  }
1740 
1741  bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1742  assert(Erased && "Not in set?");
1743 
1744  return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic());
1745 }
1746 
1748  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
1749  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
1750 
1751  if (!isFuncTypeConvertible(FPT))
1752  return llvm::StructType::get(getLLVMContext());
1753 
1754  return GetFunctionType(GD);
1755 }
1756 
1758  llvm::AttrBuilder &FuncAttrs,
1759  const FunctionProtoType *FPT) {
1760  if (!FPT)
1761  return;
1762 
1764  FPT->isNothrow())
1765  FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1766 }
1767 
1768 static void AddAttributesFromAssumes(llvm::AttrBuilder &FuncAttrs,
1769  const Decl *Callee) {
1770  if (!Callee)
1771  return;
1772 
1774 
1775  for (const AssumptionAttr *AA : Callee->specific_attrs<AssumptionAttr>())
1776  AA->getAssumption().split(Attrs, ",");
1777 
1778  if (!Attrs.empty())
1779  FuncAttrs.addAttribute(llvm::AssumptionAttrKey,
1780  llvm::join(Attrs.begin(), Attrs.end(), ","));
1781 }
1782 
1784  QualType ReturnType) {
1785  // We can't just discard the return value for a record type with a
1786  // complex destructor or a non-trivially copyable type.
1787  if (const RecordType *RT =
1788  ReturnType.getCanonicalType()->getAs<RecordType>()) {
1789  if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1790  return ClassDecl->hasTrivialDestructor();
1791  }
1792  return ReturnType.isTriviallyCopyableType(Context);
1793 }
1794 
1795 void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
1796  bool HasOptnone,
1797  bool AttrOnCallSite,
1798  llvm::AttrBuilder &FuncAttrs) {
1799  // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1800  if (!HasOptnone) {
1801  if (CodeGenOpts.OptimizeSize)
1802  FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1803  if (CodeGenOpts.OptimizeSize == 2)
1804  FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1805  }
1806 
1807  if (CodeGenOpts.DisableRedZone)
1808  FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1809  if (CodeGenOpts.IndirectTlsSegRefs)
1810  FuncAttrs.addAttribute("indirect-tls-seg-refs");
1811  if (CodeGenOpts.NoImplicitFloat)
1812  FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1813 
1814  if (AttrOnCallSite) {
1815  // Attributes that should go on the call site only.
1816  // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
1817  // the -fno-builtin-foo list.
1818  if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))
1819  FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1820  if (!CodeGenOpts.TrapFuncName.empty())
1821  FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName);
1822  } else {
1823  StringRef FpKind;
1824  switch (CodeGenOpts.getFramePointer()) {
1826  FpKind = "none";
1827  break;
1829  FpKind = "non-leaf";
1830  break;
1832  FpKind = "all";
1833  break;
1834  }
1835  FuncAttrs.addAttribute("frame-pointer", FpKind);
1836 
1837  if (CodeGenOpts.LessPreciseFPMAD)
1838  FuncAttrs.addAttribute("less-precise-fpmad", "true");
1839 
1840  if (CodeGenOpts.NullPointerIsValid)
1841  FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
1842 
1843  if (CodeGenOpts.FPDenormalMode != llvm::DenormalMode::getIEEE())
1844  FuncAttrs.addAttribute("denormal-fp-math",
1845  CodeGenOpts.FPDenormalMode.str());
1846  if (CodeGenOpts.FP32DenormalMode != CodeGenOpts.FPDenormalMode) {
1847  FuncAttrs.addAttribute(
1848  "denormal-fp-math-f32",
1849  CodeGenOpts.FP32DenormalMode.str());
1850  }
1851 
1852  if (LangOpts.getFPExceptionMode() == LangOptions::FPE_Ignore)
1853  FuncAttrs.addAttribute("no-trapping-math", "true");
1854 
1855  // TODO: Are these all needed?
1856  // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
1857  if (LangOpts.NoHonorInfs)
1858  FuncAttrs.addAttribute("no-infs-fp-math", "true");
1859  if (LangOpts.NoHonorNaNs)
1860  FuncAttrs.addAttribute("no-nans-fp-math", "true");
1861  if (LangOpts.ApproxFunc)
1862  FuncAttrs.addAttribute("approx-func-fp-math", "true");
1863  if (LangOpts.UnsafeFPMath)
1864  FuncAttrs.addAttribute("unsafe-fp-math", "true");
1865  if (CodeGenOpts.SoftFloat)
1866  FuncAttrs.addAttribute("use-soft-float", "true");
1867  FuncAttrs.addAttribute("stack-protector-buffer-size",
1868  llvm::utostr(CodeGenOpts.SSPBufferSize));
1869  if (LangOpts.NoSignedZero)
1870  FuncAttrs.addAttribute("no-signed-zeros-fp-math", "true");
1871 
1872  // TODO: Reciprocal estimate codegen options should apply to instructions?
1873  const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
1874  if (!Recips.empty())
1875  FuncAttrs.addAttribute("reciprocal-estimates",
1876  llvm::join(Recips, ","));
1877 
1878  if (!CodeGenOpts.PreferVectorWidth.empty() &&
1879  CodeGenOpts.PreferVectorWidth != "none")
1880  FuncAttrs.addAttribute("prefer-vector-width",
1881  CodeGenOpts.PreferVectorWidth);
1882 
1883  if (CodeGenOpts.StackRealignment)
1884  FuncAttrs.addAttribute("stackrealign");
1885  if (CodeGenOpts.Backchain)
1886  FuncAttrs.addAttribute("backchain");
1887  if (CodeGenOpts.EnableSegmentedStacks)
1888  FuncAttrs.addAttribute("split-stack");
1889 
1890  if (CodeGenOpts.SpeculativeLoadHardening)
1891  FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1892 
1893  // Add zero-call-used-regs attribute.
1894  switch (CodeGenOpts.getZeroCallUsedRegs()) {
1895  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
1896  FuncAttrs.removeAttribute("zero-call-used-regs");
1897  break;
1898  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
1899  FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr-arg");
1900  break;
1901  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
1902  FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr");
1903  break;
1904  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
1905  FuncAttrs.addAttribute("zero-call-used-regs", "used-arg");
1906  break;
1908  FuncAttrs.addAttribute("zero-call-used-regs", "used");
1909  break;
1910  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
1911  FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr-arg");
1912  break;
1913  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
1914  FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr");
1915  break;
1916  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
1917  FuncAttrs.addAttribute("zero-call-used-regs", "all-arg");
1918  break;
1920  FuncAttrs.addAttribute("zero-call-used-regs", "all");
1921  break;
1922  }
1923  }
1924 
1925  if (getLangOpts().assumeFunctionsAreConvergent()) {
1926  // Conservatively, mark all functions and calls in CUDA and OpenCL as
1927  // convergent (meaning, they may call an intrinsically convergent op, such
1928  // as __syncthreads() / barrier(), and so can't have certain optimizations
1929  // applied around them). LLVM will remove this attribute where it safely
1930  // can.
1931  FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1932  }
1933 
1934  if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice) {
1935  // Exceptions aren't supported in CUDA device code.
1936  FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1937  }
1938 
1939  for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
1940  StringRef Var, Value;
1941  std::tie(Var, Value) = Attr.split('=');
1942  FuncAttrs.addAttribute(Var, Value);
1943  }
1944 }
1945 
1947  llvm::AttrBuilder FuncAttrs(F.getContext());
1948  getDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
1949  /* AttrOnCallSite = */ false, FuncAttrs);
1950  // TODO: call GetCPUAndFeaturesAttributes?
1951  F.addFnAttrs(FuncAttrs);
1952 }
1953 
1955  llvm::AttrBuilder &attrs) {
1956  getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false,
1957  /*for call*/ false, attrs);
1958  GetCPUAndFeaturesAttributes(GlobalDecl(), attrs);
1959 }
1960 
1961 static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
1962  const LangOptions &LangOpts,
1963  const NoBuiltinAttr *NBA = nullptr) {
1964  auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
1965  SmallString<32> AttributeName;
1966  AttributeName += "no-builtin-";
1967  AttributeName += BuiltinName;
1968  FuncAttrs.addAttribute(AttributeName);
1969  };
1970 
1971  // First, handle the language options passed through -fno-builtin.
1972  if (LangOpts.NoBuiltin) {
1973  // -fno-builtin disables them all.
1974  FuncAttrs.addAttribute("no-builtins");
1975  return;
1976  }
1977 
1978  // Then, add attributes for builtins specified through -fno-builtin-<name>.
1979  llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr);
1980 
1981  // Now, let's check the __attribute__((no_builtin("...")) attribute added to
1982  // the source.
1983  if (!NBA)
1984  return;
1985 
1986  // If there is a wildcard in the builtin names specified through the
1987  // attribute, disable them all.
1988  if (llvm::is_contained(NBA->builtinNames(), "*")) {
1989  FuncAttrs.addAttribute("no-builtins");
1990  return;
1991  }
1992 
1993  // And last, add the rest of the builtin names.
1994  llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
1995 }
1996 
1997 static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
1998  const llvm::DataLayout &DL, const ABIArgInfo &AI,
1999  bool CheckCoerce = true) {
2000  llvm::Type *Ty = Types.ConvertTypeForMem(QTy);
2001  if (AI.getKind() == ABIArgInfo::Indirect)
2002  return true;
2003  if (AI.getKind() == ABIArgInfo::Extend)
2004  return true;
2005  if (!DL.typeSizeEqualsStoreSize(Ty))
2006  // TODO: This will result in a modest amount of values not marked noundef
2007  // when they could be. We care about values that *invisibly* contain undef
2008  // bits from the perspective of LLVM IR.
2009  return false;
2010  if (CheckCoerce && AI.canHaveCoerceToType()) {
2011  llvm::Type *CoerceTy = AI.getCoerceToType();
2012  if (llvm::TypeSize::isKnownGT(DL.getTypeSizeInBits(CoerceTy),
2013  DL.getTypeSizeInBits(Ty)))
2014  // If we're coercing to a type with a greater size than the canonical one,
2015  // we're introducing new undef bits.
2016  // Coercing to a type of smaller or equal size is ok, as we know that
2017  // there's no internal padding (typeSizeEqualsStoreSize).
2018  return false;
2019  }
2020  if (QTy->isBitIntType())
2021  return true;
2022  if (QTy->isReferenceType())
2023  return true;
2024  if (QTy->isNullPtrType())
2025  return false;
2026  if (QTy->isMemberPointerType())
2027  // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2028  // now, never mark them.
2029  return false;
2030  if (QTy->isScalarType()) {
2031  if (const ComplexType *Complex = dyn_cast<ComplexType>(QTy))
2032  return DetermineNoUndef(Complex->getElementType(), Types, DL, AI, false);
2033  return true;
2034  }
2035  if (const VectorType *Vector = dyn_cast<VectorType>(QTy))
2036  return DetermineNoUndef(Vector->getElementType(), Types, DL, AI, false);
2037  if (const MatrixType *Matrix = dyn_cast<MatrixType>(QTy))
2038  return DetermineNoUndef(Matrix->getElementType(), Types, DL, AI, false);
2039  if (const ArrayType *Array = dyn_cast<ArrayType>(QTy))
2040  return DetermineNoUndef(Array->getElementType(), Types, DL, AI, false);
2041 
2042  // TODO: Some structs may be `noundef`, in specific situations.
2043  return false;
2044 }
2045 
2046 /// Construct the IR attribute list of a function or call.
2047 ///
2048 /// When adding an attribute, please consider where it should be handled:
2049 ///
2050 /// - getDefaultFunctionAttributes is for attributes that are essentially
2051 /// part of the global target configuration (but perhaps can be
2052 /// overridden on a per-function basis). Adding attributes there
2053 /// will cause them to also be set in frontends that build on Clang's
2054 /// target-configuration logic, as well as for code defined in library
2055 /// modules such as CUDA's libdevice.
2056 ///
2057 /// - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2058 /// and adds declaration-specific, convention-specific, and
2059 /// frontend-specific logic. The last is of particular importance:
2060 /// attributes that restrict how the frontend generates code must be
2061 /// added here rather than getDefaultFunctionAttributes.
2062 ///
2064  const CGFunctionInfo &FI,
2065  CGCalleeInfo CalleeInfo,
2066  llvm::AttributeList &AttrList,
2067  unsigned &CallingConv,
2068  bool AttrOnCallSite, bool IsThunk) {
2069  llvm::AttrBuilder FuncAttrs(getLLVMContext());
2070  llvm::AttrBuilder RetAttrs(getLLVMContext());
2071 
2072  // Collect function IR attributes from the CC lowering.
2073  // We'll collect the paramete and result attributes later.
2075  if (FI.isNoReturn())
2076  FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2077  if (FI.isCmseNSCall())
2078  FuncAttrs.addAttribute("cmse_nonsecure_call");
2079 
2080  // Collect function IR attributes from the callee prototype if we have one.
2082  CalleeInfo.getCalleeFunctionProtoType());
2083 
2084  const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2085 
2086  // Attach assumption attributes to the declaration. If this is a call
2087  // site, attach assumptions from the caller to the call as well.
2088  AddAttributesFromAssumes(FuncAttrs, TargetDecl);
2089 
2090  bool HasOptnone = false;
2091  // The NoBuiltinAttr attached to the target FunctionDecl.
2092  const NoBuiltinAttr *NBA = nullptr;
2093 
2094  // Collect function IR attributes based on declaration-specific
2095  // information.
2096  // FIXME: handle sseregparm someday...
2097  if (TargetDecl) {
2098  if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2099  FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
2100  if (TargetDecl->hasAttr<NoThrowAttr>())
2101  FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2102  if (TargetDecl->hasAttr<NoReturnAttr>())
2103  FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2104  if (TargetDecl->hasAttr<ColdAttr>())
2105  FuncAttrs.addAttribute(llvm::Attribute::Cold);
2106  if (TargetDecl->hasAttr<HotAttr>())
2107  FuncAttrs.addAttribute(llvm::Attribute::Hot);
2108  if (TargetDecl->hasAttr<NoDuplicateAttr>())
2109  FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
2110  if (TargetDecl->hasAttr<ConvergentAttr>())
2111  FuncAttrs.addAttribute(llvm::Attribute::Convergent);
2112 
2113  if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2115  getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
2116  if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2117  // A sane operator new returns a non-aliasing pointer.
2118  auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2119  if (getCodeGenOpts().AssumeSaneOperatorNew &&
2120  (Kind == OO_New || Kind == OO_Array_New))
2121  RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2122  }
2123  const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
2124  const bool IsVirtualCall = MD && MD->isVirtual();
2125  // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2126  // virtual function. These attributes are not inherited by overloads.
2127  if (!(AttrOnCallSite && IsVirtualCall)) {
2128  if (Fn->isNoReturn())
2129  FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2130  NBA = Fn->getAttr<NoBuiltinAttr>();
2131  }
2132  // Only place nomerge attribute on call sites, never functions. This
2133  // allows it to work on indirect virtual function calls.
2134  if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>())
2135  FuncAttrs.addAttribute(llvm::Attribute::NoMerge);
2136  }
2137 
2138  // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2139  if (TargetDecl->hasAttr<ConstAttr>()) {
2140  FuncAttrs.addAttribute(llvm::Attribute::ReadNone);
2141  FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2142  // gcc specifies that 'const' functions have greater restrictions than
2143  // 'pure' functions, so they also cannot have infinite loops.
2144  FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2145  } else if (TargetDecl->hasAttr<PureAttr>()) {
2146  FuncAttrs.addAttribute(llvm::Attribute::ReadOnly);
2147  FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2148  // gcc specifies that 'pure' functions cannot have infinite loops.
2149  FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2150  } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2151  FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly);
2152  FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2153  }
2154  if (TargetDecl->hasAttr<RestrictAttr>())
2155  RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2156  if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2157  !CodeGenOpts.NullPointerIsValid)
2158  RetAttrs.addAttribute(llvm::Attribute::NonNull);
2159  if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2160  FuncAttrs.addAttribute("no_caller_saved_registers");
2161  if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2162  FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
2163  if (TargetDecl->hasAttr<LeafAttr>())
2164  FuncAttrs.addAttribute(llvm::Attribute::NoCallback);
2165 
2166  HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2167  if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2168  Optional<unsigned> NumElemsParam;
2169  if (AllocSize->getNumElemsParam().isValid())
2170  NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2171  FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
2172  NumElemsParam);
2173  }
2174 
2175  if (TargetDecl->hasAttr<OpenCLKernelAttr>()) {
2176  if (getLangOpts().OpenCLVersion <= 120) {
2177  // OpenCL v1.2 Work groups are always uniform
2178  FuncAttrs.addAttribute("uniform-work-group-size", "true");
2179  } else {
2180  // OpenCL v2.0 Work groups may be whether uniform or not.
2181  // '-cl-uniform-work-group-size' compile option gets a hint
2182  // to the compiler that the global work-size be a multiple of
2183  // the work-group size specified to clEnqueueNDRangeKernel
2184  // (i.e. work groups are uniform).
2185  FuncAttrs.addAttribute("uniform-work-group-size",
2186  llvm::toStringRef(CodeGenOpts.UniformWGSize));
2187  }
2188  }
2189  }
2190 
2191  // Attach "no-builtins" attributes to:
2192  // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2193  // * definitions: "no-builtins" or "no-builtin-<name>" only.
2194  // The attributes can come from:
2195  // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2196  // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2197  addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
2198 
2199  // Collect function IR attributes based on global settiings.
2200  getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2201 
2202  // Override some default IR attributes based on declaration-specific
2203  // information.
2204  if (TargetDecl) {
2205  if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2206  FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
2207  if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2208  FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2209  if (TargetDecl->hasAttr<NoSplitStackAttr>())
2210  FuncAttrs.removeAttribute("split-stack");
2211  if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2212  // A function "__attribute__((...))" overrides the command-line flag.
2213  auto Kind =
2214  TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2215  FuncAttrs.removeAttribute("zero-call-used-regs");
2216  FuncAttrs.addAttribute(
2217  "zero-call-used-regs",
2218  ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Kind));
2219  }
2220 
2221  // Add NonLazyBind attribute to function declarations when -fno-plt
2222  // is used.
2223  // FIXME: what if we just haven't processed the function definition
2224  // yet, or if it's an external definition like C99 inline?
2225  if (CodeGenOpts.NoPLT) {
2226  if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2227  if (!Fn->isDefined() && !AttrOnCallSite) {
2228  FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
2229  }
2230  }
2231  }
2232  }
2233 
2234  // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2235  // functions with -funique-internal-linkage-names.
2236  if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2237  if (isa<FunctionDecl>(TargetDecl)) {
2238  if (this->getFunctionLinkage(CalleeInfo.getCalleeDecl()) ==
2240  FuncAttrs.addAttribute("sample-profile-suffix-elision-policy",
2241  "selected");
2242  }
2243  }
2244 
2245  // Collect non-call-site function IR attributes from declaration-specific
2246  // information.
2247  if (!AttrOnCallSite) {
2248  if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
2249  FuncAttrs.addAttribute("cmse_nonsecure_entry");
2250 
2251  // Whether tail calls are enabled.
2252  auto shouldDisableTailCalls = [&] {
2253  // Should this be honored in getDefaultFunctionAttributes?
2254  if (CodeGenOpts.DisableTailCalls)
2255  return true;
2256 
2257  if (!TargetDecl)
2258  return false;
2259 
2260  if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2261  TargetDecl->hasAttr<AnyX86InterruptAttr>())
2262  return true;
2263 
2264  if (CodeGenOpts.NoEscapingBlockTailCalls) {
2265  if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
2266  if (!BD->doesNotEscape())
2267  return true;
2268  }
2269 
2270  return false;
2271  };
2272  if (shouldDisableTailCalls())
2273  FuncAttrs.addAttribute("disable-tail-calls", "true");
2274 
2275  // CPU/feature overrides. addDefaultFunctionDefinitionAttributes
2276  // handles these separately to set them based on the global defaults.
2277  GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);
2278  }
2279 
2280  // Collect attributes from arguments and return values.
2281  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2282 
2283  QualType RetTy = FI.getReturnType();
2284  const ABIArgInfo &RetAI = FI.getReturnInfo();
2285  const llvm::DataLayout &DL = getDataLayout();
2286 
2287  // C++ explicitly makes returning undefined values UB. C's rule only applies
2288  // to used values, so we never mark them noundef for now.
2289  bool HasStrictReturn = getLangOpts().CPlusPlus;
2290  if (TargetDecl && HasStrictReturn) {
2291  if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(TargetDecl))
2292  HasStrictReturn &= !FDecl->isExternC();
2293  else if (const VarDecl *VDecl = dyn_cast<VarDecl>(TargetDecl))
2294  // Function pointer
2295  HasStrictReturn &= !VDecl->isExternC();
2296  }
2297 
2298  // We don't want to be too aggressive with the return checking, unless
2299  // it's explicit in the code opts or we're using an appropriate sanitizer.
2300  // Try to respect what the programmer intended.
2301  HasStrictReturn &= getCodeGenOpts().StrictReturn ||
2302  !MayDropFunctionReturn(getContext(), RetTy) ||
2303  getLangOpts().Sanitize.has(SanitizerKind::Memory) ||
2304  getLangOpts().Sanitize.has(SanitizerKind::Return);
2305 
2306  // Determine if the return type could be partially undef
2307  if (CodeGenOpts.EnableNoundefAttrs && HasStrictReturn) {
2308  if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&
2309  DetermineNoUndef(RetTy, getTypes(), DL, RetAI))
2310  RetAttrs.addAttribute(llvm::Attribute::NoUndef);
2311  }
2312 
2313  switch (RetAI.getKind()) {
2314  case ABIArgInfo::Extend:
2315  if (RetAI.isSignExt())
2316  RetAttrs.addAttribute(llvm::Attribute::SExt);
2317  else
2318  RetAttrs.addAttribute(llvm::Attribute::ZExt);
2319  LLVM_FALLTHROUGH;
2320  case ABIArgInfo::Direct:
2321  if (RetAI.getInReg())
2322  RetAttrs.addAttribute(llvm::Attribute::InReg);
2323  break;
2324  case ABIArgInfo::Ignore:
2325  break;
2326 
2327  case ABIArgInfo::InAlloca:
2328  case ABIArgInfo::Indirect: {
2329  // inalloca and sret disable readnone and readonly
2330  FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2331  .removeAttribute(llvm::Attribute::ReadNone);
2332  break;
2333  }
2334 
2336  break;
2337 
2338  case ABIArgInfo::Expand:
2340  llvm_unreachable("Invalid ABI kind for return argument");
2341  }
2342 
2343  if (!IsThunk) {
2344  // FIXME: fix this properly, https://reviews.llvm.org/D100388
2345  if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2346  QualType PTy = RefTy->getPointeeType();
2347  if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2348  RetAttrs.addDereferenceableAttr(
2349  getMinimumObjectSize(PTy).getQuantity());
2350  if (getContext().getTargetAddressSpace(PTy) == 0 &&
2351  !CodeGenOpts.NullPointerIsValid)
2352  RetAttrs.addAttribute(llvm::Attribute::NonNull);
2353  if (PTy->isObjectType()) {
2354  llvm::Align Alignment =
2356  RetAttrs.addAlignmentAttr(Alignment);
2357  }
2358  }
2359  }
2360 
2361  bool hasUsedSRet = false;
2362  SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2363 
2364  // Attach attributes to sret.
2365  if (IRFunctionArgs.hasSRetArg()) {
2366  llvm::AttrBuilder SRETAttrs(getLLVMContext());
2367  SRETAttrs.addStructRetAttr(getTypes().ConvertTypeForMem(RetTy));
2368  hasUsedSRet = true;
2369  if (RetAI.getInReg())
2370  SRETAttrs.addAttribute(llvm::Attribute::InReg);
2371  SRETAttrs.addAlignmentAttr(RetAI.getIndirectAlign().getQuantity());
2372  ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2373  llvm::AttributeSet::get(getLLVMContext(), SRETAttrs);
2374  }
2375 
2376  // Attach attributes to inalloca argument.
2377  if (IRFunctionArgs.hasInallocaArg()) {
2378  llvm::AttrBuilder Attrs(getLLVMContext());
2379  Attrs.addInAllocaAttr(FI.getArgStruct());
2380  ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2381  llvm::AttributeSet::get(getLLVMContext(), Attrs);
2382  }
2383 
2384  // Apply `nonnull`, `dereferencable(N)` and `align N` to the `this` argument,
2385  // unless this is a thunk function.
2386  // FIXME: fix this properly, https://reviews.llvm.org/D100388
2387  if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
2388  !FI.arg_begin()->type->isVoidPointerType() && !IsThunk) {
2389  auto IRArgs = IRFunctionArgs.getIRArgs(0);
2390 
2391  assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
2392 
2393  llvm::AttrBuilder Attrs(getLLVMContext());
2394 
2395  QualType ThisTy =
2396  FI.arg_begin()->type.castAs<PointerType>()->getPointeeType();
2397 
2398  if (!CodeGenOpts.NullPointerIsValid &&
2399  getContext().getTargetAddressSpace(FI.arg_begin()->type) == 0) {
2400  Attrs.addAttribute(llvm::Attribute::NonNull);
2401  Attrs.addDereferenceableAttr(getMinimumObjectSize(ThisTy).getQuantity());
2402  } else {
2403  // FIXME dereferenceable should be correct here, regardless of
2404  // NullPointerIsValid. However, dereferenceable currently does not always
2405  // respect NullPointerIsValid and may imply nonnull and break the program.
2406  // See https://reviews.llvm.org/D66618 for discussions.
2407  Attrs.addDereferenceableOrNullAttr(
2409  FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2410  .getQuantity());
2411  }
2412 
2413  llvm::Align Alignment =
2414  getNaturalTypeAlignment(ThisTy, /*BaseInfo=*/nullptr,
2415  /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
2416  .getAsAlign();
2417  Attrs.addAlignmentAttr(Alignment);
2418 
2419  ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(getLLVMContext(), Attrs);
2420  }
2421 
2422  unsigned ArgNo = 0;
2424  E = FI.arg_end();
2425  I != E; ++I, ++ArgNo) {
2426  QualType ParamType = I->type;
2427  const ABIArgInfo &AI = I->info;
2428  llvm::AttrBuilder Attrs(getLLVMContext());
2429 
2430  // Add attribute for padding argument, if necessary.
2431  if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2432  if (AI.getPaddingInReg()) {
2433  ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2434  llvm::AttributeSet::get(
2435  getLLVMContext(),
2436  llvm::AttrBuilder(getLLVMContext()).addAttribute(llvm::Attribute::InReg));
2437  }
2438  }
2439 
2440  // Decide whether the argument we're handling could be partially undef
2441  if (CodeGenOpts.EnableNoundefAttrs &&
2442  DetermineNoUndef(ParamType, getTypes(), DL, AI)) {
2443  Attrs.addAttribute(llvm::Attribute::NoUndef);
2444  }
2445 
2446  // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2447  // have the corresponding parameter variable. It doesn't make
2448  // sense to do it here because parameters are so messed up.
2449  switch (AI.getKind()) {
2450  case ABIArgInfo::Extend:
2451  if (AI.isSignExt())
2452  Attrs.addAttribute(llvm::Attribute::SExt);
2453  else
2454  Attrs.addAttribute(llvm::Attribute::ZExt);
2455  LLVM_FALLTHROUGH;
2456  case ABIArgInfo::Direct:
2457  if (ArgNo == 0 && FI.isChainCall())
2458  Attrs.addAttribute(llvm::Attribute::Nest);
2459  else if (AI.getInReg())
2460  Attrs.addAttribute(llvm::Attribute::InReg);
2461  Attrs.addStackAlignmentAttr(llvm::MaybeAlign(AI.getDirectAlign()));
2462  break;
2463 
2464  case ABIArgInfo::Indirect: {
2465  if (AI.getInReg())
2466  Attrs.addAttribute(llvm::Attribute::InReg);
2467 
2468  if (AI.getIndirectByVal())
2469  Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
2470 
2471  auto *Decl = ParamType->getAsRecordDecl();
2472  if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
2473  Decl->getArgPassingRestrictions() == RecordDecl::APK_CanPassInRegs)
2474  // When calling the function, the pointer passed in will be the only
2475  // reference to the underlying object. Mark it accordingly.
2476  Attrs.addAttribute(llvm::Attribute::NoAlias);
2477 
2478  // TODO: We could add the byref attribute if not byval, but it would
2479  // require updating many testcases.
2480 
2481  CharUnits Align = AI.getIndirectAlign();
2482 
2483  // In a byval argument, it is important that the required
2484  // alignment of the type is honored, as LLVM might be creating a
2485  // *new* stack object, and needs to know what alignment to give
2486  // it. (Sometimes it can deduce a sensible alignment on its own,
2487  // but not if clang decides it must emit a packed struct, or the
2488  // user specifies increased alignment requirements.)
2489  //
2490  // This is different from indirect *not* byval, where the object
2491  // exists already, and the align attribute is purely
2492  // informative.
2493  assert(!Align.isZero());
2494 
2495  // For now, only add this when we have a byval argument.
2496  // TODO: be less lazy about updating test cases.
2497  if (AI.getIndirectByVal())
2498  Attrs.addAlignmentAttr(Align.getQuantity());
2499 
2500  // byval disables readnone and readonly.
2501  FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2502  .removeAttribute(llvm::Attribute::ReadNone);
2503 
2504  break;
2505  }
2507  CharUnits Align = AI.getIndirectAlign();
2508  Attrs.addByRefAttr(getTypes().ConvertTypeForMem(ParamType));
2509  Attrs.addAlignmentAttr(Align.getQuantity());
2510  break;
2511  }
2512  case ABIArgInfo::Ignore:
2513  case ABIArgInfo::Expand:
2515  break;
2516 
2517  case ABIArgInfo::InAlloca:
2518  // inalloca disables readnone and readonly.
2519  FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2520  .removeAttribute(llvm::Attribute::ReadNone);
2521  continue;
2522  }
2523 
2524  if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2525  QualType PTy = RefTy->getPointeeType();
2526  if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2527  Attrs.addDereferenceableAttr(
2528  getMinimumObjectSize(PTy).getQuantity());
2529  if (getContext().getTargetAddressSpace(PTy) == 0 &&
2530  !CodeGenOpts.NullPointerIsValid)
2531  Attrs.addAttribute(llvm::Attribute::NonNull);
2532  if (PTy->isObjectType()) {
2533  llvm::Align Alignment =
2535  Attrs.addAlignmentAttr(Alignment);
2536  }
2537  }
2538 
2539  // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
2540  // > For arguments to a __kernel function declared to be a pointer to a
2541  // > data type, the OpenCL compiler can assume that the pointee is always
2542  // > appropriately aligned as required by the data type.
2543  if (TargetDecl && TargetDecl->hasAttr<OpenCLKernelAttr>() &&
2544  ParamType->isPointerType()) {
2545  QualType PTy = ParamType->getPointeeType();
2546  if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2547  llvm::Align Alignment =
2549  Attrs.addAlignmentAttr(Alignment);
2550  }
2551  }
2552 
2553  switch (FI.getExtParameterInfo(ArgNo).getABI()) {
2555  break;
2556 
2558  // Add 'sret' if we haven't already used it for something, but
2559  // only if the result is void.
2560  if (!hasUsedSRet && RetTy->isVoidType()) {
2561  Attrs.addStructRetAttr(getTypes().ConvertTypeForMem(ParamType));
2562  hasUsedSRet = true;
2563  }
2564 
2565  // Add 'noalias' in either case.
2566  Attrs.addAttribute(llvm::Attribute::NoAlias);
2567 
2568  // Add 'dereferenceable' and 'alignment'.
2569  auto PTy = ParamType->getPointeeType();
2570  if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2571  auto info = getContext().getTypeInfoInChars(PTy);
2572  Attrs.addDereferenceableAttr(info.Width.getQuantity());
2573  Attrs.addAlignmentAttr(info.Align.getAsAlign());
2574  }
2575  break;
2576  }
2577 
2579  Attrs.addAttribute(llvm::Attribute::SwiftError);
2580  break;
2581 
2583  Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2584  break;
2585 
2587  Attrs.addAttribute(llvm::Attribute::SwiftAsync);
2588  break;
2589  }
2590 
2591  if (FI.getExtParameterInfo(ArgNo).isNoEscape())
2592  Attrs.addAttribute(llvm::Attribute::NoCapture);
2593 
2594  if (Attrs.hasAttributes()) {
2595  unsigned FirstIRArg, NumIRArgs;
2596  std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2597  for (unsigned i = 0; i < NumIRArgs; i++)
2598  ArgAttrs[FirstIRArg + i] = ArgAttrs[FirstIRArg + i].addAttributes(
2599  getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), Attrs));
2600  }
2601  }
2602  assert(ArgNo == FI.arg_size());
2603 
2604  AttrList = llvm::AttributeList::get(
2605  getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs),
2606  llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs);
2607 }
2608 
2609 /// An argument came in as a promoted argument; demote it back to its
2610 /// declared type.
2611 static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
2612  const VarDecl *var,
2613  llvm::Value *value) {
2614  llvm::Type *varType = CGF.ConvertType(var->getType());
2615 
2616  // This can happen with promotions that actually don't change the
2617  // underlying type, like the enum promotions.
2618  if (value->getType() == varType) return value;
2619 
2620  assert((varType->isIntegerTy() || varType->isFloatingPointTy())
2621  && "unexpected promotion type");
2622 
2623  if (isa<llvm::IntegerType>(varType))
2624  return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
2625 
2626  return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
2627 }
2628 
2629 /// Returns the attribute (either parameter attribute, or function
2630 /// attribute), which declares argument ArgNo to be non-null.
2631 static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
2632  QualType ArgType, unsigned ArgNo) {
2633  // FIXME: __attribute__((nonnull)) can also be applied to:
2634  // - references to pointers, where the pointee is known to be
2635  // nonnull (apparently a Clang extension)
2636  // - transparent unions containing pointers
2637  // In the former case, LLVM IR cannot represent the constraint. In
2638  // the latter case, we have no guarantee that the transparent union
2639  // is in fact passed as a pointer.
2640  if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
2641  return nullptr;
2642  // First, check attribute on parameter itself.
2643  if (PVD) {
2644  if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
2645  return ParmNNAttr;
2646  }
2647  // Check function attributes.
2648  if (!FD)
2649  return nullptr;
2650  for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
2651  if (NNAttr->isNonNull(ArgNo))
2652  return NNAttr;
2653  }
2654  return nullptr;
2655 }
2656 
2657 namespace {
2658  struct CopyBackSwiftError final : EHScopeStack::Cleanup {
2659  Address Temp;
2660  Address Arg;
2661  CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
2662  void Emit(CodeGenFunction &CGF, Flags flags) override {
2663  llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp);
2664  CGF.Builder.CreateStore(errorValue, Arg);
2665  }
2666  };
2667 }
2668 
2670  llvm::Function *Fn,
2671  const FunctionArgList &Args) {
2672  if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
2673  // Naked functions don't have prologues.
2674  return;
2675 
2676  // If this is an implicit-return-zero function, go ahead and
2677  // initialize the return value. TODO: it might be nice to have
2678  // a more general mechanism for this that didn't require synthesized
2679  // return statements.
2680  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {
2681  if (FD->hasImplicitReturnZero()) {
2682  QualType RetTy = FD->getReturnType().getUnqualifiedType();
2683  llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
2684  llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
2686  }
2687  }
2688 
2689  // FIXME: We no longer need the types from FunctionArgList; lift up and
2690  // simplify.
2691 
2692  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
2693  assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
2694 
2695  // If we're using inalloca, all the memory arguments are GEPs off of the last
2696  // parameter, which is a pointer to the complete memory area.
2697  Address ArgStruct = Address::invalid();
2698  if (IRFunctionArgs.hasInallocaArg()) {
2699  ArgStruct = Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()),
2700  FI.getArgStruct(), FI.getArgStructAlignment());
2701 
2702  assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo());
2703  }
2704 
2705  // Name the struct return parameter.
2706  if (IRFunctionArgs.hasSRetArg()) {
2707  auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo());
2708  AI->setName("agg.result");
2709  AI->addAttr(llvm::Attribute::NoAlias);
2710  }
2711 
2712  // Track if we received the parameter as a pointer (indirect, byval, or
2713  // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
2714  // into a local alloca for us.
2716  ArgVals.reserve(Args.size());
2717 
2718  // Create a pointer value for every parameter declaration. This usually
2719  // entails copying one or more LLVM IR arguments into an alloca. Don't push
2720  // any cleanups or do anything that might unwind. We do that separately, so
2721  // we can push the cleanups in the correct order for the ABI.
2722  assert(FI.arg_size() == Args.size() &&
2723  "Mismatch between function signature & arguments.");
2724  unsigned ArgNo = 0;
2726  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
2727  i != e; ++i, ++info_it, ++ArgNo) {
2728  const VarDecl *Arg = *i;
2729  const ABIArgInfo &ArgI = info_it->info;
2730 
2731  bool isPromoted =
2732  isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();
2733  // We are converting from ABIArgInfo type to VarDecl type directly, unless
2734  // the parameter is promoted. In this case we convert to
2735  // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
2736  QualType Ty = isPromoted ? info_it->type : Arg->getType();
2737  assert(hasScalarEvaluationKind(Ty) ==
2739 
2740  unsigned FirstIRArg, NumIRArgs;
2741  std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2742 
2743  switch (ArgI.getKind()) {
2744  case ABIArgInfo::InAlloca: {
2745  assert(NumIRArgs == 0);
2746  auto FieldIndex = ArgI.getInAllocaFieldIndex();
2747  Address V =
2748  Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
2749  if (ArgI.getInAllocaIndirect())
2751  getContext().getTypeAlignInChars(Ty));
2752  ArgVals.push_back(ParamValue::forIndirect(V));
2753  break;
2754  }
2755 
2756  case ABIArgInfo::Indirect:
2758  assert(NumIRArgs == 1);
2759  Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty),
2760  ArgI.getIndirectAlign());
2761 
2762  if (!hasScalarEvaluationKind(Ty)) {
2763  // Aggregates and complex variables are accessed by reference. All we
2764  // need to do is realign the value, if requested. Also, if the address
2765  // may be aliased, copy it to ensure that the parameter variable is
2766  // mutable and has a unique adress, as C requires.
2767  Address V = ParamAddr;
2768  if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
2769  Address AlignedTemp = CreateMemTemp(Ty, "coerce");
2770 
2771  // Copy from the incoming argument pointer to the temporary with the
2772  // appropriate alignment.
2773  //
2774  // FIXME: We should have a common utility for generating an aggregate
2775  // copy.
2778  AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(),
2779  ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(),
2780  llvm::ConstantInt::get(IntPtrTy, Size.getQuantity()));
2781  V = AlignedTemp;
2782  }
2783  ArgVals.push_back(ParamValue::forIndirect(V));
2784  } else {
2785  // Load scalar value from indirect argument.
2786  llvm::Value *V =
2787  EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());
2788 
2789  if (isPromoted)
2790  V = emitArgumentDemotion(*this, Arg, V);
2791  ArgVals.push_back(ParamValue::forDirect(V));
2792  }
2793  break;
2794  }
2795 
2796  case ABIArgInfo::Extend:
2797  case ABIArgInfo::Direct: {
2798  auto AI = Fn->getArg(FirstIRArg);
2799  llvm::Type *LTy = ConvertType(Arg->getType());
2800 
2801  // Prepare parameter attributes. So far, only attributes for pointer
2802  // parameters are prepared. See
2803  // http://llvm.org/docs/LangRef.html#paramattrs.
2804  if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&
2805  ArgI.getCoerceToType()->isPointerTy()) {
2806  assert(NumIRArgs == 1);
2807 
2808  if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
2809  // Set `nonnull` attribute if any.
2810  if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
2811  PVD->getFunctionScopeIndex()) &&
2812  !CGM.getCodeGenOpts().NullPointerIsValid)
2813  AI->addAttr(llvm::Attribute::NonNull);
2814 
2815  QualType OTy = PVD->getOriginalType();
2816  if (const auto *ArrTy =
2817  getContext().getAsConstantArrayType(OTy)) {
2818  // A C99 array parameter declaration with the static keyword also
2819  // indicates dereferenceability, and if the size is constant we can
2820  // use the dereferenceable attribute (which requires the size in
2821  // bytes).
2822  if (ArrTy->getSizeModifier() == ArrayType::Static) {
2823  QualType ETy = ArrTy->getElementType();
2824  llvm::Align Alignment =
2826  AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(Alignment));
2827  uint64_t ArrSize = ArrTy->getSize().getZExtValue();
2828  if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
2829  ArrSize) {
2830  llvm::AttrBuilder Attrs(getLLVMContext());
2831  Attrs.addDereferenceableAttr(
2832  getContext().getTypeSizeInChars(ETy).getQuantity() *
2833  ArrSize);
2834  AI->addAttrs(Attrs);
2835  } else if (getContext().getTargetInfo().getNullPointerValue(
2836  ETy.getAddressSpace()) == 0 &&
2837  !CGM.getCodeGenOpts().NullPointerIsValid) {
2838  AI->addAttr(llvm::Attribute::NonNull);
2839  }
2840  }
2841  } else if (const auto *ArrTy =
2842  getContext().getAsVariableArrayType(OTy)) {
2843  // For C99 VLAs with the static keyword, we don't know the size so
2844  // we can't use the dereferenceable attribute, but in addrspace(0)
2845  // we know that it must be nonnull.
2846  if (ArrTy->getSizeModifier() == VariableArrayType::Static) {
2847  QualType ETy = ArrTy->getElementType();
2848  llvm::Align Alignment =
2850  AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(Alignment));
2851  if (!getContext().getTargetAddressSpace(ETy) &&
2852  !CGM.getCodeGenOpts().NullPointerIsValid)
2853  AI->addAttr(llvm::Attribute::NonNull);
2854  }
2855  }
2856 
2857  // Set `align` attribute if any.
2858  const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
2859  if (!AVAttr)
2860  if (const auto *TOTy = dyn_cast<TypedefType>(OTy))
2861  AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
2862  if (AVAttr && !SanOpts.has(SanitizerKind::Alignment)) {
2863  // If alignment-assumption sanitizer is enabled, we do *not* add
2864  // alignment attribute here, but emit normal alignment assumption,
2865  // so the UBSAN check could function.
2866  llvm::ConstantInt *AlignmentCI =
2867  cast<llvm::ConstantInt>(EmitScalarExpr(AVAttr->getAlignment()));
2868  uint64_t AlignmentInt =
2869  AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment);
2870  if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
2871  AI->removeAttr(llvm::Attribute::AttrKind::Alignment);
2872  AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(
2873  llvm::Align(AlignmentInt)));
2874  }
2875  }
2876  }
2877 
2878  // Set 'noalias' if an argument type has the `restrict` qualifier.
2879  if (Arg->getType().isRestrictQualified())
2880  AI->addAttr(llvm::Attribute::NoAlias);
2881  }
2882 
2883  // Prepare the argument value. If we have the trivial case, handle it
2884  // with no muss and fuss.
2885  if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
2886  ArgI.getCoerceToType() == ConvertType(Ty) &&
2887  ArgI.getDirectOffset() == 0) {
2888  assert(NumIRArgs == 1);
2889 
2890  // LLVM expects swifterror parameters to be used in very restricted
2891  // ways. Copy the value into a less-restricted temporary.
2892  llvm::Value *V = AI;
2893  if (FI.getExtParameterInfo(ArgNo).getABI()
2895  QualType pointeeTy = Ty->getPointeeType();
2896  assert(pointeeTy->isPointerType());
2897  Address temp =
2898  CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
2899  Address arg(V, ConvertTypeForMem(pointeeTy),
2900  getContext().getTypeAlignInChars(pointeeTy));
2901  llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
2902  Builder.CreateStore(incomingErrorValue, temp);
2903  V = temp.getPointer();
2904 
2905  // Push a cleanup to copy the value back at the end of the function.
2906  // The convention does not guarantee that the value will be written
2907  // back if the function exits with an unwind exception.
2908  EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg);
2909  }
2910 
2911  // Ensure the argument is the correct type.
2912  if (V->getType() != ArgI.getCoerceToType())
2913  V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
2914 
2915  if (isPromoted)
2916  V = emitArgumentDemotion(*this, Arg, V);
2917 
2918  // Because of merging of function types from multiple decls it is
2919  // possible for the type of an argument to not match the corresponding
2920  // type in the function type. Since we are codegening the callee
2921  // in here, add a cast to the argument type.
2922  llvm::Type *LTy = ConvertType(Arg->getType());
2923  if (V->getType() != LTy)
2924  V = Builder.CreateBitCast(V, LTy);
2925 
2926  ArgVals.push_back(ParamValue::forDirect(V));
2927  break;
2928  }
2929 
2930  // VLST arguments are coerced to VLATs at the function boundary for
2931  // ABI consistency. If this is a VLST that was coerced to
2932  // a VLAT at the function boundary and the types match up, use
2933  // llvm.experimental.vector.extract to convert back to the original
2934  // VLST.
2935  if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(ConvertType(Ty))) {
2936  llvm::Value *Coerced = Fn->getArg(FirstIRArg);
2937  if (auto *VecTyFrom =
2938  dyn_cast<llvm::ScalableVectorType>(Coerced->getType())) {
2939  // If we are casting a scalable 16 x i1 predicate vector to a fixed i8
2940  // vector, bitcast the source and use a vector extract.
2941  auto PredType =
2942  llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
2943  if (VecTyFrom == PredType &&
2944  VecTyTo->getElementType() == Builder.getInt8Ty()) {
2945  VecTyFrom = llvm::ScalableVectorType::get(Builder.getInt8Ty(), 2);
2946  Coerced = Builder.CreateBitCast(Coerced, VecTyFrom);
2947  }
2948  if (VecTyFrom->getElementType() == VecTyTo->getElementType()) {
2949  llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty);
2950 
2951  assert(NumIRArgs == 1);
2952  Coerced->setName(Arg->getName() + ".coerce");
2953  ArgVals.push_back(ParamValue::forDirect(Builder.CreateExtractVector(
2954  VecTyTo, Coerced, Zero, "castFixedSve")));
2955  break;
2956  }
2957  }
2958  }
2959 
2960  Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg),
2961  Arg->getName());
2962 
2963  // Pointer to store into.
2964  Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
2965 
2966  // Fast-isel and the optimizer generally like scalar values better than
2967  // FCAs, so we flatten them if this is safe to do for this argument.
2968  llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
2969  if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
2970  STy->getNumElements() > 1) {
2971  uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy);
2972  llvm::Type *DstTy = Ptr.getElementType();
2973  uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy);
2974 
2975  Address AddrToStoreInto = Address::invalid();
2976  if (SrcSize <= DstSize) {
2977  AddrToStoreInto = Builder.CreateElementBitCast(Ptr, STy);
2978  } else {
2979  AddrToStoreInto =
2980  CreateTempAlloca(STy, Alloca.getAlignment(), "coerce");
2981  }
2982 
2983  assert(STy->getNumElements() == NumIRArgs);
2984  for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
2985  auto AI = Fn->getArg(FirstIRArg + i);
2986  AI->setName(Arg->getName() + ".coerce" + Twine(i));
2987  Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i);
2988  Builder.CreateStore(AI, EltPtr);
2989  }
2990 
2991  if (SrcSize > DstSize) {
2992  Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize);
2993  }
2994 
2995  } else {
2996  // Simple case, just do a coerced store of the argument into the alloca.
2997  assert(NumIRArgs == 1);
2998  auto AI = Fn->getArg(FirstIRArg);
2999  AI->setName(Arg->getName() + ".coerce");
3000  CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this);
3001  }
3002 
3003  // Match to what EmitParmDecl is expecting for this type.
3005  llvm::Value *V =
3006  EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
3007  if (isPromoted)
3008  V = emitArgumentDemotion(*this, Arg, V);
3009  ArgVals.push_back(ParamValue::forDirect(V));
3010  } else {
3011  ArgVals.push_back(ParamValue::forIndirect(Alloca));
3012  }
3013  break;
3014  }
3015 
3017  // Reconstruct into a temporary.
3018  Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
3019  ArgVals.push_back(ParamValue::forIndirect(alloca));
3020 
3021  auto coercionType = ArgI.getCoerceAndExpandType();
3022  alloca = Builder.CreateElementBitCast(alloca, coercionType);
3023 
3024  unsigned argIndex = FirstIRArg;
3025  for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3026  llvm::Type *eltType = coercionType->getElementType(i);
3028  continue;
3029 
3030  auto eltAddr = Builder.CreateStructGEP(alloca, i);
3031  auto elt = Fn->getArg(argIndex++);
3032  Builder.CreateStore(elt, eltAddr);
3033  }
3034  assert(argIndex == FirstIRArg + NumIRArgs);
3035  break;
3036  }
3037 
3038  case ABIArgInfo::Expand: {
3039  // If this structure was expanded into multiple arguments then
3040  // we need to create a temporary and reconstruct it from the
3041  // arguments.
3042  Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
3043  LValue LV = MakeAddrLValue(Alloca, Ty);
3044  ArgVals.push_back(ParamValue::forIndirect(Alloca));
3045 
3046  auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3047  ExpandTypeFromArgs(Ty, LV, FnArgIter);
3048  assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3049  for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
3050  auto AI = Fn->getArg(FirstIRArg + i);
3051  AI->setName(Arg->getName() + "." + Twine(i));
3052  }
3053  break;
3054  }
3055 
3056  case ABIArgInfo::Ignore:
3057  assert(NumIRArgs == 0);
3058  // Initialize the local variable appropriately.
3059  if (!hasScalarEvaluationKind(Ty)) {
3060  ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty)));
3061  } else {
3062  llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
3063  ArgVals.push_back(ParamValue::forDirect(U));
3064  }
3065  break;
3066  }
3067  }
3068 
3069  if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3070  for (int I = Args.size() - 1; I >= 0; --I)
3071  EmitParmDecl(*Args[I], ArgVals[I], I + 1);
3072  } else {
3073  for (unsigned I = 0, E = Args.size(); I != E; ++I)
3074  EmitParmDecl(*Args[I], ArgVals[I], I + 1);
3075  }
3076 }
3077 
3078 static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3079  while (insn->use_empty()) {
3080  llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
3081  if (!bitcast) return;
3082 
3083  // This is "safe" because we would have used a ConstantExpr otherwise.
3084  insn = cast<llvm::Instruction>(bitcast->getOperand(0));
3085  bitcast->eraseFromParent();
3086  }
3087 }
3088 
3089 /// Try to emit a fused autorelease of a return result.
3091  llvm::Value *result) {
3092  // We must be immediately followed the cast.
3093  llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3094  if (BB->empty()) return nullptr;
3095  if (&BB->back() != result) return nullptr;
3096 
3097  llvm::Type *resultType = result->getType();
3098 
3099  // result is in a BasicBlock and is therefore an Instruction.
3100  llvm::Instruction *generator = cast<llvm::Instruction>(result);
3101 
3103 
3104  // Look for:
3105  // %generator = bitcast %type1* %generator2 to %type2*
3106  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
3107  // We would have emitted this as a constant if the operand weren't
3108  // an Instruction.
3109  generator = cast<llvm::Instruction>(bitcast->getOperand(0));
3110 
3111  // Require the generator to be immediately followed by the cast.
3112  if (generator->getNextNode() != bitcast)
3113  return nullptr;
3114 
3115  InstsToKill.push_back(bitcast);
3116  }
3117 
3118  // Look for:
3119  // %generator = call i8* @objc_retain(i8* %originalResult)
3120  // or
3121  // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3122  llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
3123  if (!call) return nullptr;
3124 
3125  bool doRetainAutorelease;
3126 
3127  if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3128  doRetainAutorelease = true;
3129  } else if (call->getCalledOperand() ==
3131  doRetainAutorelease = false;
3132 
3133  // If we emitted an assembly marker for this call (and the
3134  // ARCEntrypoints field should have been set if so), go looking
3135  // for that call. If we can't find it, we can't do this
3136  // optimization. But it should always be the immediately previous
3137  // instruction, unless we needed bitcasts around the call.
3139  llvm::Instruction *prev = call->getPrevNode();
3140  assert(prev);
3141  if (isa<llvm::BitCastInst>(prev)) {
3142  prev = prev->getPrevNode();
3143  assert(prev);
3144  }
3145  assert(isa<llvm::CallInst>(prev));
3146  assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3148  InstsToKill.push_back(prev);
3149  }
3150  } else {
3151  return nullptr;
3152  }
3153 
3154  result = call->getArgOperand(0);
3155  InstsToKill.push_back(call);
3156 
3157  // Keep killing bitcasts, for sanity. Note that we no longer care
3158  // about precise ordering as long as there's exactly one use.
3159  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
3160  if (!bitcast->hasOneUse()) break;
3161  InstsToKill.push_back(bitcast);
3162  result = bitcast->getOperand(0);
3163  }
3164 
3165  // Delete all the unnecessary instructions, from latest to earliest.
3166  for (auto *I : InstsToKill)
3167  I->eraseFromParent();
3168 
3169  // Do the fused retain/autorelease if we were asked to.
3170  if (doRetainAutorelease)
3171  result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
3172 
3173  // Cast back to the result type.
3174  return CGF.Builder.CreateBitCast(result, resultType);
3175 }
3176 
3177 /// If this is a +1 of the value of an immutable 'self', remove it.
3178 static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
3179  llvm::Value *result) {
3180  // This is only applicable to a method with an immutable 'self'.
3181  const ObjCMethodDecl *method =
3182  dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);
3183  if (!method) return nullptr;
3184  const VarDecl *self = method->getSelfDecl();
3185  if (!self->getType().isConstQualified()) return nullptr;
3186 
3187  // Look for a retain call.
3188  llvm::CallInst *retainCall =
3189  dyn_cast<llvm::CallInst>(result->stripPointerCasts());
3190  if (!retainCall || retainCall->getCalledOperand() !=
3192  return nullptr;
3193 
3194  // Look for an ordinary load of 'self'.
3195  llvm::Value *retainedValue = retainCall->getArgOperand(0);
3196  llvm::LoadInst *load =
3197  dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
3198  if (!load || load->isAtomic() || load->isVolatile() ||
3199  load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer())
3200  return nullptr;
3201 
3202  // Okay! Burn it all down. This relies for correctness on the
3203  // assumption that the retain is emitted as part of the return and
3204  // that thereafter everything is used "linearly".
3205  llvm::Type *resultType = result->getType();
3206  eraseUnusedBitCasts(cast<llvm::Instruction>(result));
3207  assert(retainCall->use_empty());
3208  retainCall->eraseFromParent();
3209  eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue));
3210 
3211  return CGF.Builder.CreateBitCast(load, resultType);
3212 }
3213 
3214 /// Emit an ARC autorelease of the result of a function.
3215 ///
3216 /// \return the value to actually return from the function
3217 static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
3218  llvm::Value *result) {
3219  // If we're returning 'self', kill the initial retain. This is a
3220  // heuristic attempt to "encourage correctness" in the really unfortunate
3221  // case where we have a return of self during a dealloc and we desperately
3222  // need to avoid the possible autorelease.
3223  if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
3224  return self;
3225 
3226  // At -O0, try to emit a fused retain/autorelease.
3227  if (CGF.shouldUseFusedARCCalls())
3228  if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
3229  return fused;
3230 
3231  return CGF.EmitARCAutoreleaseReturnValue(result);
3232 }
3233 
3234 /// Heuristically search for a dominating store to the return-value slot.
3235 static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
3236  // Check if a User is a store which pointerOperand is the ReturnValue.
3237  // We are looking for stores to the ReturnValue, not for stores of the
3238  // ReturnValue to some other location.
3239  auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
3240  auto *SI = dyn_cast<llvm::StoreInst>(U);
3241  if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer() ||
3242  SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
3243  return nullptr;
3244  // These aren't actually possible for non-coerced returns, and we
3245  // only care about non-coerced returns on this code path.
3246  assert(!SI->isAtomic() && !SI->isVolatile());
3247  return SI;
3248  };
3249  // If there are multiple uses of the return-value slot, just check
3250  // for something immediately preceding the IP. Sometimes this can
3251  // happen with how we generate implicit-returns; it can also happen
3252  // with noreturn cleanups.
3253  if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
3254  llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3255  if (IP->empty()) return nullptr;
3256 
3257  // Look at directly preceding instruction, skipping bitcasts and lifetime
3258  // markers.
3259  for (llvm::Instruction &I : make_range(IP->rbegin(), IP->rend())) {
3260  if (isa<llvm::BitCastInst>(&I))
3261  continue;
3262  if (auto *II = dyn_cast<llvm::IntrinsicInst>(&I))
3263  if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
3264  continue;
3265 
3266  return GetStoreIfValid(&I);
3267  }
3268  return nullptr;
3269  }
3270 
3271  llvm::StoreInst *store =
3272  GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
3273  if (!store) return nullptr;
3274 
3275  // Now do a first-and-dirty dominance check: just walk up the
3276  // single-predecessors chain from the current insertion point.
3277  llvm::BasicBlock *StoreBB = store->getParent();
3278  llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3279  while (IP != StoreBB) {
3280  if (!(IP = IP->getSinglePredecessor()))
3281  return nullptr;
3282  }
3283 
3284  // Okay, the store's basic block dominates the insertion point; we
3285  // can do our thing.
3286  return store;
3287 }
3288 
3289 // Helper functions for EmitCMSEClearRecord
3290 
3291 // Set the bits corresponding to a field having width `BitWidth` and located at
3292 // offset `BitOffset` (from the least significant bit) within a storage unit of
3293 // `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3294 // Use little-endian layout, i.e.`Bits[0]` is the LSB.
3295 static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3296  int BitWidth, int CharWidth) {
3297  assert(CharWidth <= 64);
3298  assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3299 
3300  int Pos = 0;
3301  if (BitOffset >= CharWidth) {
3302  Pos += BitOffset / CharWidth;
3303  BitOffset = BitOffset % CharWidth;
3304  }
3305 
3306  const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
3307  if (BitOffset + BitWidth >= CharWidth) {
3308  Bits[Pos++] |= (Used << BitOffset) & Used;
3309  BitWidth -= CharWidth - BitOffset;
3310  BitOffset = 0;
3311  }
3312 
3313  while (BitWidth >= CharWidth) {
3314  Bits[Pos++] = Used;
3315  BitWidth -= CharWidth;
3316  }
3317 
3318  if (BitWidth > 0)
3319  Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
3320 }
3321 
3322 // Set the bits corresponding to a field having width `BitWidth` and located at
3323 // offset `BitOffset` (from the least significant bit) within a storage unit of
3324 // `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3325 // `Bits` corresponds to one target byte. Use target endian layout.
3326 static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3327  int StorageSize, int BitOffset, int BitWidth,
3328  int CharWidth, bool BigEndian) {
3329 
3330  SmallVector<uint64_t, 8> TmpBits(StorageSize);
3331  setBitRange(TmpBits, BitOffset, BitWidth, CharWidth);
3332 
3333  if (BigEndian)
3334  std::reverse(TmpBits.begin(), TmpBits.end());
3335 
3336  for (uint64_t V : TmpBits)
3337  Bits[StorageOffset++] |= V;
3338 }
3339 
3340 static void setUsedBits(CodeGenModule &, QualType, int,
3341  SmallVectorImpl<uint64_t> &);
3342 
3343 // Set the bits in `Bits`, which correspond to the value representations of
3344 // the actual members of the record type `RTy`. Note that this function does
3345 // not handle base classes, virtual tables, etc, since they cannot happen in
3346 // CMSE function arguments or return. The bit mask corresponds to the target
3347 // memory layout, i.e. it's endian dependent.
3348 static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
3349  SmallVectorImpl<uint64_t> &Bits) {
3350  ASTContext &Context = CGM.getContext();
3351  int CharWidth = Context.getCharWidth();
3352  const RecordDecl *RD = RTy->getDecl()->getDefinition();
3353  const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(RD);
3354  const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3355 
3356  int Idx = 0;
3357  for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
3358  const FieldDecl *F = *I;
3359 
3360  if (F->isUnnamedBitfield() || F->isZeroLengthBitField(Context) ||
3362  continue;
3363 
3364  if (F->isBitField()) {
3365  const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(F);
3367  BFI.StorageSize / CharWidth, BFI.Offset,
3368  BFI.Size, CharWidth,
3369  CGM.getDataLayout().isBigEndian());
3370  continue;
3371  }
3372 
3373  setUsedBits(CGM, F->getType(),
3374  Offset + ASTLayout.getFieldOffset(Idx) / CharWidth, Bits);
3375  }
3376 }
3377 
3378 // Set the bits in `Bits`, which correspond to the value representations of
3379 // the elements of an array type `ATy`.
3380 static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
3381  int Offset, SmallVectorImpl<uint64_t> &Bits) {
3382  const ASTContext &Context = CGM.getContext();
3383 
3384  QualType ETy = Context.getBaseElementType(ATy);
3385  int Size = Context.getTypeSizeInChars(ETy).getQuantity();
3386  SmallVector<uint64_t, 4> TmpBits(Size);
3387  setUsedBits(CGM, ETy, 0, TmpBits);
3388 
3389  for (int I = 0, N = Context.getConstantArrayElementCount(ATy); I < N; ++I) {
3390  auto Src = TmpBits.begin();
3391  auto Dst = Bits.begin() + Offset + I * Size;
3392  for (int J = 0; J < Size; ++J)
3393  *Dst++ |= *Src++;
3394  }
3395 }
3396 
3397 // Set the bits in `Bits`, which correspond to the value representations of
3398 // the type `QTy`.
3399 static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
3400  SmallVectorImpl<uint64_t> &Bits) {
3401  if (const auto *RTy = QTy->getAs<RecordType>())
3402  return setUsedBits(CGM, RTy, Offset, Bits);
3403 
3404  ASTContext &Context = CGM.getContext();
3405  if (const auto *ATy = Context.getAsConstantArrayType(QTy))
3406  return setUsedBits(CGM, ATy, Offset, Bits);
3407 
3408  int Size = Context.getTypeSizeInChars(QTy).getQuantity();
3409  if (Size <= 0)
3410  return;
3411 
3412  std::fill_n(Bits.begin() + Offset, Size,
3413  (uint64_t(1) << Context.getCharWidth()) - 1);
3414 }
3415 
3416 static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
3417  int Pos, int Size, int CharWidth,
3418  bool BigEndian) {
3419  assert(Size > 0);
3420  uint64_t Mask = 0;
3421  if (BigEndian) {
3422  for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
3423  ++P)
3424  Mask = (Mask << CharWidth) | *P;
3425  } else {
3426  auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3427  do
3428  Mask = (Mask << CharWidth) | *--P;
3429  while (P != End);
3430  }
3431  return Mask;
3432 }
3433 
3434 // Emit code to clear the bits in a record, which aren't a part of any user
3435 // declared member, when the record is a function return.
3436 llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3437  llvm::IntegerType *ITy,
3438  QualType QTy) {
3439  assert(Src->getType() == ITy);
3440  assert(ITy->getScalarSizeInBits() <= 64);
3441 
3442  const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3443  int Size = DataLayout.getTypeStoreSize(ITy);
3444  SmallVector<uint64_t, 4> Bits(Size);
3445  setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3446 
3447  int CharWidth = CGM.getContext().getCharWidth();
3448  uint64_t Mask =
3449  buildMultiCharMask(Bits, 0, Size, CharWidth, DataLayout.isBigEndian());
3450 
3451  return Builder.CreateAnd(Src, Mask, "cmse.clear");
3452 }
3453 
3454 // Emit code to clear the bits in a record, which aren't a part of any user
3455 // declared member, when the record is a function argument.
3456 llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3457  llvm::ArrayType *ATy,
3458  QualType QTy) {
3459  const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3460  int Size = DataLayout.getTypeStoreSize(ATy);
3461  SmallVector<uint64_t, 16> Bits(Size);
3462  setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3463 
3464  // Clear each element of the LLVM array.
3465  int CharWidth = CGM.getContext().getCharWidth();
3466  int CharsPerElt =
3467  ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3468  int MaskIndex = 0;
3469  llvm::Value *R = llvm::UndefValue::get(ATy);
3470  for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
3471  uint64_t Mask = buildMultiCharMask(Bits, MaskIndex, CharsPerElt, CharWidth,
3472  DataLayout.isBigEndian());
3473  MaskIndex += CharsPerElt;
3474  llvm::Value *T0 = Builder.CreateExtractValue(Src, I);
3475  llvm::Value *T1 = Builder.CreateAnd(T0, Mask, "cmse.clear");
3476  R = Builder.CreateInsertValue(R, T1, I);
3477  }
3478 
3479  return R;
3480 }
3481 
3483  bool EmitRetDbgLoc,
3484  SourceLocation EndLoc) {
3485  if (FI.isNoReturn()) {
3486  // Noreturn functions don't return.
3487  EmitUnreachable(EndLoc);
3488  return;
3489  }
3490 
3491  if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
3492  // Naked functions don't have epilogues.
3493  Builder.CreateUnreachable();
3494  return;
3495  }
3496 
3497  // Functions with no result always return void.
3498  if (!ReturnValue.isValid()) {
3499  Builder.CreateRetVoid();
3500  return;
3501  }
3502 
3503  llvm::DebugLoc RetDbgLoc;
3504  llvm::Value *RV = nullptr;
3505  QualType RetTy = FI.getReturnType();
3506  const ABIArgInfo &RetAI = FI.getReturnInfo();
3507 
3508  switch (RetAI.getKind()) {
3509  case ABIArgInfo::InAlloca:
3510  // Aggregrates get evaluated directly into the destination. Sometimes we
3511  // need to return the sret value in a register, though.
3512  assert(hasAggregateEvaluationKind(RetTy));
3513  if (RetAI.getInAllocaSRet()) {
3514  llvm::Function::arg_iterator EI = CurFn->arg_end();
3515  --EI;
3516  llvm::Value *ArgStruct = &*EI;
3517  llvm::Value *SRet = Builder.CreateStructGEP(
3518  FI.getArgStruct(), ArgStruct, RetAI.getInAllocaFieldIndex());
3519  llvm::Type *Ty =
3520  cast<llvm::GetElementPtrInst>(SRet)->getResultElementType();
3521  RV = Builder.CreateAlignedLoad(Ty, SRet, getPointerAlign(), "sret");
3522  }
3523  break;
3524 
3525  case ABIArgInfo::Indirect: {
3526  auto AI = CurFn->arg_begin();
3527  if (RetAI.isSRetAfterThis())
3528  ++AI;
3529  switch (getEvaluationKind(RetTy)) {
3530  case TEK_Complex: {
3531  ComplexPairTy RT =
3532  EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc);
3534  /*isInit*/ true);
3535  break;
3536  }
3537  case TEK_Aggregate:
3538  // Do nothing; aggregrates get evaluated directly into the destination.
3539  break;
3540  case TEK_Scalar: {
3541  LValueBaseInfo BaseInfo;
3542  TBAAAccessInfo TBAAInfo;
3543  CharUnits Alignment =
3544  CGM.getNaturalTypeAlignment(RetTy, &BaseInfo, &TBAAInfo);
3545  Address ArgAddr(&*AI, ConvertType(RetTy), Alignment);
3546  LValue ArgVal =
3547  LValue::MakeAddr(ArgAddr, RetTy, getContext(), BaseInfo, TBAAInfo);
3549  Builder.CreateLoad(ReturnValue), ArgVal, /*isInit*/ true);
3550  break;
3551  }
3552  }
3553  break;
3554  }
3555 
3556  case ABIArgInfo::Extend:
3557  case ABIArgInfo::Direct:
3558  if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
3559  RetAI.getDirectOffset() == 0) {
3560  // The internal return value temp always will have pointer-to-return-type
3561  // type, just do a load.
3562 
3563  // If there is a dominating store to ReturnValue, we can elide
3564  // the load, zap the store, and usually zap the alloca.
3565  if (llvm::StoreInst *SI =
3567  // Reuse the debug location from the store unless there is
3568  // cleanup code to be emitted between the store and return
3569  // instruction.
3570  if (EmitRetDbgLoc && !AutoreleaseResult)
3571  RetDbgLoc = SI->getDebugLoc();
3572  // Get the stored value and nuke the now-dead store.
3573  RV = SI->getValueOperand();
3574  SI->eraseFromParent();
3575 
3576  // Otherwise, we have to do a simple load.
3577  } else {
3579  }
3580  } else {
3581  // If the value is offset in memory, apply the offset now.
3582  Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
3583 
3584  RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
3585  }
3586 
3587  // In ARC, end functions that return a retainable type with a call
3588  // to objc_autoreleaseReturnValue.
3589  if (AutoreleaseResult) {
3590 #ifndef NDEBUG
3591  // Type::isObjCRetainabletype has to be called on a QualType that hasn't
3592  // been stripped of the typedefs, so we cannot use RetTy here. Get the
3593  // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
3594  // CurCodeDecl or BlockInfo.
3595  QualType RT;
3596 
3597  if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
3598  RT = FD->getReturnType();
3599  else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
3600  RT = MD->getReturnType();
3601  else if (isa<BlockDecl>(CurCodeDecl))
3603  else
3604  llvm_unreachable("Unexpected function/method type");
3605 
3606  assert(getLangOpts().ObjCAutoRefCount &&
3607  !FI.isReturnsRetained() &&
3608  RT->isObjCRetainableType());
3609 #endif
3610  RV = emitAutoreleaseOfResult(*this, RV);
3611  }
3612 
3613  break;
3614 
3615  case ABIArgInfo::Ignore:
3616  break;
3617 
3619  auto coercionType = RetAI.getCoerceAndExpandType();
3620 
3621  // Load all of the coerced elements out into results.
3623  Address addr = Builder.CreateElementBitCast(ReturnValue, coercionType);
3624  for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3625  auto coercedEltType = coercionType->getElementType(i);
3626  if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType))
3627  continue;
3628 
3629  auto eltAddr = Builder.CreateStructGEP(addr, i);
3630  auto elt = Builder.CreateLoad(eltAddr);
3631  results.push_back(elt);
3632  }
3633 
3634  // If we have one result, it's the single direct result type.
3635  if (results.size() == 1) {
3636  RV = results[0];
3637 
3638  // Otherwise, we need to make a first-class aggregate.
3639  } else {
3640  // Construct a return type that lacks padding elements.
3641  llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
3642 
3643  RV = llvm::UndefValue::get(returnType);
3644  for (unsigned i = 0, e = results.size(); i != e; ++i) {
3645  RV = Builder.CreateInsertValue(RV, results[i], i);
3646  }
3647  }
3648  break;
3649  }
3650  case ABIArgInfo::Expand:
3652  llvm_unreachable("Invalid ABI kind for return argument");
3653  }
3654 
3655  llvm::Instruction *Ret;
3656  if (RV) {
3657  if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
3658  // For certain return types, clear padding bits, as they may reveal
3659  // sensitive information.
3660  // Small struct/union types are passed as integers.
3661  auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType());
3662  if (ITy != nullptr && isa<RecordType>(RetTy.getCanonicalType()))
3663  RV = EmitCMSEClearRecord(RV, ITy, RetTy);
3664  }
3666  Ret = Builder.CreateRet(RV);
3667  } else {
3668  Ret = Builder.CreateRetVoid();
3669  }
3670 
3671  if (RetDbgLoc)
3672  Ret->setDebugLoc(std::move(RetDbgLoc));
3673 }
3674 
3676  // A current decl may not be available when emitting vtable thunks.
3677  if (!CurCodeDecl)
3678  return;
3679 
3680  // If the return block isn't reachable, neither is this check, so don't emit
3681  // it.
3682  if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
3683  return;
3684 
3685  ReturnsNonNullAttr *RetNNAttr = nullptr;
3686  if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
3687  RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
3688 
3689  if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
3690  return;
3691 
3692  // Prefer the returns_nonnull attribute if it's present.
3693  SourceLocation AttrLoc;
3694  SanitizerMask CheckKind;
3695  SanitizerHandler Handler;
3696  if (RetNNAttr) {
3697  assert(!requiresReturnValueNullabilityCheck() &&
3698  "Cannot check nullability and the nonnull attribute");
3699  AttrLoc = RetNNAttr->getLocation();
3700  CheckKind = SanitizerKind::ReturnsNonnullAttribute;
3701  Handler = SanitizerHandler::NonnullReturn;
3702  } else {
3703  if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl))
3704  if (auto *TSI = DD->getTypeSourceInfo())
3705  if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
3706  AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
3707  CheckKind = SanitizerKind::NullabilityReturn;
3708  Handler = SanitizerHandler::NullabilityReturn;
3709  }
3710 
3711  SanitizerScope SanScope(this);
3712 
3713  // Make sure the "return" source location is valid. If we're checking a
3714  // nullability annotation, make sure the preconditions for the check are met.
3715  llvm::BasicBlock *Check = createBasicBlock("nullcheck");
3716  llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck");
3717  llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load");
3718  llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr);
3719  if (requiresReturnValueNullabilityCheck())
3720  CanNullCheck =
3721  Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);
3722  Builder.CreateCondBr(CanNullCheck, Check, NoCheck);
3723  EmitBlock(Check);
3724 
3725  // Now do the null check.
3726  llvm::Value *Cond = Builder.CreateIsNotNull(RV);
3727  llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)};
3728  llvm::Value *DynamicData[] = {SLocPtr};
3729  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);
3730 
3731  EmitBlock(NoCheck);
3732 
3733 #ifndef NDEBUG
3734  // The return location should not be used after the check has been emitted.
3735  ReturnLocation = Address::invalid();
3736 #endif
3737 }
3738 
3740  const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
3741  return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
3742 }
3743 
3745  QualType Ty) {
3746  // FIXME: Generate IR in one pass, rather than going back and fixing up these
3747  // placeholders.
3748  llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
3749  llvm::Type *IRPtrTy = IRTy->getPointerTo();
3750  llvm::Value *Placeholder = llvm::UndefValue::get(IRPtrTy->getPointerTo());
3751 
3752  // FIXME: When we generate this IR in one pass, we shouldn't need
3753  // this win32-specific alignment hack.
3755  Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align);
3756 
3757  return AggValueSlot::forAddr(Address(Placeholder, IRTy, Align),
3758  Ty.getQualifiers(),
3763 }
3764 
3766  const VarDecl *param,
3767  SourceLocation loc) {
3768  // StartFunction converted the ABI-lowered parameter(s) into a
3769  // local alloca. We need to turn that into an r-value suitable
3770  // for EmitCall.
3771  Address local = GetAddrOfLocalVar(param);
3772 
3773  QualType type = param->getType();
3774 
3776  CGM.ErrorUnsupported(param, "forwarded non-trivially copyable parameter");
3777  }
3778 
3779  // GetAddrOfLocalVar returns a pointer-to-pointer for references,
3780  // but the argument needs to be the original pointer.
3781  if (type->isReferenceType()) {
3782  args.add(RValue::get(Builder.CreateLoad(local)), type);
3783 
3784  // In ARC, move out of consumed arguments so that the release cleanup
3785  // entered by StartFunction doesn't cause an over-release. This isn't
3786  // optimal -O0 code generation, but it should get cleaned up when
3787  // optimization is enabled. This also assumes that delegate calls are
3788  // performed exactly once for a set of arguments, but that should be safe.
3789  } else if (getLangOpts().ObjCAutoRefCount &&
3790  param->hasAttr<NSConsumedAttr>() &&
3791  type->isObjCRetainableType()) {
3792  llvm::Value *ptr = Builder.CreateLoad(local);
3793  auto null =
3794  llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));
3795  Builder.CreateStore(null, local);
3796  args.add(RValue::get(ptr), type);
3797 
3798  // For the most part, we just need to load the alloca, except that
3799  // aggregate r-values are actually pointers to temporaries.
3800  } else {
3801  args.add(convertTempToRValue(local, type, loc), type);
3802  }
3803 
3804  // Deactivate the cleanup for the callee-destructed param that was pushed.
3805  if (type->isRecordType() && !CurFuncIsThunk &&
3806  type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee() &&
3807  param->needsDestruction(getContext())) {
3809  CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
3810  assert(cleanup.isValid() &&
3811  "cleanup for callee-destructed param not recorded");
3812  // This unreachable is a temporary marker which will be removed later.
3813  llvm::Instruction *isActive = Builder.CreateUnreachable();
3814  args.addArgCleanupDeactivation(cleanup, isActive);
3815  }
3816 }
3817 
3818 static bool isProvablyNull(llvm::Value *addr) {
3819  return isa<llvm::ConstantPointerNull>(addr);
3820 }
3821 
3822 /// Emit the actual writing-back of a writeback.
3824  const CallArgList::Writeback &writeback) {
3825  const LValue &srcLV = writeback.Source;
3826  Address srcAddr = srcLV.getAddress(CGF);
3827  assert(!isProvablyNull(srcAddr.getPointer()) &&
3828  "shouldn't have writeback for provably null argument");
3829 
3830  llvm::BasicBlock *contBB = nullptr;
3831 
3832  // If the argument wasn't provably non-null, we need to null check
3833  // before doing the store.
3834  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3835  CGF.CGM.getDataLayout());
3836  if (!provablyNonNull) {
3837  llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
3838  contBB = CGF.createBasicBlock("icr.done");
3839 
3840  llvm::Value *isNull =
3841  CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3842  CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
3843  CGF.EmitBlock(writebackBB);
3844  }
3845 
3846  // Load the value to writeback.
3847  llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
3848 
3849  // Cast it back, in case we're writing an id to a Foo* or something.
3850  value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(),
3851  "icr.writeback-cast");
3852 
3853  // Perform the writeback.
3854 
3855  // If we have a "to use" value, it's something we need to emit a use
3856  // of. This has to be carefully threaded in: if it's done after the
3857  // release it's potentially undefined behavior (and the optimizer
3858  // will ignore it), and if it happens before the retain then the
3859  // optimizer could move the release there.
3860  if (writeback.ToUse) {
3861  assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
3862 
3863  // Retain the new value. No need to block-copy here: the block's
3864  // being passed up the stack.
3865  value = CGF.EmitARCRetainNonBlock(value);
3866 
3867  // Emit the intrinsic use here.
3868  CGF.EmitARCIntrinsicUse(writeback.ToUse);
3869 
3870  // Load the old value (primitively).
3871  llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());
3872 
3873  // Put the new value in place (primitively).
3874  CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false);
3875 
3876  // Release the old value.
3877  CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime());
3878 
3879  // Otherwise, we can just do a normal lvalue store.
3880  } else {
3881  CGF.EmitStoreThroughLValue(RValue::get(value), srcLV);
3882  }
3883 
3884  // Jump to the continuation block.
3885  if (!provablyNonNull)
3886  CGF.EmitBlock(contBB);
3887 }
3888 
3890  const CallArgList &args) {
3891  for (const auto &I : args.writebacks())
3892  emitWriteback(CGF, I);
3893 }
3894 
3896  const CallArgList &CallArgs) {
3898  CallArgs.getCleanupsToDeactivate();
3899  // Iterate in reverse to increase the likelihood of popping the cleanup.
3900  for (const auto &I : llvm::reverse(Cleanups)) {
3901  CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP);
3902  I.IsActiveIP->eraseFromParent();
3903  }
3904 }
3905 
3906 static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
3907  if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens()))
3908  if (uop->getOpcode() == UO_AddrOf)
3909  return uop->getSubExpr();
3910  return nullptr;
3911 }
3912 
3913 /// Emit an argument that's being passed call-by-writeback. That is,
3914 /// we are passing the address of an __autoreleased temporary; it
3915 /// might be copy-initialized with the current value of the given
3916 /// address, but it will definitely be copied out of after the call.
3918  const ObjCIndirectCopyRestoreExpr *CRE) {
3919  LValue srcLV;
3920 
3921  // Make an optimistic effort to emit the address as an l-value.
3922  // This can fail if the argument expression is more complicated.
3923  if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {
3924  srcLV = CGF.EmitLValue(lvExpr);
3925 
3926  // Otherwise, just emit it as a scalar.
3927  } else {
3928  Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr());
3929 
3930  QualType srcAddrType =
3931  CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
3932  srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
3933  }
3934  Address srcAddr = srcLV.getAddress(CGF);
3935 
3936  // The dest and src types don't necessarily match in LLVM terms
3937  // because of the crazy ObjC compatibility rules.
3938 
3939  llvm::PointerType *destType =
3940  cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
3941  llvm::Type *destElemType =
3942  CGF.ConvertTypeForMem(CRE->getType()->getPointeeType());
3943 
3944  // If the address is a constant null, just pass the appropriate null.
3945  if (isProvablyNull(srcAddr.getPointer())) {
3946  args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
3947  CRE->getType());
3948  return;
3949  }
3950 
3951  // Create the temporary.
3952  Address temp =
3953  CGF.CreateTempAlloca(destElemType, CGF.getPointerAlign(), "icr.temp");
3954  // Loading an l-value can introduce a cleanup if the l-value is __weak,
3955  // and that cleanup will be conditional if we can't prove that the l-value
3956  // isn't null, so we need to register a dominating point so that the cleanups
3957  // system will make valid IR.
3959 
3960  // Zero-initialize it if we're not doing a copy-initialization.
3961  bool shouldCopy = CRE->shouldCopy();
3962  if (!shouldCopy) {
3963  llvm::Value *null =
3964  llvm::ConstantPointerNull::get(cast<llvm::PointerType>(destElemType));
3965  CGF.Builder.CreateStore(null, temp);
3966  }
3967 
3968  llvm::BasicBlock *contBB = nullptr;
3969  llvm::BasicBlock *originBB = nullptr;
3970 
3971  // If the address is *not* known to be non-null, we need to switch.
3972  llvm::Value *finalArgument;
3973 
3974  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3975  CGF.CGM.getDataLayout());
3976  if (provablyNonNull) {
3977  finalArgument = temp.getPointer();
3978  } else {
3979  llvm::Value *isNull =
3980  CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3981 
3982  finalArgument = CGF.Builder.CreateSelect(isNull,
3983  llvm::ConstantPointerNull::get(destType),
3984  temp.getPointer(), "icr.argument");
3985 
3986  // If we need to copy, then the load has to be conditional, which
3987  // means we need control flow.
3988  if (shouldCopy) {
3989  originBB = CGF.Builder.GetInsertBlock();
3990  contBB = CGF.createBasicBlock("icr.cont");
3991  llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
3992  CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
3993  CGF.EmitBlock(copyBB);
3994  condEval.begin(CGF);
3995  }
3996  }
3997 
3998  llvm::Value *valueToUse = nullptr;
3999 
4000  // Perform a copy if necessary.
4001  if (shouldCopy) {
4002  RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation());
4003  assert(srcRV.isScalar());
4004 
4005  llvm::Value *src = srcRV.getScalarVal();
4006  src = CGF.Builder.CreateBitCast(src, destElemType, "icr.cast");
4007 
4008  // Use an ordinary store, not a store-to-lvalue.
4009  CGF.Builder.CreateStore(src, temp);
4010 
4011  // If optimization is enabled, and the value was held in a
4012  // __strong variable, we need to tell the optimizer that this
4013  // value has to stay alive until we're doing the store back.
4014  // This is because the temporary is effectively unretained,
4015  // and so otherwise we can violate the high-level semantics.
4016  if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4018  valueToUse = src;
4019  }
4020  }
4021 
4022  // Finish the control flow if we needed it.
4023  if (shouldCopy && !provablyNonNull) {
4024  llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4025  CGF.EmitBlock(contBB);
4026 
4027  // Make a phi for the value to intrinsically use.
4028  if (valueToUse) {
4029  llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2,
4030  "icr.to-use");
4031  phiToUse->addIncoming(valueToUse, copyBB);
4032  phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
4033  originBB);
4034  valueToUse = phiToUse;
4035  }
4036 
4037  condEval.end(CGF);
4038  }
4039 
4040  args.addWriteback(srcLV, temp, valueToUse);
4041  args.add(RValue::get(finalArgument), CRE->getType());
4042 }
4043 
4045  assert(!StackBase);
4046 
4047  // Save the stack.
4048  llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave);
4049  StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save");
4050 }
4051 
4053  if (StackBase) {
4054  // Restore the stack after the call.
4055  llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
4056  CGF.Builder.CreateCall(F, StackBase);
4057  }
4058 }
4059 
4061  SourceLocation ArgLoc,
4062  AbstractCallee AC,
4063  unsigned ParmNum) {
4064  if (!AC.getDecl() || !(SanOpts.has(SanitizerKind::NonnullAttribute) ||
4065  SanOpts.has(SanitizerKind::NullabilityArg)))
4066  return;
4067 
4068  // The param decl may be missing in a variadic function.
4069  auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(ParmNum) : nullptr;
4070  unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
4071 
4072  // Prefer the nonnull attribute if it's present.
4073  const NonNullAttr *NNAttr = nullptr;
4074  if (SanOpts.has(SanitizerKind::NonnullAttribute))
4075  NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo);
4076 
4077  bool CanCheckNullability = false;
4078  if (SanOpts.has(SanitizerKind::NullabilityArg) && !NNAttr && PVD) {
4079  auto Nullability = PVD->getType()->getNullability(getContext());
4080  CanCheckNullability = Nullability &&
4082  PVD->getTypeSourceInfo();
4083  }
4084 
4085  if (!NNAttr && !CanCheckNullability)
4086  return;
4087 
4088  SourceLocation AttrLoc;
4089  SanitizerMask CheckKind;
4090  SanitizerHandler Handler;
4091  if (NNAttr) {
4092  AttrLoc = NNAttr->getLocation();
4093  CheckKind = SanitizerKind::NonnullAttribute;
4094  Handler = SanitizerHandler::NonnullArg;
4095  } else {
4096  AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4097  CheckKind = SanitizerKind::NullabilityArg;
4098  Handler = SanitizerHandler::NullabilityArg;
4099  }
4100 
4101  SanitizerScope SanScope(this);
4102  llvm::Value *Cond = EmitNonNullRValueCheck(RV, ArgType);
4103  llvm::Constant *StaticData[] = {
4105  llvm::ConstantInt::get(Int32Ty, ArgNo + 1),
4106  };
4107  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, None);
4108 }
4109 
4110 // Check if the call is going to use the inalloca convention. This needs to
4111 // agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4112 // later, so we can't check it directly.
4113 static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
4114  ArrayRef<QualType> ArgTypes) {
4115  // The Swift calling conventions don't go through the target-specific
4116  // argument classification, they never use inalloca.
4117  // TODO: Consider limiting inalloca use to only calling conventions supported
4118  // by MSVC.
4119  if (ExplicitCC == CC_Swift || ExplicitCC == CC_SwiftAsync)
4120  return false;
4121  if (!CGM.getTarget().getCXXABI().isMicrosoft())
4122  return false;
4123  return llvm::any_of(ArgTypes, [&](QualType Ty) {
4124  return isInAllocaArgument(CGM.getCXXABI(), Ty);
4125  });
4126 }
4127 
4128 #ifndef NDEBUG
4129 // Determine whether the given argument is an Objective-C method
4130 // that may have type parameters in its signature.
4131 static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
4132  const DeclContext *dc = method->getDeclContext();
4133  if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
4134  return classDecl->getTypeParamListAsWritten();
4135  }
4136 
4137  if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
4138  return catDecl->getTypeParamList();
4139  }
4140 
4141  return false;
4142 }
4143 #endif
4144 
4145 /// EmitCallArgs - Emit call arguments for a function.
4147  CallArgList &Args, PrototypeWrapper Prototype,
4148  llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4149  AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
4150  SmallVector<QualType, 16> ArgTypes;
4151 
4152  assert((ParamsToSkip == 0 || Prototype.P) &&
4153  "Can't skip parameters if type info is not provided");
4154 
4155  // This variable only captures *explicitly* written conventions, not those
4156  // applied by default via command line flags or target defaults, such as
4157  // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
4158  // require knowing if this is a C++ instance method or being able to see
4159  // unprototyped FunctionTypes.
4160  CallingConv ExplicitCC = CC_C;
4161 
4162  // First, if a prototype was provided, use those argument types.
4163  bool IsVariadic = false;
4164  if (Prototype.P) {
4165  const auto *MD = Prototype.P.dyn_cast<const ObjCMethodDecl *>();
4166  if (MD) {
4167  IsVariadic = MD->isVariadic();
4168  ExplicitCC = getCallingConventionForDecl(
4169  MD, CGM.getTarget().getTriple().isOSWindows());
4170  ArgTypes.assign(MD->param_type_begin() + ParamsToSkip,
4171  MD->param_type_end());
4172  } else {
4173  const auto *FPT = Prototype.P.get<const FunctionProtoType *>();
4174  IsVariadic = FPT->isVariadic();
4175  ExplicitCC = FPT->getExtInfo().getCC();
4176  ArgTypes.assign(FPT->param_type_begin() + ParamsToSkip,
4177  FPT->param_type_end());
4178  }
4179 
4180 #ifndef NDEBUG
4181  // Check that the prototyped types match the argument expression types.
4182  bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD);
4183  CallExpr::const_arg_iterator Arg = ArgRange.begin();
4184  for (QualType Ty : ArgTypes) {
4185  assert(Arg != ArgRange.end() && "Running over edge of argument list!");
4186  assert(
4187  (isGenericMethod || Ty->isVariablyModifiedType() ||
4189  getContext()
4190  .getCanonicalType(Ty.getNonReferenceType())
4191  .getTypePtr() ==
4192  getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&
4193  "type mismatch in call argument!");
4194  ++Arg;
4195  }
4196 
4197  // Either we've emitted all the call args, or we have a call to variadic
4198  // function.
4199  assert((Arg == ArgRange.end() || IsVariadic) &&
4200  "Extra arguments in non-variadic function!");
4201 #endif
4202  }
4203 
4204  // If we still have any arguments, emit them using the type of the argument.
4205  for (auto *A : llvm::drop_begin(ArgRange, ArgTypes.size()))
4206  ArgTypes.push_back(IsVariadic ? getVarArgType(A) : A->getType());
4207  assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4208 
4209  // We must evaluate arguments from right to left in the MS C++ ABI,
4210  // because arguments are destroyed left to right in the callee. As a special
4211  // case, there are certain language constructs that require left-to-right
4212  // evaluation, and in those cases we consider the evaluation order requirement
4213  // to trump the "destruction order is reverse construction order" guarantee.
4214  bool LeftToRight =
4218 
4219  auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
4220  RValue EmittedArg) {
4221  if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
4222  return;
4223  auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4224  if (PS == nullptr)
4225  return;
4226 
4227  const auto &Context = getContext();
4228  auto SizeTy = Context.getSizeType();
4229  auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
4230  assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
4231  llvm::Value *V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T,
4232  EmittedArg.getScalarVal(),
4233  PS->isDynamic());
4234  Args.add(RValue::get(V), SizeTy);
4235  // If we're emitting args in reverse, be sure to do so with
4236  // pass_object_size, as well.
4237  if (!LeftToRight)
4238  std::swap(Args.back(), *(&Args.back() - 1));
4239  };
4240 
4241  // Insert a stack save if we're going to need any inalloca args.
4242  if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
4243  assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4244  "inalloca only supported on x86");
4245  Args.allocateArgumentMemory(*this);
4246  }
4247 
4248  // Evaluate each argument in the appropriate order.
4249  size_t CallArgsStart = Args.size();
4250  for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
4251  unsigned Idx = LeftToRight ? I : E - I - 1;
4252  CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
4253  unsigned InitialArgSize = Args.size();
4254  // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
4255  // the argument and parameter match or the objc method is parameterized.
4256  assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
4257  getContext().hasSameUnqualifiedType((*Arg)->getType(),
4258  ArgTypes[Idx]) ||
4259  (isa<ObjCMethodDecl>(AC.getDecl()) &&
4260  isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
4261  "Argument and parameter types don't match");
4262  EmitCallArg(Args, *Arg, ArgTypes[Idx]);
4263  // In particular, we depend on it being the last arg in Args, and the
4264  // objectsize bits depend on there only being one arg if !LeftToRight.
4265  assert(InitialArgSize + 1 == Args.size() &&
4266  "The code below depends on only adding one arg per EmitCallArg");
4267  (void)InitialArgSize;
4268  // Since pointer argument are never emitted as LValue, it is safe to emit
4269  // non-null argument check for r-value only.
4270  if (!Args.back().hasLValue()) {
4271  RValue RVArg = Args.back().getKnownRValue();
4272  EmitNonNullArgCheck(RVArg, ArgTypes[Idx], (*Arg)->getExprLoc(), AC,
4273  ParamsToSkip + Idx);
4274  // @llvm.objectsize should never have side-effects and shouldn't need
4275  // destruction/cleanups, so we can safely "emit" it after its arg,
4276  // regardless of right-to-leftness
4277  MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4278  }
4279  }
4280 
4281  if (!LeftToRight) {
4282  // Un-reverse the arguments we just evaluated so they match up with the LLVM
4283  // IR function.
4284  std::reverse(Args.begin() + CallArgsStart, Args.end());
4285  }
4286 }
4287 
4288 namespace {
4289 
4290 struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
4291  DestroyUnpassedArg(Address Addr, QualType Ty)
4292  : Addr(Addr), Ty(Ty) {}
4293 
4294  Address Addr;
4295  QualType Ty;
4296 
4297  void Emit(CodeGenFunction &CGF, Flags flags) override {
4299  if (DtorKind == QualType::DK_cxx_destructor) {
4300  const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
4301  assert(!Dtor->isTrivial());
4302  CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,
4303  /*Delegating=*/false, Addr, Ty);
4304  } else {
4305  CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty));
4306  }
4307  }
4308 };
4309 
4310 struct DisableDebugLocationUpdates {
4311  CodeGenFunction &CGF;
4312  bool disabledDebugInfo;
4313  DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) {
4314  if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && CGF.getDebugInfo()))
4315  CGF.disableDebugInfo();
4316  }
4317  ~DisableDebugLocationUpdates() {
4318  if (disabledDebugInfo)
4319  CGF.enableDebugInfo();
4320  }
4321 };
4322 
4323 } // end anonymous namespace
4324 
4326  if (!HasLV)
4327  return RV;
4328  LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty);
4330  LV.isVolatile());
4331  IsUsed = true;
4332  return RValue::getAggregate(Copy.getAddress(CGF));
4333 }
4334 
4336  LValue Dst = CGF.MakeAddrLValue(Addr, Ty);
4337  if (!HasLV && RV.isScalar())
4338  CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true);
4339  else if (!HasLV && RV.isComplex())
4340  CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true);
4341  else {
4342  auto Addr = HasLV ? LV.getAddress(CGF) : RV.getAggregateAddress();
4343  LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty);
4344  // We assume that call args are never copied into subobjects.
4346  HasLV ? LV.isVolatileQualified()
4347  : RV.isVolatileQualified());
4348  }
4349  IsUsed = true;
4350 }
4351 
4353  QualType type) {
4354  DisableDebugLocationUpdates Dis(*this, E);
4355  if (const ObjCIndirectCopyRestoreExpr *CRE
4356  = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
4357  assert(getLangOpts().ObjCAutoRefCount);
4358  return emitWritebackArg(*this, args, CRE);
4359  }
4360 
4361  assert(type->isReferenceType() == E->isGLValue() &&
4362  "reference binding to unmaterialized r-value!");
4363 
4364  if (E->isGLValue()) {
4365  assert(E->getObjectKind() == OK_Ordinary);
4366  return args.add(EmitReferenceBindingToExpr(E), type);
4367  }
4368 
4369  bool HasAggregateEvalKind = hasAggregateEvaluationKind(type);
4370 
4371  // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
4372  // However, we still have to push an EH-only cleanup in case we unwind before
4373  // we make it to the call.
4374  if (type->isRecordType() &&
4375  type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
4376  // If we're using inalloca, use the argument memory. Otherwise, use a
4377  // temporary.
4378  AggValueSlot Slot = args.isUsingInAlloca()
4379  ? createPlaceholderSlot(*this, type) : CreateAggTemp(type, "agg.tmp");
4380 
4381  bool DestroyedInCallee = true, NeedsEHCleanup = true;
4382  if (const auto *RD = type->getAsCXXRecordDecl())
4383  DestroyedInCallee = RD->hasNonTrivialDestructor();
4384  else
4385  NeedsEHCleanup = needsEHCleanup(type.isDestructedType());
4386 
4387  if (DestroyedInCallee)
4388  Slot.setExternallyDestructed();
4389 
4390  EmitAggExpr(E, Slot);
4391  RValue RV = Slot.asRValue();
4392  args.add(RV, type);
4393 
4394  if (DestroyedInCallee && NeedsEHCleanup) {
4395  // Create a no-op GEP between the placeholder and the cleanup so we can
4396  // RAUW it successfully. It also serves as a marker of the first
4397  // instruction where the cleanup is active.
4398  pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(),
4399  type);
4400  // This unreachable is a temporary marker which will be removed later.
4401  llvm::Instruction *IsActive = Builder.CreateUnreachable();
4403  }
4404  return;
4405  }
4406 
4407  if (HasAggregateEvalKind && isa<ImplicitCastExpr>(E) &&
4408  cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue) {
4409  LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
4410  assert(L.isSimple());
4411  args.addUncopiedAggregate(L, type);
4412  return;
4413  }
4414 
4415  args.add(EmitAnyExprToTemp(E), type);
4416 }
4417 
4418 QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
4419  // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
4420  // implicitly widens null pointer constants that are arguments to varargs
4421  // functions to pointer-sized ints.
4422  if (!getTarget().getTriple().isOSWindows())
4423  return Arg->getType();
4424 
4425  if (Arg->getType()->isIntegerType() &&
4426  getContext().getTypeSize(Arg->getType()) <
4427  getContext().getTargetInfo().getPointerWidth(0) &&
4430  return getContext().getIntPtrType();
4431  }
4432 
4433  return Arg->getType();
4434 }
4435 
4436 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4437 // optimizer it can aggressively ignore unwind edges.
4438 void
4439 CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
4440  if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4441  !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
4442  Inst->setMetadata("clang.arc.no_objc_arc_exceptions",
4444 }
4445 
4446 /// Emits a call to the given no-arguments nounwind runtime function.
4447 llvm::CallInst *
4448 CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4449  const llvm::Twine &name) {
4450  return EmitNounwindRuntimeCall(callee, None, name);
4451 }
4452 
4453 /// Emits a call to the given nounwind runtime function.
4454 llvm::CallInst *
4455 CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4457  const llvm::Twine &name) {
4458  llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
4459  call->setDoesNotThrow();
4460  return call;
4461 }
4462 
4463 /// Emits a simple call (never an invoke) to the given no-arguments
4464 /// runtime function.
4465 llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4466  const llvm::Twine &name) {
4467  return EmitRuntimeCall(callee, None, name);
4468 }
4469 
4470 // Calls which may throw must have operand bundles indicating which funclet
4471 // they are nested within.
4475  // There is no need for a funclet operand bundle if we aren't inside a
4476  // funclet.
4477  if (!CurrentFuncletPad)
4478  return BundleList;
4479 
4480  // Skip intrinsics which cannot throw.
4481  auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts());
4482  if (CalleeFn && CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow())
4483  return BundleList;
4484 
4485  BundleList.emplace_back("funclet", CurrentFuncletPad);
4486  return BundleList;
4487 }
4488 
4489 /// Emits a simple call (never an invoke) to the given runtime function.
4490 llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4492  const llvm::Twine &name) {
4493  llvm::CallInst *call = Builder.CreateCall(
4494  callee, args, getBundlesForFunclet(callee.getCallee()), name);
4495  call->setCallingConv(getRuntimeCC());
4496  return call;
4497 }
4498 
4499 /// Emits a call or invoke to the given noreturn runtime function.
4501  llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
4503  getBundlesForFunclet(callee.getCallee());
4504 
4505  if (getInvokeDest()) {
4506  llvm::InvokeInst *invoke =
4507  Builder.CreateInvoke(callee,
4509  getInvokeDest(),
4510  args,
4511  BundleList);
4512  invoke->setDoesNotReturn();
4513  invoke->setCallingConv(getRuntimeCC());
4514  } else {
4515  llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList);
4516  call->setDoesNotReturn();
4517  call->setCallingConv(getRuntimeCC());
4518  Builder.CreateUnreachable();
4519  }
4520 }
4521 
4522 /// Emits a call or invoke instruction to the given nullary runtime function.
4523 llvm::CallBase *
4524 CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4525  const Twine &name) {
4526  return EmitRuntimeCallOrInvoke(callee, None, name);
4527 }
4528 
4529 /// Emits a call or invoke instruction to the given runtime function.
4530 llvm::CallBase *
4531 CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4533  const Twine &name) {
4534  llvm::CallBase *call = EmitCallOrInvoke(callee, args, name);
4535  call->setCallingConv(getRuntimeCC());
4536  return call;
4537 }
4538 
4539 /// Emits a call or invoke instruction to the given function, depending
4540 /// on the current state of the EH stack.
4541 llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
4543  const Twine &Name) {
4544  llvm::BasicBlock *InvokeDest = getInvokeDest();
4546  getBundlesForFunclet(Callee.getCallee());
4547 
4548  llvm::CallBase *Inst;
4549  if (!InvokeDest)
4550  Inst = Builder.CreateCall(Callee, Args, BundleList, Name);
4551  else {
4552  llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
4553  Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList,
4554  Name);
4555  EmitBlock(ContBB);
4556  }
4557 
4558  // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4559  // optimizer it can aggressively ignore unwind edges.
4560  if (CGM.getLangOpts().ObjCAutoRefCount)
4561  AddObjCARCExceptionMetadata(Inst);
4562 
4563  return Inst;
4564 }
4565 
4566 void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
4567  llvm::Value *New) {
4568  DeferredReplacements.push_back(
4569  std::make_pair(llvm::WeakTrackingVH(Old), New));
4570 }
4571 
4572 namespace {
4573 
4574 /// Specify given \p NewAlign as the alignment of return value attribute. If
4575 /// such attribute already exists, re-set it to the maximal one of two options.
4576 LLVM_NODISCARD llvm::AttributeList
4577 maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
4578  const llvm::AttributeList &Attrs,
4579  llvm::Align NewAlign) {
4580  llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
4581  if (CurAlign >= NewAlign)
4582  return Attrs;
4583  llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign);
4584  return Attrs.removeRetAttribute(Ctx, llvm::Attribute::AttrKind::Alignment)
4585  .addRetAttribute(Ctx, AlignAttr);
4586 }
4587 
4588 template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
4589 protected:
4590  CodeGenFunction &CGF;
4591 
4592  /// We do nothing if this is, or becomes, nullptr.
4593  const AlignedAttrTy *AA = nullptr;
4594 
4595  llvm::Value *Alignment = nullptr; // May or may not be a constant.
4596  llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
4597 
4598  AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4599  : CGF(CGF_) {
4600  if (!FuncDecl)
4601  return;
4602  AA = FuncDecl->getAttr<AlignedAttrTy>();
4603  }
4604 
4605 public:
4606  /// If we can, materialize the alignment as an attribute on return value.
4607  LLVM_NODISCARD llvm::AttributeList
4608  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4609  if (!AA || OffsetCI || CGF.SanOpts.has(SanitizerKind::Alignment))
4610  return Attrs;
4611  const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4612  if (!AlignmentCI)
4613  return Attrs;
4614  // We may legitimately have non-power-of-2 alignment here.
4615  // If so, this is UB land, emit it via `@llvm.assume` instead.
4616  if (!AlignmentCI->getValue().isPowerOf2())
4617  return Attrs;
4618  llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4619  CGF.getLLVMContext(), Attrs,
4620  llvm::Align(
4621  AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4622  AA = nullptr; // We're done. Disallow doing anything else.
4623  return NewAttrs;
4624  }
4625 
4626  /// Emit alignment assumption.
4627  /// This is a general fallback that we take if either there is an offset,
4628  /// or the alignment is variable or we are sanitizing for alignment.
4629  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4630  if (!AA)
4631  return;
4632  CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4633  AA->getLocation(), Alignment, OffsetCI);
4634  AA = nullptr; // We're done. Disallow doing anything else.
4635  }
4636 };
4637 
4638 /// Helper data structure to emit `AssumeAlignedAttr`.
4639 class AssumeAlignedAttrEmitter final
4640  : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
4641 public:
4642  AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4643  : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4644  if (!AA)
4645  return;
4646  // It is guaranteed that the alignment/offset are constants.
4647  Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment()));
4648  if (Expr *Offset = AA->getOffset()) {
4649  OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset));
4650  if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
4651  OffsetCI = nullptr;
4652  }
4653  }
4654 };
4655 
4656 /// Helper data structure to emit `AllocAlignAttr`.
4657 class AllocAlignAttrEmitter final
4658  : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
4659 public:
4660  AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
4661  const CallArgList &CallArgs)
4662  : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4663  if (!AA)
4664  return;
4665  // Alignment may or may not be a constant, and that is okay.
4666  Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
4667  .getRValue(CGF)
4668  .getScalarVal();
4669  }
4670 };
4671 
4672 } // namespace
4673 
4674 static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
4675  if (auto *VT = dyn_cast<llvm::VectorType>(Ty))
4676  return VT->getPrimitiveSizeInBits().getKnownMinSize();
4677  if (auto *AT = dyn_cast<llvm::ArrayType>(Ty))
4678  return getMaxVectorWidth(AT->getElementType());
4679 
4680  unsigned MaxVectorWidth = 0;
4681  if (auto *ST = dyn_cast<llvm::StructType>(Ty))
4682  for (auto *I : ST->elements())
4683  MaxVectorWidth = std::max(MaxVectorWidth, getMaxVectorWidth(I));
4684  return MaxVectorWidth;
4685 }
4686 
4688  const CGCallee &Callee,
4689  ReturnValueSlot ReturnValue,
4690  const CallArgList &CallArgs,
4691  llvm::CallBase **callOrInvoke, bool IsMustTail,
4692  SourceLocation Loc) {
4693  // FIXME: We no longer need the types from CallArgs; lift up and simplify.
4694 
4695  assert(Callee.isOrdinary() || Callee.isVirtual());
4696 
4697  // Handle struct-return functions by passing a pointer to the
4698  // location that we would like to return into.
4699  QualType RetTy = CallInfo.getReturnType();
4700  const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
4701 
4702  llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo);
4703 
4704  const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
4705  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
4706  // We can only guarantee that a function is called from the correct
4707  // context/function based on the appropriate target attributes,
4708  // so only check in the case where we have both always_inline and target
4709  // since otherwise we could be making a conditional call after a check for
4710  // the proper cpu features (and it won't cause code generation issues due to
4711  // function based code generation).
4712  if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
4713  TargetDecl->hasAttr<TargetAttr>())
4714  checkTargetFeatures(Loc, FD);
4715 
4716  // Some architectures (such as x86-64) have the ABI changed based on
4717  // attribute-target/features. Give them a chance to diagnose.
4719  CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl), FD, CallArgs);
4720  }
4721 
4722 #ifndef NDEBUG
4723  if (!(CallInfo.isVariadic() && CallInfo.getArgStruct())) {
4724  // For an inalloca varargs function, we don't expect CallInfo to match the
4725  // function pointer's type, because the inalloca struct a will have extra
4726  // fields in it for the varargs parameters. Code later in this function
4727  // bitcasts the function pointer to the type derived from CallInfo.
4728  //
4729  // In other cases, we assert that the types match up (until pointers stop
4730  // having pointee types).
4731  if (Callee.isVirtual())
4732  assert(IRFuncTy == Callee.getVirtualFunctionType());
4733  else {
4734  llvm::PointerType *PtrTy =
4735  llvm::cast<llvm::PointerType>(Callee.getFunctionPointer()->getType());
4736  assert(PtrTy->isOpaqueOrPointeeTypeMatches(IRFuncTy));
4737  }
4738  }
4739 #endif
4740 
4741  // 1. Set up the arguments.
4742 
4743  // If we're using inalloca, insert the allocation after the stack save.
4744  // FIXME: Do this earlier rather than hacking it in here!
4745  Address ArgMemory = Address::invalid();
4746  if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
4747  const llvm::DataLayout &DL = CGM.getDataLayout();
4748  llvm::Instruction *IP = CallArgs.getStackBase();
4749  llvm::AllocaInst *AI;
4750  if (IP) {
4751  IP = IP->getNextNode();
4752  AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(),
4753  "argmem", IP);
4754  } else {
4755  AI = CreateTempAlloca(ArgStruct, "argmem");
4756  }
4757  auto Align = CallInfo.getArgStructAlignment();
4758  AI->setAlignment(Align.getAsAlign());
4759  AI->setUsedWithInAlloca(true);
4760  assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
4761  ArgMemory = Address(AI, ArgStruct, Align);
4762  }
4763 
4764  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
4765  SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
4766 
4767  // If the call returns a temporary with struct return, create a temporary
4768  // alloca to hold the result, unless one is given to us.
4769  Address SRetPtr = Address::invalid();
4770  Address SRetAlloca = Address::invalid();
4771  llvm::Value *UnusedReturnSizePtr = nullptr;
4772  if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
4773  if (!ReturnValue.isNull()) {
4774  SRetPtr = ReturnValue.getValue();
4775  } else {
4776  SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca);
4777  if (HaveInsertPoint() && ReturnValue.isUnused()) {
4778  llvm::TypeSize size =
4779  CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy));
4780  UnusedReturnSizePtr = EmitLifetimeStart(size, SRetAlloca.getPointer());
4781  }
4782  }
4783  if (IRFunctionArgs.hasSRetArg()) {
4784  IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer();
4785  } else if (RetAI.isInAlloca()) {
4786  Address Addr =
4787  Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
4788  Builder.CreateStore(SRetPtr.getPointer(), Addr);
4789  }
4790  }
4791 
4792  Address swiftErrorTemp = Address::invalid();
4793  Address swiftErrorArg = Address::invalid();
4794 
4795  // When passing arguments using temporary allocas, we need to add the
4796  // appropriate lifetime markers. This vector keeps track of all the lifetime
4797  // markers that need to be ended right after the call.
4798  SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
4799 
4800  // Translate all of the arguments as necessary to match the IR lowering.
4801  assert(CallInfo.arg_size() == CallArgs.size() &&
4802  "Mismatch between function signature & arguments.");
4803  unsigned ArgNo = 0;
4804  CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
4805  for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
4806  I != E; ++I, ++info_it, ++ArgNo) {
4807  const ABIArgInfo &ArgInfo = info_it->info;
4808 
4809  // Insert a padding argument to ensure proper alignment.
4810  if (IRFunctionArgs.hasPaddingArg(ArgNo))
4811  IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
4812  llvm::UndefValue::get(ArgInfo.getPaddingType());
4813 
4814  unsigned FirstIRArg, NumIRArgs;
4815  std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
4816 
4817  switch (ArgInfo.getKind()) {
4818  case ABIArgInfo::InAlloca: {
4819  assert(NumIRArgs == 0);
4820  assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
4821  if (I->isAggregate()) {
4822  Address Addr = I->hasLValue()
4823  ? I->getKnownLValue().getAddress(*this)
4824  : I->getKnownRValue().getAggregateAddress();
4825  llvm::Instruction *Placeholder =
4826  cast<llvm::Instruction>(Addr.getPointer());
4827 
4828  if (!ArgInfo.getInAllocaIndirect()) {
4829  // Replace the placeholder with the appropriate argument slot GEP.
4830  CGBuilderTy::InsertPoint IP = Builder.saveIP();
4831  Builder.SetInsertPoint(Placeholder);
4832  Addr = Builder.CreateStructGEP(ArgMemory,
4833  ArgInfo.getInAllocaFieldIndex());
4834  Builder.restoreIP(IP);
4835  } else {
4836  // For indirect things such as overaligned structs, replace the
4837  // placeholder with a regular aggregate temporary alloca. Store the
4838  // address of this alloca into the struct.
4839  Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp");
4840  Address ArgSlot = Builder.CreateStructGEP(
4841  ArgMemory, ArgInfo.getInAllocaFieldIndex());
4842  Builder.CreateStore(Addr.getPointer(), ArgSlot);
4843  }
4844  deferPlaceholderReplacement(Placeholder, Addr.getPointer());
4845  } else if (ArgInfo.getInAllocaIndirect()) {
4846  // Make a temporary alloca and store the address of it into the argument
4847  // struct.
4849  I->Ty, getContext().getTypeAlignInChars(I->Ty),
4850  "indirect-arg-temp");
4851  I->copyInto(*this, Addr);
4852  Address ArgSlot =
4853  Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
4854  Builder.CreateStore(Addr.getPointer(), ArgSlot);
4855  } else {
4856  // Store the RValue into the argument struct.
4857  Address Addr =
4858  Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
4859  // There are some cases where a trivial bitcast is not avoidable. The
4860  // definition of a type later in a translation unit may change it's type
4861  // from {}* to (%struct.foo*)*.
4862  Addr = Builder.CreateElementBitCast(Addr, ConvertTypeForMem(I->Ty));
4863  I->copyInto(*this, Addr);
4864  }
4865  break;
4866  }
4867 
4868  case ABIArgInfo::Indirect:
4870  assert(NumIRArgs == 1);
4871  if (!I->isAggregate()) {
4872  // Make a temporary alloca to pass the argument.
4874  I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp");
4875  IRCallArgs[FirstIRArg] = Addr.getPointer();
4876 
4877  I->copyInto(*this, Addr);
4878  } else {
4879  // We want to avoid creating an unnecessary temporary+copy here;
4880  // however, we need one in three cases:
4881  // 1. If the argument is not byval, and we are required to copy the
4882  // source. (This case doesn't occur on any common architecture.)
4883  // 2. If the argument is byval, RV is not sufficiently aligned, and
4884  // we cannot force it to be sufficiently aligned.
4885  // 3. If the argument is byval, but RV is not located in default
4886  // or alloca address space.
4887  Address Addr = I->hasLValue()
4888  ? I->getKnownLValue().getAddress(*this)
4889  : I->getKnownRValue().getAggregateAddress();
4890  llvm::Value *V = Addr.getPointer();
4891  CharUnits Align = ArgInfo.getIndirectAlign();
4892  const llvm::DataLayout *TD = &CGM.getDataLayout();
4893 
4894  assert((FirstIRArg >= IRFuncTy->getNumParams() ||
4895  IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
4896  TD->getAllocaAddrSpace()) &&
4897  "indirect argument must be in alloca address space");
4898 
4899  bool NeedCopy = false;
4900 
4901  if (Addr.getAlignment() < Align &&
4902  llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) <
4903  Align.getAsAlign()) {
4904  NeedCopy = true;
4905  } else if (I->hasLValue()) {
4906  auto LV = I->getKnownLValue();
4907  auto AS = LV.getAddressSpace();
4908 
4909  if (!ArgInfo.getIndirectByVal() ||
4910  (LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))) {
4911  NeedCopy = true;
4912  }
4913  if (!getLangOpts().OpenCL) {
4914  if ((ArgInfo.getIndirectByVal() &&
4915  (AS != LangAS::Default &&
4916  AS != CGM.getASTAllocaAddressSpace()))) {
4917  NeedCopy = true;
4918  }
4919  }
4920  // For OpenCL even if RV is located in default or alloca address space
4921  // we don't want to perform address space cast for it.
4922  else if ((ArgInfo.getIndirectByVal() &&
4923  Addr.getType()->getAddressSpace() != IRFuncTy->
4924  getParamType(FirstIRArg)->getPointerAddressSpace())) {
4925  NeedCopy = true;
4926  }
4927  }
4928 
4929  if (NeedCopy) {
4930  // Create an aligned temporary, and copy to it.
4932  I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
4933  IRCallArgs[FirstIRArg] = AI.getPointer();
4934 
4935  // Emit lifetime markers for the temporary alloca.
4936  llvm::TypeSize ByvalTempElementSize =
4937  CGM.getDataLayout().getTypeAllocSize(AI.getElementType());
4938  llvm::Value *LifetimeSize =
4939  EmitLifetimeStart(ByvalTempElementSize, AI.getPointer());
4940 
4941  // Add cleanup code to emit the end lifetime marker after the call.
4942  if (LifetimeSize) // In case we disabled lifetime markers.
4943  CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize);
4944 
4945  // Generate the copy.
4946  I->copyInto(*this, AI);
4947  } else {
4948  // Skip the extra memcpy call.
4949  auto *T = llvm::PointerType::getWithSamePointeeType(
4950  cast<llvm::PointerType>(V->getType()),
4951  CGM.getDataLayout().getAllocaAddrSpace());
4952  IRCallArgs[FirstIRArg] = getTargetHooks().performAddrSpaceCast(
4954  true);
4955  }
4956  }
4957  break;
4958  }
4959 
4960  case ABIArgInfo::Ignore:
4961  assert(NumIRArgs == 0);
4962  break;
4963 
4964  case ABIArgInfo::Extend:
4965  case ABIArgInfo::Direct: {
4966  if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
4967  ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
4968  ArgInfo.getDirectOffset() == 0) {
4969  assert(NumIRArgs == 1);
4970  llvm::Value *V;
4971  if (!I->isAggregate())
4972  V = I->getKnownRValue().getScalarVal();
4973  else
4974  V = Builder.CreateLoad(
4975  I->hasLValue() ? I->getKnownLValue().getAddress(*this)
4976  : I->getKnownRValue().getAggregateAddress());
4977 
4978  // Implement swifterror by copying into a new swifterror argument.
4979  // We'll write back in the normal path out of the call.
4980  if (CallInfo.getExtParameterInfo(ArgNo).getABI()
4982  assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
4983 
4984  QualType pointeeTy = I->Ty->getPointeeType();
4985  swiftErrorArg = Address(V, ConvertTypeForMem(pointeeTy),
4986  getContext().getTypeAlignInChars(pointeeTy));
4987 
4988  swiftErrorTemp =
4989  CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
4990  V = swiftErrorTemp.getPointer();
4991  cast<llvm::AllocaInst>(V)->setSwiftError(true);
4992 
4993  llvm::Value *errorValue = Builder.CreateLoad(swiftErrorArg);
4994  Builder.CreateStore(errorValue, swiftErrorTemp);
4995  }
4996 
4997  // We might have to widen integers, but we should never truncate.
4998  if (ArgInfo.getCoerceToType() != V->getType() &&
4999  V->getType()->isIntegerTy())
5000  V = Builder.CreateZExt(V, ArgInfo.getCoerceToType());
5001 
5002  // If the argument doesn't match, perform a bitcast to coerce it. This
5003  // can happen due to trivial type mismatches.
5004  if (FirstIRArg < IRFuncTy->getNumParams() &&
5005  V->getType() != IRFuncTy->getParamType(FirstIRArg))
5006  V = Builder.CreateBitCast(V, IRFuncTy->getParamType(FirstIRArg));
5007 
5008  IRCallArgs[FirstIRArg] = V;
5009  break;
5010  }
5011 
5012  // FIXME: Avoid the conversion through memory if possible.
5013  Address Src = Address::invalid();
5014  if (!I->isAggregate()) {
5015  Src = CreateMemTemp(I->Ty, "coerce");
5016  I->copyInto(*this, Src);
5017  } else {
5018  Src = I->hasLValue() ? I->getKnownLValue().getAddress(*this)
5019  : I->getKnownRValue().getAggregateAddress();
5020  }
5021 
5022  // If the value is offset in memory, apply the offset now.
5023  Src = emitAddressAtOffset(*this, Src, ArgInfo);
5024 
5025  // Fast-isel and the optimizer generally like scalar values better than
5026  // FCAs, so we flatten them if this is safe to do for this argument.
5027  llvm::StructType *STy =
5028  dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
5029  if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
5030  llvm::Type *SrcTy = Src.getElementType();
5031  uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
5032  uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
5033 
5034  // If the source type is smaller than the destination type of the
5035  // coerce-to logic, copy the source value into a temp alloca the size
5036  // of the destination type to allow loading all of it. The bits past
5037  // the source value are left undef.
5038  if (SrcSize < DstSize) {
5039  Address TempAlloca
5040  = CreateTempAlloca(STy, Src.getAlignment(),
5041  Src.getName() + ".coerce");
5042  Builder.CreateMemCpy(TempAlloca, Src, SrcSize);
5043  Src = TempAlloca;
5044  } else {
5045  Src = Builder.CreateElementBitCast(Src, STy);
5046  }
5047 
5048  assert(NumIRArgs == STy->getNumElements());
5049  for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5050  Address EltPtr = Builder.CreateStructGEP(Src, i);
5051  llvm::Value *LI = Builder.CreateLoad(EltPtr);
5052  IRCallArgs[FirstIRArg + i] = LI;
5053  }
5054  } else {
5055  // In the simple case, just pass the coerced loaded value.
5056  assert(NumIRArgs == 1);
5057  llvm::Value *Load =
5058  CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);
5059 
5060  if (CallInfo.isCmseNSCall()) {
5061  // For certain parameter types, clear padding bits, as they may reveal
5062  // sensitive information.
5063  // Small struct/union types are passed as integer arrays.
5064  auto *ATy = dyn_cast<llvm::ArrayType>(Load->getType());
5065  if (ATy != nullptr && isa<RecordType>(I->Ty.getCanonicalType()))
5066  Load = EmitCMSEClearRecord(Load, ATy, I->Ty);
5067  }
5068  IRCallArgs[FirstIRArg] = Load;
5069  }
5070 
5071  break;
5072  }
5073 
5075  auto coercionType = ArgInfo.getCoerceAndExpandType();
5076  auto layout = CGM.getDataLayout().getStructLayout(coercionType);
5077 
5078  llvm::Value *tempSize = nullptr;
5079  Address addr = Address::invalid();
5080  Address AllocaAddr = Address::invalid();
5081  if (I->isAggregate()) {
5082  addr = I->hasLValue() ? I->getKnownLValue().getAddress(*this)
5083  : I->getKnownRValue().getAggregateAddress();
5084 
5085  } else {
5086  RValue RV = I->getKnownRValue();
5087  assert(RV.isScalar()); // complex should always just be direct
5088 
5089  llvm::Type *scalarType = RV.getScalarVal()->getType();
5090  auto scalarSize = CGM.getDataLayout().getTypeAllocSize(scalarType);
5091  auto scalarAlign = CGM.getDataLayout().getPrefTypeAlignment(scalarType);
5092 
5093  // Materialize to a temporary.
5094  addr =
5095  CreateTempAlloca(RV.getScalarVal()->getType(),
5097  layout->getAlignment().value(), scalarAlign)),
5098  "tmp",
5099  /*ArraySize=*/nullptr, &AllocaAddr);
5100  tempSize = EmitLifetimeStart(scalarSize, AllocaAddr.getPointer());
5101 
5102  Builder.CreateStore(RV.getScalarVal(), addr);
5103  }
5104 
5105  addr = Builder.CreateElementBitCast(addr, coercionType);
5106 
5107  unsigned IRArgPos = FirstIRArg;
5108  for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
5109  llvm::Type *eltType = coercionType->getElementType(i);
5110  if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) continue;
5111  Address eltAddr = Builder.CreateStructGEP(addr, i);
5112  llvm::Value *elt = Builder.CreateLoad(eltAddr);
5113  IRCallArgs[IRArgPos++] = elt;
5114  }
5115  assert(IRArgPos == FirstIRArg + NumIRArgs);
5116 
5117  if (tempSize) {
5118  EmitLifetimeEnd(tempSize, AllocaAddr.getPointer());
5119  }
5120 
5121  break;
5122  }
5123 
5124  case ABIArgInfo::Expand: {
5125  unsigned IRArgPos = FirstIRArg;
5126  ExpandTypeToArgs(I->Ty, *I, IRFuncTy, IRCallArgs, IRArgPos);
5127  assert(IRArgPos == FirstIRArg + NumIRArgs);
5128  break;
5129  }
5130  }
5131  }
5132 
5133  const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this);
5134  llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
5135 
5136  // If we're using inalloca, set up that argument.
5137  if (ArgMemory.isValid()) {
5138  llvm::Value *Arg = ArgMemory.getPointer();
5139  if (CallInfo.isVariadic()) {
5140  // When passing non-POD arguments by value to variadic functions, we will
5141  // end up with a variadic prototype and an inalloca call site. In such
5142  // cases, we can't do any parameter mismatch checks. Give up and bitcast
5143  // the callee.
5144  unsigned CalleeAS = CalleePtr->getType()->getPointerAddressSpace();
5145  CalleePtr =
5146  Builder.CreateBitCast(CalleePtr, IRFuncTy->getPointerTo(CalleeAS));
5147  } else {
5148  llvm::Type *LastParamTy =
5149  IRFuncTy->getParamType(IRFuncTy->getNumParams() - 1);
5150  if (Arg->getType() != LastParamTy) {
5151 #ifndef NDEBUG
5152  // Assert that these structs have equivalent element types.
5153  llvm::StructType *FullTy = CallInfo.getArgStruct();
5154  if (!LastParamTy->isOpaquePointerTy()) {
5155  llvm::StructType *DeclaredTy = cast<llvm::StructType>(
5156  LastParamTy->getNonOpaquePointerElementType());
5157  assert(DeclaredTy->getNumElements() == FullTy->getNumElements());
5158  for (auto DI = DeclaredTy->element_begin(),
5159  DE = DeclaredTy->element_end(),
5160  FI = FullTy->element_begin();
5161  DI != DE; ++DI, ++FI)
5162  assert(*DI == *FI);
5163  }
5164 #endif
5165  Arg = Builder.CreateBitCast(Arg, LastParamTy);
5166  }
5167  }
5168  assert(IRFunctionArgs.hasInallocaArg());
5169  IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
5170  }
5171 
5172  // 2. Prepare the function pointer.
5173 
5174  // If the callee is a bitcast of a non-variadic function to have a
5175  // variadic function pointer type, check to see if we can remove the
5176  // bitcast. This comes up with unprototyped functions.
5177  //
5178  // This makes the IR nicer, but more importantly it ensures that we
5179  // can inline the function at -O0 if it is marked always_inline.
5180  auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
5181  llvm::Value *Ptr) -> llvm::Function * {
5182  if (!CalleeFT->isVarArg())
5183  return nullptr;
5184 
5185  // Get underlying value if it's a bitcast
5186  if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Ptr)) {
5187  if (CE->getOpcode() == llvm::Instruction::BitCast)
5188  Ptr = CE->getOperand(0);
5189  }
5190 
5191  llvm::Function *OrigFn = dyn_cast<llvm::Function>(Ptr);
5192  if (!OrigFn)
5193  return nullptr;
5194 
5195  llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
5196 
5197  // If the original type is variadic, or if any of the component types
5198  // disagree, we cannot remove the cast.
5199  if (OrigFT->isVarArg() ||
5200  OrigFT->getNumParams() != CalleeFT->getNumParams() ||
5201  OrigFT->getReturnType() != CalleeFT->getReturnType())
5202  return nullptr;
5203 
5204  for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)
5205  if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
5206  return nullptr;
5207 
5208  return OrigFn;
5209  };
5210 
5211  if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
5212  CalleePtr = OrigFn;
5213  IRFuncTy = OrigFn->getFunctionType();
5214  }
5215 
5216  // 3. Perform the actual call.
5217 
5218  // Deactivate any cleanups that we're supposed to do immediately before
5219  // the call.
5220  if (!CallArgs.getCleanupsToDeactivate().empty())
5221  deactivateArgCleanupsBeforeCall(*this, CallArgs);
5222 
5223  // Assert that the arguments we computed match up. The IR verifier
5224  // will catch this, but this is a common enough source of problems
5225  // during IRGen changes that it's way better for debugging to catch
5226  // it ourselves here.
5227 #ifndef NDEBUG
5228  assert(IRCallArgs.size() == IRFuncTy->getNumParams() || IRFuncTy->isVarArg());
5229  for (unsigned i = 0; i < IRCallArgs.size(); ++i) {
5230  // Inalloca argument can have different type.
5231  if (IRFunctionArgs.hasInallocaArg() &&
5232  i == IRFunctionArgs.getInallocaArgNo())
5233  continue;
5234  if (i < IRFuncTy->getNumParams())
5235  assert(IRCallArgs[i]->getType() == IRFuncTy->getParamType(i));
5236  }
5237 #endif
5238 
5239  // Update the largest vector width if any arguments have vector types.
5240  for (unsigned i = 0; i < IRCallArgs.size(); ++i)
5241  LargestVectorWidth = std::max(LargestVectorWidth,
5242  getMaxVectorWidth(IRCallArgs[i]->getType()));
5243 
5244  // Compute the calling convention and attributes.
5245  unsigned CallingConv;
5246  llvm::AttributeList Attrs;
5247  CGM.ConstructAttributeList(CalleePtr->getName(), CallInfo,
5248  Callee.getAbstractInfo(), Attrs, CallingConv,
5249  /*AttrOnCallSite=*/true,
5250  /*IsThunk=*/false);
5251 
5252  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl))
5253  if (FD->hasAttr<StrictFPAttr>())
5254  // All calls within a strictfp function are marked strictfp
5255  Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::StrictFP);
5256 
5257  // Add call-site nomerge attribute if exists.
5259  Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoMerge);
5260 
5261  // Add call-site noinline attribute if exists.
5263  Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoInline);
5264 
5265  // Add call-site always_inline attribute if exists.
5267  Attrs =
5268  Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::AlwaysInline);
5269 
5270  // Apply some call-site-specific attributes.
5271  // TODO: work this into building the attribute set.
5272 
5273  // Apply always_inline to all calls within flatten functions.
5274  // FIXME: should this really take priority over __try, below?
5275  if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
5277  !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>())) {
5278  Attrs =
5279  Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::AlwaysInline);
5280  }
5281 
5282  // Disable inlining inside SEH __try blocks.
5283  if (isSEHTryScope()) {
5284  Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoInline);
5285  }
5286 
5287  // Decide whether to use a call or an invoke.
5288  bool CannotThrow;
5289  if (currentFunctionUsesSEHTry()) {
5290  // SEH cares about asynchronous exceptions, so everything can "throw."
5291  CannotThrow = false;
5292  } else if (isCleanupPadScope() &&
5294  // The MSVC++ personality will implicitly terminate the program if an
5295  // exception is thrown during a cleanup outside of a try/catch.
5296  // We don't need to model anything in IR to get this behavior.
5297  CannotThrow = true;
5298  } else {
5299  // Otherwise, nounwind call sites will never throw.
5300  CannotThrow = Attrs.hasFnAttr(llvm::Attribute::NoUnwind);
5301 
5302  if (auto *FPtr = dyn_cast<llvm::Function>(CalleePtr))
5303  if (FPtr->hasFnAttribute(llvm::Attribute::NoUnwind))
5304  CannotThrow = true;
5305  }
5306 
5307  // If we made a temporary, be sure to clean up after ourselves. Note that we
5308  // can't depend on being inside of an ExprWithCleanups, so we need to manually
5309  // pop this cleanup later on. Being eager about this is OK, since this
5310  // temporary is 'invisible' outside of the callee.
5311  if (UnusedReturnSizePtr)
5312  pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, SRetAlloca,
5313  UnusedReturnSizePtr);
5314 
5315  llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
5316 
5318  getBundlesForFunclet(CalleePtr);
5319 
5320  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl))
5321  if (FD->hasAttr<StrictFPAttr>())
5322  // All calls within a strictfp function are marked strictfp
5323  Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::StrictFP);
5324 
5325  AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
5326  Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
5327 
5328  AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
5329  Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
5330 
5331  // Emit the actual call/invoke instruction.
5332  llvm::CallBase *CI;
5333  if (!InvokeDest) {
5334  CI = Builder.CreateCall(IRFuncTy, CalleePtr, IRCallArgs, BundleList);
5335  } else {
5336  llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
5337  CI = Builder.CreateInvoke(IRFuncTy, CalleePtr, Cont, InvokeDest, IRCallArgs,
5338  BundleList);
5339  EmitBlock(Cont);
5340  }
5341  if (callOrInvoke)
5342  *callOrInvoke = CI;
5343 
5344  // If this is within a function that has the guard(nocf) attribute and is an
5345  // indirect call, add the "guard_nocf" attribute to this call to indicate that
5346  // Control Flow Guard checks should not be added, even if the call is inlined.
5347  if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
5348  if (const auto *A = FD->getAttr<CFGuardAttr>()) {
5349  if (A->getGuard() == CFGuardAttr::GuardArg::nocf && !CI->getCalledFunction())
5350  Attrs = Attrs.addFnAttribute(getLLVMContext(), "guard_nocf");
5351  }
5352  }
5353 
5354  // Apply the attributes and calling convention.
5355  CI->setAttributes(Attrs);
5356  CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
5357 
5358  // Apply various metadata.
5359 
5360  if (!CI->getType()->isVoidTy())
5361  CI->setName("call");
5362 
5363  // Update largest vector width from the return type.
5364  LargestVectorWidth =
5365  std::max(LargestVectorWidth, getMaxVectorWidth(CI->getType()));
5366 
5367  // Insert instrumentation or attach profile metadata at indirect call sites.
5368  // For more details, see the comment before the definition of
5369  // IPVK_IndirectCallTarget in InstrProfData.inc.
5370  if (!CI->getCalledFunction())
5371  PGO.valueProfile(Builder, llvm::IPVK_IndirectCallTarget,
5372  CI, CalleePtr);
5373 
5374  // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5375  // optimizer it can aggressively ignore unwind edges.
5376  if (CGM.getLangOpts().ObjCAutoRefCount)
5377  AddObjCARCExceptionMetadata(CI);
5378 
5379  // Set tail call kind if necessary.
5380  if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) {
5381  if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
5382  Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
5383  else if (IsMustTail)
5384  Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
5385  }
5386 
5387  // Add metadata for calls to MSAllocator functions
5388  if (getDebugInfo() && TargetDecl &&
5389  TargetDecl->hasAttr<MSAllocatorAttr>())
5391 
5392  // Add metadata if calling an __attribute__((error(""))) or warning fn.
5393  if (TargetDecl && TargetDecl->hasAttr<ErrorAttr>()) {
5394  llvm::ConstantInt *Line =