32#include "llvm/ADT/StringExtras.h"
33#include "llvm/Analysis/ValueTracking.h"
34#include "llvm/IR/Assumptions.h"
35#include "llvm/IR/AttributeMask.h"
36#include "llvm/IR/Attributes.h"
37#include "llvm/IR/CallingConv.h"
38#include "llvm/IR/DataLayout.h"
39#include "llvm/IR/InlineAsm.h"
40#include "llvm/IR/IntrinsicInst.h"
41#include "llvm/IR/Intrinsics.h"
42#include "llvm/IR/Type.h"
43#include "llvm/Transforms/Utils/Local.h"
46using namespace CodeGen;
52 default:
return llvm::CallingConv::C;
57 case CC_Win64:
return llvm::CallingConv::Win64;
59 case CC_AAPCS:
return llvm::CallingConv::ARM_AAPCS;
60 case CC_AAPCS_VFP:
return llvm::CallingConv::ARM_AAPCS_VFP;
73 case CC_Swift:
return llvm::CallingConv::Swift;
75 case CC_M68kRTD:
return llvm::CallingConv::M68k_RTD;
125 unsigned totalArgs) {
127 assert(paramInfos.size() <= prefixArgs);
128 assert(proto->
getNumParams() + prefixArgs <= totalArgs);
130 paramInfos.reserve(totalArgs);
133 paramInfos.resize(prefixArgs);
137 paramInfos.push_back(ParamInfo);
139 if (ParamInfo.hasPassObjectSize())
140 paramInfos.emplace_back();
143 assert(paramInfos.size() <= totalArgs &&
144 "Did we forget to insert pass_object_size args?");
146 paramInfos.resize(totalArgs);
156 if (!FPT->hasExtParameterInfos()) {
157 assert(paramInfos.empty() &&
158 "We have paramInfos, but the prototype doesn't?");
159 prefix.append(FPT->param_type_begin(), FPT->param_type_end());
163 unsigned PrefixSize = prefix.size();
167 prefix.reserve(prefix.size() + FPT->getNumParams());
169 auto ExtInfos = FPT->getExtParameterInfos();
170 assert(ExtInfos.size() == FPT->getNumParams());
171 for (
unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
172 prefix.push_back(FPT->getParamType(I));
173 if (ExtInfos[I].hasPassObjectSize())
196 FTP->getExtInfo(), paramInfos,
Required);
204 return ::arrangeLLVMFunctionInfo(*
this,
false, argTypes,
214 if (D->
hasAttr<FastCallAttr>())
220 if (D->
hasAttr<ThisCallAttr>())
223 if (D->
hasAttr<VectorCallAttr>())
229 if (PcsAttr *PCS = D->
getAttr<PcsAttr>())
232 if (D->
hasAttr<AArch64VectorPcsAttr>())
235 if (D->
hasAttr<AArch64SVEPcsAttr>())
238 if (D->
hasAttr<AMDGPUKernelCallAttr>())
241 if (D->
hasAttr<IntelOclBiccAttr>())
250 if (D->
hasAttr<PreserveMostAttr>())
253 if (D->
hasAttr<PreserveAllAttr>())
277 return ::arrangeLLVMFunctionInfo(
278 *
this,
true, argTypes,
285 if (FD->
hasAttr<CUDAGlobalAttr>()) {
298 assert(!isa<CXXConstructorDecl>(MD) &&
"wrong method for constructors!");
299 assert(!isa<CXXDestructorDecl>(MD) &&
"wrong method for destructors!");
325 auto *MD = cast<CXXMethodDecl>(GD.
getDecl());
333 bool PassParams =
true;
335 if (
auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
338 if (
auto Inherited = CD->getInheritedConstructor())
350 if (!paramInfos.empty()) {
353 paramInfos.insert(paramInfos.begin() + 1, AddedArgs.
Prefix,
356 paramInfos.append(AddedArgs.
Suffix,
361 (PassParams && MD->isVariadic() ?
RequiredArgs(argTypes.size())
371 argTypes, extInfo, paramInfos, required);
377 for (
auto &arg : args)
385 for (
auto &arg : args)
392 unsigned prefixArgs,
unsigned totalArgs) {
412 unsigned ExtraPrefixArgs,
413 unsigned ExtraSuffixArgs,
414 bool PassProtoArgs) {
417 for (
const auto &Arg : args)
421 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
426 FPT, TotalPrefixArgs + ExtraSuffixArgs)
440 if (PassProtoArgs && FPT->hasExtParameterInfos()) {
447 ArgTypes, Info, ParamInfos,
Required);
455 if (MD->isImplicitObjectMemberFunction())
460 assert(isa<FunctionType>(FTy));
467 std::nullopt, noProto->getExtInfo(), {},
502 I->hasAttr<NoEscapeAttr>());
503 extParamInfos.push_back(extParamInfo);
510 if (
getContext().getLangOpts().ObjCAutoRefCount &&
511 MD->
hasAttr<NSReturnsRetainedAttr>())
537 if (isa<CXXConstructorDecl>(GD.
getDecl()) ||
538 isa<CXXDestructorDecl>(GD.
getDecl()))
551 assert(MD->
isVirtual() &&
"only methods have thunks");
568 ArgTys.push_back(*FTP->param_type_begin());
570 ArgTys.push_back(Context.
IntTy);
585 unsigned numExtraRequiredArgs,
587 assert(args.size() >= numExtraRequiredArgs);
597 if (proto->isVariadic())
600 if (proto->hasExtParameterInfos())
610 cast<FunctionNoProtoType>(fnType))) {
616 for (
const auto &arg : args)
621 paramInfos, required);
633 chainCall ? 1 : 0, chainCall);
662 for (
const auto &Arg : args)
695 unsigned numPrefixArgs) {
696 assert(numPrefixArgs + 1 <= args.size() &&
697 "Emitting a call with less args than the required prefix?");
709 paramInfos, required);
721 assert(signature.
arg_size() <= args.size());
722 if (signature.
arg_size() == args.size())
727 if (!sigParamInfos.empty()) {
728 paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
729 paramInfos.resize(args.size());
761 assert(llvm::all_of(argTypes,
765 llvm::FoldingSetNodeID ID;
770 bool isDelegateCall =
773 info, paramInfos, required, resultType, argTypes);
775 void *insertPos =
nullptr;
776 CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
784 info, paramInfos, resultType, argTypes, required);
785 FunctionInfos.InsertNode(FI, insertPos);
787 bool inserted = FunctionsBeingProcessed.insert(FI).second;
789 assert(inserted &&
"Recursively being processed?");
792 if (CC == llvm::CallingConv::SPIR_KERNEL) {
810 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() ==
nullptr)
813 bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
814 assert(erased &&
"Not in set?");
820 bool chainCall,
bool delegateCall,
826 assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
831 operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
832 argTypes.size() + 1, paramInfos.size()));
835 FI->CallingConvention = llvmCC;
836 FI->EffectiveCallingConvention = llvmCC;
837 FI->ASTCallingConvention = info.
getCC();
838 FI->InstanceMethod = instanceMethod;
839 FI->ChainCall = chainCall;
840 FI->DelegateCall = delegateCall;
846 FI->Required = required;
849 FI->ArgStruct =
nullptr;
850 FI->ArgStructAlign = 0;
851 FI->NumArgs = argTypes.size();
852 FI->HasExtParameterInfos = !paramInfos.empty();
853 FI->getArgsBuffer()[0].
type = resultType;
854 FI->MaxVectorWidth = 0;
855 for (
unsigned i = 0, e = argTypes.size(); i != e; ++i)
856 FI->getArgsBuffer()[i + 1].
type = argTypes[i];
857 for (
unsigned i = 0, e = paramInfos.size(); i != e; ++i)
858 FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
868struct TypeExpansion {
869 enum TypeExpansionKind {
881 const TypeExpansionKind Kind;
883 TypeExpansion(TypeExpansionKind K) : Kind(K) {}
884 virtual ~TypeExpansion() {}
887struct ConstantArrayExpansion : TypeExpansion {
891 ConstantArrayExpansion(
QualType EltTy, uint64_t NumElts)
892 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
893 static bool classof(
const TypeExpansion *TE) {
894 return TE->Kind == TEK_ConstantArray;
898struct RecordExpansion : TypeExpansion {
905 : TypeExpansion(TEK_Record), Bases(
std::move(Bases)),
906 Fields(
std::move(Fields)) {}
907 static bool classof(
const TypeExpansion *TE) {
908 return TE->Kind == TEK_Record;
912struct ComplexExpansion : TypeExpansion {
916 static bool classof(
const TypeExpansion *TE) {
921struct NoExpansion : TypeExpansion {
922 NoExpansion() : TypeExpansion(TEK_None) {}
923 static bool classof(
const TypeExpansion *TE) {
924 return TE->Kind == TEK_None;
929static std::unique_ptr<TypeExpansion>
932 return std::make_unique<ConstantArrayExpansion>(
933 AT->getElementType(), AT->getSize().getZExtValue());
940 "Cannot expand structure with flexible array.");
947 for (
const auto *FD : RD->
fields()) {
948 if (FD->isZeroLengthBitField(Context))
950 assert(!FD->isBitField() &&
951 "Cannot expand structure with bit-field members.");
953 if (UnionSize < FieldSize) {
954 UnionSize = FieldSize;
959 Fields.push_back(LargestFD);
961 if (
const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
962 assert(!CXXRD->isDynamicClass() &&
963 "cannot expand vtable pointers in dynamic classes");
964 llvm::append_range(Bases, llvm::make_pointer_range(CXXRD->bases()));
967 for (
const auto *FD : RD->
fields()) {
968 if (FD->isZeroLengthBitField(Context))
970 assert(!FD->isBitField() &&
971 "Cannot expand structure with bit-field members.");
972 Fields.push_back(FD);
975 return std::make_unique<RecordExpansion>(std::move(Bases),
979 return std::make_unique<ComplexExpansion>(CT->getElementType());
981 return std::make_unique<NoExpansion>();
986 if (
auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
989 if (
auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
991 for (
auto BS : RExp->Bases)
993 for (
auto FD : RExp->Fields)
997 if (isa<ComplexExpansion>(Exp.get()))
999 assert(isa<NoExpansion>(Exp.get()));
1007 if (
auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1008 for (
int i = 0, n = CAExp->NumElts; i < n; i++) {
1011 }
else if (
auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1012 for (
auto BS : RExp->Bases)
1014 for (
auto FD : RExp->Fields)
1016 }
else if (
auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
1021 assert(isa<NoExpansion>(Exp.get()));
1027 ConstantArrayExpansion *CAE,
1029 llvm::function_ref<
void(
Address)> Fn) {
1035 for (
int i = 0, n = CAE->NumElts; i < n; i++) {
1036 llvm::Value *EltAddr = CGF.
Builder.CreateConstGEP2_32(
1038 Fn(
Address(EltAddr, EltTy, EltAlign));
1043 llvm::Function::arg_iterator &AI) {
1045 "Unexpected non-simple lvalue during struct expansion.");
1048 if (
auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1051 LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1052 ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
1054 }
else if (
auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1064 ExpandTypeFromArgs(BS->
getType(), SubLV, AI);
1066 for (
auto FD : RExp->Fields) {
1069 ExpandTypeFromArgs(FD->getType(), SubLV, AI);
1071 }
else if (isa<ComplexExpansion>(Exp.get())) {
1072 auto realValue = &*AI++;
1073 auto imagValue = &*AI++;
1078 assert(isa<NoExpansion>(Exp.get()));
1079 llvm::Value *Arg = &*AI++;
1086 if (Arg->getType()->isPointerTy()) {
1095void CodeGenFunction::ExpandTypeToArgs(
1099 if (
auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1103 *
this, CAExp, Addr, [&](
Address EltAddr) {
1107 ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,
1110 }
else if (
auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1121 ExpandTypeToArgs(BS->
getType(), BaseArg, IRFuncTy, IRCallArgs,
1126 for (
auto FD : RExp->Fields) {
1129 ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,
1132 }
else if (isa<ComplexExpansion>(Exp.get())) {
1134 IRCallArgs[IRCallArgPos++] = CV.first;
1135 IRCallArgs[IRCallArgPos++] = CV.second;
1137 assert(isa<NoExpansion>(Exp.get()));
1139 assert(RV.isScalar() &&
1140 "Unexpected non-scalar rvalue during struct expansion.");
1143 llvm::Value *
V = RV.getScalarVal();
1144 if (IRCallArgPos < IRFuncTy->getNumParams() &&
1145 V->getType() != IRFuncTy->getParamType(IRCallArgPos))
1146 V =
Builder.CreateBitCast(
V, IRFuncTy->getParamType(IRCallArgPos));
1148 IRCallArgs[IRCallArgPos++] =
V;
1155 const Twine &Name =
"tmp") {
1169 llvm::StructType *SrcSTy,
1172 if (SrcSTy->getNumElements() == 0)
return SrcPtr;
1180 uint64_t FirstEltSize =
1182 if (FirstEltSize < DstSize &&
1191 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
1207 if (Val->getType() == Ty)
1210 if (isa<llvm::PointerType>(Val->getType())) {
1212 if (isa<llvm::PointerType>(Ty))
1213 return CGF.
Builder.CreateBitCast(Val, Ty,
"coerce.val");
1219 llvm::Type *DestIntTy = Ty;
1220 if (isa<llvm::PointerType>(DestIntTy))
1223 if (Val->getType() != DestIntTy) {
1225 if (DL.isBigEndian()) {
1228 uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
1229 uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
1231 if (SrcSize > DstSize) {
1232 Val = CGF.
Builder.CreateLShr(Val, SrcSize - DstSize,
"coerce.highbits");
1233 Val = CGF.
Builder.CreateTrunc(Val, DestIntTy,
"coerce.val.ii");
1235 Val = CGF.
Builder.CreateZExt(Val, DestIntTy,
"coerce.val.ii");
1236 Val = CGF.
Builder.CreateShl(Val, DstSize - SrcSize,
"coerce.highbits");
1240 Val = CGF.
Builder.CreateIntCast(Val, DestIntTy,
false,
"coerce.val.ii");
1244 if (isa<llvm::PointerType>(Ty))
1245 Val = CGF.
Builder.CreateIntToPtr(Val, Ty,
"coerce.val.ip");
1268 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
1270 DstSize.getFixedValue(), CGF);
1278 if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
1279 (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
1285 if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1286 SrcSize.getFixedValue() >= DstSize.getFixedValue()) {
1300 if (
auto *ScalableDst = dyn_cast<llvm::ScalableVectorType>(Ty)) {
1301 if (
auto *FixedSrc = dyn_cast<llvm::FixedVectorType>(SrcTy)) {
1304 bool NeedsBitcast =
false;
1306 llvm::ScalableVectorType::get(CGF.
Builder.getInt1Ty(), 16);
1307 llvm::Type *OrigType = Ty;
1308 if (ScalableDst == PredType &&
1309 FixedSrc->getElementType() == CGF.
Builder.getInt8Ty()) {
1310 ScalableDst = llvm::ScalableVectorType::get(CGF.
Builder.getInt8Ty(), 2);
1311 NeedsBitcast =
true;
1313 if (ScalableDst->getElementType() == FixedSrc->getElementType()) {
1315 auto *UndefVec = llvm::UndefValue::get(ScalableDst);
1316 auto *Zero = llvm::Constant::getNullValue(CGF.
CGM.
Int64Ty);
1318 ScalableDst, UndefVec, Load, Zero,
"cast.scalable");
1332 llvm::ConstantInt::get(CGF.
IntPtrTy, SrcSize.getKnownMinValue()));
1341 bool DestIsVolatile) {
1343 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) {
1344 for (
unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1346 llvm::Value *Elt =
Builder.CreateExtractValue(Val, i);
1364 llvm::Type *SrcTy = Src->getType();
1366 if (SrcTy == DstTy) {
1373 if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
1375 SrcSize.getFixedValue(), CGF);
1379 llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy);
1380 llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy);
1381 if (SrcPtrTy && DstPtrTy &&
1382 SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()) {
1390 if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
1391 (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
1400 if (isa<llvm::ScalableVectorType>(SrcTy) ||
1401 isa<llvm::ScalableVectorType>(DstTy) ||
1402 SrcSize.getFixedValue() <= DstSize.getFixedValue()) {
1420 llvm::ConstantInt::get(CGF.
IntPtrTy, DstSize.getFixedValue()));
1439class ClangToLLVMArgMapping {
1440 static const unsigned InvalidIndex = ~0
U;
1441 unsigned InallocaArgNo;
1443 unsigned TotalIRArgs;
1447 unsigned PaddingArgIndex;
1450 unsigned FirstArgIndex;
1451 unsigned NumberOfArgs;
1454 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1462 bool OnlyRequiredArgs =
false)
1463 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1464 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1465 construct(Context, FI, OnlyRequiredArgs);
1468 bool hasInallocaArg()
const {
return InallocaArgNo != InvalidIndex; }
1469 unsigned getInallocaArgNo()
const {
1470 assert(hasInallocaArg());
1471 return InallocaArgNo;
1474 bool hasSRetArg()
const {
return SRetArgNo != InvalidIndex; }
1475 unsigned getSRetArgNo()
const {
1476 assert(hasSRetArg());
1480 unsigned totalIRArgs()
const {
return TotalIRArgs; }
1482 bool hasPaddingArg(
unsigned ArgNo)
const {
1483 assert(ArgNo < ArgInfo.size());
1484 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1486 unsigned getPaddingArgNo(
unsigned ArgNo)
const {
1487 assert(hasPaddingArg(ArgNo));
1488 return ArgInfo[ArgNo].PaddingArgIndex;
1493 std::pair<unsigned, unsigned> getIRArgs(
unsigned ArgNo)
const {
1494 assert(ArgNo < ArgInfo.size());
1495 return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1496 ArgInfo[ArgNo].NumberOfArgs);
1501 bool OnlyRequiredArgs);
1504void ClangToLLVMArgMapping::construct(
const ASTContext &Context,
1506 bool OnlyRequiredArgs) {
1507 unsigned IRArgNo = 0;
1508 bool SwapThisWithSRet =
false;
1513 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1524 auto &IRArgs = ArgInfo[ArgNo];
1527 IRArgs.PaddingArgIndex = IRArgNo++;
1533 llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.
getCoerceToType());
1535 IRArgs.NumberOfArgs = STy->getNumElements();
1537 IRArgs.NumberOfArgs = 1;
1543 IRArgs.NumberOfArgs = 1;
1548 IRArgs.NumberOfArgs = 0;
1558 if (IRArgs.NumberOfArgs > 0) {
1559 IRArgs.FirstArgIndex = IRArgNo;
1560 IRArgNo += IRArgs.NumberOfArgs;
1565 if (IRArgNo == 1 && SwapThisWithSRet)
1568 assert(ArgNo == ArgInfo.size());
1571 InallocaArgNo = IRArgNo++;
1573 TotalIRArgs = IRArgNo;
1581 return RI.
isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
1591 switch (BT->getKind()) {
1594 case BuiltinType::Float:
1596 case BuiltinType::Double:
1598 case BuiltinType::LongDouble:
1609 if (BT->getKind() == BuiltinType::LongDouble)
1625 bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1627 assert(Inserted &&
"Recursively being processed?");
1629 llvm::Type *resultType =
nullptr;
1634 llvm_unreachable(
"Invalid ABI kind for return argument");
1646 resultType = llvm::PointerType::get(
getLLVMContext(), addressSpace);
1662 ClangToLLVMArgMapping IRFunctionArgs(
getContext(), FI,
true);
1666 if (IRFunctionArgs.hasSRetArg()) {
1669 ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1674 if (IRFunctionArgs.hasInallocaArg())
1675 ArgTypes[IRFunctionArgs.getInallocaArgNo()] =
1682 for (; it != ie; ++it, ++ArgNo) {
1686 if (IRFunctionArgs.hasPaddingArg(ArgNo))
1687 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1690 unsigned FirstIRArg, NumIRArgs;
1691 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1696 assert(NumIRArgs == 0);
1700 assert(NumIRArgs == 1);
1702 ArgTypes[FirstIRArg] = llvm::PointerType::get(
1706 assert(NumIRArgs == 1);
1707 ArgTypes[FirstIRArg] = llvm::PointerType::get(
1715 llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
1717 assert(NumIRArgs == st->getNumElements());
1718 for (
unsigned i = 0, e = st->getNumElements(); i != e; ++i)
1719 ArgTypes[FirstIRArg + i] = st->getElementType(i);
1721 assert(NumIRArgs == 1);
1722 ArgTypes[FirstIRArg] = argType;
1728 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1730 *ArgTypesIter++ = EltTy;
1732 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1737 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1739 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1744 bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1745 assert(Erased &&
"Not in set?");
1747 return llvm::FunctionType::get(resultType, ArgTypes, FI.
isVariadic());
1761 llvm::AttrBuilder &FuncAttrs,
1768 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1771 FuncAttrs.addAttribute(
"aarch64_pstate_sm_enabled");
1773 FuncAttrs.addAttribute(
"aarch64_pstate_sm_compatible");
1775 FuncAttrs.addAttribute(
"aarch64_pstate_za_shared");
1777 FuncAttrs.addAttribute(
"aarch64_pstate_za_preserved");
1781 const Decl *Callee) {
1787 for (
const AssumptionAttr *AA : Callee->specific_attrs<AssumptionAttr>())
1788 AA->getAssumption().split(Attrs,
",");
1791 FuncAttrs.addAttribute(llvm::AssumptionAttrKey,
1792 llvm::join(Attrs.begin(), Attrs.end(),
","));
1801 if (
const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1802 return ClassDecl->hasTrivialDestructor();
1808 const Decl *TargetDecl) {
1814 if (
Module.getLangOpts().Sanitize.has(SanitizerKind::Memory))
1818 if (!
Module.getLangOpts().CPlusPlus)
1821 if (
const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(TargetDecl)) {
1822 if (FDecl->isExternC())
1824 }
else if (
const VarDecl *VDecl = dyn_cast<VarDecl>(TargetDecl)) {
1826 if (VDecl->isExternC())
1834 return Module.getCodeGenOpts().StrictReturn ||
1835 !
Module.MayDropFunctionReturn(
Module.getContext(), RetTy) ||
1836 Module.getLangOpts().Sanitize.has(SanitizerKind::Return);
1843 llvm::DenormalMode FP32DenormalMode,
1844 llvm::AttrBuilder &FuncAttrs) {
1845 if (FPDenormalMode != llvm::DenormalMode::getDefault())
1846 FuncAttrs.addAttribute(
"denormal-fp-math", FPDenormalMode.str());
1848 if (FP32DenormalMode != FPDenormalMode && FP32DenormalMode.isValid())
1849 FuncAttrs.addAttribute(
"denormal-fp-math-f32", FP32DenormalMode.str());
1857 llvm::AttrBuilder &FuncAttrs) {
1863 StringRef Name,
bool HasOptnone,
const CodeGenOptions &CodeGenOpts,
1865 llvm::AttrBuilder &FuncAttrs) {
1868 if (CodeGenOpts.OptimizeSize)
1869 FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1870 if (CodeGenOpts.OptimizeSize == 2)
1871 FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1874 if (CodeGenOpts.DisableRedZone)
1875 FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1876 if (CodeGenOpts.IndirectTlsSegRefs)
1877 FuncAttrs.addAttribute(
"indirect-tls-seg-refs");
1878 if (CodeGenOpts.NoImplicitFloat)
1879 FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1881 if (AttrOnCallSite) {
1886 FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1888 FuncAttrs.addAttribute(
"trap-func-name", CodeGenOpts.
TrapFuncName);
1890 switch (CodeGenOpts.getFramePointer()) {
1896 FuncAttrs.addAttribute(
"frame-pointer",
1898 CodeGenOpts.getFramePointer()));
1901 if (CodeGenOpts.LessPreciseFPMAD)
1902 FuncAttrs.addAttribute(
"less-precise-fpmad",
"true");
1904 if (CodeGenOpts.NullPointerIsValid)
1905 FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
1908 FuncAttrs.addAttribute(
"no-trapping-math",
"true");
1912 if (LangOpts.NoHonorInfs)
1913 FuncAttrs.addAttribute(
"no-infs-fp-math",
"true");
1914 if (LangOpts.NoHonorNaNs)
1915 FuncAttrs.addAttribute(
"no-nans-fp-math",
"true");
1916 if (LangOpts.ApproxFunc)
1917 FuncAttrs.addAttribute(
"approx-func-fp-math",
"true");
1918 if (LangOpts.AllowFPReassoc && LangOpts.AllowRecip &&
1919 LangOpts.NoSignedZero && LangOpts.ApproxFunc &&
1920 (LangOpts.getDefaultFPContractMode() ==
1922 LangOpts.getDefaultFPContractMode() ==
1924 FuncAttrs.addAttribute(
"unsafe-fp-math",
"true");
1925 if (CodeGenOpts.SoftFloat)
1926 FuncAttrs.addAttribute(
"use-soft-float",
"true");
1927 FuncAttrs.addAttribute(
"stack-protector-buffer-size",
1928 llvm::utostr(CodeGenOpts.SSPBufferSize));
1929 if (LangOpts.NoSignedZero)
1930 FuncAttrs.addAttribute(
"no-signed-zeros-fp-math",
"true");
1933 const std::vector<std::string> &Recips = CodeGenOpts.
Reciprocals;
1934 if (!Recips.empty())
1935 FuncAttrs.addAttribute(
"reciprocal-estimates",
1936 llvm::join(Recips,
","));
1940 FuncAttrs.addAttribute(
"prefer-vector-width",
1943 if (CodeGenOpts.StackRealignment)
1944 FuncAttrs.addAttribute(
"stackrealign");
1945 if (CodeGenOpts.Backchain)
1946 FuncAttrs.addAttribute(
"backchain");
1947 if (CodeGenOpts.EnableSegmentedStacks)
1948 FuncAttrs.addAttribute(
"split-stack");
1950 if (CodeGenOpts.SpeculativeLoadHardening)
1951 FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1954 switch (CodeGenOpts.getZeroCallUsedRegs()) {
1955 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
1956 FuncAttrs.removeAttribute(
"zero-call-used-regs");
1958 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
1959 FuncAttrs.addAttribute(
"zero-call-used-regs",
"used-gpr-arg");
1961 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
1962 FuncAttrs.addAttribute(
"zero-call-used-regs",
"used-gpr");
1964 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
1965 FuncAttrs.addAttribute(
"zero-call-used-regs",
"used-arg");
1967 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
1968 FuncAttrs.addAttribute(
"zero-call-used-regs",
"used");
1970 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
1971 FuncAttrs.addAttribute(
"zero-call-used-regs",
"all-gpr-arg");
1973 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
1974 FuncAttrs.addAttribute(
"zero-call-used-regs",
"all-gpr");
1976 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
1977 FuncAttrs.addAttribute(
"zero-call-used-regs",
"all-arg");
1979 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
1980 FuncAttrs.addAttribute(
"zero-call-used-regs",
"all");
1991 FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1996 if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) || LangOpts.OpenCL ||
1997 LangOpts.SYCLIsDevice) {
1998 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2002 StringRef Var,
Value;
2004 FuncAttrs.addAttribute(Var,
Value);
2015 const llvm::Function &F,
2017 auto FFeatures = F.getFnAttribute(
"target-features");
2019 llvm::StringSet<> MergedNames;
2021 MergedFeatures.reserve(TargetOpts.
Features.size());
2023 auto AddUnmergedFeatures = [&](
auto &&FeatureRange) {
2024 for (StringRef Feature : FeatureRange) {
2025 if (Feature.empty())
2027 assert(Feature[0] ==
'+' || Feature[0] ==
'-');
2028 StringRef Name = Feature.drop_front(1);
2029 bool Merged = !MergedNames.insert(Name).second;
2031 MergedFeatures.push_back(Feature);
2035 if (FFeatures.isValid())
2036 AddUnmergedFeatures(llvm::split(FFeatures.getValueAsString(),
','));
2037 AddUnmergedFeatures(TargetOpts.
Features);
2039 if (!MergedFeatures.empty()) {
2040 llvm::sort(MergedFeatures);
2041 FuncAttr.addAttribute(
"target-features", llvm::join(MergedFeatures,
","));
2048 bool WillInternalize) {
2050 llvm::AttrBuilder FuncAttrs(F.getContext());
2053 if (!TargetOpts.
CPU.empty())
2054 FuncAttrs.addAttribute(
"target-cpu", TargetOpts.
CPU);
2055 if (!TargetOpts.
TuneCPU.empty())
2056 FuncAttrs.addAttribute(
"tune-cpu", TargetOpts.
TuneCPU);
2059 CodeGenOpts, LangOpts,
2062 if (!WillInternalize && F.isInterposable()) {
2067 F.addFnAttrs(FuncAttrs);
2071 llvm::AttributeMask AttrsToRemove;
2073 llvm::DenormalMode DenormModeToMerge = F.getDenormalModeRaw();
2074 llvm::DenormalMode DenormModeToMergeF32 = F.getDenormalModeF32Raw();
2075 llvm::DenormalMode Merged =
2079 if (DenormModeToMergeF32.isValid()) {
2084 if (Merged == llvm::DenormalMode::getDefault()) {
2085 AttrsToRemove.addAttribute(
"denormal-fp-math");
2086 }
else if (Merged != DenormModeToMerge) {
2088 FuncAttrs.addAttribute(
"denormal-fp-math",
2092 if (MergedF32 == llvm::DenormalMode::getDefault()) {
2093 AttrsToRemove.addAttribute(
"denormal-fp-math-f32");
2094 }
else if (MergedF32 != DenormModeToMergeF32) {
2096 FuncAttrs.addAttribute(
"denormal-fp-math-f32",
2100 F.removeFnAttrs(AttrsToRemove);
2105 F.addFnAttrs(FuncAttrs);
2108void CodeGenModule::getTrivialDefaultFunctionAttributes(
2109 StringRef Name,
bool HasOptnone,
bool AttrOnCallSite,
2110 llvm::AttrBuilder &FuncAttrs) {
2111 ::getTrivialDefaultFunctionAttributes(Name, HasOptnone,
getCodeGenOpts(),
2116void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
2118 bool AttrOnCallSite,
2119 llvm::AttrBuilder &FuncAttrs) {
2120 getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
2124 if (!AttrOnCallSite)
2129 llvm::AttrBuilder &attrs) {
2130 getDefaultFunctionAttributes(
"",
false,
2132 GetCPUAndFeaturesAttributes(
GlobalDecl(), attrs);
2137 const NoBuiltinAttr *NBA =
nullptr) {
2138 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
2140 AttributeName +=
"no-builtin-";
2141 AttributeName += BuiltinName;
2142 FuncAttrs.addAttribute(AttributeName);
2146 if (LangOpts.NoBuiltin) {
2148 FuncAttrs.addAttribute(
"no-builtins");
2162 if (llvm::is_contained(NBA->builtinNames(),
"*")) {
2163 FuncAttrs.addAttribute(
"no-builtins");
2168 llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
2172 const llvm::DataLayout &DL,
const ABIArgInfo &AI,
2173 bool CheckCoerce =
true) {
2174 llvm::Type *Ty = Types.ConvertTypeForMem(QTy);
2180 if (!DL.typeSizeEqualsStoreSize(Ty))
2187 if (llvm::TypeSize::isKnownGT(DL.getTypeSizeInBits(CoerceTy),
2188 DL.getTypeSizeInBits(Ty)))
2212 if (
const MatrixType *Matrix = dyn_cast<MatrixType>(QTy))
2214 if (
const ArrayType *Array = dyn_cast<ArrayType>(QTy))
2223 unsigned NumRequiredArgs,
unsigned ArgNo) {
2224 const auto *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl);
2229 if (ArgNo >= NumRequiredArgs)
2233 if (ArgNo < FD->getNumParams()) {
2234 const ParmVarDecl *Param = FD->getParamDecl(ArgNo);
2235 if (Param && Param->
hasAttr<MaybeUndefAttr>())
2252 if (llvm::AttributeFuncs::isNoFPClassCompatibleType(IRTy))
2255 if (llvm::StructType *ST = dyn_cast<llvm::StructType>(IRTy)) {
2257 llvm::all_of(ST->elements(), [](llvm::Type *Ty) {
2258 return llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty);
2267 llvm::FPClassTest Mask = llvm::fcNone;
2268 if (LangOpts.NoHonorInfs)
2269 Mask |= llvm::fcInf;
2270 if (LangOpts.NoHonorNaNs)
2271 Mask |= llvm::fcNan;
2277 llvm::AttributeList &Attrs) {
2278 if (Attrs.getMemoryEffects().getModRef() == llvm::ModRefInfo::NoModRef) {
2279 Attrs = Attrs.removeFnAttribute(
getLLVMContext(), llvm::Attribute::Memory);
2280 llvm::Attribute MemoryAttr = llvm::Attribute::getWithMemoryEffects(
2306 llvm::AttributeList &AttrList,
2308 bool AttrOnCallSite,
bool IsThunk) {
2316 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2318 FuncAttrs.addAttribute(
"cmse_nonsecure_call");
2330 bool HasOptnone =
false;
2332 const NoBuiltinAttr *NBA =
nullptr;
2336 auto AddPotentialArgAccess = [&]() {
2337 llvm::Attribute A = FuncAttrs.getAttribute(llvm::Attribute::Memory);
2339 FuncAttrs.addMemoryAttr(A.getMemoryEffects() |
2340 llvm::MemoryEffects::argMemOnly());
2347 if (TargetDecl->
hasAttr<ReturnsTwiceAttr>())
2348 FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
2349 if (TargetDecl->
hasAttr<NoThrowAttr>())
2350 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2351 if (TargetDecl->
hasAttr<NoReturnAttr>())
2352 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2353 if (TargetDecl->
hasAttr<ColdAttr>())
2354 FuncAttrs.addAttribute(llvm::Attribute::Cold);
2355 if (TargetDecl->
hasAttr<HotAttr>())
2356 FuncAttrs.addAttribute(llvm::Attribute::Hot);
2357 if (TargetDecl->
hasAttr<NoDuplicateAttr>())
2358 FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
2359 if (TargetDecl->
hasAttr<ConvergentAttr>())
2360 FuncAttrs.addAttribute(llvm::Attribute::Convergent);
2362 if (
const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2365 if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2367 auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2369 (Kind == OO_New || Kind == OO_Array_New))
2370 RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2373 const bool IsVirtualCall = MD && MD->
isVirtual();
2376 if (!(AttrOnCallSite && IsVirtualCall)) {
2377 if (Fn->isNoReturn())
2378 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2379 NBA = Fn->getAttr<NoBuiltinAttr>();
2383 if (isa<FunctionDecl>(TargetDecl) || isa<VarDecl>(TargetDecl)) {
2386 if (AttrOnCallSite && TargetDecl->
hasAttr<NoMergeAttr>())
2387 FuncAttrs.addAttribute(llvm::Attribute::NoMerge);
2391 if (TargetDecl->
hasAttr<ConstAttr>()) {
2392 FuncAttrs.addMemoryAttr(llvm::MemoryEffects::none());
2393 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2396 FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2397 }
else if (TargetDecl->
hasAttr<PureAttr>()) {
2398 FuncAttrs.addMemoryAttr(llvm::MemoryEffects::readOnly());
2399 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2401 FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2402 }
else if (TargetDecl->
hasAttr<NoAliasAttr>()) {
2403 FuncAttrs.addMemoryAttr(llvm::MemoryEffects::inaccessibleOrArgMemOnly());
2404 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2406 if (TargetDecl->
hasAttr<RestrictAttr>())
2407 RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2408 if (TargetDecl->
hasAttr<ReturnsNonNullAttr>() &&
2409 !CodeGenOpts.NullPointerIsValid)
2410 RetAttrs.addAttribute(llvm::Attribute::NonNull);
2411 if (TargetDecl->
hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2412 FuncAttrs.addAttribute(
"no_caller_saved_registers");
2413 if (TargetDecl->
hasAttr<AnyX86NoCfCheckAttr>())
2414 FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
2415 if (TargetDecl->
hasAttr<LeafAttr>())
2416 FuncAttrs.addAttribute(llvm::Attribute::NoCallback);
2418 HasOptnone = TargetDecl->
hasAttr<OptimizeNoneAttr>();
2419 if (
auto *AllocSize = TargetDecl->
getAttr<AllocSizeAttr>()) {
2420 std::optional<unsigned> NumElemsParam;
2421 if (AllocSize->getNumElemsParam().isValid())
2422 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2423 FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
2427 if (TargetDecl->
hasAttr<OpenCLKernelAttr>()) {
2430 FuncAttrs.addAttribute(
"uniform-work-group-size",
"true");
2437 FuncAttrs.addAttribute(
2438 "uniform-work-group-size",
2439 llvm::toStringRef(
getLangOpts().OffloadUniformBlock));
2443 if (TargetDecl->
hasAttr<CUDAGlobalAttr>() &&
2445 FuncAttrs.addAttribute(
"uniform-work-group-size",
"true");
2447 if (TargetDecl->
hasAttr<ArmLocallyStreamingAttr>())
2448 FuncAttrs.addAttribute(
"aarch64_pstate_sm_body");
2450 if (TargetDecl->
hasAttr<ArmNewZAAttr>())
2451 FuncAttrs.addAttribute(
"aarch64_pstate_za_new");
2463 getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2468 if (TargetDecl->
hasAttr<NoSpeculativeLoadHardeningAttr>())
2469 FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
2470 if (TargetDecl->
hasAttr<SpeculativeLoadHardeningAttr>())
2471 FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2472 if (TargetDecl->
hasAttr<NoSplitStackAttr>())
2473 FuncAttrs.removeAttribute(
"split-stack");
2474 if (TargetDecl->
hasAttr<ZeroCallUsedRegsAttr>()) {
2477 TargetDecl->
getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2478 FuncAttrs.removeAttribute(
"zero-call-used-regs");
2479 FuncAttrs.addAttribute(
2480 "zero-call-used-regs",
2481 ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Kind));
2488 if (CodeGenOpts.NoPLT) {
2489 if (
auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2490 if (!Fn->isDefined() && !AttrOnCallSite) {
2491 FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
2499 if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2500 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
2501 if (!FD->isExternallyVisible())
2502 FuncAttrs.addAttribute(
"sample-profile-suffix-elision-policy",
2509 if (!AttrOnCallSite) {
2510 if (TargetDecl && TargetDecl->
hasAttr<CmseNSEntryAttr>())
2511 FuncAttrs.addAttribute(
"cmse_nonsecure_entry");
2514 auto shouldDisableTailCalls = [&] {
2516 if (CodeGenOpts.DisableTailCalls)
2522 if (TargetDecl->
hasAttr<DisableTailCallsAttr>() ||
2523 TargetDecl->
hasAttr<AnyX86InterruptAttr>())
2526 if (CodeGenOpts.NoEscapingBlockTailCalls) {
2527 if (
const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
2528 if (!BD->doesNotEscape())
2534 if (shouldDisableTailCalls())
2535 FuncAttrs.addAttribute(
"disable-tail-calls",
"true");
2539 GetCPUAndFeaturesAttributes(CalleeInfo.
getCalleeDecl(), FuncAttrs);
2543 ClangToLLVMArgMapping IRFunctionArgs(
getContext(), FI);
2550 if (CodeGenOpts.EnableNoundefAttrs &&
2554 RetAttrs.addAttribute(llvm::Attribute::NoUndef);
2560 RetAttrs.addAttribute(llvm::Attribute::SExt);
2562 RetAttrs.addAttribute(llvm::Attribute::ZExt);
2566 RetAttrs.addAttribute(llvm::Attribute::InReg);
2578 AddPotentialArgAccess();
2587 llvm_unreachable(
"Invalid ABI kind for return argument");
2595 RetAttrs.addDereferenceableAttr(
2597 if (
getTypes().getTargetAddressSpace(PTy) == 0 &&
2598 !CodeGenOpts.NullPointerIsValid)
2599 RetAttrs.addAttribute(llvm::Attribute::NonNull);
2601 llvm::Align Alignment =
2603 RetAttrs.addAlignmentAttr(Alignment);
2608 bool hasUsedSRet =
false;
2612 if (IRFunctionArgs.hasSRetArg()) {
2614 SRETAttrs.addStructRetAttr(
getTypes().ConvertTypeForMem(RetTy));
2617 SRETAttrs.addAttribute(llvm::Attribute::InReg);
2619 ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2624 if (IRFunctionArgs.hasInallocaArg()) {
2627 ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2636 auto IRArgs = IRFunctionArgs.getIRArgs(0);
2638 assert(IRArgs.second == 1 &&
"Expected only a single `this` pointer.");
2645 if (!CodeGenOpts.NullPointerIsValid &&
2647 Attrs.addAttribute(llvm::Attribute::NonNull);
2654 Attrs.addDereferenceableOrNullAttr(
2660 llvm::Align Alignment =
2664 Attrs.addAlignmentAttr(Alignment);
2666 ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(
getLLVMContext(), Attrs);
2672 I != E; ++I, ++ArgNo) {
2678 if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2680 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2681 llvm::AttributeSet::get(
2683 llvm::AttrBuilder(
getLLVMContext()).addAttribute(llvm::Attribute::InReg));
2688 if (CodeGenOpts.EnableNoundefAttrs &&
2690 Attrs.addAttribute(llvm::Attribute::NoUndef);
2699 Attrs.addAttribute(llvm::Attribute::SExt);
2701 Attrs.addAttribute(llvm::Attribute::ZExt);
2705 Attrs.addAttribute(llvm::Attribute::Nest);
2707 Attrs.addAttribute(llvm::Attribute::InReg);
2708 Attrs.addStackAlignmentAttr(llvm::MaybeAlign(AI.
getDirectAlign()));
2715 Attrs.addAttribute(llvm::Attribute::InReg);
2718 Attrs.addByValAttr(
getTypes().ConvertTypeForMem(ParamType));
2721 if (CodeGenOpts.PassByValueIsNoAlias &&
Decl &&
2722 Decl->getArgPassingRestrictions() ==
2726 Attrs.addAttribute(llvm::Attribute::NoAlias);
2751 AddPotentialArgAccess();
2756 Attrs.addByRefAttr(
getTypes().ConvertTypeForMem(ParamType));
2767 AddPotentialArgAccess();
2774 Attrs.addDereferenceableAttr(
2776 if (
getTypes().getTargetAddressSpace(PTy) == 0 &&
2777 !CodeGenOpts.NullPointerIsValid)
2778 Attrs.addAttribute(llvm::Attribute::NonNull);
2780 llvm::Align Alignment =
2782 Attrs.addAlignmentAttr(Alignment);
2790 if (TargetDecl && TargetDecl->
hasAttr<OpenCLKernelAttr>() &&
2794 llvm::Align Alignment =
2796 Attrs.addAlignmentAttr(Alignment);
2808 Attrs.addStructRetAttr(
getTypes().ConvertTypeForMem(ParamType));
2813 Attrs.addAttribute(llvm::Attribute::NoAlias);
2817 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2819 Attrs.addDereferenceableAttr(info.Width.getQuantity());
2820 Attrs.addAlignmentAttr(info.Align.getAsAlign());
2826 Attrs.addAttribute(llvm::Attribute::SwiftError);
2830 Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2834 Attrs.addAttribute(llvm::Attribute::SwiftAsync);
2839 Attrs.addAttribute(llvm::Attribute::NoCapture);
2841 if (Attrs.hasAttributes()) {
2842 unsigned FirstIRArg, NumIRArgs;
2843 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2844 for (
unsigned i = 0; i < NumIRArgs; i++)
2845 ArgAttrs[FirstIRArg + i] = ArgAttrs[FirstIRArg + i].addAttributes(
2851 AttrList = llvm::AttributeList::get(
2860 llvm::Value *value) {
2865 if (value->getType() == varType)
return value;
2867 assert((varType->isIntegerTy() || varType->isFloatingPointTy())
2868 &&
"unexpected promotion type");
2870 if (isa<llvm::IntegerType>(varType))
2871 return CGF.
Builder.CreateTrunc(value, varType,
"arg.unpromote");
2873 return CGF.
Builder.CreateFPCast(value, varType,
"arg.unpromote");
2879 QualType ArgType,
unsigned ArgNo) {
2891 if (
auto ParmNNAttr = PVD->
getAttr<NonNullAttr>())
2898 if (NNAttr->isNonNull(ArgNo))
2928 if (FD->hasImplicitReturnZero()) {
2929 QualType RetTy = FD->getReturnType().getUnqualifiedType();
2931 llvm::Constant*
Zero = llvm::Constant::getNullValue(LLVMTy);
2940 assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
2945 if (IRFunctionArgs.hasInallocaArg())
2946 ArgStruct =
Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()),
2950 if (IRFunctionArgs.hasSRetArg()) {
2951 auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo());
2952 AI->setName(
"agg.result");
2953 AI->addAttr(llvm::Attribute::NoAlias);
2960 ArgVals.reserve(Args.size());
2966 assert(FI.
arg_size() == Args.size() &&
2967 "Mismatch between function signature & arguments.");
2970 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
2971 i != e; ++i, ++info_it, ++ArgNo) {
2976 isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();
2984 unsigned FirstIRArg, NumIRArgs;
2985 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2989 assert(NumIRArgs == 0);
3002 assert(NumIRArgs == 1);
3042 auto AI = Fn->getArg(FirstIRArg);
3050 assert(NumIRArgs == 1);
3052 if (
const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
3055 PVD->getFunctionScopeIndex()) &&
3057 AI->addAttr(llvm::Attribute::NonNull);
3059 QualType OTy = PVD->getOriginalType();
3060 if (
const auto *ArrTy =
3067 QualType ETy = ArrTy->getElementType();
3068 llvm::Align Alignment =
3070 AI->addAttrs(llvm::AttrBuilder(
getLLVMContext()).addAlignmentAttr(Alignment));
3071 uint64_t ArrSize = ArrTy->getSize().getZExtValue();
3075 Attrs.addDereferenceableAttr(
3076 getContext().getTypeSizeInChars(ETy).getQuantity() *
3078 AI->addAttrs(Attrs);
3079 }
else if (
getContext().getTargetInfo().getNullPointerValue(
3082 AI->addAttr(llvm::Attribute::NonNull);
3085 }
else if (
const auto *ArrTy =
3091 QualType ETy = ArrTy->getElementType();
3092 llvm::Align Alignment =
3094 AI->addAttrs(llvm::AttrBuilder(
getLLVMContext()).addAlignmentAttr(Alignment));
3095 if (!
getTypes().getTargetAddressSpace(ETy) &&
3097 AI->addAttr(llvm::Attribute::NonNull);
3102 const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
3105 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
3106 if (AVAttr && !
SanOpts.
has(SanitizerKind::Alignment)) {
3110 llvm::ConstantInt *AlignmentCI =
3113 AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment);
3114 if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
3115 AI->removeAttr(llvm::Attribute::AttrKind::Alignment);
3116 AI->addAttrs(llvm::AttrBuilder(
getLLVMContext()).addAlignmentAttr(
3117 llvm::Align(AlignmentInt)));
3124 AI->addAttr(llvm::Attribute::NoAlias);
3132 assert(NumIRArgs == 1);
3136 llvm::Value *
V = AI;
3144 getContext().getTypeAlignInChars(pointeeTy));
3167 if (
V->getType() != LTy)
3178 if (
auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(
ConvertType(Ty))) {
3179 llvm::Value *Coerced = Fn->getArg(FirstIRArg);
3180 if (
auto *VecTyFrom =
3181 dyn_cast<llvm::ScalableVectorType>(Coerced->getType())) {
3185 llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
3186 if (VecTyFrom == PredType &&
3187 VecTyTo->getElementType() ==
Builder.getInt8Ty()) {
3188 VecTyFrom = llvm::ScalableVectorType::get(
Builder.getInt8Ty(), 2);
3189 Coerced =
Builder.CreateBitCast(Coerced, VecTyFrom);
3191 if (VecTyFrom->getElementType() == VecTyTo->getElementType()) {
3194 assert(NumIRArgs == 1);
3195 Coerced->setName(Arg->
getName() +
".coerce");
3197 VecTyTo, Coerced, Zero,
"cast.fixed")));
3211 llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.
getCoerceToType());
3213 STy->getNumElements() > 1) {
3215 llvm::TypeSize PtrElementSize =
3217 if (StructSize.isScalable()) {
3218 assert(STy->containsHomogeneousScalableVectorTypes() &&
3219 "ABI only supports structure with homogeneous scalable vector "
3221 assert(StructSize == PtrElementSize &&
3222 "Only allow non-fractional movement of structure with"
3223 "homogeneous scalable vector type");
3224 assert(STy->getNumElements() == NumIRArgs);
3226 llvm::Value *LoadedStructValue = llvm::PoisonValue::get(STy);
3227 for (
unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3228 auto *AI = Fn->getArg(FirstIRArg + i);
3229 AI->setName(Arg->
getName() +
".coerce" + Twine(i));
3231 Builder.CreateInsertValue(LoadedStructValue, AI, i);
3236 uint64_t SrcSize = StructSize.getFixedValue();
3237 uint64_t DstSize = PtrElementSize.getFixedValue();
3240 if (SrcSize <= DstSize) {
3247 assert(STy->getNumElements() == NumIRArgs);
3248 for (
unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3249 auto AI = Fn->getArg(FirstIRArg + i);
3250 AI->setName(Arg->
getName() +
".coerce" + Twine(i));
3255 if (SrcSize > DstSize) {
3261 assert(NumIRArgs == 1);
3262 auto AI = Fn->getArg(FirstIRArg);
3263 AI->setName(Arg->
getName() +
".coerce");
3288 unsigned argIndex = FirstIRArg;
3289 for (
unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3290 llvm::Type *eltType = coercionType->getElementType(i);
3295 auto elt = Fn->getArg(argIndex++);
3298 assert(argIndex == FirstIRArg + NumIRArgs);
3310 auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3311 ExpandTypeFromArgs(Ty, LV, FnArgIter);
3312 assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3313 for (
unsigned i = 0, e = NumIRArgs; i != e; ++i) {
3314 auto AI = Fn->getArg(FirstIRArg + i);
3315 AI->setName(Arg->
getName() +
"." + Twine(i));
3321 assert(NumIRArgs == 0);
3333 if (
getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3334 for (
int I = Args.size() - 1; I >= 0; --I)
3337 for (
unsigned I = 0, E = Args.size(); I != E; ++I)
3343 while (insn->use_empty()) {
3344 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
3345 if (!bitcast)
return;
3348 insn = cast<llvm::Instruction>(bitcast->getOperand(0));
3349 bitcast->eraseFromParent();
3355 llvm::Value *result) {
3357 llvm::BasicBlock *BB = CGF.
Builder.GetInsertBlock();
3358 if (BB->empty())
return nullptr;
3359 if (&BB->back() != result)
return nullptr;
3361 llvm::Type *resultType = result->getType();
3364 llvm::Instruction *generator = cast<llvm::Instruction>(result);
3370 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
3373 generator = cast<llvm::Instruction>(bitcast->getOperand(0));
3376 if (generator->getNextNode() != bitcast)
3379 InstsToKill.push_back(bitcast);
3386 llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
3387 if (!call)
return nullptr;
3389 bool doRetainAutorelease;
3392 doRetainAutorelease =
true;
3393 }
else if (call->getCalledOperand() ==
3395 doRetainAutorelease =
false;
3403 llvm::Instruction *prev = call->getPrevNode();
3405 if (isa<llvm::BitCastInst>(prev)) {
3406 prev = prev->getPrevNode();
3409 assert(isa<llvm::CallInst>(prev));
3410 assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3412 InstsToKill.push_back(prev);
3418 result = call->getArgOperand(0);
3419 InstsToKill.push_back(call);
3423 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
3424 if (!bitcast->hasOneUse())
break;
3425 InstsToKill.push_back(bitcast);
3426 result = bitcast->getOperand(0);
3430 for (
auto *I : InstsToKill)
3431 I->eraseFromParent();
3434 if (doRetainAutorelease)
3438 return CGF.
Builder.CreateBitCast(result, resultType);
3443 llvm::Value *result) {
3446 dyn_cast_or_null<ObjCMethodDecl>(CGF.
CurCodeDecl);
3447 if (!method)
return nullptr;
3453 llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(result);
3454 if (!retainCall || retainCall->getCalledOperand() !=
3459 llvm::Value *retainedValue = retainCall->getArgOperand(0);
3460 llvm::LoadInst *load =
3461 dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
3462 if (!load || load->isAtomic() || load->isVolatile() ||
3469 llvm::Type *resultType = result->getType();
3471 assert(retainCall->use_empty());
3472 retainCall->eraseFromParent();
3475 return CGF.
Builder.CreateBitCast(load, resultType);
3482 llvm::Value *result) {
3503 auto GetStoreIfValid = [&CGF](llvm::User *
U) -> llvm::StoreInst * {
3504 auto *SI = dyn_cast<llvm::StoreInst>(
U);
3511 assert(!SI->isAtomic() &&
3520 llvm::BasicBlock *IP = CGF.
Builder.GetInsertBlock();
3521 if (IP->empty())
return nullptr;
3525 for (llvm::Instruction &I : make_range(IP->rbegin(), IP->rend())) {
3526 if (isa<llvm::BitCastInst>(&I))
3528 if (
auto *II = dyn_cast<llvm::IntrinsicInst>(&I))
3529 if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
3532 return GetStoreIfValid(&I);
3537 llvm::StoreInst *store =
3539 if (!store)
return nullptr;
3543 llvm::BasicBlock *StoreBB = store->getParent();
3544 llvm::BasicBlock *IP = CGF.
Builder.GetInsertBlock();
3546 while (IP != StoreBB) {
3547 if (!SeenBBs.insert(IP).second || !(IP = IP->getSinglePredecessor()))
3563 int BitWidth,
int CharWidth) {
3564 assert(CharWidth <= 64);
3565 assert(
static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3568 if (BitOffset >= CharWidth) {
3569 Pos += BitOffset / CharWidth;
3570 BitOffset = BitOffset % CharWidth;
3573 const uint64_t
Used = (uint64_t(1) << CharWidth) - 1;
3574 if (BitOffset + BitWidth >= CharWidth) {
3575 Bits[Pos++] |= (
Used << BitOffset) &
Used;
3576 BitWidth -= CharWidth - BitOffset;
3580 while (BitWidth >= CharWidth) {
3582 BitWidth -= CharWidth;
3586 Bits[Pos++] |= (
Used >> (CharWidth - BitWidth)) << BitOffset;
3594 int StorageSize,
int BitOffset,
int BitWidth,
3595 int CharWidth,
bool BigEndian) {
3598 setBitRange(TmpBits, BitOffset, BitWidth, CharWidth);
3601 std::reverse(TmpBits.begin(), TmpBits.end());
3603 for (uint64_t
V : TmpBits)
3604 Bits[StorageOffset++] |=
V;
3635 BFI.
Size, CharWidth,
3657 auto Src = TmpBits.begin();
3658 auto Dst = Bits.begin() + Offset + I * Size;
3659 for (
int J = 0; J < Size; ++J)
3679 std::fill_n(Bits.begin() + Offset, Size,
3684 int Pos,
int Size,
int CharWidth,
3689 for (
auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size;
P != E;
3691 Mask = (Mask << CharWidth) | *
P;
3693 auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3695 Mask = (Mask << CharWidth) | *--
P;
3704 llvm::IntegerType *ITy,
3706 assert(Src->getType() == ITy);
3707 assert(ITy->getScalarSizeInBits() <= 64);
3710 int Size = DataLayout.getTypeStoreSize(ITy);
3718 return Builder.CreateAnd(Src, Mask,
"cmse.clear");
3724 llvm::ArrayType *ATy,
3727 int Size = DataLayout.getTypeStoreSize(ATy);
3734 ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3736 llvm::Value *R = llvm::PoisonValue::get(ATy);
3737 for (
int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
3739 DataLayout.isBigEndian());
3740 MaskIndex += CharsPerElt;
3741 llvm::Value *T0 =
Builder.CreateExtractValue(Src, I);
3742 llvm::Value *T1 =
Builder.CreateAnd(T0, Mask,
"cmse.clear");
3743 R =
Builder.CreateInsertValue(R, T1, I);
3770 llvm::DebugLoc RetDbgLoc;
3771 llvm::Value *RV =
nullptr;
3781 llvm::Function::arg_iterator EI =
CurFn->arg_end();
3783 llvm::Value *ArgStruct = &*EI;
3787 cast<llvm::GetElementPtrInst>(SRet)->getResultElementType();
3793 auto AI =
CurFn->arg_begin();
3832 if (llvm::StoreInst *SI =
3838 RetDbgLoc = SI->getDebugLoc();
3840 RV = SI->getValueOperand();
3841 SI->eraseFromParent();
3864 if (
auto *FD = dyn_cast<FunctionDecl>(
CurCodeDecl))
3865 RT = FD->getReturnType();
3866 else if (
auto *MD = dyn_cast<ObjCMethodDecl>(
CurCodeDecl))
3867 RT = MD->getReturnType();
3871 llvm_unreachable(
"Unexpected function/method type");
3891 for (
unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3898 results.push_back(elt);
3902 if (results.size() == 1) {
3910 RV = llvm::PoisonValue::get(returnType);
3911 for (
unsigned i = 0, e = results.size(); i != e; ++i) {
3912 RV =
Builder.CreateInsertValue(RV, results[i], i);
3919 llvm_unreachable(
"Invalid ABI kind for return argument");
3922 llvm::Instruction *
Ret;
3928 auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType());
3939 Ret->setDebugLoc(std::move(RetDbgLoc));
3952 ReturnsNonNullAttr *RetNNAttr =
nullptr;
3953 if (
SanOpts.
has(SanitizerKind::ReturnsNonnullAttribute))
3956 if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
3964 assert(!requiresReturnValueNullabilityCheck() &&
3965 "Cannot check nullability and the nonnull attribute");
3966 AttrLoc = RetNNAttr->getLocation();
3967 CheckKind = SanitizerKind::ReturnsNonnullAttribute;
3968 Handler = SanitizerHandler::NonnullReturn;
3970 if (
auto *DD = dyn_cast<DeclaratorDecl>(
CurCodeDecl))
3971 if (
auto *TSI = DD->getTypeSourceInfo())
3973 AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
3974 CheckKind = SanitizerKind::NullabilityReturn;
3975 Handler = SanitizerHandler::NullabilityReturn;
3978 SanitizerScope SanScope(
this);
3985 llvm::Value *CanNullCheck =
Builder.CreateIsNotNull(SLocPtr);
3986 if (requiresReturnValueNullabilityCheck())
3988 Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);
3989 Builder.CreateCondBr(CanNullCheck, Check, NoCheck);
3993 llvm::Value *Cond =
Builder.CreateIsNotNull(RV);
3995 llvm::Value *DynamicData[] = {SLocPtr};
3996 EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);
4016 llvm::Type *IRPtrTy = llvm::PointerType::getUnqual(CGF.
getLLVMContext());
4017 llvm::Value *Placeholder = llvm::PoisonValue::get(IRPtrTy);
4044 if (
type->isReferenceType()) {
4053 param->
hasAttr<NSConsumedAttr>() &&
4054 type->isObjCRetainableType()) {
4057 llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));
4072 CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
4074 "cleanup for callee-destructed param not recorded");
4076 llvm::Instruction *isActive =
Builder.CreateUnreachable();
4082 return isa<llvm::ConstantPointerNull>(addr);
4091 "shouldn't have writeback for provably null argument");
4093 llvm::BasicBlock *contBB =
nullptr;
4097 bool provablyNonNull = llvm::isKnownNonZero(srcAddr.
getPointer(),
4099 if (!provablyNonNull) {
4103 llvm::Value *isNull =
4105 CGF.
Builder.CreateCondBr(isNull, contBB, writebackBB);
4114 "icr.writeback-cast");
4123 if (writeback.
ToUse) {
4148 if (!provablyNonNull)
4163 for (
const auto &I : llvm::reverse(Cleanups)) {
4165 I.IsActiveIP->eraseFromParent();
4171 if (uop->getOpcode() == UO_AddrOf)
4172 return uop->getSubExpr();
4202 llvm::PointerType *destType =
4204 llvm::Type *destElemType =
4221 CodeGenFunction::ConditionalEvaluation condEval(CGF);
4227 llvm::ConstantPointerNull::get(cast<llvm::PointerType>(destElemType));
4231 llvm::BasicBlock *contBB =
nullptr;
4232 llvm::BasicBlock *originBB =
nullptr;
4235 llvm::Value *finalArgument;
4237 bool provablyNonNull = llvm::isKnownNonZero(srcAddr.
getPointer(),
4239 if (provablyNonNull) {
4242 llvm::Value *isNull =
4245 finalArgument = CGF.
Builder.CreateSelect(isNull,
4246 llvm::ConstantPointerNull::get(destType),
4252 originBB = CGF.
Builder.GetInsertBlock();
4255 CGF.
Builder.CreateCondBr(isNull, contBB, copyBB);
4257 condEval.begin(CGF);
4261 llvm::Value *valueToUse =
nullptr;
4269 src = CGF.
Builder.CreateBitCast(src, destElemType,
"icr.cast");
4286 if (shouldCopy && !provablyNonNull) {
4287 llvm::BasicBlock *copyBB = CGF.
Builder.GetInsertBlock();
4292 llvm::PHINode *phiToUse = CGF.
Builder.CreatePHI(valueToUse->getType(), 2,
4294 phiToUse->addIncoming(valueToUse, copyBB);
4295 phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
4297 valueToUse = phiToUse;
4311 StackBase = CGF.
Builder.CreateStackSave(
"inalloca.save");
4317 CGF.
Builder.CreateStackRestore(StackBase);
4325 if (!AC.getDecl() || !(
SanOpts.
has(SanitizerKind::NonnullAttribute) ||
4330 auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(ParmNum) :
nullptr;
4331 unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
4334 const NonNullAttr *NNAttr =
nullptr;
4335 if (
SanOpts.
has(SanitizerKind::NonnullAttribute))
4338 bool CanCheckNullability =
false;
4339 if (
SanOpts.
has(SanitizerKind::NullabilityArg) && !NNAttr && PVD) {
4340 auto Nullability = PVD->getType()->getNullability();
4341 CanCheckNullability = Nullability &&
4343 PVD->getTypeSourceInfo();
4346 if (!NNAttr && !CanCheckNullability)
4353 AttrLoc = NNAttr->getLocation();
4354 CheckKind = SanitizerKind::NonnullAttribute;
4355 Handler = SanitizerHandler::NonnullArg;
4357 AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4358 CheckKind = SanitizerKind::NullabilityArg;
4359 Handler = SanitizerHandler::NullabilityArg;
4362 SanitizerScope SanScope(
this);
4364 llvm::Constant *StaticData[] = {
4366 llvm::ConstantInt::get(
Int32Ty, ArgNo + 1),
4368 EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, std::nullopt);
4384 return llvm::any_of(ArgTypes, [&](
QualType Ty) {
4395 return classDecl->getTypeParamListAsWritten();
4399 return catDecl->getTypeParamList();
4409 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4410 AbstractCallee AC,
unsigned ParamsToSkip, EvaluationOrder Order) {
4413 assert((ParamsToSkip == 0 ||
Prototype.P) &&
4414 "Can't skip parameters if type info is not provided");
4424 bool IsVariadic =
false;
4431 ArgTypes.assign(MD->param_type_begin() + ParamsToSkip,
4432 MD->param_type_end());
4436 ExplicitCC = FPT->getExtInfo().getCC();
4437 ArgTypes.assign(FPT->param_type_begin() + ParamsToSkip,
4438 FPT->param_type_end());
4446 assert(Arg != ArgRange.end() &&
"Running over edge of argument list!");
4448 (isGenericMethod || Ty->isVariablyModifiedType() ||
4449 Ty.getNonReferenceType()->isObjCRetainableType() ||
4451 .getCanonicalType(Ty.getNonReferenceType())
4453 getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&
4454 "type mismatch in call argument!");
4460 assert((Arg == ArgRange.end() || IsVariadic) &&
4461 "Extra arguments in non-variadic function!");
4466 for (
auto *A : llvm::drop_begin(ArgRange, ArgTypes.size()))
4467 ArgTypes.push_back(IsVariadic ? getVarArgType(A) : A->getType());
4468 assert((
int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4480 auto MaybeEmitImplicitObjectSize = [&](
unsigned I,
const Expr *Arg,
4482 if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
4484 auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4491 assert(EmittedArg.getScalarVal() &&
"We emitted nothing for the arg?");
4492 llvm::Value *
V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T,
4493 EmittedArg.getScalarVal(),
4499 std::swap(Args.back(), *(&Args.back() - 1));
4504 assert(
getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4505 "inalloca only supported on x86");
4510 size_t CallArgsStart = Args.size();
4511 for (
unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
4512 unsigned Idx = LeftToRight ? I : E - I - 1;
4514 unsigned InitialArgSize = Args.size();
4517 assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
4518 getContext().hasSameUnqualifiedType((*Arg)->getType(),
4520 (isa<ObjCMethodDecl>(AC.getDecl()) &&
4522 "Argument and parameter types don't match");
4526 assert(InitialArgSize + 1 == Args.size() &&
4527 "The code below depends on only adding one arg per EmitCallArg");
4528 (void)InitialArgSize;
4531 if (!Args.back().hasLValue()) {
4532 RValue RVArg = Args.back().getKnownRValue();
4534 ParamsToSkip + Idx);
4538 MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4545 std::reverse(Args.begin() + CallArgsStart, Args.end());
4553 : Addr(Addr), Ty(Ty) {}
4571struct DisableDebugLocationUpdates {
4573 bool disabledDebugInfo;
4575 if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && CGF.
getDebugInfo()))
4578 ~DisableDebugLocationUpdates() {
4579 if (disabledDebugInfo)
4615 DisableDebugLocationUpdates Dis(*
this, E);
4617 = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
4623 "reference binding to unmaterialized r-value!");
4635 if (
type->isRecordType() &&
4642 bool DestroyedInCallee =
true, NeedsEHCleanup =
true;
4643 if (
const auto *RD =
type->getAsCXXRecordDecl())
4644 DestroyedInCallee = RD->hasNonTrivialDestructor();
4648 if (DestroyedInCallee)
4655 if (DestroyedInCallee && NeedsEHCleanup) {
4662 llvm::Instruction *IsActive =
Builder.CreateUnreachable();
4668 if (HasAggregateEvalKind && isa<ImplicitCastExpr>(E) &&
4669 cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue) {
4679QualType CodeGenFunction::getVarArgType(
const Expr *Arg) {
4683 if (!
getTarget().getTriple().isOSWindows())
4700CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
4703 Inst->setMetadata(
"clang.arc.no_objc_arc_exceptions",
4710 const llvm::Twine &name) {
4718 const llvm::Twine &name) {
4720 call->setDoesNotThrow();
4727 const llvm::Twine &name) {
4742 if (
auto *CalleeFn = dyn_cast<llvm::Function>(
Callee->stripPointerCasts())) {
4743 if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {
4744 auto IID = CalleeFn->getIntrinsicID();
4745 if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))
4758 const llvm::Twine &name) {
4759 llvm::CallInst *call =
Builder.CreateCall(
4772 llvm::InvokeInst *invoke =
4778 invoke->setDoesNotReturn();
4781 llvm::CallInst *call =
Builder.CreateCall(callee, args, BundleList);
4782 call->setDoesNotReturn();
4791 const Twine &name) {
4799 const Twine &name) {
4809 const Twine &Name) {
4814 llvm::CallBase *Inst;
4816 Inst =
Builder.CreateCall(Callee, Args, BundleList, Name);
4819 Inst =
Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList,
4827 AddObjCARCExceptionMetadata(Inst);
4832void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
4834 DeferredReplacements.push_back(
4835 std::make_pair(llvm::WeakTrackingVH(Old), New));
4842[[nodiscard]] llvm::AttributeList
4843maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
4844 const llvm::AttributeList &Attrs,
4845 llvm::Align NewAlign) {
4846 llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
4847 if (CurAlign >= NewAlign)
4849 llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign);
4850 return Attrs.removeRetAttribute(Ctx, llvm::Attribute::AttrKind::Alignment)
4851 .addRetAttribute(Ctx, AlignAttr);
4854template <
typename AlignedAttrTy>
class AbstractAssumeAlignedAttrEmitter {
4859 const AlignedAttrTy *AA =
nullptr;
4861 llvm::Value *Alignment =
nullptr;
4862 llvm::ConstantInt *OffsetCI =
nullptr;
4868 AA = FuncDecl->
getAttr<AlignedAttrTy>();
4873 [[nodiscard]] llvm::AttributeList
4874 TryEmitAsCallSiteAttribute(
const llvm::AttributeList &Attrs) {
4875 if (!AA || OffsetCI || CGF.
SanOpts.
has(SanitizerKind::Alignment))
4877 const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4882 if (!AlignmentCI->getValue().isPowerOf2())
4884 llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4887 AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4899 AA->getLocation(), Alignment, OffsetCI);
4905class AssumeAlignedAttrEmitter final
4906 :
public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
4909 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4913 Alignment = cast<llvm::ConstantInt>(CGF.
EmitScalarExpr(AA->getAlignment()));
4914 if (
Expr *Offset = AA->getOffset()) {
4916 if (OffsetCI->isNullValue())
4923class AllocAlignAttrEmitter final
4924 :
public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
4928 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4932 Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
4941 if (
auto *VT = dyn_cast<llvm::VectorType>(Ty))
4942 return VT->getPrimitiveSizeInBits().getKnownMinValue();
4943 if (
auto *AT = dyn_cast<llvm::ArrayType>(Ty))
4946 unsigned MaxVectorWidth = 0;
4947 if (
auto *ST = dyn_cast<llvm::StructType>(Ty))
4948 for (
auto *I : ST->elements())
4950 return MaxVectorWidth;
4957 llvm::CallBase **callOrInvoke,
bool IsMustTail,
4970 const Decl *TargetDecl =
Callee.getAbstractInfo().getCalleeDecl().getDecl();
4971 if (
const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
4978 if (TargetDecl->
hasAttr<AlwaysInlineAttr>() &&
4979 (TargetDecl->
hasAttr<TargetAttr>() ||
4986 CGM, Loc, dyn_cast_or_null<FunctionDecl>(
CurCodeDecl), FD, CallArgs);
4994 if (llvm::StructType *ArgStruct = CallInfo.
getArgStruct()) {
4997 llvm::AllocaInst *AI;
4999 IP = IP->getNextNode();
5000 AI =
new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(),
5006 AI->setAlignment(Align.getAsAlign());
5007 AI->setUsedWithInAlloca(
true);
5008 assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
5009 ArgMemory =
Address(AI, ArgStruct, Align);
5012 ClangToLLVMArgMapping IRFunctionArgs(
CGM.
getContext(), CallInfo);
5019 llvm::Value *UnusedReturnSizePtr =
nullptr;
5026 llvm::TypeSize size =
5031 if (IRFunctionArgs.hasSRetArg()) {
5032 IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.
getPointer();
5049 assert(CallInfo.
arg_size() == CallArgs.size() &&
5050 "Mismatch between function signature & arguments.");
5053 for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
5054 I != E; ++I, ++info_it, ++ArgNo) {
5058 if (IRFunctionArgs.hasPaddingArg(ArgNo))
5059 IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
5062 unsigned FirstIRArg, NumIRArgs;
5063 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
5065 bool ArgHasMaybeUndefAttr =
5070 assert(NumIRArgs == 0);
5071 assert(
getTarget().getTriple().getArch() == llvm::Triple::x86);
5072 if (I->isAggregate()) {
5074 ? I->getKnownLValue().getAddress(*
this)
5075 : I->getKnownRValue().getAggregateAddress();
5076 llvm::Instruction *Placeholder =
5081 CGBuilderTy::InsertPoint IP =
Builder.saveIP();
5082 Builder.SetInsertPoint(Placeholder);
5095 deferPlaceholderReplacement(Placeholder, Addr.
getPointer());