31#include "llvm/ADT/StringExtras.h"
32#include "llvm/Analysis/ValueTracking.h"
33#include "llvm/IR/Assumptions.h"
34#include "llvm/IR/Attributes.h"
35#include "llvm/IR/CallingConv.h"
36#include "llvm/IR/DataLayout.h"
37#include "llvm/IR/InlineAsm.h"
38#include "llvm/IR/IntrinsicInst.h"
39#include "llvm/IR/Intrinsics.h"
40#include "llvm/IR/Type.h"
41#include "llvm/Transforms/Utils/Local.h"
44using namespace CodeGen;
50 default:
return llvm::CallingConv::C;
55 case CC_Win64:
return llvm::CallingConv::Win64;
57 case CC_AAPCS:
return llvm::CallingConv::ARM_AAPCS;
58 case CC_AAPCS_VFP:
return llvm::CallingConv::ARM_AAPCS_VFP;
71 case CC_Swift:
return llvm::CallingConv::Swift;
123 unsigned totalArgs) {
125 assert(paramInfos.size() <= prefixArgs);
126 assert(proto->
getNumParams() + prefixArgs <= totalArgs);
128 paramInfos.reserve(totalArgs);
131 paramInfos.resize(prefixArgs);
135 paramInfos.push_back(ParamInfo);
137 if (ParamInfo.hasPassObjectSize())
138 paramInfos.emplace_back();
141 assert(paramInfos.size() <= totalArgs &&
142 "Did we forget to insert pass_object_size args?");
144 paramInfos.resize(totalArgs);
154 if (!FPT->hasExtParameterInfos()) {
155 assert(paramInfos.empty() &&
156 "We have paramInfos, but the prototype doesn't?");
157 prefix.append(FPT->param_type_begin(), FPT->param_type_end());
161 unsigned PrefixSize = prefix.size();
165 prefix.reserve(prefix.size() + FPT->getNumParams());
167 auto ExtInfos = FPT->getExtParameterInfos();
168 assert(ExtInfos.size() == FPT->getNumParams());
169 for (
unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
170 prefix.push_back(FPT->getParamType(I));
171 if (ExtInfos[I].hasPassObjectSize())
193 FTP->getExtInfo(), paramInfos,
202 return ::arrangeLLVMFunctionInfo(*
this,
false, argTypes,
212 if (D->
hasAttr<FastCallAttr>())
218 if (D->
hasAttr<ThisCallAttr>())
221 if (D->
hasAttr<VectorCallAttr>())
227 if (PcsAttr *PCS = D->
getAttr<PcsAttr>())
230 if (D->
hasAttr<AArch64VectorPcsAttr>())
233 if (D->
hasAttr<AArch64SVEPcsAttr>())
236 if (D->
hasAttr<AMDGPUKernelCallAttr>())
239 if (D->
hasAttr<IntelOclBiccAttr>())
248 if (D->
hasAttr<PreserveMostAttr>())
251 if (D->
hasAttr<PreserveAllAttr>())
272 return ::arrangeLLVMFunctionInfo(
273 *
this,
true, argTypes,
280 if (FD->
hasAttr<CUDAGlobalAttr>()) {
293 assert(!isa<CXXConstructorDecl>(MD) &&
"wrong method for constructors!");
294 assert(!isa<CXXDestructorDecl>(MD) &&
"wrong method for destructors!");
320 auto *MD = cast<CXXMethodDecl>(GD.
getDecl());
328 bool PassParams =
true;
330 if (
auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
333 if (
auto Inherited = CD->getInheritedConstructor())
345 if (!paramInfos.empty()) {
348 paramInfos.insert(paramInfos.begin() + 1, AddedArgs.
Prefix,
351 paramInfos.append(AddedArgs.
Suffix,
356 (PassParams && MD->isVariadic() ?
RequiredArgs(argTypes.size())
366 false, argTypes, extInfo,
367 paramInfos, required);
373 for (
auto &arg : args)
381 for (
auto &arg : args)
388 unsigned prefixArgs,
unsigned totalArgs) {
408 unsigned ExtraPrefixArgs,
409 unsigned ExtraSuffixArgs,
410 bool PassProtoArgs) {
413 for (
const auto &Arg : args)
417 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
422 FPT, TotalPrefixArgs + ExtraSuffixArgs)
436 if (PassProtoArgs && FPT->hasExtParameterInfos()) {
442 false, ArgTypes, Info,
443 ParamInfos, Required);
451 if (MD->isInstance())
456 assert(isa<FunctionType>(FTy));
463 noProto->getReturnType(),
false,
464 false, std::nullopt, noProto->getExtInfo(), {},
499 I->hasAttr<NoEscapeAttr>());
500 extParamInfos.push_back(extParamInfo);
507 if (
getContext().getLangOpts().ObjCAutoRefCount &&
508 MD->
hasAttr<NSReturnsRetainedAttr>())
516 false, argTys, einfo, extParamInfos, required);
535 if (isa<CXXConstructorDecl>(GD.
getDecl()) ||
536 isa<CXXDestructorDecl>(GD.
getDecl()))
549 assert(MD->
isVirtual() &&
"only methods have thunks");
567 ArgTys.push_back(*FTP->param_type_begin());
569 ArgTys.push_back(Context.
IntTy);
585 unsigned numExtraRequiredArgs,
587 assert(args.size() >= numExtraRequiredArgs);
597 if (proto->isVariadic())
600 if (proto->hasExtParameterInfos())
610 cast<FunctionNoProtoType>(fnType))) {
616 for (
const auto &arg : args)
633 chainCall ? 1 : 0, chainCall);
662 for (
const auto &Arg : args)
684 resultType,
false,
false,
696 unsigned numPrefixArgs) {
697 assert(numPrefixArgs + 1 <= args.size() &&
698 "Emitting a call with less args than the required prefix?");
710 false, argTypes, info, paramInfos, required);
722 assert(signature.
arg_size() <= args.size());
723 if (signature.
arg_size() == args.size())
728 if (!sigParamInfos.empty()) {
729 paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
730 paramInfos.resize(args.size());
762 assert(llvm::all_of(argTypes,
766 llvm::FoldingSetNodeID ID;
768 required, resultType, argTypes);
770 void *insertPos =
nullptr;
771 CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
779 paramInfos, resultType, argTypes, required);
780 FunctionInfos.InsertNode(FI, insertPos);
782 bool inserted = FunctionsBeingProcessed.insert(FI).second;
784 assert(inserted &&
"Recursively being processed?");
787 if (CC == llvm::CallingConv::SPIR_KERNEL) {
805 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() ==
nullptr)
808 bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
809 assert(erased &&
"Not in set?");
822 assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
827 operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
828 argTypes.size() + 1, paramInfos.size()));
831 FI->CallingConvention = llvmCC;
832 FI->EffectiveCallingConvention = llvmCC;
833 FI->ASTCallingConvention = info.
getCC();
834 FI->InstanceMethod = instanceMethod;
835 FI->ChainCall = chainCall;
841 FI->Required = required;
844 FI->ArgStruct =
nullptr;
845 FI->ArgStructAlign = 0;
846 FI->NumArgs = argTypes.size();
847 FI->HasExtParameterInfos = !paramInfos.empty();
848 FI->getArgsBuffer()[0].
type = resultType;
849 FI->MaxVectorWidth = 0;
850 for (
unsigned i = 0, e = argTypes.size(); i != e; ++i)
851 FI->getArgsBuffer()[i + 1].
type = argTypes[i];
852 for (
unsigned i = 0, e = paramInfos.size(); i != e; ++i)
853 FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
863struct TypeExpansion {
864 enum TypeExpansionKind {
876 const TypeExpansionKind Kind;
878 TypeExpansion(TypeExpansionKind K) : Kind(K) {}
879 virtual ~TypeExpansion() {}
882struct ConstantArrayExpansion : TypeExpansion {
886 ConstantArrayExpansion(
QualType EltTy, uint64_t NumElts)
887 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
888 static bool classof(
const TypeExpansion *TE) {
889 return TE->Kind == TEK_ConstantArray;
893struct RecordExpansion : TypeExpansion {
900 : TypeExpansion(TEK_Record), Bases(
std::move(Bases)),
901 Fields(
std::move(Fields)) {}
902 static bool classof(
const TypeExpansion *TE) {
903 return TE->Kind == TEK_Record;
907struct ComplexExpansion : TypeExpansion {
911 static bool classof(
const TypeExpansion *TE) {
916struct NoExpansion : TypeExpansion {
917 NoExpansion() : TypeExpansion(TEK_None) {}
918 static bool classof(
const TypeExpansion *TE) {
919 return TE->Kind == TEK_None;
924static std::unique_ptr<TypeExpansion>
927 return std::make_unique<ConstantArrayExpansion>(
928 AT->getElementType(), AT->getSize().getZExtValue());
935 "Cannot expand structure with flexible array.");
942 for (
const auto *FD : RD->
fields()) {
943 if (FD->isZeroLengthBitField(Context))
945 assert(!FD->isBitField() &&
946 "Cannot expand structure with bit-field members.");
948 if (UnionSize < FieldSize) {
949 UnionSize = FieldSize;
954 Fields.push_back(LargestFD);
956 if (
const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
957 assert(!CXXRD->isDynamicClass() &&
958 "cannot expand vtable pointers in dynamic classes");
959 llvm::append_range(Bases, llvm::make_pointer_range(CXXRD->bases()));
962 for (
const auto *FD : RD->
fields()) {
963 if (FD->isZeroLengthBitField(Context))
965 assert(!FD->isBitField() &&
966 "Cannot expand structure with bit-field members.");
967 Fields.push_back(FD);
970 return std::make_unique<RecordExpansion>(std::move(Bases),
974 return std::make_unique<ComplexExpansion>(CT->getElementType());
976 return std::make_unique<NoExpansion>();
981 if (
auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
984 if (
auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
986 for (
auto BS : RExp->Bases)
988 for (
auto FD : RExp->Fields)
992 if (isa<ComplexExpansion>(Exp.get()))
994 assert(isa<NoExpansion>(Exp.get()));
1002 if (
auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1003 for (
int i = 0, n = CAExp->NumElts; i < n; i++) {
1006 }
else if (
auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1007 for (
auto BS : RExp->Bases)
1009 for (
auto FD : RExp->Fields)
1011 }
else if (
auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
1016 assert(isa<NoExpansion>(Exp.get()));
1022 ConstantArrayExpansion *CAE,
1024 llvm::function_ref<
void(
Address)> Fn) {
1030 for (
int i = 0, n = CAE->NumElts; i < n; i++) {
1031 llvm::Value *EltAddr = CGF.
Builder.CreateConstGEP2_32(
1033 Fn(
Address(EltAddr, EltTy, EltAlign));
1038 llvm::Function::arg_iterator &AI) {
1040 "Unexpected non-simple lvalue during struct expansion.");
1043 if (
auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1046 LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1047 ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
1049 }
else if (
auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1059 ExpandTypeFromArgs(BS->
getType(), SubLV, AI);
1061 for (
auto FD : RExp->Fields) {
1064 ExpandTypeFromArgs(FD->getType(), SubLV, AI);
1066 }
else if (isa<ComplexExpansion>(Exp.get())) {
1067 auto realValue = &*AI++;
1068 auto imagValue = &*AI++;
1073 assert(isa<NoExpansion>(Exp.get()));
1074 llvm::Value *Arg = &*AI++;
1081 if (Arg->getType()->isPointerTy()) {
1090void CodeGenFunction::ExpandTypeToArgs(
1094 if (
auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1098 *
this, CAExp, Addr, [&](
Address EltAddr) {
1102 ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,
1105 }
else if (
auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1116 ExpandTypeToArgs(BS->
getType(), BaseArg, IRFuncTy, IRCallArgs,
1121 for (
auto FD : RExp->Fields) {
1124 ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,
1127 }
else if (isa<ComplexExpansion>(Exp.get())) {
1129 IRCallArgs[IRCallArgPos++] = CV.first;
1130 IRCallArgs[IRCallArgPos++] = CV.second;
1132 assert(isa<NoExpansion>(Exp.get()));
1134 assert(RV.isScalar() &&
1135 "Unexpected non-scalar rvalue during struct expansion.");
1138 llvm::Value *
V = RV.getScalarVal();
1139 if (IRCallArgPos < IRFuncTy->getNumParams() &&
1140 V->getType() != IRFuncTy->getParamType(IRCallArgPos))
1141 V =
Builder.CreateBitCast(
V, IRFuncTy->getParamType(IRCallArgPos));
1143 IRCallArgs[IRCallArgPos++] =
V;
1150 const Twine &Name =
"tmp") {
1164 llvm::StructType *SrcSTy,
1167 if (SrcSTy->getNumElements() == 0)
return SrcPtr;
1175 uint64_t FirstEltSize =
1177 if (FirstEltSize < DstSize &&
1186 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
1202 if (Val->getType() == Ty)
1205 if (isa<llvm::PointerType>(Val->getType())) {
1207 if (isa<llvm::PointerType>(Ty))
1208 return CGF.
Builder.CreateBitCast(Val, Ty,
"coerce.val");
1214 llvm::Type *DestIntTy = Ty;
1215 if (isa<llvm::PointerType>(DestIntTy))
1218 if (Val->getType() != DestIntTy) {
1220 if (DL.isBigEndian()) {
1223 uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
1224 uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
1226 if (SrcSize > DstSize) {
1227 Val = CGF.
Builder.CreateLShr(Val, SrcSize - DstSize,
"coerce.highbits");
1228 Val = CGF.
Builder.CreateTrunc(Val, DestIntTy,
"coerce.val.ii");
1230 Val = CGF.
Builder.CreateZExt(Val, DestIntTy,
"coerce.val.ii");
1231 Val = CGF.
Builder.CreateShl(Val, DstSize - SrcSize,
"coerce.highbits");
1235 Val = CGF.
Builder.CreateIntCast(Val, DestIntTy,
false,
"coerce.val.ii");
1239 if (isa<llvm::PointerType>(Ty))
1240 Val = CGF.
Builder.CreateIntToPtr(Val, Ty,
"coerce.val.ip");
1263 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
1265 DstSize.getFixedValue(), CGF);
1273 if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
1274 (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
1280 if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1281 SrcSize.getFixedValue() >= DstSize.getFixedValue()) {
1295 if (
auto *ScalableDst = dyn_cast<llvm::ScalableVectorType>(Ty)) {
1296 if (
auto *FixedSrc = dyn_cast<llvm::FixedVectorType>(SrcTy)) {
1299 bool NeedsBitcast =
false;
1301 llvm::ScalableVectorType::get(CGF.
Builder.getInt1Ty(), 16);
1302 llvm::Type *OrigType = Ty;
1303 if (ScalableDst == PredType &&
1304 FixedSrc->getElementType() == CGF.
Builder.getInt8Ty()) {
1305 ScalableDst = llvm::ScalableVectorType::get(CGF.
Builder.getInt8Ty(), 2);
1306 NeedsBitcast =
true;
1308 if (ScalableDst->getElementType() == FixedSrc->getElementType()) {
1310 auto *UndefVec = llvm::UndefValue::get(ScalableDst);
1311 auto *Zero = llvm::Constant::getNullValue(CGF.
CGM.
Int64Ty);
1313 ScalableDst, UndefVec, Load, Zero,
"castScalableSve");
1327 llvm::ConstantInt::get(CGF.
IntPtrTy, SrcSize.getKnownMinValue()));
1336 bool DestIsVolatile) {
1338 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) {
1339 for (
unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1341 llvm::Value *Elt =
Builder.CreateExtractValue(Val, i);
1359 llvm::Type *SrcTy = Src->getType();
1361 if (SrcTy == DstTy) {
1368 if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
1370 SrcSize.getFixedValue(), CGF);
1374 llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy);
1375 llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy);
1376 if (SrcPtrTy && DstPtrTy &&
1377 SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()) {
1385 if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
1386 (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
1395 if (isa<llvm::ScalableVectorType>(SrcTy) ||
1396 isa<llvm::ScalableVectorType>(DstTy) ||
1397 SrcSize.getFixedValue() <= DstSize.getFixedValue()) {
1415 llvm::ConstantInt::get(CGF.
IntPtrTy, DstSize.getFixedValue()));
1434class ClangToLLVMArgMapping {
1435 static const unsigned InvalidIndex = ~0
U;
1436 unsigned InallocaArgNo;
1438 unsigned TotalIRArgs;
1442 unsigned PaddingArgIndex;
1445 unsigned FirstArgIndex;
1446 unsigned NumberOfArgs;
1449 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1457 bool OnlyRequiredArgs =
false)
1458 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1459 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1460 construct(Context, FI, OnlyRequiredArgs);
1463 bool hasInallocaArg()
const {
return InallocaArgNo != InvalidIndex; }
1464 unsigned getInallocaArgNo()
const {
1465 assert(hasInallocaArg());
1466 return InallocaArgNo;
1469 bool hasSRetArg()
const {
return SRetArgNo != InvalidIndex; }
1470 unsigned getSRetArgNo()
const {
1471 assert(hasSRetArg());
1475 unsigned totalIRArgs()
const {
return TotalIRArgs; }
1477 bool hasPaddingArg(
unsigned ArgNo)
const {
1478 assert(ArgNo < ArgInfo.size());
1479 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1481 unsigned getPaddingArgNo(
unsigned ArgNo)
const {
1482 assert(hasPaddingArg(ArgNo));
1483 return ArgInfo[ArgNo].PaddingArgIndex;
1488 std::pair<unsigned, unsigned> getIRArgs(
unsigned ArgNo)
const {
1489 assert(ArgNo < ArgInfo.size());
1490 return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1491 ArgInfo[ArgNo].NumberOfArgs);
1496 bool OnlyRequiredArgs);
1499void ClangToLLVMArgMapping::construct(
const ASTContext &Context,
1501 bool OnlyRequiredArgs) {
1502 unsigned IRArgNo = 0;
1503 bool SwapThisWithSRet =
false;
1508 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1519 auto &IRArgs = ArgInfo[ArgNo];
1522 IRArgs.PaddingArgIndex = IRArgNo++;
1528 llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.
getCoerceToType());
1530 IRArgs.NumberOfArgs = STy->getNumElements();
1532 IRArgs.NumberOfArgs = 1;
1538 IRArgs.NumberOfArgs = 1;
1543 IRArgs.NumberOfArgs = 0;
1553 if (IRArgs.NumberOfArgs > 0) {
1554 IRArgs.FirstArgIndex = IRArgNo;
1555 IRArgNo += IRArgs.NumberOfArgs;
1560 if (IRArgNo == 1 && SwapThisWithSRet)
1563 assert(ArgNo == ArgInfo.size());
1566 InallocaArgNo = IRArgNo++;
1568 TotalIRArgs = IRArgNo;
1576 return RI.
isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
1586 switch (BT->getKind()) {
1589 case BuiltinType::Float:
1591 case BuiltinType::Double:
1593 case BuiltinType::LongDouble:
1604 if (BT->getKind() == BuiltinType::LongDouble)
1620 bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1622 assert(Inserted &&
"Recursively being processed?");
1624 llvm::Type *resultType =
nullptr;
1629 llvm_unreachable(
"Invalid ABI kind for return argument");
1642 resultType = llvm::PointerType::get(ty, addressSpace);
1658 ClangToLLVMArgMapping IRFunctionArgs(
getContext(), FI,
true);
1662 if (IRFunctionArgs.hasSRetArg()) {
1666 ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1667 llvm::PointerType::get(Ty, AddressSpace);
1671 if (IRFunctionArgs.hasInallocaArg()) {
1674 ArgTypes[IRFunctionArgs.getInallocaArgNo()] = ArgStruct->getPointerTo();
1681 for (; it != ie; ++it, ++ArgNo) {
1685 if (IRFunctionArgs.hasPaddingArg(ArgNo))
1686 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1689 unsigned FirstIRArg, NumIRArgs;
1690 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1695 assert(NumIRArgs == 0);
1699 assert(NumIRArgs == 1);
1702 ArgTypes[FirstIRArg] = LTy->getPointerTo(
1707 assert(NumIRArgs == 1);
1717 llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
1719 assert(NumIRArgs == st->getNumElements());
1720 for (
unsigned i = 0, e = st->getNumElements(); i != e; ++i)
1721 ArgTypes[FirstIRArg + i] = st->getElementType(i);
1723 assert(NumIRArgs == 1);
1724 ArgTypes[FirstIRArg] = argType;
1730 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1732 *ArgTypesIter++ = EltTy;
1734 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1739 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1741 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1746 bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1747 assert(Erased &&
"Not in set?");
1749 return llvm::FunctionType::get(resultType, ArgTypes, FI.
isVariadic());
1763 llvm::AttrBuilder &FuncAttrs,
1770 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1774 const Decl *Callee) {
1780 for (
const AssumptionAttr *AA : Callee->specific_attrs<AssumptionAttr>())
1781 AA->getAssumption().split(Attrs,
",");
1784 FuncAttrs.addAttribute(llvm::AssumptionAttrKey,
1785 llvm::join(Attrs.begin(), Attrs.end(),
","));
1794 if (
const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1795 return ClassDecl->hasTrivialDestructor();
1801 const Decl *TargetDecl) {
1807 if (
Module.getLangOpts().Sanitize.has(SanitizerKind::Memory))
1811 if (!
Module.getLangOpts().CPlusPlus)
1814 if (
const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(TargetDecl)) {
1815 if (FDecl->isExternC())
1817 }
else if (
const VarDecl *VDecl = dyn_cast<VarDecl>(TargetDecl)) {
1819 if (VDecl->isExternC())
1827 return Module.getCodeGenOpts().StrictReturn ||
1828 !
Module.MayDropFunctionReturn(
Module.getContext(), RetTy) ||
1829 Module.getLangOpts().Sanitize.has(SanitizerKind::Return);
1832void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
1834 bool AttrOnCallSite,
1835 llvm::AttrBuilder &FuncAttrs) {
1838 if (CodeGenOpts.OptimizeSize)
1839 FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1840 if (CodeGenOpts.OptimizeSize == 2)
1841 FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1844 if (CodeGenOpts.DisableRedZone)
1845 FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1846 if (CodeGenOpts.IndirectTlsSegRefs)
1847 FuncAttrs.addAttribute(
"indirect-tls-seg-refs");
1848 if (CodeGenOpts.NoImplicitFloat)
1849 FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1851 if (AttrOnCallSite) {
1856 FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1858 FuncAttrs.addAttribute(
"trap-func-name", CodeGenOpts.
TrapFuncName);
1860 switch (CodeGenOpts.getFramePointer()) {
1866 FuncAttrs.addAttribute(
"frame-pointer",
1868 CodeGenOpts.getFramePointer()));
1871 if (CodeGenOpts.LessPreciseFPMAD)
1872 FuncAttrs.addAttribute(
"less-precise-fpmad",
"true");
1874 if (CodeGenOpts.NullPointerIsValid)
1875 FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
1878 FuncAttrs.addAttribute(
"denormal-fp-math",
1881 FuncAttrs.addAttribute(
1882 "denormal-fp-math-f32",
1887 FuncAttrs.addAttribute(
"no-trapping-math",
"true");
1891 if (LangOpts.NoHonorInfs)
1892 FuncAttrs.addAttribute(
"no-infs-fp-math",
"true");
1893 if (LangOpts.NoHonorNaNs)
1894 FuncAttrs.addAttribute(
"no-nans-fp-math",
"true");
1895 if (LangOpts.ApproxFunc)
1896 FuncAttrs.addAttribute(
"approx-func-fp-math",
"true");
1897 if (LangOpts.AllowFPReassoc && LangOpts.AllowRecip &&
1898 LangOpts.NoSignedZero && LangOpts.ApproxFunc &&
1899 (LangOpts.getDefaultFPContractMode() ==
1901 LangOpts.getDefaultFPContractMode() ==
1903 FuncAttrs.addAttribute(
"unsafe-fp-math",
"true");
1904 if (CodeGenOpts.SoftFloat)
1905 FuncAttrs.addAttribute(
"use-soft-float",
"true");
1906 FuncAttrs.addAttribute(
"stack-protector-buffer-size",
1907 llvm::utostr(CodeGenOpts.SSPBufferSize));
1908 if (LangOpts.NoSignedZero)
1909 FuncAttrs.addAttribute(
"no-signed-zeros-fp-math",
"true");
1912 const std::vector<std::string> &Recips = CodeGenOpts.
Reciprocals;
1913 if (!Recips.empty())
1914 FuncAttrs.addAttribute(
"reciprocal-estimates",
1915 llvm::join(Recips,
","));
1919 FuncAttrs.addAttribute(
"prefer-vector-width",
1922 if (CodeGenOpts.StackRealignment)
1923 FuncAttrs.addAttribute(
"stackrealign");
1924 if (CodeGenOpts.Backchain)
1925 FuncAttrs.addAttribute(
"backchain");
1926 if (CodeGenOpts.EnableSegmentedStacks)
1927 FuncAttrs.addAttribute(
"split-stack");
1929 if (CodeGenOpts.SpeculativeLoadHardening)
1930 FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1933 switch (CodeGenOpts.getZeroCallUsedRegs()) {
1934 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
1935 FuncAttrs.removeAttribute(
"zero-call-used-regs");
1937 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
1938 FuncAttrs.addAttribute(
"zero-call-used-regs",
"used-gpr-arg");
1940 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
1941 FuncAttrs.addAttribute(
"zero-call-used-regs",
"used-gpr");
1943 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
1944 FuncAttrs.addAttribute(
"zero-call-used-regs",
"used-arg");
1946 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
1947 FuncAttrs.addAttribute(
"zero-call-used-regs",
"used");
1949 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
1950 FuncAttrs.addAttribute(
"zero-call-used-regs",
"all-gpr-arg");
1952 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
1953 FuncAttrs.addAttribute(
"zero-call-used-regs",
"all-gpr");
1955 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
1956 FuncAttrs.addAttribute(
"zero-call-used-regs",
"all-arg");
1958 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
1959 FuncAttrs.addAttribute(
"zero-call-used-regs",
"all");
1964 if (
getLangOpts().assumeFunctionsAreConvergent()) {
1970 FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1978 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1982 StringRef Var,
Value;
1984 FuncAttrs.addAttribute(Var,
Value);
1989 llvm::AttrBuilder FuncAttrs(F.getContext());
1990 getDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
1993 F.addFnAttrs(FuncAttrs);
1997 llvm::AttrBuilder &attrs) {
1998 getDefaultFunctionAttributes(
"",
false,
2000 GetCPUAndFeaturesAttributes(
GlobalDecl(), attrs);
2005 const NoBuiltinAttr *NBA =
nullptr) {
2006 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
2008 AttributeName +=
"no-builtin-";
2009 AttributeName += BuiltinName;
2010 FuncAttrs.addAttribute(AttributeName);
2014 if (LangOpts.NoBuiltin) {
2016 FuncAttrs.addAttribute(
"no-builtins");
2030 if (llvm::is_contained(NBA->builtinNames(),
"*")) {
2031 FuncAttrs.addAttribute(
"no-builtins");
2036 llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
2040 const llvm::DataLayout &DL,
const ABIArgInfo &AI,
2041 bool CheckCoerce =
true) {
2042 llvm::Type *Ty = Types.ConvertTypeForMem(QTy);
2047 if (!DL.typeSizeEqualsStoreSize(Ty))
2054 if (llvm::TypeSize::isKnownGT(DL.getTypeSizeInBits(CoerceTy),
2055 DL.getTypeSizeInBits(Ty)))
2077 if (
const VectorType *Vector = dyn_cast<VectorType>(QTy))
2079 if (
const MatrixType *Matrix = dyn_cast<MatrixType>(QTy))
2081 if (
const ArrayType *Array = dyn_cast<ArrayType>(QTy))
2090 unsigned NumRequiredArgs,
unsigned ArgNo) {
2091 const auto *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl);
2096 if (ArgNo >= NumRequiredArgs)
2100 if (ArgNo < FD->getNumParams()) {
2101 const ParmVarDecl *Param = FD->getParamDecl(ArgNo);
2102 if (Param && Param->
hasAttr<MaybeUndefAttr>())
2119 if (llvm::AttributeFuncs::isNoFPClassCompatibleType(IRTy))
2122 if (llvm::StructType *ST = dyn_cast<llvm::StructType>(IRTy)) {
2124 llvm::all_of(ST->elements(), [](llvm::Type *Ty) {
2125 return llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty);
2134 llvm::FPClassTest Mask = llvm::fcNone;
2135 if (LangOpts.NoHonorInfs)
2136 Mask |= llvm::fcInf;
2137 if (LangOpts.NoHonorNaNs)
2138 Mask |= llvm::fcNan;
2162 llvm::AttributeList &AttrList,
2164 bool AttrOnCallSite,
bool IsThunk) {
2172 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2174 FuncAttrs.addAttribute(
"cmse_nonsecure_call");
2186 bool HasOptnone =
false;
2188 const NoBuiltinAttr *NBA =
nullptr;
2192 auto AddPotentialArgAccess = [&]() {
2193 llvm::Attribute A = FuncAttrs.getAttribute(llvm::Attribute::Memory);
2195 FuncAttrs.addMemoryAttr(A.getMemoryEffects() |
2196 llvm::MemoryEffects::argMemOnly());
2203 if (TargetDecl->
hasAttr<ReturnsTwiceAttr>())
2204 FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
2205 if (TargetDecl->
hasAttr<NoThrowAttr>())
2206 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2207 if (TargetDecl->
hasAttr<NoReturnAttr>())
2208 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2209 if (TargetDecl->
hasAttr<ColdAttr>())
2210 FuncAttrs.addAttribute(llvm::Attribute::Cold);
2211 if (TargetDecl->
hasAttr<HotAttr>())
2212 FuncAttrs.addAttribute(llvm::Attribute::Hot);
2213 if (TargetDecl->
hasAttr<NoDuplicateAttr>())
2214 FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
2215 if (TargetDecl->
hasAttr<ConvergentAttr>())
2216 FuncAttrs.addAttribute(llvm::Attribute::Convergent);
2218 if (
const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2221 if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2223 auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2225 (Kind == OO_New || Kind == OO_Array_New))
2226 RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2229 const bool IsVirtualCall = MD && MD->
isVirtual();
2232 if (!(AttrOnCallSite && IsVirtualCall)) {
2233 if (Fn->isNoReturn())
2234 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2235 NBA = Fn->getAttr<NoBuiltinAttr>();
2239 if (AttrOnCallSite && TargetDecl->
hasAttr<NoMergeAttr>())
2240 FuncAttrs.addAttribute(llvm::Attribute::NoMerge);
2244 if (TargetDecl->
hasAttr<ConstAttr>()) {
2245 FuncAttrs.addMemoryAttr(llvm::MemoryEffects::none());
2246 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2249 FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2250 }
else if (TargetDecl->
hasAttr<PureAttr>()) {
2251 FuncAttrs.addMemoryAttr(llvm::MemoryEffects::readOnly());
2252 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2254 FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2255 }
else if (TargetDecl->
hasAttr<NoAliasAttr>()) {
2256 FuncAttrs.addMemoryAttr(llvm::MemoryEffects::argMemOnly());
2257 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2259 if (TargetDecl->
hasAttr<RestrictAttr>())
2260 RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2261 if (TargetDecl->
hasAttr<ReturnsNonNullAttr>() &&
2262 !CodeGenOpts.NullPointerIsValid)
2263 RetAttrs.addAttribute(llvm::Attribute::NonNull);
2264 if (TargetDecl->
hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2265 FuncAttrs.addAttribute(
"no_caller_saved_registers");
2266 if (TargetDecl->
hasAttr<AnyX86NoCfCheckAttr>())
2267 FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
2268 if (TargetDecl->
hasAttr<LeafAttr>())
2269 FuncAttrs.addAttribute(llvm::Attribute::NoCallback);
2271 HasOptnone = TargetDecl->
hasAttr<OptimizeNoneAttr>();
2272 if (
auto *AllocSize = TargetDecl->
getAttr<AllocSizeAttr>()) {
2273 std::optional<unsigned> NumElemsParam;
2274 if (AllocSize->getNumElemsParam().isValid())
2275 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2276 FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
2280 if (TargetDecl->
hasAttr<OpenCLKernelAttr>()) {
2283 FuncAttrs.addAttribute(
"uniform-work-group-size",
"true");
2290 FuncAttrs.addAttribute(
"uniform-work-group-size",
2291 llvm::toStringRef(CodeGenOpts.UniformWGSize));
2305 getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2310 if (TargetDecl->
hasAttr<NoSpeculativeLoadHardeningAttr>())
2311 FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
2312 if (TargetDecl->
hasAttr<SpeculativeLoadHardeningAttr>())
2313 FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2314 if (TargetDecl->
hasAttr<NoSplitStackAttr>())
2315 FuncAttrs.removeAttribute(
"split-stack");
2316 if (TargetDecl->
hasAttr<ZeroCallUsedRegsAttr>()) {
2319 TargetDecl->
getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2320 FuncAttrs.removeAttribute(
"zero-call-used-regs");
2321 FuncAttrs.addAttribute(
2322 "zero-call-used-regs",
2323 ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Kind));
2330 if (CodeGenOpts.NoPLT) {
2331 if (
auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2332 if (!Fn->isDefined() && !AttrOnCallSite) {
2333 FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
2341 if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2342 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
2343 if (!FD->isExternallyVisible())
2344 FuncAttrs.addAttribute(
"sample-profile-suffix-elision-policy",
2351 if (!AttrOnCallSite) {
2352 if (TargetDecl && TargetDecl->
hasAttr<CmseNSEntryAttr>())
2353 FuncAttrs.addAttribute(
"cmse_nonsecure_entry");
2356 auto shouldDisableTailCalls = [&] {
2358 if (CodeGenOpts.DisableTailCalls)
2364 if (TargetDecl->
hasAttr<DisableTailCallsAttr>() ||
2365 TargetDecl->
hasAttr<AnyX86InterruptAttr>())
2368 if (CodeGenOpts.NoEscapingBlockTailCalls) {
2369 if (
const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
2370 if (!BD->doesNotEscape())
2376 if (shouldDisableTailCalls())
2377 FuncAttrs.addAttribute(
"disable-tail-calls",
"true");
2381 GetCPUAndFeaturesAttributes(CalleeInfo.
getCalleeDecl(), FuncAttrs);
2385 ClangToLLVMArgMapping IRFunctionArgs(
getContext(), FI);
2392 if (CodeGenOpts.EnableNoundefAttrs &&
2396 RetAttrs.addAttribute(llvm::Attribute::NoUndef);
2402 RetAttrs.addAttribute(llvm::Attribute::SExt);
2404 RetAttrs.addAttribute(llvm::Attribute::ZExt);
2408 RetAttrs.addAttribute(llvm::Attribute::InReg);
2420 AddPotentialArgAccess();
2429 llvm_unreachable(
"Invalid ABI kind for return argument");
2437 RetAttrs.addDereferenceableAttr(
2439 if (
getTypes().getTargetAddressSpace(PTy) == 0 &&
2440 !CodeGenOpts.NullPointerIsValid)
2441 RetAttrs.addAttribute(llvm::Attribute::NonNull);
2443 llvm::Align Alignment =
2445 RetAttrs.addAlignmentAttr(Alignment);
2450 bool hasUsedSRet =
false;
2454 if (IRFunctionArgs.hasSRetArg()) {
2456 SRETAttrs.addStructRetAttr(
getTypes().ConvertTypeForMem(RetTy));
2459 SRETAttrs.addAttribute(llvm::Attribute::InReg);
2461 ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2466 if (IRFunctionArgs.hasInallocaArg()) {
2469 ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2478 auto IRArgs = IRFunctionArgs.getIRArgs(0);
2480 assert(IRArgs.second == 1 &&
"Expected only a single `this` pointer.");
2487 if (!CodeGenOpts.NullPointerIsValid &&
2489 Attrs.addAttribute(llvm::Attribute::NonNull);
2496 Attrs.addDereferenceableOrNullAttr(
2502 llvm::Align Alignment =
2506 Attrs.addAlignmentAttr(Alignment);
2508 ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(
getLLVMContext(), Attrs);
2514 I != E; ++I, ++ArgNo) {
2520 if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2522 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2523 llvm::AttributeSet::get(
2525 llvm::AttrBuilder(
getLLVMContext()).addAttribute(llvm::Attribute::InReg));
2530 if (CodeGenOpts.EnableNoundefAttrs &&
2532 Attrs.addAttribute(llvm::Attribute::NoUndef);
2541 Attrs.addAttribute(llvm::Attribute::SExt);
2543 Attrs.addAttribute(llvm::Attribute::ZExt);
2547 Attrs.addAttribute(llvm::Attribute::Nest);
2549 Attrs.addAttribute(llvm::Attribute::InReg);
2550 Attrs.addStackAlignmentAttr(llvm::MaybeAlign(AI.
getDirectAlign()));
2557 Attrs.addAttribute(llvm::Attribute::InReg);
2560 Attrs.addByValAttr(
getTypes().ConvertTypeForMem(ParamType));
2563 if (CodeGenOpts.PassByValueIsNoAlias &&
Decl &&
2567 Attrs.addAttribute(llvm::Attribute::NoAlias);
2592 AddPotentialArgAccess();
2597 Attrs.addByRefAttr(
getTypes().ConvertTypeForMem(ParamType));
2608 AddPotentialArgAccess();
2615 Attrs.addDereferenceableAttr(
2617 if (
getTypes().getTargetAddressSpace(PTy) == 0 &&
2618 !CodeGenOpts.NullPointerIsValid)
2619 Attrs.addAttribute(llvm::Attribute::NonNull);
2621 llvm::Align Alignment =
2623 Attrs.addAlignmentAttr(Alignment);
2631 if (TargetDecl && TargetDecl->
hasAttr<OpenCLKernelAttr>() &&
2635 llvm::Align Alignment =
2637 Attrs.addAlignmentAttr(Alignment);
2649 Attrs.addStructRetAttr(
getTypes().ConvertTypeForMem(ParamType));
2654 Attrs.addAttribute(llvm::Attribute::NoAlias);
2658 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2660 Attrs.addDereferenceableAttr(info.Width.getQuantity());
2661 Attrs.addAlignmentAttr(info.Align.getAsAlign());
2667 Attrs.addAttribute(llvm::Attribute::SwiftError);
2671 Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2675 Attrs.addAttribute(llvm::Attribute::SwiftAsync);
2680 Attrs.addAttribute(llvm::Attribute::NoCapture);
2682 if (Attrs.hasAttributes()) {
2683 unsigned FirstIRArg, NumIRArgs;
2684 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2685 for (
unsigned i = 0; i < NumIRArgs; i++)
2686 ArgAttrs[FirstIRArg + i] = ArgAttrs[FirstIRArg + i].addAttributes(
2692 AttrList = llvm::AttributeList::get(
2701 llvm::Value *value) {
2706 if (value->getType() == varType)
return value;
2708 assert((varType->isIntegerTy() || varType->isFloatingPointTy())
2709 &&
"unexpected promotion type");
2711 if (isa<llvm::IntegerType>(varType))
2712 return CGF.
Builder.CreateTrunc(value, varType,
"arg.unpromote");
2714 return CGF.
Builder.CreateFPCast(value, varType,
"arg.unpromote");
2720 QualType ArgType,
unsigned ArgNo) {
2732 if (
auto ParmNNAttr = PVD->
getAttr<NonNullAttr>())
2739 if (NNAttr->isNonNull(ArgNo))
2769 if (FD->hasImplicitReturnZero()) {
2770 QualType RetTy = FD->getReturnType().getUnqualifiedType();
2772 llvm::Constant*
Zero = llvm::Constant::getNullValue(LLVMTy);
2781 assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
2786 if (IRFunctionArgs.hasInallocaArg()) {
2787 ArgStruct =
Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()),
2794 if (IRFunctionArgs.hasSRetArg()) {
2795 auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo());
2796 AI->setName(
"agg.result");
2797 AI->addAttr(llvm::Attribute::NoAlias);
2804 ArgVals.reserve(Args.size());
2810 assert(FI.
arg_size() == Args.size() &&
2811 "Mismatch between function signature & arguments.");
2814 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
2815 i != e; ++i, ++info_it, ++ArgNo) {
2820 isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();
2828 unsigned FirstIRArg, NumIRArgs;
2829 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2833 assert(NumIRArgs == 0);
2846 assert(NumIRArgs == 1);
2886 auto AI = Fn->getArg(FirstIRArg);
2894 assert(NumIRArgs == 1);
2896 if (
const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
2899 PVD->getFunctionScopeIndex()) &&
2901 AI->addAttr(llvm::Attribute::NonNull);
2903 QualType OTy = PVD->getOriginalType();
2904 if (
const auto *ArrTy =
2911 QualType ETy = ArrTy->getElementType();
2912 llvm::Align Alignment =
2914 AI->addAttrs(llvm::AttrBuilder(
getLLVMContext()).addAlignmentAttr(Alignment));
2915 uint64_t ArrSize = ArrTy->getSize().getZExtValue();
2919 Attrs.addDereferenceableAttr(
2920 getContext().getTypeSizeInChars(ETy).getQuantity() *
2922 AI->addAttrs(Attrs);
2923 }
else if (
getContext().getTargetInfo().getNullPointerValue(
2926 AI->addAttr(llvm::Attribute::NonNull);
2929 }
else if (
const auto *ArrTy =
2935 QualType ETy = ArrTy->getElementType();
2936 llvm::Align Alignment =
2938 AI->addAttrs(llvm::AttrBuilder(
getLLVMContext()).addAlignmentAttr(Alignment));
2939 if (!
getTypes().getTargetAddressSpace(ETy) &&
2941 AI->addAttr(llvm::Attribute::NonNull);
2946 const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
2949 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
2950 if (AVAttr && !
SanOpts.
has(SanitizerKind::Alignment)) {
2954 llvm::ConstantInt *AlignmentCI =
2957 AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment);
2958 if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
2959 AI->removeAttr(llvm::Attribute::AttrKind::Alignment);
2960 AI->addAttrs(llvm::AttrBuilder(
getLLVMContext()).addAlignmentAttr(
2961 llvm::Align(AlignmentInt)));
2968 AI->addAttr(llvm::Attribute::NoAlias);
2976 assert(NumIRArgs == 1);
2980 llvm::Value *
V = AI;
2988 getContext().getTypeAlignInChars(pointeeTy));
3011 if (
V->getType() != LTy)
3022 if (
auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(
ConvertType(Ty))) {
3023 llvm::Value *Coerced = Fn->getArg(FirstIRArg);
3024 if (
auto *VecTyFrom =
3025 dyn_cast<llvm::ScalableVectorType>(Coerced->getType())) {
3029 llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
3030 if (VecTyFrom == PredType &&
3031 VecTyTo->getElementType() ==
Builder.getInt8Ty()) {
3032 VecTyFrom = llvm::ScalableVectorType::get(
Builder.getInt8Ty(), 2);
3033 Coerced =
Builder.CreateBitCast(Coerced, VecTyFrom);
3035 if (VecTyFrom->getElementType() == VecTyTo->getElementType()) {
3038 assert(NumIRArgs == 1);
3039 Coerced->setName(Arg->
getName() +
".coerce");
3041 VecTyTo, Coerced, Zero,
"castFixedSve")));
3055 llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.
getCoerceToType());
3057 STy->getNumElements() > 1) {
3063 if (SrcSize <= DstSize) {
3070 assert(STy->getNumElements() == NumIRArgs);
3071 for (
unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3072 auto AI = Fn->getArg(FirstIRArg + i);
3073 AI->setName(Arg->
getName() +
".coerce" + Twine(i));
3078 if (SrcSize > DstSize) {
3084 assert(NumIRArgs == 1);
3085 auto AI = Fn->getArg(FirstIRArg);
3086 AI->setName(Arg->
getName() +
".coerce");
3111 unsigned argIndex = FirstIRArg;
3112 for (
unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3113 llvm::Type *eltType = coercionType->getElementType(i);
3118 auto elt = Fn->getArg(argIndex++);
3121 assert(argIndex == FirstIRArg + NumIRArgs);
3133 auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3134 ExpandTypeFromArgs(Ty, LV, FnArgIter);
3135 assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3136 for (
unsigned i = 0, e = NumIRArgs; i != e; ++i) {
3137 auto AI = Fn->getArg(FirstIRArg + i);
3138 AI->setName(Arg->
getName() +
"." + Twine(i));
3144 assert(NumIRArgs == 0);
3156 if (
getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3157 for (
int I = Args.size() - 1; I >= 0; --I)
3160 for (
unsigned I = 0, E = Args.size(); I != E; ++I)
3166 while (insn->use_empty()) {
3167 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
3168 if (!bitcast)
return;
3171 insn = cast<llvm::Instruction>(bitcast->getOperand(0));
3172 bitcast->eraseFromParent();
3178 llvm::Value *result) {
3180 llvm::BasicBlock *BB = CGF.
Builder.GetInsertBlock();
3181 if (BB->empty())
return nullptr;
3182 if (&BB->back() != result)
return nullptr;
3184 llvm::Type *resultType = result->getType();
3187 llvm::Instruction *generator = cast<llvm::Instruction>(result);
3193 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
3196 generator = cast<llvm::Instruction>(bitcast->getOperand(0));
3199 if (generator->getNextNode() != bitcast)
3202 InstsToKill.push_back(bitcast);
3209 llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
3210 if (!call)
return nullptr;
3212 bool doRetainAutorelease;
3215 doRetainAutorelease =
true;
3216 }
else if (call->getCalledOperand() ==
3218 doRetainAutorelease =
false;
3226 llvm::Instruction *prev = call->getPrevNode();
3228 if (isa<llvm::BitCastInst>(prev)) {
3229 prev = prev->getPrevNode();
3232 assert(isa<llvm::CallInst>(prev));
3233 assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3235 InstsToKill.push_back(prev);
3241 result = call->getArgOperand(0);
3242 InstsToKill.push_back(call);
3246 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
3247 if (!bitcast->hasOneUse())
break;
3248 InstsToKill.push_back(bitcast);
3249 result = bitcast->getOperand(0);
3253 for (
auto *I : InstsToKill)
3254 I->eraseFromParent();
3257 if (doRetainAutorelease)
3261 return CGF.
Builder.CreateBitCast(result, resultType);
3266 llvm::Value *result) {
3269 dyn_cast_or_null<ObjCMethodDecl>(CGF.
CurCodeDecl);
3270 if (!method)
return nullptr;
3275 llvm::CallInst *retainCall =
3276 dyn_cast<llvm::CallInst>(result->stripPointerCasts());
3277 if (!retainCall || retainCall->getCalledOperand() !=
3282 llvm::Value *retainedValue = retainCall->getArgOperand(0);
3283 llvm::LoadInst *load =
3284 dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
3285 if (!load || load->isAtomic() || load->isVolatile() ||
3292 llvm::Type *resultType = result->getType();
3294 assert(retainCall->use_empty());
3295 retainCall->eraseFromParent();
3298 return CGF.
Builder.CreateBitCast(load, resultType);
3305 llvm::Value *result) {
3326 auto GetStoreIfValid = [&CGF](llvm::User *
U) -> llvm::StoreInst * {
3327 auto *SI = dyn_cast<llvm::StoreInst>(
U);
3333 assert(!SI->isAtomic() && !SI->isVolatile());
3341 llvm::BasicBlock *IP = CGF.
Builder.GetInsertBlock();
3342 if (IP->empty())
return nullptr;
3346 for (llvm::Instruction &I : make_range(IP->rbegin(), IP->rend())) {
3347 if (isa<llvm::BitCastInst>(&I))
3349 if (
auto *II = dyn_cast<llvm::IntrinsicInst>(&I))
3350 if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
3353 return GetStoreIfValid(&I);
3358 llvm::StoreInst *store =
3360 if (!store)
return nullptr;
3364 llvm::BasicBlock *StoreBB = store->getParent();
3365 llvm::BasicBlock *IP = CGF.
Builder.GetInsertBlock();
3366 while (IP != StoreBB) {
3367 if (!(IP = IP->getSinglePredecessor()))
3383 int BitWidth,
int CharWidth) {
3384 assert(CharWidth <= 64);
3385 assert(
static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3388 if (BitOffset >= CharWidth) {
3389 Pos += BitOffset / CharWidth;
3390 BitOffset = BitOffset % CharWidth;
3393 const uint64_t
Used = (uint64_t(1) << CharWidth) - 1;
3394 if (BitOffset + BitWidth >= CharWidth) {
3395 Bits[Pos++] |= (
Used << BitOffset) &
Used;
3396 BitWidth -= CharWidth - BitOffset;
3400 while (BitWidth >= CharWidth) {
3402 BitWidth -= CharWidth;
3406 Bits[Pos++] |= (
Used >> (CharWidth - BitWidth)) << BitOffset;
3414 int StorageSize,
int BitOffset,
int BitWidth,
3415 int CharWidth,
bool BigEndian) {
3418 setBitRange(TmpBits, BitOffset, BitWidth, CharWidth);
3421 std::reverse(TmpBits.begin(), TmpBits.end());
3423 for (uint64_t
V : TmpBits)
3424 Bits[StorageOffset++] |=
V;
3455 BFI.
Size, CharWidth,
3477 auto Src = TmpBits.begin();
3478 auto Dst = Bits.begin() +
Offset + I * Size;
3479 for (
int J = 0; J < Size; ++J)
3499 std::fill_n(Bits.begin() +
Offset, Size,
3504 int Pos,
int Size,
int CharWidth,
3509 for (
auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size;
P != E;
3511 Mask = (Mask << CharWidth) | *
P;
3513 auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3515 Mask = (Mask << CharWidth) | *--
P;
3524 llvm::IntegerType *ITy,
3526 assert(Src->getType() == ITy);
3527 assert(ITy->getScalarSizeInBits() <= 64);
3530 int Size = DataLayout.getTypeStoreSize(ITy);
3538 return Builder.CreateAnd(Src, Mask,
"cmse.clear");
3544 llvm::ArrayType *ATy,
3547 int Size = DataLayout.getTypeStoreSize(ATy);
3554 ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3556 llvm::Value *R = llvm::PoisonValue::get(ATy);
3557 for (
int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
3559 DataLayout.isBigEndian());
3560 MaskIndex += CharsPerElt;
3561 llvm::Value *T0 =
Builder.CreateExtractValue(Src, I);
3562 llvm::Value *T1 =
Builder.CreateAnd(T0, Mask,
"cmse.clear");
3563 R =
Builder.CreateInsertValue(R, T1, I);
3590 llvm::DebugLoc RetDbgLoc;
3591 llvm::Value *RV =
nullptr;
3601 llvm::Function::arg_iterator EI =
CurFn->arg_end();
3603 llvm::Value *ArgStruct = &*EI;
3607 cast<llvm::GetElementPtrInst>(SRet)->getResultElementType();
3613 auto AI =
CurFn->arg_begin();
3652 if (llvm::StoreInst *SI =
3658 RetDbgLoc = SI->getDebugLoc();
3660 RV = SI->getValueOperand();
3661 SI->eraseFromParent();
3684 if (
auto *FD = dyn_cast<FunctionDecl>(
CurCodeDecl))
3685 RT = FD->getReturnType();
3686 else if (
auto *MD = dyn_cast<ObjCMethodDecl>(
CurCodeDecl))
3687 RT = MD->getReturnType();
3691 llvm_unreachable(
"Unexpected function/method type");
3711 for (
unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3718 results.push_back(elt);
3722 if (results.size() == 1) {
3730 RV = llvm::PoisonValue::get(returnType);
3731 for (
unsigned i = 0, e = results.size(); i != e; ++i) {
3732 RV =
Builder.CreateInsertValue(RV, results[i], i);
3739 llvm_unreachable(
"Invalid ABI kind for return argument");
3742 llvm::Instruction *
Ret;
3748 auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType());
3759 Ret->setDebugLoc(std::move(RetDbgLoc));
3772 ReturnsNonNullAttr *RetNNAttr =
nullptr;
3773 if (
SanOpts.
has(SanitizerKind::ReturnsNonnullAttribute))
3776 if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
3784 assert(!requiresReturnValueNullabilityCheck() &&
3785 "Cannot check nullability and the nonnull attribute");
3786 AttrLoc = RetNNAttr->getLocation();
3787 CheckKind = SanitizerKind::ReturnsNonnullAttribute;
3788 Handler = SanitizerHandler::NonnullReturn;
3790 if (
auto *DD = dyn_cast<DeclaratorDecl>(
CurCodeDecl))
3791 if (
auto *TSI = DD->getTypeSourceInfo())
3793 AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
3794 CheckKind = SanitizerKind::NullabilityReturn;
3795 Handler = SanitizerHandler::NullabilityReturn;
3798 SanitizerScope SanScope(
this);
3805 llvm::Value *CanNullCheck =
Builder.CreateIsNotNull(SLocPtr);
3806 if (requiresReturnValueNullabilityCheck())
3808 Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);
3809 Builder.CreateCondBr(CanNullCheck, Check, NoCheck);
3813 llvm::Value *Cond =
Builder.CreateIsNotNull(RV);
3815 llvm::Value *DynamicData[] = {SLocPtr};
3816 EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);
3836 llvm::Type *IRPtrTy = IRTy->getPointerTo();
3837 llvm::Value *Placeholder = llvm::PoisonValue::get(IRPtrTy->getPointerTo());
3868 if (
type->isReferenceType()) {
3877 param->
hasAttr<NSConsumedAttr>() &&
3878 type->isObjCRetainableType()) {
3881 llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));
3896 CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
3898 "cleanup for callee-destructed param not recorded");
3900 llvm::Instruction *isActive =
Builder.CreateUnreachable();
3906 return isa<llvm::ConstantPointerNull>(addr);
3915 "shouldn't have writeback for provably null argument");
3917 llvm::BasicBlock *contBB =
nullptr;
3921 bool provablyNonNull = llvm::isKnownNonZero(srcAddr.
getPointer(),
3923 if (!provablyNonNull) {
3927 llvm::Value *isNull =
3929 CGF.
Builder.CreateCondBr(isNull, contBB, writebackBB);
3938 "icr.writeback-cast");
3947 if (writeback.
ToUse) {
3972 if (!provablyNonNull)
3987 for (
const auto &I : llvm::reverse(Cleanups)) {
3989 I.IsActiveIP->eraseFromParent();
3995 if (uop->getOpcode() == UO_AddrOf)
3996 return uop->getSubExpr();
4026 llvm::PointerType *destType =
4028 llvm::Type *destElemType =
4045 CodeGenFunction::ConditionalEvaluation condEval(CGF);
4051 llvm::ConstantPointerNull::get(cast<llvm::PointerType>(destElemType));
4055 llvm::BasicBlock *contBB =
nullptr;
4056 llvm::BasicBlock *originBB =
nullptr;
4059 llvm::Value *finalArgument;
4061 bool provablyNonNull = llvm::isKnownNonZero(srcAddr.
getPointer(),
4063 if (provablyNonNull) {
4066 llvm::Value *isNull =
4069 finalArgument = CGF.
Builder.CreateSelect(isNull,
4070 llvm::ConstantPointerNull::get(destType),
4076 originBB = CGF.
Builder.GetInsertBlock();
4079 CGF.
Builder.CreateCondBr(isNull, contBB, copyBB);
4081 condEval.begin(CGF);
4085 llvm::Value *valueToUse =
nullptr;
4093 src = CGF.
Builder.CreateBitCast(src, destElemType,
"icr.cast");
4110 if (shouldCopy && !provablyNonNull) {
4111 llvm::BasicBlock *copyBB = CGF.
Builder.GetInsertBlock();
4116 llvm::PHINode *phiToUse = CGF.
Builder.CreatePHI(valueToUse->getType(), 2,
4118 phiToUse->addIncoming(valueToUse, copyBB);
4119 phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
4121 valueToUse = phiToUse;
4136 StackBase = CGF.
Builder.CreateCall(F, {},
"inalloca.save");
4142 llvm::Function *F = CGF.
CGM.
getIntrinsic(llvm::Intrinsic::stackrestore);
4143 CGF.
Builder.CreateCall(F, StackBase);
4151 if (!AC.getDecl() || !(
SanOpts.
has(SanitizerKind::NonnullAttribute) ||
4156 auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(ParmNum) :
nullptr;
4157 unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
4160 const NonNullAttr *NNAttr =
nullptr;
4161 if (
SanOpts.
has(SanitizerKind::NonnullAttribute))
4164 bool CanCheckNullability =
false;
4165 if (
SanOpts.
has(SanitizerKind::NullabilityArg) && !NNAttr && PVD) {
4166 auto Nullability = PVD->getType()->getNullability();
4167 CanCheckNullability = Nullability &&
4169 PVD->getTypeSourceInfo();
4172 if (!NNAttr && !CanCheckNullability)
4179 AttrLoc = NNAttr->getLocation();
4180 CheckKind = SanitizerKind::NonnullAttribute;
4181 Handler = SanitizerHandler::NonnullArg;
4183 AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4184 CheckKind = SanitizerKind::NullabilityArg;
4185 Handler = SanitizerHandler::NullabilityArg;
4188 SanitizerScope SanScope(
this);
4190 llvm::Constant *StaticData[] = {
4192 llvm::ConstantInt::get(
Int32Ty, ArgNo + 1),
4194 EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, std::nullopt);
4210 return llvm::any_of(ArgTypes, [&](
QualType Ty) {
4221 return classDecl->getTypeParamListAsWritten();
4225 return catDecl->getTypeParamList();
4235 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4236 AbstractCallee AC,
unsigned ParamsToSkip, EvaluationOrder Order) {
4239 assert((ParamsToSkip == 0 ||
Prototype.P) &&
4240 "Can't skip parameters if type info is not provided");
4250 bool IsVariadic =
false;
4257 ArgTypes.assign(MD->param_type_begin() + ParamsToSkip,
4258 MD->param_type_end());
4262 ExplicitCC = FPT->getExtInfo().getCC();
4263 ArgTypes.assign(FPT->param_type_begin() + ParamsToSkip,
4264 FPT->param_type_end());
4272 assert(Arg != ArgRange.end() &&
"Running over edge of argument list!");
4274 (isGenericMethod || Ty->isVariablyModifiedType() ||
4275 Ty.getNonReferenceType()->isObjCRetainableType() ||
4277 .getCanonicalType(Ty.getNonReferenceType())
4279 getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&
4280 "type mismatch in call argument!");
4286 assert((Arg == ArgRange.end() || IsVariadic) &&
4287 "Extra arguments in non-variadic function!");
4292 for (
auto *A : llvm::drop_begin(ArgRange, ArgTypes.size()))
4293 ArgTypes.push_back(IsVariadic ? getVarArgType(A) : A->getType());
4294 assert((
int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4306 auto MaybeEmitImplicitObjectSize = [&](
unsigned I,
const Expr *Arg,
4308 if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
4310 auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4317 assert(EmittedArg.getScalarVal() &&
"We emitted nothing for the arg?");
4318 llvm::Value *
V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T,
4319 EmittedArg.getScalarVal(),
4325 std::swap(Args.back(), *(&Args.back() - 1));
4330 assert(
getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4331 "inalloca only supported on x86");
4336 size_t CallArgsStart = Args.size();
4337 for (
unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
4338 unsigned Idx = LeftToRight ? I : E - I - 1;
4340 unsigned InitialArgSize = Args.size();
4343 assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
4344 getContext().hasSameUnqualifiedType((*Arg)->getType(),
4346 (isa<ObjCMethodDecl>(AC.getDecl()) &&
4348 "Argument and parameter types don't match");
4352 assert(InitialArgSize + 1 == Args.size() &&
4353 "The code below depends on only adding one arg per EmitCallArg");
4354 (void)InitialArgSize;
4357 if (!Args.back().hasLValue()) {
4358 RValue RVArg = Args.back().getKnownRValue();
4360 ParamsToSkip + Idx);
4364 MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4371 std::reverse(Args.begin() + CallArgsStart, Args.end());
4379 : Addr(Addr), Ty(Ty) {}
4397struct DisableDebugLocationUpdates {
4399 bool disabledDebugInfo;
4401 if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && CGF.
getDebugInfo()))
4404 ~DisableDebugLocationUpdates() {
4405 if (disabledDebugInfo)
4441 DisableDebugLocationUpdates Dis(*
this, E);
4443 = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
4449 "reference binding to unmaterialized r-value!");
4461 if (
type->isRecordType() &&
4468 bool DestroyedInCallee =
true, NeedsEHCleanup =
true;
4469 if (
const auto *RD =
type->getAsCXXRecordDecl())
4470 DestroyedInCallee = RD->hasNonTrivialDestructor();
4474 if (DestroyedInCallee)
4481 if (DestroyedInCallee && NeedsEHCleanup) {
4488 llvm::Instruction *IsActive =
Builder.CreateUnreachable();
4494 if (HasAggregateEvalKind && isa<ImplicitCastExpr>(E) &&
4495 cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue) {
4505QualType CodeGenFunction::getVarArgType(
const Expr *Arg) {
4509 if (!
getTarget().getTriple().isOSWindows())
4526CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
4529 Inst->setMetadata(
"clang.arc.no_objc_arc_exceptions",
4536 const llvm::Twine &name) {
4544 const llvm::Twine &name) {
4546 call->setDoesNotThrow();
4553 const llvm::Twine &name) {
4568 if (
auto *CalleeFn = dyn_cast<llvm::Function>(
Callee->stripPointerCasts())) {
4569 if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {
4570 auto IID = CalleeFn->getIntrinsicID();
4571 if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))
4584 const llvm::Twine &name) {
4585 llvm::CallInst *call =
Builder.CreateCall(
4598 llvm::InvokeInst *invoke =
4604 invoke->setDoesNotReturn();
4607 llvm::CallInst *call =
Builder.CreateCall(callee, args, BundleList);
4608 call->setDoesNotReturn();
4617 const Twine &name) {
4625 const Twine &name) {
4635 const Twine &Name) {
4640 llvm::CallBase *Inst;
4642 Inst =
Builder.CreateCall(Callee, Args, BundleList, Name);
4645 Inst =
Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList,
4653 AddObjCARCExceptionMetadata(Inst);
4658void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
4660 DeferredReplacements.push_back(
4661 std::make_pair(llvm::WeakTrackingVH(Old), New));
4668[[nodiscard]] llvm::AttributeList
4669maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
4670 const llvm::AttributeList &Attrs,
4671 llvm::Align NewAlign) {
4672 llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
4673 if (CurAlign >= NewAlign)
4675 llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign);
4676 return Attrs.removeRetAttribute(Ctx, llvm::Attribute::AttrKind::Alignment)
4677 .addRetAttribute(Ctx, AlignAttr);
4680template <
typename AlignedAttrTy>
class AbstractAssumeAlignedAttrEmitter {
4685 const AlignedAttrTy *AA =
nullptr;
4687 llvm::Value *Alignment =
nullptr;
4688 llvm::ConstantInt *OffsetCI =
nullptr;
4694 AA = FuncDecl->
getAttr<AlignedAttrTy>();
4699 [[nodiscard]] llvm::AttributeList
4700 TryEmitAsCallSiteAttribute(
const llvm::AttributeList &Attrs) {
4701 if (!AA || OffsetCI || CGF.
SanOpts.
has(SanitizerKind::Alignment))
4703 const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4708 if (!AlignmentCI->getValue().isPowerOf2())
4710 llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4713 AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4725 AA->getLocation(), Alignment, OffsetCI);
4731class AssumeAlignedAttrEmitter final
4732 :
public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
4735 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4739 Alignment = cast<llvm::ConstantInt>(CGF.
EmitScalarExpr(AA->getAlignment()));
4742 if (OffsetCI->isNullValue())
4749class AllocAlignAttrEmitter final
4750 :
public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
4754 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4758 Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
4767 if (
auto *VT = dyn_cast<llvm::VectorType>(Ty))
4768 return VT->getPrimitiveSizeInBits().getKnownMinValue();
4769 if (
auto *AT = dyn_cast<llvm::ArrayType>(Ty))
4772 unsigned MaxVectorWidth = 0;
4773 if (
auto *ST = dyn_cast<llvm::StructType>(Ty))
4774 for (
auto *I : ST->elements())
4776 return MaxVectorWidth;
4783 llvm::CallBase **callOrInvoke,
bool IsMustTail,
4796 const Decl *TargetDecl =
Callee.getAbstractInfo().getCalleeDecl().getDecl();
4797 if (
const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
4804 if (TargetDecl->
hasAttr<AlwaysInlineAttr>() &&
4805 (TargetDecl->
hasAttr<TargetAttr>() ||
4812 CGM, Loc, dyn_cast_or_null<FunctionDecl>(
CurCodeDecl), FD, CallArgs);
4825 assert(IRFuncTy ==
Callee.getVirtualFunctionType());
4827 llvm::PointerType *PtrTy =
4828 llvm::cast<llvm::PointerType>(
Callee.getFunctionPointer()->getType());
4829 assert(PtrTy->isOpaqueOrPointeeTypeMatches(IRFuncTy));
4839 if (llvm::StructType *ArgStruct = CallInfo.
getArgStruct()) {
4842 llvm::AllocaInst *AI;
4844 IP = IP->getNextNode();
4845 AI =
new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(),
4851 AI->setAlignment(Align.getAsAlign());
4852 AI->setUsedWithInAlloca(
true);
4853 assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
4854 ArgMemory =
Address(AI, ArgStruct, Align);
4857 ClangToLLVMArgMapping IRFunctionArgs(
CGM.
getContext(), CallInfo);
4864 llvm::Value *UnusedReturnSizePtr =
nullptr;
4871 llvm::TypeSize size =
4876 if (IRFunctionArgs.hasSRetArg()) {
4877 IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.
getPointer();
4894 assert(CallInfo.
arg_size() == CallArgs.size() &&
4895 "Mismatch between function signature & arguments.");
4898 for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
4899 I != E; ++I, ++info_it, ++ArgNo) {
4903 if (IRFunctionArgs.hasPaddingArg(ArgNo))
4904 IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
4907 unsigned FirstIRArg, NumIRArgs;
4908 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
4910 bool ArgHasMaybeUndefAttr =
4915 assert(NumIRArgs == 0);
4916 assert(
getTarget().getTriple().getArch() == llvm::Triple::x86);
4917 if (I->isAggregate()) {
4919 ? I->getKnownLValue().getAddress(*
this)
4920 : I->getKnownRValue().getAggregateAddress();
4921 llvm::Instruction *Placeholder =
4926 CGBuilderTy::InsertPoint IP =
Builder.saveIP();
4927 Builder.SetInsertPoint(Placeholder);
4940 deferPlaceholderReplacement(Placeholder, Addr.
getPointer());
4945 I->Ty,
getContext().getTypeAlignInChars(I->Ty),
4946 "indirect-arg-temp");
4947 I->copyInto(*
this, Addr);
4959 I->copyInto(*
this, Addr);
4966 assert(NumIRArgs == 1);
4967 if (!I->isAggregate()) {
4973 if (ArgHasMaybeUndefAttr)
4975 IRCallArgs[FirstIRArg] = Val;
4977 I->copyInto(*
this, Addr);
4988 ? I->getKnownLValue().getAddress(*
this)
4989 : I->getKnownRValue().getAggregateAddress();
4994 assert((FirstIRArg >= IRFuncTy->getNumParams() ||
4995 IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
4996 TD->getAllocaAddrSpace()) &&
4997 "indirect argument must be in alloca address space");
4999 bool NeedCopy =
false;
5002 llvm::getOrEnforceKnownAlignment(
V, Align.
getAsAlign(), *TD) <
5005 }
else if (I->hasLValue()) {
5006 auto LV = I->getKnownLValue();
5023 Addr.
getType()->getAddressSpace() != IRFuncTy->
5034 if (ArgHasMaybeUndefAttr)
5036 IRCallArgs[FirstIRArg] = Val;
5039 llvm::TypeSize ByvalTempElementSize =
5041 llvm::Value *LifetimeSize =
5046 CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize);
5049 I->copyInto(*
this, AI);
5052 auto *T = llvm::PointerType::getWithSamePointeeType(
5053 cast<llvm::PointerType>(
V->getType()),
5059 if (ArgHasMaybeUndefAttr)
5060 Val =
Builder.CreateFreeze(Val);
5061 IRCallArgs[FirstIRArg] = Val;
5068 assert(NumIRArgs == 0);
5076 assert(NumIRArgs == 1);
5078 if (!I->isAggregate())
5079 V = I->getKnownRValue().getScalarVal();
5082 I->hasLValue() ? I->getKnownLValue().getAddress(*
this)
5083 : I->getKnownRValue().getAggregateAddress());