32#include "llvm/ADT/Hashing.h"
33#include "llvm/ADT/StringExtras.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/Intrinsics.h"
36#include "llvm/IR/IntrinsicsWebAssembly.h"
37#include "llvm/IR/LLVMContext.h"
38#include "llvm/IR/MDBuilder.h"
39#include "llvm/IR/MatrixBuilder.h"
40#include "llvm/Passes/OptimizationLevel.h"
41#include "llvm/Support/ConvertUTF.h"
42#include "llvm/Support/MathExtras.h"
43#include "llvm/Support/Path.h"
44#include "llvm/Support/SaveAndRestore.h"
45#include "llvm/Support/xxhash.h"
46#include "llvm/Transforms/Utils/SanitizerStats.h"
52using namespace CodeGen;
56 "ubsan-unique-traps", llvm::cl::Optional,
57 llvm::cl::desc(
"Deoptimize traps for UBSAN so there is 1 trap per check"),
58 llvm::cl::init(
false));
66Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty,
69 llvm::Value *ArraySize) {
79 llvm::Value *ArraySize,
84 llvm::Value *
V = Alloca.getPointer();
91 llvm::IRBuilderBase::InsertPointGuard IPG(
Builder);
99 Ty->getPointerTo(DestAddrSpace),
true);
110 llvm::Value *ArraySize) {
112 return Builder.CreateAlloca(Ty, ArraySize, Name);
140 const Twine &Name,
Address *Alloca) {
145 auto *ArrayTy = cast<llvm::ArrayType>(
Result.getElementType());
146 auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
147 ArrayTy->getNumElements());
177 CGFPOptionsRAII FPOptsRAII(*
this, E);
195 if (
const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
218 if (!ignoreResult && aggSlot.
isIgnored())
223 llvm_unreachable(
"bad evaluation kind");
265 llvm_unreachable(
"bad evaluation kind");
306 VD && isa<VarDecl>(VD) && VD->
hasAttr<ObjCPreciseLifetimeAttr>();
327 llvm_unreachable(
"temporary cannot have dynamic storage duration");
329 llvm_unreachable(
"unknown storage duration");
337 auto *ClassDecl = cast<CXXRecordDecl>(RT->getDecl());
338 if (!ClassDecl->hasTrivialDestructor())
339 ReferenceTemporaryDtor = ClassDecl->getDestructor();
342 if (!ReferenceTemporaryDtor)
349 llvm::FunctionCallee CleanupFn;
350 llvm::Constant *CleanupArg;
353 ReferenceTemporary, E->
getType(),
356 CleanupArg = llvm::Constant::getNullValue(CGF.
Int8PtrTy);
360 CleanupArg = cast<llvm::Constant>(ReferenceTemporary.
getPointer());
375 ReferenceTemporary, E->
getType(),
381 llvm_unreachable(
"temporary cannot have dynamic storage duration");
403 auto *GV =
new llvm::GlobalVariable(
405 llvm::GlobalValue::PrivateLinkage, Init,
".ref.tmp",
nullptr,
406 llvm::GlobalValue::NotThreadLocal,
410 llvm::Constant *
C = GV;
412 C = TCG.performAddrSpaceCast(
414 GV->getValueType()->getPointerTo(
417 return Address(
C, GV->getValueType(), alignment);
426 llvm_unreachable(
"temporary can't have dynamic storage duration");
428 llvm_unreachable(
"unknown storage duration");
442 "Reference should never be pseudo-strong!");
450 if (
auto *Var = dyn_cast<llvm::GlobalVariable>(
Object.getPointer())) {
453 Var, Ty->getPointerTo(
Object.getAddressSpace())),
454 Ty,
Object.getAlignment());
462 if (Var->hasInitializer())
471 default: llvm_unreachable(
"expected scalar or aggregate expression");
494 for (
const auto &Ignored : CommaLHSs)
497 if (
const auto *opaque = dyn_cast<OpaqueValueExpr>(E)) {
498 if (opaque->getType()->isRecordType()) {
499 assert(Adjustments.empty());
507 if (
auto *Var = dyn_cast<llvm::GlobalVariable>(
508 Object.getPointer()->stripPointerCasts())) {
511 cast<llvm::Constant>(
Object.getPointer()),
512 TemporaryType->getPointerTo()),
518 if (!Var->hasInitializer()) {
534 if (!ShouldEmitLifetimeMarkers)
544 ConditionalEvaluation *OldConditional =
nullptr;
545 CGBuilderTy::InsertPoint OldIP;
551 OldConditional = OutermostConditional;
552 OutermostConditional =
nullptr;
555 llvm::BasicBlock *
Block = OldConditional->getStartingBlock();
556 Builder.restoreIP(CGBuilderTy::InsertPoint(
557 Block, llvm::BasicBlock::iterator(
Block->back())));
567 if (OldConditional) {
568 OutermostConditional = OldConditional;
585 switch (Adjustment.Kind) {
589 Adjustment.DerivedToBase.BasePath->path_begin(),
590 Adjustment.DerivedToBase.BasePath->path_end(),
598 "materialized temporary field is not a simple lvalue");
639 const llvm::Constant *Elts) {
640 return cast<llvm::ConstantInt>(Elts->getAggregateElement(Idx))
647 llvm::Value *KMul = Builder.getInt64(0x9ddfea08eb382d69ULL);
648 llvm::Value *K47 = Builder.getInt64(47);
649 llvm::Value *A0 = Builder.CreateMul(Builder.CreateXor(Low, High), KMul);
650 llvm::Value *A1 = Builder.CreateXor(Builder.CreateLShr(A0, K47), A0);
651 llvm::Value *B0 = Builder.CreateMul(Builder.CreateXor(High, A1), KMul);
652 llvm::Value *B1 = Builder.CreateXor(Builder.CreateLShr(B0, K47), B0);
653 return Builder.CreateMul(B1, KMul);
680 llvm::Value *ArraySize) {
687 if (Ptr->getType()->getPointerAddressSpace())
695 SanitizerScope SanScope(
this);
698 llvm::BasicBlock *Done =
nullptr;
703 auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Ptr->stripPointerCasts());
706 llvm::Value *IsNonNull =
nullptr;
707 bool IsGuaranteedNonNull =
708 SkippedChecks.
has(SanitizerKind::Null) || PtrToAlloca;
710 if ((
SanOpts.
has(SanitizerKind::Null) || AllowNullPointers) &&
711 !IsGuaranteedNonNull) {
713 IsNonNull =
Builder.CreateIsNotNull(Ptr);
717 IsGuaranteedNonNull = IsNonNull == True;
720 if (!IsGuaranteedNonNull) {
721 if (AllowNullPointers) {
726 Builder.CreateCondBr(IsNonNull, Rest, Done);
729 Checks.push_back(std::make_pair(IsNonNull, SanitizerKind::Null));
735 !SkippedChecks.
has(SanitizerKind::ObjectSize) &&
738 llvm::Value *
Size = llvm::ConstantInt::get(
IntPtrTy, TySize);
743 llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Size);
744 if (!ConstantSize || !ConstantSize->isNullValue()) {
750 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::objectsize, Tys);
751 llvm::Value *Min =
Builder.getFalse();
752 llvm::Value *NullIsUnknown =
Builder.getFalse();
754 llvm::Value *LargeEnough =
Builder.CreateICmpUGE(
755 Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic}), Size);
756 Checks.push_back(std::make_pair(LargeEnough, SanitizerKind::ObjectSize));
760 llvm::MaybeAlign AlignVal;
761 llvm::Value *PtrAsInt =
nullptr;
764 !SkippedChecks.
has(SanitizerKind::Alignment)) {
772 if (AlignVal && *AlignVal > llvm::Align(1) &&
773 (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
775 llvm::Value *Align =
Builder.CreateAnd(
776 PtrAsInt, llvm::ConstantInt::get(
IntPtrTy, AlignVal->value() - 1));
777 llvm::Value *Aligned =
780 Checks.push_back(std::make_pair(Aligned, SanitizerKind::Alignment));
784 if (Checks.size() > 0) {
785 llvm::Constant *StaticData[] = {
787 llvm::ConstantInt::get(
Int8Ty, AlignVal ? llvm::Log2(*AlignVal) : 1),
788 llvm::ConstantInt::get(
Int8Ty, TCK)};
789 EmitCheck(Checks, SanitizerHandler::TypeMismatch, StaticData,
790 PtrAsInt ? PtrAsInt : Ptr);
805 if (!IsGuaranteedNonNull) {
807 IsNonNull =
Builder.CreateIsNotNull(Ptr);
811 Builder.CreateCondBr(IsNonNull, VptrNotNull, Done);
821 llvm::raw_svector_ostream Out(MangledName);
828 llvm::hash_code TypeHash =
hash_value(Out.str());
831 llvm::Value *Low = llvm::ConstantInt::get(
Int64Ty, TypeHash);
840 const int CacheSize = 128;
841 llvm::Type *HashTable = llvm::ArrayType::get(
IntPtrTy, CacheSize);
843 "__ubsan_vptr_type_cache");
844 llvm::Value *Slot =
Builder.CreateAnd(Hash,
847 llvm::Value *Indices[] = {
Builder.getInt32(0), Slot };
856 llvm::Value *EqualHash =
Builder.CreateICmpEQ(CacheVal, Hash);
857 llvm::Constant *StaticData[] = {
861 llvm::ConstantInt::get(
Int8Ty, TCK)
863 llvm::Value *DynamicData[] = { Ptr, Hash };
864 EmitCheck(std::make_pair(EqualHash, SanitizerKind::Vptr),
865 SanitizerHandler::DynamicTypeCacheMiss, StaticData,
879 uint64_t EltSize =
C.getTypeSizeInChars(EltTy).getQuantity();
887 auto *ParamDecl = dyn_cast<ParmVarDecl>(ArrayDeclRef->getDecl());
891 auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();
896 int POSType = POSAttr->getType();
897 if (POSType != 0 && POSType != 1)
901 auto PassedSizeIt = SizeArguments.find(ParamDecl);
902 if (PassedSizeIt == SizeArguments.end())
906 assert(LocalDeclMap.count(PassedSizeDecl) &&
"Passed size not loadable");
907 Address AddrOfSize = LocalDeclMap.find(PassedSizeDecl)->second;
910 llvm::Value *SizeOfElement =
911 llvm::ConstantInt::get(SizeInBytes->getType(), EltSize);
912 return Builder.CreateUDiv(SizeInBytes, SizeOfElement);
921 StrictFlexArraysLevel) {
924 IndexedType =
Base->getType();
925 return CGF.
Builder.getInt32(VT->getNumElements());
930 if (
const auto *CE = dyn_cast<CastExpr>(
Base)) {
931 if (CE->getCastKind() == CK_ArrayToPointerDecay &&
932 !CE->getSubExpr()->isFlexibleArrayMemberLike(CGF.
getContext(),
933 StrictFlexArraysLevel)) {
934 IndexedType = CE->getSubExpr()->getType();
936 if (
const auto *CAT = dyn_cast<ConstantArrayType>(AT))
937 return CGF.
Builder.getInt(CAT->getSize());
938 else if (
const auto *VAT = dyn_cast<VariableArrayType>(AT))
944 QualType EltTy{
Base->getType()->getPointeeOrArrayElementType(), 0};
946 IndexedType =
Base->getType();
954 llvm::Value *Index,
QualType IndexType,
956 assert(
SanOpts.
has(SanitizerKind::ArrayBounds) &&
957 "should not be called unless adding bounds checks");
958 SanitizerScope SanScope(
this);
970 llvm::Value *IndexVal =
Builder.CreateIntCast(Index,
SizeTy, IndexSigned);
971 llvm::Value *BoundVal =
Builder.CreateIntCast(Bound,
SizeTy,
false);
973 llvm::Constant *StaticData[] = {
978 llvm::Value *Check = Accessed ?
Builder.CreateICmpULT(IndexVal, BoundVal)
979 :
Builder.CreateICmpULE(IndexVal, BoundVal);
980 EmitCheck(std::make_pair(Check, SanitizerKind::ArrayBounds),
981 SanitizerHandler::OutOfBounds, StaticData, Index);
987 bool isInc,
bool isPre) {
990 llvm::Value *NextVal;
991 if (isa<llvm::IntegerType>(InVal.first->getType())) {
992 uint64_t AmountVal = isInc ? 1 : -1;
993 NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal,
true);
996 NextVal =
Builder.CreateAdd(InVal.first, NextVal, isInc ?
"inc" :
"dec");
999 llvm::APFloat FVal(
getContext().getFloatTypeSemantics(ElemTy), 1);
1005 NextVal =
Builder.CreateFAdd(InVal.first, NextVal, isInc ?
"inc" :
"dec");
1018 return isPre ? IncVal : InVal;
1028 DI->EmitExplicitCastType(E->
getType());
1045 if (
const CastExpr *CE = dyn_cast<CastExpr>(E)) {
1046 if (
const auto *ECE = dyn_cast<ExplicitCastExpr>(CE))
1049 switch (CE->getCastKind()) {
1053 case CK_AddressSpaceConversion:
1054 if (
auto PtrTy = CE->getSubExpr()->getType()->getAs<
PointerType>()) {
1055 if (PtrTy->getPointeeType()->isVoidType())
1061 CE->getSubExpr(), &InnerBaseInfo, &InnerTBAAInfo, IsKnownNonNull);
1062 if (BaseInfo) *BaseInfo = InnerBaseInfo;
1063 if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;
1065 if (isa<ExplicitCastExpr>(CE)) {
1069 E->
getType(), &TargetTypeBaseInfo, &TargetTypeTBAAInfo);
1083 if (CGF.
SanOpts.
has(SanitizerKind::CFIUnrelatedCast) &&
1084 CE->getCastKind() == CK_BitCast) {
1092 llvm::Type *ElemTy =
1095 if (CE->getCastKind() == CK_AddressSpaceConversion)
1103 case CK_ArrayToPointerDecay:
1107 case CK_UncheckedDerivedToBase:
1108 case CK_DerivedToBase: {
1115 CE->getSubExpr(), BaseInfo,
nullptr,
1117 CE->getCastKind() == CK_UncheckedDerivedToBase));
1118 auto Derived = CE->getSubExpr()->
getType()->getPointeeCXXRecordDecl();
1120 Addr, Derived, CE->path_begin(), CE->path_end(),
1133 if (UO->getOpcode() == UO_AddrOf) {
1142 if (
auto *Call = dyn_cast<CallExpr>(E)) {
1143 switch (Call->getBuiltinCallee()) {
1146 case Builtin::BIaddressof:
1147 case Builtin::BI__addressof:
1148 case Builtin::BI__builtin_addressof: {
1182 return Builder.CreateICmpNE(
V, llvm::Constant::getNullValue(
V->getType()));
1193 llvm::Value *
U = llvm::UndefValue::get(EltTy);
1208 llvm_unreachable(
"bad evaluation kind");
1228 while (!isa<CXXThisExpr>(
Base)) {
1230 if (isa<CXXDynamicCastExpr>(
Base))
1233 if (
const auto *CE = dyn_cast<CastExpr>(
Base)) {
1234 Base = CE->getSubExpr();
1235 }
else if (
const auto *PE = dyn_cast<ParenExpr>(
Base)) {
1236 Base = PE->getSubExpr();
1237 }
else if (
const auto *UO = dyn_cast<UnaryOperator>(
Base)) {
1238 if (UO->getOpcode() == UO_Extension)
1239 Base = UO->getSubExpr();
1251 if (
SanOpts.
has(SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(E))
1257 if (
const auto *ME = dyn_cast<MemberExpr>(E)) {
1260 SkippedChecks.
set(SanitizerKind::Alignment,
true);
1261 if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase()))
1262 SkippedChecks.
set(SanitizerKind::Null,
true);
1287 LValue LV = EmitLValueHelper(E, IsKnownNonNull);
1293LValue CodeGenFunction::EmitLValueHelper(
const Expr *E,
1299 case Expr::ObjCPropertyRefExprClass:
1300 llvm_unreachable(
"cannot emit a property reference directly");
1302 case Expr::ObjCSelectorExprClass:
1304 case Expr::ObjCIsaExprClass:
1306 case Expr::BinaryOperatorClass:
1308 case Expr::CompoundAssignOperatorClass: {
1311 Ty = AT->getValueType();
1316 case Expr::CallExprClass:
1317 case Expr::CXXMemberCallExprClass:
1318 case Expr::CXXOperatorCallExprClass:
1319 case Expr::UserDefinedLiteralClass:
1321 case Expr::CXXRewrittenBinaryOperatorClass:
1322 return EmitLValue(cast<CXXRewrittenBinaryOperator>(E)->getSemanticForm(),
1324 case Expr::VAArgExprClass:
1326 case Expr::DeclRefExprClass:
1328 case Expr::ConstantExprClass: {
1336 return EmitLValue(cast<ConstantExpr>(E)->getSubExpr(), IsKnownNonNull);
1338 case Expr::ParenExprClass:
1339 return EmitLValue(cast<ParenExpr>(E)->getSubExpr(), IsKnownNonNull);
1340 case Expr::GenericSelectionExprClass:
1341 return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr(),
1343 case Expr::PredefinedExprClass:
1345 case Expr::StringLiteralClass:
1347 case Expr::ObjCEncodeExprClass:
1349 case Expr::PseudoObjectExprClass:
1351 case Expr::InitListExprClass:
1353 case Expr::CXXTemporaryObjectExprClass:
1354 case Expr::CXXConstructExprClass:
1356 case Expr::CXXBindTemporaryExprClass:
1358 case Expr::CXXUuidofExprClass:
1360 case Expr::LambdaExprClass:
1363 case Expr::ExprWithCleanupsClass: {
1364 const auto *cleanups = cast<ExprWithCleanups>(E);
1365 RunCleanupsScope
Scope(*
this);
1372 Scope.ForceCleanup({&
V});
1382 case Expr::CXXDefaultArgExprClass: {
1383 auto *DAE = cast<CXXDefaultArgExpr>(E);
1384 CXXDefaultArgExprScope
Scope(*
this, DAE);
1385 return EmitLValue(DAE->getExpr(), IsKnownNonNull);
1387 case Expr::CXXDefaultInitExprClass: {
1388 auto *DIE = cast<CXXDefaultInitExpr>(E);
1389 CXXDefaultInitExprScope
Scope(*
this, DIE);
1390 return EmitLValue(DIE->getExpr(), IsKnownNonNull);
1392 case Expr::CXXTypeidExprClass:
1395 case Expr::ObjCMessageExprClass:
1397 case Expr::ObjCIvarRefExprClass:
1399 case Expr::StmtExprClass:
1401 case Expr::UnaryOperatorClass:
1403 case Expr::ArraySubscriptExprClass:
1405 case Expr::MatrixSubscriptExprClass:
1407 case Expr::OMPArraySectionExprClass:
1409 case Expr::ExtVectorElementExprClass:
1411 case Expr::CXXThisExprClass:
1413 case Expr::MemberExprClass:
1415 case Expr::CompoundLiteralExprClass:
1417 case Expr::ConditionalOperatorClass:
1419 case Expr::BinaryConditionalOperatorClass:
1421 case Expr::ChooseExprClass:
1422 return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr(), IsKnownNonNull);
1423 case Expr::OpaqueValueExprClass:
1425 case Expr::SubstNonTypeTemplateParmExprClass:
1426 return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement(),
1428 case Expr::ImplicitCastExprClass:
1429 case Expr::CStyleCastExprClass:
1430 case Expr::CXXFunctionalCastExprClass:
1431 case Expr::CXXStaticCastExprClass:
1432 case Expr::CXXDynamicCastExprClass:
1433 case Expr::CXXReinterpretCastExprClass:
1434 case Expr::CXXConstCastExprClass:
1435 case Expr::CXXAddrspaceCastExprClass:
1436 case Expr::ObjCBridgedCastExprClass:
1439 case Expr::MaterializeTemporaryExprClass:
1442 case Expr::CoawaitExprClass:
1444 case Expr::CoyieldExprClass:
1452 assert(
type.isCanonical());
1453 assert(!
type->isReferenceType());
1461 if (
const auto *RT = dyn_cast<RecordType>(
type))
1462 if (
const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1484 if (
const auto *ref = dyn_cast<ReferenceType>(
type)) {
1500CodeGenFunction::ConstantEmission
1506 if (isa<ParmVarDecl>(value)) {
1508 }
else if (
auto *var = dyn_cast<VarDecl>(value)) {
1510 }
else if (isa<EnumConstantDecl>(value)) {
1515 if (CEK ==
CEK_None)
return ConstantEmission();
1518 bool resultIsReference;
1524 resultIsReference =
false;
1525 resultType = refExpr->
getType();
1530 resultIsReference =
true;
1531 resultType = value->
getType();
1535 return ConstantEmission();
1540 return ConstantEmission();
1551 auto *MD = dyn_cast_or_null<CXXMethodDecl>(
CurCodeDecl);
1552 if (MD && MD->getParent()->isLambda() &&
1553 MD->getOverloadedOperator() == OO_Call) {
1556 if (
const VarDecl *VD = dyn_cast<const VarDecl>(D)) {
1557 if (!VD->hasAttr<CUDADeviceAttr>()) {
1558 return ConstantEmission();
1567 result.
Val, resultType);
1571 if (isa<VarDecl>(value)) {
1572 if (!
getContext().DeclMustBeEmitted(cast<VarDecl>(value)))
1575 assert(isa<EnumConstantDecl>(value));
1580 if (resultIsReference)
1598CodeGenFunction::ConstantEmission
1602 return ConstantEmission();
1606 const CodeGenFunction::ConstantEmission &Constant,
Expr *E) {
1607 assert(Constant &&
"not a constant");
1608 if (Constant.isReference())
1612 return Constant.getValue();
1627 return ET->getDecl()->getIntegerType()->isBooleanType();
1636 llvm::APInt &Min, llvm::APInt &End,
1637 bool StrictEnums,
bool IsBool) {
1639 bool IsRegularCPlusPlusEnum = CGF.
getLangOpts().CPlusPlus && StrictEnums &&
1641 if (!IsBool && !IsRegularCPlusPlusEnum)
1654llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(
QualType Ty) {
1655 llvm::APInt Min, End;
1661 return MDHelper.createRange(Min, End);
1666 bool HasBoolCheck =
SanOpts.
has(SanitizerKind::Bool);
1667 bool HasEnumCheck =
SanOpts.
has(SanitizerKind::Enum);
1668 if (!HasBoolCheck && !HasEnumCheck)
1673 bool NeedsBoolCheck = HasBoolCheck && IsBool;
1674 bool NeedsEnumCheck = HasEnumCheck && Ty->
getAs<
EnumType>();
1675 if (!NeedsBoolCheck && !NeedsEnumCheck)
1682 cast<llvm::IntegerType>(
Value->
getType())->getBitWidth() == 1)
1685 llvm::APInt Min, End;
1690 SanitizerScope SanScope(
this);
1694 Check =
Builder.CreateICmpULE(
Value, llvm::ConstantInt::get(Ctx, End));
1696 llvm::Value *Upper =
1697 Builder.CreateICmpSLE(
Value, llvm::ConstantInt::get(Ctx, End));
1698 llvm::Value *Lower =
1699 Builder.CreateICmpSGE(
Value, llvm::ConstantInt::get(Ctx, Min));
1700 Check =
Builder.CreateAnd(Upper, Lower);
1705 NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool;
1706 EmitCheck(std::make_pair(Check, Kind), SanitizerHandler::LoadInvalidValue,
1716 bool isNontemporal) {
1717 if (
auto *GV = dyn_cast<llvm::GlobalValue>(Addr.
getPointer()))
1718 if (GV->isThreadLocal())
1724 if (ClangVecTy->isExtVectorBoolType()) {
1726 unsigned ValNumElems =
1727 cast<llvm::FixedVectorType>(ValTy)->getNumElements();
1730 const auto *RawIntTy = RawIntV->getType();
1731 assert(RawIntTy->isIntegerTy() &&
"compressed iN storage for bitvectors");
1733 auto *PaddedVecTy = llvm::FixedVectorType::get(
1734 Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
1735 llvm::Value *
V =
Builder.CreateBitCast(RawIntV, PaddedVecTy);
1744 const auto *VTy = cast<llvm::FixedVectorType>(EltTy);
1748 llvm::VectorType *vec4Ty =
1749 llvm::FixedVectorType::get(VTy->getElementType(), 4);
1768 if (isNontemporal) {
1769 llvm::MDNode *
Node = llvm::MDNode::get(
1770 Load->getContext(), llvm::ConstantAsMetadata::get(
Builder.getInt32(1)));
1771 Load->setMetadata(llvm::LLVMContext::MD_nontemporal,
Node);
1780 if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {
1781 Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo);
1782 Load->setMetadata(llvm::LLVMContext::MD_noundef,
1797 "wrong value rep of bool");
1807 "wrong value rep of bool");
1813 auto *PaddedVecTy = llvm::FixedVectorType::get(
1814 Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
1818 unsigned ValNumElems = cast<llvm::FixedVectorType>(ValTy)->getNumElements();
1828 bool IsVector =
true) {
1829 auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.
getElementType());
1830 if (ArrayTy && IsVector) {
1831 auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
1832 ArrayTy->getNumElements());
1836 auto *VectorTy = dyn_cast<llvm::VectorType>(Addr.
getElementType());
1837 if (VectorTy && !IsVector) {
1838 auto *ArrayTy = llvm::ArrayType::get(
1839 VectorTy->getElementType(),
1840 cast<llvm::FixedVectorType>(VectorTy)->getNumElements());
1854 value->getType()->isVectorTy());
1864 bool isInit,
bool isNontemporal) {
1865 if (
auto *GV = dyn_cast<llvm::GlobalValue>(Addr.
getPointer()))
1866 if (GV->isThreadLocal())
1872 auto *VecTy = dyn_cast<llvm::FixedVectorType>(SrcTy);
1873 if (VecTy && ClangVecTy->isExtVectorBoolType()) {
1876 unsigned MemNumElems = MemIntTy->getPrimitiveSizeInBits();
1883 if (VecTy && cast<llvm::FixedVectorType>(VecTy)->getNumElements() == 3) {
1887 SrcTy = llvm::FixedVectorType::get(VecTy->getElementType(), 4);
1906 if (isNontemporal) {
1907 llvm::MDNode *
Node =
1908 llvm::MDNode::get(
Store->getContext(),
1909 llvm::ConstantAsMetadata::get(
Builder.getInt32(1)));
1910 Store->setMetadata(llvm::LLVMContext::MD_nontemporal,
Node);
1991 llvm::MatrixBuilder MB(
Builder);
1992 MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
1994 llvm::LoadInst *
Load =
1999 assert(LV.
isBitField() &&
"Unknown LValue type!");
2017 const unsigned StorageSize =
2020 assert(
static_cast<unsigned>(Offset + Info.
Size) <= StorageSize);
2021 unsigned HighBits = StorageSize - Offset - Info.
Size;
2023 Val =
Builder.CreateShl(Val, HighBits,
"bf.shl");
2024 if (Offset + HighBits)
2025 Val =
Builder.CreateAShr(Val, Offset + HighBits,
"bf.ashr");
2028 Val =
Builder.CreateLShr(Val, Offset,
"bf.lshr");
2029 if (
static_cast<unsigned>(Offset) + Info.
Size < StorageSize)
2031 Val, llvm::APInt::getLowBitsSet(StorageSize, Info.
Size),
"bf.clear");
2051 llvm::Value *Elt = llvm::ConstantInt::get(
SizeTy, InIdx);
2059 for (
unsigned i = 0; i != NumResultElts; ++i)
2062 Vec =
Builder.CreateShuffleVector(Vec, Mask);
2081 return VectorBasePtrPlusIx;
2087 "Bad type for register variable");
2088 llvm::MDNode *RegName = cast<llvm::MDNode>(
2089 cast<llvm::MetadataAsValue>(LV.
getGlobalReg())->getMetadata());
2093 llvm::Type *Ty = OrigTy;
2094 if (OrigTy->isPointerTy())
2096 llvm::Type *Types[] = { Ty };
2098 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::read_register, Types);
2100 F, llvm::MetadataAsValue::get(Ty->getContext(), RegName));
2101 if (OrigTy->isPointerTy())
2116 auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Vec->getType());
2118 auto *IRVecTy = llvm::FixedVectorType::get(
2119 Builder.getInt1Ty(), IRStoreTy->getPrimitiveSizeInBits());
2120 Vec =
Builder.CreateBitCast(Vec, IRVecTy);
2127 Vec =
Builder.CreateBitCast(Vec, IRStoreTy);
2146 llvm::MatrixBuilder MB(
Builder);
2147 MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
2157 assert(Dst.
isBitField() &&
"Unknown LValue type");
2165 llvm_unreachable(
"present but none");
2213 RHS =
Builder.CreatePtrToInt(RHS, ResultType,
"sub.ptr.rhs.cast");
2216 "sub.ptr.lhs.cast");
2217 llvm::Value *BytesBetween =
Builder.CreateSub(LHS, RHS,
"ivar.offset");
2229 assert(Src.
isScalar() &&
"Can't emit an agg store with this method");
2245 llvm::Value *MaskedVal = SrcVal;
2247 const bool UseVolatile =
2250 const unsigned StorageSize =
2255 if (StorageSize != Info.
Size) {
2256 assert(StorageSize > Info.
Size &&
"Invalid bitfield size.");
2263 SrcVal, llvm::APInt::getLowBitsSet(StorageSize, Info.
Size),
2267 SrcVal =
Builder.CreateShl(SrcVal, Offset,
"bf.shl");
2271 Val, ~llvm::APInt::getBitsSet(StorageSize, Offset, Offset + Info.
Size),
2275 SrcVal =
Builder.CreateOr(Val, SrcVal,
"bf.set");
2277 assert(Offset == 0);
2293 llvm::Value *ResultVal = MaskedVal;
2297 assert(Info.
Size <= StorageSize);
2298 unsigned HighBits = StorageSize - Info.
Size;
2300 ResultVal =
Builder.CreateShl(ResultVal, HighBits,
"bf.result.shl");
2301 ResultVal =
Builder.CreateAShr(ResultVal, HighBits,
"bf.result.ashr");
2322 unsigned NumSrcElts = VTy->getNumElements();
2323 unsigned NumDstElts =
2324 cast<llvm::FixedVectorType>(Vec->getType())->getNumElements();
2325 if (NumDstElts == NumSrcElts) {
2330 for (
unsigned i = 0; i != NumSrcElts; ++i)
2333 Vec =
Builder.CreateShuffleVector(SrcVal, Mask);
2334 }
else if (NumDstElts > NumSrcElts) {
2340 for (
unsigned i = 0; i != NumSrcElts; ++i)
2341 ExtMask.push_back(i);
2342 ExtMask.resize(NumDstElts, -1);
2343 llvm::Value *ExtSrcVal =
Builder.CreateShuffleVector(SrcVal, ExtMask);
2346 for (
unsigned i = 0; i != NumDstElts; ++i)
2356 for (
unsigned i = 0; i != NumSrcElts; ++i)
2358 Vec =
Builder.CreateShuffleVector(Vec, ExtSrcVal, Mask);
2361 llvm_unreachable(
"unexpected shorten vector length");
2366 llvm::Value *Elt = llvm::ConstantInt::get(
SizeTy, InIdx);
2367 Vec =
Builder.CreateInsertElement(Vec, SrcVal, Elt);
2377 "Bad type for register variable");
2378 llvm::MDNode *RegName = cast<llvm::MDNode>(
2379 cast<llvm::MetadataAsValue>(Dst.
getGlobalReg())->getMetadata());
2380 assert(RegName &&
"Register LValue is not metadata");
2384 llvm::Type *Ty = OrigTy;
2385 if (OrigTy->isPointerTy())
2387 llvm::Type *Types[] = { Ty };
2389 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
2391 if (OrigTy->isPointerTy())
2394 F, {llvm::MetadataAsValue::get(Ty->getContext(), RegName),
Value});
2402 bool IsMemberAccess=
false) {
2406 if (isa<ObjCIvarRefExpr>(E)) {
2419 auto *Exp = cast<ObjCIvarRefExpr>(
const_cast<Expr *
>(E));
2425 if (
const auto *Exp = dyn_cast<DeclRefExpr>(E)) {
2426 if (
const auto *VD = dyn_cast<VarDecl>(Exp->getDecl())) {
2427 if (VD->hasGlobalStorage()) {
2436 if (
const auto *Exp = dyn_cast<UnaryOperator>(E)) {
2441 if (
const auto *Exp = dyn_cast<ParenExpr>(E)) {
2455 if (
const auto *Exp = dyn_cast<GenericSelectionExpr>(E)) {
2460 if (
const auto *Exp = dyn_cast<ImplicitCastExpr>(E)) {
2465 if (
const auto *Exp = dyn_cast<CStyleCastExpr>(E)) {
2470 if (
const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) {
2475 if (
const auto *Exp = dyn_cast<ArraySubscriptExpr>(E)) {
2488 if (
const auto *Exp = dyn_cast<MemberExpr>(E)) {
2502 CGF, VD, Addr, Loc);
2513 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2514 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2518 if (!Res || ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2519 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2522 assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
2523 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2524 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2526 "Expected link clause OR to clause with unified memory enabled.");
2536 llvm::LoadInst *
Load =
2542 PointeeType, PointeeBaseInfo, PointeeTBAAInfo,
2553 PointeeBaseInfo, PointeeTBAAInfo);
2594 V = CGF.
Builder.CreateThreadLocalAddress(
V);
2598 Address Addr(
V, RealVarTy, Alignment);
2602 VD->
hasAttr<OMPThreadPrivateDeclAttr>()) {
2617 if (FD->
hasAttr<WeakRefAttr>()) {
2632 V = llvm::ConstantExpr::getBitCast(
V,
2649 llvm::Value *ThisValue) {
2663 AsmLabelAttr *
Asm = VD->
getAttr<AsmLabelAttr>();
2664 assert(
Asm->getLabel().size() < 64-Name.size() &&
2665 "Register name too big");
2666 Name.append(
Asm->getLabel());
2667 llvm::NamedMDNode *M =
2668 CGM.
getModule().getOrInsertNamedMetadata(Name);
2669 if (M->getNumOperands() == 0) {
2672 llvm::Metadata *Ops[] = {Str};
2679 llvm::MetadataAsValue::get(CGM.
getLLVMContext(), M->getOperand(0));
2722 case llvm::GlobalValue::ExternalLinkage:
2723 case llvm::GlobalValue::LinkOnceODRLinkage:
2724 case llvm::GlobalValue::WeakODRLinkage:
2725 case llvm::GlobalValue::InternalLinkage:
2726 case llvm::GlobalValue::PrivateLinkage:
2738 "should not emit an unevaluated operand");
2740 if (
const auto *VD = dyn_cast<VarDecl>(ND)) {
2743 VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
2751 (VD->getType()->isReferenceType() ||
2753 VD->getAnyInitializer(VD);
2755 E->
getLocation(), *VD->evaluateValue(), VD->getType());
2756 assert(Val &&
"failed to emit constant expression");
2759 if (!VD->getType()->isReferenceType()) {
2764 auto *PTy = llvm::PointerType::get(
2765 VarTy,
getTypes().getTargetAddressSpace(VD->getType()));
2783 VD = VD->getCanonicalDecl();
2787 auto I = LocalDeclMap.find(VD);
2788 if (I != LocalDeclMap.end()) {
2790 if (VD->getType()->isReferenceType())
2830 "Should not use decl without marking it used!");
2832 if (ND->
hasAttr<WeakRefAttr>()) {
2833 const auto *VD = cast<ValueDecl>(ND);
2838 if (
const auto *VD = dyn_cast<VarDecl>(ND)) {
2840 if (VD->hasLinkage() || VD->isStaticDataMember())
2846 auto iter = LocalDeclMap.find(VD);
2847 if (iter != LocalDeclMap.end()) {
2848 addr = iter->second;
2852 }
else if (VD->isStaticLocal()) {
2860 llvm_unreachable(
"DeclRefExpr for Decl not entered in LocalDeclMap?");
2870 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
2877 bool isBlockByref = VD->isEscapingByref();
2887 bool isLocalStorage = VD->hasLocalStorage();
2889 bool NonGCable = isLocalStorage &&
2897 bool isImpreciseLifetime =
2898 (isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());
2899 if (isImpreciseLifetime)
2905 if (
const auto *FD = dyn_cast<FunctionDecl>(ND)) {
2909 if (
getContext().getTargetInfo().allowDebugInfoForExternalRef()) {
2912 cast<llvm::Function>(LV.
getPointer(*this)->stripPointerCasts());
2913 if (!Fn->getSubprogram())
2914 DI->EmitFunctionDecl(FD, FD->getLocation(), T, Fn);
2924 if (
const auto *BD = dyn_cast<BindingDecl>(ND)) {
2934 if (
const auto *GD = dyn_cast<MSGuidDecl>(ND))
2938 if (
const auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND))
2942 llvm_unreachable(
"Unhandled DeclRefExpr");
2952 default: llvm_unreachable(
"Unknown unary operator lvalue!");
2955 assert(!T.
isNull() &&
"CodeGenFunction::EmitUnaryOpLValue: Illegal type");
2977 assert(LV.
isSimple() &&
"real/imag on non-ordinary l-value");
3001 bool isInc = E->
getOpcode() == UO_PreInc;
3024 assert(SL !=
nullptr &&
"No StringLiteral name in PredefinedExpr");
3025 StringRef FnName =
CurFn->getName();
3026 if (FnName.startswith(
"\01"))
3027 FnName = FnName.substr(1);
3028 StringRef NameItems[] = {
3030 std::string GVName = llvm::join(NameItems, NameItems + 2,
".");
3031 if (
auto *BD = dyn_cast_or_null<BlockDecl>(
CurCodeDecl)) {
3032 std::string Name = std::string(SL->getString());
3033 if (!Name.empty()) {
3034 unsigned Discriminator =
3037 Name +=
"_" + Twine(Discriminator + 1).str();
3064 uint16_t TypeKind = -1;
3081 StringRef(), std::nullopt, Buffer, std::nullopt);
3083 llvm::Constant *Components[] = {
3087 llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(Components);
3089 auto *GV =
new llvm::GlobalVariable(
3091 true, llvm::GlobalVariable::PrivateLinkage, Descriptor);
3092 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3104 if (
V->getType() == TargetTy)
3109 if (
V->getType()->isFloatingPointTy()) {
3110 unsigned Bits =
V->getType()->getPrimitiveSizeInBits().getFixedValue();
3111 if (Bits <= TargetTy->getIntegerBitWidth())
3117 if (
V->getType()->isIntegerTy() &&
3118 V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
3119 return Builder.CreateZExt(
V, TargetTy);
3122 if (!
V->getType()->isPointerTy()) {
3127 return Builder.CreatePtrToInt(
V, TargetTy);
3147 int PathComponentsToStrip =
3149 if (PathComponentsToStrip < 0) {
3150 assert(PathComponentsToStrip !=
INT_MIN);
3151 int PathComponentsToKeep = -PathComponentsToStrip;
3152 auto I = llvm::sys::path::rbegin(FilenameString);
3153 auto E = llvm::sys::path::rend(FilenameString);
3154 while (I != E && --PathComponentsToKeep)
3157 FilenameString = FilenameString.substr(I - E);
3158 }
else if (PathComponentsToStrip > 0) {
3159 auto I = llvm::sys::path::begin(FilenameString);
3160 auto E = llvm::sys::path::end(FilenameString);
3161 while (I != E && PathComponentsToStrip--)
3166 FilenameString.substr(I - llvm::sys::path::begin(FilenameString));
3168 FilenameString = llvm::sys::path::filename(FilenameString);
3174 cast<llvm::GlobalVariable>(
3175 FilenameGV.getPointer()->stripPointerCasts()));
3176 Filename = FilenameGV.getPointer();
3187 return llvm::ConstantStruct::getAnon(
Data);
3192enum class CheckRecoverableKind {
3204 assert(Kind.countPopulation() == 1);
3205 if (Kind == SanitizerKind::Vptr)
3206 return CheckRecoverableKind::AlwaysRecoverable;
3207 else if (Kind == SanitizerKind::Return || Kind == SanitizerKind::Unreachable)
3208 return CheckRecoverableKind::Unrecoverable;
3210 return CheckRecoverableKind::Recoverable;
3214struct SanitizerHandlerInfo {
3215 char const *
const Name;
3221#define SANITIZER_CHECK(Enum, Name, Version) {#Name, Version},
3223#undef SANITIZER_CHECK
3227 llvm::FunctionType *FnType,
3230 CheckRecoverableKind RecoverKind,
bool IsFatal,
3231 llvm::BasicBlock *ContBB) {
3232 assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
3233 std::optional<ApplyDebugLocation> DL;
3234 if (!CGF.
Builder.getCurrentDebugLocation()) {
3238 bool NeedsAbortSuffix =
3239 IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;
3242 const StringRef CheckName = CheckInfo.Name;
3243 std::string FnName =
"__ubsan_handle_" + CheckName.str();
3244 if (CheckInfo.Version && !MinimalRuntime)
3245 FnName +=
"_v" + llvm::utostr(CheckInfo.Version);
3247 FnName +=
"_minimal";
3248 if (NeedsAbortSuffix)
3251 !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;
3255 B.addAttribute(llvm::Attribute::NoReturn)
3256 .addAttribute(llvm::Attribute::NoUnwind);
3258 B.addUWTableAttr(llvm::UWTableKind::Default);
3263 llvm::AttributeList::FunctionIndex, B),
3267 HandlerCall->setDoesNotReturn();
3268 CGF.
Builder.CreateUnreachable();
3275 ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked,
3279 assert(Checked.size() > 0);
3280 assert(CheckHandler >= 0 &&
3284 llvm::Value *FatalCond =
nullptr;
3285 llvm::Value *RecoverableCond =
nullptr;
3286 llvm::Value *TrapCond =
nullptr;
3287 for (
int i = 0, n = Checked.size(); i < n; ++i) {
3288 llvm::Value *Check = Checked[i].first;
3290 llvm::Value *&Cond =
3296 Cond = Cond ?
Builder.CreateAnd(Cond, Check) : Check;
3301 if (!FatalCond && !RecoverableCond)
3304 llvm::Value *JointCond;
3305 if (FatalCond && RecoverableCond)
3306 JointCond =
Builder.CreateAnd(FatalCond, RecoverableCond);
3308 JointCond = FatalCond ? FatalCond : RecoverableCond;
3314 for (
int i = 1, n = Checked.size(); i < n; ++i) {
3316 "All recoverable kinds in a single check must be same!");
3323 llvm::Instruction *Branch =
Builder.CreateCondBr(JointCond, Cont, Handlers);
3327 llvm::MDNode *
Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
3328 Branch->setMetadata(llvm::LLVMContext::MD_prof,
Node);
3337 Args.reserve(DynamicArgs.size() + 1);
3338 ArgTypes.reserve(DynamicArgs.size() + 1);
3341 if (!StaticArgs.empty()) {
3342 llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3343 auto *InfoPtr =
new llvm::GlobalVariable(
3345 llvm::GlobalVariable::PrivateLinkage, Info,
"",
nullptr,
3346 llvm::GlobalVariable::NotThreadLocal,
3348 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3350 Args.push_back(InfoPtr);
3351 ArgTypes.push_back(Args.back()->getType());
3354 for (
size_t i = 0, n = DynamicArgs.size(); i != n; ++i) {
3360 llvm::FunctionType *FnType =
3361 llvm::FunctionType::get(
CGM.
VoidTy, ArgTypes,
false);
3363 if (!FatalCond || !RecoverableCond) {
3367 (FatalCond !=
nullptr), Cont);
3371 llvm::BasicBlock *NonFatalHandlerBB =
3374 Builder.CreateCondBr(FatalCond, NonFatalHandlerBB, FatalHandlerBB);
3387 SanitizerMask Kind, llvm::Value *Cond, llvm::ConstantInt *TypeId,
3392 llvm::BranchInst *BI =
Builder.CreateCondBr(Cond, Cont, CheckBB);
3395 llvm::MDNode *
Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
3396 BI->setMetadata(llvm::LLVMContext::MD_prof,
Node);
3402 llvm::CallInst *CheckCall;
3403 llvm::FunctionCallee SlowPathFn;
3405 llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3407 new llvm::GlobalVariable(
CGM.
getModule(), Info->getType(),
false,
3408 llvm::GlobalVariable::PrivateLinkage, Info);
3409 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3413 "__cfi_slowpath_diag",
3416 CheckCall =
Builder.CreateCall(SlowPathFn, {TypeId, Ptr, InfoPtr});
3421 CheckCall =
Builder.CreateCall(SlowPathFn, {TypeId, Ptr});
3425 cast<llvm::GlobalValue>(SlowPathFn.getCallee()->stripPointerCasts()));
3426 CheckCall->setDoesNotThrow();
3435 auto &Ctx = M->getContext();
3436 llvm::Function *F = llvm::Function::Create(
3438 llvm::GlobalValue::WeakAnyLinkage,
"__cfi_check", M);
3439 F->setAlignment(llvm::Align(4096));
3441 llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx,
"entry", F);
3444 llvm::CallInst::Create(M->getFunction(
"__cfi_check_fail"), Args,
"", BB);
3445 llvm::ReturnInst::Create(Ctx,
nullptr, BB);
3456 SanitizerScope SanScope(
this);
3462 Args.push_back(&ArgData);
3463 Args.push_back(&ArgAddr);
3468 llvm::Function *F = llvm::Function::Create(
3470 llvm::GlobalValue::WeakODRLinkage,
"__cfi_check_fail", &
CGM.
getModule());
3474 F->setVisibility(llvm::GlobalValue::HiddenVisibility);
3492 llvm::Value *DataIsNotNullPtr =
3494 EmitTrapCheck(DataIsNotNullPtr, SanitizerHandler::CFICheckFail);
3496 llvm::StructType *SourceLocationTy =
3498 llvm::StructType *CfiCheckFailDataTy =
3501 llvm::Value *
V =
Builder.CreateConstGEP2_32(
3503 Builder.CreatePointerCast(
Data, CfiCheckFailDataTy->getPointerTo(0)), 0,
3509 llvm::Value *AllVtables = llvm::MetadataAsValue::get(
3512 llvm::Value *ValidVtable =
Builder.CreateZExt(
3514 {Addr, AllVtables}),
3517 const std::pair<int, SanitizerMask> CheckKinds[] = {
3525 for (
auto CheckKindMaskPair : CheckKinds) {
3526 int Kind = CheckKindMaskPair.first;
3529 Builder.CreateICmpNE(CheckKind, llvm::ConstantInt::get(
Int8Ty, Kind));
3531 EmitCheck(std::make_pair(Cond, Mask), SanitizerHandler::CFICheckFail, {},
3532 {
Data, Addr, ValidVtable});
3544 if (
SanOpts.
has(SanitizerKind::Unreachable)) {
3545 SanitizerScope SanScope(
this);
3547 SanitizerKind::Unreachable),
3548 SanitizerHandler::BuiltinUnreachable,
3560 if (TrapBBs.size() <= CheckHandlerID)
3561 TrapBBs.resize(CheckHandlerID + 1);
3563 llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
3568 auto Call = TrapBB->begin();
3569 assert(isa<llvm::CallInst>(Call) &&
"Expected call in trap BB");
3571 Call->applyMergedLocation(
Call->getDebugLoc(),
3572 Builder.getCurrentDebugLocation());
3573 Builder.CreateCondBr(Checked, Cont, TrapBB);
3576 Builder.CreateCondBr(Checked, Cont, TrapBB);
3579 llvm::CallInst *TrapCall =
Builder.CreateCall(
3582 ? TrapBB->getParent()->size()
3586 auto A = llvm::Attribute::get(
getLLVMContext(),
"trap-func-name",
3588 TrapCall->addFnAttr(A);
3590 TrapCall->setDoesNotReturn();
3591 TrapCall->setDoesNotThrow();
3599 llvm::CallInst *TrapCall =
3603 auto A = llvm::Attribute::get(
getLLVMContext(),
"trap-func-name",
3605 TrapCall->addFnAttr(A);
3615 "Array to pointer decay must have array source type!");
3630 "Expected pointer to array");
3650 const auto *CE = dyn_cast<CastExpr>(E);
3651 if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)
3663 llvm::Type *elemType,
3669 const llvm::Twine &name =
"arrayidx") {
3684 if (
auto constantIdx = dyn_cast<llvm::ConstantInt>(idx)) {
3685 CharUnits offset = constantIdx->getZExtValue() * eltSize;
3717 if (
const auto *ME = dyn_cast<MemberExpr>(E))
3720 if (
const auto *DRE = dyn_cast<DeclRefExpr>(E)) {
3721 const auto *VarDef = dyn_cast<VarDecl>(DRE->getDecl());
3725 const auto *PtrT = VarDef->getType()->getAs<
PointerType>();
3731 if (
const auto *RecT = dyn_cast<RecordType>(PointeeT))
3732 return RecT->getDecl()->
hasAttr<BPFPreserveAccessIndexAttr>();
3745 const llvm::Twine &name =
"arrayidx") {
3748 for (
auto *idx : indices.drop_back())
3749 assert(isa<llvm::ConstantInt>(idx) &&
3750 cast<llvm::ConstantInt>(idx)->isZero());
3764 llvm::Value *eltPtr;
3765 auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back());
3770 signedIndices, loc, name);
3773 unsigned idx = LastIndex->getZExtValue();
3774 llvm::DIType *DbgInfo =
nullptr;
3790 llvm::Value *IdxPre =
3792 bool SignedIndices =
false;
3793 auto EmitIdxAfterBase = [&, IdxPre](
bool Promote) -> llvm::Value * {
3796 assert(E->
getRHS() == E->
getIdx() &&
"index was neither LHS nor RHS");
3802 SignedIndices |= IdxSigned;
3808 if (Promote && Idx->getType() !=
IntPtrTy)
3818 !isa<ExtVectorElementExpr>(E->
getBase())) {
3821 auto *Idx = EmitIdxAfterBase(
false);
3822 assert(LHS.
isSimple() &&
"Can only subscript lvalue vectors here!");
3831 if (isa<ExtVectorElementExpr>(E->
getBase())) {
3833 auto *Idx = EmitIdxAfterBase(
true);
3852 auto *Idx = EmitIdxAfterBase(
true);
3862 Idx =
Builder.CreateMul(Idx, numElements);
3864 Idx =
Builder.CreateNSWMul(Idx, numElements);
3876 auto *Idx = EmitIdxAfterBase(
true);
3879 llvm::Value *InterfaceSizeVal =
3880 llvm::ConstantInt::get(Idx->getType(), InterfaceSize.
getQuantity());
3882 llvm::Value *ScaledIdx =
Builder.CreateMul(Idx, InterfaceSizeVal);
3893 llvm::Value *EltPtr =
3896 Addr =
Address(EltPtr, OrigBaseElemTy, EltAlign);
3902 assert(Array->getType()->isArrayType() &&
3903 "Array to pointer decay must have array source type!");
3907 if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
3911 auto *Idx = EmitIdxAfterBase(
true);
3916 *
this, ArrayLV.
getAddress(*
this), {CGM.getSize(CharUnits::Zero()), Idx},
3924 auto *Idx = EmitIdxAfterBase(
true);
3945 "incomplete matrix subscript expressions should be rejected during Sema");
3949 llvm::Value *NumRows =
Builder.getIntN(
3950 RowIdx->getType()->getScalarSizeInBits(),
3952 llvm::Value *FinalIdx =
3963 bool IsLowerBound) {
3965 if (
auto *ASE = dyn_cast<OMPArraySectionExpr>(
Base->IgnoreParenImpCasts())) {
3980 "Expected pointer to array");
3999 bool IsLowerBound) {
4002 if (
auto *AT =
getContext().getAsArrayType(BaseTy))
4003 ResultExprTy = AT->getElementType();
4006 llvm::Value *Idx =
nullptr;
4014 LowerBound->getType()->hasSignedIntegerRepresentation());
4016 Idx = llvm::ConstantInt::getNullValue(
IntPtrTy);
4023 llvm::APSInt ConstLength;
4026 if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(
C)) {
4033 if (std::optional<llvm::APSInt> LB =
4034 LowerBound->getIntegerConstantExpr(
C)) {
4036 LowerBound =
nullptr;
4041 else if (!LowerBound)
4044 if (Length || LowerBound) {
4045 auto *LowerBoundVal =
4049 LowerBound->getType()->hasSignedIntegerRepresentation())
4050 : llvm::ConstantInt::get(
IntPtrTy, ConstLowerBound);
4055 Length->getType()->hasSignedIntegerRepresentation())
4056 : llvm::ConstantInt::get(
IntPtrTy, ConstLength);
4057 Idx =
Builder.CreateAdd(LowerBoundVal, LengthVal,
"lb_add_len",
4060 if (Length && LowerBound) {
4062 Idx, llvm::ConstantInt::get(
IntPtrTy, 1),
"idx_sub_1",
4066 Idx = llvm::ConstantInt::get(
IntPtrTy, ConstLength + ConstLowerBound);
4072 if (
auto *VAT =
C.getAsVariableArrayType(ArrayTy)) {
4073 Length = VAT->getSizeExpr();
4074 if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(
C)) {
4079 auto *CAT =
C.getAsConstantArrayType(ArrayTy);
4080 assert(CAT &&
"unexpected type for array initializer");
4081 ConstLength = CAT->getSize();
4084 auto *LengthVal =
Builder.CreateIntCast(
4086 Length->getType()->hasSignedIntegerRepresentation());
4088 LengthVal, llvm::ConstantInt::get(
IntPtrTy, 1),
"len_sub_1",
4093 Idx = llvm::ConstantInt::get(
IntPtrTy, ConstLength);
4102 if (
auto *VLA =
getContext().getAsVariableArrayType(ResultExprTy)) {
4108 BaseTy, VLA->getElementType(), IsLowerBound);
4117 Idx =
Builder.CreateMul(Idx, NumElements);
4119 Idx =
Builder.CreateNSWMul(Idx, NumElements);
4128 assert(Array->getType()->isArrayType() &&
4129 "Array to pointer decay must have array source type!");
4133 if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
4140 *
this, ArrayLV.
getAddress(*
this), {CGM.getSize(CharUnits::Zero()), Idx},
4141 ResultExprTy, !
getLangOpts().isSignedOverflowDefined(),
4147 TBAAInfo, BaseTy, ResultExprTy,
4171 Base.getQuals().removeObjCGCAttr();
4180 "Result must be a vector");
4197 if (
Base.isSimple()) {
4198 llvm::Constant *CV =
4203 assert(
Base.isExtVectorElt() &&
"Can only subscript lvalue vec elts here!");
4205 llvm::Constant *BaseElts =
Base.getExtVectorElts();
4208 for (
unsigned i = 0, e = Indices.size(); i != e; ++i)
4209 CElts.push_back(BaseElts->getAggregateElement(Indices[i]));
4210 llvm::Constant *CV = llvm::ConstantVector::get(CElts);
4232 SkippedChecks.
set(SanitizerKind::Alignment,
true);
4233 if (IsBaseCXXThis || isa<DeclRefExpr>(BaseExpr))
4234 SkippedChecks.
set(SanitizerKind::Null,
true);
4242 if (
auto *Field = dyn_cast<FieldDecl>(ND)) {
4257 if (
const auto *FD = dyn_cast<FunctionDecl>(ND))
4260 llvm_unreachable(
"Unhandled member declaration!");
4267 assert(cast<CXXMethodDecl>(
CurCodeDecl)->getParent()->isLambda());
4268 assert(cast<CXXMethodDecl>(
CurCodeDecl)->getParent() ==
Field->getParent());
4279 unsigned FieldIndex) {
4280 unsigned I = 0, Skipped = 0;
4283 if (I == FieldIndex)
4285 if (F->isUnnamedBitfield())
4290 return FieldIndex - Skipped;
4299 if (Offset.isZero())
4388 llvm::Type *FieldIntTy = llvm::Type::getIntNTy(
getLLVMContext(), SS);
4424 assert(!FieldTBAAInfo.
Offset &&
4425 "Nonzero offset for an access with no base type!");
4438 FieldTBAAInfo.
Size =
4443 if (
auto *ClassDef = dyn_cast<CXXRecordDecl>(rec)) {
4445 ClassDef->isDynamicClass()) {
4470 Builder.CreatePreserveUnionAccessIndex(
4504 if (field->
hasAttr<AnnotateAttr>())
4529 V =
V.withElementType(llvmType);