34#include "llvm/ADT/STLExtras.h"
35#include "llvm/ADT/ScopeExit.h"
36#include "llvm/ADT/StringExtras.h"
37#include "llvm/IR/DataLayout.h"
38#include "llvm/IR/Intrinsics.h"
39#include "llvm/IR/LLVMContext.h"
40#include "llvm/IR/MDBuilder.h"
41#include "llvm/IR/MatrixBuilder.h"
42#include "llvm/Support/ConvertUTF.h"
43#include "llvm/Support/Endian.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/Path.h"
46#include "llvm/Support/xxhash.h"
47#include "llvm/Transforms/Utils/SanitizerStats.h"
53using namespace CodeGen;
58 "ubsan-guard-checks", llvm::cl::Optional,
59 llvm::cl::desc(
"Guard UBSAN checks with `llvm.allow.ubsan.check()`."));
84CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty,
CharUnits Align,
86 llvm::Value *ArraySize) {
96 llvm::Value *ArraySize,
100 *AllocaAddr = Alloca;
101 llvm::Value *
V = Alloca.getPointer();
108 llvm::IRBuilderBase::InsertPointGuard IPG(
Builder);
116 Builder.getPtrTy(DestAddrSpace),
true);
127 llvm::Value *ArraySize) {
128 llvm::AllocaInst *Alloca;
130 Alloca =
Builder.CreateAlloca(Ty, ArraySize, Name);
136 Allocas->Add(Alloca);
170 auto *ArrayTy = cast<llvm::ArrayType>(
Result.getElementType());
171 auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
172 ArrayTy->getNumElements());
203 CGFPOptionsRAII FPOptsRAII(*
this,
E);
221 if (
const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
244 if (!ignoreResult && aggSlot.
isIgnored())
249 llvm_unreachable(
"bad evaluation kind");
291 llvm_unreachable(
"bad evaluation kind");
314 llvm_unreachable(
"bad evaluation kind");
354 bool Precise = isa_and_nonnull<VarDecl>(VD) &&
355 VD->
hasAttr<ObjCPreciseLifetimeAttr>();
376 llvm_unreachable(
"temporary cannot have dynamic storage duration");
378 llvm_unreachable(
"unknown storage duration");
386 auto *ClassDecl = cast<CXXRecordDecl>(RT->getDecl());
387 if (!ClassDecl->hasTrivialDestructor())
388 ReferenceTemporaryDtor = ClassDecl->getDestructor();
391 if (!ReferenceTemporaryDtor)
398 llvm::FunctionCallee CleanupFn;
399 llvm::Constant *CleanupArg;
405 CleanupArg = llvm::Constant::getNullValue(CGF.
Int8PtrTy);
409 CleanupArg = cast<llvm::Constant>(ReferenceTemporary.
emitRawPointer(CGF));
430 llvm_unreachable(
"temporary cannot have dynamic storage duration");
452 auto *GV =
new llvm::GlobalVariable(
454 llvm::GlobalValue::PrivateLinkage,
Init,
".ref.tmp",
nullptr,
455 llvm::GlobalValue::NotThreadLocal,
459 llvm::Constant *
C = GV;
461 C = TCG.performAddrSpaceCast(
463 llvm::PointerType::get(
467 return RawAddress(
C, GV->getValueType(), alignment);
476 llvm_unreachable(
"temporary can't have dynamic storage duration");
478 llvm_unreachable(
"unknown storage duration");
492 "Reference should never be pseudo-strong!");
500 if (
auto *Var = dyn_cast<llvm::GlobalVariable>(
Object.getPointer())) {
510 if (Var->hasInitializer())
519 default: llvm_unreachable(
"expected scalar or aggregate expression");
542 for (
const auto &Ignored : CommaLHSs)
545 if (
const auto *opaque = dyn_cast<OpaqueValueExpr>(
E)) {
546 if (opaque->getType()->isRecordType()) {
547 assert(Adjustments.empty());
555 if (
auto *Var = dyn_cast<llvm::GlobalVariable>(
556 Object.getPointer()->stripPointerCasts())) {
562 if (!Var->hasInitializer()) {
578 if (!ShouldEmitLifetimeMarkers)
588 ConditionalEvaluation *OldConditional =
nullptr;
589 CGBuilderTy::InsertPoint OldIP;
595 OldConditional = OutermostConditional;
596 OutermostConditional =
nullptr;
599 llvm::BasicBlock *
Block = OldConditional->getStartingBlock();
600 Builder.restoreIP(CGBuilderTy::InsertPoint(
601 Block, llvm::BasicBlock::iterator(
Block->back())));
611 if (OldConditional) {
612 OutermostConditional = OldConditional;
629 switch (Adjustment.Kind) {
633 Adjustment.DerivedToBase.BasePath->path_begin(),
634 Adjustment.DerivedToBase.BasePath->path_end(),
642 "materialized temporary field is not a simple lvalue");
683 const llvm::Constant *Elts) {
684 return cast<llvm::ConstantInt>(Elts->getAggregateElement(Idx))
691 Builder.CreateMul(Ptr, Builder.getInt64(0xbf58476d1ce4e5b9u));
693 Builder.CreateXor(A0, Builder.CreateLShr(A0, Builder.getInt64(31)));
694 return Builder.CreateXor(Acc, A1);
721 llvm::Value *ArraySize) {
728 if (Ptr->getType()->getPointerAddressSpace())
736 SanitizerScope SanScope(
this);
739 llvm::BasicBlock *Done =
nullptr;
744 auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Ptr->stripPointerCasts());
748 bool IsGuaranteedNonNull =
749 SkippedChecks.
has(SanitizerKind::Null) || PtrToAlloca;
751 if ((
SanOpts.
has(SanitizerKind::Null) || AllowNullPointers) &&
752 !IsGuaranteedNonNull) {
761 if (!IsGuaranteedNonNull) {
762 if (AllowNullPointers) {
767 Builder.CreateCondBr(IsNonNull, Rest, Done);
770 Checks.push_back(std::make_pair(IsNonNull, SanitizerKind::Null));
776 !SkippedChecks.
has(SanitizerKind::ObjectSize) &&
779 llvm::Value *
Size = llvm::ConstantInt::get(
IntPtrTy, TySize);
784 llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Size);
785 if (!ConstantSize || !ConstantSize->isNullValue()) {
791 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::objectsize, Tys);
793 llvm::Value *NullIsUnknown =
Builder.getFalse();
795 llvm::Value *LargeEnough =
Builder.CreateICmpUGE(
796 Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic}), Size);
797 Checks.push_back(std::make_pair(LargeEnough, SanitizerKind::ObjectSize));
801 llvm::MaybeAlign AlignVal;
802 llvm::Value *PtrAsInt =
nullptr;
805 !SkippedChecks.
has(SanitizerKind::Alignment)) {
813 if (AlignVal && *AlignVal > llvm::Align(1) &&
814 (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
816 llvm::Value *Align =
Builder.CreateAnd(
817 PtrAsInt, llvm::ConstantInt::get(
IntPtrTy, AlignVal->value() - 1));
818 llvm::Value *Aligned =
821 Checks.push_back(std::make_pair(Aligned, SanitizerKind::Alignment));
825 if (Checks.size() > 0) {
826 llvm::Constant *StaticData[] = {
828 llvm::ConstantInt::get(
Int8Ty, AlignVal ? llvm::Log2(*AlignVal) : 1),
829 llvm::ConstantInt::get(
Int8Ty, TCK)};
830 EmitCheck(Checks, SanitizerHandler::TypeMismatch, StaticData,
831 PtrAsInt ? PtrAsInt : Ptr);
846 if (!IsGuaranteedNonNull) {
852 Builder.CreateCondBr(IsNonNull, VptrNotNull, Done);
858 llvm::raw_svector_ostream Out(MangledName);
866 llvm::Value *TypeHash =
867 llvm::ConstantInt::get(
Int64Ty, xxh3_64bits(Out.str()));
869 llvm::Type *VPtrTy = llvm::PointerType::get(
IntPtrTy, 0);
881 const int CacheSize = 128;
882 llvm::Type *HashTable = llvm::ArrayType::get(
IntPtrTy, CacheSize);
884 "__ubsan_vptr_type_cache");
885 llvm::Value *Slot =
Builder.CreateAnd(Hash,
888 llvm::Value *Indices[] = {
Builder.getInt32(0), Slot };
897 llvm::Value *EqualHash =
Builder.CreateICmpEQ(CacheVal, Hash);
898 llvm::Constant *StaticData[] = {
902 llvm::ConstantInt::get(
Int8Ty, TCK)
904 llvm::Value *DynamicData[] = { Ptr, Hash };
905 EmitCheck(std::make_pair(EqualHash, SanitizerKind::Vptr),
906 SanitizerHandler::DynamicTypeCacheMiss, StaticData,
920 uint64_t EltSize =
C.getTypeSizeInChars(EltTy).getQuantity();
928 auto *ParamDecl = dyn_cast<ParmVarDecl>(ArrayDeclRef->getDecl());
932 auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();
937 int POSType = POSAttr->getType();
938 if (POSType != 0 && POSType != 1)
942 auto PassedSizeIt = SizeArguments.find(ParamDecl);
943 if (PassedSizeIt == SizeArguments.end())
947 assert(LocalDeclMap.count(PassedSizeDecl) &&
"Passed size not loadable");
948 Address AddrOfSize = LocalDeclMap.find(PassedSizeDecl)->second;
951 llvm::Value *SizeOfElement =
952 llvm::ConstantInt::get(SizeInBytes->getType(), EltSize);
953 return Builder.CreateUDiv(SizeInBytes, SizeOfElement);
962 StrictFlexArraysLevel) {
965 IndexedType =
Base->getType();
966 return CGF.
Builder.getInt32(VT->getNumElements());
971 if (
const auto *CE = dyn_cast<CastExpr>(
Base)) {
972 if (CE->getCastKind() == CK_ArrayToPointerDecay &&
973 !CE->getSubExpr()->isFlexibleArrayMemberLike(CGF.
getContext(),
974 StrictFlexArraysLevel)) {
975 CodeGenFunction::SanitizerScope SanScope(&CGF);
977 IndexedType = CE->getSubExpr()->getType();
979 if (
const auto *CAT = dyn_cast<ConstantArrayType>(AT))
980 return CGF.
Builder.getInt(CAT->getSize());
982 if (
const auto *VAT = dyn_cast<VariableArrayType>(AT))
988 CodeGenFunction::SanitizerScope SanScope(&CGF);
990 QualType EltTy{
Base->getType()->getPointeeOrArrayElementType(), 0};
992 IndexedType =
Base->getType();
1017class StructAccessBase
1021 bool IsExpectedRecordDecl(
const Expr *
E)
const {
1029 StructAccessBase(
const RecordDecl *ExpectedRD) : ExpectedRD(ExpectedRD) {}
1052 const Expr *VisitStmt(
const Stmt *S) {
return nullptr; }
1068 return IsExpectedRecordDecl(
E) ?
E :
nullptr;
1071 if (IsExpectedRecordDecl(
E) &&
E->isArrow())
1073 const Expr *Res = Visit(
E->getBase());
1074 return !Res && IsExpectedRecordDecl(
E) ?
E : Res;
1077 return IsExpectedRecordDecl(
E) ?
E :
nullptr;
1080 return IsExpectedRecordDecl(
E) ?
E :
nullptr;
1084 if (IsExpectedRecordDecl(
E))
1086 return Visit(
E->getBase());
1089 if (
E->getCastKind() == CK_LValueToRValue)
1090 return IsExpectedRecordDecl(
E) ?
E :
nullptr;
1091 return Visit(
E->getSubExpr());
1094 return Visit(
E->getSubExpr());
1097 return Visit(
E->getSubExpr());
1100 return Visit(
E->getSubExpr());
1112 int64_t FieldNo = -1;
1121 Indices.emplace_back(CGF.
Builder.getInt32(FieldNo));
1130 Indices.emplace_back(CGF.
Builder.getInt32(FieldNo));
1144 const Expr *StructBase = StructAccessBase(RD).Visit(
Base);
1148 llvm::Value *Res =
nullptr;
1154 }
else if (StructBase->
isLValue()) {
1164 if (Indices.empty())
1167 Indices.push_back(
Builder.getInt32(0));
1170 RecIndicesTy(llvm::reverse(Indices)),
"counted_by.gep");
1190 llvm::Value *Index,
QualType IndexType,
1192 assert(
SanOpts.
has(SanitizerKind::ArrayBounds) &&
1193 "should not be called unless adding bounds checks");
1197 llvm::Value *Bound =
1206 QualType IndexedType,
bool Accessed) {
1210 SanitizerScope SanScope(
this);
1213 llvm::Value *IndexVal =
Builder.CreateIntCast(Index,
SizeTy, IndexSigned);
1214 llvm::Value *BoundVal =
Builder.CreateIntCast(Bound,
SizeTy,
false);
1216 llvm::Constant *StaticData[] = {
1221 llvm::Value *Check = Accessed ?
Builder.CreateICmpULT(IndexVal, BoundVal)
1222 :
Builder.CreateICmpULE(IndexVal, BoundVal);
1223 EmitCheck(std::make_pair(Check, SanitizerKind::ArrayBounds),
1224 SanitizerHandler::OutOfBounds, StaticData, Index);
1229 bool isInc,
bool isPre) {
1232 llvm::Value *NextVal;
1233 if (isa<llvm::IntegerType>(InVal.first->getType())) {
1234 uint64_t AmountVal = isInc ? 1 : -1;
1235 NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal,
true);
1238 NextVal =
Builder.CreateAdd(InVal.first, NextVal, isInc ?
"inc" :
"dec");
1241 llvm::APFloat FVal(
getContext().getFloatTypeSemantics(ElemTy), 1);
1247 NextVal =
Builder.CreateFAdd(InVal.first, NextVal, isInc ?
"inc" :
"dec");
1260 return isPre ? IncVal : InVal;
1270 DI->EmitExplicitCastType(
E->
getType());
1287 if (
const CastExpr *CE = dyn_cast<CastExpr>(
E)) {
1288 if (
const auto *ECE = dyn_cast<ExplicitCastExpr>(CE))
1291 switch (CE->getCastKind()) {
1295 case CK_AddressSpaceConversion:
1296 if (
auto PtrTy = CE->getSubExpr()->getType()->getAs<
PointerType>()) {
1297 if (PtrTy->getPointeeType()->isVoidType())
1303 CE->getSubExpr(), &InnerBaseInfo, &InnerTBAAInfo, IsKnownNonNull);
1304 if (BaseInfo) *BaseInfo = InnerBaseInfo;
1305 if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;
1307 if (isa<ExplicitCastExpr>(CE)) {
1311 E->
getType(), &TargetTypeBaseInfo, &TargetTypeTBAAInfo);
1324 if (CGF.
SanOpts.
has(SanitizerKind::CFIUnrelatedCast) &&
1325 CE->getCastKind() == CK_BitCast) {
1333 llvm::Type *ElemTy =
1336 if (CE->getCastKind() == CK_AddressSpaceConversion)
1345 case CK_ArrayToPointerDecay:
1349 case CK_UncheckedDerivedToBase:
1350 case CK_DerivedToBase: {
1357 CE->getSubExpr(), BaseInfo,
nullptr,
1359 CE->getCastKind() == CK_UncheckedDerivedToBase));
1360 auto Derived = CE->getSubExpr()->
getType()->getPointeeCXXRecordDecl();
1362 Addr, Derived, CE->path_begin(), CE->path_end(),
1375 if (UO->getOpcode() == UO_AddrOf) {
1384 if (
auto *
Call = dyn_cast<CallExpr>(
E)) {
1385 switch (
Call->getBuiltinCallee()) {
1388 case Builtin::BIaddressof:
1389 case Builtin::BI__addressof:
1390 case Builtin::BI__builtin_addressof: {
1404 true, BaseInfo, TBAAInfo, IsKnownNonNull);
1423 return Builder.CreateICmpNE(
V, llvm::Constant::getNullValue(
V->getType()));
1434 llvm::Value *
U = llvm::UndefValue::get(EltTy);
1449 llvm_unreachable(
"bad evaluation kind");
1469 while (!isa<CXXThisExpr>(
Base)) {
1471 if (isa<CXXDynamicCastExpr>(
Base))
1474 if (
const auto *CE = dyn_cast<CastExpr>(
Base)) {
1475 Base = CE->getSubExpr();
1476 }
else if (
const auto *PE = dyn_cast<ParenExpr>(
Base)) {
1477 Base = PE->getSubExpr();
1478 }
else if (
const auto *UO = dyn_cast<UnaryOperator>(
Base)) {
1479 if (UO->getOpcode() == UO_Extension)
1480 Base = UO->getSubExpr();
1492 if (
SanOpts.
has(SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(
E))
1498 if (
const auto *ME = dyn_cast<MemberExpr>(
E)) {
1501 SkippedChecks.
set(SanitizerKind::Alignment,
true);
1502 if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase()))
1503 SkippedChecks.
set(SanitizerKind::Null,
true);
1531 E->
getExprLoc(), [&] { LV = EmitLValueHelper(E, IsKnownNonNull); });
1541 if (isa<OpaqueValueExpr>(SE))
1543 return cast<CallExpr>(SE)->getCallReturnType(Ctx)->
getPointeeType();
1546LValue CodeGenFunction::EmitLValueHelper(
const Expr *
E,
1552 case Expr::ObjCPropertyRefExprClass:
1553 llvm_unreachable(
"cannot emit a property reference directly");
1555 case Expr::ObjCSelectorExprClass:
1557 case Expr::ObjCIsaExprClass:
1559 case Expr::BinaryOperatorClass:
1561 case Expr::CompoundAssignOperatorClass: {
1564 Ty = AT->getValueType();
1569 case Expr::CallExprClass:
1570 case Expr::CXXMemberCallExprClass:
1571 case Expr::CXXOperatorCallExprClass:
1572 case Expr::UserDefinedLiteralClass:
1574 case Expr::CXXRewrittenBinaryOperatorClass:
1575 return EmitLValue(cast<CXXRewrittenBinaryOperator>(
E)->getSemanticForm(),
1577 case Expr::VAArgExprClass:
1579 case Expr::DeclRefExprClass:
1581 case Expr::ConstantExprClass: {
1587 return EmitLValue(cast<ConstantExpr>(
E)->getSubExpr(), IsKnownNonNull);
1589 case Expr::ParenExprClass:
1590 return EmitLValue(cast<ParenExpr>(
E)->getSubExpr(), IsKnownNonNull);
1591 case Expr::GenericSelectionExprClass:
1592 return EmitLValue(cast<GenericSelectionExpr>(
E)->getResultExpr(),
1594 case Expr::PredefinedExprClass:
1596 case Expr::StringLiteralClass:
1598 case Expr::ObjCEncodeExprClass:
1600 case Expr::PseudoObjectExprClass:
1602 case Expr::InitListExprClass:
1604 case Expr::CXXTemporaryObjectExprClass:
1605 case Expr::CXXConstructExprClass:
1607 case Expr::CXXBindTemporaryExprClass:
1609 case Expr::CXXUuidofExprClass:
1611 case Expr::LambdaExprClass:
1614 case Expr::ExprWithCleanupsClass: {
1615 const auto *cleanups = cast<ExprWithCleanups>(
E);
1616 RunCleanupsScope
Scope(*
this);
1623 Scope.ForceCleanup({&
V});
1633 case Expr::CXXDefaultArgExprClass: {
1634 auto *DAE = cast<CXXDefaultArgExpr>(
E);
1635 CXXDefaultArgExprScope
Scope(*
this, DAE);
1636 return EmitLValue(DAE->getExpr(), IsKnownNonNull);
1638 case Expr::CXXDefaultInitExprClass: {
1639 auto *DIE = cast<CXXDefaultInitExpr>(
E);
1640 CXXDefaultInitExprScope
Scope(*
this, DIE);
1641 return EmitLValue(DIE->getExpr(), IsKnownNonNull);
1643 case Expr::CXXTypeidExprClass:
1646 case Expr::ObjCMessageExprClass:
1648 case Expr::ObjCIvarRefExprClass:
1650 case Expr::StmtExprClass:
1652 case Expr::UnaryOperatorClass:
1654 case Expr::ArraySubscriptExprClass:
1656 case Expr::MatrixSubscriptExprClass:
1658 case Expr::ArraySectionExprClass:
1660 case Expr::ExtVectorElementExprClass:
1662 case Expr::CXXThisExprClass:
1664 case Expr::MemberExprClass:
1666 case Expr::CompoundLiteralExprClass:
1668 case Expr::ConditionalOperatorClass:
1670 case Expr::BinaryConditionalOperatorClass:
1672 case Expr::ChooseExprClass:
1673 return EmitLValue(cast<ChooseExpr>(
E)->getChosenSubExpr(), IsKnownNonNull);
1674 case Expr::OpaqueValueExprClass:
1676 case Expr::SubstNonTypeTemplateParmExprClass:
1677 return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(
E)->getReplacement(),
1679 case Expr::ImplicitCastExprClass:
1680 case Expr::CStyleCastExprClass:
1681 case Expr::CXXFunctionalCastExprClass:
1682 case Expr::CXXStaticCastExprClass:
1683 case Expr::CXXDynamicCastExprClass:
1684 case Expr::CXXReinterpretCastExprClass:
1685 case Expr::CXXConstCastExprClass:
1686 case Expr::CXXAddrspaceCastExprClass:
1687 case Expr::ObjCBridgedCastExprClass:
1690 case Expr::MaterializeTemporaryExprClass:
1693 case Expr::CoawaitExprClass:
1695 case Expr::CoyieldExprClass:
1697 case Expr::PackIndexingExprClass:
1698 return EmitLValue(cast<PackIndexingExpr>(
E)->getSelectedExpr());
1699 case Expr::HLSLOutArgExprClass:
1700 llvm_unreachable(
"cannot emit a HLSL out argument directly");
1707 assert(
type.isCanonical());
1708 assert(!
type->isReferenceType());
1716 if (
const auto *RT = dyn_cast<RecordType>(
type))
1717 if (
const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1718 if (RD->hasMutableFields() || !RD->isTrivial())
1739 if (
const auto *ref = dyn_cast<ReferenceType>(
type)) {
1755CodeGenFunction::ConstantEmission
1761 if (isa<ParmVarDecl>(value)) {
1763 }
else if (
auto *var = dyn_cast<VarDecl>(value)) {
1765 }
else if (isa<EnumConstantDecl>(value)) {
1770 if (CEK ==
CEK_None)
return ConstantEmission();
1773 bool resultIsReference;
1779 resultIsReference =
false;
1780 resultType = refExpr->
getType();
1785 resultIsReference =
true;
1786 resultType = value->
getType();
1790 return ConstantEmission();
1795 return ConstantEmission();
1806 auto *MD = dyn_cast_or_null<CXXMethodDecl>(
CurCodeDecl);
1807 if (MD && MD->getParent()->isLambda() &&
1808 MD->getOverloadedOperator() == OO_Call) {
1811 if (
const VarDecl *VD = dyn_cast<const VarDecl>(
D)) {
1812 if (!VD->hasAttr<CUDADeviceAttr>()) {
1813 return ConstantEmission();
1822 result.
Val, resultType);
1826 if (isa<VarDecl>(value)) {
1827 if (!
getContext().DeclMustBeEmitted(cast<VarDecl>(value)))
1830 assert(isa<EnumConstantDecl>(value));
1835 if (resultIsReference)
1853CodeGenFunction::ConstantEmission
1857 return ConstantEmission();
1861 const CodeGenFunction::ConstantEmission &Constant,
Expr *
E) {
1862 assert(Constant &&
"not a constant");
1863 if (Constant.isReference())
1867 return Constant.getValue();
1882 return ET->getDecl()->getIntegerType()->isBooleanType();
1891 llvm::APInt &
Min, llvm::APInt &End,
1892 bool StrictEnums,
bool IsBool) {
1894 bool IsRegularCPlusPlusEnum = CGF.
getLangOpts().CPlusPlus && StrictEnums &&
1896 if (!IsBool && !IsRegularCPlusPlusEnum)
1909llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(
QualType Ty) {
1910 llvm::APInt
Min, End;
1916 return MDHelper.createRange(
Min, End);
1921 bool HasBoolCheck =
SanOpts.
has(SanitizerKind::Bool);
1922 bool HasEnumCheck =
SanOpts.
has(SanitizerKind::Enum);
1923 if (!HasBoolCheck && !HasEnumCheck)
1928 bool NeedsBoolCheck = HasBoolCheck && IsBool;
1929 bool NeedsEnumCheck = HasEnumCheck && Ty->
getAs<
EnumType>();
1930 if (!NeedsBoolCheck && !NeedsEnumCheck)
1937 cast<llvm::IntegerType>(
Value->
getType())->getBitWidth() == 1)
1940 if (NeedsEnumCheck &&
1944 llvm::APInt
Min, End;
1949 SanitizerScope SanScope(
this);
1953 Check =
Builder.CreateICmpULE(
Value, llvm::ConstantInt::get(Ctx, End));
1955 llvm::Value *Upper =
1956 Builder.CreateICmpSLE(
Value, llvm::ConstantInt::get(Ctx, End));
1957 llvm::Value *Lower =
1959 Check =
Builder.CreateAnd(Upper, Lower);
1964 NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool;
1965 EmitCheck(std::make_pair(Check, Kind), SanitizerHandler::LoadInvalidValue,
1975 bool isNontemporal) {
1976 if (
auto *GV = dyn_cast<llvm::GlobalValue>(Addr.
getBasePointer()))
1977 if (GV->isThreadLocal())
1983 if (ClangVecTy->isExtVectorBoolType()) {
1985 unsigned ValNumElems =
1986 cast<llvm::FixedVectorType>(ValTy)->getNumElements();
1989 const auto *RawIntTy = RawIntV->getType();
1990 assert(RawIntTy->isIntegerTy() &&
"compressed iN storage for bitvectors");
1992 auto *PaddedVecTy = llvm::FixedVectorType::get(
1993 Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
1994 llvm::Value *
V =
Builder.CreateBitCast(RawIntV, PaddedVecTy);
2003 const auto *VTy = cast<llvm::FixedVectorType>(EltTy);
2007 llvm::VectorType *vec4Ty =
2008 llvm::FixedVectorType::get(VTy->getElementType(), 4);
2030 if (isNontemporal) {
2031 llvm::MDNode *
Node = llvm::MDNode::get(
2032 Load->getContext(), llvm::ConstantAsMetadata::get(
Builder.getInt32(1)));
2033 Load->setMetadata(llvm::LLVMContext::MD_nontemporal,
Node);
2042 if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {
2043 Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo);
2044 Load->setMetadata(llvm::LLVMContext::MD_noundef,
2064 unsigned MemNumElems = StoreTy->getPrimitiveSizeInBits();
2081 auto *PaddedVecTy = llvm::FixedVectorType::get(
2082 Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
2086 unsigned ValNumElems = cast<llvm::FixedVectorType>(ValTy)->getNumElements();
2102 bool IsVector =
true) {
2103 auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.
getElementType());
2104 if (ArrayTy && IsVector) {
2105 auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
2106 ArrayTy->getNumElements());
2110 auto *VectorTy = dyn_cast<llvm::VectorType>(Addr.
getElementType());
2111 if (VectorTy && !IsVector) {
2112 auto *ArrayTy = llvm::ArrayType::get(
2113 VectorTy->getElementType(),
2114 cast<llvm::FixedVectorType>(VectorTy)->getNumElements());
2128 value->getType()->isVectorTy());
2138 bool isInit,
bool isNontemporal) {
2139 if (
auto *GV = dyn_cast<llvm::GlobalValue>(Addr.
getBasePointer()))
2140 if (GV->isThreadLocal())
2146 auto *VecTy = dyn_cast<llvm::FixedVectorType>(SrcTy);
2149 if (VecTy && !ClangVecTy->isExtVectorBoolType() &&
2150 cast<llvm::FixedVectorType>(VecTy)->getNumElements() == 3) {
2154 SrcTy = llvm::FixedVectorType::get(VecTy->getElementType(), 4);
2173 if (isNontemporal) {
2174 llvm::MDNode *
Node =
2175 llvm::MDNode::get(
Store->getContext(),
2176 llvm::ConstantAsMetadata::get(
Builder.getInt32(1)));
2177 Store->setMetadata(llvm::LLVMContext::MD_nontemporal,
Node);
2217 llvm_unreachable(
"bad evaluation kind");
2273 llvm::MatrixBuilder MB(
Builder);
2274 MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
2276 llvm::LoadInst *
Load =
2281 assert(LV.
isBitField() &&
"Unknown LValue type!");
2299 const unsigned StorageSize =
2302 assert(
static_cast<unsigned>(Offset + Info.
Size) <= StorageSize);
2303 unsigned HighBits = StorageSize - Offset - Info.
Size;
2305 Val =
Builder.CreateShl(Val, HighBits,
"bf.shl");
2306 if (Offset + HighBits)
2307 Val =
Builder.CreateAShr(Val, Offset + HighBits,
"bf.ashr");
2310 Val =
Builder.CreateLShr(Val, Offset,
"bf.lshr");
2311 if (
static_cast<unsigned>(Offset) + Info.
Size < StorageSize)
2313 Val, llvm::APInt::getLowBitsSet(StorageSize, Info.
Size),
"bf.clear");
2329 llvm::Type *DstTy = llvm::FixedVectorType::get(Vec->getType(), 1);
2331 Vec =
Builder.CreateInsertElement(DstTy, Vec, Zero,
"cast.splat");
2341 llvm::Value *Elt = llvm::ConstantInt::get(
SizeTy, InIdx);
2349 for (
unsigned i = 0; i != NumResultElts; ++i)
2352 Vec =
Builder.CreateShuffleVector(Vec, Mask);
2371 return VectorBasePtrPlusIx;
2377 "Bad type for register variable");
2378 llvm::MDNode *RegName = cast<llvm::MDNode>(
2379 cast<llvm::MetadataAsValue>(LV.
getGlobalReg())->getMetadata());
2383 llvm::Type *Ty = OrigTy;
2384 if (OrigTy->isPointerTy())
2386 llvm::Type *Types[] = { Ty };
2388 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::read_register, Types);
2390 F, llvm::MetadataAsValue::get(Ty->getContext(), RegName));
2391 if (OrigTy->isPointerTy())
2406 auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Vec->getType());
2408 auto *IRVecTy = llvm::FixedVectorType::get(
2409 Builder.getInt1Ty(), IRStoreTy->getPrimitiveSizeInBits());
2410 Vec =
Builder.CreateBitCast(Vec, IRVecTy);
2417 Vec =
Builder.CreateBitCast(Vec, IRStoreTy);
2436 llvm::MatrixBuilder MB(
Builder);
2437 MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
2447 assert(Dst.
isBitField() &&
"Unknown LValue type");
2455 llvm_unreachable(
"present but none");
2503 RHS =
Builder.CreatePtrToInt(RHS, ResultType,
"sub.ptr.rhs.cast");
2505 ResultType,
"sub.ptr.lhs.cast");
2506 llvm::Value *BytesBetween =
Builder.CreateSub(LHS, RHS,
"ivar.offset");
2517 assert(Src.
isScalar() &&
"Can't emit an agg store with this method");
2533 llvm::Value *MaskedVal = SrcVal;
2535 const bool UseVolatile =
2538 const unsigned StorageSize =
2543 if (StorageSize != Info.
Size) {
2544 assert(StorageSize > Info.
Size &&
"Invalid bitfield size.");
2551 SrcVal, llvm::APInt::getLowBitsSet(StorageSize, Info.
Size),
2555 SrcVal =
Builder.CreateShl(SrcVal, Offset,
"bf.shl");
2559 Val, ~llvm::APInt::getBitsSet(StorageSize, Offset, Offset + Info.
Size),
2563 SrcVal =
Builder.CreateOr(Val, SrcVal,
"bf.set");
2565 assert(Offset == 0);
2581 llvm::Value *ResultVal = MaskedVal;
2585 assert(Info.
Size <= StorageSize);
2586 unsigned HighBits = StorageSize - Info.
Size;
2588 ResultVal =
Builder.CreateShl(ResultVal, HighBits,
"bf.result.shl");
2589 ResultVal =
Builder.CreateAShr(ResultVal, HighBits,
"bf.result.ashr");
2607 "this should only occur for non-vector l-values");
2620 unsigned NumSrcElts = VTy->getNumElements();
2621 unsigned NumDstElts =
2622 cast<llvm::FixedVectorType>(Vec->getType())->getNumElements();
2623 if (NumDstElts == NumSrcElts) {
2628 for (
unsigned i = 0; i != NumSrcElts; ++i)
2631 Vec =
Builder.CreateShuffleVector(SrcVal, Mask);
2632 }
else if (NumDstElts > NumSrcElts) {
2638 for (
unsigned i = 0; i != NumSrcElts; ++i)
2639 ExtMask.push_back(i);
2640 ExtMask.resize(NumDstElts, -1);
2641 llvm::Value *ExtSrcVal =
Builder.CreateShuffleVector(SrcVal, ExtMask);
2644 for (
unsigned i = 0; i != NumDstElts; ++i)
2654 for (
unsigned i = 0; i != NumSrcElts; ++i)
2656 Vec =
Builder.CreateShuffleVector(Vec, ExtSrcVal, Mask);
2659 llvm_unreachable(
"unexpected shorten vector length");
2665 llvm::Value *Elt = llvm::ConstantInt::get(
SizeTy, InIdx);
2666 Vec =
Builder.CreateInsertElement(Vec, SrcVal, Elt);
2676 "Bad type for register variable");
2677 llvm::MDNode *RegName = cast<llvm::MDNode>(
2678 cast<llvm::MetadataAsValue>(Dst.
getGlobalReg())->getMetadata());
2679 assert(RegName &&
"Register LValue is not metadata");
2683 llvm::Type *Ty = OrigTy;
2684 if (OrigTy->isPointerTy())
2686 llvm::Type *Types[] = { Ty };
2688 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
2690 if (OrigTy->isPointerTy())
2693 F, {llvm::MetadataAsValue::get(Ty->getContext(), RegName),
Value});
2701 bool IsMemberAccess=
false) {
2705 if (isa<ObjCIvarRefExpr>(
E)) {
2718 auto *Exp = cast<ObjCIvarRefExpr>(
const_cast<Expr *
>(
E));
2724 if (
const auto *Exp = dyn_cast<DeclRefExpr>(
E)) {
2725 if (
const auto *VD = dyn_cast<VarDecl>(Exp->getDecl())) {
2726 if (VD->hasGlobalStorage()) {
2735 if (
const auto *Exp = dyn_cast<UnaryOperator>(
E)) {
2740 if (
const auto *Exp = dyn_cast<ParenExpr>(
E)) {
2754 if (
const auto *Exp = dyn_cast<GenericSelectionExpr>(
E)) {
2759 if (
const auto *Exp = dyn_cast<ImplicitCastExpr>(
E)) {
2764 if (
const auto *Exp = dyn_cast<CStyleCastExpr>(
E)) {
2769 if (
const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(
E)) {
2774 if (
const auto *Exp = dyn_cast<ArraySubscriptExpr>(
E)) {
2787 if (
const auto *Exp = dyn_cast<MemberExpr>(
E)) {
2801 CGF, VD, Addr,
Loc);
2812 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2813 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2817 if (!Res || ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2818 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2821 assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
2822 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2823 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2825 "Expected link clause OR to clause with unified memory enabled.");
2835 llvm::LoadInst *
Load =
2840 PointeeBaseInfo, PointeeTBAAInfo);
2849 PointeeBaseInfo, PointeeTBAAInfo);
2859 BaseInfo, TBAAInfo);
2889 V = CGF.
Builder.CreateThreadLocalAddress(
V);
2893 Address Addr(
V, RealVarTy, Alignment);
2897 VD->
hasAttr<OMPThreadPrivateDeclAttr>()) {
2912 if (FD->
hasAttr<WeakRefAttr>()) {
2931 llvm::Value *ThisValue) {
2944 AsmLabelAttr *
Asm = VD->
getAttr<AsmLabelAttr>();
2945 assert(
Asm->getLabel().size() < 64-Name.size() &&
2946 "Register name too big");
2947 Name.append(
Asm->getLabel());
2948 llvm::NamedMDNode *M =
2949 CGM.
getModule().getOrInsertNamedMetadata(Name);
2950 if (M->getNumOperands() == 0) {
2953 llvm::Metadata *Ops[] = {Str};
2960 llvm::MetadataAsValue::get(CGM.
getLLVMContext(), M->getOperand(0));
2974 if (
E->refersToEnclosingVariableOrCapture())
3003 case llvm::GlobalValue::ExternalLinkage:
3004 case llvm::GlobalValue::LinkOnceODRLinkage:
3005 case llvm::GlobalValue::WeakODRLinkage:
3006 case llvm::GlobalValue::InternalLinkage:
3007 case llvm::GlobalValue::PrivateLinkage:
3019 "should not emit an unevaluated operand");
3021 if (
const auto *VD = dyn_cast<VarDecl>(ND)) {
3024 VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
3032 (VD->getType()->isReferenceType() ||
3034 VD->getAnyInitializer(VD);
3036 E->getLocation(), *VD->evaluateValue(), VD->
getType());
3037 assert(Val &&
"failed to emit constant expression");
3040 if (!VD->getType()->isReferenceType()) {
3045 auto *PTy = llvm::PointerType::get(
3046 VarTy,
getTypes().getTargetAddressSpace(VD->getType()));
3063 if (
E->refersToEnclosingVariableOrCapture()) {
3064 VD = VD->getCanonicalDecl();
3068 auto I = LocalDeclMap.find(VD);
3069 if (I != LocalDeclMap.end()) {
3071 if (VD->getType()->isReferenceType())
3110 assert((ND->
isUsed(
false) || !isa<VarDecl>(ND) ||
E->isNonOdrUse() ||
3111 !
E->getLocation().isValid()) &&
3112 "Should not use decl without marking it used!");
3114 if (ND->
hasAttr<WeakRefAttr>()) {
3115 const auto *VD = cast<ValueDecl>(ND);
3120 if (
const auto *VD = dyn_cast<VarDecl>(ND)) {
3122 if (VD->hasLinkage() || VD->isStaticDataMember())
3128 auto iter = LocalDeclMap.find(VD);
3129 if (iter != LocalDeclMap.end()) {
3130 addr = iter->second;
3134 }
else if (VD->isStaticLocal()) {
3142 llvm_unreachable(
"DeclRefExpr for Decl not entered in LocalDeclMap?");
3153 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
3160 bool isBlockByref = VD->isEscapingByref();
3170 bool isLocalStorage = VD->hasLocalStorage();
3172 bool NonGCable = isLocalStorage &&
3180 bool isImpreciseLifetime =
3181 (isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());
3182 if (isImpreciseLifetime)
3188 if (
const auto *FD = dyn_cast<FunctionDecl>(ND))
3194 if (
const auto *BD = dyn_cast<BindingDecl>(ND)) {
3195 if (
E->refersToEnclosingVariableOrCapture()) {
3204 if (
const auto *GD = dyn_cast<MSGuidDecl>(ND))
3208 if (
const auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND)) {
3212 if (AS !=
T.getAddressSpace()) {
3216 CGM, ATPO.getPointer(), AS,
T.getAddressSpace(), PtrTy);
3217 ATPO =
ConstantAddress(ASC, ATPO.getElementType(), ATPO.getAlignment());
3223 llvm_unreachable(
"Unhandled DeclRefExpr");
3228 if (
E->getOpcode() == UO_Extension)
3232 switch (
E->getOpcode()) {
3233 default: llvm_unreachable(
"Unknown unary operator lvalue!");
3236 assert(!
T.isNull() &&
"CodeGenFunction::EmitUnaryOpLValue: Illegal type");
3258 assert(LV.
isSimple() &&
"real/imag on non-ordinary l-value");
3262 if (
E->getOpcode() == UO_Real &&
3271 (
E->getOpcode() == UO_Real
3282 bool isInc =
E->getOpcode() == UO_PreInc;
3304 auto SL =
E->getFunctionName();
3305 assert(SL !=
nullptr &&
"No StringLiteral name in PredefinedExpr");
3306 StringRef FnName =
CurFn->getName();
3307 if (FnName.starts_with(
"\01"))
3308 FnName = FnName.substr(1);
3309 StringRef NameItems[] = {
3311 std::string GVName = llvm::join(NameItems, NameItems + 2,
".");
3312 if (
auto *BD = dyn_cast_or_null<BlockDecl>(
CurCodeDecl)) {
3313 std::string Name = std::string(SL->getString());
3314 if (!Name.empty()) {
3315 unsigned Discriminator =
3318 Name +=
"_" + Twine(Discriminator + 1).str();
3349 bool IsBitInt =
false;
3361 " non positive amount of bits in __BitInt type");
3363 " too many bits in __BitInt type");
3379 (
intptr_t)
T.getAsOpaquePtr(), StringRef(),
3380 StringRef(), {}, Buffer, {});
3385 char S[6] = {
'\0',
'\0',
'\0',
'\0',
'\0',
'\0'};
3388 llvm::support::endian::write32(S + 1, Bits,
3390 ? llvm::endianness::big
3391 : llvm::endianness::little);
3392 StringRef Str = StringRef(S,
sizeof(S) /
sizeof(
decltype(S[0])));
3396 llvm::Constant *Components[] = {
3400 llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(Components);
3402 auto *GV =
new llvm::GlobalVariable(
3404 true, llvm::GlobalVariable::PrivateLinkage, Descriptor);
3405 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3417 if (
V->getType() == TargetTy)
3422 if (
V->getType()->isFloatingPointTy()) {
3423 unsigned Bits =
V->getType()->getPrimitiveSizeInBits().getFixedValue();
3424 if (Bits <= TargetTy->getIntegerBitWidth())
3430 if (
V->getType()->isIntegerTy() &&
3431 V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
3432 return Builder.CreateZExt(
V, TargetTy);
3435 if (!
V->getType()->isPointerTy()) {
3440 return Builder.CreatePtrToInt(
V, TargetTy);
3460 int PathComponentsToStrip =
3462 if (PathComponentsToStrip < 0) {
3463 assert(PathComponentsToStrip !=
INT_MIN);
3464 int PathComponentsToKeep = -PathComponentsToStrip;
3465 auto I = llvm::sys::path::rbegin(FilenameString);
3466 auto E = llvm::sys::path::rend(FilenameString);
3467 while (I !=
E && --PathComponentsToKeep)
3470 FilenameString = FilenameString.substr(I -
E);
3471 }
else if (PathComponentsToStrip > 0) {
3472 auto I = llvm::sys::path::begin(FilenameString);
3473 auto E = llvm::sys::path::end(FilenameString);
3474 while (I !=
E && PathComponentsToStrip--)
3479 FilenameString.substr(I - llvm::sys::path::begin(FilenameString));
3481 FilenameString = llvm::sys::path::filename(FilenameString);
3487 cast<llvm::GlobalVariable>(
3488 FilenameGV.getPointer()->stripPointerCasts()));
3489 Filename = FilenameGV.getPointer();
3500 return llvm::ConstantStruct::getAnon(
Data);
3505enum class CheckRecoverableKind {
3517 assert(Kind.countPopulation() == 1);
3518 if (Kind == SanitizerKind::Vptr)
3519 return CheckRecoverableKind::AlwaysRecoverable;
3520 else if (Kind == SanitizerKind::Return || Kind == SanitizerKind::Unreachable)
3521 return CheckRecoverableKind::Unrecoverable;
3523 return CheckRecoverableKind::Recoverable;
3527struct SanitizerHandlerInfo {
3528 char const *
const Name;
3534#define SANITIZER_CHECK(Enum, Name, Version) {#Name, Version},
3536#undef SANITIZER_CHECK
3540 llvm::FunctionType *FnType,
3543 CheckRecoverableKind RecoverKind,
bool IsFatal,
3544 llvm::BasicBlock *ContBB,
bool NoMerge) {
3545 assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
3546 std::optional<ApplyDebugLocation> DL;
3547 if (!CGF.
Builder.getCurrentDebugLocation()) {
3551 bool NeedsAbortSuffix =
3552 IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;
3555 const StringRef CheckName = CheckInfo.Name;
3556 std::string FnName =
"__ubsan_handle_" + CheckName.str();
3557 if (CheckInfo.Version && !MinimalRuntime)
3558 FnName +=
"_v" + llvm::utostr(CheckInfo.Version);
3560 FnName +=
"_minimal";
3561 if (NeedsAbortSuffix)
3564 !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;
3568 B.addAttribute(llvm::Attribute::NoReturn)
3569 .addAttribute(llvm::Attribute::NoUnwind);
3571 B.addUWTableAttr(llvm::UWTableKind::Default);
3576 llvm::AttributeList::FunctionIndex, B),
3582 HandlerCall->addFnAttr(llvm::Attribute::NoMerge);
3584 HandlerCall->setDoesNotReturn();
3585 CGF.
Builder.CreateUnreachable();
3592 ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked,
3596 assert(Checked.size() > 0);
3597 assert(CheckHandler >= 0 &&
3601 llvm::Value *FatalCond =
nullptr;
3602 llvm::Value *RecoverableCond =
nullptr;
3603 llvm::Value *TrapCond =
nullptr;
3604 bool NoMerge =
false;
3605 for (
int i = 0, n = Checked.size(); i < n; ++i) {
3606 llvm::Value *Check = Checked[i].first;
3608 llvm::Value *&Cond =
3614 Cond = Cond ?
Builder.CreateAnd(Cond, Check) : Check;
3621 llvm::Value *Allow =
3623 llvm::ConstantInt::get(
CGM.
Int8Ty, CheckHandler));
3625 for (llvm::Value **Cond : {&FatalCond, &RecoverableCond, &TrapCond}) {
3633 if (!FatalCond && !RecoverableCond)
3636 llvm::Value *JointCond;
3637 if (FatalCond && RecoverableCond)
3638 JointCond =
Builder.CreateAnd(FatalCond, RecoverableCond);
3640 JointCond = FatalCond ? FatalCond : RecoverableCond;
3646 for (
int i = 1, n = Checked.size(); i < n; ++i) {
3648 "All recoverable kinds in a single check must be same!");
3655 llvm::Instruction *Branch =
Builder.CreateCondBr(JointCond, Cont, Handlers);
3658 llvm::MDNode *
Node = MDHelper.createLikelyBranchWeights();
3659 Branch->setMetadata(llvm::LLVMContext::MD_prof,
Node);
3668 Args.reserve(DynamicArgs.size() + 1);
3669 ArgTypes.reserve(DynamicArgs.size() + 1);
3672 if (!StaticArgs.empty()) {
3673 llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3674 auto *InfoPtr =
new llvm::GlobalVariable(
3676 llvm::GlobalVariable::PrivateLinkage, Info,
"",
nullptr,
3677 llvm::GlobalVariable::NotThreadLocal,
3679 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3681 Args.push_back(InfoPtr);
3682 ArgTypes.push_back(Args.back()->getType());
3685 for (
size_t i = 0, n = DynamicArgs.size(); i != n; ++i) {
3691 llvm::FunctionType *FnType =
3692 llvm::FunctionType::get(
CGM.
VoidTy, ArgTypes,
false);
3694 if (!FatalCond || !RecoverableCond) {
3698 (FatalCond !=
nullptr), Cont, NoMerge);
3702 llvm::BasicBlock *NonFatalHandlerBB =
3705 Builder.CreateCondBr(FatalCond, NonFatalHandlerBB, FatalHandlerBB);
3708 NonFatalHandlerBB, NoMerge);
3718 SanitizerMask Kind, llvm::Value *Cond, llvm::ConstantInt *TypeId,
3723 llvm::BranchInst *BI =
Builder.CreateCondBr(Cond, Cont, CheckBB);
3726 llvm::MDNode *
Node = MDHelper.createLikelyBranchWeights();
3727 BI->setMetadata(llvm::LLVMContext::MD_prof,
Node);
3733 llvm::CallInst *CheckCall;
3734 llvm::FunctionCallee SlowPathFn;
3736 llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3738 new llvm::GlobalVariable(
CGM.
getModule(), Info->getType(),
false,
3739 llvm::GlobalVariable::PrivateLinkage, Info);
3740 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3744 "__cfi_slowpath_diag",
3747 CheckCall =
Builder.CreateCall(SlowPathFn, {TypeId, Ptr, InfoPtr});
3752 CheckCall =
Builder.CreateCall(SlowPathFn, {TypeId, Ptr});
3756 cast<llvm::GlobalValue>(SlowPathFn.getCallee()->stripPointerCasts()));
3757 CheckCall->setDoesNotThrow();
3767 QualType QInt64Ty =
C.getIntTypeForBitwidth(64,
false);
3774 FnArgs.push_back(&ArgCallsiteTypeId);
3775 FnArgs.push_back(&ArgAddr);
3776 FnArgs.push_back(&ArgCFICheckFailData);
3780 llvm::Function *F = llvm::Function::Create(
3782 llvm::GlobalValue::WeakAnyLinkage,
"__cfi_check", M);
3785 F->setAlignment(llvm::Align(4096));
3788 llvm::LLVMContext &Ctx = M->getContext();
3789 llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx,
"entry", F);
3792 llvm::CallInst::Create(M->getFunction(
"__cfi_check_fail"), Args,
"", BB);
3793 llvm::ReturnInst::Create(Ctx,
nullptr, BB);
3804 SanitizerScope SanScope(
this);
3810 Args.push_back(&ArgData);
3811 Args.push_back(&ArgAddr);
3816 llvm::Function *F = llvm::Function::Create(
3818 llvm::GlobalValue::WeakODRLinkage,
"__cfi_check_fail", &
CGM.
getModule());
3822 F->setVisibility(llvm::GlobalValue::HiddenVisibility);
3840 llvm::Value *DataIsNotNullPtr =
3842 EmitTrapCheck(DataIsNotNullPtr, SanitizerHandler::CFICheckFail);
3844 llvm::StructType *SourceLocationTy =
3846 llvm::StructType *CfiCheckFailDataTy =
3855 llvm::Value *AllVtables = llvm::MetadataAsValue::get(
3858 llvm::Value *ValidVtable =
Builder.CreateZExt(
3860 {Addr, AllVtables}),
3863 const std::pair<int, SanitizerMask> CheckKinds[] = {
3871 for (
auto CheckKindMaskPair : CheckKinds) {
3872 int Kind = CheckKindMaskPair.first;
3875 Builder.CreateICmpNE(CheckKind, llvm::ConstantInt::get(
Int8Ty, Kind));
3877 EmitCheck(std::make_pair(Cond, Mask), SanitizerHandler::CFICheckFail, {},
3878 {
Data, Addr, ValidVtable});
3890 if (
SanOpts.
has(SanitizerKind::Unreachable)) {
3891 SanitizerScope SanScope(
this);
3893 SanitizerKind::Unreachable),
3894 SanitizerHandler::BuiltinUnreachable,
3907 if ((
int)TrapBBs.size() <= CheckHandlerID)
3908 TrapBBs.resize(CheckHandlerID + 1);
3910 llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
3915 if (TrapBB && !NoMerge) {
3916 auto Call = TrapBB->begin();
3917 assert(isa<llvm::CallInst>(
Call) &&
"Expected call in trap BB");
3919 Call->applyMergedLocation(
Call->getDebugLoc(),
3920 Builder.getCurrentDebugLocation());
3921 Builder.CreateCondBr(Checked, Cont, TrapBB);
3924 Builder.CreateCondBr(Checked, Cont, TrapBB);
3927 llvm::CallInst *TrapCall =
3929 llvm::ConstantInt::get(
CGM.
Int8Ty, CheckHandlerID));
3932 auto A = llvm::Attribute::get(
getLLVMContext(),
"trap-func-name",
3934 TrapCall->addFnAttr(A);
3937 TrapCall->addFnAttr(llvm::Attribute::NoMerge);
3938 TrapCall->setDoesNotReturn();
3939 TrapCall->setDoesNotThrow();
3947 llvm::CallInst *TrapCall =
3951 auto A = llvm::Attribute::get(
getLLVMContext(),
"trap-func-name",
3953 TrapCall->addFnAttr(A);
3957 TrapCall->addFnAttr(llvm::Attribute::NoMerge);
3965 "Array to pointer decay must have array source type!");
3980 "Expected pointer to array");
4000 const auto *CE = dyn_cast<CastExpr>(
E);
4001 if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)
4013 llvm::Type *elemType,
4019 const llvm::Twine &name =
"arrayidx") {
4031 llvm::Type *elementType,
bool inbounds,
4034 const llvm::Twine &name =
"arrayidx") {
4049 if (
auto constantIdx = dyn_cast<llvm::ConstantInt>(idx)) {
4050 CharUnits offset = constantIdx->getZExtValue() * eltSize;
4069 return D &&
D->
hasAttr<BPFPreserveStaticOffsetAttr>();
4076 if (PointeeType.
isNull())
4089 llvm::Function *Fn =
4109 if (
const auto *ME = dyn_cast<MemberExpr>(
E))
4112 if (
const auto *DRE = dyn_cast<DeclRefExpr>(
E)) {
4113 const auto *VarDef = dyn_cast<VarDecl>(DRE->getDecl());
4117 const auto *PtrT = VarDef->getType()->getAs<
PointerType>();
4123 if (
const auto *RecT = dyn_cast<RecordType>(PointeeT))
4124 return RecT->getDecl()->
hasAttr<BPFPreserveAccessIndexAttr>();
4137 const llvm::Twine &name =
"arrayidx") {
4140 for (
auto *idx : indices.drop_back())
4141 assert(isa<llvm::ConstantInt>(idx) &&
4142 cast<llvm::ConstantInt>(idx)->isZero());
4159 llvm::Value *eltPtr;
4160 auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back());
4165 signedIndices, loc, eltAlign, name);
4169 unsigned idx = LastIndex->getZExtValue();
4170 llvm::DIType *DbgInfo =
nullptr;
4173 eltPtr = CGF.
Builder.CreatePreserveArrayAccessIndex(
4183 const FieldDecl *Field, int64_t &Offset) {
4186 unsigned FieldNo = 0;
4221 if (FD1OuterRec != FD2OuterRec)
4223 return std::optional<int64_t>();
4225 int64_t FD1Offset = 0;
4227 return std::optional<int64_t>();
4229 int64_t FD2Offset = 0;
4231 return std::optional<int64_t>();
4233 return std::make_optional<int64_t>(FD1Offset - FD2Offset);
4240 llvm::Value *IdxPre =
4242 bool SignedIndices =
false;
4243 auto EmitIdxAfterBase = [&, IdxPre](
bool Promote) -> llvm::Value * {
4245 if (
E->getLHS() !=
E->getIdx()) {
4246 assert(
E->getRHS() ==
E->getIdx() &&
"index was neither LHS nor RHS");
4252 SignedIndices |= IdxSigned;
4258 if (Promote && Idx->getType() !=
IntPtrTy)
4268 !isa<ExtVectorElementExpr>(
E->getBase())) {
4271 auto *Idx = EmitIdxAfterBase(
false);
4272 assert(LHS.
isSimple() &&
"Can only subscript lvalue vectors here!");
4280 if (isa<ExtVectorElementExpr>(
E->getBase())) {
4282 auto *Idx = EmitIdxAfterBase(
true);
4301 auto *Idx = EmitIdxAfterBase(
true);
4311 Idx =
Builder.CreateMul(Idx, numElements);
4313 Idx =
Builder.CreateNSWMul(Idx, numElements);
4325 auto *Idx = EmitIdxAfterBase(
true);
4328 llvm::Value *InterfaceSizeVal =
4329 llvm::ConstantInt::get(Idx->getType(), InterfaceSize.
getQuantity());
4331 llvm::Value *ScaledIdx =
Builder.CreateMul(Idx, InterfaceSizeVal);
4342 llvm::Value *EltPtr =
4344 ScaledIdx,
false, SignedIndices,
E->
getExprLoc());
4345 Addr =
Address(EltPtr, OrigBaseElemTy, EltAlign);
4351 assert(Array->getType()->isArrayType() &&
4352 "Array to pointer decay must have array source type!");
4356 if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
4360 auto *Idx = EmitIdxAfterBase(
true);
4362 if (
SanOpts.
has(SanitizerKind::ArrayBounds)) {
4373 if (
const auto *ME = dyn_cast<MemberExpr>(Array);
4379 if (std::optional<int64_t> Diff =
4388 llvm::Type *CountTy =
ConvertType(CountFD->getType());
4393 ".counted_by.load");
4397 Array->getType(), Accessed);
4406 *
this, ArrayLV.
getAddress(), {CGM.getSize(CharUnits::Zero()), Idx},
4414 auto *Idx = EmitIdxAfterBase(
true);