36 #include "llvm/ADT/ArrayRef.h"
37 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
38 #include "llvm/IR/DataLayout.h"
39 #include "llvm/IR/Dominators.h"
40 #include "llvm/IR/FPEnv.h"
41 #include "llvm/IR/IntrinsicInst.h"
42 #include "llvm/IR/Intrinsics.h"
43 #include "llvm/IR/MDBuilder.h"
44 #include "llvm/IR/Operator.h"
45 #include "llvm/Support/CRC.h"
46 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
47 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
49 using namespace clang;
50 using namespace CodeGen;
56 if (CGOpts.DisableLifetimeMarkers)
60 if (CGOpts.SanitizeAddressUseAfterScope ||
66 return CGOpts.OptimizationLevel != 0;
69 CodeGenFunction::CodeGenFunction(
CodeGenModule &cgm,
bool suppressNewContext)
71 Builder(cgm, cgm.getModule().getContext(),
llvm::ConstantFolder(),
73 SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
74 DebugInfo(CGM.getModuleDebugInfo()), PGO(cgm),
75 ShouldEmitLifetimeMarkers(
77 if (!suppressNewContext)
101 llvm::fp::ExceptionBehavior
109 llvm_unreachable(
"Unsupported FP Exception Behavior");
114 llvm::FastMathFlags FMF;
115 FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
116 FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
117 FMF.setNoInfs(FPFeatures.getNoHonorInfs());
118 FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
119 FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
120 FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
134 ConstructorHelper(FPFeatures);
137 void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(
FPOptions FPFeatures) {
138 OldFPFeatures = CGF.CurFPFeatures;
139 CGF.CurFPFeatures = FPFeatures;
141 OldExcept = CGF.Builder.getDefaultConstrainedExcept();
142 OldRounding = CGF.Builder.getDefaultConstrainedRounding();
144 if (OldFPFeatures == FPFeatures)
147 FMFGuard.emplace(CGF.Builder);
150 CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
151 auto NewExceptionBehavior =
154 CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
156 CGF.SetFastMathFlags(FPFeatures);
158 assert((CGF.CurFuncDecl ==
nullptr || CGF.Builder.getIsFPConstrained() ||
159 isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||
160 isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||
161 (NewExceptionBehavior == llvm::fp::ebIgnore &&
162 NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
163 "FPConstrained should be enabled on entire function");
165 auto mergeFnAttrValue = [&](StringRef Name,
bool Value) {
167 CGF.CurFn->getFnAttribute(Name).getValueAsBool();
168 auto NewValue = OldValue &
Value;
169 if (OldValue != NewValue)
170 CGF.CurFn->addFnAttr(Name, llvm::toStringRef(NewValue));
172 mergeFnAttrValue(
"no-infs-fp-math", FPFeatures.getNoHonorInfs());
173 mergeFnAttrValue(
"no-nans-fp-math", FPFeatures.getNoHonorNaNs());
174 mergeFnAttrValue(
"no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
175 mergeFnAttrValue(
"unsafe-fp-math", FPFeatures.getAllowFPReassociate() &&
176 FPFeatures.getAllowReciprocal() &&
177 FPFeatures.getAllowApproxFunc() &&
178 FPFeatures.getNoSignedZero());
182 CGF.CurFPFeatures = OldFPFeatures;
183 CGF.Builder.setDefaultConstrainedExcept(OldExcept);
184 CGF.Builder.setDefaultConstrainedRounding(OldRounding);
219 switch (
type->getTypeClass()) {
220 #define TYPE(name, parent)
221 #define ABSTRACT_TYPE(name, parent)
222 #define NON_CANONICAL_TYPE(name, parent) case Type::name:
223 #define DEPENDENT_TYPE(name, parent) case Type::name:
224 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
225 #include "clang/AST/TypeNodes.inc"
226 llvm_unreachable(
"non-canonical or dependent type in IR-generation");
229 case Type::DeducedTemplateSpecialization:
230 llvm_unreachable(
"undeduced type in IR-generation");
235 case Type::BlockPointer:
236 case Type::LValueReference:
237 case Type::RValueReference:
238 case Type::MemberPointer:
240 case Type::ExtVector:
241 case Type::ConstantMatrix:
242 case Type::FunctionProto:
243 case Type::FunctionNoProto:
245 case Type::ObjCObjectPointer:
255 case Type::ConstantArray:
256 case Type::IncompleteArray:
257 case Type::VariableArray:
259 case Type::ObjCObject:
260 case Type::ObjCInterface:
265 type = cast<AtomicType>(
type)->getValueType();
268 llvm_unreachable(
"unknown type kind!");
275 llvm::BasicBlock *CurBB =
Builder.GetInsertBlock();
278 assert(!CurBB->getTerminator() &&
"Unexpected terminated block.");
288 return llvm::DebugLoc();
295 llvm::BranchInst *BI =
297 if (BI && BI->isUnconditional() &&
301 llvm::DebugLoc Loc = BI->getDebugLoc();
302 Builder.SetInsertPoint(BI->getParent());
303 BI->eraseFromParent();
315 return llvm::DebugLoc();
320 if (!BB->use_empty())
321 return CGF.
CurFn->getBasicBlockList().push_back(BB);
326 assert(BreakContinueStack.empty() &&
327 "mismatched push/pop in break/continue stack!");
329 bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
330 && NumSimpleReturnExprs == NumReturnExprs
345 if (OnlySimpleReturnStmts)
346 DI->EmitLocation(
Builder, LastStopPoint);
348 DI->EmitLocation(
Builder, EndLoc);
356 bool HasOnlyLifetimeMarkers =
358 bool EmitRetDbgLoc = !HasCleanups || HasOnlyLifetimeMarkers;
364 if (OnlySimpleReturnStmts)
365 DI->EmitLocation(
Builder, EndLoc);
380 CurFn->addFnAttr(
"instrument-function-exit",
"__cyg_profile_func_exit");
382 CurFn->addFnAttr(
"instrument-function-exit-inlined",
383 "__cyg_profile_func_exit");
397 "did not remove all scopes from cleanup stack!");
401 if (IndirectBranch) {
408 if (!EscapedLocals.empty()) {
412 EscapeArgs.resize(EscapedLocals.size());
413 for (
auto &Pair : EscapedLocals)
414 EscapeArgs[Pair.second] = Pair.first;
415 llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration(
423 Ptr->eraseFromParent();
427 if (PostAllocaInsertPt) {
428 llvm::Instruction *PostPtr = PostAllocaInsertPt;
429 PostAllocaInsertPt =
nullptr;
430 PostPtr->eraseFromParent();
435 if (IndirectBranch) {
436 llvm::PHINode *PN = cast<llvm::PHINode>(IndirectBranch->getAddress());
437 if (PN->getNumIncomingValues() == 0) {
438 PN->replaceAllUsesWith(llvm::UndefValue::get(PN->getType()));
439 PN->eraseFromParent();
448 for (
const auto &FuncletAndParent : TerminateFunclets)
454 for (
const auto &R : DeferredReplacements) {
455 if (llvm::Value *Old = R.first) {
456 Old->replaceAllUsesWith(R.second);
457 cast<llvm::Instruction>(Old)->eraseFromParent();
460 DeferredReplacements.clear();
469 llvm::DominatorTree DT(*
CurFn);
470 llvm::PromoteMemToReg(
476 for (llvm::Argument &A :
CurFn->args())
477 if (
auto *VT = dyn_cast<llvm::VectorType>(A.getType()))
480 VT->getPrimitiveSizeInBits().getKnownMinSize());
483 if (
auto *VT = dyn_cast<llvm::VectorType>(
CurFn->getReturnType()))
486 VT->getPrimitiveSizeInBits().getKnownMinSize());
498 CurFn->addFnAttr(
"min-legal-vector-width", llvm::utostr(LargestVectorWidth));
504 CurFn->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(
515 if (RetAlloca && RetAlloca->use_empty()) {
516 RetAlloca->eraseFromParent();
564 llvm::Value *EncodedAddr) {
567 auto *FuncAsInt =
Builder.CreatePtrToInt(F,
IntPtrTy,
"func_addr.int");
568 auto *GOTAsInt =
Builder.CreateAdd(PCRelAsInt, FuncAsInt,
"global_addr.int");
576 void CodeGenFunction::EmitKernelMetadata(
const FunctionDecl *FD,
577 llvm::Function *Fn) {
578 if (!FD->
hasAttr<OpenCLKernelAttr>() && !FD->
hasAttr<CUDAGlobalAttr>())
588 if (
const VecTypeHintAttr *A = FD->
getAttr<VecTypeHintAttr>()) {
589 QualType HintQTy = A->getTypeHint();
591 bool IsSignedInteger =
594 llvm::Metadata *AttrMDArgs[] = {
595 llvm::ConstantAsMetadata::get(llvm::UndefValue::get(
597 llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
598 llvm::IntegerType::get(Context, 32),
600 Fn->setMetadata(
"vec_type_hint", llvm::MDNode::get(Context, AttrMDArgs));
603 if (
const WorkGroupSizeHintAttr *A = FD->
getAttr<WorkGroupSizeHintAttr>()) {
604 llvm::Metadata *AttrMDArgs[] = {
605 llvm::ConstantAsMetadata::get(
Builder.getInt32(A->getXDim())),
606 llvm::ConstantAsMetadata::get(
Builder.getInt32(A->getYDim())),
607 llvm::ConstantAsMetadata::get(
Builder.getInt32(A->getZDim()))};
608 Fn->setMetadata(
"work_group_size_hint", llvm::MDNode::get(Context, AttrMDArgs));
611 if (
const ReqdWorkGroupSizeAttr *A = FD->
getAttr<ReqdWorkGroupSizeAttr>()) {
612 llvm::Metadata *AttrMDArgs[] = {
613 llvm::ConstantAsMetadata::get(
Builder.getInt32(A->getXDim())),
614 llvm::ConstantAsMetadata::get(
Builder.getInt32(A->getYDim())),
615 llvm::ConstantAsMetadata::get(
Builder.getInt32(A->getZDim()))};
616 Fn->setMetadata(
"reqd_work_group_size", llvm::MDNode::get(Context, AttrMDArgs));
619 if (
const OpenCLIntelReqdSubGroupSizeAttr *A =
620 FD->
getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
621 llvm::Metadata *AttrMDArgs[] = {
622 llvm::ConstantAsMetadata::get(
Builder.getInt32(A->getSubGroupSize()))};
623 Fn->setMetadata(
"intel_reqd_sub_group_size",
624 llvm::MDNode::get(Context, AttrMDArgs));
630 const Stmt *Body =
nullptr;
631 if (
auto *FD = dyn_cast_or_null<FunctionDecl>(F))
633 else if (
auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(F))
634 Body = OMD->getBody();
636 if (
auto *CS = dyn_cast_or_null<CompoundStmt>(Body)) {
637 auto LastStmt = CS->body_rbegin();
638 if (LastStmt != CS->body_rend())
639 return isa<ReturnStmt>(*LastStmt);
646 Fn->addFnAttr(
"sanitize_thread_no_checking_at_run_time");
647 Fn->removeFnAttr(llvm::Attribute::SanitizeThread);
652 bool CodeGenFunction::requiresReturnValueCheck()
const {
653 return requiresReturnValueNullabilityCheck() ||
659 auto *MD = dyn_cast_or_null<CXXMethodDecl>(D);
660 if (!MD || !MD->getDeclName().getAsIdentifierInfo() ||
661 !MD->getDeclName().getAsIdentifierInfo()->isStr(
"allocate") ||
662 (MD->getNumParams() != 1 && MD->getNumParams() != 2))
665 if (MD->parameters()[0]->getType().getCanonicalType() != Ctx.
getSizeType())
668 if (MD->getNumParams() == 2) {
669 auto *PT = MD->parameters()[1]->getType()->getAs<
PointerType>();
670 if (!PT || !PT->isVoidPointerType() ||
671 !PT->getPointeeType().isConstQualified())
681 if (
const auto *MD = dyn_cast<CXXMethodDecl>(FD))
694 "Do not use a CodeGenFunction object for more than one function");
698 DidCallStackSave =
false;
700 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
707 assert(
CurFn->isDeclaration() &&
"Function already has body?");
712 #define SANITIZER(NAME, ID) \
713 if (SanOpts.empty()) \
715 if (SanOpts.has(SanitizerKind::ID)) \
716 if (CGM.isInNoSanitizeList(SanitizerKind::ID, Fn, Loc)) \
717 SanOpts.set(SanitizerKind::ID, false);
719 #include "clang/Basic/Sanitizers.def"
725 bool NoSanitizeCoverage =
false;
731 if (mask & SanitizerKind::Address)
732 SanOpts.
set(SanitizerKind::KernelAddress,
false);
733 if (mask & SanitizerKind::KernelAddress)
735 if (mask & SanitizerKind::HWAddress)
736 SanOpts.
set(SanitizerKind::KernelHWAddress,
false);
737 if (mask & SanitizerKind::KernelHWAddress)
741 if (
Attr->hasCoverage())
742 NoSanitizeCoverage =
true;
746 Fn->addFnAttr(llvm::Attribute::NoSanitizeBounds);
749 Fn->addFnAttr(llvm::Attribute::NoSanitizeCoverage);
753 CurFn->addFnAttr(llvm::Attribute::DisableSanitizerInstrumentation);
756 if (
SanOpts.
hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress))
757 Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
759 SanitizerKind::KernelHWAddress))
760 Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
762 Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
764 Fn->addFnAttr(llvm::Attribute::SanitizeThread);
765 if (
SanOpts.
hasOneOf(SanitizerKind::Memory | SanitizerKind::KernelMemory))
766 Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
769 Fn->addFnAttr(llvm::Attribute::SafeStack);
770 if (
SanOpts.
has(SanitizerKind::ShadowCallStack))
771 Fn->addFnAttr(llvm::Attribute::ShadowCallStack);
774 if (
SanOpts.
hasOneOf(SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
775 Fn->addFnAttr(llvm::Attribute::OptForFuzzing);
780 if (
const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(D)) {
781 IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(0);
784 (OMD->getSelector().isUnarySelector() && II->
isStr(
".cxx_destruct"))) {
793 if (D &&
SanOpts.
has(SanitizerKind::CFIUnrelatedCast)) {
807 bool AlwaysXRayAttr =
false;
808 if (
const auto *XRayAttr = D ? D->
getAttr<XRayInstrumentAttr>() :
nullptr) {
814 Fn->addFnAttr(
"function-instrument",
"xray-always");
815 AlwaysXRayAttr =
true;
817 if (XRayAttr->neverXRayInstrument())
818 Fn->addFnAttr(
"function-instrument",
"xray-never");
819 if (
const auto *LogArgs = D->
getAttr<XRayLogArgsAttr>())
821 Fn->addFnAttr(
"xray-log-args",
822 llvm::utostr(LogArgs->getArgumentCount()));
827 "xray-instruction-threshold",
833 Fn->addFnAttr(
"xray-ignore-loops");
837 Fn->addFnAttr(
"xray-skip-exit");
841 Fn->addFnAttr(
"xray-skip-entry");
844 if (FuncGroups > 1) {
845 auto FuncName = llvm::makeArrayRef<uint8_t>(
846 CurFn->getName().bytes_begin(),
CurFn->getName().bytes_end());
847 auto Group = crc32(FuncName) % FuncGroups;
850 Fn->addFnAttr(
"function-instrument",
"xray-never");
856 Fn->addFnAttr(llvm::Attribute::NoProfile);
859 if (
const auto *
Attr =
860 D ? D->
getAttr<PatchableFunctionEntryAttr>() :
nullptr) {
861 Count =
Attr->getCount();
867 if (Count &&
Offset <= Count) {
868 Fn->addFnAttr(
"patchable-function-entry", std::to_string(Count -
Offset));
870 Fn->addFnAttr(
"patchable-function-prefix", std::to_string(
Offset));
877 getContext().getTargetInfo().getTriple().isX86())
878 Fn->addFnAttr(
"patchable-function",
"prologue-short-redirect");
882 Fn->addFnAttr(
"no-jump-tables",
"true");
886 Fn->addFnAttr(
"no-inline-line-tables");
890 Fn->addFnAttr(
"profile-sample-accurate");
893 Fn->addFnAttr(
"use-sample-profile");
895 if (D && D->
hasAttr<CFICanonicalJumpTableAttr>())
896 Fn->addFnAttr(
"cfi-canonical-jump-table");
898 if (D && D->
hasAttr<NoProfileFunctionAttr>())
899 Fn->addFnAttr(llvm::Attribute::NoProfile);
904 EmitKernelMetadata(FD, Fn);
915 llvm::Constant *FTRTTIConst =
917 llvm::GlobalVariable *FTRTTIProxy =
919 llvm::LLVMContext &Ctx = Fn->getContext();
920 llvm::MDBuilder MDB(Ctx);
921 Fn->setMetadata(llvm::LLVMContext::MD_func_sanitize,
922 MDB.createRTTIPointerPrologue(PrologueSig, FTRTTIProxy));
929 if (
SanOpts.
has(SanitizerKind::NullabilityReturn)) {
932 if (!(
SanOpts.
has(SanitizerKind::ReturnsNonnullAttribute) &&
934 RetValNullabilityPrecondition =
953 Fn->addFnAttr(llvm::Attribute::NoRecurse);
956 llvm::fp::ExceptionBehavior FPExceptionBehavior =
958 Builder.setDefaultConstrainedRounding(RM);
959 Builder.setDefaultConstrainedExcept(FPExceptionBehavior);
961 (!FD && (FPExceptionBehavior != llvm::fp::ebIgnore ||
962 RM != llvm::RoundingMode::NearestTiesToEven))) {
963 Builder.setIsFPConstrained(
true);
964 Fn->addFnAttr(llvm::Attribute::StrictFP);
971 Fn->addFnAttr(
"stackrealign");
975 Fn->removeFnAttr(
"zero-call-used-regs");
982 llvm::Value *Undef = llvm::UndefValue::get(
Int32Ty);
987 Builder.SetInsertPoint(EntryBB);
991 if (requiresReturnValueCheck()) {
1002 DI->emitFunctionStart(GD, Loc, StartLoc,
1003 DI->getFunctionType(FD, RetTy, Args),
CurFn,
1009 CurFn->addFnAttr(
"instrument-function-entry",
"__cyg_profile_func_enter");
1011 CurFn->addFnAttr(
"instrument-function-entry-inlined",
1012 "__cyg_profile_func_enter");
1014 CurFn->addFnAttr(
"instrument-function-entry-inlined",
1015 "__cyg_profile_func_enter_bare");
1027 Fn->addFnAttr(
"fentry-call",
"true");
1029 Fn->addFnAttr(
"instrument-function-entry-inlined",
1035 <<
"-mnop-mcount" <<
"-mfentry";
1036 Fn->addFnAttr(
"mnop-mcount");
1042 <<
"-mrecord-mcount" <<
"-mfentry";
1043 Fn->addFnAttr(
"mrecord-mcount");
1049 if (
getContext().getTargetInfo().getTriple().getArch() !=
1050 llvm::Triple::systemz)
1052 <<
"-mpacked-stack";
1053 Fn->addFnAttr(
"packed-stack");
1058 Fn->addFnAttr(
"warn-stack-size",
1071 auto AI =
CurFn->arg_begin();
1087 llvm::Function::arg_iterator EI =
CurFn->arg_end();
1092 cast<llvm::GetElementPtrInst>(Addr)->getResultElementType();
1119 if (isa_and_nonnull<CXXMethodDecl>(D) &&
1120 cast<CXXMethodDecl>(D)->isInstance()) {
1147 if (FD->hasCapturedVLAType()) {
1150 auto VAT = FD->getCapturedVLAType();
1151 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1158 CXXThisValue = CXXABIThisValue;
1162 if (CXXABIThisValue) {
1164 SkippedChecks.
set(SanitizerKind::ObjectSize,
true);
1176 Loc, CXXABIThisValue, ThisTy, CXXABIThisAlignment, SkippedChecks);
1183 if (!FD || !FD->
hasAttr<NakedAttr>()) {
1184 for (
const VarDecl *VD : Args) {
1189 if (
const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(VD))
1190 Ty = PVD->getOriginalType();
1200 DI->EmitLocation(
Builder, StartLoc);
1205 LargestVectorWidth = VecWidth->getVectorWidth();
1210 if (
const CompoundStmt *S = dyn_cast<CompoundStmt>(Body))
1218 CurFn->addFnAttr(llvm::Attribute::MustProgress);
1227 llvm::BasicBlock *SkipCountBB =
nullptr;
1249 if (F->isInterposable())
return;
1251 for (llvm::BasicBlock &BB : *F)
1252 for (llvm::Instruction &I : BB)
1256 F->setDoesNotThrow();
1276 bool PassedParams =
true;
1278 if (
auto Inherited = CD->getInheritedConstructor())
1284 Args.push_back(Param);
1285 if (!Param->hasAttr<PassObjectSizeAttr>())
1289 getContext(), Param->getDeclContext(), Param->getLocation(),
1291 SizeArguments[Param] = Implicit;
1292 Args.push_back(Implicit);
1296 if (MD && (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD)))
1304 assert(Fn &&
"generating code for null Function");
1315 std::string FDInlineName = (Fn->getName() +
".inline").str();
1316 llvm::Module *M = Fn->getParent();
1317 llvm::Function *Clone = M->getFunction(FDInlineName);
1319 Clone = llvm::Function::Create(Fn->getFunctionType(),
1321 Fn->getAddressSpace(), FDInlineName, M);
1322 Clone->addFnAttr(llvm::Attribute::AlwaysInline);
1334 if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) {
1335 std::string FDInlineName = (Fn->getName() +
".inline").str();
1336 llvm::Module *M = Fn->getParent();
1337 if (llvm::Function *Clone = M->getFunction(FDInlineName)) {
1338 Clone->replaceAllUsesWith(Fn);
1339 Clone->eraseFromParent();
1347 if (FD->
hasAttr<NoDebugAttr>()) {
1350 Fn->setSubprogram(
nullptr);
1352 DebugInfo =
nullptr;
1359 BodyRange = Body->getSourceRange();
1362 CurEHLocation = BodyRange.
getEnd();
1374 if (SpecDecl->hasBody(SpecDecl))
1375 Loc = SpecDecl->getLocation();
1381 if (isa<CoroutineBodyStmt>(Body))
1382 ShouldEmitLifetimeMarkers =
true;
1386 if (ShouldEmitLifetimeMarkers)
1394 if (Body && isa_and_nonnull<CoroutineBodyStmt>(Body))
1399 if (isa<CXXDestructorDecl>(FD))
1401 else if (isa<CXXConstructorDecl>(FD))
1405 FD->
hasAttr<CUDAGlobalAttr>())
1407 else if (isa<CXXMethodDecl>(FD) &&
1408 cast<CXXMethodDecl>(FD)->isLambdaStaticInvoker()) {
1412 }
else if (FD->
isDefaulted() && isa<CXXMethodDecl>(FD) &&
1413 (cast<CXXMethodDecl>(FD)->isCopyAssignmentOperator() ||
1414 cast<CXXMethodDecl>(FD)->isMoveAssignmentOperator())) {
1421 llvm_unreachable(
"no definition for emitted function");
1431 bool ShouldEmitUnreachable =
1436 llvm::Value *IsFalse =
Builder.getFalse();
1437 EmitCheck(std::make_pair(IsFalse, SanitizerKind::Return),
1438 SanitizerHandler::MissingReturn,
1440 }
else if (ShouldEmitUnreachable) {
1444 if (
SanOpts.
has(SanitizerKind::Return) || ShouldEmitUnreachable) {
1446 Builder.ClearInsertionPoint();
1455 if (!
CurFn->doesNotThrow())
1464 if (!S)
return false;
1471 if (isa<LabelStmt>(S))
1476 if (isa<SwitchCase>(S) && !IgnoreCaseStmts)
1480 if (isa<SwitchStmt>(S))
1481 IgnoreCaseStmts =
true;
1484 for (
const Stmt *SubStmt : S->children())
1496 if (!S)
return false;
1500 if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) || isa<DoStmt>(S) ||
1504 if (isa<BreakStmt>(S))
1508 for (
const Stmt *SubStmt : S->children())
1516 if (!S)
return false;
1522 if (isa<IfStmt>(S) || isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||
1523 isa<DoStmt>(S) || isa<ForStmt>(S) || isa<CompoundStmt>(S) ||
1524 isa<CXXForRangeStmt>(S) || isa<CXXTryStmt>(S) ||
1525 isa<ObjCForCollectionStmt>(S) || isa<ObjCAtTryStmt>(S))
1528 if (isa<DeclStmt>(S))
1531 for (
const Stmt *SubStmt : S->children())
1548 ResultBool = ResultInt.getBoolValue();
1577 if (
const UnaryOperator *UnOp = dyn_cast<UnaryOperator>(C->IgnoreParens()))
1578 if (UnOp->getOpcode() == UO_LNot)
1579 C = UnOp->getSubExpr();
1581 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(C->IgnoreParens());
1591 llvm::BasicBlock *FalseBlock,
uint64_t TrueCount ,
1598 llvm::BasicBlock *ThenBlock =
nullptr;
1599 llvm::BasicBlock *ElseBlock =
nullptr;
1600 llvm::BasicBlock *NextBlock =
nullptr;
1617 if (LOp == BO_LAnd) {
1618 ThenBlock = CounterIncrBlock;
1619 ElseBlock = FalseBlock;
1620 NextBlock = TrueBlock;
1635 else if (LOp == BO_LOr) {
1636 ThenBlock = TrueBlock;
1637 ElseBlock = CounterIncrBlock;
1638 NextBlock = FalseBlock;
1640 llvm_unreachable(
"Expected Opcode must be that of a Logical Operator");
1661 llvm::BasicBlock *TrueBlock,
1662 llvm::BasicBlock *FalseBlock,
1667 if (
const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Cond)) {
1670 if (CondBOp->getOpcode() == BO_LAnd) {
1673 bool ConstantBool =
false;
1679 FalseBlock, TrueCount, LH);
1688 FalseBlock, TrueCount, LH, CondBOp);
1715 FalseBlock, TrueCount, LH);
1721 if (CondBOp->getOpcode() == BO_LOr) {
1724 bool ConstantBool =
false;
1730 FalseBlock, TrueCount, LH);
1739 FalseBlock, TrueCount, LH, CondBOp);
1750 uint64_t RHSCount = TrueCount - LHSCount;
1777 if (
const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Cond)) {
1779 if (CondUOp->getOpcode() == UO_LNot) {
1810 LHSScaledTrueCount = TrueCount * LHSRatio;
1819 LHSScaledTrueCount, LH);
1826 TrueCount - LHSScaledTrueCount, LH);
1832 if (
const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Cond)) {
1849 llvm::MDNode *Weights =
nullptr;
1850 llvm::MDNode *Unpredictable =
nullptr;
1857 auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
1858 if (FD && FD->
getBuiltinID() == Builtin::BI__builtin_unpredictable) {
1860 Unpredictable = MDHelper.createUnpredictable();
1866 llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(CondV, LH);
1867 if (CondV != NewCondV)
1872 Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
1875 Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
1893 llvm::Value *sizeInChars) {
1897 llvm::Value *baseSizeInChars
1902 llvm::Value *end =
Builder.CreateInBoundsGEP(
1905 llvm::BasicBlock *originBB = CGF.
Builder.GetInsertBlock();
1913 llvm::PHINode *cur =
Builder.CreatePHI(begin.
getType(), 2,
"vla.cur");
1914 cur->addIncoming(begin.
getPointer(), originBB);
1925 Builder.CreateInBoundsGEP(CGF.
Int8Ty, cur, baseSizeInChars,
"vla.next");
1928 llvm::Value *done =
Builder.CreateICmpEQ(next, end,
"vla-init.isdone");
1929 Builder.CreateCondBr(done, contBB, loopBB);
1930 cur->addIncoming(next, loopBB);
1940 if (cast<CXXRecordDecl>(RT->getDecl())->isEmpty())
1952 llvm::Value *SizeVal;
1959 dyn_cast_or_null<VariableArrayType>(
1962 SizeVal = VlaSize.NumElts;
1964 if (!eltSize.
isOne())
1985 llvm::GlobalVariable *NullVariable =
1986 new llvm::GlobalVariable(
CGM.
getModule(), NullConstant->getType(),
1988 llvm::GlobalVariable::PrivateLinkage,
1989 NullConstant, Twine());
1991 NullVariable->setAlignment(NullAlign.
getAsAlign());
1993 Builder.getInt8Ty(), NullAlign);
2010 if (!IndirectBranch)
2016 IndirectBranch->addDestination(BB);
2017 return llvm::BlockAddress::get(
CurFn, BB);
2022 if (IndirectBranch)
return IndirectBranch->getParent();
2027 llvm::Value *DestVal = TmpBuilder.CreatePHI(
Int8PtrTy, 0,
2028 "indirect.goto.dest");
2031 IndirectBranch = TmpBuilder.CreateIndirectBr(DestVal);
2032 return IndirectBranch->getParent();
2044 llvm::Value *numVLAElements =
nullptr;
2045 if (isa<VariableArrayType>(
arrayType)) {
2056 baseType = elementType;
2057 return numVLAElements;
2059 }
while (isa<VariableArrayType>(
arrayType));
2071 llvm::ConstantInt *zero =
Builder.getInt32(0);
2072 gepIndices.push_back(zero);
2077 llvm::ArrayType *llvmArrayType =
2079 while (llvmArrayType) {
2080 assert(isa<ConstantArrayType>(
arrayType));
2081 assert(cast<ConstantArrayType>(
arrayType)->getSize().getZExtValue()
2082 == llvmArrayType->getNumElements());
2084 gepIndices.push_back(zero);
2085 countFromCLAs *= llvmArrayType->getNumElements();
2089 dyn_cast<llvm::ArrayType>(llvmArrayType->getElementType());
2092 "LLVM and Clang types are out-of-synch");
2101 cast<ConstantArrayType>(
arrayType)->getSize().getZExtValue();
2118 llvm::Value *numElements
2119 = llvm::ConstantInt::get(
SizeTy, countFromCLAs);
2123 numElements =
Builder.CreateNUWMul(numVLAElements, numElements);
2130 assert(vla &&
"type was not a variable array type!");
2137 llvm::Value *numElements =
nullptr;
2141 elementType =
type->getElementType();
2142 llvm::Value *vlaSize = VLASizeMap[
type->getSizeExpr()];
2143 assert(vlaSize &&
"no size for VLA!");
2144 assert(vlaSize->getType() ==
SizeTy);
2147 numElements = vlaSize;
2151 numElements =
Builder.CreateNUWMul(numElements, vlaSize);
2153 }
while ((
type =
getContext().getAsVariableArrayType(elementType)));
2155 return { numElements, elementType };
2161 assert(vla &&
"type was not a variable array type!");
2167 llvm::Value *VlaSize = VLASizeMap[Vla->
getSizeExpr()];
2168 assert(VlaSize &&
"no size for VLA!");
2169 assert(VlaSize->getType() ==
SizeTy);
2174 assert(
type->isVariablyModifiedType() &&
2175 "Must pass variably modified type to EmitVLASizes!");
2182 assert(
type->isVariablyModifiedType());
2184 const Type *ty =
type.getTypePtr();
2187 #define TYPE(Class, Base)
2188 #define ABSTRACT_TYPE(Class, Base)
2189 #define NON_CANONICAL_TYPE(Class, Base)
2190 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
2191 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2192 #include "clang/AST/TypeNodes.inc"
2193 llvm_unreachable(
"unexpected dependent type!");
2199 case Type::ExtVector:
2200 case Type::ConstantMatrix:
2203 case Type::Elaborated:
2205 case Type::TemplateSpecialization:
2206 case Type::ObjCTypeParam:
2207 case Type::ObjCObject:
2208 case Type::ObjCInterface:
2209 case Type::ObjCObjectPointer:
2211 llvm_unreachable(
"type class is never variably-modified!");
2213 case Type::Adjusted:
2214 type = cast<AdjustedType>(ty)->getAdjustedType();
2218 type = cast<DecayedType>(ty)->getPointeeType();
2222 type = cast<PointerType>(ty)->getPointeeType();
2225 case Type::BlockPointer:
2226 type = cast<BlockPointerType>(ty)->getPointeeType();
2229 case Type::LValueReference:
2230 case Type::RValueReference:
2231 type = cast<ReferenceType>(ty)->getPointeeType();
2234 case Type::MemberPointer:
2235 type = cast<MemberPointerType>(ty)->getPointeeType();
2238 case Type::ConstantArray:
2239 case Type::IncompleteArray:
2241 type = cast<ArrayType>(ty)->getElementType();
2244 case Type::VariableArray: {
2253 llvm::Value *&entry = VLASizeMap[sizeExpr];
2263 llvm::Value *
Zero = llvm::Constant::getNullValue(size->getType());
2265 llvm::Value *CheckCondition =
2269 llvm::Constant *StaticArgs[] = {
2272 EmitCheck(std::make_pair(CheckCondition, SanitizerKind::VLABound),
2273 SanitizerHandler::VLABoundNotPositive, StaticArgs, size);
2286 case Type::FunctionProto:
2287 case Type::FunctionNoProto:
2288 type = cast<FunctionType>(ty)->getReturnType();
2293 case Type::UnaryTransform:
2294 case Type::Attributed:
2295 case Type::BTFTagAttributed:
2296 case Type::SubstTemplateTypeParm:
2297 case Type::MacroQualified:
2303 case Type::Decltype:
2305 case Type::DeducedTemplateSpecialization:
2309 case Type::TypeOfExpr:
2315 type = cast<AtomicType>(ty)->getValueType();
2319 type = cast<PipeType>(ty)->getElementType();
2322 }
while (
type->isVariablyModifiedType());
2326 if (
getContext().getBuiltinVaListType()->isArrayType())
2337 assert(Init.hasValue() &&
"Invalid DeclRefExpr initializer!");
2340 Dbg->EmitGlobalVariable(E->
getDecl(), Init);
2355 llvm::Instruction *inst =
new llvm::BitCastInst(value, value->getType(),
"",
2359 protection.Inst = inst;
2364 if (!protection.Inst)
return;
2367 protection.Inst->eraseFromParent();
2373 llvm::Value *Alignment,
2374 llvm::Value *OffsetValue) {
2375 if (Alignment->getType() !=
IntPtrTy)
2378 if (OffsetValue && OffsetValue->getType() !=
IntPtrTy)
2381 llvm::Value *TheCheck =
nullptr;
2383 llvm::Value *PtrIntValue =
2387 bool IsOffsetZero =
false;
2388 if (
const auto *CI = dyn_cast<llvm::ConstantInt>(OffsetValue))
2389 IsOffsetZero = CI->isZero();
2392 PtrIntValue =
Builder.CreateSub(PtrIntValue, OffsetValue,
"offsetptr");
2395 llvm::Value *
Zero = llvm::ConstantInt::get(
IntPtrTy, 0);
2398 llvm::Value *MaskedPtr =
Builder.CreateAnd(PtrIntValue, Mask,
"maskedptr");
2399 TheCheck =
Builder.CreateICmpEQ(MaskedPtr,
Zero,
"maskcond");
2401 llvm::Instruction *Assumption =
Builder.CreateAlignmentAssumption(
2407 OffsetValue, TheCheck, Assumption);
2413 llvm::Value *Alignment,
2414 llvm::Value *OffsetValue) {
2415 if (
auto *CE = dyn_cast<CastExpr>(E))
2416 E = CE->getSubExprAsWritten();
2425 llvm::Value *AnnotatedVal,
2426 StringRef AnnotationStr,
2428 const AnnotateAttr *
Attr) {
2437 return Builder.CreateCall(AnnotationFn, Args);
2441 assert(D->
hasAttr<AnnotateAttr>() &&
"no annotate attribute");
2452 assert(D->
hasAttr<AnnotateAttr>() &&
"no annotate attribute");
2454 llvm::Type *VTy =
V->getType();
2455 auto *PTy = dyn_cast<llvm::PointerType>(VTy);
2456 unsigned AS = PTy ? PTy->getAddressSpace() : 0;
2457 llvm::PointerType *IntrinTy =
2458 llvm::PointerType::getWithSamePointeeType(
CGM.
Int8PtrTy, AS);
2466 if (VTy != IntrinTy)
2484 CGF->IsSanitizerScope =
false;
2488 const llvm::Twine &Name,
2489 llvm::BasicBlock *BB,
2490 llvm::BasicBlock::iterator InsertPt)
const {
2497 llvm::Instruction *I,
const llvm::Twine &Name, llvm::BasicBlock *BB,
2498 llvm::BasicBlock::iterator InsertPt)
const {
2499 llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
2501 CGF->InsertHelper(I, Name, BB, InsertPt);
2530 llvm::StringMap<bool> CallerFeatureMap;
2535 FeatureList, CallerFeatureMap)) {
2541 TargetDecl->
hasAttr<TargetAttr>()) {
2544 const TargetAttr *TD = TargetDecl->
getAttr<TargetAttr>();
2549 llvm::StringMap<bool> CalleeFeatureMap;
2553 if (F[0] ==
'+' && CalleeFeatureMap.lookup(F.substr(1)))
2554 ReqFeatures.push_back(StringRef(F).substr(1));
2557 for (
const auto &F : CalleeFeatureMap) {
2560 ReqFeatures.push_back(F.getKey());
2562 if (!llvm::all_of(ReqFeatures, [&](StringRef Feature) {
2563 if (!CallerFeatureMap.lookup(Feature)) {
2564 MissingFeature = Feature.str();
2578 llvm::IRBuilder<> IRB(
Builder.GetInsertBlock(),
Builder.GetInsertPoint());
2579 IRB.SetCurrentDebugLocation(
Builder.getCurrentDebugLocation());
2584 CodeGenFunction::FormResolverCondition(
const MultiVersionResolverOption &RO) {
2587 if (!RO.Conditions.Architecture.empty())
2588 Condition = EmitX86CpuIs(RO.Conditions.Architecture);
2590 if (!RO.Conditions.Features.empty()) {
2591 llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Conditions.Features);
2599 llvm::Function *Resolver,
2601 llvm::Function *FuncToReturn,
2602 bool SupportsIFunc) {
2603 if (SupportsIFunc) {
2604 Builder.CreateRet(FuncToReturn);
2609 llvm::make_pointer_range(Resolver->args()));
2611 llvm::CallInst *Result =
Builder.CreateCall(FuncToReturn, Args);
2612 Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
2614 if (Resolver->getReturnType()->isVoidTy())
2622 assert(
getContext().getTargetInfo().getTriple().isX86() &&
2623 "Only implemented for x86 targets");
2629 Builder.SetInsertPoint(CurBlock);
2633 Builder.SetInsertPoint(CurBlock);
2634 llvm::Value *
Condition = FormResolverCondition(RO);
2638 assert(&RO == Options.end() - 1 &&
2639 "Default or Generic case must be last");
2645 llvm::BasicBlock *RetBlock =
createBasicBlock(
"resolver_return", Resolver);
2654 Builder.SetInsertPoint(CurBlock);
2655 llvm::CallInst *TrapCall =
EmitTrapCall(llvm::Intrinsic::trap);
2656 TrapCall->setDoesNotReturn();
2657 TrapCall->setDoesNotThrow();
2659 Builder.ClearInsertionPoint();
2671 llvm::Value *OffsetValue, llvm::Value *TheCheck,
2672 llvm::Instruction *Assumption) {
2673 assert(Assumption && isa<llvm::CallInst>(Assumption) &&
2674 cast<llvm::CallInst>(Assumption)->getCalledOperand() ==
2675 llvm::Intrinsic::getDeclaration(
2676 Builder.GetInsertBlock()->getParent()->getParent(),
2677 llvm::Intrinsic::assume) &&
2678 "Assumption should be a call to llvm.assume().");
2679 assert(&(
Builder.GetInsertBlock()->back()) == Assumption &&
2680 "Assumption should be the last instruction of the basic block, "
2681 "since the basic block is still being generated.");
2693 Assumption->removeFromParent();
2699 OffsetValue =
Builder.getInt1(
false);
2707 EmitCheck({std::make_pair(TheCheck, SanitizerKind::Alignment)},
2708 SanitizerHandler::AlignmentAssumption, StaticData, DynamicData);
2719 return DI->SourceLocToDebugLoc(Location);
2721 return llvm::DebugLoc();
2725 CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
2736 llvm::Type *CondTy = Cond->getType();
2737 assert(CondTy->isIntegerTy(1) &&
"expecting condition to be a boolean");
2738 llvm::Function *FnExpect =
2740 llvm::Value *ExpectedValueOfCond =
2742 return Builder.CreateCall(FnExpect, {Cond, ExpectedValueOfCond},
2743 Cond->getName() +
".expval");
2745 llvm_unreachable(
"Unknown Likelihood");
2749 unsigned NumElementsDst,
2750 const llvm::Twine &Name) {
2751 auto *SrcTy = cast<llvm::FixedVectorType>(SrcVec->getType());
2752 unsigned NumElementsSrc = SrcTy->getNumElements();
2753 if (NumElementsSrc == NumElementsDst)
2756 std::vector<int> ShuffleMask(NumElementsDst, -1);
2757 for (
unsigned MaskIdx = 0;
2758 MaskIdx < std::min<>(NumElementsDst, NumElementsSrc); ++MaskIdx)
2759 ShuffleMask[MaskIdx] = MaskIdx;
2761 return Builder.CreateShuffleVector(SrcVec, ShuffleMask, Name);