18#include "llvm/IR/Intrinsics.h"
19#include "llvm/IR/MDBuilder.h"
20#include "llvm/Support/CommandLine.h"
21#include "llvm/Support/Endian.h"
22#include "llvm/Support/FileSystem.h"
23#include "llvm/Support/MD5.h"
30static llvm::cl::opt<bool>
32 llvm::cl::desc(
"Enable value profiling"),
33 llvm::cl::Hidden, llvm::cl::init(
false));
38using namespace CodeGen;
40void CodeGenPGO::setFuncName(StringRef Name,
41 llvm::GlobalValue::LinkageTypes
Linkage) {
42 llvm::IndexedInstrProfReader *PGOReader = CGM.
getPGOReader();
43 FuncName = llvm::getPGOFuncName(
45 PGOReader ? PGOReader->getVersion() : llvm::IndexedInstrProf::Version);
49 FuncNameVar = llvm::createPGOFuncNameVar(CGM.
getModule(),
Linkage, FuncName);
52void CodeGenPGO::setFuncName(llvm::Function *Fn) {
53 setFuncName(Fn->getName(), Fn->getLinkage());
55 llvm::createPGOFuncNameMetadata(*Fn, FuncName);
85 static const int NumBitsPerType = 6;
86 static const unsigned NumTypesPerWord =
sizeof(
uint64_t) * 8 / NumBitsPerType;
87 static const unsigned TooBig = 1u << NumBitsPerType;
97 enum HashType :
unsigned char {
138 static_assert(LastHashType <= TooBig,
"Too many types in HashType");
141 : Working(0), Count(0), HashVersion(HashVersion) {}
142 void combine(HashType
Type);
146const int PGOHash::NumBitsPerType;
147const unsigned PGOHash::NumTypesPerWord;
148const unsigned PGOHash::TooBig;
151static PGOHashVersion getPGOHashVersion(llvm::IndexedInstrProfReader *PGOReader,
153 if (PGOReader->getVersion() <= 4)
155 if (PGOReader->getVersion() <= 5)
165 unsigned NextCounter;
169 llvm::DenseMap<const Stmt *, unsigned> &CounterMap;
171 unsigned NextMCDCBitmapIdx;
175 unsigned MCDCMaxCond;
181 MapRegionCounters(
PGOHashVersion HashVersion, uint64_t ProfileVersion,
182 llvm::DenseMap<const Stmt *, unsigned> &CounterMap,
185 : NextCounter(0), Hash(HashVersion), CounterMap(CounterMap),
186 NextMCDCBitmapIdx(0), MCDCState(MCDCState), MCDCMaxCond(MCDCMaxCond),
187 ProfileVersion(ProfileVersion),
Diag(
Diag) {}
191 bool TraverseBlockExpr(
BlockExpr *BE) {
return true; }
194 for (
auto C : zip(
LE->captures(),
LE->capture_inits()))
195 TraverseLambdaCapture(LE, &std::get<0>(
C), std::get<1>(
C));
198 bool TraverseCapturedStmt(
CapturedStmt *CS) {
return true; }
200 bool VisitDecl(
const Decl *D) {
205 case Decl::CXXMethod:
206 case Decl::CXXConstructor:
207 case Decl::CXXDestructor:
208 case Decl::CXXConversion:
209 case Decl::ObjCMethod:
212 CounterMap[D->
getBody()] = NextCounter++;
220 PGOHash::HashType updateCounterMappings(
Stmt *S) {
222 if (
Type != PGOHash::None)
223 CounterMap[S] = NextCounter++;
236 unsigned NumCond = 0;
237 bool SplitNestedLogicalOp =
false;
242 bool dataTraverseStmtPre(
Stmt *S) {
244 if (MCDCMaxCond == 0)
249 if (LogOpStack.empty()) {
251 SplitNestedLogicalOp =
false;
254 if (
const Expr *E = dyn_cast<Expr>(S)) {
255 const BinaryOperator *BinOp = dyn_cast<BinaryOperator>(E->IgnoreParens());
264 SplitNestedLogicalOp = SplitNestedLogicalOp || !NonLogOpStack.empty();
266 LogOpStack.push_back(BinOp);
273 if (!LogOpStack.empty())
274 NonLogOpStack.push_back(S);
282 bool dataTraverseStmtPost(
Stmt *S) {
284 if (MCDCMaxCond == 0)
287 if (
const Expr *E = dyn_cast<Expr>(S)) {
288 const BinaryOperator *BinOp = dyn_cast<BinaryOperator>(E->IgnoreParens());
290 assert(LogOpStack.back() == BinOp);
291 LogOpStack.pop_back();
294 if (LogOpStack.empty()) {
296 if (SplitNestedLogicalOp) {
297 unsigned DiagID =
Diag.getCustomDiagID(
299 "unsupported MC/DC boolean expression; "
300 "contains an operation with a nested boolean expression. "
301 "Expression will not be covered");
302 Diag.Report(S->getBeginLoc(), DiagID);
307 if (NumCond > MCDCMaxCond) {
308 unsigned DiagID =
Diag.getCustomDiagID(
310 "unsupported MC/DC boolean expression; "
311 "number of conditions (%0) exceeds max (%1). "
312 "Expression will not be covered");
313 Diag.Report(S->getBeginLoc(), DiagID) << NumCond << MCDCMaxCond;
320 unsigned SizeInBits = std::max<unsigned>(1L << NumCond,
CHAR_BIT);
321 NextMCDCBitmapIdx += SizeInBits /
CHAR_BIT;
327 if (!LogOpStack.empty())
328 NonLogOpStack.pop_back();
339 if (S->isLogicalOp()) {
340 if (CodeGenFunction::isInstrumentedCondition(S->getLHS()))
343 if (CodeGenFunction::isInstrumentedCondition(S->getRHS())) {
344 if (ProfileVersion >= llvm::IndexedInstrProf::Version7)
345 CounterMap[S->getRHS()] = NextCounter++;
350 return Base::VisitBinaryOperator(S);
355 CounterMap[S->getTrueExpr()] = NextCounter++;
357 CounterMap[S->getFalseExpr()] = NextCounter++;
358 return Base::VisitConditionalOperator(S);
362 bool VisitStmt(
Stmt *S) {
363 auto Type = updateCounterMappings(S);
365 Type = getHashType(Hash.getHashVersion(), S);
366 if (
Type != PGOHash::None)
374 return Base::TraverseIfStmt(
If);
379 for (
Stmt *CS :
If->children()) {
380 if (!CS || NoSingleByteCoverage)
382 if (CS ==
If->getThen())
383 CounterMap[
If->getThen()] = NextCounter++;
384 else if (CS ==
If->getElse())
385 CounterMap[
If->getElse()] = NextCounter++;
391 for (
Stmt *CS :
If->children()) {
394 if (CS ==
If->getThen())
395 Hash.combine(PGOHash::IfThenBranch);
396 else if (CS ==
If->getElse())
397 Hash.combine(PGOHash::IfElseBranch);
400 Hash.combine(PGOHash::EndOfScope);
404 bool TraverseWhileStmt(
WhileStmt *While) {
409 if (!CS || NoSingleByteCoverage)
412 CounterMap[While->
getCond()] = NextCounter++;
413 else if (CS == While->
getBody())
414 CounterMap[While->
getBody()] = NextCounter++;
417 Base::TraverseWhileStmt(While);
419 Hash.combine(PGOHash::EndOfScope);
423 bool TraverseDoStmt(
DoStmt *Do) {
428 if (!CS || NoSingleByteCoverage)
431 CounterMap[Do->
getCond()] = NextCounter++;
433 CounterMap[Do->
getBody()] = NextCounter++;
436 Base::TraverseDoStmt(Do);
438 Hash.combine(PGOHash::EndOfScope);
442 bool TraverseForStmt(
ForStmt *For) {
447 if (!CS || NoSingleByteCoverage)
450 CounterMap[For->
getCond()] = NextCounter++;
451 else if (CS == For->
getInc())
452 CounterMap[For->
getInc()] = NextCounter++;
454 CounterMap[For->
getBody()] = NextCounter++;
457 Base::TraverseForStmt(For);
459 Hash.combine(PGOHash::EndOfScope);
467 if (!CS || NoSingleByteCoverage)
470 CounterMap[ForRange->
getBody()] = NextCounter++;
473 Base::TraverseCXXForRangeStmt(ForRange);
475 Hash.combine(PGOHash::EndOfScope);
482#define DEFINE_NESTABLE_TRAVERSAL(N) \
483 bool Traverse##N(N *S) { \
484 Base::Traverse##N(S); \
485 if (Hash.getHashVersion() != PGO_HASH_V1) \
486 Hash.combine(PGOHash::EndOfScope); \
496 switch (S->getStmtClass()) {
499 case Stmt::LabelStmtClass:
500 return PGOHash::LabelStmt;
501 case Stmt::WhileStmtClass:
502 return PGOHash::WhileStmt;
503 case Stmt::DoStmtClass:
504 return PGOHash::DoStmt;
505 case Stmt::ForStmtClass:
506 return PGOHash::ForStmt;
507 case Stmt::CXXForRangeStmtClass:
508 return PGOHash::CXXForRangeStmt;
509 case Stmt::ObjCForCollectionStmtClass:
510 return PGOHash::ObjCForCollectionStmt;
511 case Stmt::SwitchStmtClass:
512 return PGOHash::SwitchStmt;
513 case Stmt::CaseStmtClass:
514 return PGOHash::CaseStmt;
515 case Stmt::DefaultStmtClass:
516 return PGOHash::DefaultStmt;
517 case Stmt::IfStmtClass:
518 return PGOHash::IfStmt;
519 case Stmt::CXXTryStmtClass:
520 return PGOHash::CXXTryStmt;
521 case Stmt::CXXCatchStmtClass:
522 return PGOHash::CXXCatchStmt;
523 case Stmt::ConditionalOperatorClass:
524 return PGOHash::ConditionalOperator;
525 case Stmt::BinaryConditionalOperatorClass:
526 return PGOHash::BinaryConditionalOperator;
527 case Stmt::BinaryOperatorClass: {
530 return PGOHash::BinaryOperatorLAnd;
532 return PGOHash::BinaryOperatorLOr;
538 return PGOHash::BinaryOperatorLT;
540 return PGOHash::BinaryOperatorGT;
542 return PGOHash::BinaryOperatorLE;
544 return PGOHash::BinaryOperatorGE;
546 return PGOHash::BinaryOperatorEQ;
548 return PGOHash::BinaryOperatorNE;
556 switch (S->getStmtClass()) {
559 case Stmt::GotoStmtClass:
560 return PGOHash::GotoStmt;
561 case Stmt::IndirectGotoStmtClass:
562 return PGOHash::IndirectGotoStmt;
563 case Stmt::BreakStmtClass:
564 return PGOHash::BreakStmt;
565 case Stmt::ContinueStmtClass:
566 return PGOHash::ContinueStmt;
567 case Stmt::ReturnStmtClass:
568 return PGOHash::ReturnStmt;
569 case Stmt::CXXThrowExprClass:
570 return PGOHash::ThrowExpr;
571 case Stmt::UnaryOperatorClass: {
574 return PGOHash::UnaryOperatorLNot;
580 return PGOHash::None;
592 bool RecordNextStmtCount;
598 llvm::DenseMap<const Stmt *, uint64_t> &
CountMap;
601 struct BreakContinue {
604 BreakContinue() =
default;
608 ComputeRegionCounts(llvm::DenseMap<const Stmt *, uint64_t> &
CountMap,
612 void RecordStmtCount(
const Stmt *S) {
613 if (RecordNextStmtCount) {
615 RecordNextStmtCount =
false;
621 CurrentCount = Count;
625 void VisitStmt(
const Stmt *S) {
627 for (
const Stmt *Child : S->children())
658 void VisitBlockDecl(
const BlockDecl *D) {
667 if (S->getRetValue())
668 Visit(S->getRetValue());
670 RecordNextStmtCount =
true;
678 RecordNextStmtCount =
true;
681 void VisitGotoStmt(
const GotoStmt *S) {
684 RecordNextStmtCount =
true;
687 void VisitLabelStmt(
const LabelStmt *S) {
688 RecordNextStmtCount =
false;
692 Visit(S->getSubStmt());
695 void VisitBreakStmt(
const BreakStmt *S) {
697 assert(!BreakContinueStack.empty() &&
"break not in a loop or switch!");
698 BreakContinueStack.back().BreakCount += CurrentCount;
700 RecordNextStmtCount =
true;
705 assert(!BreakContinueStack.empty() &&
"continue stmt not in a loop!");
706 BreakContinueStack.back().ContinueCount += CurrentCount;
708 RecordNextStmtCount =
true;
711 void VisitWhileStmt(
const WhileStmt *S) {
713 uint64_t ParentCount = CurrentCount;
715 BreakContinueStack.push_back(BreakContinue());
719 CountMap[S->getBody()] = CurrentCount;
721 uint64_t BackedgeCount = CurrentCount;
727 BreakContinue BC = BreakContinueStack.pop_back_val();
729 setCount(ParentCount + BackedgeCount + BC.ContinueCount);
732 setCount(BC.BreakCount + CondCount - BodyCount);
733 RecordNextStmtCount =
true;
736 void VisitDoStmt(
const DoStmt *S) {
740 BreakContinueStack.push_back(BreakContinue());
742 uint64_t BodyCount = setCount(LoopCount + CurrentCount);
745 uint64_t BackedgeCount = CurrentCount;
747 BreakContinue BC = BreakContinueStack.pop_back_val();
750 uint64_t CondCount = setCount(BackedgeCount + BC.ContinueCount);
753 setCount(BC.BreakCount + CondCount - LoopCount);
754 RecordNextStmtCount =
true;
757 void VisitForStmt(
const ForStmt *S) {
762 uint64_t ParentCount = CurrentCount;
764 BreakContinueStack.push_back(BreakContinue());
770 uint64_t BackedgeCount = CurrentCount;
771 BreakContinue BC = BreakContinueStack.pop_back_val();
776 uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount);
783 setCount(ParentCount + BackedgeCount + BC.ContinueCount);
788 setCount(BC.BreakCount + CondCount - BodyCount);
789 RecordNextStmtCount =
true;
796 Visit(S->getLoopVarStmt());
797 Visit(S->getRangeStmt());
798 Visit(S->getBeginStmt());
799 Visit(S->getEndStmt());
801 uint64_t ParentCount = CurrentCount;
802 BreakContinueStack.push_back(BreakContinue());
808 uint64_t BackedgeCount = CurrentCount;
809 BreakContinue BC = BreakContinueStack.pop_back_val();
813 uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount);
819 setCount(ParentCount + BackedgeCount + BC.ContinueCount);
822 setCount(BC.BreakCount + CondCount - BodyCount);
823 RecordNextStmtCount =
true;
828 Visit(S->getElement());
829 uint64_t ParentCount = CurrentCount;
830 BreakContinueStack.push_back(BreakContinue());
835 uint64_t BackedgeCount = CurrentCount;
836 BreakContinue BC = BreakContinueStack.pop_back_val();
838 setCount(BC.BreakCount + ParentCount + BackedgeCount + BC.ContinueCount -
840 RecordNextStmtCount =
true;
849 BreakContinueStack.push_back(BreakContinue());
852 BreakContinue BC = BreakContinueStack.pop_back_val();
853 if (!BreakContinueStack.empty())
854 BreakContinueStack.back().ContinueCount += BC.ContinueCount;
857 RecordNextStmtCount =
true;
861 RecordNextStmtCount =
false;
866 setCount(CurrentCount + CaseCount);
870 RecordNextStmtCount =
true;
871 Visit(S->getSubStmt());
874 void VisitIfStmt(
const IfStmt *S) {
877 if (S->isConsteval()) {
878 const Stmt *Stm = S->isNegatedConsteval() ? S->getThen() : S->getElse();
884 uint64_t ParentCount = CurrentCount;
896 uint64_t ElseCount = ParentCount - ThenCount;
901 OutCount += CurrentCount;
903 OutCount += ElseCount;
905 RecordNextStmtCount =
true;
910 Visit(S->getTryBlock());
911 for (
unsigned I = 0, E = S->getNumHandlers(); I < E; ++I)
912 Visit(S->getHandler(I));
915 RecordNextStmtCount =
true;
919 RecordNextStmtCount =
false;
923 Visit(S->getHandlerBlock());
928 uint64_t ParentCount = CurrentCount;
938 uint64_t FalseCount = setCount(ParentCount - TrueCount);
941 OutCount += CurrentCount;
944 RecordNextStmtCount =
true;
949 uint64_t ParentCount = CurrentCount;
955 setCount(ParentCount + RHSCount - CurrentCount);
956 RecordNextStmtCount =
true;
961 uint64_t ParentCount = CurrentCount;
967 setCount(ParentCount + RHSCount - CurrentCount);
968 RecordNextStmtCount =
true;
973void PGOHash::combine(HashType
Type) {
975 assert(
Type &&
"Hash is invalid: unexpected type 0");
976 assert(
unsigned(
Type) < TooBig &&
"Hash is invalid: too many types");
979 if (Count && Count % NumTypesPerWord == 0) {
980 using namespace llvm::support;
982 endian::byte_swap<uint64_t, llvm::endianness::little>(Working);
989 Working = Working << NumBitsPerType |
Type;
994 if (Count <= NumTypesPerWord)
1005 MD5.update({(uint8_t)Working});
1007 using namespace llvm::support;
1009 endian::byte_swap<uint64_t, llvm::endianness::little>(Working);
1010 MD5.update(
llvm::ArrayRef((uint8_t *)&Swapped,
sizeof(Swapped)));
1015 llvm::MD5::MD5Result
Result;
1031 llvm::IndexedInstrProfReader *PGOReader = CGM.
getPGOReader();
1032 if (!InstrumentRegions && !PGOReader)
1040 if (
const auto *CCD = dyn_cast<CXXConstructorDecl>(D))
1049 if (Fn->hasFnAttribute(llvm::Attribute::NoProfile))
1051 if (Fn->hasFnAttribute(llvm::Attribute::SkipProfile))
1056 mapRegionCounters(D);
1058 emitCounterRegionMapping(D);
1061 loadRegionCounts(PGOReader,
SM.isInMainFile(D->
getLocation()));
1062 computeRegionCounts(D);
1063 applyFunctionAttributes(PGOReader, Fn);
1067void CodeGenPGO::mapRegionCounters(
const Decl *D) {
1071 uint64_t ProfileVersion = llvm::IndexedInstrProf::Version;
1073 HashVersion = getPGOHashVersion(PGOReader, CGM);
1074 ProfileVersion = PGOReader->getVersion();
1086 unsigned MCDCMaxConditions = (CGM.
getCodeGenOpts().MCDCCoverage) ? 6 : 0;
1088 RegionCounterMap.reset(
new llvm::DenseMap<const Stmt *, unsigned>);
1090 MapRegionCounters Walker(HashVersion, ProfileVersion, *RegionCounterMap,
1091 *RegionMCDCState, MCDCMaxConditions, CGM.
getDiags());
1092 if (
const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
1094 else if (
const ObjCMethodDecl *MD = dyn_cast_or_null<ObjCMethodDecl>(D))
1096 else if (
const BlockDecl *BD = dyn_cast_or_null<BlockDecl>(D))
1097 Walker.TraverseDecl(
const_cast<BlockDecl *
>(BD));
1098 else if (
const CapturedDecl *CD = dyn_cast_or_null<CapturedDecl>(D))
1100 assert(Walker.NextCounter > 0 &&
"no entry counter mapped for decl");
1101 NumRegionCounters = Walker.NextCounter;
1102 RegionMCDCState->BitmapBytes = Walker.NextMCDCBitmapIdx;
1103 FunctionHash = Walker.Hash.finalize();
1106bool CodeGenPGO::skipRegionMappingForDecl(
const Decl *D) {
1116 !D->
hasAttr<CUDAGlobalAttr>()) ||
1118 (D->
hasAttr<CUDAGlobalAttr>() ||
1119 (!D->
hasAttr<CUDAHostAttr>() && D->
hasAttr<CUDADeviceAttr>())))))
1128void CodeGenPGO::emitCounterRegionMapping(
const Decl *D) {
1129 if (skipRegionMappingForDecl(D))
1132 std::string CoverageMapping;
1133 llvm::raw_string_ostream OS(CoverageMapping);
1134 RegionMCDCState->BranchByStmt.clear();
1137 CGM.
getLangOpts(), RegionCounterMap.get(), RegionMCDCState.get());
1138 MappingGen.emitCounterMapping(D, OS);
1141 if (CoverageMapping.empty())
1145 FuncNameVar, FuncName, FunctionHash, CoverageMapping);
1150 llvm::GlobalValue::LinkageTypes
Linkage) {
1151 if (skipRegionMappingForDecl(D))
1154 std::string CoverageMapping;
1155 llvm::raw_string_ostream OS(CoverageMapping);
1162 if (CoverageMapping.empty())
1167 FuncNameVar, FuncName, FunctionHash, CoverageMapping,
false);
1170void CodeGenPGO::computeRegionCounts(
const Decl *D) {
1171 StmtCountMap.reset(
new llvm::DenseMap<const Stmt *, uint64_t>);
1172 ComputeRegionCounts Walker(*StmtCountMap, *
this);
1173 if (
const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
1174 Walker.VisitFunctionDecl(FD);
1175 else if (
const ObjCMethodDecl *MD = dyn_cast_or_null<ObjCMethodDecl>(D))
1176 Walker.VisitObjCMethodDecl(MD);
1177 else if (
const BlockDecl *BD = dyn_cast_or_null<BlockDecl>(D))
1178 Walker.VisitBlockDecl(BD);
1179 else if (
const CapturedDecl *CD = dyn_cast_or_null<CapturedDecl>(D))
1180 Walker.VisitCapturedDecl(
const_cast<CapturedDecl *
>(CD));
1184CodeGenPGO::applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader,
1185 llvm::Function *Fn) {
1190 Fn->setEntryCount(FunctionCount);
1194 llvm::Value *StepV) {
1195 if (!RegionCounterMap || !Builder.GetInsertBlock())
1198 unsigned Counter = (*RegionCounterMap)[S];
1200 llvm::Value *Args[] = {FuncNameVar,
1201 Builder.getInt64(FunctionHash),
1202 Builder.getInt32(NumRegionCounters),
1203 Builder.getInt32(Counter), StepV};
1206 Builder.CreateCall(CGM.
getIntrinsic(llvm::Intrinsic::instrprof_cover),
1210 Builder.CreateCall(CGM.
getIntrinsic(llvm::Intrinsic::instrprof_increment),
1214 CGM.
getIntrinsic(llvm::Intrinsic::instrprof_increment_step),
1219bool CodeGenPGO::canEmitMCDCCoverage(
const CGBuilderTy &Builder) {
1225 if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState)
1228 auto *I8PtrTy = llvm::PointerType::getUnqual(CGM.
getLLVMContext());
1233 llvm::Value *Args[3] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
1234 Builder.getInt64(FunctionHash),
1235 Builder.getInt32(RegionMCDCState->BitmapBytes)};
1237 CGM.
getIntrinsic(llvm::Intrinsic::instrprof_mcdc_parameters), Args);
1243 if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState)
1246 S = S->IgnoreParens();
1248 auto DecisionStateIter = RegionMCDCState->DecisionByStmt.find(S);
1249 if (DecisionStateIter == RegionMCDCState->DecisionByStmt.end())
1253 unsigned MCDCTestVectorBitmapOffset = DecisionStateIter->second.BitmapIdx;
1254 auto *I8PtrTy = llvm::PointerType::getUnqual(CGM.
getLLVMContext());
1261 llvm::Value *Args[5] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
1262 Builder.getInt64(FunctionHash),
1263 Builder.getInt32(RegionMCDCState->BitmapBytes),
1264 Builder.getInt32(MCDCTestVectorBitmapOffset),
1267 CGM.
getIntrinsic(llvm::Intrinsic::instrprof_mcdc_tvbitmap_update), Args);
1272 if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState)
1275 S = S->IgnoreParens();
1277 if (!RegionMCDCState->DecisionByStmt.contains(S))
1281 Builder.CreateStore(Builder.getInt32(0), MCDCCondBitmapAddr);
1287 if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState)
1299 auto BranchStateIter = RegionMCDCState->BranchByStmt.find(S);
1300 if (BranchStateIter == RegionMCDCState->BranchByStmt.end())
1304 const auto &Branch = BranchStateIter->second;
1305 assert(Branch.ID >= 0 &&
"Condition has no ID!");
1307 auto *I8PtrTy = llvm::PointerType::getUnqual(CGM.
getLLVMContext());
1312 llvm::Value *Args[5] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
1313 Builder.getInt64(FunctionHash),
1314 Builder.getInt32(Branch.ID),
1317 CGM.
getIntrinsic(llvm::Intrinsic::instrprof_mcdc_condbitmap_update),
1323 M.addModuleFlag(llvm::Module::Warning,
"EnableValueProfiling",
1330 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1331 llvm::Type *IntTy64 = llvm::Type::getInt64Ty(M.getContext());
1332 uint64_t ProfileVersion =
1333 (INSTR_PROF_RAW_VERSION | VARIANT_MASK_BYTE_COVERAGE);
1335 auto IRLevelVersionVariable =
new llvm::GlobalVariable(
1336 M, IntTy64,
true, llvm::GlobalValue::WeakAnyLinkage,
1337 llvm::Constant::getIntegerValue(IntTy64,
1338 llvm::APInt(64, ProfileVersion)),
1341 IRLevelVersionVariable->setVisibility(llvm::GlobalValue::DefaultVisibility);
1342 llvm::Triple TT(M.getTargetTriple());
1343 if (TT.supportsCOMDAT()) {
1344 IRLevelVersionVariable->setLinkage(llvm::GlobalValue::ExternalLinkage);
1345 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));
1347 IRLevelVersionVariable->setDSOLocal(
true);
1354 llvm::Instruction *ValueSite, llvm::Value *ValuePtr) {
1359 if (!ValuePtr || !ValueSite || !Builder.GetInsertBlock())
1362 if (isa<llvm::Constant>(ValuePtr))
1366 if (InstrumentValueSites && RegionCounterMap) {
1367 auto BuilderInsertPoint = Builder.saveIP();
1368 Builder.SetInsertPoint(ValueSite);
1369 llvm::Value *Args[5] = {
1371 Builder.getInt64(FunctionHash),
1372 Builder.CreatePtrToInt(ValuePtr, Builder.getInt64Ty()),
1373 Builder.getInt32(ValueKind),
1374 Builder.getInt32(NumValueSites[ValueKind]++)
1377 CGM.
getIntrinsic(llvm::Intrinsic::instrprof_value_profile), Args);
1378 Builder.restoreIP(BuilderInsertPoint);
1382 llvm::IndexedInstrProfReader *PGOReader = CGM.
getPGOReader();
1390 if (NumValueSites[ValueKind] >= ProfRecord->getNumValueSites(ValueKind))
1393 llvm::annotateValueSite(CGM.
getModule(), *ValueSite, *ProfRecord,
1394 (llvm::InstrProfValueKind)ValueKind,
1395 NumValueSites[ValueKind]);
1397 NumValueSites[ValueKind]++;
1401void CodeGenPGO::loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader,
1402 bool IsInMainFile) {
1404 RegionCounts.clear();
1406 PGOReader->getInstrProfRecord(FuncName, FunctionHash);
1407 if (
auto E = RecordExpected.takeError()) {
1408 auto IPE = std::get<0>(llvm::InstrProfError::take(std::move(E)));
1409 if (IPE == llvm::instrprof_error::unknown_function)
1411 else if (IPE == llvm::instrprof_error::hash_mismatch)
1413 else if (IPE == llvm::instrprof_error::malformed)
1419 std::make_unique<llvm::InstrProfRecord>(std::move(RecordExpected.get()));
1420 RegionCounts = ProfRecord->Counts;
1428 return MaxWeight < UINT32_MAX ? 1 : MaxWeight / UINT32_MAX + 1;
1441 assert(Scale &&
"scale by 0?");
1442 uint64_t Scaled = Weight / Scale + 1;
1443 assert(Scaled <= UINT32_MAX &&
"overflow 32-bits");
1447llvm::MDNode *CodeGenFunction::createProfileWeights(uint64_t TrueCount,
1448 uint64_t FalseCount)
const {
1450 if (!TrueCount && !FalseCount)
1464 if (Weights.size() < 2)
1468 uint64_t MaxWeight = *std::max_element(Weights.begin(), Weights.end());
1476 ScaledWeights.reserve(Weights.size());
1477 for (uint64_t W : Weights)
1481 return MDHelper.createBranchWeights(ScaledWeights);
1485CodeGenFunction::createProfileWeightsForLoop(
const Stmt *Cond,
1486 uint64_t LoopCount)
const {
1489 std::optional<uint64_t> CondCount = PGO.
getStmtCount(Cond);
1490 if (!CondCount || *CondCount == 0)
1492 return createProfileWeights(LoopCount,
1493 std::max(*CondCount, LoopCount) - LoopCount);
llvm::ImmutableMap< CountKey, unsigned > CountMap
#define DEFINE_NESTABLE_TRAVERSAL(N)
static llvm::cl::opt< bool > EnableValueProfiling("enable-value-profiling", llvm::cl::desc("Enable value profiling"), llvm::cl::Hidden, llvm::cl::init(false))
llvm::cl::opt< bool > SystemHeadersCoverage
PGOHashVersion
The version of the PGO hash algorithm.
static uint64_t calculateWeightScale(uint64_t MaxWeight)
Calculate what to divide by to scale weights.
static uint32_t scaleBranchWeight(uint64_t Weight, uint64_t Scale)
Scale an individual branch weight (and add 1).
static DiagnosticBuilder Diag(DiagnosticsEngine *Diags, const LangOptions &Features, FullSourceLoc TokLoc, const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd, unsigned DiagID)
Produce a diagnostic highlighting some portion of a literal.
SourceManager & getSourceManager()
AbstractConditionalOperator - An abstract base class for ConditionalOperator and BinaryConditionalOpe...
Expr * getCond() const
getCond - Return the expression representing the condition for the ?: operator.
Expr * getTrueExpr() const
getTrueExpr - Return the subexpression representing the value of the expression if the condition eval...
Expr * getFalseExpr() const
getFalseExpr - Return the subexpression representing the value of the expression if the condition eva...
BinaryConditionalOperator - The GNU extension to the conditional operator which allows the middle ope...
A builtin binary operation expression such as "x + y" or "x <= y".
static bool isLogicalOp(Opcode Opc)
Represents a block literal declaration, which is like an unnamed FunctionDecl.
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
BlockExpr - Adaptor class for mixing a BlockDecl with expressions.
BreakStmt - This represents a break.
CXXCatchStmt - This represents a C++ catch block.
CXXForRangeStmt - This represents C++0x [stmt.ranged]'s ranged for statement, represented as 'for (ra...
A C++ throw-expression (C++ [except.throw]).
const Expr * getSubExpr() const
CXXTryStmt - A C++ try block, including all handlers.
Represents the body of a CapturedStmt, and serves as its DeclContext.
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
This captures a statement into a function.
CaseStmt - Represent a case statement.
bool hasProfileClangInstr() const
Check if Clang profile instrumenation is on.
std::string MainFileName
The user provided name for the "main file", if non-empty.
llvm::Value * getPointer() const
static const Expr * stripCond(const Expr *C)
Ignore parentheses and logical-NOT to track conditions consistently.
static bool IsConstructorDelegationValid(const CXXConstructorDecl *Ctor)
This class organizes the cross-function state that is used while generating LLVM code.
llvm::Module & getModule() const
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
const TargetInfo & getTarget() const
llvm::IndexedInstrProfReader * getPGOReader() const
CoverageMappingModuleGen * getCoverageMapping() const
InstrProfStats & getPGOStats()
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
llvm::LLVMContext & getLLVMContext()
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys=std::nullopt)
void ClearUnusedCoverageMapping(const Decl *D)
Remove the deferred empty coverage mapping as this declaration is actually instrumented.
void assignRegionCounters(GlobalDecl GD, llvm::Function *Fn)
Assign counters to regions and configure them for PGO of a given function.
std::optional< uint64_t > getStmtCount(const Stmt *S) const
Check if an execution count is known for a given statement.
uint64_t getRegionCount(const Stmt *S)
Return the region count for the counter at the given index.
void setValueProfilingFlag(llvm::Module &M)
void valueProfile(CGBuilderTy &Builder, uint32_t ValueKind, llvm::Instruction *ValueSite, llvm::Value *ValuePtr)
void emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, Address MCDCCondBitmapAddr, llvm::Value *Val)
void emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S, Address MCDCCondBitmapAddr)
void setProfileVersion(llvm::Module &M)
void emitEmptyCounterMapping(const Decl *D, StringRef FuncName, llvm::GlobalValue::LinkageTypes Linkage)
Emit a coverage mapping range with a counter zero for an unused declaration.
void emitMCDCParameters(CGBuilderTy &Builder)
void emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S, Address MCDCCondBitmapAddr)
bool haveRegionCounts() const
Whether or not we have PGO region data for the current function.
void emitCounterSetOrIncrement(CGBuilderTy &Builder, const Stmt *S, llvm::Value *StepV)
Organizes the per-function state that is used while generating code coverage mapping data.
void emitEmptyMapping(const Decl *D, llvm::raw_ostream &OS)
Emit the coverage mapping data for an unused function.
void addFunctionMappingRecord(llvm::GlobalVariable *FunctionName, StringRef FunctionNameValue, uint64_t FunctionHash, const std::string &CoverageMapping, bool IsUsed=true)
Add a function's coverage mapping record to the collection of the function mapping records.
void addMissing(bool MainFile)
Record that a function we've visited has no profile data.
void addMismatched(bool MainFile)
Record that a function we've visited has mismatched profile data.
void addVisited(bool MainFile)
Record that we've visited a function and whether or not that function was in the main source file.
ConditionalOperator - The ?: ternary operator.
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
ContinueStmt - This represents a continue.
Decl - This represents one declaration (or definition), e.g.
bool isImplicit() const
isImplicit - Indicates whether the declaration was implicitly generated by the implementation.
virtual Stmt * getBody() const
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
SourceLocation getLocation() const
Concrete class used by the front-end to report problems and issues.
DoStmt - This represents a 'do/while' stmt.
This represents one expression.
ForStmt - This represents a 'for (init;cond;inc)' stmt.
Represents a function declaration or definition.
Stmt * getBody(const FunctionDecl *&Definition) const
Retrieve the body (definition) of the function.
GlobalDecl - represents a global declaration.
CXXCtorType getCtorType() const
CXXDtorType getDtorType() const
const Decl * getDecl() const
GotoStmt - This represents a direct goto.
IfStmt - This represents an if/then/else.
IndirectGotoStmt - This represents an indirect goto.
LabelStmt - Represents a label, which has a substatement.
A C++ lambda expression, which produces a function object (of unspecified type) that can be invoked l...
Represents Objective-C's collection statement.
ObjCMethodDecl - Represents an instance or class method declaration.
Stmt * getBody() const override
Retrieve the body of this method, if it has one.
A class that does preorder or postorder depth-first traversal on the entire Clang AST and visits each...
ReturnStmt - This represents a return, optionally of an expression: return; return 4;.
This class handles loading and caching of source files into memory.
Stmt - This represents one statement.
SourceLocation getBeginLoc() const LLVM_READONLY
SwitchStmt - This represents a 'switch' stmt.
bool hasConstructorVariants() const
Does this ABI have different entrypoints for complete-object and base-subobject constructors?
TargetCXXABI getCXXABI() const
Get the C++ ABI currently in use.
The base class of the type hierarchy.
UnaryOperator - This represents the unary-expression's (except sizeof and alignof),...
WhileStmt - This represents a 'while' stmt.
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
bool LE(InterpState &S, CodePtr OpPC)
The JSON file list parser is used to communicate input to InstallAPI.
@ If
'if' clause, allowed on all the Compute Constructs, Data Constructs, Executable Constructs,...
@ Ctor_Base
Base object ctor.
Linkage
Describes the different kinds of linkage (C++ [basic.link], C99 6.2.2) that an entity may have.
@ Result
The result type of a method or function.
@ Dtor_Base
Base object dtor.
void finalize(TemplateInstantiationCallbackPtrs &Callbacks, const Sema &TheSema)
@ None
The alignment was not explicit in code.
YAML serialization mapping.
cl::opt< bool > EnableSingleByteCoverage
Per-Function MC/DC state.
llvm::DenseMap< const Stmt *, Decision > DecisionByStmt