17 #include "llvm/ADT/SmallSet.h"
18 #include "llvm/ADT/StringExtras.h"
19 #include "llvm/ADT/StringSwitch.h"
20 #include "llvm/ADT/Triple.h"
21 #include "llvm/Analysis/AliasAnalysis.h"
22 #include "llvm/Analysis/StackSafetyAnalysis.h"
23 #include "llvm/Analysis/TargetLibraryInfo.h"
24 #include "llvm/Analysis/TargetTransformInfo.h"
25 #include "llvm/Bitcode/BitcodeReader.h"
26 #include "llvm/Bitcode/BitcodeWriter.h"
27 #include "llvm/Bitcode/BitcodeWriterPass.h"
28 #include "llvm/CodeGen/RegAllocRegistry.h"
29 #include "llvm/CodeGen/SchedulerRegistry.h"
30 #include "llvm/CodeGen/TargetSubtargetInfo.h"
31 #include "llvm/IR/DataLayout.h"
32 #include "llvm/IR/IRPrintingPasses.h"
33 #include "llvm/IR/LegacyPassManager.h"
34 #include "llvm/IR/Module.h"
35 #include "llvm/IR/ModuleSummaryIndex.h"
36 #include "llvm/IR/PassManager.h"
37 #include "llvm/IR/Verifier.h"
38 #include "llvm/LTO/LTOBackend.h"
39 #include "llvm/MC/MCAsmInfo.h"
40 #include "llvm/MC/SubtargetFeature.h"
41 #include "llvm/MC/TargetRegistry.h"
42 #include "llvm/Object/OffloadBinary.h"
43 #include "llvm/Passes/PassBuilder.h"
44 #include "llvm/Passes/PassPlugin.h"
45 #include "llvm/Passes/StandardInstrumentations.h"
46 #include "llvm/Support/BuryPointer.h"
47 #include "llvm/Support/CommandLine.h"
48 #include "llvm/Support/MemoryBuffer.h"
49 #include "llvm/Support/PrettyStackTrace.h"
50 #include "llvm/Support/TimeProfiler.h"
51 #include "llvm/Support/Timer.h"
52 #include "llvm/Support/ToolOutputFile.h"
53 #include "llvm/Support/raw_ostream.h"
54 #include "llvm/Target/TargetMachine.h"
55 #include "llvm/Target/TargetOptions.h"
56 #include "llvm/Transforms/Coroutines/CoroCleanup.h"
57 #include "llvm/Transforms/Coroutines/CoroEarly.h"
58 #include "llvm/Transforms/Coroutines/CoroElide.h"
59 #include "llvm/Transforms/Coroutines/CoroSplit.h"
60 #include "llvm/Transforms/IPO.h"
61 #include "llvm/Transforms/IPO/AlwaysInliner.h"
62 #include "llvm/Transforms/IPO/LowerTypeTests.h"
63 #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
64 #include "llvm/Transforms/InstCombine/InstCombine.h"
65 #include "llvm/Transforms/Instrumentation.h"
66 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
67 #include "llvm/Transforms/Instrumentation/AddressSanitizerOptions.h"
68 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
69 #include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
70 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
71 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
72 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
73 #include "llvm/Transforms/Instrumentation/MemProfiler.h"
74 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
75 #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
76 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
77 #include "llvm/Transforms/ObjCARC.h"
78 #include "llvm/Transforms/Scalar.h"
79 #include "llvm/Transforms/Scalar/EarlyCSE.h"
80 #include "llvm/Transforms/Scalar/GVN.h"
81 #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
82 #include "llvm/Transforms/Utils.h"
83 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
84 #include "llvm/Transforms/Utils/Debugify.h"
85 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
86 #include "llvm/Transforms/Utils/ModuleUtils.h"
87 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
88 #include "llvm/Transforms/Utils/SymbolRewriter.h"
90 using namespace clang;
93 #define HANDLE_EXTENSION(Ext) \
94 llvm::PassPluginLibraryInfo get##Ext##PluginInfo();
95 #include "llvm/Support/Extension.def"
108 class EmitAssemblyHelper {
116 Timer CodeGenerationTime;
118 std::unique_ptr<raw_pwrite_stream>
OS;
122 TargetIRAnalysis getTargetIRAnalysis()
const {
124 return TM->getTargetIRAnalysis();
126 return TargetIRAnalysis();
137 void CreateTargetMachine(
bool MustCreateTM);
142 bool AddEmitPasses(legacy::PassManager &CodeGenPasses,
BackendAction Action,
143 raw_pwrite_stream &OS, raw_pwrite_stream *DwoOS);
145 std::unique_ptr<llvm::ToolOutputFile> openOutputFile(StringRef Path) {
147 auto F = std::make_unique<llvm::ToolOutputFile>(Path, EC,
148 llvm::sys::fs::OF_None);
150 Diags.
Report(diag::err_fe_unable_to_open_output) << Path << EC.message();
158 std::unique_ptr<raw_pwrite_stream> &OS,
159 std::unique_ptr<llvm::ToolOutputFile> &ThinLinkOS);
161 std::unique_ptr<raw_pwrite_stream> &OS,
162 std::unique_ptr<llvm::ToolOutputFile> &DwoOS);
169 bool shouldEmitRegularLTOSummary()
const {
170 return CodeGenOpts.PrepareForLTO && !CodeGenOpts.DisableLLVMPasses &&
171 TargetTriple.getVendor() != llvm::Triple::Apple;
180 : Diags(_Diags), HSOpts(HeaderSearchOpts), CodeGenOpts(CGOpts),
181 TargetOpts(TOpts), LangOpts(LOpts), TheModule(M),
182 CodeGenerationTime(
"codegen",
"Code Generation Time"),
183 TargetTriple(TheModule->getTargetTriple()) {}
185 ~EmitAssemblyHelper() {
186 if (CodeGenOpts.DisableFree)
187 BuryPointer(std::move(TM));
190 std::unique_ptr<TargetMachine> TM;
194 std::unique_ptr<raw_pwrite_stream> OS);
198 static SanitizerCoverageOptions
200 SanitizerCoverageOptions Opts;
203 Opts.IndirectCalls = CGOpts.SanitizeCoverageIndirectCalls;
204 Opts.TraceBB = CGOpts.SanitizeCoverageTraceBB;
205 Opts.TraceCmp = CGOpts.SanitizeCoverageTraceCmp;
206 Opts.TraceDiv = CGOpts.SanitizeCoverageTraceDiv;
207 Opts.TraceGep = CGOpts.SanitizeCoverageTraceGep;
208 Opts.Use8bitCounters = CGOpts.SanitizeCoverage8bitCounters;
209 Opts.TracePC = CGOpts.SanitizeCoverageTracePC;
210 Opts.TracePCGuard = CGOpts.SanitizeCoverageTracePCGuard;
211 Opts.NoPrune = CGOpts.SanitizeCoverageNoPrune;
212 Opts.Inline8bitCounters = CGOpts.SanitizeCoverageInline8bitCounters;
213 Opts.InlineBoolFlag = CGOpts.SanitizeCoverageInlineBoolFlag;
214 Opts.PCTable = CGOpts.SanitizeCoveragePCTable;
215 Opts.StackDepth = CGOpts.SanitizeCoverageStackDepth;
216 Opts.TraceLoads = CGOpts.SanitizeCoverageTraceLoads;
217 Opts.TraceStores = CGOpts.SanitizeCoverageTraceStores;
226 if (!CGOpts.SanitizeAddressGlobalsDeadStripping)
228 switch (T.getObjectFormat()) {
233 return !CGOpts.DisableIntegratedAS;
235 llvm::report_fatal_error(
"ASan not implemented for GOFF");
237 llvm::report_fatal_error(
"ASan not implemented for XCOFF.");
239 case Triple::DXContainer:
241 case Triple::UnknownObjectFormat:
247 static TargetLibraryInfoImpl *
createTLII(llvm::Triple &TargetTriple,
249 TargetLibraryInfoImpl *TLII =
new TargetLibraryInfoImpl(TargetTriple);
251 switch (CodeGenOpts.getVecLib()) {
253 TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::Accelerate);
256 switch(TargetTriple.getArch()) {
259 case llvm::Triple::x86_64:
260 TLII->addVectorizableFunctionsFromVecLib
261 (TargetLibraryInfoImpl::LIBMVEC_X86);
266 TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::MASSV);
269 TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::SVML);
272 TLII->addVectorizableFunctionsFromVecLib(
273 TargetLibraryInfoImpl::DarwinLibSystemM);
282 switch (CodeGenOpts.OptimizationLevel) {
284 llvm_unreachable(
"Invalid optimization level!");
288 return CodeGenOpt::Less;
290 return CodeGenOpt::Default;
292 return CodeGenOpt::Aggressive;
298 unsigned CodeModel = llvm::StringSwitch<unsigned>(CodeGenOpts.
CodeModel)
299 .Case(
"tiny", llvm::CodeModel::Tiny)
300 .Case(
"small", llvm::CodeModel::Small)
301 .Case(
"kernel", llvm::CodeModel::Kernel)
302 .Case(
"medium", llvm::CodeModel::Medium)
303 .Case(
"large", llvm::CodeModel::Large)
304 .Case(
"default", ~1u)
306 assert(CodeModel != ~0u &&
"invalid code model!");
307 if (CodeModel == ~1u)
309 return static_cast<llvm::CodeModel::Model
>(CodeModel);
314 return CGFT_ObjectFile;
319 return CGFT_AssemblyFile;
329 llvm::TargetOptions &Options,
334 switch (LangOpts.getThreadModel()) {
336 Options.ThreadModel = llvm::ThreadModel::POSIX;
339 Options.ThreadModel = llvm::ThreadModel::Single;
344 assert((CodeGenOpts.
FloatABI ==
"soft" || CodeGenOpts.
FloatABI ==
"softfp" ||
346 "Invalid Floating Point ABI!");
347 Options.FloatABIType =
348 llvm::StringSwitch<llvm::FloatABI::ABIType>(CodeGenOpts.
FloatABI)
349 .Case(
"soft", llvm::FloatABI::Soft)
350 .Case(
"softfp", llvm::FloatABI::Soft)
351 .Case(
"hard", llvm::FloatABI::Hard)
352 .Default(llvm::FloatABI::Default);
355 switch (LangOpts.getDefaultFPContractMode()) {
359 Options.AllowFPOpFusion = llvm::FPOpFusion::Standard;
363 Options.AllowFPOpFusion = llvm::FPOpFusion::Standard;
366 Options.AllowFPOpFusion = llvm::FPOpFusion::Fast;
370 Options.BinutilsVersion =
371 llvm::TargetMachine::parseBinutilsVersion(CodeGenOpts.
BinutilsVersion);
372 Options.UseInitArray = CodeGenOpts.UseInitArray;
373 Options.LowerGlobalDtorsViaCxaAtExit =
374 CodeGenOpts.RegisterGlobalDtorsWithAtExit;
375 Options.DisableIntegratedAS = CodeGenOpts.DisableIntegratedAS;
376 Options.CompressDebugSections = CodeGenOpts.getCompressDebugSections();
377 Options.RelaxELFRelocations = CodeGenOpts.RelaxELFRelocations;
383 Options.ExceptionModel = llvm::ExceptionHandling::SjLj;
385 Options.ExceptionModel = llvm::ExceptionHandling::WinEH;
387 Options.ExceptionModel = llvm::ExceptionHandling::DwarfCFI;
389 Options.ExceptionModel = llvm::ExceptionHandling::Wasm;
391 Options.NoInfsFPMath = LangOpts.NoHonorInfs;
392 Options.NoNaNsFPMath = LangOpts.NoHonorNaNs;
393 Options.NoZerosInBSS = CodeGenOpts.NoZeroInitializedInBSS;
394 Options.UnsafeFPMath = LangOpts.UnsafeFPMath;
395 Options.ApproxFuncFPMath = LangOpts.ApproxFunc;
398 llvm::StringSwitch<llvm::BasicBlockSection>(CodeGenOpts.
BBSections)
400 .Case(
"labels", llvm::BasicBlockSection::Labels)
401 .StartsWith(
"list=", llvm::BasicBlockSection::List)
405 if (Options.BBSections == llvm::BasicBlockSection::List) {
406 ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr =
407 MemoryBuffer::getFile(CodeGenOpts.
BBSections.substr(5));
409 Diags.
Report(diag::err_fe_unable_to_load_basic_block_sections_file)
410 << MBOrErr.getError().message();
413 Options.BBSectionsFuncListBuf = std::move(*MBOrErr);
416 Options.EnableMachineFunctionSplitter = CodeGenOpts.SplitMachineFunctions;
417 Options.FunctionSections = CodeGenOpts.FunctionSections;
418 Options.DataSections = CodeGenOpts.DataSections;
419 Options.IgnoreXCOFFVisibility = LangOpts.IgnoreXCOFFVisibility;
420 Options.UniqueSectionNames = CodeGenOpts.UniqueSectionNames;
421 Options.UniqueBasicBlockSectionNames =
422 CodeGenOpts.UniqueBasicBlockSectionNames;
423 Options.TLSSize = CodeGenOpts.TLSSize;
424 Options.EmulatedTLS = CodeGenOpts.EmulatedTLS;
425 Options.ExplicitEmulatedTLS = CodeGenOpts.ExplicitEmulatedTLS;
426 Options.DebuggerTuning = CodeGenOpts.getDebuggerTuning();
427 Options.EmitStackSizeSection = CodeGenOpts.StackSizeSection;
429 Options.EmitAddrsig = CodeGenOpts.Addrsig;
430 Options.ForceDwarfFrameSection = CodeGenOpts.ForceDwarfFrameSection;
431 Options.EmitCallSiteInfo = CodeGenOpts.EmitCallSiteInfo;
432 Options.EnableAIXExtendedAltivecABI = CodeGenOpts.EnableAIXExtendedAltivecABI;
433 Options.XRayOmitFunctionIndex = CodeGenOpts.XRayOmitFunctionIndex;
434 Options.LoopAlignment = CodeGenOpts.LoopAlignment;
435 Options.DebugStrictDwarf = CodeGenOpts.DebugStrictDwarf;
437 Options.Hotpatch = CodeGenOpts.HotPatch;
438 Options.JMCInstrument = CodeGenOpts.JMCInstrument;
440 switch (CodeGenOpts.getSwiftAsyncFramePointer()) {
442 Options.SwiftAsyncFramePointer =
443 SwiftAsyncFramePointerMode::DeploymentBased;
447 Options.SwiftAsyncFramePointer = SwiftAsyncFramePointerMode::Always;
451 Options.SwiftAsyncFramePointer = SwiftAsyncFramePointerMode::Never;
456 Options.MCOptions.EmitDwarfUnwind = CodeGenOpts.getEmitDwarfUnwind();
457 Options.MCOptions.MCRelaxAll = CodeGenOpts.RelaxAll;
458 Options.MCOptions.MCSaveTempLabels = CodeGenOpts.SaveTempLabels;
459 Options.MCOptions.MCUseDwarfDirectory =
460 CodeGenOpts.NoDwarfDirectoryAsm
461 ? llvm::MCTargetOptions::DisableDwarfDirectory
462 : llvm::MCTargetOptions::EnableDwarfDirectory;
463 Options.MCOptions.MCNoExecStack = CodeGenOpts.NoExecStack;
464 Options.MCOptions.MCIncrementalLinkerCompatible =
465 CodeGenOpts.IncrementalLinkerCompatible;
466 Options.MCOptions.MCFatalWarnings = CodeGenOpts.FatalWarnings;
467 Options.MCOptions.MCNoWarn = CodeGenOpts.NoWarn;
468 Options.MCOptions.AsmVerbose = CodeGenOpts.AsmVerbose;
469 Options.MCOptions.Dwarf64 = CodeGenOpts.Dwarf64;
470 Options.MCOptions.PreserveAsmComments = CodeGenOpts.PreserveAsmComments;
471 Options.MCOptions.ABIName = TargetOpts.
ABI;
473 if (!Entry.IsFramework &&
477 Options.MCOptions.IASSearchPaths.push_back(
478 Entry.IgnoreSysRoot ? Entry.Path : HSOpts.
Sysroot + Entry.Path);
479 Options.MCOptions.Argv0 = CodeGenOpts.
Argv0;
481 Options.MisExpect = CodeGenOpts.MisExpect;
488 if (!CodeGenOpts.EmitGcovArcs && !CodeGenOpts.EmitGcovNotes)
493 Options.EmitNotes = CodeGenOpts.EmitGcovNotes;
494 Options.EmitData = CodeGenOpts.EmitGcovArcs;
496 Options.NoRedZone = CodeGenOpts.DisableRedZone;
499 Options.Atomic = CodeGenOpts.AtomicProfileUpdate;
508 InstrProfOptions Options;
509 Options.NoRedZone = CodeGenOpts.DisableRedZone;
511 Options.Atomic = CodeGenOpts.AtomicProfileUpdate;
517 BackendArgs.push_back(
"clang");
519 BackendArgs.push_back(
"-debug-pass");
520 BackendArgs.push_back(CodeGenOpts.
DebugPass.c_str());
523 BackendArgs.push_back(
"-limit-float-precision");
529 if (BackendArgs.size() == 1)
531 BackendArgs.push_back(
nullptr);
535 llvm::cl::ParseCommandLineOptions(BackendArgs.size() - 1,
539 void EmitAssemblyHelper::CreateTargetMachine(
bool MustCreateTM) {
543 const llvm::Target *TheTarget = TargetRegistry::lookupTarget(Triple,
Error);
546 Diags.
Report(diag::err_fe_unable_to_create_target) <<
Error;
556 llvm::TargetOptions Options;
560 TM.reset(TheTarget->createTargetMachine(Triple, TargetOpts.
CPU, FeaturesStr,
561 Options, RM, CM, OptLevel));
564 bool EmitAssemblyHelper::AddEmitPasses(legacy::PassManager &CodeGenPasses,
566 raw_pwrite_stream &OS,
567 raw_pwrite_stream *DwoOS) {
569 std::unique_ptr<TargetLibraryInfoImpl> TLII(
571 CodeGenPasses.add(
new TargetLibraryInfoWrapperPass(*TLII));
580 if (CodeGenOpts.OptimizationLevel > 0)
581 CodeGenPasses.add(createObjCARCContractPass());
583 if (TM->addPassesToEmitFile(CodeGenPasses, OS, DwoOS, CGFT,
584 !CodeGenOpts.VerifyModule)) {
585 Diags.
Report(diag::err_fe_unable_to_interface_with_target);
593 switch (Opts.OptimizationLevel) {
595 llvm_unreachable(
"Invalid optimization level!");
598 return OptimizationLevel::O0;
601 return OptimizationLevel::O1;
604 switch (Opts.OptimizeSize) {
606 llvm_unreachable(
"Invalid optimization level for size!");
609 return OptimizationLevel::O2;
612 return OptimizationLevel::Os;
615 return OptimizationLevel::Oz;
619 return OptimizationLevel::O3;
626 PB.registerOptimizerLastEPCallback([&](ModulePassManager &MPM,
627 OptimizationLevel
Level) {
629 auto SancovOpts = getSancovOptsFromCGOpts(CodeGenOpts);
630 MPM.addPass(ModuleSanitizerCoveragePass(
631 SancovOpts, CodeGenOpts.SanitizeCoverageAllowlistFiles,
632 CodeGenOpts.SanitizeCoverageIgnorelistFiles));
635 auto MSanPass = [&](
SanitizerMask Mask,
bool CompileKernel) {
636 if (LangOpts.Sanitize.has(Mask)) {
637 int TrackOrigins = CodeGenOpts.SanitizeMemoryTrackOrigins;
638 bool Recover = CodeGenOpts.SanitizeRecover.has(Mask);
640 MemorySanitizerOptions options(TrackOrigins, Recover, CompileKernel,
641 CodeGenOpts.SanitizeMemoryParamRetval);
642 MPM.addPass(ModuleMemorySanitizerPass(options));
643 FunctionPassManager FPM;
644 FPM.addPass(MemorySanitizerPass(options));
645 if (Level != OptimizationLevel::O0) {
650 FPM.addPass(EarlyCSEPass());
657 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
660 MSanPass(SanitizerKind::Memory,
false);
661 MSanPass(SanitizerKind::KernelMemory,
true);
663 if (LangOpts.
Sanitize.
has(SanitizerKind::Thread)) {
664 MPM.addPass(ModuleThreadSanitizerPass());
665 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
668 auto ASanPass = [&](
SanitizerMask Mask,
bool CompileKernel) {
671 bool UseOdrIndicator = CodeGenOpts.SanitizeAddressUseOdrIndicator;
672 llvm::AsanDtorKind DestructorKind =
673 CodeGenOpts.getSanitizeAddressDtor();
674 AddressSanitizerOptions Opts;
675 Opts.CompileKernel = CompileKernel;
677 Opts.UseAfterScope = CodeGenOpts.SanitizeAddressUseAfterScope;
678 Opts.UseAfterReturn = CodeGenOpts.getSanitizeAddressUseAfterReturn();
679 MPM.addPass(ModuleAddressSanitizerPass(
680 Opts, UseGlobalGC, UseOdrIndicator, DestructorKind));
683 ASanPass(SanitizerKind::Address,
false);
684 ASanPass(SanitizerKind::KernelAddress,
true);
686 auto HWASanPass = [&](
SanitizerMask Mask,
bool CompileKernel) {
689 MPM.addPass(HWAddressSanitizerPass(
690 {CompileKernel, Recover,
691 CodeGenOpts.OptimizationLevel == 0}));
694 HWASanPass(SanitizerKind::HWAddress,
false);
695 HWASanPass(SanitizerKind::KernelHWAddress,
true);
697 if (LangOpts.
Sanitize.
has(SanitizerKind::DataFlow)) {
703 void EmitAssemblyHelper::RunOptimizationPipeline(
704 BackendAction Action, std::unique_ptr<raw_pwrite_stream> &OS,
705 std::unique_ptr<llvm::ToolOutputFile> &ThinLinkOS) {
711 ? getDefaultProfileGenName()
713 "",
"", PGOOptions::IRInstr, PGOOptions::NoCSAction,
714 CodeGenOpts.DebugInfoForProfiling);
718 : PGOOptions::NoCSAction;
721 CSAction, CodeGenOpts.DebugInfoForProfiling);
726 PGOOptions::SampleUse, PGOOptions::NoCSAction,
727 CodeGenOpts.DebugInfoForProfiling, CodeGenOpts.PseudoProbeForProfiling);
728 else if (CodeGenOpts.PseudoProbeForProfiling)
731 PGOOptions(
"",
"",
"", PGOOptions::NoAction, PGOOptions::NoCSAction,
732 CodeGenOpts.DebugInfoForProfiling,
true);
733 else if (CodeGenOpts.DebugInfoForProfiling)
735 PGOOpt = PGOOptions(
"",
"",
"", PGOOptions::NoAction,
736 PGOOptions::NoCSAction,
true);
741 "Cannot have both CSProfileUse pass and CSProfileGen pass at "
744 assert(PGOOpt->Action != PGOOptions::IRInstr &&
745 PGOOpt->Action != PGOOptions::SampleUse &&
746 "Cannot run CSProfileGen pass with ProfileGen or SampleUse "
749 ? getDefaultProfileGenName()
750 : CodeGenOpts.InstrProfileOutput;
751 PGOOpt->CSAction = PGOOptions::CSIRInstr;
753 PGOOpt = PGOOptions(
"",
755 ? getDefaultProfileGenName()
757 "", PGOOptions::NoAction, PGOOptions::CSIRInstr,
758 CodeGenOpts.DebugInfoForProfiling);
761 TM->setPGOOption(PGOOpt);
763 PipelineTuningOptions PTO;
764 PTO.LoopUnrolling = CodeGenOpts.UnrollLoops;
767 PTO.LoopInterleaving = CodeGenOpts.UnrollLoops;
768 PTO.LoopVectorization = CodeGenOpts.VectorizeLoop;
769 PTO.SLPVectorization = CodeGenOpts.VectorizeSLP;
770 PTO.MergeFunctions = CodeGenOpts.MergeFunctions;
773 PTO.CallGraphProfile = !CodeGenOpts.DisableIntegratedAS;
775 LoopAnalysisManager LAM;
776 FunctionAnalysisManager FAM;
777 CGSCCAnalysisManager CGAM;
778 ModuleAnalysisManager MAM;
780 bool DebugPassStructure = CodeGenOpts.
DebugPass ==
"Structure";
781 PassInstrumentationCallbacks PIC;
782 PrintPassOptions PrintPassOpts;
783 PrintPassOpts.Indent = DebugPassStructure;
784 PrintPassOpts.SkipAnalyses = DebugPassStructure;
785 StandardInstrumentations SI(CodeGenOpts.DebugPassManager ||
787 false, PrintPassOpts);
788 SI.registerCallbacks(PIC, &FAM);
789 PassBuilder PB(TM.get(), PTO, PGOOpt, &PIC);
795 PassPlugin->registerPassBuilderCallbacks(PB);
797 Diags.
Report(diag::err_fe_unable_to_load_plugin)
798 << PluginFN <<
toString(PassPlugin.takeError());
801 #define HANDLE_EXTENSION(Ext) \
802 get##Ext##PluginInfo().RegisterPassBuilderCallbacks(PB);
803 #include "llvm/Support/Extension.def"
807 std::unique_ptr<TargetLibraryInfoImpl> TLII(
809 FAM.registerPass([&] {
return TargetLibraryAnalysis(*TLII); });
812 PB.registerModuleAnalyses(MAM);
813 PB.registerCGSCCAnalyses(CGAM);
814 PB.registerFunctionAnalyses(FAM);
815 PB.registerLoopAnalyses(LAM);
816 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
818 ModulePassManager MPM;
820 if (!CodeGenOpts.DisableLLVMPasses) {
825 bool IsThinLTO = CodeGenOpts.PrepareForThinLTO;
826 bool IsLTO = CodeGenOpts.PrepareForLTO;
828 if (LangOpts.ObjCAutoRefCount) {
829 PB.registerPipelineStartEPCallback(
830 [](ModulePassManager &MPM, OptimizationLevel
Level) {
831 if (
Level != OptimizationLevel::O0)
833 createModuleToFunctionPassAdaptor(ObjCARCExpandPass()));
835 PB.registerPipelineEarlySimplificationEPCallback(
836 [](ModulePassManager &MPM, OptimizationLevel
Level) {
837 if (
Level != OptimizationLevel::O0)
838 MPM.addPass(ObjCARCAPElimPass());
840 PB.registerScalarOptimizerLateEPCallback(
841 [](FunctionPassManager &FPM, OptimizationLevel
Level) {
842 if (
Level != OptimizationLevel::O0)
843 FPM.addPass(ObjCARCOptPass());
853 if (IsThinLTOPostLink)
854 PB.registerPipelineStartEPCallback(
855 [](ModulePassManager &MPM, OptimizationLevel
Level) {
856 MPM.addPass(LowerTypeTestsPass(
nullptr,
861 if (CodeGenOpts.InstrumentFunctions ||
862 CodeGenOpts.InstrumentFunctionEntryBare ||
863 CodeGenOpts.InstrumentFunctionsAfterInlining ||
864 CodeGenOpts.InstrumentForProfiling) {
865 PB.registerPipelineStartEPCallback(
866 [](ModulePassManager &MPM, OptimizationLevel
Level) {
867 MPM.addPass(createModuleToFunctionPassAdaptor(
868 EntryExitInstrumenterPass(
false)));
870 PB.registerOptimizerLastEPCallback(
871 [](ModulePassManager &MPM, OptimizationLevel
Level) {
872 MPM.addPass(createModuleToFunctionPassAdaptor(
873 EntryExitInstrumenterPass(
true)));
879 if (LangOpts.
Sanitize.
has(SanitizerKind::LocalBounds))
880 PB.registerScalarOptimizerLateEPCallback(
881 [](FunctionPassManager &FPM, OptimizationLevel
Level) {
882 FPM.addPass(BoundsCheckingPass());
887 if (!IsThinLTOPostLink)
891 PB.registerPipelineStartEPCallback(
892 [Options](ModulePassManager &MPM, OptimizationLevel
Level) {
893 MPM.addPass(GCOVProfilerPass(*Options));
897 PB.registerPipelineStartEPCallback(
898 [Options](ModulePassManager &MPM, OptimizationLevel
Level) {
899 MPM.addPass(InstrProfiling(*Options,
false));
902 if (CodeGenOpts.OptimizationLevel == 0) {
903 MPM = PB.buildO0DefaultPipeline(
Level, IsLTO || IsThinLTO);
904 }
else if (IsThinLTO) {
905 MPM = PB.buildThinLTOPreLinkDefaultPipeline(
Level);
907 MPM = PB.buildLTOPreLinkDefaultPipeline(
Level);
909 MPM = PB.buildPerModuleDefaultPipeline(
Level);
913 MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
914 MPM.addPass(ModuleMemProfilerPass());
922 MPM.addPass(VerifierPass());
926 if (CodeGenOpts.PrepareForThinLTO && !CodeGenOpts.DisableLLVMPasses) {
932 if (!TheModule->getModuleFlag(
"EnableSplitLTOUnit"))
933 TheModule->addModuleFlag(
Module::Error,
"EnableSplitLTOUnit",
934 CodeGenOpts.EnableSplitLTOUnit);
935 MPM.addPass(ThinLTOBitcodeWriterPass(*OS, ThinLinkOS ? &ThinLinkOS->os()
940 bool EmitLTOSummary = shouldEmitRegularLTOSummary();
941 if (EmitLTOSummary) {
942 if (!TheModule->getModuleFlag(
"ThinLTO"))
943 TheModule->addModuleFlag(
Module::Error,
"ThinLTO", uint32_t(0));
944 if (!TheModule->getModuleFlag(
"EnableSplitLTOUnit"))
945 TheModule->addModuleFlag(
Module::Error,
"EnableSplitLTOUnit",
949 BitcodeWriterPass(*OS, CodeGenOpts.EmitLLVMUseLists, EmitLTOSummary));
954 MPM.addPass(PrintModulePass(*OS,
"", CodeGenOpts.EmitLLVMUseLists));
963 PrettyStackTraceString CrashInfo(
"Optimizer");
964 llvm::TimeTraceScope TimeScope(
"Optimizer");
965 MPM.run(*TheModule, MAM);
969 void EmitAssemblyHelper::RunCodegenPipeline(
970 BackendAction Action, std::unique_ptr<raw_pwrite_stream> &OS,
971 std::unique_ptr<llvm::ToolOutputFile> &DwoOS) {
975 legacy::PassManager CodeGenPasses;
983 createTargetTransformInfoWrapperPass(getTargetIRAnalysis()));
989 if (!AddEmitPasses(CodeGenPasses, Action, *OS,
990 DwoOS ? &DwoOS->os() :
nullptr))
999 PrettyStackTraceString CrashInfo(
"Code generation");
1000 llvm::TimeTraceScope TimeScope(
"CodeGenPasses");
1001 CodeGenPasses.run(*TheModule);
1006 std::unique_ptr<raw_pwrite_stream> OS) {
1007 TimeRegion Region(CodeGenOpts.TimePasses ? &CodeGenerationTime :
nullptr);
1011 CreateTargetMachine(RequiresCodeGen);
1013 if (RequiresCodeGen && !TM)
1016 TheModule->setDataLayout(TM->createDataLayout());
1019 cl::PrintOptionValues();
1021 std::unique_ptr<llvm::ToolOutputFile> ThinLinkOS, DwoOS;
1022 RunOptimizationPipeline(Action, OS, ThinLinkOS);
1023 RunCodegenPipeline(Action, OS, DwoOS);
1035 std::unique_ptr<raw_pwrite_stream> OS,
std::string SampleProfile,
1037 StringMap<DenseMap<GlobalValue::GUID, GlobalValueSummary *>>
1038 ModuleToDefinedGVSummaries;
1039 CombinedIndex->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
1046 FunctionImporter::ImportMapTy ImportList;
1047 if (!lto::initImportList(*M, *CombinedIndex, ImportList))
1050 auto AddStream = [&](
size_t Task) {
1051 return std::make_unique<CachedFileStream>(std::move(OS),
1058 handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) {
1059 errs() <<
"Error setting up ThinLTO save-temps: " << EIB.message()
1064 Conf.CPU = TOpts.
CPU;
1069 Conf.OptLevel = CGOpts.OptimizationLevel;
1071 Conf.SampleProfile = std::move(SampleProfile);
1072 Conf.PTO.LoopUnrolling = CGOpts.UnrollLoops;
1075 Conf.PTO.LoopInterleaving = CGOpts.UnrollLoops;
1076 Conf.PTO.LoopVectorization = CGOpts.VectorizeLoop;
1077 Conf.PTO.SLPVectorization = CGOpts.VectorizeSLP;
1080 Conf.PTO.CallGraphProfile = !CGOpts.DisableIntegratedAS;
1084 Conf.RunCSIRInstr =
true;
1087 Conf.RunCSIRInstr =
false;
1091 Conf.ProfileRemapping = std::move(ProfileRemapping);
1092 Conf.DebugPassManager = CGOpts.DebugPassManager;
1093 Conf.RemarksWithHotness = CGOpts.DiagnosticsWithHotness;
1101 Conf.PreCodeGenModuleHook = [](
size_t Task,
const Module &Mod) {
1106 Conf.PreCodeGenModuleHook = [&](
size_t Task,
const Module &Mod) {
1107 M->
print(*OS,
nullptr, CGOpts.EmitLLVMUseLists);
1112 Conf.PreCodeGenModuleHook = [&](
size_t Task,
const Module &Mod) {
1113 WriteBitcodeToFile(*M, *OS, CGOpts.EmitLLVMUseLists);
1122 thinBackend(Conf, -1, AddStream, *M, *CombinedIndex, ImportList,
1123 ModuleToDefinedGVSummaries[M->getModuleIdentifier()],
1125 handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) {
1126 errs() <<
"Error running ThinLTO backend: " << EIB.message() <<
'\n';
1136 StringRef TDesc,
Module *M,
1138 std::unique_ptr<raw_pwrite_stream> OS) {
1140 llvm::TimeTraceScope TimeScope(
"Backend");
1142 std::unique_ptr<llvm::Module> EmptyModule;
1147 std::unique_ptr<ModuleSummaryIndex> CombinedIndex;
1148 if (
Error E = llvm::getModuleSummaryIndexForFile(
1151 .moveInto(CombinedIndex)) {
1152 logAllUnhandledErrors(std::move(E), errs(),
1153 "Error loading index file '" +
1161 if (CombinedIndex) {
1162 if (!CombinedIndex->skipModuleByDistributedBackend()) {
1174 EmptyModule = std::make_unique<llvm::Module>(
"empty", M->getContext());
1175 EmptyModule->setTargetTriple(M->getTargetTriple());
1176 M = EmptyModule.get();
1180 EmitAssemblyHelper AsmHelper(Diags, HeaderOpts, CGOpts, TOpts, LOpts, M);
1181 AsmHelper.EmitAssembly(Action, std::move(OS));
1186 std::string DLDesc = M->getDataLayout().getStringRepresentation();
1187 if (DLDesc != TDesc) {
1190 "expected target description '%1'");
1191 Diags.
Report(DiagID) << DLDesc << TDesc;
1199 llvm::MemoryBufferRef Buf) {
1202 llvm::embedBitcodeInModule(
1214 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> ObjectOrErr =
1215 llvm::MemoryBuffer::getFileOrSTDIN(OffloadObject);
1216 if (std::error_code EC = ObjectOrErr.getError()) {
1218 "could not open '%0' for embedding");
1219 Diags.
Report(DiagID) << OffloadObject;
1223 llvm::embedBufferInModule(*M, **ObjectOrErr,
".llvm.offloading",
1224 Align(object::OffloadBinary::getAlignment()));