clang  15.0.0git
DependencyScanningWorker.cpp
Go to the documentation of this file.
1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
15 #include "clang/Frontend/Utils.h"
19 #include "clang/Tooling/Tooling.h"
20 
21 using namespace clang;
22 using namespace tooling;
23 using namespace dependencies;
24 
25 namespace {
26 
27 /// Forwards the gatherered dependencies to the consumer.
28 class DependencyConsumerForwarder : public DependencyFileGenerator {
29 public:
30  DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
31  DependencyConsumer &C)
32  : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {}
33 
34  void finishedMainFile(DiagnosticsEngine &Diags) override {
35  C.handleDependencyOutputOpts(*Opts);
36  llvm::SmallString<256> CanonPath;
37  for (const auto &File : getDependencies()) {
38  CanonPath = File;
39  llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
40  C.handleFileDependency(CanonPath);
41  }
42  }
43 
44 private:
45  std::unique_ptr<DependencyOutputOptions> Opts;
46  DependencyConsumer &C;
47 };
48 
49 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
50 
51 /// A listener that collects the imported modules and optionally the input
52 /// files.
53 class PrebuiltModuleListener : public ASTReaderListener {
54 public:
55  PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
56  llvm::StringSet<> &InputFiles, bool VisitInputFiles,
57  llvm::SmallVector<std::string> &NewModuleFiles)
58  : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
59  VisitInputFiles(VisitInputFiles), NewModuleFiles(NewModuleFiles) {}
60 
61  bool needsImportVisitation() const override { return true; }
62  bool needsInputFileVisitation() override { return VisitInputFiles; }
63  bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
64 
65  void visitImport(StringRef ModuleName, StringRef Filename) override {
66  if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
67  NewModuleFiles.push_back(Filename.str());
68  }
69 
70  bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
71  bool isExplicitModule) override {
72  InputFiles.insert(Filename);
73  return true;
74  }
75 
76 private:
77  PrebuiltModuleFilesT &PrebuiltModuleFiles;
78  llvm::StringSet<> &InputFiles;
79  bool VisitInputFiles;
80  llvm::SmallVector<std::string> &NewModuleFiles;
81 };
82 
83 /// Visit the given prebuilt module and collect all of the modules it
84 /// transitively imports and contributing input files.
85 static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
86  CompilerInstance &CI,
87  PrebuiltModuleFilesT &ModuleFiles,
88  llvm::StringSet<> &InputFiles,
89  bool VisitInputFiles) {
90  // List of module files to be processed.
91  llvm::SmallVector<std::string> Worklist{PrebuiltModuleFilename.str()};
92  PrebuiltModuleListener Listener(ModuleFiles, InputFiles, VisitInputFiles,
93  Worklist);
94 
95  while (!Worklist.empty())
97  Worklist.pop_back_val(), CI.getFileManager(),
99  /*FindModuleFileExtensions=*/false, Listener,
100  /*ValidateDiagnosticOptions=*/false);
101 }
102 
103 /// Transform arbitrary file name into an object-like file name.
104 static std::string makeObjFileName(StringRef FileName) {
105  SmallString<128> ObjFileName(FileName);
106  llvm::sys::path::replace_extension(ObjFileName, "o");
107  return std::string(ObjFileName.str());
108 }
109 
110 /// Deduce the dependency target based on the output file and input files.
111 static std::string
112 deduceDepTarget(const std::string &OutputFile,
113  const SmallVectorImpl<FrontendInputFile> &InputFiles) {
114  if (OutputFile != "-")
115  return OutputFile;
116 
117  if (InputFiles.empty() || !InputFiles.front().isFile())
118  return "clang-scan-deps\\ dependency";
119 
120  return makeObjFileName(InputFiles.front().getFile());
121 }
122 
123 /// Sanitize diagnostic options for dependency scan.
124 static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
125  // Don't print 'X warnings and Y errors generated'.
126  DiagOpts.ShowCarets = false;
127  // Don't write out diagnostic file.
128  DiagOpts.DiagnosticSerializationFile.clear();
129  // Don't treat warnings as errors.
130  DiagOpts.Warnings.push_back("no-error");
131 }
132 
133 /// A clang tool that runs the preprocessor in a mode that's optimized for
134 /// dependency scanning for the given compiler invocation.
135 class DependencyScanningAction : public tooling::ToolAction {
136 public:
137  DependencyScanningAction(
138  StringRef WorkingDirectory, DependencyConsumer &Consumer,
141  ScanningOutputFormat Format, bool OptimizeArgs,
142  llvm::Optional<StringRef> ModuleName = None)
143  : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
144  DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings), Format(Format),
145  OptimizeArgs(OptimizeArgs), ModuleName(ModuleName) {}
146 
147  bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
148  FileManager *FileMgr,
149  std::shared_ptr<PCHContainerOperations> PCHContainerOps,
150  DiagnosticConsumer *DiagConsumer) override {
151  // Make a deep copy of the original Clang invocation.
152  CompilerInvocation OriginalInvocation(*Invocation);
153 
154  // Create a compiler instance to handle the actual work.
155  CompilerInstance ScanInstance(std::move(PCHContainerOps));
156  ScanInstance.setInvocation(std::move(Invocation));
157 
158  // Create the compiler's actual diagnostics engine.
159  sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
160  ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
161  if (!ScanInstance.hasDiagnostics())
162  return false;
163 
164  ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
165  true;
166 
167  ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
168  ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
169 
170  FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
171  ScanInstance.setFileManager(FileMgr);
172  ScanInstance.createSourceManager(*FileMgr);
173 
174  llvm::StringSet<> PrebuiltModulesInputFiles;
175  // Store the list of prebuilt module files into header search options. This
176  // will prevent the implicit build to create duplicate modules and will
177  // force reuse of the existing prebuilt module files instead.
178  if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
179  visitPrebuiltModule(
180  ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
181  ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
182  PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
183 
184  // Use the dependency scanning optimized file system if requested to do so.
185  if (DepFS) {
186  DepFS->enableMinimizationOfAllFiles();
187  // Don't minimize any files that contributed to prebuilt modules. The
188  // implicit build validates the modules by comparing the reported sizes of
189  // their inputs to the current state of the filesystem. Minimization would
190  // throw this mechanism off.
191  for (const auto &File : PrebuiltModulesInputFiles)
192  DepFS->disableMinimization(File.getKey());
193  // Don't minimize any files that were explicitly passed in the build
194  // settings and that might be opened.
195  for (const auto &E : ScanInstance.getHeaderSearchOpts().UserEntries)
196  DepFS->disableMinimization(E.Path);
197  for (const auto &F : ScanInstance.getHeaderSearchOpts().VFSOverlayFiles)
198  DepFS->disableMinimization(F);
199 
200  // Support for virtual file system overlays on top of the caching
201  // filesystem.
203  ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS));
204 
205  // Pass the skip mappings which should speed up excluded conditional block
206  // skipping in the preprocessor.
207  ScanInstance.getPreprocessorOpts()
208  .ExcludedConditionalDirectiveSkipMappings = &PPSkipMappings;
209  }
210 
211  // Create the dependency collector that will collect the produced
212  // dependencies.
213  //
214  // This also moves the existing dependency output options from the
215  // invocation to the collector. The options in the invocation are reset,
216  // which ensures that the compiler won't create new dependency collectors,
217  // and thus won't write out the extra '.d' files to disk.
218  auto Opts = std::make_unique<DependencyOutputOptions>();
219  std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
220  // We need at least one -MT equivalent for the generator of make dependency
221  // files to work.
222  if (Opts->Targets.empty())
223  Opts->Targets = {
224  deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
225  ScanInstance.getFrontendOpts().Inputs)};
226  Opts->IncludeSystemHeaders = true;
227 
228  switch (Format) {
230  ScanInstance.addDependencyCollector(
231  std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
232  Consumer));
233  break;
235  ScanInstance.addDependencyCollector(std::make_shared<ModuleDepCollector>(
236  std::move(Opts), ScanInstance, Consumer,
237  std::move(OriginalInvocation), OptimizeArgs));
238  break;
239  }
240 
241  // Consider different header search and diagnostic options to create
242  // different modules. This avoids the unsound aliasing of module PCMs.
243  //
244  // TODO: Implement diagnostic bucketing to reduce the impact of strict
245  // context hashing.
246  ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
247 
248  std::unique_ptr<FrontendAction> Action;
249 
250  if (ModuleName.hasValue())
251  Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
252  else
253  Action = std::make_unique<ReadPCHAndPreprocessAction>();
254 
255  const bool Result = ScanInstance.ExecuteAction(*Action);
256  if (!DepFS)
257  FileMgr->clearStatCache();
258  return Result;
259  }
260 
261 private:
262  StringRef WorkingDirectory;
263  DependencyConsumer &Consumer;
266  ScanningOutputFormat Format;
267  bool OptimizeArgs;
268  llvm::Optional<StringRef> ModuleName;
269 };
270 
271 } // end anonymous namespace
272 
274  DependencyScanningService &Service)
275  : Format(Service.getFormat()), OptimizeArgs(Service.canOptimizeArgs()) {
276  PCHContainerOps = std::make_shared<PCHContainerOperations>();
277  PCHContainerOps->registerReader(
278  std::make_unique<ObjectFilePCHContainerReader>());
279  // We don't need to write object files, but the current PCH implementation
280  // requires the writer to be registered as well.
281  PCHContainerOps->registerWriter(
282  std::make_unique<ObjectFilePCHContainerWriter>());
283 
284  auto OverlayFS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
285  llvm::vfs::createPhysicalFileSystem());
286  InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
287  OverlayFS->pushOverlay(InMemoryFS);
288  RealFS = OverlayFS;
289 
292  RealFS, PPSkipMappings);
293  if (Service.canReuseFileManager())
294  Files = new FileManager(FileSystemOptions(), RealFS);
295 }
296 
297 static llvm::Error
299  llvm::function_ref<bool(DiagnosticConsumer &, DiagnosticOptions &)>
300  BodyShouldSucceed) {
301  sanitizeDiagOpts(*DiagOpts);
302 
303  // Capture the emitted diagnostics and report them to the client
304  // in the case of a failure.
305  std::string DiagnosticOutput;
306  llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
307  TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts);
308 
309  if (BodyShouldSucceed(DiagPrinter, *DiagOpts))
310  return llvm::Error::success();
311  return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
312  llvm::inconvertibleErrorCode());
313 }
314 
316  StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
317  DependencyConsumer &Consumer, llvm::Optional<StringRef> ModuleName) {
318  // Reset what might have been modified in the previous worker invocation.
319  RealFS->setCurrentWorkingDirectory(WorkingDirectory);
320  if (Files)
321  Files->setVirtualFileSystem(RealFS);
322 
324  Files ? Files : new FileManager(FileSystemOptions(), RealFS);
325 
326  Optional<std::vector<std::string>> ModifiedCommandLine;
327  if (ModuleName.hasValue()) {
328  ModifiedCommandLine = CommandLine;
329  InMemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer(""));
330  ModifiedCommandLine->emplace_back(*ModuleName);
331  }
332 
333  const std::vector<std::string> &FinalCommandLine =
334  ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
335 
336  std::vector<const char *> FinalCCommandLine(CommandLine.size(), nullptr);
337  llvm::transform(CommandLine, FinalCCommandLine.begin(),
338  [](const std::string &Str) { return Str.c_str(); });
339 
340  return runWithDiags(CreateAndPopulateDiagOpts(FinalCCommandLine).release(),
341  [&](DiagnosticConsumer &DC, DiagnosticOptions &DiagOpts) {
342  DependencyScanningAction Action(
343  WorkingDirectory, Consumer, DepFS, PPSkipMappings,
344  Format, OptimizeArgs, ModuleName);
345  // Create an invocation that uses the underlying file
346  // system to ensure that any file system requests that
347  // are made by the driver do not go through the
348  // dependency scanning filesystem.
349  ToolInvocation Invocation(FinalCommandLine, &Action,
350  CurrentFiles.get(),
351  PCHContainerOps);
352  Invocation.setDiagnosticConsumer(&DC);
353  Invocation.setDiagnosticOptions(&DiagOpts);
354  return Invocation.run();
355  });
356 }
DependencyScanningService.h
clang::tooling::ToolInvocation
Utility to run a FrontendAction in a single clang invocation.
Definition: Tooling.h:239
clang::DiagnosticOptions::DiagnosticSerializationFile
std::string DiagnosticSerializationFile
The file to serialize diagnostics to (non-appending).
Definition: DiagnosticOptions.h:107
clang::DeclaratorContext::File
@ File
string
string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} 0 ${PATH_LIB_START} PATH_HEAD) string(SUBSTRING $
Definition: CMakeLists.txt:22
clang::tooling::dependencies::DependencyConsumer
Definition: DependencyScanningWorker.h:32
llvm::SmallVector
Definition: LLVM.h:38
clang::FileManager::clearStatCache
void clearStatCache()
Removes the FileSystemStatCache object from the manager.
Definition: FileManager.cpp:68
clang::DiagnosticConsumer
Abstract interface, implemented by clients of the front-end, which formats and prints fully processed...
Definition: Diagnostic.h:1738
AttributeLangSupport::C
@ C
Definition: SemaDeclAttr.cpp:55
Filename
StringRef Filename
Definition: Format.cpp:2551
clang::DiagnosticsEngine
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
clang::tooling::ToolInvocation::setDiagnosticConsumer
void setDiagnosticConsumer(DiagnosticConsumer *DiagConsumer)
Set a DiagnosticConsumer to use during driver command-line parsing and the action invocation itself.
Definition: Tooling.h:272
llvm::Optional
Definition: LLVM.h:40
clang::tooling::dependencies::DependencyScanningService::canReuseFileManager
bool canReuseFileManager() const
Definition: DependencyScanningService.h:57
clang::ExcludedPreprocessorDirectiveSkipMapping
llvm::DenseMap< const char *, const PreprocessorSkippedRangeMapping * > ExcludedPreprocessorDirectiveSkipMapping
The datastructure that holds the mapping between the active memory buffers and the individual skip ma...
Definition: PreprocessorExcludedConditionalDirectiveSkipMapping.h:26
clang::FileSystemOptions::WorkingDir
std::string WorkingDir
If set, paths are resolved as if the working directory was set to the value of WorkingDir.
Definition: FileSystemOptions.h:26
clang::CompilerInstance::getPCHContainerReader
const PCHContainerReader & getPCHContainerReader() const
Return the appropriate PCHContainerReader depending on the current CodeGenOptions.
Definition: CompilerInstance.h:542
CompilerInvocation.h
clang::FileSystemOptions
Keeps track of options that affect how file operations are performed.
Definition: FileSystemOptions.h:22
clang::tooling::ToolAction
Interface to process a clang::CompilerInvocation.
Definition: Tooling.h:80
clang::CompilerInstance::getFileManager
FileManager & getFileManager() const
Return the current file manager to the caller.
Definition: CompilerInstance.h:402
clang::tooling::dependencies::ScanningOutputFormat
ScanningOutputFormat
The format that is output by the dependency scanner.
Definition: DependencyScanningService.h:34
clang::FileManager::setVirtualFileSystem
void setVirtualFileSystem(IntrusiveRefCntPtr< llvm::vfs::FileSystem > FS)
Definition: FileManager.h:247
Utils.h
llvm::SmallString
Definition: LLVM.h:37
clang::ASTReader::readASTFileControlBlock
static bool readASTFileControlBlock(StringRef Filename, FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions, ASTReaderListener &Listener, bool ValidateDiagnosticOptions)
Read the control block for the named AST file.
Definition: ASTReader.cpp:5181
DependencyScanningWorker.h
clang::tooling::ToolInvocation::setDiagnosticOptions
void setDiagnosticOptions(DiagnosticOptions *DiagOpts)
Set a DiagnosticOptions to use during driver command-line parsing.
Definition: Tooling.h:277
clang::tooling::dependencies::DependencyScanningWorker::computeDependencies
llvm::Error computeDependencies(StringRef WorkingDirectory, const std::vector< std::string > &CommandLine, DependencyConsumer &Consumer, llvm::Optional< StringRef > ModuleName=None)
Run the dependency scanning tool for a given clang driver command-line, and report the discovered dep...
Definition: DependencyScanningWorker.cpp:315
clang::CompilerInstance
CompilerInstance - Helper class for managing a single instance of the Clang compiler.
Definition: CompilerInstance.h:72
clang::HeaderSearchOptions::PrebuiltModuleFiles
std::map< std::string, std::string, std::less<> > PrebuiltModuleFiles
The mapping of module names to prebuilt module files.
Definition: HeaderSearchOptions.h:119
clang::FileManager::getFileSystemOpts
FileSystemOptions & getFileSystemOpts()
Returns the current file system options.
Definition: FileManager.h:242
clang::DependencyFileGenerator
Builds a dependency file when attached to a Preprocessor (for includes) and ASTReader (for module imp...
Definition: Utils.h:102
clang::tooling::dependencies::DependencyScanningWorker::DependencyScanningWorker
DependencyScanningWorker(DependencyScanningService &Service)
Definition: DependencyScanningWorker.cpp:273
runWithDiags
static llvm::Error runWithDiags(DiagnosticOptions *DiagOpts, llvm::function_ref< bool(DiagnosticConsumer &, DiagnosticOptions &)> BodyShouldSucceed)
Definition: DependencyScanningWorker.cpp:298
clang::tooling::dependencies::DependencyScanningService::getMode
ScanningMode getMode() const
Definition: DependencyScanningService.h:53
ModuleDepCollector.h
Tooling.h
clang::tooling::dependencies::ScanningMode::MinimizedSourcePreprocessing
@ MinimizedSourcePreprocessing
This mode is used to compute the dependencies by running the preprocessor over the source files that ...
clang::tooling::dependencies::DependencyScanningService::getSharedCache
DependencyScanningFilesystemSharedCache & getSharedCache()
Definition: DependencyScanningService.h:61
clang::DiagnosticOptions::Warnings
std::vector< std::string > Warnings
The list of -W...
Definition: DiagnosticOptions.h:111
TextDiagnosticPrinter.h
clang::tooling::ToolInvocation::run
bool run()
Run the clang invocation.
Definition: Tooling.cpp:341
FrontendActions.h
clang::tooling::dependencies::DependencyScanningService
The dependency scanning service contains the shared state that is used by the invidual dependency sca...
Definition: DependencyScanningService.h:47
clang::createVFSFromCompilerInvocation
IntrusiveRefCntPtr< llvm::vfs::FileSystem > createVFSFromCompilerInvocation(const CompilerInvocation &CI, DiagnosticsEngine &Diags)
Definition: CompilerInvocation.cpp:4661
clang::tooling::dependencies::DependencyScanningWorkerFilesystem
A virtual file system optimized for the dependency discovery.
Definition: DependencyScanningFilesystem.h:291
std
Definition: Format.h:4296
clang::tooling::dependencies::ScanningOutputFormat::Make
@ Make
This is the Makefile compatible dep format.
clang
Definition: CalledOnceCheck.h:17
clang::FileManager
Implements support for file system lookup, file system caching, and directory search management.
Definition: FileManager.h:53
clang::ASTReaderListener
Abstract interface for callback invocations by the ASTReader.
Definition: ASTReader.h:113
clang::CompilerInvocation
Helper class for holding the data necessary to invoke the compiler.
Definition: CompilerInvocation.h:193
clang::SrcMgr::isSystem
bool isSystem(CharacteristicKind CK)
Determine whether a file / directory characteristic is for system code.
Definition: SourceManager.h:88
clang::TextDiagnosticPrinter
Definition: TextDiagnosticPrinter.h:27
clang::CreateAndPopulateDiagOpts
std::unique_ptr< DiagnosticOptions > CreateAndPopulateDiagOpts(ArrayRef< const char * > Argv)
Definition: CompilerInvocation.cpp:2333
CompilerInstance.h
llvm::SmallVectorImpl
Definition: Randstruct.h:18
PreprocessorOptions.h
clang::tooling::dependencies::ScanningOutputFormat::Full
@ Full
This outputs the full module dependency graph suitable for use for explicitly building modules.
ObjectFilePCHContainerOperations.h
llvm::IntrusiveRefCntPtr
Definition: LLVM.h:47
clang::DiagnosticOptions
Options for controlling the compiler diagnostics engine.
Definition: DiagnosticOptions.h:70