clang  14.0.0git
DependencyScanningFilesystem.cpp
Go to the documentation of this file.
1 //===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
11 #include "llvm/Support/MemoryBuffer.h"
12 #include "llvm/Support/Threading.h"
13 
14 using namespace clang;
15 using namespace tooling;
16 using namespace dependencies;
17 
19  StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) {
20  // Load the file and its content from the file system.
21  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile =
22  FS.openFileForRead(Filename);
23  if (!MaybeFile)
24  return MaybeFile.getError();
25  llvm::ErrorOr<llvm::vfs::Status> Stat = (*MaybeFile)->status();
26  if (!Stat)
27  return Stat.getError();
28 
29  llvm::vfs::File &F = **MaybeFile;
30  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MaybeBuffer =
31  F.getBuffer(Stat->getName());
32  if (!MaybeBuffer)
33  return MaybeBuffer.getError();
34 
35  llvm::SmallString<1024> MinimizedFileContents;
36  // Minimize the file down to directives that might affect the dependencies.
37  const auto &Buffer = *MaybeBuffer;
39  if (!Minimize || minimizeSourceToDependencyDirectives(
40  Buffer->getBuffer(), MinimizedFileContents, Tokens)) {
41  // Use the original file unless requested otherwise, or
42  // if the minimization failed.
43  // FIXME: Propage the diagnostic if desired by the client.
44  CachedFileSystemEntry Result;
45  Result.MaybeStat = std::move(*Stat);
46  Result.Contents.reserve(Buffer->getBufferSize() + 1);
47  Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd());
48  // Implicitly null terminate the contents for Clang's lexer.
49  Result.Contents.push_back('\0');
50  Result.Contents.pop_back();
51  return Result;
52  }
53 
54  CachedFileSystemEntry Result;
55  size_t Size = MinimizedFileContents.size();
56  Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(),
57  Stat->getLastModificationTime(),
58  Stat->getUser(), Stat->getGroup(), Size,
59  Stat->getType(), Stat->getPermissions());
60  // The contents produced by the minimizer must be null terminated.
61  assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' &&
62  "not null terminated contents");
63  // Even though there's an implicit null terminator in the minimized contents,
64  // we want to temporarily make it explicit. This will ensure that the
65  // std::move will preserve it even if it needs to do a copy if the
66  // SmallString still has the small capacity.
67  MinimizedFileContents.push_back('\0');
68  Result.Contents = std::move(MinimizedFileContents);
69  // Now make the null terminator implicit again, so that Clang's lexer can find
70  // it right where the buffer ends.
71  Result.Contents.pop_back();
72 
73  // Compute the skipped PP ranges that speedup skipping over inactive
74  // preprocessor blocks.
76  SkippedRanges;
78  SkippedRanges);
80  for (const auto &Range : SkippedRanges) {
81  if (Range.Length < 16) {
82  // Ignore small ranges as non-profitable.
83  // FIXME: This is a heuristic, its worth investigating the tradeoffs
84  // when it should be applied.
85  continue;
86  }
87  Mapping[Range.Offset] = Range.Length;
88  }
89  Result.PPSkippedRangeMapping = std::move(Mapping);
90 
91  return Result;
92 }
93 
96  assert(Stat.isDirectory() && "not a directory!");
97  auto Result = CachedFileSystemEntry();
98  Result.MaybeStat = std::move(Stat);
99  return Result;
100 }
101 
102 DependencyScanningFilesystemSharedCache::SingleCache::SingleCache() {
103  // This heuristic was chosen using a empirical testing on a
104  // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
105  // sharding gives a performance edge by reducing the lock contention.
106  // FIXME: A better heuristic might also consider the OS to account for
107  // the different cost of lock contention on different OSes.
108  NumShards =
109  std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4);
110  CacheShards = std::make_unique<CacheShard[]>(NumShards);
111 }
112 
113 DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
114 DependencyScanningFilesystemSharedCache::SingleCache::get(StringRef Key) {
115  CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards];
116  std::unique_lock<std::mutex> LockGuard(Shard.CacheLock);
117  auto It = Shard.Cache.try_emplace(Key);
118  return It.first->getValue();
119 }
120 
121 DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
122 DependencyScanningFilesystemSharedCache::get(StringRef Key, bool Minimized) {
123  SingleCache &Cache = Minimized ? CacheMinimized : CacheOriginal;
124  return Cache.get(Key);
125 }
126 
127 /// Whitelist file extensions that should be minimized, treating no extension as
128 /// a source file that should be minimized.
129 ///
130 /// This is kinda hacky, it would be better if we knew what kind of file Clang
131 /// was expecting instead.
132 static bool shouldMinimizeBasedOnExtension(StringRef Filename) {
133  StringRef Ext = llvm::sys::path::extension(Filename);
134  if (Ext.empty())
135  return true; // C++ standard library
136  return llvm::StringSwitch<bool>(Ext)
137  .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true)
138  .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true)
139  .CasesLower(".m", ".mm", true)
140  .CasesLower(".i", ".ii", ".mi", ".mmi", true)
141  .CasesLower(".def", ".inc", true)
142  .Default(false);
143 }
144 
145 
146 static bool shouldCacheStatFailures(StringRef Filename) {
147  StringRef Ext = llvm::sys::path::extension(Filename);
148  if (Ext.empty())
149  return false; // This may be the module cache directory.
150  // Only cache stat failures on source files.
152 }
153 
155  StringRef RawFilename) {
157  llvm::sys::path::native(RawFilename, Filename);
158  NotToBeMinimized.insert(Filename);
159 }
160 
161 bool DependencyScanningWorkerFilesystem::shouldMinimize(StringRef RawFilename) {
162  if (!shouldMinimizeBasedOnExtension(RawFilename))
163  return false;
164 
166  llvm::sys::path::native(RawFilename, Filename);
167  return !NotToBeMinimized.contains(Filename);
168 }
169 
170 CachedFileSystemEntry DependencyScanningWorkerFilesystem::createFileSystemEntry(
171  llvm::ErrorOr<llvm::vfs::Status> &&MaybeStatus, StringRef Filename,
172  bool ShouldMinimize) {
173  if (!MaybeStatus)
174  return CachedFileSystemEntry(MaybeStatus.getError());
175 
176  if (MaybeStatus->isDirectory())
177  return CachedFileSystemEntry::createDirectoryEntry(std::move(*MaybeStatus));
178 
179  return CachedFileSystemEntry::createFileEntry(Filename, getUnderlyingFS(),
180  ShouldMinimize);
181 }
182 
183 llvm::ErrorOr<const CachedFileSystemEntry *>
184 DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
185  const StringRef Filename) {
186  bool ShouldMinimize = shouldMinimize(Filename);
187 
188  if (const auto *Entry = Cache.getCachedEntry(Filename, ShouldMinimize))
189  return Entry;
190 
191  // FIXME: Handle PCM/PCH files.
192  // FIXME: Handle module map files.
193 
194  DependencyScanningFilesystemSharedCache::SharedFileSystemEntry
195  &SharedCacheEntry = SharedCache.get(Filename, ShouldMinimize);
196  const CachedFileSystemEntry *Result;
197  {
198  std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock);
199  CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value;
200 
201  if (!CacheEntry.isValid()) {
202  auto MaybeStatus = getUnderlyingFS().status(Filename);
203  if (!MaybeStatus && !shouldCacheStatFailures(Filename))
204  // HACK: We need to always restat non source files if the stat fails.
205  // This is because Clang first looks up the module cache and module
206  // files before building them, and then looks for them again. If we
207  // cache the stat failure, it won't see them the second time.
208  return MaybeStatus.getError();
209  CacheEntry = createFileSystemEntry(std::move(MaybeStatus), Filename,
210  ShouldMinimize);
211  }
212 
213  Result = &CacheEntry;
214  }
215 
216  // Store the result in the local cache.
217  Cache.setCachedEntry(Filename, ShouldMinimize, Result);
218  return Result;
219 }
220 
221 llvm::ErrorOr<llvm::vfs::Status>
223  SmallString<256> OwnedFilename;
224  StringRef Filename = Path.toStringRef(OwnedFilename);
225  const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
226  getOrCreateFileSystemEntry(Filename);
227  if (!Result)
228  return Result.getError();
229  return (*Result)->getStatus();
230 }
231 
232 namespace {
233 
234 /// The VFS that is used by clang consumes the \c CachedFileSystemEntry using
235 /// this subclass.
236 class MinimizedVFSFile final : public llvm::vfs::File {
237 public:
238  MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
239  llvm::vfs::Status Stat)
240  : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
241 
242  static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
243  create(const CachedFileSystemEntry *Entry,
245 
246  llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
247 
248  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
249  getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
250  bool IsVolatile) override {
251  return std::move(Buffer);
252  }
253 
254  std::error_code close() override { return {}; }
255 
256 private:
257  std::unique_ptr<llvm::MemoryBuffer> Buffer;
258  llvm::vfs::Status Stat;
259 };
260 
261 } // end anonymous namespace
262 
263 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MinimizedVFSFile::create(
264  const CachedFileSystemEntry *Entry,
266  if (Entry->isDirectory())
267  return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
268  std::make_error_code(std::errc::is_a_directory));
269  llvm::ErrorOr<StringRef> Contents = Entry->getContents();
270  if (!Contents)
271  return Contents.getError();
272  auto Result = std::make_unique<MinimizedVFSFile>(
273  llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(),
274  /*RequiresNullTerminator=*/false),
275  *Entry->getStatus());
276  if (!Entry->getPPSkippedRangeMapping().empty() && PPSkipMappings)
277  (*PPSkipMappings)[Result->Buffer->getBufferStart()] =
278  &Entry->getPPSkippedRangeMapping();
279  return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
280  std::unique_ptr<llvm::vfs::File>(std::move(Result)));
281 }
282 
283 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
285  SmallString<256> OwnedFilename;
286  StringRef Filename = Path.toStringRef(OwnedFilename);
287 
288  const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
289  getOrCreateFileSystemEntry(Filename);
290  if (!Result)
291  return Result.getError();
292  return MinimizedVFSFile::create(Result.get(), PPSkipMappings);
293 }
clang::tooling::dependencies::CachedFileSystemEntry::createDirectoryEntry
static CachedFileSystemEntry createDirectoryEntry(llvm::vfs::Status &&Stat)
Create an entry that represents a directory on the filesystem.
Definition: DependencyScanningFilesystem.cpp:95
max
__DEVICE__ int max(int __a, int __b)
Definition: __clang_cuda_math.h:196
shouldCacheStatFailures
static bool shouldCacheStatFailures(StringRef Filename)
Definition: DependencyScanningFilesystem.cpp:146
clang::tooling::dependencies::CachedFileSystemEntry
An in-memory representation of a file system entity that is of interest to the dependency scanning fi...
Definition: DependencyScanningFilesystem.h:34
llvm::SmallVector
Definition: LLVM.h:38
clang::tooling::dependencies::DependencyScanningFilesystemSharedCache::get
SharedFileSystemEntry & get(StringRef Key, bool Minimized)
Returns a cache entry for the corresponding key.
Definition: DependencyScanningFilesystem.cpp:122
clang::minimizeSourceToDependencyDirectives
bool minimizeSourceToDependencyDirectives(llvm::StringRef Input, llvm::SmallVectorImpl< char > &Output, llvm::SmallVectorImpl< minimize_source_to_dependency_directives::Token > &Tokens, DiagnosticsEngine *Diags=nullptr, SourceLocation InputSourceLoc=SourceLocation())
Minimize the input down to the preprocessor directives that might have an effect on the dependencies ...
clang::tooling::dependencies::CachedFileSystemEntry::createFileEntry
static CachedFileSystemEntry createFileEntry(StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize=true)
Create an entry that represents an opened source file with minimized or original contents.
Definition: DependencyScanningFilesystem.cpp:18
Filename
StringRef Filename
Definition: Format.cpp:2379
clang::tooling::Range
A source range independent of the SourceManager.
Definition: Replacement.h:44
clang::TypePropertyCache
The type-property cache.
Definition: Type.cpp:3789
clang::ExcludedPreprocessorDirectiveSkipMapping
llvm::DenseMap< const char *, const PreprocessorSkippedRangeMapping * > ExcludedPreprocessorDirectiveSkipMapping
The datastructure that holds the mapping between the active memory buffers and the individual skip ma...
Definition: PreprocessorExcludedConditionalDirectiveSkipMapping.h:26
clang::tooling::dependencies::DependencyScanningWorkerFilesystem::status
llvm::ErrorOr< llvm::vfs::Status > status(const Twine &Path) override
Definition: DependencyScanningFilesystem.cpp:222
clang::hash_value
llvm::hash_code hash_value(const clang::SanitizerMask &Arg)
Definition: Sanitizers.cpp:68
llvm::SmallString
Definition: LLVM.h:37
clang::TypePropertyCache::get
static CachedProperties get(QualType T)
Definition: Type.cpp:3791
clang::serialized_diags::create
std::unique_ptr< DiagnosticConsumer > create(StringRef OutputFile, DiagnosticOptions *Diags, bool MergeChildRecords=false)
Returns a DiagnosticConsumer that serializes diagnostics to a bitcode file.
Definition: SerializedDiagnosticPrinter.cpp:302
clang::minimize_source_to_dependency_directives::computeSkippedRanges
bool computeSkippedRanges(ArrayRef< Token > Input, llvm::SmallVectorImpl< SkippedRange > &Range)
Computes the potential source ranges that can be skipped by the preprocessor when skipping a directiv...
Definition: DependencyDirectivesSourceMinimizer.cpp:925
clang::tooling::dependencies::CachedFileSystemEntry::CachedFileSystemEntry
CachedFileSystemEntry()
Default constructor creates an entry with an invalid stat.
Definition: DependencyScanningFilesystem.h:37
shouldMinimizeBasedOnExtension
static bool shouldMinimizeBasedOnExtension(StringRef Filename)
Whitelist file extensions that should be minimized, treating no extension as a source file that shoul...
Definition: DependencyScanningFilesystem.cpp:132
clang::PreprocessorSkippedRangeMapping
llvm::DenseMap< unsigned, unsigned > PreprocessorSkippedRangeMapping
A mapping from an offset into a buffer to the number of bytes that can be skipped by the preprocessor...
Definition: PreprocessorExcludedConditionalDirectiveSkipMapping.h:21
std
Definition: Format.h:4125
clang
Definition: CalledOnceCheck.h:17
clang::tooling::dependencies::DependencyScanningWorkerFilesystem::openFileForRead
llvm::ErrorOr< std::unique_ptr< llvm::vfs::File > > openFileForRead(const Twine &Path) override
Definition: DependencyScanningFilesystem.cpp:284
clang::tooling::dependencies::DependencyScanningWorkerFilesystem::disableMinimization
void disableMinimization(StringRef Filename)
Disable minimization of the given file.
Definition: DependencyScanningFilesystem.cpp:154
DependencyScanningFilesystem.h
clang::format::make_error_code
std::error_code make_error_code(ParseError e)
Definition: Format.cpp:922
DependencyDirectivesSourceMinimizer.h