clang 19.0.0git
DependencyScanningFilesystem.cpp
Go to the documentation of this file.
1//===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
10#include "llvm/Support/MemoryBuffer.h"
11#include "llvm/Support/SmallVectorMemoryBuffer.h"
12#include "llvm/Support/Threading.h"
13#include <optional>
14
15using namespace clang;
16using namespace tooling;
17using namespace dependencies;
18
19llvm::ErrorOr<DependencyScanningWorkerFilesystem::TentativeEntry>
20DependencyScanningWorkerFilesystem::readFile(StringRef Filename) {
21 // Load the file and its content from the file system.
22 auto MaybeFile = getUnderlyingFS().openFileForRead(Filename);
23 if (!MaybeFile)
24 return MaybeFile.getError();
25 auto File = std::move(*MaybeFile);
26
27 auto MaybeStat = File->status();
28 if (!MaybeStat)
29 return MaybeStat.getError();
30 auto Stat = std::move(*MaybeStat);
31
32 auto MaybeBuffer = File->getBuffer(Stat.getName());
33 if (!MaybeBuffer)
34 return MaybeBuffer.getError();
35 auto Buffer = std::move(*MaybeBuffer);
36
37 // If the file size changed between read and stat, pretend it didn't.
38 if (Stat.getSize() != Buffer->getBufferSize())
39 Stat = llvm::vfs::Status::copyWithNewSize(Stat, Buffer->getBufferSize());
40
41 return TentativeEntry(Stat, std::move(Buffer));
42}
43
45 EntryRef Ref) {
46 auto &Entry = Ref.Entry;
47
48 if (Entry.isError() || Entry.isDirectory())
49 return false;
50
51 CachedFileContents *Contents = Entry.getCachedContents();
52 assert(Contents && "contents not initialized");
53
54 // Double-checked locking.
55 if (Contents->DepDirectives.load())
56 return true;
57
58 std::lock_guard<std::mutex> GuardLock(Contents->ValueLock);
59
60 // Double-checked locking.
61 if (Contents->DepDirectives.load())
62 return true;
63
65 // Scan the file for preprocessor directives that might affect the
66 // dependencies.
67 if (scanSourceForDependencyDirectives(Contents->Original->getBuffer(),
68 Contents->DepDirectiveTokens,
69 Directives)) {
70 Contents->DepDirectiveTokens.clear();
71 // FIXME: Propagate the diagnostic if desired by the client.
72 Contents->DepDirectives.store(new std::optional<DependencyDirectivesTy>());
73 return false;
74 }
75
76 // This function performed double-checked locking using `DepDirectives`.
77 // Assigning it must be the last thing this function does, otherwise other
78 // threads may skip the critical section (`DepDirectives != nullptr`), leading
79 // to a data race.
80 Contents->DepDirectives.store(
81 new std::optional<DependencyDirectivesTy>(std::move(Directives)));
82 return true;
83}
84
87 // This heuristic was chosen using a empirical testing on a
88 // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
89 // sharding gives a performance edge by reducing the lock contention.
90 // FIXME: A better heuristic might also consider the OS to account for
91 // the different cost of lock contention on different OSes.
92 NumShards =
93 std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4);
94 CacheShards = std::make_unique<CacheShard[]>(NumShards);
95}
96
99 StringRef Filename) const {
100 assert(llvm::sys::path::is_absolute_gnu(Filename));
101 return CacheShards[llvm::hash_value(Filename) % NumShards];
102}
103
106 llvm::sys::fs::UniqueID UID) const {
107 auto Hash = llvm::hash_combine(UID.getDevice(), UID.getFile());
108 return CacheShards[Hash % NumShards];
109}
110
113 StringRef Filename) const {
114 assert(llvm::sys::path::is_absolute_gnu(Filename));
115 std::lock_guard<std::mutex> LockGuard(CacheLock);
116 auto It = EntriesByFilename.find(Filename);
117 return It == EntriesByFilename.end() ? nullptr : It->getValue();
118}
119
122 llvm::sys::fs::UniqueID UID) const {
123 std::lock_guard<std::mutex> LockGuard(CacheLock);
124 auto It = EntriesByUID.find(UID);
125 return It == EntriesByUID.end() ? nullptr : It->getSecond();
126}
127
131 llvm::ErrorOr<llvm::vfs::Status> Stat) {
132 std::lock_guard<std::mutex> LockGuard(CacheLock);
133 auto Insertion = EntriesByFilename.insert({Filename, nullptr});
134 if (Insertion.second)
135 Insertion.first->second =
136 new (EntryStorage.Allocate()) CachedFileSystemEntry(std::move(Stat));
137 return *Insertion.first->second;
138}
139
142 llvm::sys::fs::UniqueID UID, llvm::vfs::Status Stat,
143 std::unique_ptr<llvm::MemoryBuffer> Contents) {
144 std::lock_guard<std::mutex> LockGuard(CacheLock);
145 auto Insertion = EntriesByUID.insert({UID, nullptr});
146 if (Insertion.second) {
147 CachedFileContents *StoredContents = nullptr;
148 if (Contents)
149 StoredContents = new (ContentsStorage.Allocate())
150 CachedFileContents(std::move(Contents));
151 Insertion.first->second = new (EntryStorage.Allocate())
152 CachedFileSystemEntry(std::move(Stat), StoredContents);
153 }
154 return *Insertion.first->second;
155}
156
160 const CachedFileSystemEntry &Entry) {
161 std::lock_guard<std::mutex> LockGuard(CacheLock);
162 return *EntriesByFilename.insert({Filename, &Entry}).first->getValue();
163}
164
165static bool shouldCacheStatFailures(StringRef Filename) {
166 StringRef Ext = llvm::sys::path::extension(Filename);
167 if (Ext.empty())
168 return false; // This may be the module cache directory.
169 return true;
170}
171
176 llvm::vfs::ProxyFileSystem>(std::move(FS)),
177 SharedCache(SharedCache),
178 WorkingDirForCacheLookup(llvm::errc::invalid_argument) {
179 updateWorkingDirForCacheLookup();
180}
181
183DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForUID(
184 TentativeEntry TEntry) {
185 auto &Shard = SharedCache.getShardForUID(TEntry.Status.getUniqueID());
186 return Shard.getOrEmplaceEntryForUID(TEntry.Status.getUniqueID(),
187 std::move(TEntry.Status),
188 std::move(TEntry.Contents));
189}
190
192DependencyScanningWorkerFilesystem::findEntryByFilenameWithWriteThrough(
193 StringRef Filename) {
194 if (const auto *Entry = LocalCache.findEntryByFilename(Filename))
195 return Entry;
196 auto &Shard = SharedCache.getShardForFilename(Filename);
197 if (const auto *Entry = Shard.findEntryByFilename(Filename))
198 return &LocalCache.insertEntryForFilename(Filename, *Entry);
199 return nullptr;
200}
201
202llvm::ErrorOr<const CachedFileSystemEntry &>
203DependencyScanningWorkerFilesystem::computeAndStoreResult(
204 StringRef OriginalFilename, StringRef FilenameForLookup) {
205 llvm::ErrorOr<llvm::vfs::Status> Stat =
206 getUnderlyingFS().status(OriginalFilename);
207 if (!Stat) {
208 if (!shouldCacheStatFailures(OriginalFilename))
209 return Stat.getError();
210 const auto &Entry =
211 getOrEmplaceSharedEntryForFilename(FilenameForLookup, Stat.getError());
212 return insertLocalEntryForFilename(FilenameForLookup, Entry);
213 }
214
215 if (const auto *Entry = findSharedEntryByUID(*Stat))
216 return insertLocalEntryForFilename(FilenameForLookup, *Entry);
217
218 auto TEntry =
219 Stat->isDirectory() ? TentativeEntry(*Stat) : readFile(OriginalFilename);
220
221 const CachedFileSystemEntry *SharedEntry = [&]() {
222 if (TEntry) {
223 const auto &UIDEntry = getOrEmplaceSharedEntryForUID(std::move(*TEntry));
224 return &getOrInsertSharedEntryForFilename(FilenameForLookup, UIDEntry);
225 }
226 return &getOrEmplaceSharedEntryForFilename(FilenameForLookup,
227 TEntry.getError());
228 }();
229
230 return insertLocalEntryForFilename(FilenameForLookup, *SharedEntry);
231}
232
233llvm::ErrorOr<EntryRef>
235 StringRef OriginalFilename) {
236 StringRef FilenameForLookup;
237 SmallString<256> PathBuf;
238 if (llvm::sys::path::is_absolute_gnu(OriginalFilename)) {
239 FilenameForLookup = OriginalFilename;
240 } else if (!WorkingDirForCacheLookup) {
241 return WorkingDirForCacheLookup.getError();
242 } else {
243 StringRef RelFilename = OriginalFilename;
244 RelFilename.consume_front("./");
245 PathBuf = *WorkingDirForCacheLookup;
246 llvm::sys::path::append(PathBuf, RelFilename);
247 FilenameForLookup = PathBuf.str();
248 }
249 assert(llvm::sys::path::is_absolute_gnu(FilenameForLookup));
250 if (const auto *Entry =
251 findEntryByFilenameWithWriteThrough(FilenameForLookup))
252 return EntryRef(OriginalFilename, *Entry).unwrapError();
253 auto MaybeEntry = computeAndStoreResult(OriginalFilename, FilenameForLookup);
254 if (!MaybeEntry)
255 return MaybeEntry.getError();
256 return EntryRef(OriginalFilename, *MaybeEntry).unwrapError();
257}
258
259llvm::ErrorOr<llvm::vfs::Status>
261 SmallString<256> OwnedFilename;
262 StringRef Filename = Path.toStringRef(OwnedFilename);
263
264 if (Filename.ends_with(".pcm"))
265 return getUnderlyingFS().status(Path);
266
267 llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename);
268 if (!Result)
269 return Result.getError();
270 return Result->getStatus();
271}
272
273namespace {
274
275/// The VFS that is used by clang consumes the \c CachedFileSystemEntry using
276/// this subclass.
277class DepScanFile final : public llvm::vfs::File {
278public:
279 DepScanFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
280 llvm::vfs::Status Stat)
281 : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
282
283 static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> create(EntryRef Entry);
284
285 llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
286
287 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
288 getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
289 bool IsVolatile) override {
290 return std::move(Buffer);
291 }
292
293 std::error_code close() override { return {}; }
294
295private:
296 std::unique_ptr<llvm::MemoryBuffer> Buffer;
297 llvm::vfs::Status Stat;
298};
299
300} // end anonymous namespace
301
302llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
303DepScanFile::create(EntryRef Entry) {
304 assert(!Entry.isError() && "error");
305
306 if (Entry.isDirectory())
307 return std::make_error_code(std::errc::is_a_directory);
308
309 auto Result = std::make_unique<DepScanFile>(
310 llvm::MemoryBuffer::getMemBuffer(Entry.getContents(),
311 Entry.getStatus().getName(),
312 /*RequiresNullTerminator=*/false),
313 Entry.getStatus());
314
315 return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
316 std::unique_ptr<llvm::vfs::File>(std::move(Result)));
317}
318
319llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
321 SmallString<256> OwnedFilename;
322 StringRef Filename = Path.toStringRef(OwnedFilename);
323
324 if (Filename.ends_with(".pcm"))
325 return getUnderlyingFS().openFileForRead(Path);
326
327 llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename);
328 if (!Result)
329 return Result.getError();
330 return DepScanFile::create(Result.get());
331}
332
334 const Twine &Path) {
335 std::error_code EC = ProxyFileSystem::setCurrentWorkingDirectory(Path);
336 updateWorkingDirForCacheLookup();
337 return EC;
338}
339
340void DependencyScanningWorkerFilesystem::updateWorkingDirForCacheLookup() {
341 llvm::ErrorOr<std::string> CWD =
342 getUnderlyingFS().getCurrentWorkingDirectory();
343 if (!CWD) {
344 WorkingDirForCacheLookup = CWD.getError();
345 } else if (!llvm::sys::path::is_absolute_gnu(*CWD)) {
346 WorkingDirForCacheLookup = llvm::errc::invalid_argument;
347 } else {
348 WorkingDirForCacheLookup = *CWD;
349 }
350 assert(!WorkingDirForCacheLookup ||
351 llvm::sys::path::is_absolute_gnu(*WorkingDirForCacheLookup));
352}
353
static bool shouldCacheStatFailures(StringRef Filename)
StringRef Filename
Definition: Format.cpp:2969
An in-memory representation of a file system entity that is of interest to the dependency scanning fi...
const CachedFileSystemEntry & insertEntryForFilename(StringRef Filename, const CachedFileSystemEntry &Entry)
Associates the given entry with the filename and returns the given entry pointer (for convenience).
const CachedFileSystemEntry * findEntryByFilename(StringRef Filename) const
Returns entry associated with the filename or nullptr if none is found.
This class is a shared cache, that caches the 'stat' and 'open' calls to the underlying real file sys...
CacheShard & getShardForFilename(StringRef Filename) const
Returns shard for the given key.
A virtual file system optimized for the dependency discovery.
bool ensureDirectiveTokensArePopulated(EntryRef Entry)
Ensure the directive tokens are populated for this file entry.
llvm::ErrorOr< EntryRef > getOrCreateFileSystemEntry(StringRef Filename)
Returns entry for the given filename.
llvm::ErrorOr< std::unique_ptr< llvm::vfs::File > > openFileForRead(const Twine &Path) override
DependencyScanningWorkerFilesystem(DependencyScanningFilesystemSharedCache &SharedCache, IntrusiveRefCntPtr< llvm::vfs::FileSystem > FS)
llvm::ErrorOr< llvm::vfs::Status > status(const Twine &Path) override
Reference to a CachedFileSystemEntry.
llvm::ErrorOr< EntryRef > unwrapError() const
If the cached entry represents an error, promotes it into ErrorOr.
The JSON file list parser is used to communicate input to InstallAPI.
bool scanSourceForDependencyDirectives(StringRef Input, SmallVectorImpl< dependency_directives_scan::Token > &Tokens, SmallVectorImpl< dependency_directives_scan::Directive > &Directives, DiagnosticsEngine *Diags=nullptr, SourceLocation InputSourceLoc=SourceLocation())
Scan the input for the preprocessor directives that might have an effect on the dependencies for a co...
@ Result
The result type of a method or function.
YAML serialization mapping.
Definition: Dominators.h:30
hash_code hash_value(const clang::tooling::dependencies::ModuleID &ID)
Definition: Format.h:5378
Contents and directive tokens of a cached file entry.
std::mutex ValueLock
The mutex that must be locked before mutating directive tokens.
std::atomic< const std::optional< DependencyDirectivesTy > * > DepDirectives
Accessor to the directive tokens that's atomic to avoid data races.
std::unique_ptr< llvm::MemoryBuffer > Original
Owning storage for the original contents.
SmallVector< dependency_directives_scan::Token, 10 > DepDirectiveTokens
const CachedFileSystemEntry & getOrEmplaceEntryForUID(llvm::sys::fs::UniqueID UID, llvm::vfs::Status Stat, std::unique_ptr< llvm::MemoryBuffer > Contents)
Returns entry associated with the unique ID if there is some.
std::mutex CacheLock
The mutex that needs to be locked before mutation of any member.
const CachedFileSystemEntry * findEntryByUID(llvm::sys::fs::UniqueID UID) const
Returns entry associated with the unique ID or nullptr if none is found.
const CachedFileSystemEntry & getOrInsertEntryForFilename(StringRef Filename, const CachedFileSystemEntry &Entry)
Returns entry associated with the filename if there is some.
const CachedFileSystemEntry * findEntryByFilename(StringRef Filename) const
Returns entry associated with the filename or nullptr if none is found.
const CachedFileSystemEntry & getOrEmplaceEntryForFilename(StringRef Filename, llvm::ErrorOr< llvm::vfs::Status > Stat)
Returns entry associated with the filename if there is some.
llvm::StringMap< const CachedFileSystemEntry *, llvm::BumpPtrAllocator > EntriesByFilename
Map from filenames to cached entries.