11#include "llvm/Support/MemoryBuffer.h"
12#include "llvm/Support/Threading.h"
18llvm::ErrorOr<DependencyScanningWorkerFilesystem::TentativeEntry>
19DependencyScanningWorkerFilesystem::readFile(StringRef Filename,
20 llvm::vfs::FileSystem &FS) {
22 auto MaybeFile = FS.openFileForRead(Filename);
24 return MaybeFile.getError();
25 auto File = std::move(*MaybeFile);
27 auto MaybeStat =
File->status();
29 return MaybeStat.getError();
30 auto Stat = std::move(*MaybeStat);
32 auto MaybeBuffer =
File->getBuffer(Stat.getName());
34 return MaybeBuffer.getError();
35 auto Buffer = std::move(*MaybeBuffer);
38 if (Stat.getSize() != Buffer->getBufferSize())
39 Stat = llvm::vfs::Status::copyWithNewSize(Stat, Buffer->getBufferSize());
41 return TentativeEntry(Stat, std::move(Buffer));
46 auto &Entry = Ref.Entry;
48 if (Entry.isError() || Entry.isDirectory())
52 assert(Contents &&
"contents not initialized");
58 std::lock_guard<std::mutex> GuardLock(Contents->
ValueLock);
72 Contents->
DepDirectives.store(
new std::optional<DependencyDirectivesTy>());
81 new std::optional<DependencyDirectivesTy>(std::move(Directives)));
93 std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4);
94 CacheShards = std::make_unique<CacheShard[]>(NumShards);
99 StringRef Filename)
const {
100 assert(llvm::sys::path::is_absolute_gnu(Filename));
106 llvm::sys::fs::UniqueID UID)
const {
107 auto Hash = llvm::hash_combine(UID.getDevice(), UID.getFile());
108 return CacheShards[Hash % NumShards];
111std::vector<DependencyScanningFilesystemSharedCache::OutOfDateEntry>
113 llvm::vfs::FileSystem &UnderlyingFS)
const {
115 std::vector<OutOfDateEntry> InvalidDiagInfo;
116 for (
unsigned i = 0; i < NumShards; i++) {
118 std::lock_guard<std::mutex> LockGuard(Shard.
CacheLock);
126 llvm::ErrorOr<llvm::vfs::Status> Status = UnderlyingFS.status(Path);
133 InvalidDiagInfo.emplace_back(Path.data());
135 llvm::vfs::Status CachedStatus = Entry->
getStatus();
136 if (Status->getType() == llvm::sys::fs::file_type::regular_file &&
137 Status->getType() == CachedStatus.getType()) {
145 uint64_t CachedSize = CachedStatus.getSize();
146 uint64_t ActualSize = Status->getSize();
147 if (CachedSize != ActualSize) {
150 InvalidDiagInfo.emplace_back(Path.data(), CachedSize, ActualSize);
157 return InvalidDiagInfo;
161using InProgressEntry =
163using SlotResolved = llvm::ErrorOr<const CachedFileSystemEntry *>;
164using SlotProducer = std::shared_ptr<InProgressEntry>;
165using SlotAcquisitionResult = std::variant<SlotResolved, SlotProducer>;
170template <
typename Map,
typename Key>
171SlotAcquisitionResult acquireSlot(std::mutex &CacheLock, Map &M,
const Key &K) {
172 std::shared_ptr<InProgressEntry> Pending;
174 std::lock_guard<std::mutex> ShardLock(CacheLock);
179 return SlotResolved{State.Entry};
181 if (!State.InProgress) {
182 State.InProgress = std::make_shared<InProgressEntry>();
183 return SlotProducer{State.InProgress};
188 Pending = State.InProgress;
192 std::unique_lock<std::mutex> EntryLock(Pending->Mutex);
193 Pending->CondVar.wait(EntryLock, [&] {
return Pending->Done; });
194 return SlotResolved{Pending->Result};
200 StringRef Filename)
const {
201 assert(llvm::sys::path::is_absolute_gnu(Filename));
202 std::lock_guard<std::mutex> LockGuard(
CacheLock);
209 llvm::ErrorOr<llvm::StringRef> RealPath) {
210 std::lock_guard<std::mutex> LockGuard(
CacheLock);
213 if (!StoredRealPath) {
216 return RealPath.getError();
217 return RealPath->str();
224 return *StoredRealPath;
233 auto OverlayVFS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
234 std::move(GlobalFS));
235 OverlayVFS->pushOverlay(std::move(LocalFS));
248 Service(Service), WorkingDirForCacheLookup(
llvm::errc::invalid_argument),
249 LocalFS(
std::move(LocalFS)), BaseFS(FS) {
250 updateWorkingDirForCacheLookup();
254 llvm::ErrorOr<TentativeEntry> &TEntry,
255 llvm::SpecificBumpPtrAllocator<CachedFileContents> &ContentsStorage,
256 llvm::SpecificBumpPtrAllocator<CachedFileSystemEntry> &EntryStorage) {
259 if (TEntry->Contents)
260 StoredContents =
new (ContentsStorage.Allocate())
262 return new (EntryStorage.Allocate())
265 return new (EntryStorage.Allocate())
271DependencyScanningWorkerFilesystem::resolveUIDThroughSharedCache(
272 StringRef OriginalFilename,
const llvm::vfs::Status &Stat) {
273 auto &UIDShard = Service.getSharedCache().getShardForUID(Stat.getUniqueID());
274 auto UIDSlot = acquireSlot(UIDShard.CacheLock, UIDShard.EntriesByUID,
276 if (
auto *Resolved = std::get_if<SlotResolved>(&UIDSlot)) {
277 assert(*Resolved && **Resolved &&
278 "in-progress UID slot fulfilled without an entry");
281 auto UIDProducer = std::move(std::get<SlotProducer>(UIDSlot));
283 auto TEntry = Stat.isDirectory() ? TentativeEntry(Stat)
284 : readFile(OriginalFilename, *BaseFS);
290 const CachedFileSystemEntry *SharedEntry;
292 std::lock_guard<std::mutex> ShardLock(UIDShard.CacheLock);
293 auto &State = UIDShard.EntriesByUID[Stat.getUniqueID()];
294 assert(!State.Entry &&
"UID slot already published an entry");
296 createEntry(TEntry, UIDShard.ContentsStorage, UIDShard.EntryStorage);
297 State.Entry = SharedEntry;
298 State.InProgress.reset();
300 UIDProducer->publish(SharedEntry);
305DependencyScanningWorkerFilesystem::createCachedFSEntryForLocal(
306 StringRef OriginalFilename,
const llvm::vfs::Status &Stat) {
308 auto TEntry = Stat.isDirectory() ? TentativeEntry(Stat)
309 : readFile(OriginalFilename, *LocalFS);
310 return createEntry(TEntry, LocalContentsStorage, LocalEntryStorage);
313llvm::ErrorOr<const CachedFileSystemEntry *>
314DependencyScanningWorkerFilesystem::resolveFilenameThroughSharedCache(
315 StringRef OriginalFilename, StringRef FilenameForLookup) {
317 if (
auto Stat = LocalFS->status(OriginalFilename)) {
318 return createCachedFSEntryForLocal(OriginalFilename, *Stat);
321 assert(llvm::sys::path::is_absolute_gnu(FilenameForLookup));
322 auto &FilenameShard =
323 Service.getSharedCache().getShardForFilename(FilenameForLookup);
325 acquireSlot(FilenameShard.CacheLock, FilenameShard.CacheByFilename,
327 if (
auto *Resolved = std::get_if<SlotResolved>(&FilenameSlot))
329 auto FilenameProducer = std::move(std::get<SlotProducer>(FilenameSlot));
338 auto Stat = BaseFS->status(OriginalFilename);
339 const bool ShouldCacheNegativeStat =
340 !Stat && Service.getOpts().CacheNegativeStats &&
342 llvm::ErrorOr<const CachedFileSystemEntry *>
Result = std::error_code{};
344 Result = resolveUIDThroughSharedCache(OriginalFilename, *Stat);
345 else if (!ShouldCacheNegativeStat)
352 std::lock_guard<std::mutex> ShardLock(FilenameShard.CacheLock);
353 auto &State = FilenameShard.CacheByFilename[FilenameForLookup];
354 assert(!State.Entry &&
"filename slot already published an entry");
355 if (ShouldCacheNegativeStat) {
356 auto *Entry =
new (FilenameShard.EntryStorage.Allocate())
357 CachedFileSystemEntry(Stat.getError());
363 State.InProgress.reset();
365 FilenameProducer->publish(
Result);
369llvm::ErrorOr<EntryRef>
371 StringRef OriginalFilename) {
373 auto FilenameForLookup = tryGetFilenameForLookup(OriginalFilename, PathBuf);
374 if (!FilenameForLookup)
375 return FilenameForLookup.getError();
377 auto &
Local = LocalCache[*FilenameForLookup];
382 resolveFilenameThroughSharedCache(OriginalFilename, *FilenameForLookup);
384 return MaybeEntry.getError();
385 Local.File = *MaybeEntry;
389llvm::ErrorOr<llvm::vfs::Status>
392 StringRef Filename = Path.toStringRef(OwnedFilename);
397 return Result->getStatus();
407 llvm::ErrorOr<llvm::vfs::Status> Status =
status(Path);
408 return Status && Status->exists();
415class DepScanFile final :
public llvm::vfs::File {
417 DepScanFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
418 llvm::vfs::Status Stat)
419 : Buffer(
std::move(Buffer)), Stat(
std::move(Stat)) {}
421 static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> create(
EntryRef Entry);
423 llvm::ErrorOr<llvm::vfs::Status> status()
override {
return Stat; }
425 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
426 getBuffer(
const Twine &Name,
int64_t FileSize,
bool RequiresNullTerminator,
427 bool IsVolatile)
override {
428 return llvm::MemoryBuffer::getMemBuffer(Buffer->getMemBufferRef(),
429 RequiresNullTerminator);
432 std::error_code close()
override {
return {}; }
435 std::unique_ptr<llvm::MemoryBuffer> Buffer;
436 llvm::vfs::Status Stat;
441llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
442DepScanFile::create(
EntryRef Entry) {
443 assert(!Entry.
isError() &&
"error");
446 return std::make_error_code(std::errc::is_a_directory);
448 auto Result = std::make_unique<DepScanFile>(
449 llvm::MemoryBuffer::getMemBuffer(Entry.
getContents(),
454 return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
455 std::unique_ptr<llvm::vfs::File>(std::move(
Result)));
458llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
461 StringRef Filename = Path.toStringRef(OwnedFilename);
466 return DepScanFile::create(
Result.get());
473 StringRef OriginalFilename = Path.toStringRef(OwnedFilename);
476 auto FilenameForLookup = tryGetFilenameForLookup(OriginalFilename, PathBuf);
477 if (!FilenameForLookup)
478 return FilenameForLookup.getError();
480 auto HandleCachedRealPath =
483 return RealPath.getError();
484 Output.assign(RealPath->begin(), RealPath->end());
489 auto &
Local = LocalCache[*FilenameForLookup];
491 return HandleCachedRealPath(*
Local.RealPath);
494 if (LocalFS && LocalFS->exists(OriginalFilename)) {
495 std::error_code EC = LocalFS->getRealPath(OriginalFilename, Output);
498 RealPath = std::string(Output.begin(), Output.end());
499 Local.RealPath =
new (LocalRealPathStorage.Allocate())
501 return HandleCachedRealPath(*
Local.RealPath);
506 Service.getSharedCache().getShardForFilename(*FilenameForLookup);
507 if (
const auto *ShardRealPath =
508 Shard.findRealPathByFilename(*FilenameForLookup)) {
509 Local.RealPath = ShardRealPath;
510 return HandleCachedRealPath(*
Local.RealPath);
514 std::error_code EC = BaseFS->getRealPath(OriginalFilename, Output);
515 llvm::ErrorOr<llvm::StringRef> ComputedRealPath = EC;
517 ComputedRealPath = StringRef{Output.data(), Output.size()};
522 const auto &RealPath = Shard.getOrEmplaceRealPathForFilename(
523 *FilenameForLookup, ComputedRealPath);
524 Local.RealPath = &RealPath;
525 return HandleCachedRealPath(*
Local.RealPath);
530 std::error_code EC = ProxyFileSystem::setCurrentWorkingDirectory(Path);
531 updateWorkingDirForCacheLookup();
535void DependencyScanningWorkerFilesystem::updateWorkingDirForCacheLookup() {
536 llvm::ErrorOr<std::string> CWD =
537 getUnderlyingFS().getCurrentWorkingDirectory();
539 WorkingDirForCacheLookup = CWD.getError();
540 }
else if (!llvm::sys::path::is_absolute_gnu(*CWD)) {
541 WorkingDirForCacheLookup = llvm::errc::invalid_argument;
543 WorkingDirForCacheLookup = *CWD;
545 assert(!WorkingDirForCacheLookup ||
546 llvm::sys::path::is_absolute_gnu(*WorkingDirForCacheLookup));
549llvm::ErrorOr<StringRef>
550DependencyScanningWorkerFilesystem::tryGetFilenameForLookup(
551 StringRef OriginalFilename, llvm::SmallVectorImpl<char> &PathBuf)
const {
552 StringRef FilenameForLookup;
553 if (llvm::sys::path::is_absolute_gnu(OriginalFilename)) {
554 FilenameForLookup = OriginalFilename;
555 }
else if (!WorkingDirForCacheLookup) {
556 return WorkingDirForCacheLookup.getError();
558 StringRef RelFilename = OriginalFilename;
559 RelFilename.consume_front(
"./");
560 PathBuf.assign(WorkingDirForCacheLookup->begin(),
561 WorkingDirForCacheLookup->end());
562 llvm::sys::path::append(PathBuf, RelFilename);
563 FilenameForLookup = StringRef{PathBuf.begin(), PathBuf.size()};
565 assert(llvm::sys::path::is_absolute_gnu(FilenameForLookup));
566 return FilenameForLookup;
static IntrusiveRefCntPtr< llvm::vfs::FileSystem > combineGlobalAndLocalFS(IntrusiveRefCntPtr< llvm::vfs::FileSystem > GlobalFS, IntrusiveRefCntPtr< llvm::vfs::FileSystem > LocalFS)
An in-memory representation of a file system entity that is of interest to the dependency scanning fi...
llvm::vfs::Status getStatus() const
std::error_code getError() const
CacheShard & getShardForUID(llvm::sys::fs::UniqueID UID) const
CacheShard & getShardForFilename(StringRef Filename) const
Returns shard for the given key.
std::vector< OutOfDateEntry > getOutOfDateEntries(llvm::vfs::FileSystem &UnderlyingFS) const
Visits all cached entries and re-stat an entry using UnderlyingFS to check if the cache contains out-...
DependencyScanningFilesystemSharedCache()
The dependency scanning service contains shared configuration and state that is used by the individua...
std::error_code getRealPath(const Twine &Path, SmallVectorImpl< char > &Output) override
DependencyScanningWorkerFilesystem(DependencyScanningService &Service, IntrusiveRefCntPtr< llvm::vfs::FileSystem > FS, IntrusiveRefCntPtr< llvm::vfs::FileSystem > LocalFS=nullptr)
bool ensureDirectiveTokensArePopulated(EntryRef Entry)
Ensure the directive tokens are populated for this file entry.
bool exists(const Twine &Path) override
Check whether Path exists.
llvm::ErrorOr< EntryRef > getOrCreateFileSystemEntry(StringRef Filename)
Returns entry for the given filename.
llvm::ErrorOr< std::unique_ptr< llvm::vfs::File > > openFileForRead(const Twine &Path) override
std::error_code setCurrentWorkingDirectory(const Twine &Path) override
llvm::ErrorOr< llvm::vfs::Status > status(const Twine &Path) override
Reference to a CachedFileSystemEntry.
llvm::vfs::Status getStatus() const
StringRef getContents() const
llvm::ErrorOr< EntryRef > unwrapError() const
If the cached entry represents an error, promotes it into ErrorOr.
llvm::ErrorOr< std::string > CachedRealPath
bool shouldCacheNegativeStatsForPath(StringRef Path)
Top level wrappers for InstallAPI frontend operations.
nullptr
This class represents a compute construct, representing a 'Kind' of ‘parallel’, 'serial',...
bool scanSourceForDependencyDirectives(StringRef Input, SmallVectorImpl< dependency_directives_scan::Token > &Tokens, SmallVectorImpl< dependency_directives_scan::Directive > &Directives, DiagnosticsEngine *Diags=nullptr, SourceLocation InputSourceLoc=SourceLocation())
Scan the input for the preprocessor directives that might have an effect on the dependencies for a co...
@ Result
The result type of a method or function.
Diagnostic wrappers for TextAPI types for error reporting.
hash_code hash_value(const clang::dependencies::ModuleID &ID)
__builtin_elementwise_add_sat __builtin_elementwise_sub_sat __builtin_riscv_ssha_i32 __builtin_riscv_sshl_u32 __builtin_riscv_nclipu_u32 int64_t
Contents and directive tokens of a cached file entry.
std::unique_ptr< llvm::MemoryBuffer > Original
Owning storage for the original contents.
SmallVector< dependency_directives_scan::Token, 10 > DepDirectiveTokens
std::atomic< const std::optional< DependencyDirectivesTy > * > DepDirectives
Accessor to the directive tokens that's atomic to avoid data races.
std::mutex ValueLock
The mutex that must be locked before mutating directive tokens.
llvm::StringMap< FilenameCacheState, llvm::BumpPtrAllocator > CacheByFilename
Map from filenames to their cached state.
llvm::SpecificBumpPtrAllocator< CachedRealPath > RealPathStorage
The backing storage for cached real paths.
const CachedRealPath * findRealPathByFilename(StringRef Filename) const
Returns the real path associated with the filename or nullptr if none is found.
std::mutex CacheLock
The mutex that needs to be locked before mutation of any member.
const CachedRealPath & getOrEmplaceRealPathForFilename(StringRef Filename, llvm::ErrorOr< StringRef > RealPath)
Returns the real path associated with the filename if there is some.
In-flight slot used to dedup concurrent producers for the same key.