21#include "llvm/ADT/SmallString.h"
22#include "llvm/ADT/Statistic.h"
23#include "llvm/Config/llvm-config.h"
24#include "llvm/Support/Error.h"
25#include "llvm/Support/FileSystem.h"
26#include "llvm/Support/IOSandbox.h"
27#include "llvm/Support/MemoryBuffer.h"
28#include "llvm/Support/Path.h"
29#include "llvm/Support/raw_ostream.h"
40#define DEBUG_TYPE "file-search"
43 std::optional<std::string> &Storage) {
44 using namespace llvm::sys::path;
48 if (Path.size() > 1 && root_path(Path) != Path && is_separator(Path.back()))
49 Path = Path.drop_back();
54 if (is_style_windows(Style::native)) {
55 if (Path.size() > 1 && Path.back() ==
':' &&
56 Path.equals_insensitive(root_name(Path))) {
57 Storage = Path.str() +
".";
69 : FS(
std::move(FS)), FileSystemOpts(FSO), SeenDirEntries(64),
70 SeenFileEntries(64), NextFileUID(0) {
74 this->FS = llvm::vfs::getRealFileSystem();
80 assert(statCache &&
"No stat cache provided?");
81 StatCache = std::move(statCache);
86llvm::ErrorOr<DirectoryEntryRef>
87FileManager::getDirectoryFromFile(StringRef Filename,
bool CacheFailure) {
91 if (llvm::sys::path::is_separator(Filename[Filename.size() - 1]))
94 StringRef DirName = llvm::sys::path::parent_path(Filename);
99 return getDirectoryRefImpl(DirName, CacheFailure);
102DirectoryEntry *&FileManager::getRealDirEntry(
const llvm::vfs::Status &Status) {
103 assert(Status.isDirectory() &&
"The directory should exist!");
120void FileManager::addAncestorsAsVirtualDirs(StringRef Path) {
121 StringRef DirName = llvm::sys::path::parent_path(Path);
128 std::optional<std::string>
Storage;
129 StringRef OriginalDirName = DirName;
132 auto &NamedDirEnt = *SeenDirEntries.insert(
133 {DirName, std::errc::no_such_file_or_directory}).first;
139 if (NamedDirEnt.second)
143 llvm::vfs::Status Status;
145 getStatValue(DirName, Status,
false,
nullptr );
149 auto *UDE =
new (DirsAlloc.Allocate()) DirectoryEntry();
150 NamedDirEnt.second = *UDE;
151 VirtualDirectoryEntries.push_back(UDE);
154 DirectoryEntry *&UDE = getRealDirEntry(Status);
155 NamedDirEnt.second = *UDE;
159 addAncestorsAsVirtualDirs(OriginalDirName);
162llvm::ErrorOr<DirectoryEntryRef>
163FileManager::getDirectoryRefImpl(StringRef DirName,
bool CacheFailure) {
164 std::optional<std::string> DirNameStr;
171 auto SeenDirInsertResult =
172 SeenDirEntries.insert({DirName, std::errc::no_such_file_or_directory});
173 if (!SeenDirInsertResult.second) {
174 if (SeenDirInsertResult.first->second)
175 return DirectoryEntryRef(*SeenDirInsertResult.first);
176 return SeenDirInsertResult.first->second.getError();
181 auto &NamedDirEnt = *SeenDirInsertResult.first;
182 assert(!NamedDirEnt.second &&
"should be newly-created");
186 StringRef InterndDirName = NamedDirEnt.first();
189 llvm::vfs::Status Status;
190 auto statError = getStatValue(InterndDirName, Status,
false,
195 NamedDirEnt.second = statError;
197 SeenDirEntries.erase(DirName);
202 DirectoryEntry *&UDE = getRealDirEntry(Status);
203 NamedDirEnt.second = *UDE;
205 return DirectoryEntryRef(NamedDirEnt);
208llvm::ErrorOr<FileEntryRef> FileManager::getFileRefImpl(StringRef Filename,
215 auto SeenFileInsertResult =
216 SeenFileEntries.insert({Filename, std::errc::no_such_file_or_directory});
217 if (!SeenFileInsertResult.second) {
218 if (!SeenFileInsertResult.first->second)
219 return SeenFileInsertResult.first->second.getError();
220 return FileEntryRef(*SeenFileInsertResult.first);
224 ++NumFileCacheMisses;
225 auto *NamedFileEnt = &*SeenFileInsertResult.first;
226 assert(!NamedFileEnt->second &&
"should be newly-created");
230 StringRef InterndFileName = NamedFileEnt->first();
237 auto DirInfoOrErr = getDirectoryFromFile(Filename, CacheFailure);
239 std::error_code Err = DirInfoOrErr.getError();
241 NamedFileEnt->second = Err;
243 SeenFileEntries.erase(Filename);
247 DirectoryEntryRef DirInfo = *DirInfoOrErr;
253 std::unique_ptr<llvm::vfs::File> F;
254 llvm::vfs::Status Status;
255 auto statError = getStatValue(InterndFileName, Status,
true,
256 openFile ? &F :
nullptr, IsText);
260 NamedFileEnt->second = statError;
262 SeenFileEntries.erase(Filename);
267 assert((openFile || !F) &&
"undesired open file");
271 FileEntry *&UFE = UniqueRealFiles[Status.getUniqueID()];
272 bool ReusingEntry = UFE !=
nullptr;
274 UFE =
new (FilesAlloc.Allocate()) FileEntry();
276 if (!Status.ExposesExternalVFSPath || Status.getName() == Filename) {
278 NamedFileEnt->second = FileEntryRef::MapValue(*UFE, DirInfo);
317 .insert({Status.getName(), FileEntryRef::MapValue(*UFE, DirInfo)})
320 "filename redirected to a non-canonical filename?");
322 "filename from getStatValue() refers to wrong file");
326 NamedFileEnt->second = FileEntryRef::MapValue(Redirection, DirInfo);
329 FileEntryRef ReturnedRef(*NamedFileEnt);
335 UFE->Size = Status.getSize();
336 UFE->ModTime = llvm::sys::toTimeT(Status.getLastModificationTime());
338 UFE->UID = NextFileUID++;
339 UFE->UniqueID = Status.getUniqueID();
340 UFE->IsNamedPipe = Status.getType() == llvm::sys::fs::file_type::fifo_file;
342 Status.getType() == llvm::sys::fs::file_type::character_file;
343 UFE->File = std::move(F);
346 if (
auto PathName = UFE->File->getName())
347 fillRealPathName(UFE, *PathName);
348 }
else if (!openFile) {
350 fillRealPathName(UFE, InterndFileName);
360 auto ContentOrError = [] {
361 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
362 return llvm::MemoryBuffer::getSTDIN();
366 return llvm::createFileError(
"-", ContentOrError.getError());
368 auto Content = std::move(*ContentOrError);
370 Content->getBufferSize(), 0);
372 FE.Content = std::move(Content);
373 FE.IsNamedPipe =
true;
378 FS->visit([Active](llvm::vfs::FileSystem &FileSys) {
379 if (
auto *RFS = dyn_cast<llvm::vfs::RedirectingFileSystem>(&FileSys))
380 RFS->setUsageTrackingActive(Active);
385 time_t ModificationTime) {
389 auto &NamedFileEnt = *SeenFileEntries.insert(
390 {Filename, std::errc::no_such_file_or_directory}).first;
391 if (NamedFileEnt.second) {
399 ++NumFileCacheMisses;
400 addAncestorsAsVirtualDirs(Filename);
409 auto DirInfo = getDirectoryFromFile(Filename.empty() ?
"." : Filename,
412 "The directory of a virtual file should already be in the cache.");
415 llvm::vfs::Status Status;
416 const char *InterndFileName = NamedFileEnt.first().data();
417 if (!getStatValue(InterndFileName, Status,
true,
nullptr)) {
418 Status = llvm::vfs::Status(
419 Status.getName(), Status.getUniqueID(),
420 llvm::sys::toTimePoint(ModificationTime),
421 Status.getUser(), Status.getGroup(), Size,
422 Status.getType(), Status.getPermissions());
424 auto &RealFE = UniqueRealFiles[Status.getUniqueID()];
439 RealFE =
new (FilesAlloc.Allocate())
FileEntry();
440 RealFE->UniqueID = Status.getUniqueID();
441 RealFE->IsNamedPipe =
442 Status.getType() == llvm::sys::fs::file_type::fifo_file;
443 fillRealPathName(RealFE, Status.getName());
448 UFE =
new (FilesAlloc.Allocate())
FileEntry();
449 VirtualFileEntries.push_back(UFE);
454 UFE->ModTime = ModificationTime;
456 UFE->UID = NextFileUID++;
463 llvm::vfs::Status Status;
464 if (getStatValue(VF.
getName(), Status,
true,
nullptr))
467 if (!SeenBypassFileEntries)
468 SeenBypassFileEntries = std::make_unique<
469 llvm::StringMap<llvm::ErrorOr<FileEntryRef::MapValue>>>();
472 auto Insertion = SeenBypassFileEntries->insert(
473 {VF.
getName(), std::errc::no_such_file_or_directory});
474 if (!Insertion.second)
479 BypassFileEntries.push_back(BFE);
481 BFE->Size = Status.getSize();
483 BFE->ModTime = llvm::sys::toTimeT(Status.getLastModificationTime());
484 BFE->UID = NextFileUID++;
492 StringRef pathRef(Path.data(), Path.size());
494 if (FileSystemOpts.WorkingDir.empty()
495 || llvm::sys::path::is_absolute(pathRef))
499 llvm::sys::path::append(NewPath, pathRef);
500 Path = std::move(NewPath);
505 bool Canonicalize)
const {
508 if (!llvm::sys::path::is_absolute(StringRef(Path.data(), Path.size()))) {
509 FS->makeAbsolute(Path);
514 Changed |= llvm::sys::path::remove_dots(Path);
526 llvm::sys::path::remove_dots(AbsPath,
true);
527 UFE->RealPathName = std::string(AbsPath);
530llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
532 bool RequiresNullTerminator,
533 std::optional<int64_t> MaybeLimit,
bool IsText) {
537 return llvm::MemoryBuffer::getMemBuffer(Entry->Content->getMemBufferRef());
539 uint64_t FileSize = Entry->
getSize();
542 FileSize = *MaybeLimit;
549 StringRef Filename = FE.
getName();
552 auto Result = Entry->File->getBuffer(Filename, FileSize,
553 RequiresNullTerminator, isVolatile);
559 return getBufferForFileImpl(Filename, FileSize, isVolatile,
560 RequiresNullTerminator, IsText);
563llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
564FileManager::getBufferForFileImpl(StringRef Filename, int64_t FileSize,
565 bool isVolatile,
bool RequiresNullTerminator,
568 return FS->getBufferForFile(Filename, FileSize, RequiresNullTerminator,
573 return FS->getBufferForFile(FilePath, FileSize, RequiresNullTerminator,
582std::error_code FileManager::getStatValue(StringRef Path,
583 llvm::vfs::Status &Status,
585 std::unique_ptr<llvm::vfs::File> *F,
597 StatCache.get(), *FS, IsText);
602 llvm::vfs::Status &
Result) {
606 llvm::ErrorOr<llvm::vfs::Status> S = FS->status(FilePath.c_str());
610 return std::error_code();
616 UIDToFiles.resize(NextFileUID);
618 for (
const auto &Entry : SeenFileEntries) {
640 llvm::DenseMap<const void *, llvm::StringRef>::iterator Known =
641 CanonicalNames.find(Entry);
642 if (Known != CanonicalNames.end())
643 return Known->second;
647 StringRef CanonicalName(Name);
651 if (!FS->getRealPath(Name, RealPathBuf)) {
652 if (is_style_windows(llvm::sys::path::Style::native)) {
656 if (!FS->makeAbsolute(AbsPathBuf)) {
657 if (llvm::sys::path::root_name(RealPathBuf) ==
658 llvm::sys::path::root_name(AbsPathBuf)) {
659 CanonicalName = RealPathBuf.str().copy(CanonicalNameStorage);
664 llvm::sys::path::remove_dots(AbsPathBuf,
true);
665 CanonicalName = AbsPathBuf.str().copy(CanonicalNameStorage);
669 CanonicalName = RealPathBuf.str().copy(CanonicalNameStorage);
673 CanonicalNames.insert({Entry, CanonicalName});
674 return CanonicalName;
678 assert(&
Other !=
this &&
"Collecting stats into the same FileManager");
679 NumDirLookups +=
Other.NumDirLookups;
680 NumFileLookups +=
Other.NumFileLookups;
681 NumDirCacheMisses +=
Other.NumDirCacheMisses;
682 NumFileCacheMisses +=
Other.NumFileCacheMisses;
686 llvm::errs() <<
"\n*** File Manager Stats:\n";
687 llvm::errs() << UniqueRealFiles.size() <<
" real files found, "
688 << UniqueRealDirs.size() <<
" real dirs found.\n";
689 llvm::errs() << VirtualFileEntries.size() <<
" virtual files found, "
690 << VirtualDirectoryEntries.size() <<
" virtual dirs found.\n";
691 llvm::errs() << NumDirLookups <<
" dir lookups, "
692 << NumDirCacheMisses <<
" dir cache misses.\n";
693 llvm::errs() << NumFileLookups <<
" file lookups, "
694 << NumFileCacheMisses <<
" file cache misses.\n";
697 if (
auto *T = dyn_cast_or_null<llvm::vfs::TracingFileSystem>(&VFS))
698 llvm::errs() <<
"\n*** Virtual File System Stats:\n"
699 << T->NumStatusCalls <<
" status() calls\n"
700 << T->NumOpenFileForReadCalls <<
" openFileForRead() calls\n"
701 << T->NumDirBeginCalls <<
" dir_begin() calls\n"
702 << T->NumGetRealPathCalls <<
" getRealPath() calls\n"
703 << T->NumExistsCalls <<
" exists() calls\n"
704 << T->NumIsLocalCalls <<
" isLocal() calls\n";
static void normalizeCacheKey(StringRef &Path, std::optional< std::string > &Storage)
Defines the clang::FileManager interface and associated types.
Defines the FileSystemStatCache interface.
A reference to a DirectoryEntry that includes the name of the directory as it was accessed by the Fil...
StringRef getName() const
const DirectoryEntry & getDirEntry() const
Cached information about one directory (either on disk or in the virtual file system).
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
const FileEntry & getFileEntry() const
StringRef getName() const
The name of this FileEntry.
DirectoryEntryRef getDir() const
Cached information about one file (either on disk or in the virtual file system).
bool isNamedPipe() const
Check whether the file is a named pipe (and thus can't be opened by the native FileManager methods).
void AddStats(const FileManager &Other)
Import statistics from a child FileManager and add them to this current FileManager.
void trackVFSUsage(bool Active)
Enable or disable tracking of VFS usage.
void clearStatCache()
Removes the FileSystemStatCache object from the manager.
llvm::vfs::FileSystem & getVirtualFileSystem() const
llvm::ErrorOr< std::unique_ptr< llvm::MemoryBuffer > > getBufferForFile(FileEntryRef Entry, bool isVolatile=false, bool RequiresNullTerminator=true, std::optional< int64_t > MaybeLimit=std::nullopt, bool IsText=true)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...
std::error_code getNoncachedStatValue(StringRef Path, llvm::vfs::Status &Result)
Get the 'stat' information for the given Path.
FileManager(const FileSystemOptions &FileSystemOpts, IntrusiveRefCntPtr< llvm::vfs::FileSystem > FS=nullptr)
Construct a file manager, optionally with a custom VFS.
llvm::Expected< FileEntryRef > getSTDIN()
Get the FileEntryRef for stdin, returning an error if stdin cannot be read.
StringRef getCanonicalName(DirectoryEntryRef Dir)
Retrieve the canonical name for a given directory.
void GetUniqueIDMapping(SmallVectorImpl< OptionalFileEntryRef > &UIDToFiles) const
Produce an array mapping from the unique IDs assigned to each file to the corresponding FileEntryRef.
void setStatCache(std::unique_ptr< FileSystemStatCache > statCache)
Installs the provided FileSystemStatCache object within the FileManager.
FileEntryRef getVirtualFileRef(StringRef Filename, off_t Size, time_t ModificationTime)
Retrieve a file entry for a "virtual" file that acts as if there were a file with the given name on d...
bool FixupRelativePath(SmallVectorImpl< char > &Path) const
If path is not absolute and FileSystemOptions set the working directory, the path is modified to be r...
bool makeAbsolutePath(SmallVectorImpl< char > &Path, bool Canonicalize=false) const
Makes Path absolute taking into account FileSystemOptions and the working directory option,...
OptionalFileEntryRef getBypassFile(FileEntryRef VFE)
Retrieve a FileEntry that bypasses VFE, which is expected to be a virtual file entry,...
static bool fixupRelativePath(const FileSystemOptions &FileSystemOpts, SmallVectorImpl< char > &Path)
Keeps track of options that affect how file operations are performed.
std::string WorkingDir
If set, paths are resolved as if the working directory was set to the value of WorkingDir.
static std::error_code get(StringRef Path, llvm::vfs::Status &Status, bool isFile, std::unique_ptr< llvm::vfs::File > *F, FileSystemStatCache *Cache, llvm::vfs::FileSystem &FS, bool IsText=true)
Get the 'stat' information for the specified path, using the cache to accelerate it if possible.
The JSON file list parser is used to communicate input to InstallAPI.
bool isa(CodeGen::Address addr)
CustomizableOptional< FileEntryRef > OptionalFileEntryRef
std::error_code make_error_code(BuildPreambleError Error)
@ Result
The result type of a method or function.
U cast(CodeGen::Address addr)
@ Other
Other implicit parameter.
Type stored in the StringMap.