20#include "llvm/ADT/SmallString.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/Config/llvm-config.h"
23#include "llvm/Support/Error.h"
24#include "llvm/Support/FileSystem.h"
25#include "llvm/Support/IOSandbox.h"
26#include "llvm/Support/MemoryBuffer.h"
27#include "llvm/Support/Path.h"
28#include "llvm/Support/raw_ostream.h"
39#define DEBUG_TYPE "file-search"
42 std::optional<std::string> &Storage) {
43 using namespace llvm::sys::path;
47 if (Path.size() > 1 && root_path(Path) != Path && is_separator(Path.back()))
48 Path = Path.drop_back();
53 if (is_style_windows(Style::native)) {
54 if (Path.size() > 1 && Path.back() ==
':' &&
55 Path.equals_insensitive(root_name(Path))) {
56 Storage = Path.str() +
".";
68 : FS(
std::move(FS)), FileSystemOpts(FSO), SeenDirEntries(64),
69 SeenFileEntries(64), NextFileUID(0) {
73 this->FS = llvm::vfs::getRealFileSystem();
78llvm::ErrorOr<DirectoryEntryRef>
79FileManager::getDirectoryFromFile(StringRef Filename,
bool CacheFailure) {
83 if (llvm::sys::path::is_separator(Filename[Filename.size() - 1]))
86 StringRef DirName = llvm::sys::path::parent_path(Filename);
91 return getDirectoryRefImpl(DirName, CacheFailure);
94DirectoryEntry *&FileManager::getRealDirEntry(
const llvm::vfs::Status &Status) {
95 assert(Status.isDirectory() &&
"The directory should exist!");
112void FileManager::addAncestorsAsVirtualDirs(StringRef Path) {
113 StringRef DirName = llvm::sys::path::parent_path(Path);
120 std::optional<std::string>
Storage;
121 StringRef OriginalDirName = DirName;
124 auto &NamedDirEnt = *SeenDirEntries.insert(
125 {DirName, std::errc::no_such_file_or_directory}).first;
131 if (NamedDirEnt.second)
135 llvm::vfs::Status Status;
137 getStatValue(DirName, Status,
false,
nullptr );
141 auto *UDE =
new (DirsAlloc.Allocate()) DirectoryEntry();
142 NamedDirEnt.second = *UDE;
143 VirtualDirectoryEntries.push_back(UDE);
146 DirectoryEntry *&UDE = getRealDirEntry(Status);
147 NamedDirEnt.second = *UDE;
151 addAncestorsAsVirtualDirs(OriginalDirName);
154llvm::ErrorOr<DirectoryEntryRef>
155FileManager::getDirectoryRefImpl(StringRef DirName,
bool CacheFailure) {
156 std::optional<std::string> DirNameStr;
163 auto SeenDirInsertResult =
164 SeenDirEntries.insert({DirName, std::errc::no_such_file_or_directory});
165 if (!SeenDirInsertResult.second) {
166 if (SeenDirInsertResult.first->second)
167 return DirectoryEntryRef(*SeenDirInsertResult.first);
168 return SeenDirInsertResult.first->second.getError();
173 auto &NamedDirEnt = *SeenDirInsertResult.first;
174 assert(!NamedDirEnt.second &&
"should be newly-created");
178 StringRef InterndDirName = NamedDirEnt.first();
181 llvm::vfs::Status Status;
182 auto statError = getStatValue(InterndDirName, Status,
false,
187 NamedDirEnt.second = statError;
189 SeenDirEntries.erase(DirName);
194 DirectoryEntry *&UDE = getRealDirEntry(Status);
195 NamedDirEnt.second = *UDE;
197 return DirectoryEntryRef(NamedDirEnt);
200llvm::ErrorOr<FileEntryRef> FileManager::getFileRefImpl(StringRef Filename,
207 auto SeenFileInsertResult =
208 SeenFileEntries.insert({Filename, std::errc::no_such_file_or_directory});
209 if (!SeenFileInsertResult.second) {
210 if (!SeenFileInsertResult.first->second)
211 return SeenFileInsertResult.first->second.getError();
212 return FileEntryRef(*SeenFileInsertResult.first);
216 ++NumFileCacheMisses;
217 auto *NamedFileEnt = &*SeenFileInsertResult.first;
218 assert(!NamedFileEnt->second &&
"should be newly-created");
222 StringRef InterndFileName = NamedFileEnt->first();
229 auto DirInfoOrErr = getDirectoryFromFile(Filename, CacheFailure);
231 std::error_code Err = DirInfoOrErr.getError();
233 NamedFileEnt->second = Err;
235 SeenFileEntries.erase(Filename);
239 DirectoryEntryRef DirInfo = *DirInfoOrErr;
245 std::unique_ptr<llvm::vfs::File> F;
246 llvm::vfs::Status Status;
247 auto statError = getStatValue(InterndFileName, Status,
true,
248 openFile ? &F :
nullptr, IsText);
252 NamedFileEnt->second = statError;
254 SeenFileEntries.erase(Filename);
259 assert((openFile || !F) &&
"undesired open file");
263 FileEntry *&UFE = UniqueRealFiles[Status.getUniqueID()];
264 bool ReusingEntry = UFE !=
nullptr;
266 UFE =
new (FilesAlloc.Allocate()) FileEntry();
268 if (!Status.ExposesExternalVFSPath || Status.getName() == Filename) {
270 NamedFileEnt->second = FileEntryRef::MapValue(*UFE, DirInfo);
309 .insert({Status.getName(), FileEntryRef::MapValue(*UFE, DirInfo)})
312 "filename redirected to a non-canonical filename?");
314 "filename from getStatValue() refers to wrong file");
318 NamedFileEnt->second = FileEntryRef::MapValue(Redirection, DirInfo);
321 FileEntryRef ReturnedRef(*NamedFileEnt);
327 UFE->Size = Status.getSize();
328 UFE->ModTime = llvm::sys::toTimeT(Status.getLastModificationTime());
330 UFE->UID = NextFileUID++;
331 UFE->UniqueID = Status.getUniqueID();
332 UFE->IsNamedPipe = Status.getType() == llvm::sys::fs::file_type::fifo_file;
334 Status.getType() == llvm::sys::fs::file_type::character_file;
335 UFE->File = std::move(F);
338 if (
auto PathName = UFE->File->getName())
339 fillRealPathName(UFE, *PathName);
340 }
else if (!openFile) {
342 fillRealPathName(UFE, InterndFileName);
352 auto ContentOrError = [] {
353 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
354 return llvm::MemoryBuffer::getSTDIN();
358 return llvm::createFileError(
"-", ContentOrError.getError());
360 auto Content = std::move(*ContentOrError);
362 Content->getBufferSize(), 0);
364 FE.Content = std::move(Content);
365 FE.IsNamedPipe =
true;
370 FS->visit([Active](llvm::vfs::FileSystem &FileSys) {
371 if (
auto *RFS = dyn_cast<llvm::vfs::RedirectingFileSystem>(&FileSys))
372 RFS->setUsageTrackingActive(Active);
377 time_t ModificationTime) {
381 auto &NamedFileEnt = *SeenFileEntries.insert(
382 {Filename, std::errc::no_such_file_or_directory}).first;
383 if (NamedFileEnt.second) {
391 ++NumFileCacheMisses;
392 addAncestorsAsVirtualDirs(Filename);
401 auto DirInfo = getDirectoryFromFile(Filename.empty() ?
"." : Filename,
404 "The directory of a virtual file should already be in the cache.");
407 llvm::vfs::Status Status;
408 const char *InterndFileName = NamedFileEnt.first().data();
409 if (!getStatValue(InterndFileName, Status,
true,
nullptr)) {
410 Status = llvm::vfs::Status(
411 Status.getName(), Status.getUniqueID(),
412 llvm::sys::toTimePoint(ModificationTime),
413 Status.getUser(), Status.getGroup(), Size,
414 Status.getType(), Status.getPermissions());
416 auto &RealFE = UniqueRealFiles[Status.getUniqueID()];
431 RealFE =
new (FilesAlloc.Allocate())
FileEntry();
432 RealFE->UniqueID = Status.getUniqueID();
433 RealFE->IsNamedPipe =
434 Status.getType() == llvm::sys::fs::file_type::fifo_file;
435 fillRealPathName(RealFE, Status.getName());
440 UFE =
new (FilesAlloc.Allocate())
FileEntry();
441 VirtualFileEntries.push_back(UFE);
446 UFE->ModTime = ModificationTime;
448 UFE->UID = NextFileUID++;
455 llvm::vfs::Status Status;
456 if (getStatValue(VF.
getName(), Status,
true,
nullptr))
459 if (!SeenBypassFileEntries)
460 SeenBypassFileEntries = std::make_unique<
461 llvm::StringMap<llvm::ErrorOr<FileEntryRef::MapValue>>>();
464 auto Insertion = SeenBypassFileEntries->insert(
465 {VF.
getName(), std::errc::no_such_file_or_directory});
466 if (!Insertion.second)
471 BypassFileEntries.push_back(BFE);
473 BFE->Size = Status.getSize();
475 BFE->ModTime = llvm::sys::toTimeT(Status.getLastModificationTime());
476 BFE->UID = NextFileUID++;
484 StringRef pathRef(Path.data(), Path.size());
486 if (FileSystemOpts.WorkingDir.empty()
487 || llvm::sys::path::is_absolute(pathRef))
491 llvm::sys::path::append(NewPath, pathRef);
492 Path = std::move(NewPath);
497 bool Canonicalize)
const {
500 if (!llvm::sys::path::is_absolute(StringRef(Path.data(), Path.size()))) {
501 FS->makeAbsolute(Path);
506 Changed |= llvm::sys::path::remove_dots(Path);
518 llvm::sys::path::remove_dots(AbsPath,
true);
519 UFE->RealPathName = std::string(AbsPath);
522llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
524 bool RequiresNullTerminator,
525 std::optional<int64_t> MaybeLimit,
bool IsText) {
529 return llvm::MemoryBuffer::getMemBuffer(Entry->Content->getMemBufferRef());
531 uint64_t FileSize = Entry->
getSize();
534 FileSize = *MaybeLimit;
541 StringRef Filename = FE.
getName();
544 auto Result = Entry->File->getBuffer(Filename, FileSize,
545 RequiresNullTerminator, isVolatile);
551 return getBufferForFileImpl(Filename, FileSize, isVolatile,
552 RequiresNullTerminator, IsText);
555llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
556FileManager::getBufferForFileImpl(StringRef Filename, int64_t FileSize,
557 bool isVolatile,
bool RequiresNullTerminator,
560 return FS->getBufferForFile(Filename, FileSize, RequiresNullTerminator,
565 return FS->getBufferForFile(FilePath, FileSize, RequiresNullTerminator,
569std::error_code FileManager::getStatValue(StringRef Path,
570 llvm::vfs::Status &Status,
572 std::unique_ptr<llvm::vfs::File> *F,
584 bool isForDir = !isFile;
585 std::error_code RetCode;
587 if (isForDir || !F) {
590 llvm::ErrorOr<llvm::vfs::Status> StatusOrErr = FS->status(Path);
592 RetCode = StatusOrErr.getError();
594 Status = *StatusOrErr;
605 IsText ? FS->openFileForRead(Path) : FS->openFileForReadBinary(Path);
609 RetCode = OwnedFile.getError();
614 llvm::ErrorOr<llvm::vfs::Status> StatusOrErr = (*OwnedFile)->status();
616 Status = *StatusOrErr;
617 *F = std::move(*OwnedFile);
622 RetCode = StatusOrErr.getError();
633 if (Status.isDirectory() != isForDir) {
637 return std::make_error_code(Status.isDirectory()
638 ? std::errc::is_a_directory
639 : std::errc::not_a_directory);
642 return std::error_code();
654 llvm::DenseMap<const void *, llvm::StringRef>::iterator Known =
655 CanonicalNames.find(Entry);
656 if (Known != CanonicalNames.end())
657 return Known->second;
661 StringRef CanonicalName(Name);
665 if (!FS->getRealPath(Name, RealPathBuf)) {
666 if (is_style_windows(llvm::sys::path::Style::native)) {
670 if (!FS->makeAbsolute(AbsPathBuf)) {
671 if (llvm::sys::path::root_name(RealPathBuf) ==
672 llvm::sys::path::root_name(AbsPathBuf)) {
673 CanonicalName = RealPathBuf.str().copy(CanonicalNameStorage);
678 llvm::sys::path::remove_dots(AbsPathBuf,
true);
679 CanonicalName = AbsPathBuf.str().copy(CanonicalNameStorage);
683 CanonicalName = RealPathBuf.str().copy(CanonicalNameStorage);
687 CanonicalNames.insert({Entry, CanonicalName});
688 return CanonicalName;
692 assert(&
Other !=
this &&
"Collecting stats into the same FileManager");
693 NumDirLookups +=
Other.NumDirLookups;
694 NumFileLookups +=
Other.NumFileLookups;
695 NumDirCacheMisses +=
Other.NumDirCacheMisses;
696 NumFileCacheMisses +=
Other.NumFileCacheMisses;
700 llvm::errs() <<
"\n*** File Manager Stats:\n";
701 llvm::errs() << UniqueRealFiles.size() <<
" real files found, "
702 << UniqueRealDirs.size() <<
" real dirs found.\n";
703 llvm::errs() << VirtualFileEntries.size() <<
" virtual files found, "
704 << VirtualDirectoryEntries.size() <<
" virtual dirs found.\n";
705 llvm::errs() << NumDirLookups <<
" dir lookups, "
706 << NumDirCacheMisses <<
" dir cache misses.\n";
707 llvm::errs() << NumFileLookups <<
" file lookups, "
708 << NumFileCacheMisses <<
" file cache misses.\n";
711 if (
auto *T = dyn_cast_or_null<llvm::vfs::TracingFileSystem>(&VFS))
712 llvm::errs() <<
"\n*** Virtual File System Stats:\n"
713 << T->NumStatusCalls <<
" status() calls\n"
714 << T->NumOpenFileForReadCalls <<
" openFileForRead() calls\n"
715 << T->NumDirBeginCalls <<
" dir_begin() calls\n"
716 << T->NumGetRealPathCalls <<
" getRealPath() calls\n"
717 << T->NumExistsCalls <<
" exists() calls\n"
718 << T->NumIsLocalCalls <<
" isLocal() calls\n";
static void normalizeCacheKey(StringRef &Path, std::optional< std::string > &Storage)
Defines the clang::FileManager interface and associated types.
A reference to a DirectoryEntry that includes the name of the directory as it was accessed by the Fil...
StringRef getName() const
const DirectoryEntry & getDirEntry() const
Cached information about one directory (either on disk or in the virtual file system).
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
const FileEntry & getFileEntry() const
StringRef getName() const
The name of this FileEntry.
DirectoryEntryRef getDir() const
Cached information about one file (either on disk or in the virtual file system).
bool isNamedPipe() const
Check whether the file is a named pipe (and thus can't be opened by the native FileManager methods).
void AddStats(const FileManager &Other)
Import statistics from a child FileManager and add them to this current FileManager.
void trackVFSUsage(bool Active)
Enable or disable tracking of VFS usage.
llvm::vfs::FileSystem & getVirtualFileSystem() const
llvm::ErrorOr< std::unique_ptr< llvm::MemoryBuffer > > getBufferForFile(FileEntryRef Entry, bool isVolatile=false, bool RequiresNullTerminator=true, std::optional< int64_t > MaybeLimit=std::nullopt, bool IsText=true)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...
FileManager(const FileSystemOptions &FileSystemOpts, IntrusiveRefCntPtr< llvm::vfs::FileSystem > FS=nullptr)
Construct a file manager, optionally with a custom VFS.
llvm::Expected< FileEntryRef > getSTDIN()
Get the FileEntryRef for stdin, returning an error if stdin cannot be read.
StringRef getCanonicalName(DirectoryEntryRef Dir)
Retrieve the canonical name for a given directory.
FileEntryRef getVirtualFileRef(StringRef Filename, off_t Size, time_t ModificationTime)
Retrieve a file entry for a "virtual" file that acts as if there were a file with the given name on d...
bool FixupRelativePath(SmallVectorImpl< char > &Path) const
If path is not absolute and FileSystemOptions set the working directory, the path is modified to be r...
bool makeAbsolutePath(SmallVectorImpl< char > &Path, bool Canonicalize=false) const
Makes Path absolute taking into account FileSystemOptions and the working directory option,...
OptionalFileEntryRef getBypassFile(FileEntryRef VFE)
Retrieve a FileEntry that bypasses VFE, which is expected to be a virtual file entry,...
static bool fixupRelativePath(const FileSystemOptions &FileSystemOpts, SmallVectorImpl< char > &Path)
Keeps track of options that affect how file operations are performed.
std::string WorkingDir
If set, paths are resolved as if the working directory was set to the value of WorkingDir.
The JSON file list parser is used to communicate input to InstallAPI.
bool isa(CodeGen::Address addr)
CustomizableOptional< FileEntryRef > OptionalFileEntryRef
std::error_code make_error_code(BuildPreambleError Error)
@ Result
The result type of a method or function.
U cast(CodeGen::Address addr)
@ Other
Other implicit parameter.
Type stored in the StringMap.