clang  8.0.0svn
VirtualFileSystem.h
Go to the documentation of this file.
1 //===- VirtualFileSystem.h - Virtual File System Layer ----------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// Defines the virtual file system interface vfs::FileSystem.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_CLANG_BASIC_VIRTUALFILESYSTEM_H
16 #define LLVM_CLANG_BASIC_VIRTUALFILESYSTEM_H
17 
18 #include "clang/Basic/LLVM.h"
19 #include "llvm/ADT/IntrusiveRefCntPtr.h"
20 #include "llvm/ADT/None.h"
21 #include "llvm/ADT/Optional.h"
22 #include "llvm/ADT/SmallVector.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/Twine.h"
25 #include "llvm/Support/Chrono.h"
26 #include "llvm/Support/ErrorOr.h"
27 #include "llvm/Support/FileSystem.h"
28 #include "llvm/Support/SourceMgr.h"
29 #include <cassert>
30 #include <cstdint>
31 #include <ctime>
32 #include <memory>
33 #include <stack>
34 #include <string>
35 #include <system_error>
36 #include <utility>
37 #include <vector>
38 
39 namespace llvm {
40 
41 class MemoryBuffer;
42 
43 } // namespace llvm
44 
45 namespace clang {
46 namespace vfs {
47 
48 /// The result of a \p status operation.
49 class Status {
50  std::string Name;
51  llvm::sys::fs::UniqueID UID;
52  llvm::sys::TimePoint<> MTime;
53  uint32_t User;
54  uint32_t Group;
55  uint64_t Size;
56  llvm::sys::fs::file_type Type = llvm::sys::fs::file_type::status_error;
57  llvm::sys::fs::perms Perms;
58 
59 public:
60  // FIXME: remove when files support multiple names
61  bool IsVFSMapped = false;
62 
63  Status() = default;
64  Status(const llvm::sys::fs::file_status &Status);
65  Status(StringRef Name, llvm::sys::fs::UniqueID UID,
66  llvm::sys::TimePoint<> MTime, uint32_t User, uint32_t Group,
67  uint64_t Size, llvm::sys::fs::file_type Type,
68  llvm::sys::fs::perms Perms);
69 
70  /// Get a copy of a Status with a different name.
71  static Status copyWithNewName(const Status &In, StringRef NewName);
72  static Status copyWithNewName(const llvm::sys::fs::file_status &In,
73  StringRef NewName);
74 
75  /// Returns the name that should be used for this file or directory.
76  StringRef getName() const { return Name; }
77 
78  /// @name Status interface from llvm::sys::fs
79  /// @{
80  llvm::sys::fs::file_type getType() const { return Type; }
81  llvm::sys::fs::perms getPermissions() const { return Perms; }
82  llvm::sys::TimePoint<> getLastModificationTime() const { return MTime; }
83  llvm::sys::fs::UniqueID getUniqueID() const { return UID; }
84  uint32_t getUser() const { return User; }
85  uint32_t getGroup() const { return Group; }
86  uint64_t getSize() const { return Size; }
87  /// @}
88  /// @name Status queries
89  /// These are static queries in llvm::sys::fs.
90  /// @{
91  bool equivalent(const Status &Other) const;
92  bool isDirectory() const;
93  bool isRegularFile() const;
94  bool isOther() const;
95  bool isSymlink() const;
96  bool isStatusKnown() const;
97  bool exists() const;
98  /// @}
99 };
100 
101 /// Represents an open file.
102 class File {
103 public:
104  /// Destroy the file after closing it (if open).
105  /// Sub-classes should generally call close() inside their destructors. We
106  /// cannot do that from the base class, since close is virtual.
107  virtual ~File();
108 
109  /// Get the status of the file.
110  virtual llvm::ErrorOr<Status> status() = 0;
111 
112  /// Get the name of the file
113  virtual llvm::ErrorOr<std::string> getName() {
114  if (auto Status = status())
115  return Status->getName().str();
116  else
117  return Status.getError();
118  }
119 
120  /// Get the contents of the file as a \p MemoryBuffer.
121  virtual llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
122  getBuffer(const Twine &Name, int64_t FileSize = -1,
123  bool RequiresNullTerminator = true, bool IsVolatile = false) = 0;
124 
125  /// Closes the file.
126  virtual std::error_code close() = 0;
127 };
128 
129 /// A member of a directory, yielded by a directory_iterator.
130 /// Only information available on most platforms is included.
132  std::string Path;
133  llvm::sys::fs::file_type Type;
134 
135 public:
136  directory_entry() = default;
137  directory_entry(std::string Path, llvm::sys::fs::file_type Type)
138  : Path(std::move(Path)), Type(Type) {}
139 
140  llvm::StringRef path() const { return Path; }
141  llvm::sys::fs::file_type type() const { return Type; }
142 };
143 
144 namespace detail {
145 
146 /// An interface for virtual file systems to provide an iterator over the
147 /// (non-recursive) contents of a directory.
148 struct DirIterImpl {
149  virtual ~DirIterImpl();
150 
151  /// Sets \c CurrentEntry to the next entry in the directory on success,
152  /// to directory_entry() at end, or returns a system-defined \c error_code.
153  virtual std::error_code increment() = 0;
154 
156 };
157 
158 } // namespace detail
159 
160 /// An input iterator over the entries in a virtual path, similar to
161 /// llvm::sys::fs::directory_iterator.
163  std::shared_ptr<detail::DirIterImpl> Impl; // Input iterator semantics on copy
164 
165 public:
166  directory_iterator(std::shared_ptr<detail::DirIterImpl> I)
167  : Impl(std::move(I)) {
168  assert(Impl.get() != nullptr && "requires non-null implementation");
169  if (Impl->CurrentEntry.path().empty())
170  Impl.reset(); // Normalize the end iterator to Impl == nullptr.
171  }
172 
173  /// Construct an 'end' iterator.
174  directory_iterator() = default;
175 
176  /// Equivalent to operator++, with an error code.
177  directory_iterator &increment(std::error_code &EC) {
178  assert(Impl && "attempting to increment past end");
179  EC = Impl->increment();
180  if (Impl->CurrentEntry.path().empty())
181  Impl.reset(); // Normalize the end iterator to Impl == nullptr.
182  return *this;
183  }
184 
185  const directory_entry &operator*() const { return Impl->CurrentEntry; }
186  const directory_entry *operator->() const { return &Impl->CurrentEntry; }
187 
188  bool operator==(const directory_iterator &RHS) const {
189  if (Impl && RHS.Impl)
190  return Impl->CurrentEntry.path() == RHS.Impl->CurrentEntry.path();
191  return !Impl && !RHS.Impl;
192  }
193  bool operator!=(const directory_iterator &RHS) const {
194  return !(*this == RHS);
195  }
196 };
197 
198 class FileSystem;
199 
200 /// An input iterator over the recursive contents of a virtual path,
201 /// similar to llvm::sys::fs::recursive_directory_iterator.
203  using IterState =
204  std::stack<directory_iterator, std::vector<directory_iterator>>;
205 
206  FileSystem *FS;
207  std::shared_ptr<IterState> State; // Input iterator semantics on copy.
208 
209 public:
210  recursive_directory_iterator(FileSystem &FS, const Twine &Path,
211  std::error_code &EC);
212 
213  /// Construct an 'end' iterator.
214  recursive_directory_iterator() = default;
215 
216  /// Equivalent to operator++, with an error code.
217  recursive_directory_iterator &increment(std::error_code &EC);
218 
219  const directory_entry &operator*() const { return *State->top(); }
220  const directory_entry *operator->() const { return &*State->top(); }
221 
222  bool operator==(const recursive_directory_iterator &Other) const {
223  return State == Other.State; // identity
224  }
225  bool operator!=(const recursive_directory_iterator &RHS) const {
226  return !(*this == RHS);
227  }
228 
229  /// Gets the current level. Starting path is at level 0.
230  int level() const {
231  assert(!State->empty() && "Cannot get level without any iteration state");
232  return State->size()-1;
233  }
234 };
235 
236 /// The virtual file system interface.
237 class FileSystem : public llvm::ThreadSafeRefCountedBase<FileSystem> {
238 public:
239  virtual ~FileSystem();
240 
241  /// Get the status of the entry at \p Path, if one exists.
242  virtual llvm::ErrorOr<Status> status(const Twine &Path) = 0;
243 
244  /// Get a \p File object for the file at \p Path, if one exists.
245  virtual llvm::ErrorOr<std::unique_ptr<File>>
246  openFileForRead(const Twine &Path) = 0;
247 
248  /// This is a convenience method that opens a file, gets its content and then
249  /// closes the file.
250  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
251  getBufferForFile(const Twine &Name, int64_t FileSize = -1,
252  bool RequiresNullTerminator = true, bool IsVolatile = false);
253 
254  /// Get a directory_iterator for \p Dir.
255  /// \note The 'end' iterator is directory_iterator().
256  virtual directory_iterator dir_begin(const Twine &Dir,
257  std::error_code &EC) = 0;
258 
259  /// Set the working directory. This will affect all following operations on
260  /// this file system and may propagate down for nested file systems.
261  virtual std::error_code setCurrentWorkingDirectory(const Twine &Path) = 0;
262 
263  /// Get the working directory of this file system.
264  virtual llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const = 0;
265 
266  /// Gets real path of \p Path e.g. collapse all . and .. patterns, resolve
267  /// symlinks. For real file system, this uses `llvm::sys::fs::real_path`.
268  /// This returns errc::operation_not_permitted if not implemented by subclass.
269  virtual std::error_code getRealPath(const Twine &Path,
270  SmallVectorImpl<char> &Output) const;
271 
272  /// Check whether a file exists. Provided for convenience.
273  bool exists(const Twine &Path);
274 
275  /// Make \a Path an absolute path.
276  ///
277  /// Makes \a Path absolute using the current directory if it is not already.
278  /// An empty \a Path will result in the current directory.
279  ///
280  /// /absolute/path => /absolute/path
281  /// relative/../path => <current-directory>/relative/../path
282  ///
283  /// \param Path A path that is modified to be an absolute path.
284  /// \returns success if \a path has been made absolute, otherwise a
285  /// platform-specific error_code.
286  std::error_code makeAbsolute(SmallVectorImpl<char> &Path) const;
287 };
288 
289 /// Gets an \p vfs::FileSystem for the 'real' file system, as seen by
290 /// the operating system.
292 
293 /// A file system that allows overlaying one \p AbstractFileSystem on top
294 /// of another.
295 ///
296 /// Consists of a stack of >=1 \p FileSystem objects, which are treated as being
297 /// one merged file system. When there is a directory that exists in more than
298 /// one file system, the \p OverlayFileSystem contains a directory containing
299 /// the union of their contents. The attributes (permissions, etc.) of the
300 /// top-most (most recently added) directory are used. When there is a file
301 /// that exists in more than one file system, the file in the top-most file
302 /// system overrides the other(s).
305 
306  /// The stack of file systems, implemented as a list in order of
307  /// their addition.
308  FileSystemList FSList;
309 
310 public:
312 
313  /// Pushes a file system on top of the stack.
314  void pushOverlay(IntrusiveRefCntPtr<FileSystem> FS);
315 
316  llvm::ErrorOr<Status> status(const Twine &Path) override;
317  llvm::ErrorOr<std::unique_ptr<File>>
318  openFileForRead(const Twine &Path) override;
319  directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
320  llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override;
321  std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
322  std::error_code getRealPath(const Twine &Path,
323  SmallVectorImpl<char> &Output) const override;
324 
325  using iterator = FileSystemList::reverse_iterator;
326  using const_iterator = FileSystemList::const_reverse_iterator;
327 
328  /// Get an iterator pointing to the most recently added file system.
329  iterator overlays_begin() { return FSList.rbegin(); }
330  const_iterator overlays_begin() const { return FSList.rbegin(); }
331 
332  /// Get an iterator pointing one-past the least recently added file
333  /// system.
334  iterator overlays_end() { return FSList.rend(); }
335  const_iterator overlays_end() const { return FSList.rend(); }
336 };
337 
338 namespace detail {
339 
340 class InMemoryDirectory;
341 class InMemoryFile;
342 
343 } // namespace detail
344 
345 /// An in-memory file system.
347  std::unique_ptr<detail::InMemoryDirectory> Root;
348  std::string WorkingDirectory;
349  bool UseNormalizedPaths = true;
350 
351  /// If HardLinkTarget is non-null, a hardlink is created to the To path which
352  /// must be a file. If it is null then it adds the file as the public addFile.
353  bool addFile(const Twine &Path, time_t ModificationTime,
354  std::unique_ptr<llvm::MemoryBuffer> Buffer,
358  const detail::InMemoryFile *HardLinkTarget);
359 
360 public:
361  explicit InMemoryFileSystem(bool UseNormalizedPaths = true);
362  ~InMemoryFileSystem() override;
363 
364  /// Add a file containing a buffer or a directory to the VFS with a
365  /// path. The VFS owns the buffer. If present, User, Group, Type
366  /// and Perms apply to the newly-created file or directory.
367  /// \return true if the file or directory was successfully added,
368  /// false if the file or directory already exists in the file system with
369  /// different contents.
370  bool addFile(const Twine &Path, time_t ModificationTime,
371  std::unique_ptr<llvm::MemoryBuffer> Buffer,
372  Optional<uint32_t> User = None, Optional<uint32_t> Group = None,
374  Optional<llvm::sys::fs::perms> Perms = None);
375 
376  /// Add a hard link to a file.
377  /// Here hard links are not intended to be fully equivalent to the classical
378  /// filesystem. Both the hard link and the file share the same buffer and
379  /// status (and thus have the same UniqueID). Because of this there is no way
380  /// to distinguish between the link and the file after the link has been
381  /// added.
382  ///
383  /// The To path must be an existing file or a hardlink. The From file must not
384  /// have been added before. The To Path must not be a directory. The From Node
385  /// is added as a hard link which points to the resolved file of To Node.
386  /// \return true if the above condition is satisfied and hardlink was
387  /// successfully created, false otherwise.
388  bool addHardLink(const Twine &From, const Twine &To);
389 
390  /// Add a buffer to the VFS with a path. The VFS does not own the buffer.
391  /// If present, User, Group, Type and Perms apply to the newly-created file
392  /// or directory.
393  /// \return true if the file or directory was successfully added,
394  /// false if the file or directory already exists in the file system with
395  /// different contents.
396  bool addFileNoOwn(const Twine &Path, time_t ModificationTime,
397  llvm::MemoryBuffer *Buffer,
398  Optional<uint32_t> User = None,
399  Optional<uint32_t> Group = None,
401  Optional<llvm::sys::fs::perms> Perms = None);
402 
403  std::string toString() const;
404 
405  /// Return true if this file system normalizes . and .. in paths.
406  bool useNormalizedPaths() const { return UseNormalizedPaths; }
407 
408  llvm::ErrorOr<Status> status(const Twine &Path) override;
409  llvm::ErrorOr<std::unique_ptr<File>>
410  openFileForRead(const Twine &Path) override;
411  directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
412 
413  llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
414  return WorkingDirectory;
415  }
416  /// Canonicalizes \p Path by combining with the current working
417  /// directory and normalizing the path (e.g. remove dots). If the current
418  /// working directory is not set, this returns errc::operation_not_permitted.
419  ///
420  /// This doesn't resolve symlinks as they are not supported in in-memory file
421  /// system.
422  std::error_code getRealPath(const Twine &Path,
423  SmallVectorImpl<char> &Output) const override;
424 
425  std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
426 };
427 
428 /// Get a globally unique ID for a virtual file or directory.
429 llvm::sys::fs::UniqueID getNextVirtualUniqueID();
430 
431 /// Gets a \p FileSystem for a virtual file system described in YAML
432 /// format.
434 getVFSFromYAML(std::unique_ptr<llvm::MemoryBuffer> Buffer,
435  llvm::SourceMgr::DiagHandlerTy DiagHandler,
436  StringRef YAMLFilePath,
437  void *DiagContext = nullptr,
439 
440 struct YAMLVFSEntry {
441  template <typename T1, typename T2> YAMLVFSEntry(T1 &&VPath, T2 &&RPath)
442  : VPath(std::forward<T1>(VPath)), RPath(std::forward<T2>(RPath)) {}
443  std::string VPath;
444  std::string RPath;
445 };
446 
447 /// Collect all pairs of <virtual path, real path> entries from the
448 /// \p YAMLFilePath. This is used by the module dependency collector to forward
449 /// the entries into the reproducer output VFS YAML file.
450 void collectVFSFromYAML(
451  std::unique_ptr<llvm::MemoryBuffer> Buffer,
452  llvm::SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath,
453  SmallVectorImpl<YAMLVFSEntry> &CollectedEntries,
454  void *DiagContext = nullptr,
456 
458  std::vector<YAMLVFSEntry> Mappings;
459  Optional<bool> IsCaseSensitive;
460  Optional<bool> IsOverlayRelative;
461  Optional<bool> UseExternalNames;
462  Optional<bool> IgnoreNonExistentContents;
463  std::string OverlayDir;
464 
465 public:
466  YAMLVFSWriter() = default;
467 
468  void addFileMapping(StringRef VirtualPath, StringRef RealPath);
469 
470  void setCaseSensitivity(bool CaseSensitive) {
471  IsCaseSensitive = CaseSensitive;
472  }
473 
474  void setUseExternalNames(bool UseExtNames) {
475  UseExternalNames = UseExtNames;
476  }
477 
478  void setIgnoreNonExistentContents(bool IgnoreContents) {
479  IgnoreNonExistentContents = IgnoreContents;
480  }
481 
482  void setOverlayDir(StringRef OverlayDirectory) {
483  IsOverlayRelative = true;
484  OverlayDir.assign(OverlayDirectory.str());
485  }
486 
487  void write(llvm::raw_ostream &OS);
488 };
489 
490 } // namespace vfs
491 } // namespace clang
492 
493 #endif // LLVM_CLANG_BASIC_VIRTUALFILESYSTEM_H
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:30
const directory_entry & operator*() const
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the &#39;real&#39; file system, as seen by the operating system.
void setOverlayDir(StringRef OverlayDirectory)
llvm::sys::fs::perms getPermissions() const
The base class of the type hierarchy.
Definition: Type.h:1415
StringRef getName() const
Returns the name that should be used for this file or directory.
void setIgnoreNonExistentContents(bool IgnoreContents)
uint32_t getUser() const
bool operator==(const recursive_directory_iterator &Other) const
The virtual file system interface.
llvm::ErrorOr< std::string > getCurrentWorkingDirectory() const override
Get the working directory of this file system.
LineState State
IntrusiveRefCntPtr< FileSystem > getVFSFromYAML(std::unique_ptr< llvm::MemoryBuffer > Buffer, llvm::SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath, void *DiagContext=nullptr, IntrusiveRefCntPtr< FileSystem > ExternalFS=getRealFileSystem())
Gets a FileSystem for a virtual file system described in YAML format.
const directory_entry * operator->() const
Definition: Format.h:2031
An input iterator over the recursive contents of a virtual path, similar to llvm::sys::fs::recursive_...
directory_iterator(std::shared_ptr< detail::DirIterImpl > I)
An in-memory file system.
bool useNormalizedPaths() const
Return true if this file system normalizes . and .. in paths.
A file system that allows overlaying one AbstractFileSystem on top of another.
directory_iterator & increment(std::error_code &EC)
Equivalent to operator++, with an error code.
A member of a directory, yielded by a directory_iterator.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
int level() const
Gets the current level. Starting path is at level 0.
The result of a status operation.
iterator overlays_end()
Get an iterator pointing one-past the least recently added file system.
Represents an open file.
void setCaseSensitivity(bool CaseSensitive)
virtual llvm::ErrorOr< std::string > getName()
Get the name of the file.
const directory_entry * operator->() const
FileSystemList::reverse_iterator iterator
llvm::sys::fs::file_type getType() const
llvm::sys::fs::file_type type() const
void collectVFSFromYAML(std::unique_ptr< llvm::MemoryBuffer > Buffer, llvm::SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath, SmallVectorImpl< YAMLVFSEntry > &CollectedEntries, void *DiagContext=nullptr, IntrusiveRefCntPtr< FileSystem > ExternalFS=getRealFileSystem())
Collect all pairs of <virtual path, real path> entries from the YAMLFilePath.
iterator overlays_begin()
Get an iterator pointing to the most recently added file system.
llvm::sys::TimePoint getLastModificationTime() const
llvm::StringRef path() const
YAMLVFSEntry(T1 &&VPath, T2 &&RPath)
Dataflow Directional Tag Classes.
uint64_t getSize() const
llvm::sys::fs::UniqueID getNextVirtualUniqueID()
Get a globally unique ID for a virtual file or directory.
bool operator==(const directory_iterator &RHS) const
FileSystemList::const_reverse_iterator const_iterator
std::string toString(const til::SExpr *E)
const directory_entry & operator*() const
void setUseExternalNames(bool UseExtNames)
const_iterator overlays_end() const
llvm::sys::fs::UniqueID getUniqueID() const
An input iterator over the entries in a virtual path, similar to llvm::sys::fs::directory_iterator.
const_iterator overlays_begin() const
An interface for virtual file systems to provide an iterator over the (non-recursive) contents of a d...
bool operator!=(const directory_iterator &RHS) const
uint32_t getGroup() const
bool operator!=(const recursive_directory_iterator &RHS) const
directory_entry(std::string Path, llvm::sys::fs::file_type Type)