clang  14.0.0git
DependencyScanningFilesystem.h
Go to the documentation of this file.
1 //===- DependencyScanningFilesystem.h - clang-scan-deps fs ===---*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H
10 #define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H
11 
12 #include "clang/Basic/LLVM.h"
14 #include "llvm/ADT/StringMap.h"
15 #include "llvm/ADT/StringSet.h"
16 #include "llvm/Support/Allocator.h"
17 #include "llvm/Support/ErrorOr.h"
18 #include "llvm/Support/VirtualFileSystem.h"
19 #include <mutex>
20 
21 namespace clang {
22 namespace tooling {
23 namespace dependencies {
24 
25 /// An in-memory representation of a file system entity that is of interest to
26 /// the dependency scanning filesystem.
27 ///
28 /// It represents one of the following:
29 /// - an opened source file with minimized contents and a stat value.
30 /// - an opened source file with original contents and a stat value.
31 /// - a directory entry with its stat value.
32 /// - an error value to represent a file system error.
33 /// - a placeholder with an invalid stat indicating a not yet initialized entry.
35 public:
36  /// Default constructor creates an entry with an invalid stat.
37  CachedFileSystemEntry() : MaybeStat(llvm::vfs::Status()) {}
38 
39  CachedFileSystemEntry(std::error_code Error) : MaybeStat(std::move(Error)) {}
40 
41  /// Create an entry that represents an opened source file with minimized or
42  /// original contents.
43  ///
44  /// The filesystem opens the file even for `stat` calls open to avoid the
45  /// issues with stat + open of minimized files that might lead to a
46  /// mismatching size of the file. If file is not minimized, the full file is
47  /// read and copied into memory to ensure that it's not memory mapped to avoid
48  /// running out of file descriptors.
50  llvm::vfs::FileSystem &FS,
51  bool Minimize = true);
52 
53  /// Create an entry that represents a directory on the filesystem.
54  static CachedFileSystemEntry createDirectoryEntry(llvm::vfs::Status &&Stat);
55 
56  /// \returns True if the entry is valid.
57  bool isValid() const { return !MaybeStat || MaybeStat->isStatusKnown(); }
58 
59  /// \returns True if the current entry points to a directory.
60  bool isDirectory() const { return MaybeStat && MaybeStat->isDirectory(); }
61 
62  /// \returns The error or the file's contents.
63  llvm::ErrorOr<StringRef> getContents() const {
64  if (!MaybeStat)
65  return MaybeStat.getError();
66  assert(!MaybeStat->isDirectory() && "not a file");
67  assert(isValid() && "not initialized");
68  return Contents.str();
69  }
70 
71  /// \returns The error or the status of the entry.
72  llvm::ErrorOr<llvm::vfs::Status> getStatus() const {
73  assert(isValid() && "not initialized");
74  return MaybeStat;
75  }
76 
77  /// \returns the name of the file.
78  StringRef getName() const {
79  assert(isValid() && "not initialized");
80  return MaybeStat->getName();
81  }
82 
83  /// Return the mapping between location -> distance that is used to speed up
84  /// the block skipping in the preprocessor.
86  return PPSkippedRangeMapping;
87  }
88 
91 
94 
95 private:
96  llvm::ErrorOr<llvm::vfs::Status> MaybeStat;
97  // Store the contents in a small string to allow a
98  // move from the small string for the minimized contents.
99  // Note: small size of 1 allows us to store an empty string with an implicit
100  // null terminator without any allocations.
101  llvm::SmallString<1> Contents;
102  PreprocessorSkippedRangeMapping PPSkippedRangeMapping;
103 };
104 
105 /// This class is a shared cache, that caches the 'stat' and 'open' calls to the
106 /// underlying real file system. It distinguishes between minimized and original
107 /// files.
108 ///
109 /// It is sharded based on the hash of the key to reduce the lock contention for
110 /// the worker threads.
112 public:
114  std::mutex ValueLock;
116  };
117 
118  /// Returns a cache entry for the corresponding key.
119  ///
120  /// A new cache entry is created if the key is not in the cache. This is a
121  /// thread safe call.
122  SharedFileSystemEntry &get(StringRef Key, bool Minimized);
123 
124 private:
125  class SingleCache {
126  public:
127  SingleCache();
128 
129  SharedFileSystemEntry &get(StringRef Key);
130 
131  private:
132  struct CacheShard {
133  std::mutex CacheLock;
134  llvm::StringMap<SharedFileSystemEntry, llvm::BumpPtrAllocator> Cache;
135  };
136  std::unique_ptr<CacheShard[]> CacheShards;
137  unsigned NumShards;
138  };
139 
140  SingleCache CacheMinimized;
141  SingleCache CacheOriginal;
142 };
143 
144 /// This class is a local cache, that caches the 'stat' and 'open' calls to the
145 /// underlying real file system. It distinguishes between minimized and original
146 /// files.
148 private:
149  using SingleCache =
150  llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator>;
151 
152  SingleCache CacheMinimized;
153  SingleCache CacheOriginal;
154 
155  SingleCache &selectCache(bool Minimized) {
156  return Minimized ? CacheMinimized : CacheOriginal;
157  }
158 
159 public:
160  void setCachedEntry(StringRef Filename, bool Minimized,
161  const CachedFileSystemEntry *Entry) {
162  SingleCache &Cache = selectCache(Minimized);
163  bool IsInserted = Cache.try_emplace(Filename, Entry).second;
164  (void)IsInserted;
165  assert(IsInserted && "local cache is updated more than once");
166  }
167 
169  bool Minimized) {
170  SingleCache &Cache = selectCache(Minimized);
171  auto It = Cache.find(Filename);
172  return It == Cache.end() ? nullptr : It->getValue();
173  }
174 };
175 
176 /// A virtual file system optimized for the dependency discovery.
177 ///
178 /// It is primarily designed to work with source files whose contents was was
179 /// preprocessed to remove any tokens that are unlikely to affect the dependency
180 /// computation.
181 ///
182 /// This is not a thread safe VFS. A single instance is meant to be used only in
183 /// one thread. Multiple instances are allowed to service multiple threads
184 /// running in parallel.
186 public:
191  : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache),
192  PPSkipMappings(PPSkipMappings) {}
193 
194  llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
195  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
196  openFileForRead(const Twine &Path) override;
197 
198  void clearIgnoredFiles() { IgnoredFiles.clear(); }
199  void ignoreFile(StringRef Filename);
200 
201 private:
202  bool shouldIgnoreFile(StringRef Filename);
203 
204  llvm::ErrorOr<const CachedFileSystemEntry *>
205  getOrCreateFileSystemEntry(const StringRef Filename);
206 
207  /// The global cache shared between worker threads.
209  /// The local cache is used by the worker thread to cache file system queries
210  /// locally instead of querying the global cache every time.
212  /// The optional mapping structure which records information about the
213  /// excluded conditional directive skip mappings that are used by the
214  /// currently active preprocessor.
216  /// The set of files that should not be minimized.
217  llvm::StringSet<> IgnoredFiles;
218 };
219 
220 } // end namespace dependencies
221 } // end namespace tooling
222 } // end namespace clang
223 
224 #endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H
clang::tooling::dependencies::CachedFileSystemEntry::createDirectoryEntry
static CachedFileSystemEntry createDirectoryEntry(llvm::vfs::Status &&Stat)
Create an entry that represents a directory on the filesystem.
Definition: DependencyScanningFilesystem.cpp:95
llvm
Definition: Dominators.h:30
clang::tooling::dependencies::DependencyScanningFilesystemSharedCache::SharedFileSystemEntry
Definition: DependencyScanningFilesystem.h:113
clang::tooling::dependencies::CachedFileSystemEntry
An in-memory representation of a file system entity that is of interest to the dependency scanning fi...
Definition: DependencyScanningFilesystem.h:34
clang::tooling::dependencies::DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem
DependencyScanningWorkerFilesystem(DependencyScanningFilesystemSharedCache &SharedCache, IntrusiveRefCntPtr< llvm::vfs::FileSystem > FS, ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings)
Definition: DependencyScanningFilesystem.h:187
Error
llvm::Error Error
Definition: ByteCodeEmitter.cpp:20
clang::tooling::dependencies::CachedFileSystemEntry::CachedFileSystemEntry
CachedFileSystemEntry(std::error_code Error)
Definition: DependencyScanningFilesystem.h:39
clang::tooling::dependencies::DependencyScanningFilesystemSharedCache::get
SharedFileSystemEntry & get(StringRef Key, bool Minimized)
Returns a cache entry for the corresponding key.
Definition: DependencyScanningFilesystem.cpp:122
clang::tooling::dependencies::CachedFileSystemEntry::getContents
llvm::ErrorOr< StringRef > getContents() const
Definition: DependencyScanningFilesystem.h:63
clang::tooling::dependencies::CachedFileSystemEntry::getName
StringRef getName() const
Definition: DependencyScanningFilesystem.h:78
clang::tooling::dependencies::CachedFileSystemEntry::createFileEntry
static CachedFileSystemEntry createFileEntry(StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize=true)
Create an entry that represents an opened source file with minimized or original contents.
Definition: DependencyScanningFilesystem.cpp:18
Filename
StringRef Filename
Definition: Format.cpp:2333
clang::tooling::dependencies::DependencyScanningWorkerFilesystem::clearIgnoredFiles
void clearIgnoredFiles()
Definition: DependencyScanningFilesystem.h:198
clang::tooling::dependencies::DependencyScanningFilesystemSharedCache::SharedFileSystemEntry::ValueLock
std::mutex ValueLock
Definition: DependencyScanningFilesystem.h:114
clang::ExcludedPreprocessorDirectiveSkipMapping
llvm::DenseMap< const char *, const PreprocessorSkippedRangeMapping * > ExcludedPreprocessorDirectiveSkipMapping
The datastructure that holds the mapping between the active memory buffers and the individual skip ma...
Definition: PreprocessorExcludedConditionalDirectiveSkipMapping.h:26
clang::tooling::dependencies::CachedFileSystemEntry::isDirectory
bool isDirectory() const
Definition: DependencyScanningFilesystem.h:60
clang::tooling::dependencies::DependencyScanningWorkerFilesystem::status
llvm::ErrorOr< llvm::vfs::Status > status(const Twine &Path) override
Definition: DependencyScanningFilesystem.cpp:213
clang::tooling::dependencies::DependencyScanningWorkerFilesystem::ignoreFile
void ignoreFile(StringRef Filename)
Definition: DependencyScanningFilesystem.cpp:153
clang::tooling::dependencies::CachedFileSystemEntry::operator=
CachedFileSystemEntry & operator=(CachedFileSystemEntry &&)=default
clang::tooling::dependencies::DependencyScanningFilesystemLocalCache::setCachedEntry
void setCachedEntry(StringRef Filename, bool Minimized, const CachedFileSystemEntry *Entry)
Definition: DependencyScanningFilesystem.h:160
PreprocessorExcludedConditionalDirectiveSkipMapping.h
llvm::SmallString< 1 >
clang::tooling::dependencies::CachedFileSystemEntry::getPPSkippedRangeMapping
const PreprocessorSkippedRangeMapping & getPPSkippedRangeMapping() const
Return the mapping between location -> distance that is used to speed up the block skipping in the pr...
Definition: DependencyScanningFilesystem.h:85
Cache
TypePropertyCache< Private > Cache
Definition: Type.cpp:3828
clang::tooling::dependencies::DependencyScanningFilesystemSharedCache
This class is a shared cache, that caches the 'stat' and 'open' calls to the underlying real file sys...
Definition: DependencyScanningFilesystem.h:111
clang::tooling::dependencies::DependencyScanningFilesystemLocalCache
This class is a local cache, that caches the 'stat' and 'open' calls to the underlying real file syst...
Definition: DependencyScanningFilesystem.h:147
clang::tooling::dependencies::CachedFileSystemEntry::CachedFileSystemEntry
CachedFileSystemEntry()
Default constructor creates an entry with an invalid stat.
Definition: DependencyScanningFilesystem.h:37
LLVM.h
clang::tooling::dependencies::DependencyScanningWorkerFilesystem
A virtual file system optimized for the dependency discovery.
Definition: DependencyScanningFilesystem.h:185
clang::PreprocessorSkippedRangeMapping
llvm::DenseMap< unsigned, unsigned > PreprocessorSkippedRangeMapping
A mapping from an offset into a buffer to the number of bytes that can be skipped by the preprocessor...
Definition: PreprocessorExcludedConditionalDirectiveSkipMapping.h:21
clang::tooling::dependencies::DependencyScanningFilesystemSharedCache::SharedFileSystemEntry::Value
CachedFileSystemEntry Value
Definition: DependencyScanningFilesystem.h:115
std
Definition: Format.h:4034
clang::tooling::dependencies::DependencyScanningFilesystemLocalCache::getCachedEntry
const CachedFileSystemEntry * getCachedEntry(StringRef Filename, bool Minimized)
Definition: DependencyScanningFilesystem.h:168
clang
Definition: CalledOnceCheck.h:17
clang::tooling::dependencies::DependencyScanningWorkerFilesystem::openFileForRead
llvm::ErrorOr< std::unique_ptr< llvm::vfs::File > > openFileForRead(const Twine &Path) override
Definition: DependencyScanningFilesystem.cpp:275
ProxyFileSystem
clang::tooling::dependencies::CachedFileSystemEntry::isValid
bool isValid() const
Definition: DependencyScanningFilesystem.h:57
llvm::IntrusiveRefCntPtr< llvm::vfs::FileSystem >
clang::tooling::dependencies::CachedFileSystemEntry::getStatus
llvm::ErrorOr< llvm::vfs::Status > getStatus() const
Definition: DependencyScanningFilesystem.h:72