clang  6.0.0svn
ModuleManager.cpp
Go to the documentation of this file.
1 //===--- ModuleManager.cpp - Module Manager ---------------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines the ModuleManager class, which manages a set of loaded
11 // modules for the ASTReader.
12 //
13 //===----------------------------------------------------------------------===//
17 #include "clang/Lex/HeaderSearch.h"
18 #include "clang/Lex/ModuleMap.h"
20 #include "llvm/Support/MemoryBuffer.h"
21 #include "llvm/Support/Path.h"
22 #include <system_error>
23 
24 #ifndef NDEBUG
25 #include "llvm/Support/GraphWriter.h"
26 #endif
27 
28 using namespace clang;
29 using namespace serialization;
30 
32  const FileEntry *Entry = FileMgr.getFile(Name, /*openFile=*/false,
33  /*cacheFailure=*/false);
34  if (Entry)
35  return lookup(Entry);
36 
37  return nullptr;
38 }
39 
41  if (const Module *Mod = HeaderSearchInfo.getModuleMap().findModule(Name))
42  if (const FileEntry *File = Mod->getASTFile())
43  return lookup(File);
44 
45  return nullptr;
46 }
47 
49  auto Known = Modules.find(File);
50  if (Known == Modules.end())
51  return nullptr;
52 
53  return Known->second;
54 }
55 
56 std::unique_ptr<llvm::MemoryBuffer>
57 ModuleManager::lookupBuffer(StringRef Name) {
58  const FileEntry *Entry = FileMgr.getFile(Name, /*openFile=*/false,
59  /*cacheFailure=*/false);
60  return std::move(InMemoryBuffers[Entry]);
61 }
62 
63 static bool checkSignature(ASTFileSignature Signature,
64  ASTFileSignature ExpectedSignature,
65  std::string &ErrorStr) {
66  if (!ExpectedSignature || Signature == ExpectedSignature)
67  return false;
68 
69  ErrorStr =
70  Signature ? "signature mismatch" : "could not read module signature";
71  return true;
72 }
73 
74 static void updateModuleImports(ModuleFile &MF, ModuleFile *ImportedBy,
75  SourceLocation ImportLoc) {
76  if (ImportedBy) {
77  MF.ImportedBy.insert(ImportedBy);
78  ImportedBy->Imports.insert(&MF);
79  } else {
80  if (!MF.DirectlyImported)
81  MF.ImportLoc = ImportLoc;
82 
83  MF.DirectlyImported = true;
84  }
85 }
86 
89  SourceLocation ImportLoc, ModuleFile *ImportedBy,
90  unsigned Generation,
91  off_t ExpectedSize, time_t ExpectedModTime,
92  ASTFileSignature ExpectedSignature,
93  ASTFileSignatureReader ReadSignature,
95  std::string &ErrorStr) {
96  Module = nullptr;
97 
98  // Look for the file entry. This only fails if the expected size or
99  // modification time differ.
100  const FileEntry *Entry;
101  if (Type == MK_ExplicitModule || Type == MK_PrebuiltModule) {
102  // If we're not expecting to pull this file out of the module cache, it
103  // might have a different mtime due to being moved across filesystems in
104  // a distributed build. The size must still match, though. (As must the
105  // contents, but we can't check that.)
106  ExpectedModTime = 0;
107  }
108  if (lookupModuleFile(FileName, ExpectedSize, ExpectedModTime, Entry)) {
109  ErrorStr = "module file out of date";
110  return OutOfDate;
111  }
112 
113  if (!Entry && FileName != "-") {
114  ErrorStr = "module file not found";
115  return Missing;
116  }
117 
118  // Check whether we already loaded this module, before
119  if (ModuleFile *ModuleEntry = Modules.lookup(Entry)) {
120  // Check the stored signature.
121  if (checkSignature(ModuleEntry->Signature, ExpectedSignature, ErrorStr))
122  return OutOfDate;
123 
124  Module = ModuleEntry;
125  updateModuleImports(*ModuleEntry, ImportedBy, ImportLoc);
126  return AlreadyLoaded;
127  }
128 
129  // Allocate a new module.
130  auto NewModule = llvm::make_unique<ModuleFile>(Type, Generation);
131  NewModule->Index = Chain.size();
132  NewModule->FileName = FileName.str();
133  NewModule->File = Entry;
134  NewModule->ImportLoc = ImportLoc;
135  NewModule->InputFilesValidationTimestamp = 0;
136 
137  if (NewModule->Kind == MK_ImplicitModule) {
138  std::string TimestampFilename = NewModule->getTimestampFilename();
139  vfs::Status Status;
140  // A cached stat value would be fine as well.
141  if (!FileMgr.getNoncachedStatValue(TimestampFilename, Status))
142  NewModule->InputFilesValidationTimestamp =
143  llvm::sys::toTimeT(Status.getLastModificationTime());
144  }
145 
146  // Load the contents of the module
147  if (std::unique_ptr<llvm::MemoryBuffer> Buffer = lookupBuffer(FileName)) {
148  // The buffer was already provided for us.
149  NewModule->Buffer = &PCMCache->addBuffer(FileName, std::move(Buffer));
150  } else if (llvm::MemoryBuffer *Buffer = PCMCache->lookupBuffer(FileName)) {
151  NewModule->Buffer = Buffer;
152  } else {
153  // Open the AST file.
154  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buf((std::error_code()));
155  if (FileName == "-") {
156  Buf = llvm::MemoryBuffer::getSTDIN();
157  } else {
158  // Leave the FileEntry open so if it gets read again by another
159  // ModuleManager it must be the same underlying file.
160  // FIXME: Because FileManager::getFile() doesn't guarantee that it will
161  // give us an open file, this may not be 100% reliable.
162  Buf = FileMgr.getBufferForFile(NewModule->File,
163  /*IsVolatile=*/false,
164  /*ShouldClose=*/false);
165  }
166 
167  if (!Buf) {
168  ErrorStr = Buf.getError().message();
169  return Missing;
170  }
171 
172  NewModule->Buffer = &PCMCache->addBuffer(FileName, std::move(*Buf));
173  }
174 
175  // Initialize the stream.
176  NewModule->Data = PCHContainerRdr.ExtractPCH(*NewModule->Buffer);
177 
178  // Read the signature eagerly now so that we can check it. Avoid calling
179  // ReadSignature unless there's something to check though.
180  if (ExpectedSignature && checkSignature(ReadSignature(NewModule->Data),
181  ExpectedSignature, ErrorStr)) {
182  // Try to remove the buffer. If it can't be removed, then it was already
183  // validated by this process.
184  if (!PCMCache->tryToRemoveBuffer(NewModule->FileName))
185  FileMgr.invalidateCache(NewModule->File);
186  return OutOfDate;
187  }
188 
189  // We're keeping this module. Store it everywhere.
190  Module = Modules[Entry] = NewModule.get();
191 
192  updateModuleImports(*NewModule, ImportedBy, ImportLoc);
193 
194  if (!NewModule->isModule())
195  PCHChain.push_back(NewModule.get());
196  if (!ImportedBy)
197  Roots.push_back(NewModule.get());
198 
199  Chain.push_back(std::move(NewModule));
200  return NewlyLoaded;
201 }
202 
204  ModuleIterator First,
205  llvm::SmallPtrSetImpl<ModuleFile *> &LoadedSuccessfully,
206  ModuleMap *modMap) {
207  auto Last = end();
208  if (First == Last)
209  return;
210 
211 
212  // Explicitly clear VisitOrder since we might not notice it is stale.
213  VisitOrder.clear();
214 
215  // Collect the set of module file pointers that we'll be removing.
216  llvm::SmallPtrSet<ModuleFile *, 4> victimSet(
217  (llvm::pointer_iterator<ModuleIterator>(First)),
218  (llvm::pointer_iterator<ModuleIterator>(Last)));
219 
220  auto IsVictim = [&](ModuleFile *MF) {
221  return victimSet.count(MF);
222  };
223  // Remove any references to the now-destroyed modules.
224  for (auto I = begin(); I != First; ++I) {
225  I->Imports.remove_if(IsVictim);
226  I->ImportedBy.remove_if(IsVictim);
227  }
228  Roots.erase(std::remove_if(Roots.begin(), Roots.end(), IsVictim),
229  Roots.end());
230 
231  // Remove the modules from the PCH chain.
232  for (auto I = First; I != Last; ++I) {
233  if (!I->isModule()) {
234  PCHChain.erase(std::find(PCHChain.begin(), PCHChain.end(), &*I),
235  PCHChain.end());
236  break;
237  }
238  }
239 
240  // Delete the modules and erase them from the various structures.
241  for (ModuleIterator victim = First; victim != Last; ++victim) {
242  Modules.erase(victim->File);
243 
244  if (modMap) {
245  StringRef ModuleName = victim->ModuleName;
246  if (Module *mod = modMap->findModule(ModuleName)) {
247  mod->setASTFile(nullptr);
248  }
249  }
250 
251  // Files that didn't make it through ReadASTCore successfully will be
252  // rebuilt (or there was an error). Invalidate them so that we can load the
253  // new files that will be renamed over the old ones.
254  //
255  // The PCMCache tracks whether the module was successfully loaded in another
256  // thread/context; in that case, it won't need to be rebuilt (and we can't
257  // safely invalidate it anyway).
258  if (LoadedSuccessfully.count(&*victim) == 0 &&
259  !PCMCache->tryToRemoveBuffer(victim->FileName))
260  FileMgr.invalidateCache(victim->File);
261  }
262 
263  // Delete the modules.
264  Chain.erase(Chain.begin() + (First - begin()), Chain.end());
265 }
266 
267 void
269  std::unique_ptr<llvm::MemoryBuffer> Buffer) {
270 
271  const FileEntry *Entry =
272  FileMgr.getVirtualFile(FileName, Buffer->getBufferSize(), 0);
273  InMemoryBuffers[Entry] = std::move(Buffer);
274 }
275 
276 ModuleManager::VisitState *ModuleManager::allocateVisitState() {
277  // Fast path: if we have a cached state, use it.
278  if (FirstVisitState) {
279  VisitState *Result = FirstVisitState;
280  FirstVisitState = FirstVisitState->NextState;
281  Result->NextState = nullptr;
282  return Result;
283  }
284 
285  // Allocate and return a new state.
286  return new VisitState(size());
287 }
288 
289 void ModuleManager::returnVisitState(VisitState *State) {
290  assert(State->NextState == nullptr && "Visited state is in list?");
291  State->NextState = FirstVisitState;
292  FirstVisitState = State;
293 }
294 
296  GlobalIndex = Index;
297  if (!GlobalIndex) {
298  ModulesInCommonWithGlobalIndex.clear();
299  return;
300  }
301 
302  // Notify the global module index about all of the modules we've already
303  // loaded.
304  for (ModuleFile &M : *this)
305  if (!GlobalIndex->loadedModuleFile(&M))
306  ModulesInCommonWithGlobalIndex.push_back(&M);
307 }
308 
310  if (!GlobalIndex || GlobalIndex->loadedModuleFile(MF))
311  return;
312 
313  ModulesInCommonWithGlobalIndex.push_back(MF);
314 }
315 
317  const PCHContainerReader &PCHContainerRdr,
318  const HeaderSearch& HeaderSearchInfo)
319  : FileMgr(FileMgr), PCMCache(&PCMCache), PCHContainerRdr(PCHContainerRdr),
320  HeaderSearchInfo (HeaderSearchInfo), GlobalIndex(),
321  FirstVisitState(nullptr) {}
322 
323 ModuleManager::~ModuleManager() { delete FirstVisitState; }
324 
325 void ModuleManager::visit(llvm::function_ref<bool(ModuleFile &M)> Visitor,
326  llvm::SmallPtrSetImpl<ModuleFile *> *ModuleFilesHit) {
327  // If the visitation order vector is the wrong size, recompute the order.
328  if (VisitOrder.size() != Chain.size()) {
329  unsigned N = size();
330  VisitOrder.clear();
331  VisitOrder.reserve(N);
332 
333  // Record the number of incoming edges for each module. When we
334  // encounter a module with no incoming edges, push it into the queue
335  // to seed the queue.
337  Queue.reserve(N);
338  llvm::SmallVector<unsigned, 4> UnusedIncomingEdges;
339  UnusedIncomingEdges.resize(size());
340  for (ModuleFile &M : llvm::reverse(*this)) {
341  unsigned Size = M.ImportedBy.size();
342  UnusedIncomingEdges[M.Index] = Size;
343  if (!Size)
344  Queue.push_back(&M);
345  }
346 
347  // Traverse the graph, making sure to visit a module before visiting any
348  // of its dependencies.
349  while (!Queue.empty()) {
350  ModuleFile *CurrentModule = Queue.pop_back_val();
351  VisitOrder.push_back(CurrentModule);
352 
353  // For any module that this module depends on, push it on the
354  // stack (if it hasn't already been marked as visited).
355  for (auto M = CurrentModule->Imports.rbegin(),
356  MEnd = CurrentModule->Imports.rend();
357  M != MEnd; ++M) {
358  // Remove our current module as an impediment to visiting the
359  // module we depend on. If we were the last unvisited module
360  // that depends on this particular module, push it into the
361  // queue to be visited.
362  unsigned &NumUnusedEdges = UnusedIncomingEdges[(*M)->Index];
363  if (NumUnusedEdges && (--NumUnusedEdges == 0))
364  Queue.push_back(*M);
365  }
366  }
367 
368  assert(VisitOrder.size() == N && "Visitation order is wrong?");
369 
370  delete FirstVisitState;
371  FirstVisitState = nullptr;
372  }
373 
374  VisitState *State = allocateVisitState();
375  unsigned VisitNumber = State->NextVisitNumber++;
376 
377  // If the caller has provided us with a hit-set that came from the global
378  // module index, mark every module file in common with the global module
379  // index that is *not* in that set as 'visited'.
380  if (ModuleFilesHit && !ModulesInCommonWithGlobalIndex.empty()) {
381  for (unsigned I = 0, N = ModulesInCommonWithGlobalIndex.size(); I != N; ++I)
382  {
383  ModuleFile *M = ModulesInCommonWithGlobalIndex[I];
384  if (!ModuleFilesHit->count(M))
385  State->VisitNumber[M->Index] = VisitNumber;
386  }
387  }
388 
389  for (unsigned I = 0, N = VisitOrder.size(); I != N; ++I) {
390  ModuleFile *CurrentModule = VisitOrder[I];
391  // Should we skip this module file?
392  if (State->VisitNumber[CurrentModule->Index] == VisitNumber)
393  continue;
394 
395  // Visit the module.
396  assert(State->VisitNumber[CurrentModule->Index] == VisitNumber - 1);
397  State->VisitNumber[CurrentModule->Index] = VisitNumber;
398  if (!Visitor(*CurrentModule))
399  continue;
400 
401  // The visitor has requested that cut off visitation of any
402  // module that the current module depends on. To indicate this
403  // behavior, we mark all of the reachable modules as having been visited.
404  ModuleFile *NextModule = CurrentModule;
405  do {
406  // For any module that this module depends on, push it on the
407  // stack (if it hasn't already been marked as visited).
408  for (llvm::SetVector<ModuleFile *>::iterator
409  M = NextModule->Imports.begin(),
410  MEnd = NextModule->Imports.end();
411  M != MEnd; ++M) {
412  if (State->VisitNumber[(*M)->Index] != VisitNumber) {
413  State->Stack.push_back(*M);
414  State->VisitNumber[(*M)->Index] = VisitNumber;
415  }
416  }
417 
418  if (State->Stack.empty())
419  break;
420 
421  // Pop the next module off the stack.
422  NextModule = State->Stack.pop_back_val();
423  } while (true);
424  }
425 
426  returnVisitState(State);
427 }
428 
429 bool ModuleManager::lookupModuleFile(StringRef FileName,
430  off_t ExpectedSize,
431  time_t ExpectedModTime,
432  const FileEntry *&File) {
433  if (FileName == "-") {
434  File = nullptr;
435  return false;
436  }
437 
438  // Open the file immediately to ensure there is no race between stat'ing and
439  // opening the file.
440  File = FileMgr.getFile(FileName, /*openFile=*/true, /*cacheFailure=*/false);
441  if (!File)
442  return false;
443 
444  if ((ExpectedSize && ExpectedSize != File->getSize()) ||
445  (ExpectedModTime && ExpectedModTime != File->getModificationTime()))
446  // Do not destroy File, as it may be referenced. If we need to rebuild it,
447  // it will be destroyed by removeModules.
448  return true;
449 
450  return false;
451 }
452 
453 #ifndef NDEBUG
454 namespace llvm {
455  template<>
457  typedef ModuleFile *NodeRef;
458  typedef llvm::SetVector<ModuleFile *>::const_iterator ChildIteratorType;
459  typedef pointer_iterator<ModuleManager::ModuleConstIterator> nodes_iterator;
460 
461  static ChildIteratorType child_begin(NodeRef Node) {
462  return Node->Imports.begin();
463  }
464 
465  static ChildIteratorType child_end(NodeRef Node) {
466  return Node->Imports.end();
467  }
468 
469  static nodes_iterator nodes_begin(const ModuleManager &Manager) {
470  return nodes_iterator(Manager.begin());
471  }
472 
473  static nodes_iterator nodes_end(const ModuleManager &Manager) {
474  return nodes_iterator(Manager.end());
475  }
476  };
477 
478  template<>
479  struct DOTGraphTraits<ModuleManager> : public DefaultDOTGraphTraits {
480  explicit DOTGraphTraits(bool IsSimple = false)
481  : DefaultDOTGraphTraits(IsSimple) { }
482 
483  static bool renderGraphFromBottomUp() {
484  return true;
485  }
486 
487  std::string getNodeLabel(ModuleFile *M, const ModuleManager&) {
488  return M->ModuleName;
489  }
490  };
491 }
492 
494  llvm::ViewGraph(*this, "Modules");
495 }
496 #endif
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:116
ModuleFile * lookup(const FileEntry *File) const
Returns the module associated with the given module file.
time_t getModificationTime() const
Definition: FileManager.h:91
virtual StringRef ExtractPCH(llvm::MemoryBufferRef Buffer) const =0
Returns the serialized AST inside the PCH container Buffer.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:26
std::string ModuleName
The name of the module.
Definition: Module.h:118
The module file is out-of-date.
The base class of the type hierarchy.
Definition: Type.h:1300
unsigned Index
The index of this module in the list of modules.
Definition: Module.h:109
ModuleFile * lookupByFileName(StringRef FileName) const
Returns the module associated with the given file name.
Manages the set of modules loaded by an AST reader.
Definition: ModuleManager.h:35
Manage memory buffers across multiple users.
ModuleMap & getModuleMap()
Retrieve the module map.
Definition: HeaderSearch.h:636
SourceLocation ImportLoc
The source location where this module was first imported.
Definition: Module.h:194
AddModuleResult
The result of attempting to add a new module.
ModuleIterator begin()
Forward iterator to traverse all loaded modules.
LineState State
ModuleManager(FileManager &FileMgr, MemoryBufferCache &PCMCache, const PCHContainerReader &PCHContainerRdr, const HeaderSearch &HeaderSearchInfo)
void viewGraph()
View the graphviz representation of the module graph.
void visit(llvm::function_ref< bool(ModuleFile &M)> Visitor, llvm::SmallPtrSetImpl< ModuleFile *> *ModuleFilesHit=nullptr)
Visit each of the modules.
The signature of a module, which is a hash of the AST content.
Definition: Module.h:47
void setGlobalIndex(GlobalModuleIndex *Index)
Set the global module index.
Describes a module or submodule.
Definition: Module.h:57
Module * findModule(StringRef Name) const
Retrieve a module with the given name.
Definition: ModuleMap.cpp:704
bool lookupModuleFile(StringRef FileName, off_t ExpectedSize, time_t ExpectedModTime, const FileEntry *&File)
Attempt to resolve the given module file name to a file entry.
ModuleIterator end()
Forward iterator end-point to traverse all loaded modules.
The result of a status operation.
This abstract interface provides operations for unwrapping containers for serialized ASTs (precompile...
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:137
const FileEntry * getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
ModuleKind
Specifies the kind of module that has been loaded.
Definition: Module.h:47
File is from a prebuilt module path.
Definition: Module.h:53
llvm::pointee_iterator< SmallVectorImpl< std::unique_ptr< ModuleFile > >::iterator > ModuleIterator
static nodes_iterator nodes_end(const ModuleManager &Manager)
Information about a module that has been loaded by the ASTReader.
Definition: Module.h:100
AddModuleResult addModule(StringRef FileName, ModuleKind Type, SourceLocation ImportLoc, ModuleFile *ImportedBy, unsigned Generation, off_t ExpectedSize, time_t ExpectedModTime, ASTFileSignature ExpectedSignature, ASTFileSignatureReader ReadSignature, ModuleFile *&Module, std::string &ErrorStr)
Attempts to create a new module and add it to the list of known modules.
void addInMemoryBuffer(StringRef FileName, std::unique_ptr< llvm::MemoryBuffer > Buffer)
Add an in-memory buffer the list of known buffers.
static nodes_iterator nodes_begin(const ModuleManager &Manager)
Encodes a location in the source.
File is an implicitly-loaded module.
Definition: Module.h:48
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
llvm::sys::TimePoint getLastModificationTime() const
static ChildIteratorType child_begin(NodeRef Node)
llvm::SetVector< ModuleFile * >::const_iterator ChildIteratorType
const FileEntry * getVirtualFile(StringRef Filename, off_t Size, time_t ModificationTime)
Retrieve a file entry for a "virtual" file that acts as if there were a file with the given name on d...
llvm::SetVector< ModuleFile * > ImportedBy
List of modules which depend on this module.
Definition: Module.h:448
A global index for a set of module files, providing information about the identifiers within those mo...
The module file was just loaded in response to this call.
ast_type_traits::DynTypedNode Node
Dataflow Directional Tag Classes.
pointer_iterator< ModuleManager::ModuleConstIterator > nodes_iterator
off_t getSize() const
Definition: FileManager.h:87
File is an explicitly-loaded module.
Definition: Module.h:49
ModuleFile * lookupByModuleName(StringRef ModName) const
Returns the module associated with the given module name.
llvm::ErrorOr< std::unique_ptr< llvm::MemoryBuffer > > getBufferForFile(const FileEntry *Entry, bool isVolatile=false, bool ShouldCloseOpenFile=true)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful, otherwise returning null.
static void updateModuleImports(ModuleFile &MF, ModuleFile *ImportedBy, SourceLocation ImportLoc)
void removeModules(ModuleIterator First, llvm::SmallPtrSetImpl< ModuleFile *> &LoadedSuccessfully, ModuleMap *modMap)
Remove the modules starting from First (to the end).
void moduleFileAccepted(ModuleFile *MF)
Notification from the AST reader that the given module file has been "accepted", and will not (can no...
ASTFileSignature(* ASTFileSignatureReader)(StringRef)
std::string getNodeLabel(ModuleFile *M, const ModuleManager &)
unsigned size() const
Number of modules loaded.
bool getNoncachedStatValue(StringRef Path, vfs::Status &Result)
Get the &#39;stat&#39; information for the given Path.
The module file had already been loaded.
void invalidateCache(const FileEntry *Entry)
Remove the real file Entry from the cache.
static ChildIteratorType child_end(NodeRef Node)
bool DirectlyImported
Whether this module has been directly imported by the user.
Definition: Module.h:161
llvm::SetVector< ModuleFile * > Imports
List of modules which this module depends on.
Definition: Module.h:451
std::unique_ptr< llvm::MemoryBuffer > lookupBuffer(StringRef Name)
Returns the in-memory (virtual file) buffer with the given name.
bool loadedModuleFile(ModuleFile *File)
Note that the given module file has been loaded.
static bool checkSignature(ASTFileSignature Signature, ASTFileSignature ExpectedSignature, std::string &ErrorStr)