ModuleManager.cpp

Go to the documentation of this file.

//===- ModuleManager.cpp - Module Manager ---------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
//  This file defines the ModuleManager class, which manages a set of loaded
//  modules for the ASTReader.
//
//===----------------------------------------------------------------------===//
 
#include "clang/Serialization/ModuleManager.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/LLVM.h"
#include "clang/Lex/HeaderSearch.h"
#include "clang/Lex/ModuleMap.h"
#include "clang/Serialization/GlobalModuleIndex.h"
#include "clang/Serialization/InMemoryModuleCache.h"
#include "clang/Serialization/ModuleCache.h"
#include "clang/Serialization/ModuleFile.h"
#include "clang/Serialization/PCHContainerOperations.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/iterator.h"
#include "llvm/Support/DOTGraphTraits.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/GraphWriter.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/VirtualFileSystem.h"
#include <cassert>
#include <memory>
#include <string>
#include <system_error>
 
using namespace clang;
using namespace serialization;
 
std::optional<ModuleFileKey>
ModuleManager::makeKey(const ModuleFileName &Name) const {
  if (unsigned SuffixLen = Name.getImplicitModuleSuffixLength()) {
    StringRef ModuleCachePath = StringRef(Name).drop_back(SuffixLen);
    StringRef ImplicitModuleSuffix = StringRef(Name).take_back(SuffixLen);
    if (auto *ModuleCacheDir = ModCache.getDirectoryPtr(ModuleCachePath))
      return ModuleFileKey(ModuleCacheDir, ImplicitModuleSuffix);
  } else {
    if (auto ModuleFile = FileMgr.getOptionalFileRef(Name, /*OpenFile=*/true,
                                                     /*CacheFailure=*/false,
                                                     /*IsText=*/false))
      return ModuleFileKey(*ModuleFile);
  }
 
  return std::nullopt;
}
 
ModuleFile *ModuleManager::lookupByModuleName(StringRef Name) const {
  if (const Module *Mod = HeaderSearchInfo.getModuleMap().findModule(Name))
    if (const ModuleFileName *FileName = Mod->getASTFileName())
      return lookupByFileName(*FileName);
 
  return nullptr;
}
 
ModuleFile *ModuleManager::lookupByFileName(ModuleFileName Name) const {
  std::optional<ModuleFileKey> Key = makeKey(Name);
  return Key ? lookup(*Key) : nullptr;
}
 
ModuleFile *ModuleManager::lookup(ModuleFileKey Key) const {
  return Modules.lookup(Key);
}
 
std::unique_ptr<llvm::MemoryBuffer>
ModuleManager::lookupBuffer(StringRef Name, off_t &Size, time_t &ModTime) {
  auto Entry = FileMgr.getOptionalFileRef(Name, /*OpenFile=*/false,
                                          /*CacheFailure=*/false,
                                          /*IsText=*/false);
  if (!Entry)
    return nullptr;
  Size = Entry->getSize();
  ModTime = Entry->getModificationTime();
  return std::move(InMemoryBuffers[*Entry]);
}
 
bool ModuleManager::isModuleFileOutOfDate(off_t Size, time_t ModTime,
                                          off_t ExpectedSize,
                                          time_t ExpectedModTime,
                                          AddModuleResult &Result) {
  bool OutOfDate = false;
  if (ExpectedSize && ExpectedSize != Size) {
    Result.Changes.push_back({Change::Size, ExpectedSize, Size});
    OutOfDate = true;
  }
 
  if (ExpectedModTime && ExpectedModTime != ModTime) {
    Result.Changes.push_back({Change::ModTime, ExpectedModTime, ModTime});
    OutOfDate = true;
  }
 
  return OutOfDate;
}
 
bool ModuleManager::checkSignature(ASTFileSignature Signature,
                                   ASTFileSignature ExpectedSignature,
                                   AddModuleResult &Result) {
  if (!ExpectedSignature || Signature == ExpectedSignature)
    return false;
  Result.SignatureError =
      Signature ? "signature mismatch" : "could not read module signature";
  return true;
}
 
static void updateModuleImports(ModuleFile &MF, ModuleFile *ImportedBy,
                                SourceLocation ImportLoc) {
  if (ImportedBy) {
    MF.ImportedBy.insert(ImportedBy);
    ImportedBy->Imports.insert(&MF);
  } else {
    if (!MF.DirectlyImported)
      MF.ImportLoc = ImportLoc;
 
    MF.DirectlyImported = true;
  }
}
 
AddModuleResult ModuleManager::addModule(
    ModuleFileName FileName, ModuleKind Type, SourceLocation ImportLoc,
    ModuleFile *ImportedBy, unsigned Generation, off_t ExpectedSize,
    time_t ExpectedModTime, ASTFileSignature ExpectedSignature,
    ASTFileSignatureReader ReadSignature) {
  AddModuleResult Result;
 
  uint64_t InputFilesValidationTimestamp = 0;
  if (Type == MK_ImplicitModule)
    InputFilesValidationTimestamp = ModCache.getModuleTimestamp(FileName);
 
  bool IgnoreModTime = Type == MK_ExplicitModule || Type == MK_PrebuiltModule;
  if (ImportedBy)
    IgnoreModTime &= ImportedBy->Kind == MK_ExplicitModule ||
                     ImportedBy->Kind == MK_PrebuiltModule;
  if (IgnoreModTime) {
    // If neither this file nor the importer are in the module cache, this file
    // might have a different mtime due to being moved across filesystems in
    // a distributed build. The size must still match, though. (As must the
    // contents, but we can't check that.)
    ExpectedModTime = 0;
  }
 
  std::optional<ModuleFileKey> FileKey = makeKey(FileName);
  if (!FileKey) {
    Result.K = AddModuleResult::Missing;
    return Result;
  }
 
  // Check whether we already loaded this module before.
  // Note: `isModuleFileOutOfDate` and `checkSignature` are mutually exclusive
  // in practice. If a signature is stored, it means size/mtime values have been
  // zeroed out. If size/mtime are non-NULL, the signature is empty.
  if (ModuleFile *ModuleEntry = lookup(*FileKey)) {
    if (isModuleFileOutOfDate(ModuleEntry->Size, ModuleEntry->ModTime,
                              ExpectedSize, ExpectedModTime, Result)) {
      Result.setOutOfDate(ModuleEntry->InputFilesValidationStatus);
      return Result;
    }
 
    // Check the stored signature.
    if (checkSignature(ModuleEntry->Signature, ExpectedSignature, Result)) {
      Result.setOutOfDate(ModuleEntry->InputFilesValidationStatus);
      return Result;
    }
 
    Result.Module = ModuleEntry;
    updateModuleImports(*ModuleEntry, ImportedBy, ImportLoc);
    Result.K = AddModuleResult::AlreadyLoaded;
    return Result;
  }
 
  // Load the contents of the module
  off_t Size = ExpectedSize;
  time_t ModTime = ExpectedModTime;
  llvm::MemoryBuffer *ModuleBuffer = nullptr;
  std::unique_ptr<llvm::MemoryBuffer> NewFileBuffer = nullptr;
  if (std::unique_ptr<llvm::MemoryBuffer> Buffer =
          lookupBuffer(FileName, Size, ModTime)) {
    // The buffer was already provided for us.
    ModuleBuffer = &getModuleCache().getInMemoryModuleCache().addBuiltPCM(
        FileName, std::move(Buffer), Size, ModTime);
  } else if (llvm::MemoryBuffer *Buffer =
                 getModuleCache().getInMemoryModuleCache().lookupPCM(
                     FileName, Size, ModTime)) {
    ModuleBuffer = Buffer;
  } else if (getModuleCache().getInMemoryModuleCache().shouldBuildPCM(
                 FileName)) {
    // Report that the module is out of date, since we tried (and failed) to
    // import it earlier. No ModuleFile exists yet, so derive the validation
    // status from the module kind being loaded.
    Result.setOutOfDate(Type == MK_ExplicitModule || Type == MK_PrebuiltModule
                            ? InputFilesValidation::Disabled
                            : InputFilesValidation::NotStarted);
    return Result;
  } else {
    auto Buf = [&]() -> Expected<std::unique_ptr<llvm::MemoryBuffer>> {
      // Implicit modules live in the module cache.
      if (FileName.getImplicitModuleSuffixLength())
        return ModCache.read(FileName, Size, ModTime);
 
      // Explicit modules are treated as any other compiler input file, load
      // them via FileManager.
      Expected<FileEntryRef> Entry =
          FileName == StringRef("-")
              ? FileMgr.getSTDIN()
              : FileMgr.getFileRef(FileName, /*OpenFile=*/true,
                                   /*CacheFailure=*/false,
                                   /*IsText=*/false);
      if (!Entry)
        return Entry.takeError();
 
      Size = Entry->getSize();
      ModTime = Entry->getModificationTime();
 
      // RequiresNullTerminator is false because module files don't need it, and
      // this allows the file to still be mmapped.
      return llvm::errorOrToExpected(
          FileMgr.getBufferForFile(*Entry, /*IsVolatile=*/false,
                                   /*RequiresNullTerminator=*/false,
                                   /*MaybeLimit=*/std::nullopt,
                                   /*IsText=*/false));
    }();
 
    if (!Buf) {
      Result.BufferError = llvm::toString(Buf.takeError());
      Result.K = AddModuleResult::Missing;
      return Result;
    }
 
    NewFileBuffer = std::move(*Buf);
    ModuleBuffer = NewFileBuffer.get();
  }
 
  // Allocate bookkeeping for a module file not yet loaded into this reader.
  auto NewModule = std::make_unique<ModuleFile>(Type, *FileKey, Generation);
  NewModule->Index = Chain.size();
  NewModule->FileName = FileName;
  NewModule->ImportLoc = ImportLoc;
  NewModule->InputFilesValidationTimestamp = InputFilesValidationTimestamp;
  NewModule->Size = Size;
  NewModule->ModTime = ModTime;
  NewModule->Buffer = ModuleBuffer;
  // Initialize the stream.
  NewModule->Data = PCHContainerRdr.ExtractPCH(*NewModule->Buffer);
 
  // Check file properties.
  if (isModuleFileOutOfDate(Size, ModTime, ExpectedSize, ExpectedModTime,
                            Result)) {
    Result.setOutOfDate(NewModule->InputFilesValidationStatus);
    return Result;
  }
 
  // Read the signature eagerly now so that we can check it.  Avoid calling
  // ReadSignature unless there's something to check though.
  if (ExpectedSignature && checkSignature(ReadSignature(NewModule->Data),
                                          ExpectedSignature, Result)) {
    Result.setOutOfDate(NewModule->InputFilesValidationStatus);
    return Result;
  }
 
  if (NewFileBuffer)
    getModuleCache().getInMemoryModuleCache().addPCM(
        FileName, std::move(NewFileBuffer), Size, ModTime);
 
  // We're keeping this module. Store it in the map.
  Result.Module = Modules[*FileKey] = NewModule.get();
 
  updateModuleImports(*NewModule, ImportedBy, ImportLoc);
 
  if (!NewModule->isModule())
    PCHChain.push_back(NewModule.get());
  if (!ImportedBy)
    Roots.push_back(NewModule.get());
 
  Chain.push_back(std::move(NewModule));
  Result.K = AddModuleResult::NewlyLoaded;
  return Result;
}
 
void ModuleManager::removeModules(ModuleIterator First) {
  auto Last = end();
  if (First == Last)
    return;
 
  // Explicitly clear VisitOrder since we might not notice it is stale.
  VisitOrder.clear();
 
  // Collect the set of module file pointers that we'll be removing.
  llvm::SmallPtrSet<ModuleFile *, 4> victimSet(
      (llvm::pointer_iterator<ModuleIterator>(First)),
      (llvm::pointer_iterator<ModuleIterator>(Last)));
 
  auto IsVictim = [&](ModuleFile *MF) {
    return victimSet.count(MF);
  };
  // Remove any references to the now-destroyed modules.
  for (auto I = begin(); I != First; ++I) {
    I->Imports.remove_if(IsVictim);
    I->ImportedBy.remove_if(IsVictim);
  }
  llvm::erase_if(Roots, IsVictim);
 
  // Remove the modules from the PCH chain.
  for (auto I = First; I != Last; ++I) {
    if (!I->isModule()) {
      PCHChain.erase(llvm::find(PCHChain, &*I), PCHChain.end());
      break;
    }
  }
 
  // Delete the modules.
  for (ModuleIterator victim = First; victim != Last; ++victim)
    Modules.erase(victim->FileKey);
 
  Chain.erase(Chain.begin() + (First - begin()), Chain.end());
}
 
void
ModuleManager::addInMemoryBuffer(StringRef FileName,
                                 std::unique_ptr<llvm::MemoryBuffer> Buffer) {
  FileEntryRef Entry =
      FileMgr.getVirtualFileRef(FileName, Buffer->getBufferSize(), 0);
  InMemoryBuffers[Entry] = std::move(Buffer);
}
 
std::unique_ptr<ModuleManager::VisitState> ModuleManager::allocateVisitState() {
  // Fast path: if we have a cached state, use it.
  if (FirstVisitState) {
    auto Result = std::move(FirstVisitState);
    FirstVisitState = std::move(Result->NextState);
    return Result;
  }
 
  // Allocate and return a new state.
  return std::make_unique<VisitState>(size());
}
 
void ModuleManager::returnVisitState(std::unique_ptr<VisitState> State) {
  assert(State->NextState == nullptr && "Visited state is in list?");
  State->NextState = std::move(FirstVisitState);
  FirstVisitState = std::move(State);
}
 
void ModuleManager::setGlobalIndex(GlobalModuleIndex *Index) {
  GlobalIndex = Index;
  if (!GlobalIndex) {
    ModulesInCommonWithGlobalIndex.clear();
    return;
  }
 
  // Notify the global module index about all of the modules we've already
  // loaded.
  for (ModuleFile &M : *this)
    if (!GlobalIndex->loadedModuleFile(&M))
      ModulesInCommonWithGlobalIndex.push_back(&M);
}
 
void ModuleManager::moduleFileAccepted(ModuleFile *MF) {
  if (!GlobalIndex || GlobalIndex->loadedModuleFile(MF))
    return;
 
  ModulesInCommonWithGlobalIndex.push_back(MF);
}
 
ModuleManager::ModuleManager(FileManager &FileMgr, ModuleCache &ModCache,
                             const PCHContainerReader &PCHContainerRdr,
                             const HeaderSearch &HeaderSearchInfo)
    : FileMgr(FileMgr), ModCache(ModCache), PCHContainerRdr(PCHContainerRdr),
      HeaderSearchInfo(HeaderSearchInfo) {}
 
void ModuleManager::visit(llvm::function_ref<bool(ModuleFile &M)> Visitor,
                          llvm::SmallPtrSetImpl<ModuleFile *> *ModuleFilesHit) {
  // If the visitation order vector is the wrong size, recompute the order.
  if (VisitOrder.size() != Chain.size()) {
    unsigned N = size();
    VisitOrder.clear();
    VisitOrder.reserve(N);
 
    // Record the number of incoming edges for each module. When we
    // encounter a module with no incoming edges, push it into the queue
    // to seed the queue.
    SmallVector<ModuleFile *, 4> Queue;
    Queue.reserve(N);
    llvm::SmallVector<unsigned, 4> UnusedIncomingEdges;
    UnusedIncomingEdges.resize(size());
    for (ModuleFile &M : llvm::reverse(*this)) {
      unsigned Size = M.ImportedBy.size();
      UnusedIncomingEdges[M.Index] = Size;
      if (!Size)
        Queue.push_back(&M);
    }
 
    // Traverse the graph, making sure to visit a module before visiting any
    // of its dependencies.
    while (!Queue.empty()) {
      ModuleFile *CurrentModule = Queue.pop_back_val();
      VisitOrder.push_back(CurrentModule);
 
      // For any module that this module depends on, push it on the
      // stack (if it hasn't already been marked as visited).
      for (ModuleFile *M : llvm::reverse(CurrentModule->Imports)) {
        // Remove our current module as an impediment to visiting the
        // module we depend on. If we were the last unvisited module
        // that depends on this particular module, push it into the
        // queue to be visited.
        unsigned &NumUnusedEdges = UnusedIncomingEdges[M->Index];
        if (NumUnusedEdges && (--NumUnusedEdges == 0))
          Queue.push_back(M);
      }
    }
 
    assert(VisitOrder.size() == N && "Visitation order is wrong?");
 
    FirstVisitState = nullptr;
  }
 
  auto State = allocateVisitState();
  unsigned VisitNumber = State->NextVisitNumber++;
 
  // If the caller has provided us with a hit-set that came from the global
  // module index, mark every module file in common with the global module
  // index that is *not* in that set as 'visited'.
  if (ModuleFilesHit && !ModulesInCommonWithGlobalIndex.empty()) {
    for (unsigned I = 0, N = ModulesInCommonWithGlobalIndex.size(); I != N; ++I)
    {
      ModuleFile *M = ModulesInCommonWithGlobalIndex[I];
      if (!ModuleFilesHit->count(M))
        State->VisitNumber[M->Index] = VisitNumber;
    }
  }
 
  for (unsigned I = 0, N = VisitOrder.size(); I != N; ++I) {
    ModuleFile *CurrentModule = VisitOrder[I];
    // Should we skip this module file?
    if (State->VisitNumber[CurrentModule->Index] == VisitNumber)
      continue;
 
    // Visit the module.
    assert(State->VisitNumber[CurrentModule->Index] == VisitNumber - 1);
    State->VisitNumber[CurrentModule->Index] = VisitNumber;
    if (!Visitor(*CurrentModule))
      continue;
 
    // The visitor has requested that cut off visitation of any
    // module that the current module depends on. To indicate this
    // behavior, we mark all of the reachable modules as having been visited.
    ModuleFile *NextModule = CurrentModule;
    do {
      // For any module that this module depends on, push it on the
      // stack (if it hasn't already been marked as visited).
      for (llvm::SetVector<ModuleFile *>::iterator
             M = NextModule->Imports.begin(),
             MEnd = NextModule->Imports.end();
           M != MEnd; ++M) {
        if (State->VisitNumber[(*M)->Index] != VisitNumber) {
          State->Stack.push_back(*M);
          State->VisitNumber[(*M)->Index] = VisitNumber;
        }
      }
 
      if (State->Stack.empty())
        break;
 
      // Pop the next module off the stack.
      NextModule = State->Stack.pop_back_val();
    } while (true);
  }
 
  returnVisitState(std::move(State));
}
 
#ifndef NDEBUG
namespace llvm {
 
  template<>
  struct GraphTraits<ModuleManager> {
    using NodeRef = ModuleFile *;
    using ChildIteratorType = llvm::SetVector<ModuleFile *>::const_iterator;
    using nodes_iterator = pointer_iterator<ModuleManager::ModuleConstIterator>;
 
    static ChildIteratorType child_begin(NodeRef Node) {
      return Node->Imports.begin();
    }
 
    static ChildIteratorType child_end(NodeRef Node) {
      return Node->Imports.end();
    }
 
    static nodes_iterator nodes_begin(const ModuleManager &Manager) {
      return nodes_iterator(Manager.begin());
    }
 
    static nodes_iterator nodes_end(const ModuleManager &Manager) {
      return nodes_iterator(Manager.end());
    }
  };
 
  template<>
  struct DOTGraphTraits<ModuleManager> : public DefaultDOTGraphTraits {
    explicit DOTGraphTraits(bool IsSimple = false)
        : DefaultDOTGraphTraits(IsSimple) {}
 
    static bool renderGraphFromBottomUp() { return true; }
 
    std::string getNodeLabel(ModuleFile *M, const ModuleManager&) {
      return M->ModuleName;
    }
  };
 
} // namespace llvm
 
void ModuleManager::viewGraph() {
  llvm::ViewGraph(*this, "Modules");
}
#endif