ModulesBuilder.cpp

Go to the documentation of this file.

//===----------------- ModulesBuilder.cpp ------------------------*- C++-*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#include "ModulesBuilder.h"
#include "Compiler.h"
#include "support/Logger.h"
#include "clang/Frontend/FrontendAction.h"
#include "clang/Frontend/FrontendActions.h"
#include "clang/Serialization/ASTReader.h"
 
namespace clang {
namespace clangd {
 
namespace {
 
// Create a path to store module files. Generally it should be:
//
//   {TEMP_DIRS}/clangd/module_files/{hashed-file-name}-%%-%%-%%-%%-%%-%%/.
//
// {TEMP_DIRS} is the temporary directory for the system, e.g., "/var/tmp"
// or "C:/TEMP".
//
// '%%' means random value to make the generated path unique.
//
// \param MainFile is used to get the root of the project from global
// compilation database.
//
// TODO: Move these module fils out of the temporary directory if the module
// files are persistent.
llvm::SmallString<256> getUniqueModuleFilesPath(PathRef MainFile) {
  llvm::SmallString<128> HashedPrefix = llvm::sys::path::filename(MainFile);
  // There might be multiple files with the same name in a project. So appending
  // the hash value of the full path to make sure they won't conflict.
  HashedPrefix += std::to_string(llvm::hash_value(MainFile));
 
  llvm::SmallString<256> ResultPattern;
 
  llvm::sys::path::system_temp_directory(/*erasedOnReboot=*/true,
                                         ResultPattern);
 
  llvm::sys::path::append(ResultPattern, "clangd");
  llvm::sys::path::append(ResultPattern, "module_files");
 
  llvm::sys::path::append(ResultPattern, HashedPrefix);
 
  ResultPattern.append("-%%-%%-%%-%%-%%-%%");
 
  llvm::SmallString<256> Result;
  llvm::sys::fs::createUniquePath(ResultPattern, Result,
                                  /*MakeAbsolute=*/false);
 
  llvm::sys::fs::create_directories(Result);
  return Result;
}
 
// Get a unique module file path under \param ModuleFilesPrefix.
std::string getModuleFilePath(llvm::StringRef ModuleName,
                              PathRef ModuleFilesPrefix) {
  llvm::SmallString<256> ModuleFilePath(ModuleFilesPrefix);
  auto [PrimaryModuleName, PartitionName] = ModuleName.split(':');
  llvm::sys::path::append(ModuleFilePath, PrimaryModuleName);
  if (!PartitionName.empty()) {
    ModuleFilePath.append("-");
    ModuleFilePath.append(PartitionName);
  }
 
  ModuleFilePath.append(".pcm");
  return std::string(ModuleFilePath);
}
 
// FailedPrerequisiteModules - stands for the PrerequisiteModules which has
// errors happened during the building process.
class FailedPrerequisiteModules : public PrerequisiteModules {
public:
  ~FailedPrerequisiteModules() override = default;
 
  // We shouldn't adjust the compilation commands based on
  // FailedPrerequisiteModules.
  void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override {
  }
 
  // FailedPrerequisiteModules can never be reused.
  bool
  canReuse(const CompilerInvocation &CI,
           llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>) const override {
    return false;
  }
};
 
// StandalonePrerequisiteModules - stands for PrerequisiteModules for which all
// the required modules are built successfully. All the module files
// are owned by the StandalonePrerequisiteModules class.
//
// Any of the built module files won't be shared with other instances of the
// class. So that we can avoid worrying thread safety.
//
// We don't need to worry about duplicated module names here since the standard
// guarantees the module names should be unique to a program.
class StandalonePrerequisiteModules : public PrerequisiteModules {
public:
  StandalonePrerequisiteModules() = default;
 
  StandalonePrerequisiteModules(const StandalonePrerequisiteModules &) = delete;
  StandalonePrerequisiteModules
  operator=(const StandalonePrerequisiteModules &) = delete;
  StandalonePrerequisiteModules(StandalonePrerequisiteModules &&) = delete;
  StandalonePrerequisiteModules
  operator=(StandalonePrerequisiteModules &&) = delete;
 
  ~StandalonePrerequisiteModules() override = default;
 
  void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override {
    // Appending all built module files.
    for (auto &RequiredModule : RequiredModules)
      Options.PrebuiltModuleFiles.insert_or_assign(
          RequiredModule.ModuleName, RequiredModule.ModuleFilePath);
  }
 
  bool canReuse(const CompilerInvocation &CI,
                llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>) const override;
 
  bool isModuleUnitBuilt(llvm::StringRef ModuleName) const {
    return BuiltModuleNames.contains(ModuleName);
  }
 
  void addModuleFile(llvm::StringRef ModuleName,
                     llvm::StringRef ModuleFilePath) {
    RequiredModules.emplace_back(ModuleName, ModuleFilePath);
    BuiltModuleNames.insert(ModuleName);
  }
 
private:
  struct ModuleFile {
    ModuleFile(llvm::StringRef ModuleName, PathRef ModuleFilePath)
        : ModuleName(ModuleName.str()), ModuleFilePath(ModuleFilePath.str()) {}
 
    ModuleFile(const ModuleFile &) = delete;
    ModuleFile operator=(const ModuleFile &) = delete;
 
    // The move constructor is needed for llvm::SmallVector.
    ModuleFile(ModuleFile &&Other)
        : ModuleName(std::move(Other.ModuleName)),
          ModuleFilePath(std::move(Other.ModuleFilePath)) {}
 
    ModuleFile &operator=(ModuleFile &&Other) = delete;
 
    ~ModuleFile() {
      if (!ModuleFilePath.empty())
        llvm::sys::fs::remove(ModuleFilePath);
    }
 
    std::string ModuleName;
    std::string ModuleFilePath;
  };
 
  llvm::SmallVector<ModuleFile, 8> RequiredModules;
  // A helper class to speedup the query if a module is built.
  llvm::StringSet<> BuiltModuleNames;
};
 
// Build a module file for module with `ModuleName`. The information of built
// module file are stored in \param BuiltModuleFiles.
llvm::Error buildModuleFile(llvm::StringRef ModuleName,
                            const GlobalCompilationDatabase &CDB,
                            const ThreadsafeFS &TFS, ProjectModules &MDB,
                            PathRef ModuleFilesPrefix,
                            StandalonePrerequisiteModules &BuiltModuleFiles) {
  if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName))
    return llvm::Error::success();
 
  PathRef ModuleUnitFileName = MDB.getSourceForModuleName(ModuleName);
  // It is possible that we're meeting third party modules (modules whose
  // source are not in the project. e.g, the std module may be a third-party
  // module for most projects) or something wrong with the implementation of
  // ProjectModules.
  // FIXME: How should we treat third party modules here? If we want to ignore
  // third party modules, we should return true instead of false here.
  // Currently we simply bail out.
  if (ModuleUnitFileName.empty())
    return llvm::createStringError("Failed to get the primary source");
 
  // Try cheap operation earlier to boil-out cheaply if there are problems.
  auto Cmd = CDB.getCompileCommand(ModuleUnitFileName);
  if (!Cmd)
    return llvm::createStringError(
        llvm::formatv("No compile command for {0}", ModuleUnitFileName));
 
  for (auto &RequiredModuleName : MDB.getRequiredModules(ModuleUnitFileName)) {
    // Return early if there are errors building the module file.
    if (llvm::Error Err = buildModuleFile(RequiredModuleName, CDB, TFS, MDB,
                                          ModuleFilesPrefix, BuiltModuleFiles))
      return llvm::createStringError(
          llvm::formatv("Failed to build dependency {0}: {1}",
                        RequiredModuleName, llvm::toString(std::move(Err))));
  }
 
  Cmd->Output = getModuleFilePath(ModuleName, ModuleFilesPrefix);
 
  ParseInputs Inputs;
  Inputs.TFS = &TFS;
  Inputs.CompileCommand = std::move(*Cmd);
 
  IgnoreDiagnostics IgnoreDiags;
  auto CI = buildCompilerInvocation(Inputs, IgnoreDiags);
  if (!CI)
    return llvm::createStringError("Failed to build compiler invocation");
 
  auto FS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
  auto Buf = FS->getBufferForFile(Inputs.CompileCommand.Filename);
  if (!Buf)
    return llvm::createStringError("Failed to create buffer");
 
  // In clang's driver, we will suppress the check for ODR violation in GMF.
  // See the implementation of RenderModulesOptions in Clang.cpp.
  CI->getLangOpts().SkipODRCheckInGMF = true;
 
  // Hash the contents of input files and store the hash value to the BMI files.
  // So that we can check if the files are still valid when we want to reuse the
  // BMI files.
  CI->getHeaderSearchOpts().ValidateASTInputFilesContent = true;
 
  BuiltModuleFiles.adjustHeaderSearchOptions(CI->getHeaderSearchOpts());
 
  CI->getFrontendOpts().OutputFile = Inputs.CompileCommand.Output;
  auto Clang =
      prepareCompilerInstance(std::move(CI), /*Preamble=*/nullptr,
                              std::move(*Buf), std::move(FS), IgnoreDiags);
  if (!Clang)
    return llvm::createStringError("Failed to prepare compiler instance");
 
  GenerateReducedModuleInterfaceAction Action;
  Clang->ExecuteAction(Action);
 
  if (Clang->getDiagnostics().hasErrorOccurred())
    return llvm::createStringError("Compilation failed");
 
  BuiltModuleFiles.addModuleFile(ModuleName, Inputs.CompileCommand.Output);
  return llvm::Error::success();
}
} // namespace
 
std::unique_ptr<PrerequisiteModules>
ModulesBuilder::buildPrerequisiteModulesFor(PathRef File,
                                            const ThreadsafeFS &TFS) const {
  std::unique_ptr<ProjectModules> MDB = CDB.getProjectModules(File);
  if (!MDB) {
    elog("Failed to get Project Modules information for {0}", File);
    return std::make_unique<FailedPrerequisiteModules>();
  }
 
  std::vector<std::string> RequiredModuleNames = MDB->getRequiredModules(File);
  if (RequiredModuleNames.empty())
    return std::make_unique<StandalonePrerequisiteModules>();
 
  llvm::SmallString<256> ModuleFilesPrefix = getUniqueModuleFilesPath(File);
 
  log("Trying to build required modules for {0} in {1}", File,
      ModuleFilesPrefix);
 
  auto RequiredModules = std::make_unique<StandalonePrerequisiteModules>();
 
  for (llvm::StringRef RequiredModuleName : RequiredModuleNames) {
    // Return early if there is any error.
    if (llvm::Error Err =
            buildModuleFile(RequiredModuleName, CDB, TFS, *MDB.get(),
                            ModuleFilesPrefix, *RequiredModules.get())) {
      elog("Failed to build module {0}; due to {1}", RequiredModuleName,
           toString(std::move(Err)));
      return std::make_unique<FailedPrerequisiteModules>();
    }
  }
 
  log("Built required modules for {0} in {1}", File, ModuleFilesPrefix);
 
  return std::move(RequiredModules);
}
 
bool StandalonePrerequisiteModules::canReuse(
    const CompilerInvocation &CI,
    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) const {
  if (RequiredModules.empty())
    return true;
 
  CompilerInstance Clang;
 
  Clang.setInvocation(std::make_shared<CompilerInvocation>(CI));
  IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
      CompilerInstance::createDiagnostics(new DiagnosticOptions());
  Clang.setDiagnostics(Diags.get());
 
  FileManager *FM = Clang.createFileManager(VFS);
  Clang.createSourceManager(*FM);
 
  if (!Clang.createTarget())
    return false;
 
  assert(Clang.getHeaderSearchOptsPtr());
  adjustHeaderSearchOptions(Clang.getHeaderSearchOpts());
  // Since we don't need to compile the source code actually, the TU kind here
  // doesn't matter.
  Clang.createPreprocessor(TU_Complete);
  Clang.getHeaderSearchOpts().ForceCheckCXX20ModulesInputFiles = true;
  Clang.getHeaderSearchOpts().ValidateASTInputFilesContent = true;
 
  // Following the practice of clang's driver to suppres the checking for ODR
  // violation in GMF.
  // See
  // https://clang.llvm.org/docs/StandardCPlusPlusModules.html#object-definition-consistency
  // for example.
  Clang.getLangOpts().SkipODRCheckInGMF = true;
 
  Clang.createASTReader();
  for (auto &RequiredModule : RequiredModules) {
    llvm::StringRef BMIPath = RequiredModule.ModuleFilePath;
    // FIXME: Loading BMI fully is too heavy considering something cheaply to
    // check if we can reuse the BMI.
    auto ReadResult =
        Clang.getASTReader()->ReadAST(BMIPath, serialization::MK_MainFile,
                                      SourceLocation(), ASTReader::ARR_None);
 
    if (ReadResult != ASTReader::Success) {
      elog("Can't reuse {0}: {1}", BMIPath, ReadResult);
      return false;
    }
  }
 
  return true;
}
 
} // namespace clangd
} // namespace clang