ClangDocMain.cpp

Go to the documentation of this file.

//===-- ClangDocMain.cpp - ClangDoc -----------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This tool for generating C and C++ documentation from source code
// and comments. Generally, it runs a LibTooling FrontendAction on source files,
// mapping each declaration in those files to its USR and serializing relevant
// information into LLVM bitcode. It then runs a pass over the collected
// declaration information, reducing by USR. There is an option to dump this
// intermediate result to bitcode. Finally, it hands the reduced information
// off to a generator, which does the final parsing from the intermediate
// representation to the desired output format.
//
//===----------------------------------------------------------------------===//
 
#include "BitcodeReader.h"
#include "ClangDoc.h"
#include "Generators.h"
#include "Representation.h"
#include "support/Utils.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Tooling/AllTUsExecution.h"
#include "clang/Tooling/CommonOptionsParser.h"
#include "clang/Tooling/Execution.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Mutex.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/ThreadPool.h"
#include "llvm/Support/TimeProfiler.h"
#include "llvm/Support/raw_ostream.h"
#include <atomic>
#include <mutex>
#include <string>
 
using namespace clang::tooling;
using namespace clang;
using clang::doc::OutputFormatTy;
 
static llvm::cl::extrahelp CommonHelp(CommonOptionsParser::HelpMessage);
static llvm::cl::OptionCategory ClangDocCategory("clang-doc options");
 
static llvm::cl::opt<std::string>
    ProjectName("project-name", llvm::cl::desc("Name of project."),
                llvm::cl::cat(ClangDocCategory));
 
static llvm::cl::opt<bool> IgnoreMappingFailures(
    "ignore-map-errors",
    llvm::cl::desc("Continue if files are not mapped correctly."),
    llvm::cl::init(true), llvm::cl::cat(ClangDocCategory));
 
static llvm::cl::opt<std::string>
    OutDirectory("output",
                 llvm::cl::desc("Directory for outputting generated files."),
                 llvm::cl::init("docs"), llvm::cl::cat(ClangDocCategory));
 
static llvm::cl::opt<std::string>
    BaseDirectory("base",
                  llvm::cl::desc(R"(Base Directory for generated documentation.
URLs will be rooted at this directory for HTML links.)"),
                  llvm::cl::init(""), llvm::cl::cat(ClangDocCategory));
 
static llvm::cl::opt<bool>
    PublicOnly("public", llvm::cl::desc("Document only public declarations."),
               llvm::cl::init(false), llvm::cl::cat(ClangDocCategory));
 
static llvm::cl::opt<bool> DoxygenOnly(
    "doxygen",
    llvm::cl::desc("Use only doxygen-style comments to generate docs."),
    llvm::cl::init(false), llvm::cl::cat(ClangDocCategory));
 
static llvm::cl::list<std::string> UserStylesheets(
    "stylesheets", llvm::cl::CommaSeparated,
    llvm::cl::desc("CSS stylesheets to extend the default styles."),
    llvm::cl::cat(ClangDocCategory));
 
static llvm::cl::opt<std::string> UserAssetPath(
    "asset",
    llvm::cl::desc("User supplied asset path to "
                   "override the default css and js files for html output"),
    llvm::cl::cat(ClangDocCategory));
 
static llvm::cl::opt<std::string> SourceRoot("source-root", llvm::cl::desc(R"(
Directory where processed files are stored.
Links to definition locations will only be
generated if the file is in this dir.)"),
                                             llvm::cl::cat(ClangDocCategory));
 
static llvm::cl::opt<std::string>
    RepositoryUrl("repository", llvm::cl::desc(R"(
URL of repository that hosts code.
Used for links to definition locations.)"),
                  llvm::cl::cat(ClangDocCategory));
 
static llvm::cl::opt<std::string> RepositoryCodeLinePrefix(
    "repository-line-prefix",
    llvm::cl::desc("Prefix of line code for repository."),
    llvm::cl::cat(ClangDocCategory));
 
static llvm::cl::opt<bool> FTimeTrace("ftime-trace", llvm::cl::desc(R"(
Turn on time profiler. Generates clang-doc-tracing.json)"),
                                      llvm::cl::init(false),
                                      llvm::cl::cat(ClangDocCategory));
 
static llvm::cl::opt<bool>
    Pretty("pretty-json", llvm::cl::desc("Serialize JSON with whitespace."),
           llvm::cl::cat(ClangDocCategory));
 
static llvm::cl::opt<OutputFormatTy> FormatEnum(
    "format", llvm::cl::desc("Format for outputted docs."),
    llvm::cl::values(clEnumValN(OutputFormatTy::yaml, "yaml",
                                "Documentation in YAML format."),
                     clEnumValN(OutputFormatTy::md, "md",
                                "Documentation in MD format."),
                     clEnumValN(OutputFormatTy::html, "html",
                                "Documentation in HTML format."),
                     clEnumValN(OutputFormatTy::json, "json",
                                "Documentation in JSON format"),
                     clEnumValN(OutputFormatTy::md_mustache, "md_mustache",
                                "Documentation in MD format.")),
    llvm::cl::init(OutputFormatTy::yaml), llvm::cl::cat(ClangDocCategory));
 
static llvm::ExitOnError ExitOnErr;
 
static llvm::StringRef getFormatString() {
  switch (FormatEnum) {
  case OutputFormatTy::yaml:
    return "yaml";
  case OutputFormatTy::md:
    return "md";
  case OutputFormatTy::html:
    return "html";
  case OutputFormatTy::json:
    return "json";
  case OutputFormatTy::md_mustache:
    return "md_mustache";
  }
  llvm_unreachable("Unknown OutputFormatTy");
}
 
// This function isn't referenced outside its translation unit, but it
// can't use the "static" keyword because its address is used for
// GetMainExecutable (since some platforms don't support taking the
// address of main, and some platforms can't implement GetMainExecutable
// without being given the address of a function in the main executable).
static std::string getExecutablePath(const char *Argv0, void *MainAddr) {
  return llvm::sys::fs::getMainExecutable(Argv0, MainAddr);
}
 
// TODO: Rename this, since it only gets custom CSS/JS
static llvm::Error getAssetFiles(clang::doc::ClangDocContext &CDCtx) {
  using DirIt = llvm::sys::fs::directory_iterator;
  std::error_code FileErr;
  llvm::SmallString<128> FilePath(UserAssetPath);
  for (DirIt DirStart = DirIt(UserAssetPath, FileErr), DirEnd;
       !FileErr && DirStart != DirEnd; DirStart.increment(FileErr)) {
    FilePath = DirStart->path();
    if (llvm::sys::fs::is_regular_file(FilePath)) {
      if (llvm::sys::path::extension(FilePath) == ".css")
        CDCtx.UserStylesheets.insert(CDCtx.UserStylesheets.begin(),
                                     std::string(FilePath));
      else if (llvm::sys::path::extension(FilePath) == ".js")
        CDCtx.JsScripts.emplace_back(FilePath.str());
    }
  }
  if (FileErr)
    return llvm::createFileError(FilePath, FileErr);
  return llvm::Error::success();
}
 
static llvm::Error getHtmlFiles(const char *Argv0,
                                clang::doc::ClangDocContext &CDCtx) {
  bool IsDir = llvm::sys::fs::is_directory(UserAssetPath);
  if (!UserAssetPath.empty() && !IsDir)
    llvm::outs() << "Asset path supply is not a directory: " << UserAssetPath
                 << " falling back to default\n";
  if (IsDir) {
    if (FormatEnum == OutputFormatTy::html) {
      if (auto Err = getAssetFiles(CDCtx))
        return Err;
    }
  }
  void *MainAddr = (void *)(intptr_t)getExecutablePath;
  std::string ClangDocPath = getExecutablePath(Argv0, MainAddr);
  llvm::SmallString<128> NativeClangDocPath;
  llvm::sys::path::native(ClangDocPath, NativeClangDocPath);
 
  llvm::SmallString<128> AssetsPath;
  AssetsPath = llvm::sys::path::parent_path(NativeClangDocPath);
  llvm::sys::path::append(AssetsPath, "..", "share", "clang-doc");
 
  getHtmlFiles(AssetsPath, CDCtx);
 
  return llvm::Error::success();
}
 
static llvm::Error getMdFiles(const char *Argv0,
                              clang::doc::ClangDocContext &CDCtx) {
  bool IsDir = llvm::sys::fs::is_directory(UserAssetPath);
  if (!UserAssetPath.empty() && !IsDir)
    llvm::outs() << "Asset path supply is not a directory: " << UserAssetPath
                 << " falling back to default\n";
 
  void *MainAddr = (void *)(intptr_t)getExecutablePath;
  std::string ClangDocPath = getExecutablePath(Argv0, MainAddr);
  llvm::SmallString<128> NativeClangDocPath;
  llvm::sys::path::native(ClangDocPath, NativeClangDocPath);
 
  llvm::SmallString<128> AssetsPath;
  AssetsPath = llvm::sys::path::parent_path(NativeClangDocPath);
  llvm::sys::path::append(AssetsPath, "..", "share", "clang-doc", "md");
 
  getMdFiles(AssetsPath, CDCtx);
 
  return llvm::Error::success();
}
 
/// Make the output of clang-doc deterministic by sorting the children of
/// namespaces and records.
static void sortUsrToInfo(llvm::StringMap<doc::Info *> &USRToInfo) {
  for (auto &I : USRToInfo) {
    auto &Info = I.second;
    if (Info->IT == doc::InfoType::IT_namespace) {
      auto *Namespace = static_cast<clang::doc::NamespaceInfo *>(Info);
      Namespace->Children.sort();
    }
    if (Info->IT == doc::InfoType::IT_record) {
      auto *Record = static_cast<clang::doc::RecordInfo *>(Info);
      Record->Children.sort();
    }
  }
}
 
static llvm::Error handleMappingFailures(DiagnosticsEngine &Diags,
                                         llvm::Error Err) {
  if (!Err)
    return llvm::Error::success();
  if (IgnoreMappingFailures) {
    unsigned ID = Diags.getCustomDiagID(
        DiagnosticsEngine::Warning,
        "Error mapping decls in files. Clang-doc will ignore these files and "
        "continue:\n%0");
    Diags.Report(ID) << toString(std::move(Err));
    return llvm::Error::success();
  }
  return Err;
}
 
static llvm::Error createDirectories(llvm::StringRef OutDirectory) {
  if (std::error_code Err = llvm::sys::fs::create_directories(OutDirectory))
    return llvm::createFileError(OutDirectory, Err,
                                 "failed to create directory.");
  return llvm::Error::success();
}
 
int main(int argc, const char **argv) {
  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
  std::error_code OK;
 
  ExitOnErr.setBanner("clang-doc error: ");
 
  const char *Overview =
      R"(Generates documentation from source code and comments.
 
Example usage for files without flags (default):
 
  $ clang-doc File1.cpp File2.cpp ... FileN.cpp
 
Example usage for a project using a compile commands database:
 
  $ clang-doc --executor=all-TUs compile_commands.json
)";
 
  auto Executor = ExitOnErr(clang::tooling::createExecutorFromCommandLineArgs(
      argc, argv, ClangDocCategory, Overview));
 
  // turns on ftime trace profiling
  if (FTimeTrace)
    llvm::timeTraceProfilerInitialize(200, "clang-doc");
  {
    llvm::TimeTraceScope("main");
 
    // Fail early if an invalid format was provided.
    llvm::StringRef Format = getFormatString();
    llvm::outs() << "Emiting docs in " << Format << " format.\n";
    auto G = ExitOnErr(doc::findGeneratorByName(Format));
 
    ArgumentsAdjuster ArgAdjuster;
    if (!DoxygenOnly)
      ArgAdjuster = combineAdjusters(
          getInsertArgumentAdjuster("-fparse-all-comments",
                                    tooling::ArgumentInsertPosition::END),
          ArgAdjuster);
 
    auto DiagOpts = std::make_unique<DiagnosticOptions>();
    TextDiagnosticPrinter *DiagClient =
        new TextDiagnosticPrinter(llvm::errs(), *DiagOpts);
    IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
    DiagnosticsEngine Diags(DiagID, *DiagOpts, DiagClient);
 
    clang::doc::ClangDocContext CDCtx(
        Executor->getExecutionContext(), ProjectName, PublicOnly, OutDirectory,
        SourceRoot, RepositoryUrl, RepositoryCodeLinePrefix, BaseDirectory,
        {UserStylesheets.begin(), UserStylesheets.end()}, Diags, FormatEnum,
        FTimeTrace, Pretty);
 
    if (Format == "html")
      ExitOnErr(getHtmlFiles(argv[0], CDCtx));
    else if (Format == "md_mustache")
      ExitOnErr(getMdFiles(argv[0], CDCtx));
 
    llvm::timeTraceProfilerBegin("Executor Launch", "total runtime");
    // Mapping phase
    llvm::outs() << "Mapping decls...\n";
    ExitOnErr(handleMappingFailures(
        Diags,
        Executor->execute(doc::newMapperActionFactory(CDCtx), ArgAdjuster)));
    llvm::timeTraceProfilerEnd();
 
    // Collect values into output by key.
    // In ToolResults, the Key is the hashed USR and the value is the
    // bitcode-encoded representation of the Info object.
    llvm::timeTraceProfilerBegin("Collect Info", "total runtime");
    llvm::outs() << "Collecting infos...\n";
    llvm::StringMap<std::vector<StringRef>> USRToBitcode;
    Executor->getToolResults()->forEachResult(
        [&](StringRef Key, StringRef Value) {
          USRToBitcode[Key].emplace_back(Value);
        });
    llvm::timeTraceProfilerEnd();
 
    // Collects all Infos according to their unique USR value. This map is added
    // to from the thread pool below and is protected by the USRToInfoMutex.
    llvm::sys::Mutex USRToInfoMutex;
    llvm::StringMap<doc::Info *> USRToInfo;
 
    // First reducing phase (reduce all decls into one info per decl).
    llvm::outs() << "Reducing " << USRToBitcode.size() << " infos...\n";
    std::atomic<bool> Error;
    Error = false;
    llvm::sys::Mutex IndexMutex;
    llvm::sys::Mutex DiagMutex;
    unsigned DiagIDBitcodeReading = Diags.getCustomDiagID(
        DiagnosticsEngine::Error, "error reading bitcode: %0");
    unsigned DiagIDBitcodeMerging = Diags.getCustomDiagID(
        DiagnosticsEngine::Error, "error merging bitcode: %0");
    // Note: we use per-thread arenas, so Pool must outlive the last use of this
    // memory in the generators.
    llvm::DefaultThreadPool Pool(
        // ExecutorConcurrency is a flag exposed by AllTUsExecution.h
        llvm::hardware_concurrency(ExecutorConcurrency));
    {
      llvm::TimeTraceScope TS("Reduce");
      for (const auto &Group : USRToBitcode) {
        StringRef Key = Group.getKey();
        std::vector<StringRef> Bitcodes = Group.getValue();
        Pool.async([Key, Bitcodes, &CDCtx, &Diags, &USRToInfo, &USRToInfoMutex,
                    &IndexMutex, &DiagMutex, &Error, DiagIDBitcodeReading,
                    DiagIDBitcodeMerging]() {
          if (CDCtx.FTimeTrace)
            llvm::timeTraceProfilerInitialize(200, "clang-doc");
 
          doc::Info *Reduced = nullptr;
          {
            llvm::TimeTraceScope Red("decoding and merging bitcode");
            for (const auto &Bitcode : Bitcodes) {
 
              llvm::scope_exit ArenaGuard(
                  [] { clang::doc::TransientArena.Reset(); });
              llvm::BitstreamCursor Stream(Bitcode);
              doc::ClangDocBitcodeReader Reader(Stream, Diags);
              auto ReadInfos = Reader.readBitcode();
              if (!ReadInfos) {
                std::lock_guard<llvm::sys::Mutex> Guard(DiagMutex);
 
                Diags.Report(DiagIDBitcodeReading)
                    << toString(ReadInfos.takeError());
                Error = true;
                return;
              }
              for (auto &I : *ReadInfos) {
                if (auto Err = doc::mergeSingleInfo(
                        Reduced, std::move(I), clang::doc::PersistentArena)) {
                  std::lock_guard<llvm::sys::Mutex> Guard(DiagMutex);
                  Diags.Report(DiagIDBitcodeMerging)
                      << toString(std::move(Err));
                  return;
                }
              }
            }
          } // time trace decoding and merging bitcode
 
          // Add a reference to this Info in the Index
          {
            llvm::TimeTraceScope Merge("addInfoToIndex");
            std::lock_guard<llvm::sys::Mutex> Guard(IndexMutex);
            clang::doc::Generator::addInfoToIndex(CDCtx.Idx, Reduced);
          }
          // Save in the result map (needs a lock due to threaded access).
          {
            llvm::TimeTraceScope Merge("USRToInfo");
            std::lock_guard<llvm::sys::Mutex> Guard(USRToInfoMutex);
            USRToInfo[Key] = std::move(Reduced);
          }
 
          if (CDCtx.FTimeTrace)
            llvm::timeTraceProfilerFinishThread();
        });
      }
 
      Pool.wait();
    } // time trace reduce
 
    if (Error)
      return 1;
 
    {
      llvm::TimeTraceScope Sort("Sort USRToInfo");
      sortUsrToInfo(USRToInfo);
    }
 
    llvm::timeTraceProfilerBegin("Writing output", "total runtime");
    // Ensure the root output directory exists.
    ExitOnErr(createDirectories(OutDirectory));
 
    // Run the generator.
    llvm::outs() << "Generating docs...\n";
 
    ExitOnErr(
        G->generateDocumentation(OutDirectory, std::move(USRToInfo), CDCtx));
    llvm::outs() << "Generating assets for docs...\n";
    ExitOnErr(G->createResources(CDCtx));
    llvm::timeTraceProfilerEnd();
  } // time trace main
 
  if (FTimeTrace) {
    std::error_code EC;
    llvm::raw_fd_ostream OS("clang-doc-tracing.json", EC,
                            llvm::sys::fs::OF_Text);
    if (!EC) {
      llvm::timeTraceProfilerWrite(OS);
      llvm::timeTraceProfilerCleanup();
    } else
      return 1;
  }
  return 0;
}