17#include "clang/Tooling/ArgumentsAdjusters.h"
18#include "clang/Tooling/CompilationDatabase.h"
19#include "clang/Tooling/CompilationDatabasePluginRegistry.h"
20#include "clang/Tooling/JSONCompilationDatabase.h"
21#include "clang/Tooling/Tooling.h"
22#include "llvm/ADT/PointerIntPair.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/ScopeExit.h"
25#include "llvm/ADT/SmallString.h"
26#include "llvm/ADT/StringMap.h"
27#include "llvm/Support/Path.h"
28#include "llvm/Support/VirtualFileSystem.h"
29#include "llvm/TargetParser/Host.h"
32#include <condition_variable>
54tooling::CompileCommand
56 std::vector<std::string> Argv = {
"clang"};
60 auto FileExtension = llvm::sys::path::extension(
File);
61 if (FileExtension.empty() || FileExtension ==
".h")
62 Argv.push_back(
"-xobjective-c++-header");
63 Argv.push_back(std::string(
File));
64 tooling::CompileCommand Cmd(llvm::sys::path::parent_path(
File),
65 llvm::sys::path::filename(
File), std::move(Argv),
67 Cmd.Heuristic =
"clangd fallback";
83 using stopwatch = std::chrono::steady_clock;
92 CachedFile(llvm::StringRef
Parent, llvm::StringRef Rel) {
94 llvm::sys::path::append(
Path, Rel);
95 this->Path =
Path.str().str();
98 size_t Size = NoFileCached;
99 llvm::sys::TimePoint<> ModifiedTime;
102 static constexpr size_t NoFileCached = -1;
111 std::unique_ptr<llvm::MemoryBuffer>
Buffer;
114 LoadResult load(llvm::vfs::FileSystem &FS,
bool HasOldData);
119 std::atomic<stopwatch::rep> NoCDBAt = {
120 stopwatch::time_point::min().time_since_epoch().count()};
125 stopwatch::time_point CachePopulatedAt = stopwatch::time_point::min();
127 bool NeedsBroadcast =
false;
130 std::shared_ptr<tooling::CompilationDatabase> CDB;
132 CachedFile CompileCommandsJson;
133 CachedFile BuildCompileCommandsJson;
134 CachedFile CompileFlagsTxt;
141 CachedFile *ActiveCachedFile =
nullptr;
145 : CompileCommandsJson(
Path,
"compile_commands.json"),
146 BuildCompileCommandsJson(
Path,
"build/compile_commands.json"),
147 CompileFlagsTxt(
Path,
"compile_flags.txt"),
Path(
Path) {
148 assert(llvm::sys::path::is_absolute(
Path));
161 std::shared_ptr<const tooling::CompilationDatabase>
163 stopwatch::time_point FreshTime, stopwatch::time_point FreshTimeMissing) {
165 if (stopwatch::time_point(stopwatch::duration(NoCDBAt.load())) >
167 ShouldBroadcast =
false;
171 std::lock_guard<std::mutex> Lock(Mu);
172 auto RequestBroadcast = llvm::make_scope_exit([&, OldCDB(CDB.get())] {
174 if (CDB != nullptr && CDB.get() != OldCDB)
175 NeedsBroadcast = true;
176 else if (CDB == nullptr)
177 NeedsBroadcast = false;
179 if (!ShouldBroadcast)
181 ShouldBroadcast = NeedsBroadcast;
182 NeedsBroadcast = false;
186 if (CachePopulatedAt > FreshTime)
189 if (load(*TFS.view(std::nullopt))) {
191 CachePopulatedAt = stopwatch::now();
192 NoCDBAt.store((CDB ? stopwatch::time_point::min() : CachePopulatedAt)
202 bool load(llvm::vfs::FileSystem &FS);
206DirectoryBasedGlobalCompilationDatabase::DirectoryCache::CachedFile::load(
207 llvm::vfs::FileSystem &FS,
bool HasOldData) {
208 auto Stat = FS.status(
Path);
209 if (!Stat || !Stat->isRegularFile()) {
212 return {LoadResult::FileNotFound,
nullptr};
215 if (HasOldData && Stat->getLastModificationTime() == ModifiedTime &&
216 Stat->getSize() == Size)
217 return {LoadResult::FoundSameData,
nullptr};
218 auto Buf = FS.getBufferForFile(
Path);
219 if (!Buf || (*Buf)->getBufferSize() != Stat->getSize()) {
226 elog(
"Failed to read {0}: {1}",
Path,
227 Buf ?
"size changed" : Buf.getError().message());
228 return {LoadResult::TransientError,
nullptr};
232 if (HasOldData && NewContentHash == ContentHash) {
234 ModifiedTime = Stat->getLastModificationTime();
235 return {LoadResult::FoundSameData,
nullptr};
238 Size = (*Buf)->getBufferSize();
239 ModifiedTime = Stat->getLastModificationTime();
240 ContentHash = NewContentHash;
241 return {LoadResult::FoundNewData, std::move(*Buf)};
245static std::unique_ptr<tooling::CompilationDatabase>
247 if (
auto CDB = tooling::JSONCompilationDatabase::loadFromBuffer(
248 Data,
Error, tooling::JSONCommandLineSyntax::AutoDetect)) {
256 auto FS = llvm::vfs::getRealFileSystem();
257 return tooling::inferMissingCompileCommands(
258 expandResponseFiles(std::move(CDB), std::move(FS)));
262static std::unique_ptr<tooling::CompilationDatabase>
264 return tooling::FixedCompilationDatabase::loadFromBuffer(
265 llvm::sys::path::parent_path(
Path), Data,
Error);
268bool DirectoryBasedGlobalCompilationDatabase::DirectoryCache::load(
269 llvm::vfs::FileSystem &FS) {
270 dlog(
"Probing directory {0}",
Path);
280 std::unique_ptr<tooling::CompilationDatabase> (*Parser)(
285 for (
const auto &
Entry : {CDBFile{&CompileCommandsJson,
parseJSON},
286 CDBFile{&BuildCompileCommandsJson,
parseJSON},
288 bool Active = ActiveCachedFile ==
Entry.File;
289 auto Loaded =
Entry.File->load(FS, Active);
290 switch (Loaded.Result) {
293 log(
"Unloaded compilation database from {0}",
Entry.File->Path);
294 ActiveCachedFile =
nullptr;
303 assert(Active &&
"CachedFile may not return 'same data' if !HasOldData");
308 CDB =
Entry.Parser(
Entry.File->Path, Loaded.Buffer->getBuffer(),
Error);
310 log(
"{0} compilation database from {1}", Active ?
"Reloaded" :
"Loaded",
313 elog(
"Failed to load compilation database from {0}: {1}",
315 ActiveCachedFile =
Entry.File;
326 if (CachePopulatedAt > stopwatch::time_point::min())
328 for (
const auto &
Entry :
329 tooling::CompilationDatabasePluginRegistry::entries()) {
331 if (
Entry.getName() ==
"fixed-compilation-database" ||
332 Entry.getName() ==
"json-compilation-database")
334 auto Plugin =
Entry.instantiate();
335 if (
auto CDB = Plugin->loadFromDirectory(
Path,
Error)) {
336 log(
"Loaded compilation database from {0} with plugin {1}",
Path,
338 this->CDB = std::move(CDB);
343 dlog(
"No compilation database at {0}",
Path);
359std::optional<tooling::CompileCommand>
361 CDBLookupRequest Req;
363 Req.ShouldBroadcast =
true;
364 auto Now = std::chrono::steady_clock::now();
368 auto Res = lookupCDB(Req);
370 log(
"Failed to find compilation database for {0}",
File);
374 auto Candidates = Res->CDB->getCompileCommands(
File);
375 if (!Candidates.empty())
376 return std::move(Candidates.front());
381std::vector<DirectoryBasedGlobalCompilationDatabase::DirectoryCache *>
382DirectoryBasedGlobalCompilationDatabase::getDirectoryCaches(
383 llvm::ArrayRef<llvm::StringRef> Dirs)
const {
384 std::vector<std::string> FoldedDirs;
385 FoldedDirs.reserve(Dirs.size());
386 for (
const auto &Dir : Dirs) {
388 if (!llvm::sys::path::is_absolute(Dir))
389 elog(
"Trying to cache CDB for relative {0}");
394 std::vector<DirectoryCache *> Ret;
395 Ret.reserve(Dirs.size());
397 std::lock_guard<std::mutex> Lock(DirCachesMutex);
398 for (
unsigned I = 0; I < Dirs.size(); ++I)
399 Ret.push_back(&DirCaches.try_emplace(FoldedDirs[I], Dirs[I]).first->second);
403std::optional<DirectoryBasedGlobalCompilationDatabase::CDBLookupResult>
404DirectoryBasedGlobalCompilationDatabase::lookupCDB(
405 CDBLookupRequest Request)
const {
406 assert(llvm::sys::path::is_absolute(Request.FileName) &&
407 "path must be absolute");
410 std::vector<llvm::StringRef> SearchDirs;
414 WithContext WithProvidedContext(Opts.
ContextProvider(Request.FileName));
416 switch (Spec.Policy) {
420 Storage = *Spec.FixedCDBPath;
421 SearchDirs = {Storage};
428 actOnAllParentDirectories(Storage, [&](llvm::StringRef Dir) {
429 SearchDirs.push_back(Dir);
435 std::shared_ptr<const tooling::CompilationDatabase> CDB =
nullptr;
436 bool ShouldBroadcast =
false;
437 DirectoryCache *DirCache =
nullptr;
438 for (DirectoryCache *
Candidate : getDirectoryCaches(SearchDirs)) {
439 bool CandidateShouldBroadcast = Request.ShouldBroadcast;
440 if ((CDB =
Candidate->get(Opts.
TFS, CandidateShouldBroadcast,
441 Request.FreshTime, Request.FreshTimeMissing))) {
443 ShouldBroadcast = CandidateShouldBroadcast;
451 CDBLookupResult Result;
452 Result.CDB = std::move(CDB);
453 Result.PI.SourceRoot = DirCache->Path;
456 broadcastCDB(Result);
472 std::condition_variable CV;
475 std::atomic<bool> ShouldStop = {
false};
477 CDBLookupResult Lookup;
480 std::deque<Task> Queue;
481 std::optional<Task> ActiveTask;
486 std::unique_lock<std::mutex> Lock(Mu);
488 bool Stopping =
false;
490 return (Stopping = ShouldStop.load(std::memory_order_acquire)) ||
498 ActiveTask = std::move(Queue.front());
504 process(ActiveTask->Lookup);
513 void process(
const CDBLookupResult &T);
521 assert(!Lookup.PI.SourceRoot.empty());
522 std::lock_guard<std::mutex> Lock(Mu);
524 llvm::erase_if(Queue, [&](
const Task &T) {
525 return T.Lookup.PI.SourceRoot == Lookup.PI.SourceRoot;
533 std::unique_lock<std::mutex> Lock(Mu);
534 return wait(Lock, CV, Timeout,
535 [&] {
return Queue.empty() && !ActiveTask; });
540 std::lock_guard<std::mutex> Lock(Mu);
541 ShouldStop.store(
true, std::memory_order_release);
562 llvm::StringRef ThisDir;
569 DirInfo *
Parent =
nullptr;
571 llvm::StringMap<DirInfo> Dirs;
574 using SearchPath = llvm::PointerIntPair<DirInfo *, 1>;
578 DirInfo *addParents(llvm::StringRef FilePath) {
579 DirInfo *Leaf =
nullptr;
580 DirInfo *Child =
nullptr;
581 actOnAllParentDirectories(FilePath, [&](llvm::StringRef Dir) {
582 auto &
Info = Dirs[Dir];
588 Child->Parent = &
Info;
592 return Info.Parent !=
nullptr;
603 std::vector<llvm::StringRef> DirKeys;
604 std::vector<DirInfo *> DirValues;
605 DirKeys.reserve(Dirs.size() + 1);
606 DirValues.reserve(Dirs.size());
607 for (
auto &
E : Dirs) {
608 DirKeys.push_back(
E.first());
609 DirValues.push_back(&
E.second);
615 DirKeys.push_back(ThisDir);
616 auto DirCaches =
Parent.getDirectoryCaches(DirKeys);
618 DirCaches.pop_back();
621 for (
unsigned I = 0; I < DirKeys.size(); ++I) {
622 DirValues[I]->Cache = DirCaches[I];
623 if (DirCaches[I] == ThisCache)
624 DirValues[I]->State = DirInfo::TargetCDB;
629 bool shouldInclude(SearchPath P) {
630 DirInfo *
Info = P.getPointer();
633 if (
Info->State == DirInfo::Unknown) {
634 assert(
Info->Cache &&
"grabCaches() should have filled this");
637 constexpr auto IgnoreCache = std::chrono::steady_clock::time_point::max();
639 bool ShouldBroadcast =
false;
641 nullptr !=
Info->Cache->get(
Parent.Opts.TFS, ShouldBroadcast,
644 Info->State = Exists ? DirInfo::OtherCDB : DirInfo::Missing;
647 if (
Info->State != DirInfo::Missing)
648 return Info->State == DirInfo::TargetCDB;
650 if (!P.getInt() || !
Info->Parent)
653 return shouldInclude(SearchPath(
Info->Parent, 1));
661 std::vector<std::string>
filter(std::vector<std::string> AllFiles,
662 std::atomic<bool> &ShouldStop) {
663 std::vector<std::string> Filtered;
665 auto ExitEarly = [&] {
666 if (ShouldStop.load(std::memory_order_acquire)) {
667 log(
"Giving up on broadcasting CDB, as we're shutting down");
674 std::vector<SearchPath> SearchPaths(AllFiles.size());
675 for (
unsigned I = 0; I < AllFiles.size(); ++I) {
676 if (
Parent.Opts.CompileCommandsDir) {
677 SearchPaths[I].setPointer(&Dirs[*
Parent.Opts.CompileCommandsDir]);
689 SearchPaths[I].setInt(1);
690 SearchPaths[I].setPointer(addParents(AllFiles[I]));
700 for (
unsigned I = 0; I < AllFiles.size(); ++I) {
703 if (shouldInclude(SearchPaths[I]))
704 Filtered.push_back(std::move(AllFiles[I]));
710void DirectoryBasedGlobalCompilationDatabase::BroadcastThread::process(
711 const CDBLookupResult &T) {
712 vlog(
"Broadcasting compilation database from {0}", T.PI.SourceRoot);
713 std::vector<std::string> GovernedFiles =
714 Filter(T.PI.SourceRoot, Parent).filter(T.CDB->getAllFiles(), ShouldStop);
715 if (!GovernedFiles.empty())
719void DirectoryBasedGlobalCompilationDatabase::broadcastCDB(
720 CDBLookupResult Result)
const {
721 assert(Result.CDB &&
"Trying to broadcast an invalid CDB!");
722 Broadcaster->enqueue(Result);
727 return Broadcaster->blockUntilIdle(Timeout);
730std::optional<ProjectInfo>
732 CDBLookupRequest Req;
734 Req.ShouldBroadcast =
false;
735 Req.FreshTime = Req.FreshTimeMissing =
736 std::chrono::steady_clock::time_point::min();
737 auto Res = lookupCDB(Req);
744 std::vector<std::string> FallbackFlags,
747 FallbackFlags(std::move(FallbackFlags)) {}
749std::optional<tooling::CompileCommand>
751 std::optional<tooling::CompileCommand> Cmd;
753 std::lock_guard<std::mutex> Lock(Mutex);
755 if (It != Commands.end())
764 auto FS = llvm::vfs::getRealFileSystem();
765 auto Tokenizer = llvm::Triple(llvm::sys::getProcessTriple()).isOSWindows()
766 ? llvm::cl::TokenizeWindowsCommandLine
767 : llvm::cl::TokenizeGNUCommandLine;
771 tooling::addExpandedResponseFiles(Cmd->CommandLine, Cmd->Directory,
785 std::lock_guard<std::mutex> Lock(Mutex);
786 Cmd.CommandLine.insert(Cmd.CommandLine.end(), FallbackFlags.begin(),
787 FallbackFlags.end());
794 std::optional<tooling::CompileCommand> Cmd) {
800 std::unique_lock<std::mutex> Lock(Mutex);
802 Commands[CanonPath] = std::move(*Cmd);
804 Commands.erase(CanonPath);
812 BaseChanged = Base->watch([
this](
const std::vector<std::string>
Changes) {
819 BaseOwner = std::move(Base);
822std::optional<tooling::CompileCommand>
include_cleaner::Header Missing
A context is an immutable container for per-request data that must be propagated through layers that ...
Context clone() const
Clone this context object.
static const Context & current()
Returns the context for the current thread, creating it if needed.
A point in time we can wait for.
tooling::CompileCommand getFallbackCommand(PathRef File) const override
Makes a guess at how to build a file.
DelegatingCDB(const GlobalCompilationDatabase *Base)
std::optional< tooling::CompileCommand > getCompileCommand(PathRef File) const override
If there are any known-good commands for building this file, returns one.
bool blockUntilIdle(Deadline D) const override
If the CDB does any asynchronous work, wait for it to complete.
std::optional< ProjectInfo > getProjectInfo(PathRef File) const override
Finds the closest project to File.
Filter(llvm::StringRef ThisDir, DirectoryBasedGlobalCompilationDatabase &Parent)
std::vector< std::string > filter(std::vector< std::string > AllFiles, std::atomic< bool > &ShouldStop)
bool blockUntilIdle(Deadline Timeout)
void enqueue(CDBLookupResult Lookup)
BroadcastThread(DirectoryBasedGlobalCompilationDatabase &Parent)
DirectoryCache(llvm::StringRef Path)
std::shared_ptr< const tooling::CompilationDatabase > get(const ThreadsafeFS &TFS, bool &ShouldBroadcast, stopwatch::time_point FreshTime, stopwatch::time_point FreshTimeMissing)
Gets compile args from tooling::CompilationDatabases built for parent directories.
bool blockUntilIdle(Deadline Timeout) const override
If the CDB does any asynchronous work, wait for it to complete.
DirectoryBasedGlobalCompilationDatabase(const Options &Opts)
~DirectoryBasedGlobalCompilationDatabase() override
std::optional< tooling::CompileCommand > getCompileCommand(PathRef File) const override
Scans File's parents looking for compilation databases.
std::optional< ProjectInfo > getProjectInfo(PathRef File) const override
Returns the path to first directory containing a compilation database in File's parents.
void broadcast(const T &V)
Provides compilation arguments used for parsing C and C++ files.
virtual tooling::CompileCommand getFallbackCommand(PathRef File) const
Makes a guess at how to build a file.
virtual std::optional< ProjectInfo > getProjectInfo(PathRef File) const
Finds the closest project to File.
virtual bool blockUntilIdle(Deadline D) const
If the CDB does any asynchronous work, wait for it to complete.
virtual std::optional< tooling::CompileCommand > getCompileCommand(PathRef File) const =0
If there are any known-good commands for building this file, returns one.
CommandChanged OnCommandChanged
tooling::CompileCommand getFallbackCommand(PathRef File) const override
Makes a guess at how to build a file.
void setCompileCommand(PathRef File, std::optional< tooling::CompileCommand > CompilationCommand)
Sets or clears the compilation command for a particular file.
std::optional< tooling::CompileCommand > getCompileCommand(PathRef File) const override
If there are any known-good commands for building this file, returns one.
llvm::unique_function< void(tooling::CompileCommand &, StringRef File) const > CommandMangler
OverlayCDB(const GlobalCompilationDatabase *Base, std::vector< std::string > FallbackFlags={}, CommandMangler Mangler=nullptr)
Wrapper for vfs::FileSystem for use in multithreaded programs like clangd.
WithContext replaces Context::current() with a provided scope.
@ Info
An information message.
std::string maybeCaseFoldPath(PathRef Path)
std::string Path
A typedef to represent a file path.
FileDigest digest(llvm::StringRef Content)
static std::unique_ptr< tooling::CompilationDatabase > parseJSON(PathRef Path, llvm::StringRef Data, std::string &Error)
void vlog(const char *Fmt, Ts &&... Vals)
std::array< uint8_t, 8 > FileDigest
PathRef absoluteParent(PathRef Path)
Variant of parent_path that operates only on absolute paths.
void log(const char *Fmt, Ts &&... Vals)
void wait(std::unique_lock< std::mutex > &Lock, std::condition_variable &CV, Deadline D)
Wait once on CV for the specified duration.
Path removeDots(PathRef File)
Returns a version of File that doesn't contain dots and dot dots.
static std::unique_ptr< tooling::CompilationDatabase > parseFixed(PathRef Path, llvm::StringRef Data, std::string &Error)
llvm::StringRef PathRef
A typedef to represent a ref to file path.
void elog(const char *Fmt, Ts &&... Vals)
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
enum clang::clangd::Config::CDBSearchSpec::@10 Policy
std::optional< std::string > FixedCDBPath
static const Config & current()
Returns the Config of the current Context, or an empty configuration.
CDBSearchSpec CDBSearch
Where to search for compilation databases for this file's flags.
struct clang::clangd::Config::@2 CompileFlags
Controls how the compile command for the current file is determined.
enum clang::clangd::DirectoryBasedGlobalCompilationDatabase::DirectoryCache::CachedFile::LoadResult::@14 Result
std::unique_ptr< llvm::MemoryBuffer > Buffer
std::chrono::steady_clock::duration RevalidateAfter
std::chrono::steady_clock::duration RevalidateMissingAfter
std::function< Context(llvm::StringRef)> ContextProvider
std::optional< Path > CompileCommandsDir