31 #include "clang/Basic/SourceLocation.h"
32 #include "clang/Basic/SourceManager.h"
33 #include "clang/Frontend/FrontendAction.h"
34 #include "llvm/ADT/ArrayRef.h"
35 #include "llvm/ADT/DenseSet.h"
36 #include "llvm/ADT/STLExtras.h"
37 #include "llvm/ADT/StringMap.h"
38 #include "llvm/ADT/StringRef.h"
39 #include "llvm/Support/Error.h"
40 #include "llvm/Support/Path.h"
41 #include "llvm/Support/Threading.h"
42 #include "llvm/Support/xxhash.h"
47 #include <condition_variable>
67 llvm::SmallString<128> getAbsolutePath(
const tooling::CompileCommand &Cmd) {
68 llvm::SmallString<128> AbsolutePath;
69 if (llvm::sys::path::is_absolute(Cmd.Filename)) {
70 AbsolutePath = Cmd.Filename;
72 AbsolutePath = Cmd.Directory;
73 llvm::sys::path::append(AbsolutePath, Cmd.Filename);
74 llvm::sys::path::remove_dots(AbsolutePath,
true);
79 bool shardIsStale(
const LoadedShard &LS, llvm::vfs::FileSystem *FS) {
80 auto Buf = FS->getBufferForFile(LS.AbsolutePath);
82 vlog(
"Background-index: Couldn't read {0} to validate stored index: {1}",
83 LS.AbsolutePath, Buf.getError().message());
87 return digest(Buf->get()->getBuffer()) != LS.Digest;
96 IndexingPriority(Opts.IndexingPriority),
97 ContextProvider(std::move(Opts.ContextProvider)),
99 Rebuilder(this, &IndexedSymbols, Opts.ThreadPoolSize),
100 IndexStorageFactory(std::move(IndexStorageFactory)),
101 Queue(std::move(Opts.OnProgress)),
103 CDB.watch([&](const std::vector<std::string> &ChangedFiles) {
106 assert(Opts.ThreadPoolSize > 0 &&
"Thread pool size can't be zero.");
107 assert(this->IndexStorageFactory &&
"Storage factory can not be null!");
108 for (
unsigned I = 0; I < Opts.ThreadPoolSize; ++I) {
109 ThreadPool.
runAsync(
"background-worker-" + llvm::Twine(I + 1),
112 Queue.
work([&] { Rebuilder.
idle(); });
123 const std::vector<std::string> &ChangedFiles) {
127 std::optional<WithContext> WithProvidedContext;
129 WithProvidedContext.emplace(ContextProvider(
""));
132 log(
"Enqueueing {0} commands for indexing", ChangedFiles.size());
135 auto NeedsReIndexing = loadProject(std::move(ChangedFiles));
137 std::shuffle(NeedsReIndexing.begin(), NeedsReIndexing.end(),
138 std::mt19937(std::random_device{}()));
139 std::vector<BackgroundQueue::Task> Tasks;
140 Tasks.reserve(NeedsReIndexing.size());
141 for (
const auto &File : NeedsReIndexing)
142 Tasks.push_back(indexFileTask(std::move(File)));
143 Queue.
append(std::move(Tasks));
146 T.QueuePri = LoadShards;
147 T.ThreadPri = llvm::ThreadPriority::Default;
152 Path = llvm::sys::path::filename(
Path);
153 return Path.drop_back(llvm::sys::path::extension(
Path).size());
156 BackgroundQueue::Task BackgroundIndex::indexFileTask(std::string
Path) {
158 uint64_t Key = llvm::xxHash64(
Path);
159 BackgroundQueue::Task T([
this,
Path(std::move(
Path))] {
160 std::optional<WithContext> WithProvidedContext;
162 WithProvidedContext.emplace(ContextProvider(
Path));
166 if (
auto Error = index(std::move(*Cmd)))
167 elog(
"Indexing {0} failed: {1}",
Path, std::move(Error));
169 T.QueuePri = IndexFile;
170 T.ThreadPri = IndexingPriority;
171 T.Tag = std::move(
Tag);
184 void BackgroundIndex::update(
186 const llvm::StringMap<ShardVersion> &ShardVersionsSnapshot,
189 llvm::StringMap<std::pair<Path, FileDigest>> FilesToUpdate;
192 for (
const auto &IndexIt : *Index.Sources) {
193 const auto &IGN = IndexIt.getValue();
196 elog(
"Failed to resolve URI: {0}", AbsPath.takeError());
199 const auto DigestIt = ShardVersionsSnapshot.find(*AbsPath);
201 if (DigestIt == ShardVersionsSnapshot.end() ||
202 DigestIt->getValue().Digest != IGN.Digest ||
203 (DigestIt->getValue().HadErrors && !HadErrors))
204 FilesToUpdate[IGN.URI] = {std::move(*AbsPath), IGN.Digest};
208 FileShardedIndex ShardedIndex(std::move(Index));
211 for (
const auto &FileIt : FilesToUpdate) {
212 auto Uri = FileIt.first();
213 auto IF = ShardedIndex.getShard(Uri);
214 assert(IF &&
"no shard for file in Index.Sources?");
225 if (
auto Error = IndexStorageFactory(
Path)->storeShard(
Path, *IF))
226 elog(
"Failed to write background-index shard for file {0}: {1}",
Path,
230 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
231 const auto &Hash = FileIt.getValue().second;
232 auto DigestIt = ShardVersions.try_emplace(
Path);
233 ShardVersion &SV = DigestIt.first->second;
236 if (!DigestIt.second && SV.Digest == Hash && SV.HadErrors && !HadErrors)
239 SV.HadErrors = HadErrors;
245 Uri, std::make_unique<SymbolSlab>(std::move(*IF->Symbols)),
246 std::make_unique<RefSlab>(std::move(*IF->Refs)),
247 std::make_unique<RelationSlab>(std::move(*IF->Relations)),
253 llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd) {
254 trace::Span
Tracer(
"BackgroundIndex");
256 auto AbsolutePath = getAbsolutePath(Cmd);
258 auto FS = TFS.
view(Cmd.Directory);
259 auto Buf = FS->getBufferForFile(AbsolutePath);
261 return llvm::errorCodeToError(Buf.getError());
262 auto Hash =
digest(Buf->get()->getBuffer());
265 llvm::StringMap<ShardVersion> ShardVersionsSnapshot;
267 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
268 ShardVersionsSnapshot = ShardVersions;
271 vlog(
"Indexing {0} (digest:={1})", Cmd.Filename, llvm::toHex(Hash));
274 Inputs.CompileCommand = std::move(Cmd);
278 return error(
"Couldn't build compiler invocation");
284 return error(
"Couldn't build compiler instance");
286 SymbolCollector::Options IndexOpts;
289 IndexOpts.FileFilter = [&ShardVersionsSnapshot](
const SourceManager &SM,
291 const auto *F = SM.getFileEntryForID(FID);
300 auto D = ShardVersionsSnapshot.find(*AbsPath);
301 if (
D != ShardVersionsSnapshot.end() &&
D->second.Digest == Digest &&
302 !
D->second.HadErrors)
306 IndexOpts.CollectMainFileRefs =
true;
310 IndexOpts, [&](SymbolSlab S) { Index.Symbols = std::move(S); },
311 [&](RefSlab R) { Index.Refs = std::move(R); },
312 [&](RelationSlab R) { Index.Relations = std::move(R); },
313 [&](
IncludeGraph IG) { Index.Sources = std::move(IG); });
320 const FrontendInputFile &Input =
Clang->getFrontendOpts().Inputs.front();
322 return error(
"BeginSourceFile() failed");
323 if (llvm::Error Err =
Action->Execute())
328 Index.Cmd =
Inputs.CompileCommand;
329 assert(Index.Symbols && Index.Refs && Index.Sources &&
330 "Symbols, Refs and Sources must be set.");
332 log(
"Indexed {0} ({1} symbols, {2} refs, {3} files)",
333 Inputs.CompileCommand.Filename, Index.Symbols->size(),
334 Index.Refs->numRefs(), Index.Sources->size());
339 bool HadErrors =
Clang->hasDiagnostics() &&
340 Clang->getDiagnostics().hasUncompilableErrorOccurred();
342 log(
"Failed to compile {0}, index may be incomplete", AbsolutePath);
343 for (
auto &It : *Index.Sources)
346 update(AbsolutePath, std::move(Index), ShardVersionsSnapshot, HadErrors);
349 return llvm::Error::success();
355 std::vector<std::string>
356 BackgroundIndex::loadProject(std::vector<std::string> MainFiles) {
359 llvm::erase_if(MainFiles, [&](
const std::string &TU) {
361 WithContext WithProvidedContext(ContextProvider(TU));
367 const std::vector<LoadedShard> Result =
369 size_t LoadedShards = 0;
372 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
373 for (
auto &LS : Result) {
378 ? std::make_unique<SymbolSlab>(std::move(*LS.Shard->Symbols))
380 auto RS = LS.Shard->Refs
381 ? std::make_unique<RefSlab>(std::move(*LS.Shard->Refs))
385 ? std::make_unique<RelationSlab>(std::move(*LS.Shard->Relations))
387 ShardVersion &SV = ShardVersions[LS.AbsolutePath];
388 SV.Digest = LS.Digest;
389 SV.HadErrors = LS.HadErrors;
393 std::move(SS), std::move(RS), std::move(RelS),
400 auto FS = TFS.
view(std::nullopt);
401 llvm::DenseSet<PathRef> TUsToIndex;
404 for (
auto &LS : Result) {
405 if (!shardIsStale(LS, FS.get()))
407 PathRef TUForFile = LS.DependentTU;
408 assert(!TUForFile.empty() &&
"File without a TU!");
415 TUsToIndex.insert(TUForFile);
418 return {TUsToIndex.begin(), TUsToIndex.end()};