31 #include "clang/Basic/SourceLocation.h"
32 #include "clang/Basic/SourceManager.h"
33 #include "clang/Frontend/FrontendAction.h"
34 #include "llvm/ADT/ArrayRef.h"
35 #include "llvm/ADT/DenseSet.h"
36 #include "llvm/ADT/STLExtras.h"
37 #include "llvm/ADT/StringMap.h"
38 #include "llvm/ADT/StringRef.h"
39 #include "llvm/Support/Error.h"
40 #include "llvm/Support/Path.h"
41 #include "llvm/Support/Threading.h"
42 #include "llvm/Support/xxhash.h"
47 #include <condition_variable>
66 llvm::SmallString<128> getAbsolutePath(
const tooling::CompileCommand &Cmd) {
67 llvm::SmallString<128> AbsolutePath;
68 if (llvm::sys::path::is_absolute(Cmd.Filename)) {
69 AbsolutePath = Cmd.Filename;
71 AbsolutePath = Cmd.Directory;
72 llvm::sys::path::append(AbsolutePath, Cmd.Filename);
73 llvm::sys::path::remove_dots(AbsolutePath,
true);
78 bool shardIsStale(
const LoadedShard &LS, llvm::vfs::FileSystem *FS) {
79 auto Buf = FS->getBufferForFile(LS.AbsolutePath);
81 vlog(
"Background-index: Couldn't read {0} to validate stored index: {1}",
82 LS.AbsolutePath, Buf.getError().message());
86 return digest(Buf->get()->getBuffer()) != LS.Digest;
95 IndexingPriority(Opts.IndexingPriority),
96 ContextProvider(std::move(Opts.ContextProvider)),
98 Rebuilder(this, &IndexedSymbols, Opts.ThreadPoolSize),
99 IndexStorageFactory(std::move(IndexStorageFactory)),
100 Queue(std::move(Opts.OnProgress)),
102 CDB.watch([&](const std::vector<std::string> &ChangedFiles) {
105 assert(Opts.ThreadPoolSize > 0 &&
"Thread pool size can't be zero.");
106 assert(this->IndexStorageFactory &&
"Storage factory can not be null!");
107 for (
unsigned I = 0; I < Opts.ThreadPoolSize; ++I) {
108 ThreadPool.
runAsync(
"background-worker-" + llvm::Twine(I + 1),
111 Queue.
work([&] { Rebuilder.
idle(); });
122 const std::vector<std::string> &ChangedFiles) {
126 llvm::Optional<WithContext> WithProvidedContext;
128 WithProvidedContext.emplace(ContextProvider(
""));
131 log(
"Enqueueing {0} commands for indexing", ChangedFiles.size());
134 auto NeedsReIndexing = loadProject(std::move(ChangedFiles));
136 std::shuffle(NeedsReIndexing.begin(), NeedsReIndexing.end(),
137 std::mt19937(std::random_device{}()));
138 std::vector<BackgroundQueue::Task> Tasks;
139 Tasks.reserve(NeedsReIndexing.size());
140 for (
const auto &File : NeedsReIndexing)
141 Tasks.push_back(indexFileTask(std::move(File)));
142 Queue.
append(std::move(Tasks));
145 T.QueuePri = LoadShards;
146 T.ThreadPri = llvm::ThreadPriority::Default;
151 Path = llvm::sys::path::filename(
Path);
152 return Path.drop_back(llvm::sys::path::extension(
Path).size());
155 BackgroundQueue::Task BackgroundIndex::indexFileTask(std::string
Path) {
157 uint64_t Key = llvm::xxHash64(
Path);
158 BackgroundQueue::Task T([
this,
Path(std::move(
Path))] {
159 llvm::Optional<WithContext> WithProvidedContext;
161 WithProvidedContext.emplace(ContextProvider(
Path));
165 if (
auto Error = index(std::move(*Cmd)))
166 elog(
"Indexing {0} failed: {1}",
Path, std::move(Error));
168 T.QueuePri = IndexFile;
169 T.ThreadPri = IndexingPriority;
170 T.Tag = std::move(
Tag);
183 void BackgroundIndex::update(
185 const llvm::StringMap<ShardVersion> &ShardVersionsSnapshot,
188 llvm::StringMap<std::pair<Path, FileDigest>> FilesToUpdate;
191 for (
const auto &IndexIt : *Index.Sources) {
192 const auto &IGN = IndexIt.getValue();
195 elog(
"Failed to resolve URI: {0}", AbsPath.takeError());
198 const auto DigestIt = ShardVersionsSnapshot.find(*AbsPath);
200 if (DigestIt == ShardVersionsSnapshot.end() ||
201 DigestIt->getValue().Digest != IGN.Digest ||
202 (DigestIt->getValue().HadErrors && !HadErrors))
203 FilesToUpdate[IGN.URI] = {std::move(*AbsPath), IGN.Digest};
207 FileShardedIndex ShardedIndex(std::move(Index));
210 for (
const auto &FileIt : FilesToUpdate) {
211 auto Uri = FileIt.first();
212 auto IF = ShardedIndex.getShard(Uri);
213 assert(IF &&
"no shard for file in Index.Sources?");
224 if (
auto Error = IndexStorageFactory(
Path)->storeShard(
Path, *IF))
225 elog(
"Failed to write background-index shard for file {0}: {1}",
Path,
229 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
230 const auto &Hash = FileIt.getValue().second;
231 auto DigestIt = ShardVersions.try_emplace(
Path);
232 ShardVersion &SV = DigestIt.first->second;
235 if (!DigestIt.second && SV.Digest == Hash && SV.HadErrors && !HadErrors)
238 SV.HadErrors = HadErrors;
244 Uri, std::make_unique<SymbolSlab>(std::move(*IF->Symbols)),
245 std::make_unique<RefSlab>(std::move(*IF->Refs)),
246 std::make_unique<RelationSlab>(std::move(*IF->Relations)),
252 llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd) {
253 trace::Span
Tracer(
"BackgroundIndex");
255 auto AbsolutePath = getAbsolutePath(Cmd);
257 auto FS = TFS.
view(Cmd.Directory);
258 auto Buf = FS->getBufferForFile(AbsolutePath);
260 return llvm::errorCodeToError(Buf.getError());
261 auto Hash =
digest(Buf->get()->getBuffer());
264 llvm::StringMap<ShardVersion> ShardVersionsSnapshot;
266 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
267 ShardVersionsSnapshot = ShardVersions;
270 vlog(
"Indexing {0} (digest:={1})", Cmd.Filename, llvm::toHex(Hash));
273 Inputs.CompileCommand = std::move(Cmd);
277 return error(
"Couldn't build compiler invocation");
283 return error(
"Couldn't build compiler instance");
285 SymbolCollector::Options IndexOpts;
288 IndexOpts.FileFilter = [&ShardVersionsSnapshot](
const SourceManager &SM,
290 const auto *F = SM.getFileEntryForID(FID);
299 auto D = ShardVersionsSnapshot.find(*AbsPath);
300 if (
D != ShardVersionsSnapshot.end() &&
D->second.Digest == Digest &&
301 !
D->second.HadErrors)
305 IndexOpts.CollectMainFileRefs =
true;
309 IndexOpts, [&](SymbolSlab S) { Index.Symbols = std::move(S); },
310 [&](RefSlab R) { Index.Refs = std::move(R); },
311 [&](RelationSlab R) { Index.Relations = std::move(R); },
312 [&](
IncludeGraph IG) { Index.Sources = std::move(IG); });
319 const FrontendInputFile &Input =
Clang->getFrontendOpts().Inputs.front();
321 return error(
"BeginSourceFile() failed");
322 if (llvm::Error Err =
Action->Execute())
327 Index.Cmd =
Inputs.CompileCommand;
328 assert(Index.Symbols && Index.Refs && Index.Sources &&
329 "Symbols, Refs and Sources must be set.");
331 log(
"Indexed {0} ({1} symbols, {2} refs, {3} files)",
332 Inputs.CompileCommand.Filename, Index.Symbols->size(),
333 Index.Refs->numRefs(), Index.Sources->size());
338 bool HadErrors =
Clang->hasDiagnostics() &&
339 Clang->getDiagnostics().hasUncompilableErrorOccurred();
341 log(
"Failed to compile {0}, index may be incomplete", AbsolutePath);
342 for (
auto &It : *Index.Sources)
345 update(AbsolutePath, std::move(Index), ShardVersionsSnapshot, HadErrors);
348 return llvm::Error::success();
354 std::vector<std::string>
355 BackgroundIndex::loadProject(std::vector<std::string> MainFiles) {
358 llvm::erase_if(MainFiles, [&](
const std::string &TU) {
360 WithContext WithProvidedContext(ContextProvider(TU));
366 const std::vector<LoadedShard> Result =
368 size_t LoadedShards = 0;
371 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
372 for (
auto &LS : Result) {
377 ? std::make_unique<SymbolSlab>(std::move(*LS.Shard->Symbols))
379 auto RS = LS.Shard->Refs
380 ? std::make_unique<RefSlab>(std::move(*LS.Shard->Refs))
384 ? std::make_unique<RelationSlab>(std::move(*LS.Shard->Relations))
386 ShardVersion &SV = ShardVersions[LS.AbsolutePath];
387 SV.Digest = LS.Digest;
388 SV.HadErrors = LS.HadErrors;
392 std::move(SS), std::move(RS), std::move(RelS),
399 auto FS = TFS.
view(llvm::None);
400 llvm::DenseSet<PathRef> TUsToIndex;
403 for (
auto &LS : Result) {
404 if (!shardIsStale(LS, FS.get()))
406 PathRef TUForFile = LS.DependentTU;
407 assert(!TUForFile.empty() &&
"File without a TU!");
414 TUsToIndex.insert(TUForFile);
417 return {TUsToIndex.begin(), TUsToIndex.end()};