27 #include "clang/AST/ASTContext.h"
28 #include "clang/Index/IndexingAction.h"
29 #include "clang/Index/IndexingOptions.h"
30 #include "clang/Lex/Preprocessor.h"
31 #include "llvm/ADT/DenseMap.h"
32 #include "llvm/ADT/Optional.h"
33 #include "llvm/ADT/STLExtras.h"
34 #include "llvm/ADT/StringMap.h"
35 #include "llvm/ADT/StringRef.h"
46 SlabTuple indexSymbols(ASTContext &AST, Preprocessor &
PP,
47 llvm::ArrayRef<Decl *> DeclsToIndex,
48 const MainFileMacros *MacroRefsToIndex,
49 const CanonicalIncludes &Includes,
bool IsIndexMainAST,
50 llvm::StringRef Version,
bool CollectMainFileRefs) {
62 index::IndexingOptions IndexOpts;
64 IndexOpts.SystemSymbolFilter =
65 index::IndexingOptions::SystemSymbolFilterKind::DeclarationsOnly;
67 IndexOpts.IndexFunctionLocals =
true;
75 IndexOpts.IndexMacrosInPreprocessor =
true;
82 index::indexTopLevelDecls(AST,
PP, DeclsToIndex,
Collector, IndexOpts);
84 Collector.handleMacros(*MacroRefsToIndex);
87 const auto MainFileEntry = SM.getFileEntryRefForID(SM.getMainFileID());
89 std::string(MainFileEntry ? MainFileEntry->getName() :
"");
93 auto Relations =
Collector.takeRelations();
95 vlog(
"indexed {0} AST for {1} version {2}:\n"
96 " symbol slab: {3} symbols, {4} bytes\n"
97 " ref slab: {5} symbols, {6} refs, {7} bytes\n"
98 " relations slab: {8} relations, {9} bytes",
99 IsIndexMainAST ?
"file" :
"preamble",
FileName, Version, Syms.size(),
101 Relations.size(), Relations.bytes());
102 return std::make_tuple(std::move(Syms), std::move(
Refs),
103 std::move(Relations));
111 auto Entry = IG.try_emplace(URI).first;
112 auto &Node =
Entry->getValue();
113 Node = FullGraph.lookup(
Entry->getKey());
114 Node.URI =
Entry->getKey();
117 for (
auto &Include : Node.DirectIncludes) {
118 auto I = IG.try_emplace(Include).first;
119 I->getValue().URI = I->getKey();
120 Include = I->getKey();
127 :
Index(std::move(Input)) {
129 llvm::DenseMap<SymbolID, FileShard *> SymbolIDToFile;
133 for (
const auto &S : *Index.
Symbols) {
134 auto It = Shards.try_emplace(S.CanonicalDeclaration.FileURI);
135 It.first->getValue().Symbols.insert(&S);
136 SymbolIDToFile[S.ID] = &It.first->getValue();
139 S.Definition.FileURI != S.CanonicalDeclaration.FileURI) {
140 auto It = Shards.try_emplace(S.Definition.FileURI);
141 It.first->getValue().Symbols.insert(&S);
147 for (
const auto &SymRefs : *Index.
Refs) {
148 for (
const auto &R : SymRefs.second) {
149 const auto It = Shards.try_emplace(R.Location.FileURI);
150 It.first->getValue().Refs.insert(&R);
151 RefToSymID[&R] = SymRefs.first;
164 FileShard *SubjectFile = SymbolIDToFile.lookup(R.Subject);
165 FileShard *ObjectFile = SymbolIDToFile.lookup(R.Object);
167 SubjectFile->Relations.insert(&R);
168 if (ObjectFile && ObjectFile != SubjectFile)
169 ObjectFile->Relations.insert(&R);
174 const auto &FullGraph = *Index.
Sources;
175 for (
const auto &It : FullGraph) {
176 auto ShardIt = Shards.try_emplace(It.first());
177 ShardIt.first->getValue().IG = getSubGraph(It.first(), FullGraph);
184 std::vector<PathRef> Result;
185 Result.reserve(Shards.size());
186 for (
auto Key : Shards.keys())
187 Result.push_back(
Key);
191 llvm::Optional<IndexFileIn>
193 auto It = Shards.find(Uri);
194 if (It == Shards.end())
198 IF.
Sources = It->getValue().IG;
202 for (
const auto *S : It->getValue().Symbols)
204 IF.
Symbols = std::move(SymB).build();
207 for (
const auto *
Ref : It->getValue().Refs) {
208 auto SID = RefToSymID.lookup(
Ref);
211 IF.
Refs = std::move(RefB).build();
214 for (
const auto *Rel : It->getValue().Relations) {
219 return std::move(IF);
232 std::vector<Decl *> DeclsToIndex(
233 AST.getTranslationUnitDecl()->decls().begin(),
234 AST.getTranslationUnitDecl()->decls().end());
235 return indexSymbols(AST,
PP, DeclsToIndex,
242 : IdxContents(IdxContents) {}
245 std::unique_ptr<SymbolSlab> Symbols,
246 std::unique_ptr<RefSlab>
Refs,
247 std::unique_ptr<RelationSlab> Relations,
248 bool CountReferences) {
249 std::lock_guard<std::mutex> Lock(Mutex);
252 SymbolsSnapshot.erase(
Key);
254 SymbolsSnapshot[
Key] = std::move(Symbols);
256 RefsSnapshot.erase(
Key);
258 RefSlabAndCountReferences Item;
259 Item.CountReferences = CountReferences;
260 Item.Slab = std::move(
Refs);
261 RefsSnapshot[
Key] = std::move(Item);
264 RelationsSnapshot.erase(
Key);
266 RelationsSnapshot[
Key] = std::move(Relations);
269 std::unique_ptr<SymbolIndex>
272 std::vector<std::shared_ptr<SymbolSlab>> SymbolSlabs;
273 std::vector<std::shared_ptr<RefSlab>> RefSlabs;
274 std::vector<std::shared_ptr<RelationSlab>> RelationSlabs;
275 llvm::StringSet<>
Files;
276 std::vector<RefSlab *> MainFileRefs;
278 std::lock_guard<std::mutex> Lock(Mutex);
279 for (
const auto &FileAndSymbols : SymbolsSnapshot) {
280 SymbolSlabs.push_back(FileAndSymbols.second);
281 Files.insert(FileAndSymbols.first());
283 for (
const auto &FileAndRefs : RefsSnapshot) {
284 RefSlabs.push_back(FileAndRefs.second.Slab);
285 Files.insert(FileAndRefs.first());
286 if (FileAndRefs.second.CountReferences)
287 MainFileRefs.push_back(RefSlabs.back().get());
289 for (
const auto &FileAndRelations : RelationsSnapshot) {
290 Files.insert(FileAndRelations.first());
291 RelationSlabs.push_back(FileAndRelations.second);
295 *Version = this->Version;
297 std::vector<const Symbol *> AllSymbols;
298 std::vector<Symbol> SymsStorage;
299 switch (DuplicateHandle) {
301 llvm::DenseMap<SymbolID, Symbol> Merged;
302 for (
const auto &Slab : SymbolSlabs) {
303 for (
const auto &Sym : *Slab) {
304 assert(Sym.References == 0 &&
305 "Symbol with non-zero references sent to FileSymbols");
306 auto I = Merged.try_emplace(Sym.ID, Sym);
308 I.first->second =
mergeSymbol(I.first->second, Sym);
312 for (
const auto &Sym : *
Refs) {
313 auto It = Merged.find(Sym.first);
315 if (It == Merged.end())
317 It->getSecond().References += Sym.second.size();
319 SymsStorage.reserve(Merged.size());
320 for (
auto &Sym : Merged) {
321 SymsStorage.push_back(std::move(Sym.second));
322 AllSymbols.push_back(&SymsStorage.back());
327 llvm::DenseSet<SymbolID> AddedSymbols;
328 for (
const auto &Slab : SymbolSlabs)
329 for (
const auto &Sym : *Slab) {
330 assert(Sym.References == 0 &&
331 "Symbol with non-zero references sent to FileSymbols");
332 if (AddedSymbols.insert(Sym.ID).second)
333 AllSymbols.push_back(&Sym);
339 std::vector<Ref> RefsStorage;
340 llvm::DenseMap<SymbolID, llvm::ArrayRef<Ref>> AllRefs;
342 llvm::DenseMap<SymbolID, llvm::SmallVector<Ref, 4>> MergedRefs;
344 for (
const auto &
RefSlab : RefSlabs)
345 for (
const auto &Sym : *
RefSlab) {
346 MergedRefs[Sym.first].append(Sym.second.begin(), Sym.second.end());
347 Count += Sym.second.size();
349 RefsStorage.reserve(Count);
350 AllRefs.reserve(MergedRefs.size());
351 for (
auto &Sym : MergedRefs) {
352 auto &SymRefs = Sym.second;
355 llvm::copy(SymRefs, back_inserter(RefsStorage));
358 llvm::ArrayRef<Ref>(&RefsStorage[RefsStorage.size() - SymRefs.size()],
363 std::vector<Relation> AllRelations;
366 AllRelations.push_back(R);
371 llvm::sort(AllRelations);
372 AllRelations.erase(std::unique(AllRelations.begin(), AllRelations.end()),
376 RefsStorage.size() *
sizeof(
Ref) + SymsStorage.size() *
sizeof(
Symbol);
377 for (
const auto &Slab : SymbolSlabs)
378 StorageSize += Slab->bytes();
379 for (
const auto &
RefSlab : RefSlabs)
385 return std::make_unique<MemIndex>(
386 llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
387 std::move(AllRelations), std::move(
Files), IdxContents,
388 std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
389 std::move(RefsStorage), std::move(SymsStorage)),
392 return std::make_unique<dex::Dex>(
393 llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
394 std::move(AllRelations), std::move(
Files), IdxContents,
395 std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
396 std::move(RefsStorage), std::move(SymsStorage)),
399 llvm_unreachable(
"Unknown clangd::IndexType");
403 std::lock_guard<std::mutex> Lock(Mutex);
404 for (
const auto &SymSlab : SymbolsSnapshot) {
405 MT.
detail(SymSlab.first())
409 for (
const auto &
RefSlab : RefsSnapshot) {
414 for (
const auto &RelSlab : RelationsSnapshot) {
415 MT.
detail(RelSlab.first())
424 PreambleIndex(std::make_unique<
MemIndex>()),
426 MainFileIndex(std::make_unique<
MemIndex>()) {}
431 auto IF = ShardedIndex.
getShard(Uri);
436 Uri, std::make_unique<SymbolSlab>(std::move(*IF->
Symbols)),
437 std::make_unique<RefSlab>(),
438 std::make_unique<RelationSlab>(std::move(*IF->
Relations)),
441 size_t IndexVersion = 0;
445 std::lock_guard<std::mutex> Lock(UpdateIndexMu);
446 if (IndexVersion <= PreambleIndexVersion) {
450 PreambleIndexVersion = IndexVersion;
451 PreambleIndex.
reset(std::move(NewIndex));
453 "Build dynamic index for header symbols with estimated memory usage of "
460 ASTContext &AST, Preprocessor &
PP,
472 std::make_unique<SymbolSlab>(std::move(std::get<0>(Contents))),
473 std::make_unique<RefSlab>(std::move(std::get<1>(Contents))),
474 std::make_unique<RelationSlab>(std::move(std::get<2>(Contents))),
476 size_t IndexVersion = 0;
480 std::lock_guard<std::mutex> Lock(UpdateIndexMu);
481 if (IndexVersion <= MainIndexVersion) {
485 MainIndexVersion = IndexVersion;
486 MainFileIndex.
reset(std::move(NewIndex));
488 "Build dynamic index for main-file symbols with estimated memory usage "
500 MT.
child(
"main_file")