clang-tools  10.0.0svn
BackgroundIndexTests.cpp
Go to the documentation of this file.
1 #include "Headers.h"
2 #include "SyncAPI.h"
3 #include "TestFS.h"
4 #include "TestIndex.h"
5 #include "TestTU.h"
6 #include "index/Background.h"
8 #include "clang/Tooling/CompilationDatabase.h"
9 #include "llvm/Support/ScopedPrinter.h"
10 #include "llvm/Support/Threading.h"
11 #include "gmock/gmock.h"
12 #include "gtest/gtest.h"
13 #include <deque>
14 #include <thread>
15 
16 using ::testing::_;
17 using ::testing::AllOf;
18 using ::testing::Contains;
19 using ::testing::ElementsAre;
20 using ::testing::Not;
21 using ::testing::UnorderedElementsAre;
22 
23 namespace clang {
24 namespace clangd {
25 
26 MATCHER_P(Named, N, "") { return arg.Name == N; }
27 MATCHER(Declared, "") {
28  return !StringRef(arg.CanonicalDeclaration.FileURI).empty();
29 }
30 MATCHER(Defined, "") { return !StringRef(arg.Definition.FileURI).empty(); }
31 MATCHER_P(FileURI, F, "") { return StringRef(arg.Location.FileURI) == F; }
32 ::testing::Matcher<const RefSlab &>
33 RefsAre(std::vector<::testing::Matcher<Ref>> Matchers) {
34  return ElementsAre(::testing::Pair(_, UnorderedElementsAreArray(Matchers)));
35 }
36 // URI cannot be empty since it references keys in the IncludeGraph.
37 MATCHER(EmptyIncludeNode, "") {
38  return arg.Flags == IncludeGraphNode::SourceFlag::None && !arg.URI.empty() &&
39  arg.Digest == FileDigest{{0}} && arg.DirectIncludes.empty();
40 }
41 
42 MATCHER(HadErrors, "") {
43  return arg.Flags & IncludeGraphNode::SourceFlag::HadErrors;
44 }
45 
46 MATCHER_P(NumReferences, N, "") { return arg.References == N; }
47 
49  mutable std::mutex StorageMu;
50  llvm::StringMap<std::string> &Storage;
51  size_t &CacheHits;
52 
53 public:
54  MemoryShardStorage(llvm::StringMap<std::string> &Storage, size_t &CacheHits)
55  : Storage(Storage), CacheHits(CacheHits) {}
56  llvm::Error storeShard(llvm::StringRef ShardIdentifier,
57  IndexFileOut Shard) const override {
58  std::lock_guard<std::mutex> Lock(StorageMu);
59  AccessedPaths.insert(ShardIdentifier);
60  Storage[ShardIdentifier] = llvm::to_string(Shard);
61  return llvm::Error::success();
62  }
63  std::unique_ptr<IndexFileIn>
64  loadShard(llvm::StringRef ShardIdentifier) const override {
65  std::lock_guard<std::mutex> Lock(StorageMu);
66  AccessedPaths.insert(ShardIdentifier);
67  if (Storage.find(ShardIdentifier) == Storage.end()) {
68  return nullptr;
69  }
70  auto IndexFile = readIndexFile(Storage[ShardIdentifier]);
71  if (!IndexFile) {
72  ADD_FAILURE() << "Error while reading " << ShardIdentifier << ':'
73  << IndexFile.takeError();
74  return nullptr;
75  }
76  CacheHits++;
77  return std::make_unique<IndexFileIn>(std::move(*IndexFile));
78  }
79 
80  mutable llvm::StringSet<> AccessedPaths;
81 };
82 
83 class BackgroundIndexTest : public ::testing::Test {
84 protected:
86 };
87 
88 TEST_F(BackgroundIndexTest, NoCrashOnErrorFile) {
90  FS.Files[testPath("root/A.cc")] = "error file";
91  llvm::StringMap<std::string> Storage;
92  size_t CacheHits = 0;
93  MemoryShardStorage MSS(Storage, CacheHits);
94  OverlayCDB CDB(/*Base=*/nullptr);
95  BackgroundIndex Idx(Context::empty(), FS, CDB,
96  [&](llvm::StringRef) { return &MSS; });
97 
98  tooling::CompileCommand Cmd;
99  Cmd.Filename = testPath("root/A.cc");
100  Cmd.Directory = testPath("root");
101  Cmd.CommandLine = {"clang++", "-DA=1", testPath("root/A.cc")};
102  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
103 
104  ASSERT_TRUE(Idx.blockUntilIdleForTest());
105 }
106 
107 TEST_F(BackgroundIndexTest, IndexTwoFiles) {
109  // a.h yields different symbols when included by A.cc vs B.cc.
110  FS.Files[testPath("root/A.h")] = R"cpp(
111  void common();
112  void f_b();
113  #if A
114  class A_CC {};
115  #else
116  class B_CC{};
117  #endif
118  )cpp";
119  FS.Files[testPath("root/A.cc")] =
120  "#include \"A.h\"\nvoid g() { (void)common; }";
121  FS.Files[testPath("root/B.cc")] =
122  R"cpp(
123  #define A 0
124  #include "A.h"
125  void f_b() {
126  (void)common;
127  (void)common;
128  (void)common;
129  (void)common;
130  })cpp";
131  llvm::StringMap<std::string> Storage;
132  size_t CacheHits = 0;
133  MemoryShardStorage MSS(Storage, CacheHits);
134  OverlayCDB CDB(/*Base=*/nullptr);
135  BackgroundIndex Idx(Context::empty(), FS, CDB,
136  [&](llvm::StringRef) { return &MSS; });
137 
138  tooling::CompileCommand Cmd;
139  Cmd.Filename = testPath("root/A.cc");
140  Cmd.Directory = testPath("root");
141  Cmd.CommandLine = {"clang++", "-DA=1", testPath("root/A.cc")};
142  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
143 
144  ASSERT_TRUE(Idx.blockUntilIdleForTest());
145  EXPECT_THAT(runFuzzyFind(Idx, ""),
146  UnorderedElementsAre(AllOf(Named("common"), NumReferences(1U)),
147  AllOf(Named("A_CC"), NumReferences(0U)),
148  AllOf(Named("g"), NumReferences(0U)),
149  AllOf(Named("f_b"), Declared(),
150  Not(Defined()), NumReferences(0U))));
151 
152  Cmd.Filename = testPath("root/B.cc");
153  Cmd.CommandLine = {"clang++", Cmd.Filename};
154  CDB.setCompileCommand(testPath("root/B.cc"), Cmd);
155 
156  ASSERT_TRUE(Idx.blockUntilIdleForTest());
157  // B_CC is dropped as we don't collect symbols from A.h in this compilation.
158  EXPECT_THAT(runFuzzyFind(Idx, ""),
159  UnorderedElementsAre(AllOf(Named("common"), NumReferences(5U)),
160  AllOf(Named("A_CC"), NumReferences(0U)),
161  AllOf(Named("g"), NumReferences(0U)),
162  AllOf(Named("f_b"), Declared(), Defined(),
163  NumReferences(1U))));
164 
165  auto Syms = runFuzzyFind(Idx, "common");
166  EXPECT_THAT(Syms, UnorderedElementsAre(Named("common")));
167  auto Common = *Syms.begin();
168  EXPECT_THAT(getRefs(Idx, Common.ID),
169  RefsAre({FileURI("unittest:///root/A.h"),
170  FileURI("unittest:///root/A.cc"),
171  FileURI("unittest:///root/B.cc"),
172  FileURI("unittest:///root/B.cc"),
173  FileURI("unittest:///root/B.cc"),
174  FileURI("unittest:///root/B.cc")}));
175 }
176 
177 TEST_F(BackgroundIndexTest, ShardStorageTest) {
179  FS.Files[testPath("root/A.h")] = R"cpp(
180  void common();
181  void f_b();
182  class A_CC {};
183  )cpp";
184  std::string A_CC = "";
185  FS.Files[testPath("root/A.cc")] = R"cpp(
186  #include "A.h"
187  void g() { (void)common; }
188  class B_CC : public A_CC {};
189  )cpp";
190 
191  llvm::StringMap<std::string> Storage;
192  size_t CacheHits = 0;
193  MemoryShardStorage MSS(Storage, CacheHits);
194 
195  tooling::CompileCommand Cmd;
196  Cmd.Filename = testPath("root/A.cc");
197  Cmd.Directory = testPath("root");
198  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
199  // Check nothing is loaded from Storage, but A.cc and A.h has been stored.
200  {
201  OverlayCDB CDB(/*Base=*/nullptr);
202  BackgroundIndex Idx(Context::empty(), FS, CDB,
203  [&](llvm::StringRef) { return &MSS; });
204  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
205  ASSERT_TRUE(Idx.blockUntilIdleForTest());
206  }
207  EXPECT_EQ(CacheHits, 0U);
208  EXPECT_EQ(Storage.size(), 2U);
209 
210  {
211  OverlayCDB CDB(/*Base=*/nullptr);
212  BackgroundIndex Idx(Context::empty(), FS, CDB,
213  [&](llvm::StringRef) { return &MSS; });
214  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
215  ASSERT_TRUE(Idx.blockUntilIdleForTest());
216  }
217  EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
218  EXPECT_EQ(Storage.size(), 2U);
219 
220  auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
221  EXPECT_NE(ShardHeader, nullptr);
222  EXPECT_THAT(
223  *ShardHeader->Symbols,
224  UnorderedElementsAre(Named("common"), Named("A_CC"),
225  AllOf(Named("f_b"), Declared(), Not(Defined()))));
226  for (const auto &Ref : *ShardHeader->Refs)
227  EXPECT_THAT(Ref.second,
228  UnorderedElementsAre(FileURI("unittest:///root/A.h")));
229 
230  auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
231  EXPECT_NE(ShardSource, nullptr);
232  EXPECT_THAT(*ShardSource->Symbols,
233  UnorderedElementsAre(Named("g"), Named("B_CC")));
234  for (const auto &Ref : *ShardSource->Refs)
235  EXPECT_THAT(Ref.second,
236  UnorderedElementsAre(FileURI("unittest:///root/A.cc")));
237 
238  // The BaseOf relationship between A_CC and B_CC is stored in the file
239  // containing the definition of the subject (A_CC)
240  SymbolID A = findSymbol(*ShardHeader->Symbols, "A_CC").ID;
241  SymbolID B = findSymbol(*ShardSource->Symbols, "B_CC").ID;
242  EXPECT_THAT(*ShardHeader->Relations,
243  UnorderedElementsAre(Relation{A, RelationKind::BaseOf, B}));
244  // (and not in the file containing the definition of the object (B_CC)).
245  EXPECT_EQ(ShardSource->Relations->size(), 0u);
246 }
247 
248 TEST_F(BackgroundIndexTest, DirectIncludesTest) {
250  FS.Files[testPath("root/B.h")] = "";
251  FS.Files[testPath("root/A.h")] = R"cpp(
252  #include "B.h"
253  void common();
254  void f_b();
255  class A_CC {};
256  )cpp";
257  std::string A_CC = "#include \"A.h\"\nvoid g() { (void)common; }";
258  FS.Files[testPath("root/A.cc")] = A_CC;
259 
260  llvm::StringMap<std::string> Storage;
261  size_t CacheHits = 0;
262  MemoryShardStorage MSS(Storage, CacheHits);
263 
264  tooling::CompileCommand Cmd;
265  Cmd.Filename = testPath("root/A.cc");
266  Cmd.Directory = testPath("root");
267  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
268  {
269  OverlayCDB CDB(/*Base=*/nullptr);
270  BackgroundIndex Idx(Context::empty(), FS, CDB,
271  [&](llvm::StringRef) { return &MSS; });
272  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
273  ASSERT_TRUE(Idx.blockUntilIdleForTest());
274  }
275 
276  auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
277  EXPECT_TRUE(ShardSource->Sources);
278  EXPECT_EQ(ShardSource->Sources->size(), 2U); // A.cc, A.h
279  EXPECT_THAT(
280  ShardSource->Sources->lookup("unittest:///root/A.cc").DirectIncludes,
281  UnorderedElementsAre("unittest:///root/A.h"));
282  EXPECT_NE(ShardSource->Sources->lookup("unittest:///root/A.cc").Digest,
283  FileDigest{{0}});
284  EXPECT_THAT(ShardSource->Sources->lookup("unittest:///root/A.h"),
285  EmptyIncludeNode());
286 
287  auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
288  EXPECT_TRUE(ShardHeader->Sources);
289  EXPECT_EQ(ShardHeader->Sources->size(), 2U); // A.h, B.h
290  EXPECT_THAT(
291  ShardHeader->Sources->lookup("unittest:///root/A.h").DirectIncludes,
292  UnorderedElementsAre("unittest:///root/B.h"));
293  EXPECT_NE(ShardHeader->Sources->lookup("unittest:///root/A.h").Digest,
294  FileDigest{{0}});
295  EXPECT_THAT(ShardHeader->Sources->lookup("unittest:///root/B.h"),
296  EmptyIncludeNode());
297 }
298 
299 TEST_F(BackgroundIndexTest, ShardStorageLoad) {
301  FS.Files[testPath("root/A.h")] = R"cpp(
302  void common();
303  void f_b();
304  class A_CC {};
305  )cpp";
306  FS.Files[testPath("root/A.cc")] =
307  "#include \"A.h\"\nvoid g() { (void)common; }";
308 
309  llvm::StringMap<std::string> Storage;
310  size_t CacheHits = 0;
311  MemoryShardStorage MSS(Storage, CacheHits);
312 
313  tooling::CompileCommand Cmd;
314  Cmd.Filename = testPath("root/A.cc");
315  Cmd.Directory = testPath("root");
316  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
317  // Check nothing is loaded from Storage, but A.cc and A.h has been stored.
318  {
319  OverlayCDB CDB(/*Base=*/nullptr);
320  BackgroundIndex Idx(Context::empty(), FS, CDB,
321  [&](llvm::StringRef) { return &MSS; });
322  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
323  ASSERT_TRUE(Idx.blockUntilIdleForTest());
324  }
325 
326  // Change header.
327  FS.Files[testPath("root/A.h")] = R"cpp(
328  void common();
329  void f_b();
330  class A_CC {};
331  class A_CCnew {};
332  )cpp";
333  {
334  OverlayCDB CDB(/*Base=*/nullptr);
335  BackgroundIndex Idx(Context::empty(), FS, CDB,
336  [&](llvm::StringRef) { return &MSS; });
337  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
338  ASSERT_TRUE(Idx.blockUntilIdleForTest());
339  }
340  EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
341 
342  // Check if the new symbol has arrived.
343  auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
344  EXPECT_NE(ShardHeader, nullptr);
345  EXPECT_THAT(*ShardHeader->Symbols, Contains(Named("A_CCnew")));
346 
347  // Change source.
348  FS.Files[testPath("root/A.cc")] =
349  "#include \"A.h\"\nvoid g() { (void)common; }\nvoid f_b() {}";
350  {
351  CacheHits = 0;
352  OverlayCDB CDB(/*Base=*/nullptr);
353  BackgroundIndex Idx(Context::empty(), FS, CDB,
354  [&](llvm::StringRef) { return &MSS; });
355  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
356  ASSERT_TRUE(Idx.blockUntilIdleForTest());
357  }
358  EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
359 
360  // Check if the new symbol has arrived.
361  ShardHeader = MSS.loadShard(testPath("root/A.h"));
362  EXPECT_NE(ShardHeader, nullptr);
363  EXPECT_THAT(*ShardHeader->Symbols, Contains(Named("A_CCnew")));
364  auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
365  EXPECT_NE(ShardSource, nullptr);
366  EXPECT_THAT(*ShardSource->Symbols,
367  Contains(AllOf(Named("f_b"), Declared(), Defined())));
368 }
369 
370 TEST_F(BackgroundIndexTest, ShardStorageEmptyFile) {
372  FS.Files[testPath("root/A.h")] = R"cpp(
373  void common();
374  void f_b();
375  class A_CC {};
376  )cpp";
377  FS.Files[testPath("root/B.h")] = R"cpp(
378  #include "A.h"
379  )cpp";
380  FS.Files[testPath("root/A.cc")] =
381  "#include \"B.h\"\nvoid g() { (void)common; }";
382 
383  llvm::StringMap<std::string> Storage;
384  size_t CacheHits = 0;
385  MemoryShardStorage MSS(Storage, CacheHits);
386 
387  tooling::CompileCommand Cmd;
388  Cmd.Filename = testPath("root/A.cc");
389  Cmd.Directory = testPath("root");
390  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
391  // Check that A.cc, A.h and B.h has been stored.
392  {
393  OverlayCDB CDB(/*Base=*/nullptr);
394  BackgroundIndex Idx(Context::empty(), FS, CDB,
395  [&](llvm::StringRef) { return &MSS; });
396  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
397  ASSERT_TRUE(Idx.blockUntilIdleForTest());
398  }
399  EXPECT_THAT(Storage.keys(),
400  UnorderedElementsAre(testPath("root/A.cc"), testPath("root/A.h"),
401  testPath("root/B.h")));
402  auto ShardHeader = MSS.loadShard(testPath("root/B.h"));
403  EXPECT_NE(ShardHeader, nullptr);
404  EXPECT_TRUE(ShardHeader->Symbols->empty());
405 
406  // Check that A.cc, A.h and B.h has been loaded.
407  {
408  CacheHits = 0;
409  OverlayCDB CDB(/*Base=*/nullptr);
410  BackgroundIndex Idx(Context::empty(), FS, CDB,
411  [&](llvm::StringRef) { return &MSS; });
412  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
413  ASSERT_TRUE(Idx.blockUntilIdleForTest());
414  }
415  EXPECT_EQ(CacheHits, 3U);
416 
417  // Update B.h to contain some symbols.
418  FS.Files[testPath("root/B.h")] = R"cpp(
419  #include "A.h"
420  void new_func();
421  )cpp";
422  // Check that B.h has been stored with new contents.
423  {
424  CacheHits = 0;
425  OverlayCDB CDB(/*Base=*/nullptr);
426  BackgroundIndex Idx(Context::empty(), FS, CDB,
427  [&](llvm::StringRef) { return &MSS; });
428  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
429  ASSERT_TRUE(Idx.blockUntilIdleForTest());
430  }
431  EXPECT_EQ(CacheHits, 3U);
432  ShardHeader = MSS.loadShard(testPath("root/B.h"));
433  EXPECT_NE(ShardHeader, nullptr);
434  EXPECT_THAT(*ShardHeader->Symbols,
435  Contains(AllOf(Named("new_func"), Declared(), Not(Defined()))));
436 }
437 
438 TEST_F(BackgroundIndexTest, NoDotsInAbsPath) {
440  llvm::StringMap<std::string> Storage;
441  size_t CacheHits = 0;
442  MemoryShardStorage MSS(Storage, CacheHits);
443  OverlayCDB CDB(/*Base=*/nullptr);
444  BackgroundIndex Idx(Context::empty(), FS, CDB,
445  [&](llvm::StringRef) { return &MSS; });
446 
447  tooling::CompileCommand Cmd;
448  FS.Files[testPath("root/A.cc")] = "";
449  Cmd.Filename = "../A.cc";
450  Cmd.Directory = testPath("root/build");
451  Cmd.CommandLine = {"clang++", "../A.cc"};
452  CDB.setCompileCommand(testPath("root/build/../A.cc"), Cmd);
453 
454  FS.Files[testPath("root/B.cc")] = "";
455  Cmd.Filename = "./B.cc";
456  Cmd.Directory = testPath("root");
457  Cmd.CommandLine = {"clang++", "./B.cc"};
458  CDB.setCompileCommand(testPath("root/./B.cc"), Cmd);
459 
460  ASSERT_TRUE(Idx.blockUntilIdleForTest());
461  for (llvm::StringRef AbsPath : MSS.AccessedPaths.keys()) {
462  EXPECT_FALSE(AbsPath.contains("./")) << AbsPath;
463  EXPECT_FALSE(AbsPath.contains("../")) << AbsPath;
464  }
465 }
466 
467 TEST_F(BackgroundIndexTest, UncompilableFiles) {
469  llvm::StringMap<std::string> Storage;
470  size_t CacheHits = 0;
471  MemoryShardStorage MSS(Storage, CacheHits);
472  OverlayCDB CDB(/*Base=*/nullptr);
473  BackgroundIndex Idx(Context::empty(), FS, CDB,
474  [&](llvm::StringRef) { return &MSS; });
475 
476  tooling::CompileCommand Cmd;
477  FS.Files[testPath("A.h")] = "void foo();";
478  FS.Files[testPath("B.h")] = "#include \"C.h\"\nasdf;";
479  FS.Files[testPath("C.h")] = "";
480  FS.Files[testPath("A.cc")] = R"cpp(
481  #include "A.h"
482  #include "B.h"
483  #include "not_found_header.h"
484 
485  void foo() {}
486  )cpp";
487  Cmd.Filename = "../A.cc";
488  Cmd.Directory = testPath("build");
489  Cmd.CommandLine = {"clang++", "../A.cc"};
490  CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
491  ASSERT_TRUE(Idx.blockUntilIdleForTest());
492 
493  EXPECT_THAT(Storage.keys(), ElementsAre(testPath("A.cc"), testPath("A.h"),
494  testPath("B.h"), testPath("C.h")));
495 
496  {
497  auto Shard = MSS.loadShard(testPath("A.cc"));
498  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("foo")));
499  EXPECT_THAT(Shard->Sources->keys(),
500  UnorderedElementsAre("unittest:///A.cc", "unittest:///A.h",
501  "unittest:///B.h"));
502  EXPECT_THAT(Shard->Sources->lookup("unittest:///A.cc"), HadErrors());
503  }
504 
505  {
506  auto Shard = MSS.loadShard(testPath("A.h"));
507  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("foo")));
508  EXPECT_THAT(Shard->Sources->keys(),
509  UnorderedElementsAre("unittest:///A.h"));
510  EXPECT_THAT(Shard->Sources->lookup("unittest:///A.h"), HadErrors());
511  }
512 
513  {
514  auto Shard = MSS.loadShard(testPath("B.h"));
515  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("asdf")));
516  EXPECT_THAT(Shard->Sources->keys(),
517  UnorderedElementsAre("unittest:///B.h", "unittest:///C.h"));
518  EXPECT_THAT(Shard->Sources->lookup("unittest:///B.h"), HadErrors());
519  }
520 
521  {
522  auto Shard = MSS.loadShard(testPath("C.h"));
523  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre());
524  EXPECT_THAT(Shard->Sources->keys(),
525  UnorderedElementsAre("unittest:///C.h"));
526  EXPECT_THAT(Shard->Sources->lookup("unittest:///C.h"), HadErrors());
527  }
528 }
529 
532  llvm::StringMap<std::string> Storage;
533  size_t CacheHits = 0;
534  MemoryShardStorage MSS(Storage, CacheHits);
535  OverlayCDB CDB(/*Base=*/nullptr, /*FallbackFlags=*/{},
536  /*ResourceDir=*/std::string(""));
537  BackgroundIndex Idx(Context::empty(), FS, CDB,
538  [&](llvm::StringRef) { return &MSS; });
539 
540  tooling::CompileCommand Cmd;
541  FS.Files[testPath("A.cc")] = "#include \"A.h\"";
542  FS.Files[testPath("A.h")] = "";
543  Cmd.Filename = "../A.cc";
544  Cmd.Directory = testPath("build");
545  Cmd.CommandLine = {"clang++", "../A.cc", "-fsyntax-only"};
546  CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
547  ASSERT_TRUE(Idx.blockUntilIdleForTest());
548 
549  EXPECT_THAT(Storage.keys(), ElementsAre(testPath("A.cc"), testPath("A.h")));
550  // Make sure we only store the Cmd for main file.
551  EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd);
552 
553  {
554  tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd;
555  EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine);
556  EXPECT_EQ(CmdStored.Directory, Cmd.Directory);
557  }
558 
559  // FIXME: Changing compile commands should be enough to invalidate the cache.
560  FS.Files[testPath("A.cc")] = " ";
561  Cmd.CommandLine = {"clang++", "../A.cc", "-Dfoo", "-fsyntax-only"};
562  CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
563  ASSERT_TRUE(Idx.blockUntilIdleForTest());
564 
565  EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd);
566 
567  {
568  tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd;
569  EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine);
570  EXPECT_EQ(CmdStored.Directory, Cmd.Directory);
571  }
572 }
573 
574 class BackgroundIndexRebuilderTest : public testing::Test {
575 protected:
577  : Target(std::make_unique<MemIndex>()),
578  Rebuilder(&Target, &Source, /*Threads=*/10) {
579  // Prepare FileSymbols with TestSymbol in it, for checkRebuild.
580  TestSymbol.ID = SymbolID("foo");
581  }
582 
583  // Perform Action and determine whether it rebuilt the index or not.
584  bool checkRebuild(std::function<void()> Action) {
585  // Update name so we can tell if the index updates.
586  VersionStorage.push_back("Sym" + std::to_string(++VersionCounter));
587  TestSymbol.Name = VersionStorage.back();
589  SB.insert(TestSymbol);
590  Source.update("", std::make_unique<SymbolSlab>(std::move(SB).build()),
591  nullptr, nullptr, false);
592  // Now maybe update the index.
593  Action();
594  // Now query the index to get the name count.
595  std::string ReadName;
596  LookupRequest Req;
597  Req.IDs.insert(TestSymbol.ID);
598  Target.lookup(Req, [&](const Symbol &S) { ReadName = S.Name; });
599  // The index was rebuild if the name is up to date.
600  return ReadName == VersionStorage.back();
601  }
602 
607 
608  unsigned VersionCounter = 0;
609  std::deque<std::string> VersionStorage;
610 };
611 
613  for (unsigned I = 0; I < Rebuilder.TUsBeforeFirstBuild - 1; ++I)
614  EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
615  EXPECT_TRUE(checkRebuild([&] { Rebuilder.indexedTU(); }));
616  for (unsigned I = 0; I < Rebuilder.TUsBeforeRebuild - 1; ++I)
617  EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
618  EXPECT_TRUE(checkRebuild([&] { Rebuilder.indexedTU(); }));
619 }
620 
622  Rebuilder.startLoading();
623  Rebuilder.loadedShard(10);
624  Rebuilder.loadedShard(20);
625  EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
626 
627  // No rebuild for no shards.
628  Rebuilder.startLoading();
629  EXPECT_FALSE(checkRebuild([&] { Rebuilder.doneLoading(); }));
630 
631  // Loads can overlap.
632  Rebuilder.startLoading();
633  Rebuilder.loadedShard(1);
634  Rebuilder.startLoading();
635  Rebuilder.loadedShard(1);
636  EXPECT_FALSE(checkRebuild([&] { Rebuilder.doneLoading(); }));
637  Rebuilder.loadedShard(1);
638  EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
639 
640  // No rebuilding for indexed files while loading.
641  Rebuilder.startLoading();
642  for (unsigned I = 0; I < 3 * Rebuilder.TUsBeforeRebuild; ++I)
643  EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
644  // But they get indexed when we're done, even if no shards were loaded.
645  EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
646 }
647 
648 TEST(BackgroundQueueTest, Priority) {
649  // Create high and low priority tasks.
650  // Once a bunch of high priority tasks have run, the queue is stopped.
651  // So the low priority tasks should never run.
652  BackgroundQueue Q;
653  std::atomic<unsigned> HiRan(0), LoRan(0);
654  BackgroundQueue::Task Lo([&] { ++LoRan; });
655  BackgroundQueue::Task Hi([&] {
656  if (++HiRan >= 10)
657  Q.stop();
658  });
659  Hi.QueuePri = 100;
660 
661  // Enqueuing the low-priority ones first shouldn't make them run first.
662  Q.append(std::vector<BackgroundQueue::Task>(30, Lo));
663  for (unsigned I = 0; I < 30; ++I)
664  Q.push(Hi);
665 
666  AsyncTaskRunner ThreadPool;
667  for (unsigned I = 0; I < 5; ++I)
668  ThreadPool.runAsync("worker", [&] { Q.work(); });
669  // We should test enqueue with active workers, but it's hard to avoid races.
670  // Just make sure we don't crash.
671  Q.push(Lo);
672  Q.append(std::vector<BackgroundQueue::Task>(2, Hi));
673 
674  // After finishing, check the tasks that ran.
675  ThreadPool.wait();
676  EXPECT_GE(HiRan, 10u);
677  EXPECT_EQ(LoRan, 0u);
678 }
679 
680 TEST(BackgroundQueueTest, Boost) {
681  std::string Sequence;
682 
683  BackgroundQueue::Task A([&] { Sequence.push_back('A'); });
684  A.Tag = "A";
685  A.QueuePri = 1;
686 
687  BackgroundQueue::Task B([&] { Sequence.push_back('B'); });
688  B.QueuePri = 2;
689  B.Tag = "B";
690 
691  {
692  BackgroundQueue Q;
693  Q.append({A, B});
694  Q.work([&] { Q.stop(); });
695  EXPECT_EQ("BA", Sequence) << "priority order";
696  }
697  Sequence.clear();
698  {
699  BackgroundQueue Q;
700  Q.boost("A", 3);
701  Q.append({A, B});
702  Q.work([&] { Q.stop(); });
703  EXPECT_EQ("AB", Sequence) << "A was boosted before enqueueing";
704  }
705  Sequence.clear();
706  {
707  BackgroundQueue Q;
708  Q.append({A, B});
709  Q.boost("A", 3);
710  Q.work([&] { Q.stop(); });
711  EXPECT_EQ("AB", Sequence) << "A was boosted after enqueueing";
712  }
713 }
714 
715 } // namespace clangd
716 } // namespace clang
::testing::Matcher< const RefSlab & > RefsAre(std::vector<::testing::Matcher< Ref >> Matchers)
MATCHER_P(Named, N, "")
Represents a relation between two symbols.
Definition: Relation.h:29
llvm::StringMap< std::string > Files
Definition: TestFS.h:38
std::array< uint8_t, 8 > FileDigest
Definition: SourceCode.h:38
A container of Symbols from several source files.
Definition: FileIndex.h:60
llvm::Error storeShard(llvm::StringRef ShardIdentifier, IndexFileOut Shard) const override
llvm::DenseSet< SymbolID > IDs
Definition: Index.h:64
bool checkRebuild(std::function< void()> Action)
Represents a symbol occurrence in the source file.
Definition: Ref.h:52
void insert(const Symbol &S)
Adds a symbol, overwriting any existing one with the same ID.
Definition: Symbol.cpp:50
MockFSProvider FS
llvm::Expected< IndexFileIn > readIndexFile(llvm::StringRef Data)
SymbolSlab::Builder is a mutable container that can &#39;freeze&#39; to SymbolSlab.
Definition: Symbol.h:199
SymbolID ID
The ID of the symbol.
Definition: Symbol.h:38
void boost(llvm::StringRef Tag, unsigned NewPriority)
TEST_F(BackgroundIndexTest, NoCrashOnErrorFile)
static void preventThreadStarvationInTests()
void runAsync(const llvm::Twine &Name, llvm::unique_function< void()> Action)
Definition: Threading.cpp:70
TEST(BackgroundQueueTest, Priority)
std::string testPath(PathRef File)
Definition: TestFS.cpp:82
llvm::unique_function< void()> Action
MemIndex is a naive in-memory index suitable for a small set of symbols.
Definition: MemIndex.h:19
SymbolSlab runFuzzyFind(const SymbolIndex &Index, llvm::StringRef Query)
Definition: SyncAPI.cpp:127
MATCHER(Declared, "")
Runs tasks on separate (detached) threads and wait for all tasks to finish.
Definition: Threading.h:105
void setCompileCommand(PathRef File, llvm::Optional< tooling::CompileCommand > CompilationCommand)
Sets or clears the compilation command for a particular file.
The class presents a C++ symbol, e.g.
Definition: Symbol.h:36
MemoryShardStorage(llvm::StringMap< std::string > &Storage, size_t &CacheHits)
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
llvm::StringRef Name
The unqualified name of the symbol, e.g. "bar" (for ns::bar).
Definition: Symbol.h:42
const Symbol & findSymbol(const SymbolSlab &Slab, llvm::StringRef QName)
Definition: TestTU.cpp:97
std::unique_ptr< IndexFileIn > loadShard(llvm::StringRef ShardIdentifier) const override
void append(std::vector< Task >)
A work item on the thread pool&#39;s queue.
Definition: Background.h:70
Wraps another compilation database, and supports overriding the commands using an in-memory mapping...
static Context empty()
Returns an empty root context that contains no data.
Definition: Context.cpp:15
std::array< uint8_t, 20 > SymbolID
RefSlab getRefs(const SymbolIndex &Index, SymbolID ID)
Definition: SyncAPI.cpp:140
void work(std::function< void()> OnIdle=nullptr)