clang-tools  11.0.0git
BackgroundIndexTests.cpp
Go to the documentation of this file.
1 #include "CompileCommands.h"
2 #include "Config.h"
3 #include "Headers.h"
4 #include "SyncAPI.h"
5 #include "TestFS.h"
6 #include "TestIndex.h"
7 #include "TestTU.h"
8 #include "index/Background.h"
10 #include "clang/Tooling/ArgumentsAdjusters.h"
11 #include "clang/Tooling/CompilationDatabase.h"
12 #include "llvm/Support/ScopedPrinter.h"
13 #include "llvm/Support/Threading.h"
14 #include "gmock/gmock.h"
15 #include "gtest/gtest.h"
16 #include <deque>
17 #include <thread>
18 
19 using ::testing::_;
20 using ::testing::AllOf;
21 using ::testing::Contains;
22 using ::testing::ElementsAre;
23 using ::testing::Not;
24 using ::testing::UnorderedElementsAre;
25 
26 namespace clang {
27 namespace clangd {
28 
29 MATCHER_P(Named, N, "") { return arg.Name == N; }
30 MATCHER_P(QName, N, "") { return (arg.Scope + arg.Name).str() == N; }
31 MATCHER(Declared, "") {
32  return !StringRef(arg.CanonicalDeclaration.FileURI).empty();
33 }
34 MATCHER(Defined, "") { return !StringRef(arg.Definition.FileURI).empty(); }
35 MATCHER_P(FileURI, F, "") { return StringRef(arg.Location.FileURI) == F; }
36 ::testing::Matcher<const RefSlab &>
37 RefsAre(std::vector<::testing::Matcher<Ref>> Matchers) {
38  return ElementsAre(::testing::Pair(_, UnorderedElementsAreArray(Matchers)));
39 }
40 // URI cannot be empty since it references keys in the IncludeGraph.
41 MATCHER(EmptyIncludeNode, "") {
42  return arg.Flags == IncludeGraphNode::SourceFlag::None && !arg.URI.empty() &&
43  arg.Digest == FileDigest{{0}} && arg.DirectIncludes.empty();
44 }
45 
46 MATCHER(HadErrors, "") {
47  return arg.Flags & IncludeGraphNode::SourceFlag::HadErrors;
48 }
49 
50 MATCHER_P(NumReferences, N, "") { return arg.References == N; }
51 
53  mutable std::mutex StorageMu;
54  llvm::StringMap<std::string> &Storage;
55  size_t &CacheHits;
56 
57 public:
58  MemoryShardStorage(llvm::StringMap<std::string> &Storage, size_t &CacheHits)
59  : Storage(Storage), CacheHits(CacheHits) {}
60  llvm::Error storeShard(llvm::StringRef ShardIdentifier,
61  IndexFileOut Shard) const override {
62  std::lock_guard<std::mutex> Lock(StorageMu);
63  AccessedPaths.insert(ShardIdentifier);
64  Storage[ShardIdentifier] = llvm::to_string(Shard);
65  return llvm::Error::success();
66  }
67  std::unique_ptr<IndexFileIn>
68  loadShard(llvm::StringRef ShardIdentifier) const override {
69  std::lock_guard<std::mutex> Lock(StorageMu);
70  AccessedPaths.insert(ShardIdentifier);
71  if (Storage.find(ShardIdentifier) == Storage.end()) {
72  return nullptr;
73  }
74  auto IndexFile = readIndexFile(Storage[ShardIdentifier]);
75  if (!IndexFile) {
76  ADD_FAILURE() << "Error while reading " << ShardIdentifier << ':'
77  << IndexFile.takeError();
78  return nullptr;
79  }
80  CacheHits++;
81  return std::make_unique<IndexFileIn>(std::move(*IndexFile));
82  }
83 
84  mutable llvm::StringSet<> AccessedPaths;
85 };
86 
87 class BackgroundIndexTest : public ::testing::Test {
88 protected:
90 };
91 
92 TEST_F(BackgroundIndexTest, NoCrashOnErrorFile) {
93  MockFS FS;
94  FS.Files[testPath("root/A.cc")] = "error file";
95  llvm::StringMap<std::string> Storage;
96  size_t CacheHits = 0;
97  MemoryShardStorage MSS(Storage, CacheHits);
98  OverlayCDB CDB(/*Base=*/nullptr);
99  BackgroundIndex Idx(Context::empty(), FS, CDB,
100  [&](llvm::StringRef) { return &MSS; });
101 
102  tooling::CompileCommand Cmd;
103  Cmd.Filename = testPath("root/A.cc");
104  Cmd.Directory = testPath("root");
105  Cmd.CommandLine = {"clang++", "-DA=1", testPath("root/A.cc")};
106  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
107 
108  ASSERT_TRUE(Idx.blockUntilIdleForTest());
109 }
110 
112  MockFS FS;
113  // Set up two identical TUs, foo and bar.
114  // They define foo::one and bar::one.
115  std::vector<tooling::CompileCommand> Cmds;
116  for (std::string Name : {"foo", "bar"}) {
117  std::string Filename = Name + ".cpp";
118  std::string Header = Name + ".h";
119  FS.Files[Filename] = "#include \"" + Header + "\"";
120  FS.Files[Header] = "namespace " + Name + " { int one; }";
121  tooling::CompileCommand Cmd;
122  Cmd.Filename = Filename;
123  Cmd.Directory = testRoot();
124  Cmd.CommandLine = {"clang++", Filename};
125  Cmds.push_back(std::move(Cmd));
126  }
127  // Context provider that installs a configuration mutating foo's command.
128  // This causes it to define foo::two instead of foo::one.
129  auto ContextProvider = [](PathRef P) {
130  Config C;
131  if (P.endswith("foo.cpp"))
132  C.CompileFlags.Edits.push_back(
133  [](std::vector<std::string> &Argv) { Argv.push_back("-Done=two"); });
134  return Context::current().derive(Config::Key, std::move(C));
135  };
136  // Create the background index.
137  llvm::StringMap<std::string> Storage;
138  size_t CacheHits = 0;
139  MemoryShardStorage MSS(Storage, CacheHits);
140  // We need the CommandMangler, because that applies the config we're testing.
141  OverlayCDB CDB(/*Base=*/nullptr, /*FallbackFlags=*/{},
142  tooling::ArgumentsAdjuster(CommandMangler::forTests()));
143  BackgroundIndex Idx(
144  Context::empty(), FS, CDB, [&](llvm::StringRef) { return &MSS; },
145  /*ThreadPoolSize=*/4, /*OnProgress=*/nullptr, std::move(ContextProvider));
146  // Index the two files.
147  for (auto &Cmd : Cmds) {
148  std::string FullPath = testPath(Cmd.Filename);
149  CDB.setCompileCommand(FullPath, std::move(Cmd));
150  }
151  // Wait for both files to be indexed.
152  ASSERT_TRUE(Idx.blockUntilIdleForTest());
153  EXPECT_THAT(runFuzzyFind(Idx, ""),
154  UnorderedElementsAre(QName("foo"), QName("foo::two"),
155  QName("bar"), QName("bar::one")));
156 }
157 
158 TEST_F(BackgroundIndexTest, IndexTwoFiles) {
159  MockFS FS;
160  // a.h yields different symbols when included by A.cc vs B.cc.
161  FS.Files[testPath("root/A.h")] = R"cpp(
162  void common();
163  void f_b();
164  #if A
165  class A_CC {};
166  #else
167  class B_CC{};
168  #endif
169  )cpp";
170  FS.Files[testPath("root/A.cc")] =
171  "#include \"A.h\"\nvoid g() { (void)common; }";
172  FS.Files[testPath("root/B.cc")] =
173  R"cpp(
174  #define A 0
175  #include "A.h"
176  void f_b() {
177  (void)common;
178  (void)common;
179  (void)common;
180  (void)common;
181  })cpp";
182  llvm::StringMap<std::string> Storage;
183  size_t CacheHits = 0;
184  MemoryShardStorage MSS(Storage, CacheHits);
185  OverlayCDB CDB(/*Base=*/nullptr);
186  BackgroundIndex Idx(Context::empty(), FS, CDB,
187  [&](llvm::StringRef) { return &MSS; });
188 
189  tooling::CompileCommand Cmd;
190  Cmd.Filename = testPath("root/A.cc");
191  Cmd.Directory = testPath("root");
192  Cmd.CommandLine = {"clang++", "-DA=1", testPath("root/A.cc")};
193  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
194 
195  ASSERT_TRUE(Idx.blockUntilIdleForTest());
196  EXPECT_THAT(runFuzzyFind(Idx, ""),
197  UnorderedElementsAre(AllOf(Named("common"), NumReferences(1U)),
198  AllOf(Named("A_CC"), NumReferences(0U)),
199  AllOf(Named("g"), NumReferences(0U)),
200  AllOf(Named("f_b"), Declared(),
201  Not(Defined()), NumReferences(0U))));
202 
203  Cmd.Filename = testPath("root/B.cc");
204  Cmd.CommandLine = {"clang++", Cmd.Filename};
205  CDB.setCompileCommand(testPath("root/B.cc"), Cmd);
206 
207  ASSERT_TRUE(Idx.blockUntilIdleForTest());
208  // B_CC is dropped as we don't collect symbols from A.h in this compilation.
209  EXPECT_THAT(runFuzzyFind(Idx, ""),
210  UnorderedElementsAre(AllOf(Named("common"), NumReferences(5U)),
211  AllOf(Named("A_CC"), NumReferences(0U)),
212  AllOf(Named("g"), NumReferences(0U)),
213  AllOf(Named("f_b"), Declared(), Defined(),
214  NumReferences(1U))));
215 
216  auto Syms = runFuzzyFind(Idx, "common");
217  EXPECT_THAT(Syms, UnorderedElementsAre(Named("common")));
218  auto Common = *Syms.begin();
219  EXPECT_THAT(getRefs(Idx, Common.ID),
220  RefsAre({FileURI("unittest:///root/A.h"),
221  FileURI("unittest:///root/A.cc"),
222  FileURI("unittest:///root/B.cc"),
223  FileURI("unittest:///root/B.cc"),
224  FileURI("unittest:///root/B.cc"),
225  FileURI("unittest:///root/B.cc")}));
226 }
227 
228 TEST_F(BackgroundIndexTest, ShardStorageTest) {
229  MockFS FS;
230  FS.Files[testPath("root/A.h")] = R"cpp(
231  void common();
232  void f_b();
233  class A_CC {};
234  )cpp";
235  std::string A_CC = "";
236  FS.Files[testPath("root/A.cc")] = R"cpp(
237  #include "A.h"
238  void g() { (void)common; }
239  class B_CC : public A_CC {};
240  )cpp";
241 
242  llvm::StringMap<std::string> Storage;
243  size_t CacheHits = 0;
244  MemoryShardStorage MSS(Storage, CacheHits);
245 
246  tooling::CompileCommand Cmd;
247  Cmd.Filename = testPath("root/A.cc");
248  Cmd.Directory = testPath("root");
249  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
250  // Check nothing is loaded from Storage, but A.cc and A.h has been stored.
251  {
252  OverlayCDB CDB(/*Base=*/nullptr);
253  BackgroundIndex Idx(Context::empty(), FS, CDB,
254  [&](llvm::StringRef) { return &MSS; });
255  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
256  ASSERT_TRUE(Idx.blockUntilIdleForTest());
257  }
258  EXPECT_EQ(CacheHits, 0U);
259  EXPECT_EQ(Storage.size(), 2U);
260 
261  {
262  OverlayCDB CDB(/*Base=*/nullptr);
263  BackgroundIndex Idx(Context::empty(), FS, CDB,
264  [&](llvm::StringRef) { return &MSS; });
265  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
266  ASSERT_TRUE(Idx.blockUntilIdleForTest());
267  }
268  EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
269  EXPECT_EQ(Storage.size(), 2U);
270 
271  auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
272  EXPECT_NE(ShardHeader, nullptr);
273  EXPECT_THAT(
274  *ShardHeader->Symbols,
275  UnorderedElementsAre(Named("common"), Named("A_CC"),
276  AllOf(Named("f_b"), Declared(), Not(Defined()))));
277  for (const auto &Ref : *ShardHeader->Refs)
278  EXPECT_THAT(Ref.second,
279  UnorderedElementsAre(FileURI("unittest:///root/A.h")));
280 
281  auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
282  EXPECT_NE(ShardSource, nullptr);
283  EXPECT_THAT(*ShardSource->Symbols,
284  UnorderedElementsAre(Named("g"), Named("B_CC")));
285  for (const auto &Ref : *ShardSource->Refs)
286  EXPECT_THAT(Ref.second,
287  UnorderedElementsAre(FileURI("unittest:///root/A.cc")));
288 
289  // The BaseOf relationship between A_CC and B_CC is stored in the file
290  // containing the definition of the subject (A_CC)
291  SymbolID A = findSymbol(*ShardHeader->Symbols, "A_CC").ID;
292  SymbolID B = findSymbol(*ShardSource->Symbols, "B_CC").ID;
293  EXPECT_THAT(*ShardHeader->Relations,
294  UnorderedElementsAre(Relation{A, RelationKind::BaseOf, B}));
295  // (and not in the file containing the definition of the object (B_CC)).
296  EXPECT_EQ(ShardSource->Relations->size(), 0u);
297 }
298 
299 TEST_F(BackgroundIndexTest, DirectIncludesTest) {
300  MockFS FS;
301  FS.Files[testPath("root/B.h")] = "";
302  FS.Files[testPath("root/A.h")] = R"cpp(
303  #include "B.h"
304  void common();
305  void f_b();
306  class A_CC {};
307  )cpp";
308  std::string A_CC = "#include \"A.h\"\nvoid g() { (void)common; }";
309  FS.Files[testPath("root/A.cc")] = A_CC;
310 
311  llvm::StringMap<std::string> Storage;
312  size_t CacheHits = 0;
313  MemoryShardStorage MSS(Storage, CacheHits);
314 
315  tooling::CompileCommand Cmd;
316  Cmd.Filename = testPath("root/A.cc");
317  Cmd.Directory = testPath("root");
318  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
319  {
320  OverlayCDB CDB(/*Base=*/nullptr);
321  BackgroundIndex Idx(Context::empty(), FS, CDB,
322  [&](llvm::StringRef) { return &MSS; });
323  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
324  ASSERT_TRUE(Idx.blockUntilIdleForTest());
325  }
326 
327  auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
328  EXPECT_TRUE(ShardSource->Sources);
329  EXPECT_EQ(ShardSource->Sources->size(), 2U); // A.cc, A.h
330  EXPECT_THAT(
331  ShardSource->Sources->lookup("unittest:///root/A.cc").DirectIncludes,
332  UnorderedElementsAre("unittest:///root/A.h"));
333  EXPECT_NE(ShardSource->Sources->lookup("unittest:///root/A.cc").Digest,
334  FileDigest{{0}});
335  EXPECT_THAT(ShardSource->Sources->lookup("unittest:///root/A.h"),
336  EmptyIncludeNode());
337 
338  auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
339  EXPECT_TRUE(ShardHeader->Sources);
340  EXPECT_EQ(ShardHeader->Sources->size(), 2U); // A.h, B.h
341  EXPECT_THAT(
342  ShardHeader->Sources->lookup("unittest:///root/A.h").DirectIncludes,
343  UnorderedElementsAre("unittest:///root/B.h"));
344  EXPECT_NE(ShardHeader->Sources->lookup("unittest:///root/A.h").Digest,
345  FileDigest{{0}});
346  EXPECT_THAT(ShardHeader->Sources->lookup("unittest:///root/B.h"),
347  EmptyIncludeNode());
348 }
349 
350 TEST_F(BackgroundIndexTest, ShardStorageLoad) {
351  MockFS FS;
352  FS.Files[testPath("root/A.h")] = R"cpp(
353  void common();
354  void f_b();
355  class A_CC {};
356  )cpp";
357  FS.Files[testPath("root/A.cc")] =
358  "#include \"A.h\"\nvoid g() { (void)common; }";
359 
360  llvm::StringMap<std::string> Storage;
361  size_t CacheHits = 0;
362  MemoryShardStorage MSS(Storage, CacheHits);
363 
364  tooling::CompileCommand Cmd;
365  Cmd.Filename = testPath("root/A.cc");
366  Cmd.Directory = testPath("root");
367  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
368  // Check nothing is loaded from Storage, but A.cc and A.h has been stored.
369  {
370  OverlayCDB CDB(/*Base=*/nullptr);
371  BackgroundIndex Idx(Context::empty(), FS, CDB,
372  [&](llvm::StringRef) { return &MSS; });
373  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
374  ASSERT_TRUE(Idx.blockUntilIdleForTest());
375  }
376 
377  // Change header.
378  FS.Files[testPath("root/A.h")] = R"cpp(
379  void common();
380  void f_b();
381  class A_CC {};
382  class A_CCnew {};
383  )cpp";
384  {
385  OverlayCDB CDB(/*Base=*/nullptr);
386  BackgroundIndex Idx(Context::empty(), FS, CDB,
387  [&](llvm::StringRef) { return &MSS; });
388  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
389  ASSERT_TRUE(Idx.blockUntilIdleForTest());
390  }
391  EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
392 
393  // Check if the new symbol has arrived.
394  auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
395  EXPECT_NE(ShardHeader, nullptr);
396  EXPECT_THAT(*ShardHeader->Symbols, Contains(Named("A_CCnew")));
397 
398  // Change source.
399  FS.Files[testPath("root/A.cc")] =
400  "#include \"A.h\"\nvoid g() { (void)common; }\nvoid f_b() {}";
401  {
402  CacheHits = 0;
403  OverlayCDB CDB(/*Base=*/nullptr);
404  BackgroundIndex Idx(Context::empty(), FS, CDB,
405  [&](llvm::StringRef) { return &MSS; });
406  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
407  ASSERT_TRUE(Idx.blockUntilIdleForTest());
408  }
409  EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
410 
411  // Check if the new symbol has arrived.
412  ShardHeader = MSS.loadShard(testPath("root/A.h"));
413  EXPECT_NE(ShardHeader, nullptr);
414  EXPECT_THAT(*ShardHeader->Symbols, Contains(Named("A_CCnew")));
415  auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
416  EXPECT_NE(ShardSource, nullptr);
417  EXPECT_THAT(*ShardSource->Symbols,
418  Contains(AllOf(Named("f_b"), Declared(), Defined())));
419 }
420 
421 TEST_F(BackgroundIndexTest, ShardStorageEmptyFile) {
422  MockFS FS;
423  FS.Files[testPath("root/A.h")] = R"cpp(
424  void common();
425  void f_b();
426  class A_CC {};
427  )cpp";
428  FS.Files[testPath("root/B.h")] = R"cpp(
429  #include "A.h"
430  )cpp";
431  FS.Files[testPath("root/A.cc")] =
432  "#include \"B.h\"\nvoid g() { (void)common; }";
433 
434  llvm::StringMap<std::string> Storage;
435  size_t CacheHits = 0;
436  MemoryShardStorage MSS(Storage, CacheHits);
437 
438  tooling::CompileCommand Cmd;
439  Cmd.Filename = testPath("root/A.cc");
440  Cmd.Directory = testPath("root");
441  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
442  // Check that A.cc, A.h and B.h has been stored.
443  {
444  OverlayCDB CDB(/*Base=*/nullptr);
445  BackgroundIndex Idx(Context::empty(), FS, CDB,
446  [&](llvm::StringRef) { return &MSS; });
447  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
448  ASSERT_TRUE(Idx.blockUntilIdleForTest());
449  }
450  EXPECT_THAT(Storage.keys(),
451  UnorderedElementsAre(testPath("root/A.cc"), testPath("root/A.h"),
452  testPath("root/B.h")));
453  auto ShardHeader = MSS.loadShard(testPath("root/B.h"));
454  EXPECT_NE(ShardHeader, nullptr);
455  EXPECT_TRUE(ShardHeader->Symbols->empty());
456 
457  // Check that A.cc, A.h and B.h has been loaded.
458  {
459  CacheHits = 0;
460  OverlayCDB CDB(/*Base=*/nullptr);
461  BackgroundIndex Idx(Context::empty(), FS, CDB,
462  [&](llvm::StringRef) { return &MSS; });
463  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
464  ASSERT_TRUE(Idx.blockUntilIdleForTest());
465  }
466  EXPECT_EQ(CacheHits, 3U);
467 
468  // Update B.h to contain some symbols.
469  FS.Files[testPath("root/B.h")] = R"cpp(
470  #include "A.h"
471  void new_func();
472  )cpp";
473  // Check that B.h has been stored with new contents.
474  {
475  CacheHits = 0;
476  OverlayCDB CDB(/*Base=*/nullptr);
477  BackgroundIndex Idx(Context::empty(), FS, CDB,
478  [&](llvm::StringRef) { return &MSS; });
479  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
480  ASSERT_TRUE(Idx.blockUntilIdleForTest());
481  }
482  EXPECT_EQ(CacheHits, 3U);
483  ShardHeader = MSS.loadShard(testPath("root/B.h"));
484  EXPECT_NE(ShardHeader, nullptr);
485  EXPECT_THAT(*ShardHeader->Symbols,
486  Contains(AllOf(Named("new_func"), Declared(), Not(Defined()))));
487 }
488 
489 TEST_F(BackgroundIndexTest, NoDotsInAbsPath) {
490  MockFS FS;
491  llvm::StringMap<std::string> Storage;
492  size_t CacheHits = 0;
493  MemoryShardStorage MSS(Storage, CacheHits);
494  OverlayCDB CDB(/*Base=*/nullptr);
495  BackgroundIndex Idx(Context::empty(), FS, CDB,
496  [&](llvm::StringRef) { return &MSS; });
497  ASSERT_TRUE(Idx.blockUntilIdleForTest());
498 
499  tooling::CompileCommand Cmd;
500  FS.Files[testPath("root/A.cc")] = "";
501  Cmd.Filename = "../A.cc";
502  Cmd.Directory = testPath("root/build");
503  Cmd.CommandLine = {"clang++", "../A.cc"};
504  CDB.setCompileCommand(testPath("root/build/../A.cc"), Cmd);
505  ASSERT_TRUE(Idx.blockUntilIdleForTest());
506 
507  FS.Files[testPath("root/B.cc")] = "";
508  Cmd.Filename = "./B.cc";
509  Cmd.Directory = testPath("root");
510  Cmd.CommandLine = {"clang++", "./B.cc"};
511  CDB.setCompileCommand(testPath("root/./B.cc"), Cmd);
512  ASSERT_TRUE(Idx.blockUntilIdleForTest());
513 
514  for (llvm::StringRef AbsPath : MSS.AccessedPaths.keys()) {
515  EXPECT_FALSE(AbsPath.contains("./")) << AbsPath;
516  EXPECT_FALSE(AbsPath.contains("../")) << AbsPath;
517  }
518 }
519 
520 TEST_F(BackgroundIndexTest, UncompilableFiles) {
521  MockFS FS;
522  llvm::StringMap<std::string> Storage;
523  size_t CacheHits = 0;
524  MemoryShardStorage MSS(Storage, CacheHits);
525  OverlayCDB CDB(/*Base=*/nullptr);
526  BackgroundIndex Idx(Context::empty(), FS, CDB,
527  [&](llvm::StringRef) { return &MSS; });
528 
529  tooling::CompileCommand Cmd;
530  FS.Files[testPath("A.h")] = "void foo();";
531  FS.Files[testPath("B.h")] = "#include \"C.h\"\nasdf;";
532  FS.Files[testPath("C.h")] = "";
533  FS.Files[testPath("A.cc")] = R"cpp(
534  #include "A.h"
535  #include "B.h"
536  #include "not_found_header.h"
537 
538  void foo() {}
539  )cpp";
540  Cmd.Filename = "../A.cc";
541  Cmd.Directory = testPath("build");
542  Cmd.CommandLine = {"clang++", "../A.cc"};
543  CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
544  ASSERT_TRUE(Idx.blockUntilIdleForTest());
545 
546  EXPECT_THAT(Storage.keys(), ElementsAre(testPath("A.cc"), testPath("A.h"),
547  testPath("B.h"), testPath("C.h")));
548 
549  {
550  auto Shard = MSS.loadShard(testPath("A.cc"));
551  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("foo")));
552  EXPECT_THAT(Shard->Sources->keys(),
553  UnorderedElementsAre("unittest:///A.cc", "unittest:///A.h",
554  "unittest:///B.h"));
555  EXPECT_THAT(Shard->Sources->lookup("unittest:///A.cc"), HadErrors());
556  }
557 
558  {
559  auto Shard = MSS.loadShard(testPath("A.h"));
560  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("foo")));
561  EXPECT_THAT(Shard->Sources->keys(),
562  UnorderedElementsAre("unittest:///A.h"));
563  EXPECT_THAT(Shard->Sources->lookup("unittest:///A.h"), HadErrors());
564  }
565 
566  {
567  auto Shard = MSS.loadShard(testPath("B.h"));
568  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("asdf")));
569  EXPECT_THAT(Shard->Sources->keys(),
570  UnorderedElementsAre("unittest:///B.h", "unittest:///C.h"));
571  EXPECT_THAT(Shard->Sources->lookup("unittest:///B.h"), HadErrors());
572  }
573 
574  {
575  auto Shard = MSS.loadShard(testPath("C.h"));
576  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre());
577  EXPECT_THAT(Shard->Sources->keys(),
578  UnorderedElementsAre("unittest:///C.h"));
579  EXPECT_THAT(Shard->Sources->lookup("unittest:///C.h"), HadErrors());
580  }
581 }
582 
584  MockFS FS;
585  llvm::StringMap<std::string> Storage;
586  size_t CacheHits = 0;
587  MemoryShardStorage MSS(Storage, CacheHits);
588  OverlayCDB CDB(/*Base=*/nullptr);
589  BackgroundIndex Idx(Context::empty(), FS, CDB,
590  [&](llvm::StringRef) { return &MSS; });
591 
592  tooling::CompileCommand Cmd;
593  FS.Files[testPath("A.cc")] = "#include \"A.h\"";
594  FS.Files[testPath("A.h")] = "";
595  Cmd.Filename = "../A.cc";
596  Cmd.Directory = testPath("build");
597  Cmd.CommandLine = {"clang++", "../A.cc", "-fsyntax-only"};
598  CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
599  ASSERT_TRUE(Idx.blockUntilIdleForTest());
600 
601  EXPECT_THAT(Storage.keys(), ElementsAre(testPath("A.cc"), testPath("A.h")));
602  // Make sure we only store the Cmd for main file.
603  EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd);
604 
605  {
606  tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd;
607  EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine);
608  EXPECT_EQ(CmdStored.Directory, Cmd.Directory);
609  }
610 
611  // FIXME: Changing compile commands should be enough to invalidate the cache.
612  FS.Files[testPath("A.cc")] = " ";
613  Cmd.CommandLine = {"clang++", "../A.cc", "-Dfoo", "-fsyntax-only"};
614  CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
615  ASSERT_TRUE(Idx.blockUntilIdleForTest());
616 
617  EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd);
618 
619  {
620  tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd;
621  EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine);
622  EXPECT_EQ(CmdStored.Directory, Cmd.Directory);
623  }
624 }
625 
626 class BackgroundIndexRebuilderTest : public testing::Test {
627 protected:
629  : Target(std::make_unique<MemIndex>()),
630  Rebuilder(&Target, &Source, /*Threads=*/10) {
631  // Prepare FileSymbols with TestSymbol in it, for checkRebuild.
632  TestSymbol.ID = SymbolID("foo");
633  }
634 
635  // Perform Action and determine whether it rebuilt the index or not.
636  bool checkRebuild(std::function<void()> Action) {
637  // Update name so we can tell if the index updates.
638  VersionStorage.push_back("Sym" + std::to_string(++VersionCounter));
639  TestSymbol.Name = VersionStorage.back();
641  SB.insert(TestSymbol);
642  Source.update("", std::make_unique<SymbolSlab>(std::move(SB).build()),
643  nullptr, nullptr, false);
644  // Now maybe update the index.
645  Action();
646  // Now query the index to get the name count.
647  std::string ReadName;
648  LookupRequest Req;
649  Req.IDs.insert(TestSymbol.ID);
650  Target.lookup(Req,
651  [&](const Symbol &S) { ReadName = std::string(S.Name); });
652  // The index was rebuild if the name is up to date.
653  return ReadName == VersionStorage.back();
654  }
655 
660 
661  unsigned VersionCounter = 0;
662  std::deque<std::string> VersionStorage;
663 };
664 
666  for (unsigned I = 0; I < Rebuilder.TUsBeforeFirstBuild - 1; ++I)
667  EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
668  EXPECT_TRUE(checkRebuild([&] { Rebuilder.indexedTU(); }));
669  for (unsigned I = 0; I < Rebuilder.TUsBeforeRebuild - 1; ++I)
670  EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
671  EXPECT_TRUE(checkRebuild([&] { Rebuilder.indexedTU(); }));
672 }
673 
675  Rebuilder.startLoading();
676  Rebuilder.loadedShard(10);
677  Rebuilder.loadedShard(20);
678  EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
679 
680  // No rebuild for no shards.
681  Rebuilder.startLoading();
682  EXPECT_FALSE(checkRebuild([&] { Rebuilder.doneLoading(); }));
683 
684  // Loads can overlap.
685  Rebuilder.startLoading();
686  Rebuilder.loadedShard(1);
687  Rebuilder.startLoading();
688  Rebuilder.loadedShard(1);
689  EXPECT_FALSE(checkRebuild([&] { Rebuilder.doneLoading(); }));
690  Rebuilder.loadedShard(1);
691  EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
692 
693  // No rebuilding for indexed files while loading.
694  Rebuilder.startLoading();
695  for (unsigned I = 0; I < 3 * Rebuilder.TUsBeforeRebuild; ++I)
696  EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
697  // But they get indexed when we're done, even if no shards were loaded.
698  EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
699 }
700 
701 TEST(BackgroundQueueTest, Priority) {
702  // Create high and low priority tasks.
703  // Once a bunch of high priority tasks have run, the queue is stopped.
704  // So the low priority tasks should never run.
705  BackgroundQueue Q;
706  std::atomic<unsigned> HiRan(0), LoRan(0);
707  BackgroundQueue::Task Lo([&] { ++LoRan; });
708  BackgroundQueue::Task Hi([&] {
709  if (++HiRan >= 10)
710  Q.stop();
711  });
712  Hi.QueuePri = 100;
713 
714  // Enqueuing the low-priority ones first shouldn't make them run first.
715  Q.append(std::vector<BackgroundQueue::Task>(30, Lo));
716  for (unsigned I = 0; I < 30; ++I)
717  Q.push(Hi);
718 
719  AsyncTaskRunner ThreadPool;
720  for (unsigned I = 0; I < 5; ++I)
721  ThreadPool.runAsync("worker", [&] { Q.work(); });
722  // We should test enqueue with active workers, but it's hard to avoid races.
723  // Just make sure we don't crash.
724  Q.push(Lo);
725  Q.append(std::vector<BackgroundQueue::Task>(2, Hi));
726 
727  // After finishing, check the tasks that ran.
728  ThreadPool.wait();
729  EXPECT_GE(HiRan, 10u);
730  EXPECT_EQ(LoRan, 0u);
731 }
732 
733 TEST(BackgroundQueueTest, Boost) {
734  std::string Sequence;
735 
736  BackgroundQueue::Task A([&] { Sequence.push_back('A'); });
737  A.Tag = "A";
738  A.QueuePri = 1;
739 
740  BackgroundQueue::Task B([&] { Sequence.push_back('B'); });
741  B.QueuePri = 2;
742  B.Tag = "B";
743 
744  {
745  BackgroundQueue Q;
746  Q.append({A, B});
747  Q.work([&] { Q.stop(); });
748  EXPECT_EQ("BA", Sequence) << "priority order";
749  }
750  Sequence.clear();
751  {
752  BackgroundQueue Q;
753  Q.boost("A", 3);
754  Q.append({A, B});
755  Q.work([&] { Q.stop(); });
756  EXPECT_EQ("AB", Sequence) << "A was boosted before enqueueing";
757  }
758  Sequence.clear();
759  {
760  BackgroundQueue Q;
761  Q.append({A, B});
762  Q.boost("A", 3);
763  Q.work([&] { Q.stop(); });
764  EXPECT_EQ("AB", Sequence) << "A was boosted after enqueueing";
765  }
766 }
767 
768 TEST(BackgroundQueueTest, Progress) {
769  using testing::AnyOf;
772  // Verify values are sane.
773  // Items are enqueued one at a time (at least in this test).
774  EXPECT_THAT(New.Enqueued, AnyOf(S.Enqueued, S.Enqueued + 1));
775  // Items are completed one at a time.
776  EXPECT_THAT(New.Completed, AnyOf(S.Completed, S.Completed + 1));
777  // Items are started or completed one at a time.
778  EXPECT_THAT(New.Active, AnyOf(S.Active - 1, S.Active, S.Active + 1));
779  // Idle point only advances in time.
780  EXPECT_GE(New.LastIdle, S.LastIdle);
781  // Idle point is a task that has been completed in the past.
782  EXPECT_LE(New.LastIdle, New.Completed);
783  // LastIdle is now only if we're really idle.
784  EXPECT_EQ(New.LastIdle == New.Enqueued,
785  New.Completed == New.Enqueued && New.Active == 0u);
786  S = New;
787  });
788 
789  // Two types of tasks: a ping task enqueues a pong task.
790  // This avoids all enqueues followed by all completions (boring!)
791  std::atomic<int> PingCount(0), PongCount(0);
792  BackgroundQueue::Task Pong([&] { ++PongCount; });
793  BackgroundQueue::Task Ping([&] {
794  ++PingCount;
795  Q.push(Pong);
796  });
797 
798  for (int I = 0; I < 1000; ++I)
799  Q.push(Ping);
800  // Spin up some workers and stop while idle.
801  AsyncTaskRunner ThreadPool;
802  for (unsigned I = 0; I < 5; ++I)
803  ThreadPool.runAsync("worker", [&] { Q.work([&] { Q.stop(); }); });
804  ThreadPool.wait();
805 
806  // Everything's done, check final stats.
807  // Assertions above ensure we got from 0 to 2000 in a reasonable way.
808  EXPECT_EQ(PingCount.load(), 1000);
809  EXPECT_EQ(PongCount.load(), 1000);
810  EXPECT_EQ(S.Active, 0u);
811  EXPECT_EQ(S.Enqueued, 2000u);
812  EXPECT_EQ(S.Completed, 2000u);
813  EXPECT_EQ(S.LastIdle, 2000u);
814 }
815 
816 } // namespace clangd
817 } // namespace clang
::testing::Matcher< const RefSlab & > RefsAre(std::vector<::testing::Matcher< Ref >> Matchers)
MATCHER_P(Named, N, "")
Represents a relation between two symbols.
Definition: Relation.h:29
std::array< uint8_t, 8 > FileDigest
Definition: SourceCode.h:40
A container of slabs associated with a key.
Definition: FileIndex.h:71
MockFS FS
llvm::Error storeShard(llvm::StringRef ShardIdentifier, IndexFileOut Shard) const override
llvm::DenseSet< SymbolID > IDs
Definition: Index.h:64
bool checkRebuild(std::function< void()> Action)
Represents a symbol occurrence in the source file.
Definition: Ref.h:87
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:23
void insert(const Symbol &S)
Adds a symbol, overwriting any existing one with the same ID.
Definition: Symbol.cpp:50
llvm::Expected< IndexFileIn > readIndexFile(llvm::StringRef Data)
static CommandMangler forTests()
SymbolSlab::Builder is a mutable container that can &#39;freeze&#39; to SymbolSlab.
Definition: Symbol.h:200
SymbolID ID
The ID of the symbol.
Definition: Symbol.h:38
void boost(llvm::StringRef Tag, unsigned NewPriority)
TEST_F(BackgroundIndexTest, NoCrashOnErrorFile)
std::string QName
std::string Filename
Filename as a string.
static void preventThreadStarvationInTests()
void runAsync(const llvm::Twine &Name, llvm::unique_function< void()> Action)
Definition: Threading.cpp:72
TEST(BackgroundQueueTest, Priority)
llvm::unique_function< void()> Action
std::string testPath(PathRef File, llvm::sys::path::Style Style)
Definition: TestFS.cpp:82
MemIndex is a naive in-memory index suitable for a small set of symbols.
Definition: MemIndex.h:19
static const Context & current()
Returns the context for the current thread, creating it if needed.
Definition: Context.cpp:27
SymbolSlab runFuzzyFind(const SymbolIndex &Index, llvm::StringRef Query)
Definition: SyncAPI.cpp:121
MATCHER(Declared, "")
static constexpr llvm::StringLiteral Name
const char * testRoot()
Definition: TestFS.cpp:74
static clangd::Key< Config > Key
Context key which can be used to set the current Config.
Definition: Config.h:44
Runs tasks on separate (detached) threads and wait for all tasks to finish.
Definition: Threading.h:105
void setCompileCommand(PathRef File, llvm::Optional< tooling::CompileCommand > CompilationCommand)
Sets or clears the compilation command for a particular file.
Settings that express user/project preferences and control clangd behavior.
Definition: Config.h:40
The class presents a C++ symbol, e.g.
Definition: Symbol.h:36
llvm::StringMap< std::string > Files
Definition: TestFS.h:41
MemoryShardStorage(llvm::StringMap< std::string > &Storage, size_t &CacheHits)
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
llvm::StringRef Name
The unqualified name of the symbol, e.g. "bar" (for ns::bar).
Definition: Symbol.h:42
const Symbol & findSymbol(const SymbolSlab &Slab, llvm::StringRef QName)
Definition: TestTU.cpp:144
Context derive(const Key< Type > &Key, typename std::decay< Type >::type Value) const &
Derives a child context It is safe to move or destroy a parent context after calling derive()...
Definition: Context.h:121
std::unique_ptr< IndexFileIn > loadShard(llvm::StringRef ShardIdentifier) const override
void append(std::vector< Task >)
std::vector< llvm::unique_function< void(std::vector< std::string > &)> > Edits
Definition: Config.h:56
A work item on the thread pool&#39;s queue.
Definition: Background.h:70
Wraps another compilation database, and supports overriding the commands using an in-memory mapping...
static Context empty()
Returns an empty root context that contains no data.
Definition: Context.cpp:15
std::array< uint8_t, 20 > SymbolID
struct clang::clangd::Config::@0 CompileFlags
Controls how the compile command for the current file is determined.
RefSlab getRefs(const SymbolIndex &Index, SymbolID ID)
Definition: SyncAPI.cpp:134
void work(std::function< void()> OnIdle=nullptr)