clang-tools  15.0.0git
TUSchedulerTests.cpp
Go to the documentation of this file.
1 //===-- TUSchedulerTests.cpp ------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "Annotations.h"
10 #include "ClangdServer.h"
11 #include "Diagnostics.h"
13 #include "Matchers.h"
14 #include "ParsedAST.h"
15 #include "Preamble.h"
16 #include "TUScheduler.h"
17 #include "TestFS.h"
18 #include "TestIndex.h"
19 #include "support/Cancellation.h"
20 #include "support/Context.h"
21 #include "support/Path.h"
22 #include "support/TestTracer.h"
23 #include "support/Threading.h"
24 #include "support/ThreadsafeFS.h"
25 #include "clang/Basic/DiagnosticDriver.h"
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/FunctionExtras.h"
28 #include "llvm/ADT/ScopeExit.h"
29 #include "llvm/ADT/StringExtras.h"
30 #include "llvm/ADT/StringMap.h"
31 #include "llvm/ADT/StringRef.h"
32 #include "gmock/gmock.h"
33 #include "gtest/gtest.h"
34 #include <algorithm>
35 #include <atomic>
36 #include <chrono>
37 #include <cstdint>
38 #include <memory>
39 #include <string>
40 #include <utility>
41 
42 namespace clang {
43 namespace clangd {
44 namespace {
45 
46 using ::testing::AllOf;
47 using ::testing::AnyOf;
48 using ::testing::Contains;
49 using ::testing::Each;
50 using ::testing::ElementsAre;
51 using ::testing::Eq;
53 using ::testing::IsEmpty;
54 using ::testing::Not;
55 using ::testing::Pair;
56 using ::testing::Pointee;
57 using ::testing::SizeIs;
58 using ::testing::UnorderedElementsAre;
59 
60 MATCHER_P2(TUState, PreambleActivity, ASTActivity, "") {
61  if (arg.PreambleActivity != PreambleActivity) {
62  *result_listener << "preamblestate is "
63  << static_cast<uint8_t>(arg.PreambleActivity);
64  return false;
65  }
66  if (arg.ASTActivity.K != ASTActivity) {
67  *result_listener << "aststate is " << arg.ASTActivity.K;
68  return false;
69  }
70  return true;
71 }
72 
73 // Simple ContextProvider to verify the provider is invoked & contexts are used.
74 static Key<std::string> BoundPath;
75 Context bindPath(PathRef F) {
76  return Context::current().derive(BoundPath, F.str());
77 }
78 llvm::StringRef boundPath() {
79  const std::string *V = Context::current().get(BoundPath);
80  return V ? *V : llvm::StringRef("");
81 }
82 
83 TUScheduler::Options optsForTest() {
84  TUScheduler::Options Opts(ClangdServer::optsForTest());
85  Opts.ContextProvider = bindPath;
86  return Opts;
87 }
88 
89 class TUSchedulerTests : public ::testing::Test {
90 protected:
91  ParseInputs getInputs(PathRef File, std::string Contents) {
92  ParseInputs Inputs;
93  Inputs.CompileCommand = *CDB.getCompileCommand(File);
94  Inputs.TFS = &FS;
95  Inputs.Contents = std::move(Contents);
96  Inputs.Opts = ParseOptions();
97  return Inputs;
98  }
99 
100  void updateWithCallback(TUScheduler &S, PathRef File,
101  llvm::StringRef Contents, WantDiagnostics WD,
102  llvm::unique_function<void()> CB) {
103  updateWithCallback(S, File, getInputs(File, std::string(Contents)), WD,
104  std::move(CB));
105  }
106 
107  void updateWithCallback(TUScheduler &S, PathRef File, ParseInputs Inputs,
108  WantDiagnostics WD,
109  llvm::unique_function<void()> CB) {
110  WithContextValue Ctx(llvm::make_scope_exit(std::move(CB)));
111  S.update(File, Inputs, WD);
112  }
113 
114  static Key<llvm::unique_function<void(PathRef File, std::vector<Diag>)>>
116 
117  /// A diagnostics callback that should be passed to TUScheduler when it's used
118  /// in updateWithDiags.
119  static std::unique_ptr<ParsingCallbacks> captureDiags() {
120  class CaptureDiags : public ParsingCallbacks {
121  public:
122  void onMainAST(PathRef File, ParsedAST &AST, PublishFn Publish) override {
123  reportDiagnostics(File, *AST.getDiagnostics(), Publish);
124  }
125 
126  void onFailedAST(PathRef File, llvm::StringRef Version,
127  std::vector<Diag> Diags, PublishFn Publish) override {
128  reportDiagnostics(File, Diags, Publish);
129  }
130 
131  private:
132  void reportDiagnostics(PathRef File, llvm::ArrayRef<Diag> Diags,
133  PublishFn Publish) {
135  if (!D)
136  return;
137  Publish([&]() {
138  const_cast<
139  llvm::unique_function<void(PathRef, std::vector<Diag>)> &> (*D)(
140  File, std::move(Diags));
141  });
142  }
143  };
144  return std::make_unique<CaptureDiags>();
145  }
146 
147  /// Schedule an update and call \p CB with the diagnostics it produces, if
148  /// any. The TUScheduler should be created with captureDiags as a
149  /// DiagsCallback for this to work.
150  void updateWithDiags(TUScheduler &S, PathRef File, ParseInputs Inputs,
151  WantDiagnostics WD,
152  llvm::unique_function<void(std::vector<Diag>)> CB) {
153  Path OrigFile = File.str();
154  WithContextValue Ctx(DiagsCallbackKey,
155  [OrigFile, CB = std::move(CB)](
156  PathRef File, std::vector<Diag> Diags) mutable {
157  assert(File == OrigFile);
158  CB(std::move(Diags));
159  });
160  S.update(File, std::move(Inputs), WD);
161  }
162 
163  void updateWithDiags(TUScheduler &S, PathRef File, llvm::StringRef Contents,
164  WantDiagnostics WD,
165  llvm::unique_function<void(std::vector<Diag>)> CB) {
166  return updateWithDiags(S, File, getInputs(File, std::string(Contents)), WD,
167  std::move(CB));
168  }
169 
170  MockFS FS;
171  MockCompilationDatabase CDB;
172 };
173 
174 Key<llvm::unique_function<void(PathRef File, std::vector<Diag>)>>
176 
177 TEST_F(TUSchedulerTests, MissingFiles) {
178  TUScheduler S(CDB, optsForTest());
179 
180  auto Added = testPath("added.cpp");
181  FS.Files[Added] = "x";
182 
183  auto Missing = testPath("missing.cpp");
184  FS.Files[Missing] = "";
185 
186  S.update(Added, getInputs(Added, "x"), WantDiagnostics::No);
187 
188  // Assert each operation for missing file is an error (even if it's
189  // available in VFS).
190  S.runWithAST("", Missing,
191  [&](Expected<InputsAndAST> AST) { EXPECT_ERROR(AST); });
192  S.runWithPreamble(
193  "", Missing, TUScheduler::Stale,
194  [&](Expected<InputsAndPreamble> Preamble) { EXPECT_ERROR(Preamble); });
195  // remove() shouldn't crash on missing files.
196  S.remove(Missing);
197 
198  // Assert there aren't any errors for added file.
199  S.runWithAST("", Added,
200  [&](Expected<InputsAndAST> AST) { EXPECT_TRUE(bool(AST)); });
201  S.runWithPreamble("", Added, TUScheduler::Stale,
202  [&](Expected<InputsAndPreamble> Preamble) {
203  EXPECT_TRUE(bool(Preamble));
204  });
205  S.remove(Added);
206 
207  // Assert that all operations fail after removing the file.
208  S.runWithAST("", Added,
209  [&](Expected<InputsAndAST> AST) { EXPECT_ERROR(AST); });
210  S.runWithPreamble("", Added, TUScheduler::Stale,
211  [&](Expected<InputsAndPreamble> Preamble) {
212  ASSERT_FALSE(bool(Preamble));
213  llvm::consumeError(Preamble.takeError());
214  });
215  // remove() shouldn't crash on missing files.
216  S.remove(Added);
217 }
218 
219 TEST_F(TUSchedulerTests, WantDiagnostics) {
220  std::atomic<int> CallbackCount(0);
221  {
222  // To avoid a racy test, don't allow tasks to actually run on the worker
223  // thread until we've scheduled them all.
224  Notification Ready;
225  TUScheduler S(CDB, optsForTest(), captureDiags());
226  auto Path = testPath("foo.cpp");
227  updateWithDiags(S, Path, "", WantDiagnostics::Yes,
228  [&](std::vector<Diag>) { Ready.wait(); });
229  updateWithDiags(S, Path, "request diags", WantDiagnostics::Yes,
230  [&](std::vector<Diag>) { ++CallbackCount; });
231  updateWithDiags(S, Path, "auto (clobbered)", WantDiagnostics::Auto,
232  [&](std::vector<Diag>) {
233  ADD_FAILURE()
234  << "auto should have been cancelled by auto";
235  });
236  updateWithDiags(S, Path, "request no diags", WantDiagnostics::No,
237  [&](std::vector<Diag>) {
238  ADD_FAILURE() << "no diags should not be called back";
239  });
240  updateWithDiags(S, Path, "auto (produces)", WantDiagnostics::Auto,
241  [&](std::vector<Diag>) { ++CallbackCount; });
242  Ready.notify();
243 
244  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
245  }
246  EXPECT_EQ(2, CallbackCount);
247 }
248 
249 TEST_F(TUSchedulerTests, Debounce) {
250  auto Opts = optsForTest();
251  Opts.UpdateDebounce = DebouncePolicy::fixed(std::chrono::milliseconds(500));
252  TUScheduler S(CDB, Opts, captureDiags());
253  auto Path = testPath("foo.cpp");
254  // Issue a write that's going to be debounced away.
255  updateWithDiags(S, Path, "auto (debounced)", WantDiagnostics::Auto,
256  [&](std::vector<Diag>) {
257  ADD_FAILURE()
258  << "auto should have been debounced and canceled";
259  });
260  // Sleep a bit to verify that it's really debounce that's holding diagnostics.
261  std::this_thread::sleep_for(std::chrono::milliseconds(50));
262 
263  // Issue another write, this time we'll wait for its diagnostics.
264  Notification N;
265  updateWithDiags(S, Path, "auto (timed out)", WantDiagnostics::Auto,
266  [&](std::vector<Diag>) { N.notify(); });
267  EXPECT_TRUE(N.wait(timeoutSeconds(1)));
268 
269  // Once we start shutting down the TUScheduler, this one becomes a dead write.
270  updateWithDiags(S, Path, "auto (discarded)", WantDiagnostics::Auto,
271  [&](std::vector<Diag>) {
272  ADD_FAILURE()
273  << "auto should have been discarded (dead write)";
274  });
275 }
276 
277 TEST_F(TUSchedulerTests, Cancellation) {
278  // We have the following update/read sequence
279  // U0
280  // U1(WantDiags=Yes) <-- cancelled
281  // R1 <-- cancelled
282  // U2(WantDiags=Yes) <-- cancelled
283  // R2A <-- cancelled
284  // R2B
285  // U3(WantDiags=Yes)
286  // R3 <-- cancelled
287  std::vector<StringRef> DiagsSeen, ReadsSeen, ReadsCanceled;
288  {
289  Notification Proceed; // Ensure we schedule everything.
290  TUScheduler S(CDB, optsForTest(), captureDiags());
291  auto Path = testPath("foo.cpp");
292  // Helper to schedule a named update and return a function to cancel it.
293  auto Update = [&](StringRef ID) -> Canceler {
294  auto T = cancelableTask();
295  WithContext C(std::move(T.first));
296  updateWithDiags(
297  S, Path, ("//" + ID).str(), WantDiagnostics::Yes,
298  [&, ID](std::vector<Diag> Diags) { DiagsSeen.push_back(ID); });
299  return std::move(T.second);
300  };
301  // Helper to schedule a named read and return a function to cancel it.
302  auto Read = [&](StringRef ID) -> Canceler {
303  auto T = cancelableTask();
304  WithContext C(std::move(T.first));
305  S.runWithAST(ID, Path, [&, ID](llvm::Expected<InputsAndAST> E) {
306  if (auto Err = E.takeError()) {
307  if (Err.isA<CancelledError>()) {
308  ReadsCanceled.push_back(ID);
309  consumeError(std::move(Err));
310  } else {
311  ADD_FAILURE() << "Non-cancelled error for " << ID << ": "
312  << llvm::toString(std::move(Err));
313  }
314  } else {
315  ReadsSeen.push_back(ID);
316  }
317  });
318  return std::move(T.second);
319  };
320 
321  updateWithCallback(S, Path, "", WantDiagnostics::Yes,
322  [&]() { Proceed.wait(); });
323  // The second parens indicate cancellation, where present.
324  Update("U1")();
325  Read("R1")();
326  Update("U2")();
327  Read("R2A")();
328  Read("R2B");
329  Update("U3");
330  Read("R3")();
331  Proceed.notify();
332 
333  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
334  }
335  EXPECT_THAT(DiagsSeen, ElementsAre("U2", "U3"))
336  << "U1 and all dependent reads were cancelled. "
337  "U2 has a dependent read R2A. "
338  "U3 was not cancelled.";
339  EXPECT_THAT(ReadsSeen, ElementsAre("R2B"))
340  << "All reads other than R2B were cancelled";
341  EXPECT_THAT(ReadsCanceled, ElementsAre("R1", "R2A", "R3"))
342  << "All reads other than R2B were cancelled";
343 }
344 
345 TEST_F(TUSchedulerTests, InvalidationNoCrash) {
346  auto Path = testPath("foo.cpp");
347  TUScheduler S(CDB, optsForTest(), captureDiags());
348 
349  Notification StartedRunning;
350  Notification ScheduledChange;
351  // We expect invalidation logic to not crash by trying to invalidate a running
352  // request.
353  S.update(Path, getInputs(Path, ""), WantDiagnostics::Auto);
354  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
355  S.runWithAST(
356  "invalidatable-but-running", Path,
357  [&](llvm::Expected<InputsAndAST> AST) {
358  StartedRunning.notify();
359  ScheduledChange.wait();
360  ASSERT_TRUE(bool(AST));
361  },
363  StartedRunning.wait();
364  S.update(Path, getInputs(Path, ""), WantDiagnostics::Auto);
365  ScheduledChange.notify();
366  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
367 }
368 
369 TEST_F(TUSchedulerTests, Invalidation) {
370  auto Path = testPath("foo.cpp");
371  TUScheduler S(CDB, optsForTest(), captureDiags());
372  std::atomic<int> Builds(0), Actions(0);
373 
374  Notification Start;
375  updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {
376  ++Builds;
377  Start.wait();
378  });
379  S.runWithAST(
380  "invalidatable", Path,
381  [&](llvm::Expected<InputsAndAST> AST) {
382  ++Actions;
383  EXPECT_FALSE(bool(AST));
384  llvm::Error E = AST.takeError();
385  EXPECT_TRUE(E.isA<CancelledError>());
386  handleAllErrors(std::move(E), [&](const CancelledError &E) {
387  EXPECT_EQ(E.Reason, static_cast<int>(ErrorCode::ContentModified));
388  });
389  },
391  S.runWithAST(
392  "not-invalidatable", Path,
393  [&](llvm::Expected<InputsAndAST> AST) {
394  ++Actions;
395  EXPECT_TRUE(bool(AST));
396  },
398  updateWithDiags(S, Path, "b", WantDiagnostics::Auto, [&](std::vector<Diag>) {
399  ++Builds;
400  ADD_FAILURE() << "Shouldn't build, all dependents invalidated";
401  });
402  S.runWithAST(
403  "invalidatable", Path,
404  [&](llvm::Expected<InputsAndAST> AST) {
405  ++Actions;
406  EXPECT_FALSE(bool(AST));
407  llvm::Error E = AST.takeError();
408  EXPECT_TRUE(E.isA<CancelledError>());
409  consumeError(std::move(E));
410  },
412  updateWithDiags(S, Path, "c", WantDiagnostics::Auto,
413  [&](std::vector<Diag>) { ++Builds; });
414  S.runWithAST(
415  "invalidatable", Path,
416  [&](llvm::Expected<InputsAndAST> AST) {
417  ++Actions;
418  EXPECT_TRUE(bool(AST)) << "Shouldn't be invalidated, no update follows";
419  },
421  Start.notify();
422  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
423 
424  EXPECT_EQ(2, Builds.load()) << "Middle build should be skipped";
425  EXPECT_EQ(4, Actions.load()) << "All actions should run (some with error)";
426 }
427 
428 // We don't invalidate requests for updates that don't change the file content.
429 // These are mostly "refresh this file" events synthesized inside clangd itself.
430 // (Usually the AST rebuild is elided after verifying that all inputs are
431 // unchanged, but invalidation decisions happen earlier and so independently).
432 // See https://github.com/clangd/clangd/issues/620
433 TEST_F(TUSchedulerTests, InvalidationUnchanged) {
434  auto Path = testPath("foo.cpp");
435  TUScheduler S(CDB, optsForTest(), captureDiags());
436  std::atomic<int> Actions(0);
437 
438  Notification Start;
439  updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {
440  Start.wait();
441  });
442  S.runWithAST(
443  "invalidatable", Path,
444  [&](llvm::Expected<InputsAndAST> AST) {
445  ++Actions;
446  EXPECT_TRUE(bool(AST))
447  << "Should not invalidate based on an update with same content: "
448  << llvm::toString(AST.takeError());
449  },
451  updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {
452  ADD_FAILURE() << "Shouldn't build, identical to previous";
453  });
454  Start.notify();
455  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
456 
457  EXPECT_EQ(1, Actions.load()) << "All actions should run";
458 }
459 
460 TEST_F(TUSchedulerTests, ManyUpdates) {
461  const int FilesCount = 3;
462  const int UpdatesPerFile = 10;
463 
464  std::mutex Mut;
465  int TotalASTReads = 0;
466  int TotalPreambleReads = 0;
467  int TotalUpdates = 0;
468  llvm::StringMap<int> LatestDiagVersion;
469 
470  // Run TUScheduler and collect some stats.
471  {
472  auto Opts = optsForTest();
473  Opts.UpdateDebounce = DebouncePolicy::fixed(std::chrono::milliseconds(50));
474  TUScheduler S(CDB, Opts, captureDiags());
475 
476  std::vector<std::string> Files;
477  for (int I = 0; I < FilesCount; ++I) {
478  std::string Name = "foo" + std::to_string(I) + ".cpp";
479  Files.push_back(testPath(Name));
480  this->FS.Files[Files.back()] = "";
481  }
482 
483  StringRef Contents1 = R"cpp(int a;)cpp";
484  StringRef Contents2 = R"cpp(int main() { return 1; })cpp";
485  StringRef Contents3 = R"cpp(int a; int b; int sum() { return a + b; })cpp";
486 
487  StringRef AllContents[] = {Contents1, Contents2, Contents3};
488  const int AllContentsSize = 3;
489 
490  // Scheduler may run tasks asynchronously, but should propagate the
491  // context. We stash a nonce in the context, and verify it in the task.
492  static Key<int> NonceKey;
493  int Nonce = 0;
494 
495  for (int FileI = 0; FileI < FilesCount; ++FileI) {
496  for (int UpdateI = 0; UpdateI < UpdatesPerFile; ++UpdateI) {
497  auto Contents = AllContents[(FileI + UpdateI) % AllContentsSize];
498 
499  auto File = Files[FileI];
500  auto Inputs = getInputs(File, Contents.str());
501  {
502  WithContextValue WithNonce(NonceKey, ++Nonce);
503  Inputs.Version = std::to_string(UpdateI);
504  updateWithDiags(
505  S, File, Inputs, WantDiagnostics::Auto,
506  [File, Nonce, Version(Inputs.Version), &Mut, &TotalUpdates,
507  &LatestDiagVersion](std::vector<Diag>) {
508  EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
509  EXPECT_EQ(File, boundPath());
510 
511  std::lock_guard<std::mutex> Lock(Mut);
512  ++TotalUpdates;
513  EXPECT_EQ(File, *TUScheduler::getFileBeingProcessedInContext());
514  // Make sure Diags are for a newer version.
515  auto It = LatestDiagVersion.try_emplace(File, -1);
516  const int PrevVersion = It.first->second;
517  int CurVersion;
518  ASSERT_TRUE(llvm::to_integer(Version, CurVersion, 10));
519  EXPECT_LT(PrevVersion, CurVersion);
520  It.first->getValue() = CurVersion;
521  });
522  }
523  {
524  WithContextValue WithNonce(NonceKey, ++Nonce);
525  S.runWithAST(
526  "CheckAST", File,
527  [File, Inputs, Nonce, &Mut,
528  &TotalASTReads](Expected<InputsAndAST> AST) {
529  EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
530  EXPECT_EQ(File, boundPath());
531 
532  ASSERT_TRUE((bool)AST);
533  EXPECT_EQ(AST->Inputs.Contents, Inputs.Contents);
534  EXPECT_EQ(AST->Inputs.Version, Inputs.Version);
535  EXPECT_EQ(AST->AST.version(), Inputs.Version);
536 
537  std::lock_guard<std::mutex> Lock(Mut);
538  ++TotalASTReads;
540  });
541  }
542 
543  {
544  WithContextValue WithNonce(NonceKey, ++Nonce);
545  S.runWithPreamble(
546  "CheckPreamble", File, TUScheduler::Stale,
547  [File, Inputs, Nonce, &Mut,
548  &TotalPreambleReads](Expected<InputsAndPreamble> Preamble) {
549  EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
550  EXPECT_EQ(File, boundPath());
551 
552  ASSERT_TRUE((bool)Preamble);
553  EXPECT_EQ(Preamble->Contents, Inputs.Contents);
554 
555  std::lock_guard<std::mutex> Lock(Mut);
556  ++TotalPreambleReads;
558  });
559  }
560  }
561  }
562  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
563  } // TUScheduler destructor waits for all operations to finish.
564 
565  std::lock_guard<std::mutex> Lock(Mut);
566  // Updates might get coalesced in preamble thread and result in dropping
567  // diagnostics for intermediate snapshots.
568  EXPECT_GE(TotalUpdates, FilesCount);
569  EXPECT_LE(TotalUpdates, FilesCount * UpdatesPerFile);
570  // We should receive diags for last update.
571  for (const auto &Entry : LatestDiagVersion)
572  EXPECT_EQ(Entry.second, UpdatesPerFile - 1);
573  EXPECT_EQ(TotalASTReads, FilesCount * UpdatesPerFile);
574  EXPECT_EQ(TotalPreambleReads, FilesCount * UpdatesPerFile);
575 }
576 
577 TEST_F(TUSchedulerTests, EvictedAST) {
578  std::atomic<int> BuiltASTCounter(0);
579  auto Opts = optsForTest();
580  Opts.AsyncThreadsCount = 1;
581  Opts.RetentionPolicy.MaxRetainedASTs = 2;
582  trace::TestTracer Tracer;
583  TUScheduler S(CDB, Opts);
584 
585  llvm::StringLiteral SourceContents = R"cpp(
586  int* a;
587  double* b = a;
588  )cpp";
589  llvm::StringLiteral OtherSourceContents = R"cpp(
590  int* a;
591  double* b = a + 0;
592  )cpp";
593 
594  auto Foo = testPath("foo.cpp");
595  auto Bar = testPath("bar.cpp");
596  auto Baz = testPath("baz.cpp");
597 
598  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
599  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
600  // Build one file in advance. We will not access it later, so it will be the
601  // one that the cache will evict.
602  updateWithCallback(S, Foo, SourceContents, WantDiagnostics::Yes,
603  [&BuiltASTCounter]() { ++BuiltASTCounter; });
604  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
605  ASSERT_EQ(BuiltASTCounter.load(), 1);
606  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
607  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(1));
608 
609  // Build two more files. Since we can retain only 2 ASTs, these should be
610  // the ones we see in the cache later.
611  updateWithCallback(S, Bar, SourceContents, WantDiagnostics::Yes,
612  [&BuiltASTCounter]() { ++BuiltASTCounter; });
613  updateWithCallback(S, Baz, SourceContents, WantDiagnostics::Yes,
614  [&BuiltASTCounter]() { ++BuiltASTCounter; });
615  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
616  ASSERT_EQ(BuiltASTCounter.load(), 3);
617  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
618  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(2));
619 
620  // Check only the last two ASTs are retained.
621  ASSERT_THAT(S.getFilesWithCachedAST(), UnorderedElementsAre(Bar, Baz));
622 
623  // Access the old file again.
624  updateWithCallback(S, Foo, OtherSourceContents, WantDiagnostics::Yes,
625  [&BuiltASTCounter]() { ++BuiltASTCounter; });
626  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
627  ASSERT_EQ(BuiltASTCounter.load(), 4);
628  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
629  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(1));
630 
631  // Check the AST for foo.cpp is retained now and one of the others got
632  // evicted.
633  EXPECT_THAT(S.getFilesWithCachedAST(),
634  UnorderedElementsAre(Foo, AnyOf(Bar, Baz)));
635 }
636 
637 // We send "empty" changes to TUScheduler when we think some external event
638 // *might* have invalidated current state (e.g. a header was edited).
639 // Verify that this doesn't evict our cache entries.
640 TEST_F(TUSchedulerTests, NoopChangesDontThrashCache) {
641  auto Opts = optsForTest();
642  Opts.RetentionPolicy.MaxRetainedASTs = 1;
643  TUScheduler S(CDB, Opts);
644 
645  auto Foo = testPath("foo.cpp");
646  auto FooInputs = getInputs(Foo, "int x=1;");
647  auto Bar = testPath("bar.cpp");
648  auto BarInputs = getInputs(Bar, "int x=2;");
649 
650  // After opening Foo then Bar, AST cache contains Bar.
651  S.update(Foo, FooInputs, WantDiagnostics::Auto);
652  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
653  S.update(Bar, BarInputs, WantDiagnostics::Auto);
654  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
655  ASSERT_THAT(S.getFilesWithCachedAST(), ElementsAre(Bar));
656 
657  // Any number of no-op updates to Foo don't dislodge Bar from the cache.
658  S.update(Foo, FooInputs, WantDiagnostics::Auto);
659  S.update(Foo, FooInputs, WantDiagnostics::Auto);
660  S.update(Foo, FooInputs, WantDiagnostics::Auto);
661  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
662  ASSERT_THAT(S.getFilesWithCachedAST(), ElementsAre(Bar));
663  // In fact each file has been built only once.
664  ASSERT_EQ(S.fileStats().lookup(Foo).ASTBuilds, 1u);
665  ASSERT_EQ(S.fileStats().lookup(Bar).ASTBuilds, 1u);
666 }
667 
668 TEST_F(TUSchedulerTests, EmptyPreamble) {
669  TUScheduler S(CDB, optsForTest());
670 
671  auto Foo = testPath("foo.cpp");
672  auto Header = testPath("foo.h");
673 
674  FS.Files[Header] = "void foo()";
675  FS.Timestamps[Header] = time_t(0);
676  auto *WithPreamble = R"cpp(
677  #include "foo.h"
678  int main() {}
679  )cpp";
680  auto *WithEmptyPreamble = R"cpp(int main() {})cpp";
681  S.update(Foo, getInputs(Foo, WithPreamble), WantDiagnostics::Auto);
682  S.runWithPreamble(
683  "getNonEmptyPreamble", Foo, TUScheduler::Stale,
684  [&](Expected<InputsAndPreamble> Preamble) {
685  // We expect to get a non-empty preamble.
686  EXPECT_GT(
687  cantFail(std::move(Preamble)).Preamble->Preamble.getBounds().Size,
688  0u);
689  });
690  // Wait while the preamble is being built.
691  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
692 
693  // Update the file which results in an empty preamble.
694  S.update(Foo, getInputs(Foo, WithEmptyPreamble), WantDiagnostics::Auto);
695  // Wait while the preamble is being built.
696  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
697  S.runWithPreamble(
698  "getEmptyPreamble", Foo, TUScheduler::Stale,
699  [&](Expected<InputsAndPreamble> Preamble) {
700  // We expect to get an empty preamble.
701  EXPECT_EQ(
702  cantFail(std::move(Preamble)).Preamble->Preamble.getBounds().Size,
703  0u);
704  });
705 }
706 
707 TEST_F(TUSchedulerTests, ASTSignalsSmokeTests) {
708  TUScheduler S(CDB, optsForTest());
709  auto Foo = testPath("foo.cpp");
710  auto Header = testPath("foo.h");
711 
712  FS.Files[Header] = "namespace tar { int foo(); }";
713  const char *Contents = R"cpp(
714  #include "foo.h"
715  namespace ns {
716  int func() {
717  return tar::foo());
718  }
719  } // namespace ns
720  )cpp";
721  // Update the file which results in an empty preamble.
722  S.update(Foo, getInputs(Foo, Contents), WantDiagnostics::Yes);
723  // Wait while the preamble is being built.
724  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
725  Notification TaskRun;
726  S.runWithPreamble(
727  "ASTSignals", Foo, TUScheduler::Stale,
728  [&](Expected<InputsAndPreamble> IP) {
729  ASSERT_FALSE(!IP);
730  std::vector<std::pair<StringRef, int>> NS;
731  for (const auto &P : IP->Signals->RelatedNamespaces)
732  NS.emplace_back(P.getKey(), P.getValue());
733  EXPECT_THAT(NS,
734  UnorderedElementsAre(Pair("ns::", 1), Pair("tar::", 1)));
735 
736  std::vector<std::pair<SymbolID, int>> Sym;
737  for (const auto &P : IP->Signals->ReferencedSymbols)
738  Sym.emplace_back(P.getFirst(), P.getSecond());
739  EXPECT_THAT(Sym, UnorderedElementsAre(Pair(ns("tar").ID, 1),
740  Pair(ns("ns").ID, 1),
741  Pair(func("tar::foo").ID, 1),
742  Pair(func("ns::func").ID, 1)));
743  TaskRun.notify();
744  });
745  TaskRun.wait();
746 }
747 
748 TEST_F(TUSchedulerTests, RunWaitsForPreamble) {
749  // Testing strategy: we update the file and schedule a few preamble reads at
750  // the same time. All reads should get the same non-null preamble.
751  TUScheduler S(CDB, optsForTest());
752  auto Foo = testPath("foo.cpp");
753  auto *NonEmptyPreamble = R"cpp(
754  #define FOO 1
755  #define BAR 2
756 
757  int main() {}
758  )cpp";
759  constexpr int ReadsToSchedule = 10;
760  std::mutex PreamblesMut;
761  std::vector<const void *> Preambles(ReadsToSchedule, nullptr);
762  S.update(Foo, getInputs(Foo, NonEmptyPreamble), WantDiagnostics::Auto);
763  for (int I = 0; I < ReadsToSchedule; ++I) {
764  S.runWithPreamble(
765  "test", Foo, TUScheduler::Stale,
766  [I, &PreamblesMut, &Preambles](Expected<InputsAndPreamble> IP) {
767  std::lock_guard<std::mutex> Lock(PreamblesMut);
768  Preambles[I] = cantFail(std::move(IP)).Preamble;
769  });
770  }
771  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
772  // Check all actions got the same non-null preamble.
773  std::lock_guard<std::mutex> Lock(PreamblesMut);
774  ASSERT_NE(Preambles[0], nullptr);
775  ASSERT_THAT(Preambles, Each(Preambles[0]));
776 }
777 
778 TEST_F(TUSchedulerTests, NoopOnEmptyChanges) {
779  TUScheduler S(CDB, optsForTest(), captureDiags());
780 
781  auto Source = testPath("foo.cpp");
782  auto Header = testPath("foo.h");
783 
784  FS.Files[Header] = "int a;";
785  FS.Timestamps[Header] = time_t(0);
786 
787  std::string SourceContents = R"cpp(
788  #include "foo.h"
789  int b = a;
790  )cpp";
791 
792  // Return value indicates if the updated callback was received.
793  auto DoUpdate = [&](std::string Contents) -> bool {
794  std::atomic<bool> Updated(false);
795  Updated = false;
796  updateWithDiags(S, Source, Contents, WantDiagnostics::Yes,
797  [&Updated](std::vector<Diag>) { Updated = true; });
798  bool UpdateFinished = S.blockUntilIdle(timeoutSeconds(10));
799  if (!UpdateFinished)
800  ADD_FAILURE() << "Updated has not finished in one second. Threading bug?";
801  return Updated;
802  };
803 
804  // Test that subsequent updates with the same inputs do not cause rebuilds.
805  ASSERT_TRUE(DoUpdate(SourceContents));
806  ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 1u);
807  ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 1u);
808  ASSERT_FALSE(DoUpdate(SourceContents));
809  ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 1u);
810  ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 1u);
811 
812  // Update to a header should cause a rebuild, though.
813  FS.Timestamps[Header] = time_t(1);
814  ASSERT_TRUE(DoUpdate(SourceContents));
815  ASSERT_FALSE(DoUpdate(SourceContents));
816  ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 2u);
817  ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 2u);
818 
819  // Update to the contents should cause a rebuild.
820  SourceContents += "\nint c = b;";
821  ASSERT_TRUE(DoUpdate(SourceContents));
822  ASSERT_FALSE(DoUpdate(SourceContents));
823  ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 3u);
824  ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 2u);
825 
826  // Update to the compile commands should also cause a rebuild.
827  CDB.ExtraClangFlags.push_back("-DSOMETHING");
828  ASSERT_TRUE(DoUpdate(SourceContents));
829  ASSERT_FALSE(DoUpdate(SourceContents));
830  ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 4u);
831  ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 3u);
832 }
833 
834 // We rebuild if a completely missing header exists, but not if one is added
835 // on a higher-priority include path entry (for performance).
836 // (Previously we wouldn't automatically rebuild when files were added).
837 TEST_F(TUSchedulerTests, MissingHeader) {
838  CDB.ExtraClangFlags.push_back("-I" + testPath("a"));
839  CDB.ExtraClangFlags.push_back("-I" + testPath("b"));
840  // Force both directories to exist so they don't get pruned.
841  FS.Files.try_emplace("a/__unused__");
842  FS.Files.try_emplace("b/__unused__");
843  TUScheduler S(CDB, optsForTest(), captureDiags());
844 
845  auto Source = testPath("foo.cpp");
846  auto HeaderA = testPath("a/foo.h");
847  auto HeaderB = testPath("b/foo.h");
848 
849  auto *SourceContents = R"cpp(
850  #include "foo.h"
851  int c = b;
852  )cpp";
853 
854  ParseInputs Inputs = getInputs(Source, SourceContents);
855  std::atomic<size_t> DiagCount(0);
856 
857  // Update the source contents, which should trigger an initial build with
858  // the header file missing.
859  updateWithDiags(
860  S, Source, Inputs, WantDiagnostics::Yes,
861  [&DiagCount](std::vector<Diag> Diags) {
862  ++DiagCount;
863  EXPECT_THAT(Diags,
864  ElementsAre(Field(&Diag::Message, "'foo.h' file not found"),
866  "use of undeclared identifier 'b'")));
867  });
868  S.blockUntilIdle(timeoutSeconds(10));
869 
870  FS.Files[HeaderB] = "int b;";
871  FS.Timestamps[HeaderB] = time_t(1);
872 
873  // The addition of the missing header file triggers a rebuild, no errors.
874  updateWithDiags(S, Source, Inputs, WantDiagnostics::Yes,
875  [&DiagCount](std::vector<Diag> Diags) {
876  ++DiagCount;
877  EXPECT_THAT(Diags, IsEmpty());
878  });
879 
880  // Ensure previous assertions are done before we touch the FS again.
881  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
882  // Add the high-priority header file, which should reintroduce the error.
883  FS.Files[HeaderA] = "int a;";
884  FS.Timestamps[HeaderA] = time_t(1);
885 
886  // This isn't detected: we don't stat a/foo.h to validate the preamble.
887  updateWithDiags(S, Source, Inputs, WantDiagnostics::Yes,
888  [&DiagCount](std::vector<Diag> Diags) {
889  ++DiagCount;
890  ADD_FAILURE()
891  << "Didn't expect new diagnostics when adding a/foo.h";
892  });
893 
894  // Forcing the reload should should cause a rebuild.
895  Inputs.ForceRebuild = true;
896  updateWithDiags(
897  S, Source, Inputs, WantDiagnostics::Yes,
898  [&DiagCount](std::vector<Diag> Diags) {
899  ++DiagCount;
900  ElementsAre(Field(&Diag::Message, "use of undeclared identifier 'b'"));
901  });
902 
903  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
904  EXPECT_EQ(DiagCount, 3U);
905 }
906 
907 TEST_F(TUSchedulerTests, NoChangeDiags) {
908  trace::TestTracer Tracer;
909  TUScheduler S(CDB, optsForTest(), captureDiags());
910 
911  auto FooCpp = testPath("foo.cpp");
912  const auto *Contents = "int a; int b;";
913 
914  EXPECT_THAT(Tracer.takeMetric("ast_access_read", "hit"), SizeIs(0));
915  EXPECT_THAT(Tracer.takeMetric("ast_access_read", "miss"), SizeIs(0));
916  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
917  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
918  updateWithDiags(
919  S, FooCpp, Contents, WantDiagnostics::No,
920  [](std::vector<Diag>) { ADD_FAILURE() << "Should not be called."; });
921  S.runWithAST("touchAST", FooCpp, [](Expected<InputsAndAST> IA) {
922  // Make sure the AST was actually built.
923  cantFail(std::move(IA));
924  });
925  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
926  EXPECT_THAT(Tracer.takeMetric("ast_access_read", "hit"), SizeIs(0));
927  EXPECT_THAT(Tracer.takeMetric("ast_access_read", "miss"), SizeIs(1));
928 
929  // Even though the inputs didn't change and AST can be reused, we need to
930  // report the diagnostics, as they were not reported previously.
931  std::atomic<bool> SeenDiags(false);
932  updateWithDiags(S, FooCpp, Contents, WantDiagnostics::Auto,
933  [&](std::vector<Diag>) { SeenDiags = true; });
934  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
935  ASSERT_TRUE(SeenDiags);
936  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(1));
937  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
938 
939  // Subsequent request does not get any diagnostics callback because the same
940  // diags have previously been reported and the inputs didn't change.
941  updateWithDiags(
942  S, FooCpp, Contents, WantDiagnostics::Auto,
943  [&](std::vector<Diag>) { ADD_FAILURE() << "Should not be called."; });
944  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
945 }
946 
947 TEST_F(TUSchedulerTests, Run) {
948  for (bool Sync : {false, true}) {
949  auto Opts = optsForTest();
950  if (Sync)
951  Opts.AsyncThreadsCount = 0;
952  TUScheduler S(CDB, Opts);
953  std::atomic<int> Counter(0);
954  S.run("add 1", /*Path=*/"", [&] { ++Counter; });
955  S.run("add 2", /*Path=*/"", [&] { Counter += 2; });
956  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
957  EXPECT_EQ(Counter.load(), 3);
958 
959  Notification TaskRun;
960  Key<int> TestKey;
961  WithContextValue CtxWithKey(TestKey, 10);
962  const char *Path = "somepath";
963  S.run("props context", Path, [&] {
964  EXPECT_EQ(Context::current().getExisting(TestKey), 10);
965  EXPECT_EQ(Path, boundPath());
966  TaskRun.notify();
967  });
968  TaskRun.wait();
969  }
970 }
971 
972 TEST_F(TUSchedulerTests, TUStatus) {
973  class CaptureTUStatus : public ClangdServer::Callbacks {
974  public:
975  void onFileUpdated(PathRef File, const TUStatus &Status) override {
976  auto ASTAction = Status.ASTActivity.K;
977  auto PreambleAction = Status.PreambleActivity;
978  std::lock_guard<std::mutex> Lock(Mutex);
979  // Only push the action if it has changed. Since TUStatus can be published
980  // from either Preamble or AST thread and when one changes the other stays
981  // the same.
982  // Note that this can result in missing some updates when something other
983  // than action kind changes, e.g. when AST is built/reused the action kind
984  // stays as Building.
985  if (ASTActions.empty() || ASTActions.back() != ASTAction)
986  ASTActions.push_back(ASTAction);
987  if (PreambleActions.empty() || PreambleActions.back() != PreambleAction)
988  PreambleActions.push_back(PreambleAction);
989  }
990 
991  std::vector<PreambleAction> preambleStatuses() {
992  std::lock_guard<std::mutex> Lock(Mutex);
993  return PreambleActions;
994  }
995 
996  std::vector<ASTAction::Kind> astStatuses() {
997  std::lock_guard<std::mutex> Lock(Mutex);
998  return ASTActions;
999  }
1000 
1001  private:
1002  std::mutex Mutex;
1003  std::vector<ASTAction::Kind> ASTActions;
1004  std::vector<PreambleAction> PreambleActions;
1005  } CaptureTUStatus;
1006  MockFS FS;
1007  MockCompilationDatabase CDB;
1008  ClangdServer Server(CDB, FS, ClangdServer::optsForTest(), &CaptureTUStatus);
1009  Annotations Code("int m^ain () {}");
1010 
1011  // We schedule the following tasks in the queue:
1012  // [Update] [GoToDefinition]
1013  Server.addDocument(testPath("foo.cpp"), Code.code(), "1",
1015  ASSERT_TRUE(Server.blockUntilIdleForTest());
1016  Server.locateSymbolAt(testPath("foo.cpp"), Code.point(),
1017  [](Expected<std::vector<LocatedSymbol>> Result) {
1018  ASSERT_TRUE((bool)Result);
1019  });
1020  ASSERT_TRUE(Server.blockUntilIdleForTest());
1021 
1022  EXPECT_THAT(CaptureTUStatus.preambleStatuses(),
1023  ElementsAre(
1024  // PreambleThread starts idle, as the update is first handled
1025  // by ASTWorker.
1027  // Then it starts building first preamble and releases that to
1028  // ASTWorker.
1030  // Then goes idle and stays that way as we don't receive any
1031  // more update requests.
1033  EXPECT_THAT(CaptureTUStatus.astStatuses(),
1034  ElementsAre(
1035  // Starts handling the update action and blocks until the
1036  // first preamble is built.
1038  // Afterwqards it builds an AST for that preamble to publish
1039  // diagnostics.
1041  // Then goes idle.
1043  // Afterwards we start executing go-to-def.
1045  // Then go idle.
1046  ASTAction::Idle));
1047 }
1048 
1049 TEST_F(TUSchedulerTests, CommandLineErrors) {
1050  // We should see errors from command-line parsing inside the main file.
1051  CDB.ExtraClangFlags = {"-fsome-unknown-flag"};
1052 
1053  // (!) 'Ready' must live longer than TUScheduler.
1054  Notification Ready;
1055 
1056  TUScheduler S(CDB, optsForTest(), captureDiags());
1057  std::vector<Diag> Diagnostics;
1058  updateWithDiags(S, testPath("foo.cpp"), "void test() {}",
1059  WantDiagnostics::Yes, [&](std::vector<Diag> D) {
1060  Diagnostics = std::move(D);
1061  Ready.notify();
1062  });
1063  Ready.wait();
1064 
1065  EXPECT_THAT(
1066  Diagnostics,
1067  ElementsAre(AllOf(
1068  Field(&Diag::ID, Eq(diag::err_drv_unknown_argument)),
1069  Field(&Diag::Name, Eq("drv_unknown_argument")),
1070  Field(&Diag::Message, "unknown argument: '-fsome-unknown-flag'"))));
1071 }
1072 
1073 TEST_F(TUSchedulerTests, CommandLineWarnings) {
1074  // We should not see warnings from command-line parsing.
1075  CDB.ExtraClangFlags = {"-Wsome-unknown-warning"};
1076 
1077  // (!) 'Ready' must live longer than TUScheduler.
1078  Notification Ready;
1079 
1080  TUScheduler S(CDB, optsForTest(), captureDiags());
1081  std::vector<Diag> Diagnostics;
1082  updateWithDiags(S, testPath("foo.cpp"), "void test() {}",
1083  WantDiagnostics::Yes, [&](std::vector<Diag> D) {
1084  Diagnostics = std::move(D);
1085  Ready.notify();
1086  });
1087  Ready.wait();
1088 
1089  EXPECT_THAT(Diagnostics, IsEmpty());
1090 }
1091 
1092 TEST(DebouncePolicy, Compute) {
1093  namespace c = std::chrono;
1094  DebouncePolicy::clock::duration History[] = {
1095  c::seconds(0),
1096  c::seconds(5),
1097  c::seconds(10),
1098  c::seconds(20),
1099  };
1100  DebouncePolicy Policy;
1101  Policy.Min = c::seconds(3);
1102  Policy.Max = c::seconds(25);
1103  // Call Policy.compute(History) and return seconds as a float.
1104  auto Compute = [&](llvm::ArrayRef<DebouncePolicy::clock::duration> History) {
1105  return c::duration_cast<c::duration<float, c::seconds::period>>(
1106  Policy.compute(History))
1107  .count();
1108  };
1109  EXPECT_NEAR(10, Compute(History), 0.01) << "(upper) median = 10";
1110  Policy.RebuildRatio = 1.5;
1111  EXPECT_NEAR(15, Compute(History), 0.01) << "median = 10, ratio = 1.5";
1112  Policy.RebuildRatio = 3;
1113  EXPECT_NEAR(25, Compute(History), 0.01) << "constrained by max";
1114  Policy.RebuildRatio = 0;
1115  EXPECT_NEAR(3, Compute(History), 0.01) << "constrained by min";
1116  EXPECT_NEAR(25, Compute({}), 0.01) << "no history -> max";
1117 }
1118 
1119 TEST_F(TUSchedulerTests, AsyncPreambleThread) {
1120  // Blocks preamble thread while building preamble with \p BlockVersion until
1121  // \p N is notified.
1122  class BlockPreambleThread : public ParsingCallbacks {
1123  public:
1124  BlockPreambleThread(llvm::StringRef BlockVersion, Notification &N)
1125  : BlockVersion(BlockVersion), N(N) {}
1126  void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx,
1127  Preprocessor &, const CanonicalIncludes &) override {
1128  if (Version == BlockVersion)
1129  N.wait();
1130  }
1131 
1132  private:
1133  llvm::StringRef BlockVersion;
1134  Notification &N;
1135  };
1136 
1137  static constexpr llvm::StringLiteral InputsV0 = "v0";
1138  static constexpr llvm::StringLiteral InputsV1 = "v1";
1139  Notification Ready;
1140  TUScheduler S(CDB, optsForTest(),
1141  std::make_unique<BlockPreambleThread>(InputsV1, Ready));
1142 
1143  Path File = testPath("foo.cpp");
1144  auto PI = getInputs(File, "");
1145  PI.Version = InputsV0.str();
1146  S.update(File, PI, WantDiagnostics::Auto);
1147  S.blockUntilIdle(timeoutSeconds(10));
1148 
1149  // Block preamble builds.
1150  PI.Version = InputsV1.str();
1151  // Issue second update which will block preamble thread.
1152  S.update(File, PI, WantDiagnostics::Auto);
1153 
1154  Notification RunASTAction;
1155  // Issue an AST read, which shouldn't be blocked and see latest version of the
1156  // file.
1157  S.runWithAST("test", File, [&](Expected<InputsAndAST> AST) {
1158  ASSERT_TRUE(bool(AST));
1159  // Make sure preamble is built with stale inputs, but AST was built using
1160  // new ones.
1161  EXPECT_THAT(AST->AST.preambleVersion(), InputsV0);
1162  EXPECT_THAT(AST->Inputs.Version, InputsV1.str());
1163  RunASTAction.notify();
1164  });
1165  RunASTAction.wait();
1166  Ready.notify();
1167 }
1168 
1169 TEST_F(TUSchedulerTests, OnlyPublishWhenPreambleIsBuilt) {
1170  struct PreamblePublishCounter : public ParsingCallbacks {
1171  PreamblePublishCounter(int &PreamblePublishCount)
1172  : PreamblePublishCount(PreamblePublishCount) {}
1173  void onPreamblePublished(PathRef File) override { ++PreamblePublishCount; }
1174  int &PreamblePublishCount;
1175  };
1176 
1177  int PreamblePublishCount = 0;
1178  TUScheduler S(CDB, optsForTest(),
1179  std::make_unique<PreamblePublishCounter>(PreamblePublishCount));
1180 
1181  Path File = testPath("foo.cpp");
1182  S.update(File, getInputs(File, ""), WantDiagnostics::Auto);
1183  S.blockUntilIdle(timeoutSeconds(10));
1184  EXPECT_EQ(PreamblePublishCount, 1);
1185  // Same contents, no publish.
1186  S.update(File, getInputs(File, ""), WantDiagnostics::Auto);
1187  S.blockUntilIdle(timeoutSeconds(10));
1188  EXPECT_EQ(PreamblePublishCount, 1);
1189  // New contents, should publish.
1190  S.update(File, getInputs(File, "#define FOO"), WantDiagnostics::Auto);
1191  S.blockUntilIdle(timeoutSeconds(10));
1192  EXPECT_EQ(PreamblePublishCount, 2);
1193 }
1194 
1195 // If a header file is missing from the CDB (or inferred using heuristics), and
1196 // it's included by another open file, then we parse it using that files flags.
1197 TEST_F(TUSchedulerTests, IncluderCache) {
1198  static std::string Main = testPath("main.cpp"), Main2 = testPath("main2.cpp"),
1199  Main3 = testPath("main3.cpp"),
1200  NoCmd = testPath("no_cmd.h"),
1201  Unreliable = testPath("unreliable.h"),
1202  OK = testPath("ok.h"),
1203  NotIncluded = testPath("not_included.h");
1204  struct NoHeadersCDB : public GlobalCompilationDatabase {
1205  llvm::Optional<tooling::CompileCommand>
1206  getCompileCommand(PathRef File) const override {
1207  if (File == NoCmd || File == NotIncluded || FailAll)
1208  return llvm::None;
1209  auto Basic = getFallbackCommand(File);
1210  Basic.Heuristic.clear();
1211  if (File == Unreliable) {
1212  Basic.Heuristic = "not reliable";
1213  } else if (File == Main) {
1214  Basic.CommandLine.push_back("-DMAIN");
1215  } else if (File == Main2) {
1216  Basic.CommandLine.push_back("-DMAIN2");
1217  } else if (File == Main3) {
1218  Basic.CommandLine.push_back("-DMAIN3");
1219  }
1220  return Basic;
1221  }
1222 
1223  std::atomic<bool> FailAll{false};
1224  } CDB;
1225  TUScheduler S(CDB, optsForTest());
1226  auto GetFlags = [&](PathRef Header) {
1227  S.update(Header, getInputs(Header, ";"), WantDiagnostics::Yes);
1228  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1229  tooling::CompileCommand Cmd;
1230  S.runWithPreamble("GetFlags", Header, TUScheduler::StaleOrAbsent,
1231  [&](llvm::Expected<InputsAndPreamble> Inputs) {
1232  ASSERT_FALSE(!Inputs) << Inputs.takeError();
1233  Cmd = std::move(Inputs->Command);
1234  });
1235  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1236  return Cmd.CommandLine;
1237  };
1238 
1239  for (const auto &Path : {NoCmd, Unreliable, OK, NotIncluded})
1240  FS.Files[Path] = ";";
1241 
1242  // Initially these files have normal commands from the CDB.
1243  EXPECT_THAT(GetFlags(Main), Contains("-DMAIN")) << "sanity check";
1244  EXPECT_THAT(GetFlags(NoCmd), Not(Contains("-DMAIN"))) << "no includes yet";
1245 
1246  // Now make Main include the others, and some should pick up its flags.
1247  const char *AllIncludes = R"cpp(
1248  #include "no_cmd.h"
1249  #include "ok.h"
1250  #include "unreliable.h"
1251  )cpp";
1252  S.update(Main, getInputs(Main, AllIncludes), WantDiagnostics::Yes);
1253  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1254  EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN"))
1255  << "Included from main file, has no own command";
1256  EXPECT_THAT(GetFlags(Unreliable), Contains("-DMAIN"))
1257  << "Included from main file, own command is heuristic";
1258  EXPECT_THAT(GetFlags(OK), Not(Contains("-DMAIN")))
1259  << "Included from main file, but own command is used";
1260  EXPECT_THAT(GetFlags(NotIncluded), Not(Contains("-DMAIN")))
1261  << "Not included from main file";
1262 
1263  // Open another file - it won't overwrite the associations with Main.
1264  std::string SomeIncludes = R"cpp(
1265  #include "no_cmd.h"
1266  #include "not_included.h"
1267  )cpp";
1268  S.update(Main2, getInputs(Main2, SomeIncludes), WantDiagnostics::Yes);
1269  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1270  EXPECT_THAT(GetFlags(NoCmd),
1271  AllOf(Contains("-DMAIN"), Not(Contains("-DMAIN2"))))
1272  << "mainfile association is stable";
1273  EXPECT_THAT(GetFlags(NotIncluded),
1274  AllOf(Contains("-DMAIN2"), Not(Contains("-DMAIN"))))
1275  << "new headers are associated with new mainfile";
1276 
1277  // Remove includes from main - this marks the associations as invalid but
1278  // doesn't actually remove them until another preamble claims them.
1279  S.update(Main, getInputs(Main, ""), WantDiagnostics::Yes);
1280  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1281  EXPECT_THAT(GetFlags(NoCmd),
1282  AllOf(Contains("-DMAIN"), Not(Contains("-DMAIN2"))))
1283  << "mainfile association not updated yet!";
1284 
1285  // Open yet another file - this time it claims the associations.
1286  S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);
1287  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1288  EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN3"))
1289  << "association invalidated and then claimed by main3";
1290  EXPECT_THAT(GetFlags(Unreliable), Contains("-DMAIN"))
1291  << "association invalidated but not reclaimed";
1292  EXPECT_THAT(GetFlags(NotIncluded), Contains("-DMAIN2"))
1293  << "association still valid";
1294 
1295  // Delete the file from CDB, it should invalidate the associations.
1296  CDB.FailAll = true;
1297  EXPECT_THAT(GetFlags(NoCmd), Not(Contains("-DMAIN3")))
1298  << "association should've been invalidated.";
1299  // Also run update for Main3 to invalidate the preeamble to make sure next
1300  // update populates include cache associations.
1301  S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);
1302  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1303  // Re-add the file and make sure nothing crashes.
1304  CDB.FailAll = false;
1305  S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);
1306  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1307  EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN3"))
1308  << "association invalidated and then claimed by main3";
1309 }
1310 
1311 TEST_F(TUSchedulerTests, PreservesLastActiveFile) {
1312  for (bool Sync : {false, true}) {
1313  auto Opts = optsForTest();
1314  if (Sync)
1315  Opts.AsyncThreadsCount = 0;
1316  TUScheduler S(CDB, Opts);
1317 
1318  auto CheckNoFileActionsSeesLastActiveFile =
1319  [&](llvm::StringRef LastActiveFile) {
1320  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1321  std::atomic<int> Counter(0);
1322  // We only check for run and runQuick as runWithAST and
1323  // runWithPreamble is always bound to a file.
1324  S.run("run-UsesLastActiveFile", /*Path=*/"", [&] {
1325  ++Counter;
1326  EXPECT_EQ(LastActiveFile, boundPath());
1327  });
1328  S.runQuick("runQuick-UsesLastActiveFile", /*Path=*/"", [&] {
1329  ++Counter;
1330  EXPECT_EQ(LastActiveFile, boundPath());
1331  });
1332  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1333  EXPECT_EQ(2, Counter.load());
1334  };
1335 
1336  // Check that we see no file initially
1337  CheckNoFileActionsSeesLastActiveFile("");
1338 
1339  // Now check that every action scheduled with a particular file changes the
1340  // LastActiveFile.
1341  auto Path = testPath("run.cc");
1342  S.run(Path, Path, [] {});
1343  CheckNoFileActionsSeesLastActiveFile(Path);
1344 
1345  Path = testPath("runQuick.cc");
1346  S.runQuick(Path, Path, [] {});
1347  CheckNoFileActionsSeesLastActiveFile(Path);
1348 
1349  Path = testPath("runWithAST.cc");
1350  S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1351  S.runWithAST(Path, Path, [](llvm::Expected<InputsAndAST> Inp) {
1352  EXPECT_TRUE(bool(Inp));
1353  });
1354  CheckNoFileActionsSeesLastActiveFile(Path);
1355 
1356  Path = testPath("runWithPreamble.cc");
1357  S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1358  S.runWithPreamble(
1360  [](llvm::Expected<InputsAndPreamble> Inp) { EXPECT_TRUE(bool(Inp)); });
1361  CheckNoFileActionsSeesLastActiveFile(Path);
1362 
1363  Path = testPath("update.cc");
1364  S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1365  CheckNoFileActionsSeesLastActiveFile(Path);
1366 
1367  // An update with the same contents should not change LastActiveFile.
1368  auto LastActive = Path;
1369  Path = testPath("runWithAST.cc");
1370  S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1371  CheckNoFileActionsSeesLastActiveFile(LastActive);
1372  }
1373 }
1374 } // namespace
1375 } // namespace clangd
1376 } // namespace clang
clang::clangd::ParseInputs::Version
std::string Version
Definition: Compiler.h:51
clang::clangd::cancelableTask
std::pair< Context, Canceler > cancelableTask(int Reason)
Defines a new task whose cancellation may be requested.
Definition: Cancellation.cpp:24
clang::clangd::TEST
TEST(BackgroundQueueTest, Priority)
Definition: BackgroundIndexTests.cpp:750
clang::clangd::MATCHER_P2
MATCHER_P2(hasFlag, Flag, Path, "")
Definition: GlobalCompilationDatabaseTests.cpp:446
clang::clangd::timeoutSeconds
Deadline timeoutSeconds(llvm::Optional< double > Seconds)
Makes a deadline from a timeout in seconds. None means wait forever.
Definition: Threading.cpp:112
E
const Expr * E
Definition: AvoidBindCheck.cpp:88
clang::clangd::ParseInputs::ForceRebuild
bool ForceRebuild
Definition: Compiler.h:54
clang::clangd::testPath
std::string testPath(PathRef File, llvm::sys::path::Style Style)
Definition: TestFS.cpp:94
DiagsCallbackKey
static Key< llvm::unique_function< void(PathRef File, std::vector< Diag >)> > DiagsCallbackKey
Definition: TUSchedulerTests.cpp:115
clang::clangd::ParseInputs::Contents
std::string Contents
Definition: Compiler.h:49
clang::clangd::Path
std::string Path
A typedef to represent a file path.
Definition: Path.h:26
Update
llvm::Optional< UpdateType > Update
Definition: TUScheduler.cpp:644
Expected
std::vector< const char * > Expected
Definition: PrintASTTests.cpp:26
Tracer
std::unique_ptr< trace::EventTracer > Tracer
Definition: TraceTests.cpp:161
Path.h
clang::clangd::Diag::Name
std::string Name
Definition: Diagnostics.h:95
clang::clangd::Context::current
static const Context & current()
Returns the context for the current thread, creating it if needed.
Definition: Context.cpp:27
clang::clangd::WantDiagnostics::Yes
@ Yes
Diagnostics
WantDiagnostics Diagnostics
Definition: TUScheduler.cpp:602
clang::clangd::Context::get
const Type * get(const Key< Type > &Key) const
Get data stored for a typed Key.
Definition: Context.h:98
clang::clangd::WantDiagnostics::Auto
@ Auto
Diagnostics must not be generated for this snapshot.
clang::clangd::PreambleData::Preamble
PrecompiledPreamble Preamble
Definition: Preamble.h:55
clang::tidy::utils::fixit::QualifierTarget::Pointee
@ Pointee
clang::clangd::PreambleAction
PreambleAction
Definition: TUScheduler.h:90
Preamble.h
Ctx
Context Ctx
Definition: TUScheduler.cpp:495
clang::clangd::ClangdServer::optsForTest
static Options optsForTest()
Definition: ClangdServer.cpp:127
clang::clangd::TEST_F
TEST_F(BackgroundIndexTest, NoCrashOnErrorFile)
Definition: BackgroundIndexTests.cpp:94
clang::clangd::func
Symbol func(llvm::StringRef Name)
Definition: TestIndex.cpp:60
clang::clangd::ns
Symbol ns(llvm::StringRef Name)
Definition: TestIndex.cpp:76
FS
MockFS FS
Definition: TUSchedulerTests.cpp:170
clang::clangd::TUScheduler::Stale
@ Stale
The preamble may be generated from an older version of the file.
Definition: TUScheduler.h:286
Preamble
const PreambleData & Preamble
Definition: CodeComplete.cpp:1192
Cancellation.h
clang::clangd::TUScheduler::StaleOrAbsent
@ StaleOrAbsent
Besides accepting stale preamble, this also allow preamble to be absent (not ready or failed to build...
Definition: TUScheduler.h:289
clang::clangd::WantDiagnostics
WantDiagnostics
Determines whether diagnostics should be generated for a file snapshot.
Definition: TUScheduler.h:52
ThreadsafeFS.h
Inputs
ParseInputs Inputs
Definition: TUScheduler.cpp:492
Code
std::string Code
Definition: FindTargetTests.cpp:67
Foo
Definition: sample.h:4
ns1::ns2::D
@ D
Definition: CategoricalFeature.h:3
clang::clangd::ParsedAST::version
llvm::StringRef version() const
Returns the version of the ParseInputs this AST was built from.
Definition: ParsedAST.h:109
clang::clangd::DiagBase::Message
std::string Message
Definition: Diagnostics.h:59
GlobalCompilationDatabase.h
clang::clangd::PreambleAction::Idle
@ Idle
clang::clangd::SymbolKind::Key
@ Key
clang::clangd::MockFS::Timestamps
llvm::StringMap< time_t > Timestamps
Definition: TestFS.h:46
TestFS.h
Threading.h
Diagnostics.h
clang::clangd::ASTAction::Idle
@ Idle
Definition: TUScheduler.h:100
Name
Token Name
Definition: MacroToEnumCheck.cpp:89
Files
llvm::DenseSet< FileID > Files
Definition: IncludeCleaner.cpp:194
EXPECT_ERROR
#define EXPECT_ERROR(expectedValue)
Definition: clangd/unittests/Matchers.h:118
Annotations.h
Bar
Definition: sample.cpp:5
clang::clangd::ASTAction::RunningAction
@ RunningAction
Definition: TUScheduler.h:98
TestTracer.h
Entry
Definition: Modularize.cpp:428
Counter
trace::Metric Counter
Definition: TraceTests.cpp:149
clang::clangd::Context::derive
Context derive(const Key< Type > &Key, typename std::decay< Type >::type Value) const &
Derives a child context It is safe to move or destroy a parent context after calling derive().
Definition: Context.h:119
ID
static char ID
Definition: Logger.cpp:74
TestIndex.h
clang::clangd::PathRef
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:29
C
const Criteria C
Definition: FunctionCognitiveComplexityCheck.cpp:93
clang::clangd::Canceler
std::function< void()> Canceler
A canceller requests cancellation of a task, when called.
Definition: Cancellation.h:70
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
clang::clangd::TUScheduler::NoInvalidation
@ NoInvalidation
The request will run unless explicitly cancelled.
Definition: TUScheduler.h:257
ClangdServer.h
TUScheduler.h
clang::clangd::DiagBase::ID
unsigned ID
Definition: Diagnostics.h:72
CDB
MockCompilationDatabase CDB
Definition: TUSchedulerTests.cpp:171
Diags
CapturedDiags Diags
Definition: ConfigCompileTests.cpp:39
clang::tidy::cppcoreguidelines::toString
static llvm::StringRef toString(SpecialMemberFunctionsCheck::SpecialMemberFunctionKind K)
Definition: SpecialMemberFunctionsCheck.cpp:55
clang::clangd::ASTAction::Building
@ Building
Definition: TUScheduler.h:99
clang::clangd::MockFS::Files
llvm::StringMap< std::string > Files
Definition: TestFS.h:45
clang::clangd::DebouncePolicy::fixed
static DebouncePolicy fixed(clock::duration)
A policy that always returns the same duration, useful for tests.
Definition: TUScheduler.cpp:1770
clang::clangd::CompletionItemKind::Missing
@ Missing
clang::clangd::DocumentHighlightKind::Read
@ Read
clang::clangd::PreambleAction::Building
@ Building
clang::clangd::WantDiagnostics::No
@ No
Diagnostics must be generated for this snapshot.
clang::clangd::MockCompilationDatabase::ExtraClangFlags
std::vector< std::string > ExtraClangFlags
Definition: TestFS.h:68
Field
const FieldDecl * Field
Definition: MemberwiseConstructor.cpp:260
Matchers.h
clang::clangd::ErrorCode::ContentModified
@ ContentModified
clang::clangd::ParsedAST::preambleVersion
llvm::Optional< llvm::StringRef > preambleVersion() const
Returns the version of the ParseInputs used to build Preamble part of this AST.
Definition: ParsedAST.cpp:782
Context.h
clang::clangd::TUScheduler::getFileBeingProcessedInContext
static llvm::Optional< llvm::StringRef > getFileBeingProcessedInContext()
Definition: TUScheduler.cpp:140
ParsedAST.h
clang::clangd::TUScheduler::InvalidateOnUpdate
@ InvalidateOnUpdate
The request will be implicitly cancelled by a subsequent update().
Definition: TUScheduler.h:262