clang-tools  14.0.0git
TUSchedulerTests.cpp
Go to the documentation of this file.
1 //===-- TUSchedulerTests.cpp ------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "Annotations.h"
10 #include "ClangdServer.h"
11 #include "Diagnostics.h"
13 #include "Matchers.h"
14 #include "ParsedAST.h"
15 #include "Preamble.h"
16 #include "TUScheduler.h"
17 #include "TestFS.h"
18 #include "TestIndex.h"
19 #include "support/Cancellation.h"
20 #include "support/Context.h"
21 #include "support/Path.h"
22 #include "support/TestTracer.h"
23 #include "support/Threading.h"
24 #include "support/ThreadsafeFS.h"
25 #include "clang/Basic/DiagnosticDriver.h"
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/FunctionExtras.h"
28 #include "llvm/ADT/STLExtras.h"
29 #include "llvm/ADT/ScopeExit.h"
30 #include "llvm/ADT/StringExtras.h"
31 #include "llvm/ADT/StringMap.h"
32 #include "llvm/ADT/StringRef.h"
33 #include "gmock/gmock.h"
34 #include "gtest/gtest.h"
35 #include <algorithm>
36 #include <atomic>
37 #include <chrono>
38 #include <cstdint>
39 #include <memory>
40 #include <string>
41 #include <utility>
42 
43 namespace clang {
44 namespace clangd {
45 namespace {
46 
47 using ::testing::AllOf;
48 using ::testing::AnyOf;
49 using ::testing::Contains;
50 using ::testing::Each;
51 using ::testing::ElementsAre;
52 using ::testing::Eq;
53 using ::testing::Field;
54 using ::testing::IsEmpty;
55 using ::testing::Not;
56 using ::testing::Pair;
57 using ::testing::Pointee;
58 using ::testing::SizeIs;
59 using ::testing::UnorderedElementsAre;
60 
61 MATCHER_P2(TUState, PreambleActivity, ASTActivity, "") {
62  if (arg.PreambleActivity != PreambleActivity) {
63  *result_listener << "preamblestate is "
64  << static_cast<uint8_t>(arg.PreambleActivity);
65  return false;
66  }
67  if (arg.ASTActivity.K != ASTActivity) {
68  *result_listener << "aststate is " << arg.ASTActivity.K;
69  return false;
70  }
71  return true;
72 }
73 
74 // Simple ContextProvider to verify the provider is invoked & contexts are used.
75 static Key<std::string> BoundPath;
76 Context bindPath(PathRef F) {
77  return Context::current().derive(BoundPath, F.str());
78 }
79 llvm::StringRef boundPath() {
80  const std::string *V = Context::current().get(BoundPath);
81  return V ? *V : llvm::StringRef("");
82 }
83 
84 TUScheduler::Options optsForTest() {
85  TUScheduler::Options Opts(ClangdServer::optsForTest());
86  Opts.ContextProvider = bindPath;
87  return Opts;
88 }
89 
90 class TUSchedulerTests : public ::testing::Test {
91 protected:
92  ParseInputs getInputs(PathRef File, std::string Contents) {
93  ParseInputs Inputs;
94  Inputs.CompileCommand = *CDB.getCompileCommand(File);
95  Inputs.TFS = &FS;
96  Inputs.Contents = std::move(Contents);
97  Inputs.Opts = ParseOptions();
98  return Inputs;
99  }
100 
101  void updateWithCallback(TUScheduler &S, PathRef File,
102  llvm::StringRef Contents, WantDiagnostics WD,
103  llvm::unique_function<void()> CB) {
104  updateWithCallback(S, File, getInputs(File, std::string(Contents)), WD,
105  std::move(CB));
106  }
107 
108  void updateWithCallback(TUScheduler &S, PathRef File, ParseInputs Inputs,
109  WantDiagnostics WD,
110  llvm::unique_function<void()> CB) {
111  WithContextValue Ctx(llvm::make_scope_exit(std::move(CB)));
112  S.update(File, Inputs, WD);
113  }
114 
115  static Key<llvm::unique_function<void(PathRef File, std::vector<Diag>)>>
117 
118  /// A diagnostics callback that should be passed to TUScheduler when it's used
119  /// in updateWithDiags.
120  static std::unique_ptr<ParsingCallbacks> captureDiags() {
121  class CaptureDiags : public ParsingCallbacks {
122  public:
123  void onMainAST(PathRef File, ParsedAST &AST, PublishFn Publish) override {
124  reportDiagnostics(File, *AST.getDiagnostics(), Publish);
125  }
126 
127  void onFailedAST(PathRef File, llvm::StringRef Version,
128  std::vector<Diag> Diags, PublishFn Publish) override {
129  reportDiagnostics(File, Diags, Publish);
130  }
131 
132  private:
133  void reportDiagnostics(PathRef File, llvm::ArrayRef<Diag> Diags,
134  PublishFn Publish) {
136  if (!D)
137  return;
138  Publish([&]() {
139  const_cast<
140  llvm::unique_function<void(PathRef, std::vector<Diag>)> &> (*D)(
141  File, std::move(Diags));
142  });
143  }
144  };
145  return std::make_unique<CaptureDiags>();
146  }
147 
148  /// Schedule an update and call \p CB with the diagnostics it produces, if
149  /// any. The TUScheduler should be created with captureDiags as a
150  /// DiagsCallback for this to work.
151  void updateWithDiags(TUScheduler &S, PathRef File, ParseInputs Inputs,
152  WantDiagnostics WD,
153  llvm::unique_function<void(std::vector<Diag>)> CB) {
154  Path OrigFile = File.str();
155  WithContextValue Ctx(DiagsCallbackKey,
156  [OrigFile, CB = std::move(CB)](
157  PathRef File, std::vector<Diag> Diags) mutable {
158  assert(File == OrigFile);
159  CB(std::move(Diags));
160  });
161  S.update(File, std::move(Inputs), WD);
162  }
163 
164  void updateWithDiags(TUScheduler &S, PathRef File, llvm::StringRef Contents,
165  WantDiagnostics WD,
166  llvm::unique_function<void(std::vector<Diag>)> CB) {
167  return updateWithDiags(S, File, getInputs(File, std::string(Contents)), WD,
168  std::move(CB));
169  }
170 
171  MockFS FS;
172  MockCompilationDatabase CDB;
173 };
174 
175 Key<llvm::unique_function<void(PathRef File, std::vector<Diag>)>>
177 
178 TEST_F(TUSchedulerTests, MissingFiles) {
179  TUScheduler S(CDB, optsForTest());
180 
181  auto Added = testPath("added.cpp");
182  FS.Files[Added] = "x";
183 
184  auto Missing = testPath("missing.cpp");
185  FS.Files[Missing] = "";
186 
187  S.update(Added, getInputs(Added, "x"), WantDiagnostics::No);
188 
189  // Assert each operation for missing file is an error (even if it's
190  // available in VFS).
191  S.runWithAST("", Missing,
192  [&](Expected<InputsAndAST> AST) { EXPECT_ERROR(AST); });
193  S.runWithPreamble(
194  "", Missing, TUScheduler::Stale,
195  [&](Expected<InputsAndPreamble> Preamble) { EXPECT_ERROR(Preamble); });
196  // remove() shouldn't crash on missing files.
197  S.remove(Missing);
198 
199  // Assert there aren't any errors for added file.
200  S.runWithAST("", Added,
201  [&](Expected<InputsAndAST> AST) { EXPECT_TRUE(bool(AST)); });
202  S.runWithPreamble("", Added, TUScheduler::Stale,
203  [&](Expected<InputsAndPreamble> Preamble) {
204  EXPECT_TRUE(bool(Preamble));
205  });
206  S.remove(Added);
207 
208  // Assert that all operations fail after removing the file.
209  S.runWithAST("", Added,
210  [&](Expected<InputsAndAST> AST) { EXPECT_ERROR(AST); });
211  S.runWithPreamble("", Added, TUScheduler::Stale,
212  [&](Expected<InputsAndPreamble> Preamble) {
213  ASSERT_FALSE(bool(Preamble));
214  llvm::consumeError(Preamble.takeError());
215  });
216  // remove() shouldn't crash on missing files.
217  S.remove(Added);
218 }
219 
220 TEST_F(TUSchedulerTests, WantDiagnostics) {
221  std::atomic<int> CallbackCount(0);
222  {
223  // To avoid a racy test, don't allow tasks to actually run on the worker
224  // thread until we've scheduled them all.
225  Notification Ready;
226  TUScheduler S(CDB, optsForTest(), captureDiags());
227  auto Path = testPath("foo.cpp");
228  updateWithDiags(S, Path, "", WantDiagnostics::Yes,
229  [&](std::vector<Diag>) { Ready.wait(); });
230  updateWithDiags(S, Path, "request diags", WantDiagnostics::Yes,
231  [&](std::vector<Diag>) { ++CallbackCount; });
232  updateWithDiags(S, Path, "auto (clobbered)", WantDiagnostics::Auto,
233  [&](std::vector<Diag>) {
234  ADD_FAILURE()
235  << "auto should have been cancelled by auto";
236  });
237  updateWithDiags(S, Path, "request no diags", WantDiagnostics::No,
238  [&](std::vector<Diag>) {
239  ADD_FAILURE() << "no diags should not be called back";
240  });
241  updateWithDiags(S, Path, "auto (produces)", WantDiagnostics::Auto,
242  [&](std::vector<Diag>) { ++CallbackCount; });
243  Ready.notify();
244 
245  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
246  }
247  EXPECT_EQ(2, CallbackCount);
248 }
249 
250 TEST_F(TUSchedulerTests, Debounce) {
251  std::atomic<int> CallbackCount(0);
252  {
253  auto Opts = optsForTest();
254  Opts.UpdateDebounce = DebouncePolicy::fixed(std::chrono::seconds(1));
255  TUScheduler S(CDB, Opts, captureDiags());
256  // FIXME: we could probably use timeouts lower than 1 second here.
257  auto Path = testPath("foo.cpp");
258  updateWithDiags(S, Path, "auto (debounced)", WantDiagnostics::Auto,
259  [&](std::vector<Diag>) {
260  ADD_FAILURE()
261  << "auto should have been debounced and canceled";
262  });
263  std::this_thread::sleep_for(std::chrono::milliseconds(200));
264  updateWithDiags(S, Path, "auto (timed out)", WantDiagnostics::Auto,
265  [&](std::vector<Diag>) { ++CallbackCount; });
266  std::this_thread::sleep_for(std::chrono::seconds(2));
267  updateWithDiags(S, Path, "auto (shut down)", WantDiagnostics::Auto,
268  [&](std::vector<Diag>) { ++CallbackCount; });
269 
270  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
271  }
272  EXPECT_EQ(2, CallbackCount);
273 }
274 
275 TEST_F(TUSchedulerTests, Cancellation) {
276  // We have the following update/read sequence
277  // U0
278  // U1(WantDiags=Yes) <-- cancelled
279  // R1 <-- cancelled
280  // U2(WantDiags=Yes) <-- cancelled
281  // R2A <-- cancelled
282  // R2B
283  // U3(WantDiags=Yes)
284  // R3 <-- cancelled
285  std::vector<std::string> DiagsSeen, ReadsSeen, ReadsCanceled;
286  {
287  Notification Proceed; // Ensure we schedule everything.
288  TUScheduler S(CDB, optsForTest(), captureDiags());
289  auto Path = testPath("foo.cpp");
290  // Helper to schedule a named update and return a function to cancel it.
291  auto Update = [&](std::string ID) -> Canceler {
292  auto T = cancelableTask();
293  WithContext C(std::move(T.first));
294  updateWithDiags(
295  S, Path, "//" + ID, WantDiagnostics::Yes,
296  [&, ID](std::vector<Diag> Diags) { DiagsSeen.push_back(ID); });
297  return std::move(T.second);
298  };
299  // Helper to schedule a named read and return a function to cancel it.
300  auto Read = [&](std::string ID) -> Canceler {
301  auto T = cancelableTask();
302  WithContext C(std::move(T.first));
303  S.runWithAST(ID, Path, [&, ID](llvm::Expected<InputsAndAST> E) {
304  if (auto Err = E.takeError()) {
305  if (Err.isA<CancelledError>()) {
306  ReadsCanceled.push_back(ID);
307  consumeError(std::move(Err));
308  } else {
309  ADD_FAILURE() << "Non-cancelled error for " << ID << ": "
310  << llvm::toString(std::move(Err));
311  }
312  } else {
313  ReadsSeen.push_back(ID);
314  }
315  });
316  return std::move(T.second);
317  };
318 
319  updateWithCallback(S, Path, "", WantDiagnostics::Yes,
320  [&]() { Proceed.wait(); });
321  // The second parens indicate cancellation, where present.
322  Update("U1")();
323  Read("R1")();
324  Update("U2")();
325  Read("R2A")();
326  Read("R2B");
327  Update("U3");
328  Read("R3")();
329  Proceed.notify();
330 
331  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
332  }
333  EXPECT_THAT(DiagsSeen, ElementsAre("U2", "U3"))
334  << "U1 and all dependent reads were cancelled. "
335  "U2 has a dependent read R2A. "
336  "U3 was not cancelled.";
337  EXPECT_THAT(ReadsSeen, ElementsAre("R2B"))
338  << "All reads other than R2B were cancelled";
339  EXPECT_THAT(ReadsCanceled, ElementsAre("R1", "R2A", "R3"))
340  << "All reads other than R2B were cancelled";
341 }
342 
343 TEST_F(TUSchedulerTests, InvalidationNoCrash) {
344  auto Path = testPath("foo.cpp");
345  TUScheduler S(CDB, optsForTest(), captureDiags());
346 
347  Notification StartedRunning;
348  Notification ScheduledChange;
349  // We expect invalidation logic to not crash by trying to invalidate a running
350  // request.
351  S.update(Path, getInputs(Path, ""), WantDiagnostics::Auto);
352  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
353  S.runWithAST(
354  "invalidatable-but-running", Path,
355  [&](llvm::Expected<InputsAndAST> AST) {
356  StartedRunning.notify();
357  ScheduledChange.wait();
358  ASSERT_TRUE(bool(AST));
359  },
361  StartedRunning.wait();
362  S.update(Path, getInputs(Path, ""), WantDiagnostics::Auto);
363  ScheduledChange.notify();
364  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
365 }
366 
367 TEST_F(TUSchedulerTests, Invalidation) {
368  auto Path = testPath("foo.cpp");
369  TUScheduler S(CDB, optsForTest(), captureDiags());
370  std::atomic<int> Builds(0), Actions(0);
371 
372  Notification Start;
373  updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {
374  ++Builds;
375  Start.wait();
376  });
377  S.runWithAST(
378  "invalidatable", Path,
379  [&](llvm::Expected<InputsAndAST> AST) {
380  ++Actions;
381  EXPECT_FALSE(bool(AST));
382  llvm::Error E = AST.takeError();
383  EXPECT_TRUE(E.isA<CancelledError>());
384  handleAllErrors(std::move(E), [&](const CancelledError &E) {
385  EXPECT_EQ(E.Reason, static_cast<int>(ErrorCode::ContentModified));
386  });
387  },
389  S.runWithAST(
390  "not-invalidatable", Path,
391  [&](llvm::Expected<InputsAndAST> AST) {
392  ++Actions;
393  EXPECT_TRUE(bool(AST));
394  },
396  updateWithDiags(S, Path, "b", WantDiagnostics::Auto, [&](std::vector<Diag>) {
397  ++Builds;
398  ADD_FAILURE() << "Shouldn't build, all dependents invalidated";
399  });
400  S.runWithAST(
401  "invalidatable", Path,
402  [&](llvm::Expected<InputsAndAST> AST) {
403  ++Actions;
404  EXPECT_FALSE(bool(AST));
405  llvm::Error E = AST.takeError();
406  EXPECT_TRUE(E.isA<CancelledError>());
407  consumeError(std::move(E));
408  },
410  updateWithDiags(S, Path, "c", WantDiagnostics::Auto,
411  [&](std::vector<Diag>) { ++Builds; });
412  S.runWithAST(
413  "invalidatable", Path,
414  [&](llvm::Expected<InputsAndAST> AST) {
415  ++Actions;
416  EXPECT_TRUE(bool(AST)) << "Shouldn't be invalidated, no update follows";
417  },
419  Start.notify();
420  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
421 
422  EXPECT_EQ(2, Builds.load()) << "Middle build should be skipped";
423  EXPECT_EQ(4, Actions.load()) << "All actions should run (some with error)";
424 }
425 
426 // We don't invalidate requests for updates that don't change the file content.
427 // These are mostly "refresh this file" events synthesized inside clangd itself.
428 // (Usually the AST rebuild is elided after verifying that all inputs are
429 // unchanged, but invalidation decisions happen earlier and so independently).
430 // See https://github.com/clangd/clangd/issues/620
431 TEST_F(TUSchedulerTests, InvalidationUnchanged) {
432  auto Path = testPath("foo.cpp");
433  TUScheduler S(CDB, optsForTest(), captureDiags());
434  std::atomic<int> Actions(0);
435 
436  Notification Start;
437  updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {
438  Start.wait();
439  });
440  S.runWithAST(
441  "invalidatable", Path,
442  [&](llvm::Expected<InputsAndAST> AST) {
443  ++Actions;
444  EXPECT_TRUE(bool(AST))
445  << "Should not invalidate based on an update with same content: "
446  << llvm::toString(AST.takeError());
447  },
449  updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {
450  ADD_FAILURE() << "Shouldn't build, identical to previous";
451  });
452  Start.notify();
453  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
454 
455  EXPECT_EQ(1, Actions.load()) << "All actions should run";
456 }
457 
458 TEST_F(TUSchedulerTests, ManyUpdates) {
459  const int FilesCount = 3;
460  const int UpdatesPerFile = 10;
461 
462  std::mutex Mut;
463  int TotalASTReads = 0;
464  int TotalPreambleReads = 0;
465  int TotalUpdates = 0;
466  llvm::StringMap<int> LatestDiagVersion;
467 
468  // Run TUScheduler and collect some stats.
469  {
470  auto Opts = optsForTest();
471  Opts.UpdateDebounce = DebouncePolicy::fixed(std::chrono::milliseconds(50));
472  TUScheduler S(CDB, Opts, captureDiags());
473 
474  std::vector<std::string> Files;
475  for (int I = 0; I < FilesCount; ++I) {
476  std::string Name = "foo" + std::to_string(I) + ".cpp";
477  Files.push_back(testPath(Name));
478  this->FS.Files[Files.back()] = "";
479  }
480 
481  StringRef Contents1 = R"cpp(int a;)cpp";
482  StringRef Contents2 = R"cpp(int main() { return 1; })cpp";
483  StringRef Contents3 = R"cpp(int a; int b; int sum() { return a + b; })cpp";
484 
485  StringRef AllContents[] = {Contents1, Contents2, Contents3};
486  const int AllContentsSize = 3;
487 
488  // Scheduler may run tasks asynchronously, but should propagate the
489  // context. We stash a nonce in the context, and verify it in the task.
490  static Key<int> NonceKey;
491  int Nonce = 0;
492 
493  for (int FileI = 0; FileI < FilesCount; ++FileI) {
494  for (int UpdateI = 0; UpdateI < UpdatesPerFile; ++UpdateI) {
495  auto Contents = AllContents[(FileI + UpdateI) % AllContentsSize];
496 
497  auto File = Files[FileI];
498  auto Inputs = getInputs(File, Contents.str());
499  {
500  WithContextValue WithNonce(NonceKey, ++Nonce);
501  Inputs.Version = std::to_string(UpdateI);
502  updateWithDiags(
503  S, File, Inputs, WantDiagnostics::Auto,
504  [File, Nonce, Version(Inputs.Version), &Mut, &TotalUpdates,
505  &LatestDiagVersion](std::vector<Diag>) {
506  EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
507  EXPECT_EQ(File, boundPath());
508 
509  std::lock_guard<std::mutex> Lock(Mut);
510  ++TotalUpdates;
511  EXPECT_EQ(File, *TUScheduler::getFileBeingProcessedInContext());
512  // Make sure Diags are for a newer version.
513  auto It = LatestDiagVersion.try_emplace(File, -1);
514  const int PrevVersion = It.first->second;
515  int CurVersion;
516  ASSERT_TRUE(llvm::to_integer(Version, CurVersion, 10));
517  EXPECT_LT(PrevVersion, CurVersion);
518  It.first->getValue() = CurVersion;
519  });
520  }
521  {
522  WithContextValue WithNonce(NonceKey, ++Nonce);
523  S.runWithAST(
524  "CheckAST", File,
525  [File, Inputs, Nonce, &Mut,
526  &TotalASTReads](Expected<InputsAndAST> AST) {
527  EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
528  EXPECT_EQ(File, boundPath());
529 
530  ASSERT_TRUE((bool)AST);
531  EXPECT_EQ(AST->Inputs.Contents, Inputs.Contents);
532  EXPECT_EQ(AST->Inputs.Version, Inputs.Version);
533  EXPECT_EQ(AST->AST.version(), Inputs.Version);
534 
535  std::lock_guard<std::mutex> Lock(Mut);
536  ++TotalASTReads;
538  });
539  }
540 
541  {
542  WithContextValue WithNonce(NonceKey, ++Nonce);
543  S.runWithPreamble(
544  "CheckPreamble", File, TUScheduler::Stale,
545  [File, Inputs, Nonce, &Mut,
546  &TotalPreambleReads](Expected<InputsAndPreamble> Preamble) {
547  EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
548  EXPECT_EQ(File, boundPath());
549 
550  ASSERT_TRUE((bool)Preamble);
551  EXPECT_EQ(Preamble->Contents, Inputs.Contents);
552 
553  std::lock_guard<std::mutex> Lock(Mut);
554  ++TotalPreambleReads;
556  });
557  }
558  }
559  }
560  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
561  } // TUScheduler destructor waits for all operations to finish.
562 
563  std::lock_guard<std::mutex> Lock(Mut);
564  // Updates might get coalesced in preamble thread and result in dropping
565  // diagnostics for intermediate snapshots.
566  EXPECT_GE(TotalUpdates, FilesCount);
567  EXPECT_LE(TotalUpdates, FilesCount * UpdatesPerFile);
568  // We should receive diags for last update.
569  for (const auto &Entry : LatestDiagVersion)
570  EXPECT_EQ(Entry.second, UpdatesPerFile - 1);
571  EXPECT_EQ(TotalASTReads, FilesCount * UpdatesPerFile);
572  EXPECT_EQ(TotalPreambleReads, FilesCount * UpdatesPerFile);
573 }
574 
575 TEST_F(TUSchedulerTests, EvictedAST) {
576  std::atomic<int> BuiltASTCounter(0);
577  auto Opts = optsForTest();
578  Opts.AsyncThreadsCount = 1;
579  Opts.RetentionPolicy.MaxRetainedASTs = 2;
580  trace::TestTracer Tracer;
581  TUScheduler S(CDB, Opts);
582 
583  llvm::StringLiteral SourceContents = R"cpp(
584  int* a;
585  double* b = a;
586  )cpp";
587  llvm::StringLiteral OtherSourceContents = R"cpp(
588  int* a;
589  double* b = a + 0;
590  )cpp";
591 
592  auto Foo = testPath("foo.cpp");
593  auto Bar = testPath("bar.cpp");
594  auto Baz = testPath("baz.cpp");
595 
596  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
597  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
598  // Build one file in advance. We will not access it later, so it will be the
599  // one that the cache will evict.
600  updateWithCallback(S, Foo, SourceContents, WantDiagnostics::Yes,
601  [&BuiltASTCounter]() { ++BuiltASTCounter; });
602  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
603  ASSERT_EQ(BuiltASTCounter.load(), 1);
604  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
605  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(1));
606 
607  // Build two more files. Since we can retain only 2 ASTs, these should be
608  // the ones we see in the cache later.
609  updateWithCallback(S, Bar, SourceContents, WantDiagnostics::Yes,
610  [&BuiltASTCounter]() { ++BuiltASTCounter; });
611  updateWithCallback(S, Baz, SourceContents, WantDiagnostics::Yes,
612  [&BuiltASTCounter]() { ++BuiltASTCounter; });
613  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
614  ASSERT_EQ(BuiltASTCounter.load(), 3);
615  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
616  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(2));
617 
618  // Check only the last two ASTs are retained.
619  ASSERT_THAT(S.getFilesWithCachedAST(), UnorderedElementsAre(Bar, Baz));
620 
621  // Access the old file again.
622  updateWithCallback(S, Foo, OtherSourceContents, WantDiagnostics::Yes,
623  [&BuiltASTCounter]() { ++BuiltASTCounter; });
624  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
625  ASSERT_EQ(BuiltASTCounter.load(), 4);
626  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
627  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(1));
628 
629  // Check the AST for foo.cpp is retained now and one of the others got
630  // evicted.
631  EXPECT_THAT(S.getFilesWithCachedAST(),
632  UnorderedElementsAre(Foo, AnyOf(Bar, Baz)));
633 }
634 
635 // We send "empty" changes to TUScheduler when we think some external event
636 // *might* have invalidated current state (e.g. a header was edited).
637 // Verify that this doesn't evict our cache entries.
638 TEST_F(TUSchedulerTests, NoopChangesDontThrashCache) {
639  auto Opts = optsForTest();
640  Opts.RetentionPolicy.MaxRetainedASTs = 1;
641  TUScheduler S(CDB, Opts);
642 
643  auto Foo = testPath("foo.cpp");
644  auto FooInputs = getInputs(Foo, "int x=1;");
645  auto Bar = testPath("bar.cpp");
646  auto BarInputs = getInputs(Bar, "int x=2;");
647 
648  // After opening Foo then Bar, AST cache contains Bar.
649  S.update(Foo, FooInputs, WantDiagnostics::Auto);
650  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
651  S.update(Bar, BarInputs, WantDiagnostics::Auto);
652  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
653  ASSERT_THAT(S.getFilesWithCachedAST(), ElementsAre(Bar));
654 
655  // Any number of no-op updates to Foo don't dislodge Bar from the cache.
656  S.update(Foo, FooInputs, WantDiagnostics::Auto);
657  S.update(Foo, FooInputs, WantDiagnostics::Auto);
658  S.update(Foo, FooInputs, WantDiagnostics::Auto);
659  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
660  ASSERT_THAT(S.getFilesWithCachedAST(), ElementsAre(Bar));
661  // In fact each file has been built only once.
662  ASSERT_EQ(S.fileStats().lookup(Foo).ASTBuilds, 1u);
663  ASSERT_EQ(S.fileStats().lookup(Bar).ASTBuilds, 1u);
664 }
665 
666 TEST_F(TUSchedulerTests, EmptyPreamble) {
667  TUScheduler S(CDB, optsForTest());
668 
669  auto Foo = testPath("foo.cpp");
670  auto Header = testPath("foo.h");
671 
672  FS.Files[Header] = "void foo()";
673  FS.Timestamps[Header] = time_t(0);
674  auto WithPreamble = R"cpp(
675  #include "foo.h"
676  int main() {}
677  )cpp";
678  auto WithEmptyPreamble = R"cpp(int main() {})cpp";
679  S.update(Foo, getInputs(Foo, WithPreamble), WantDiagnostics::Auto);
680  S.runWithPreamble(
681  "getNonEmptyPreamble", Foo, TUScheduler::Stale,
682  [&](Expected<InputsAndPreamble> Preamble) {
683  // We expect to get a non-empty preamble.
684  EXPECT_GT(
685  cantFail(std::move(Preamble)).Preamble->Preamble.getBounds().Size,
686  0u);
687  });
688  // Wait while the preamble is being built.
689  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
690 
691  // Update the file which results in an empty preamble.
692  S.update(Foo, getInputs(Foo, WithEmptyPreamble), WantDiagnostics::Auto);
693  // Wait while the preamble is being built.
694  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
695  S.runWithPreamble(
696  "getEmptyPreamble", Foo, TUScheduler::Stale,
697  [&](Expected<InputsAndPreamble> Preamble) {
698  // We expect to get an empty preamble.
699  EXPECT_EQ(
700  cantFail(std::move(Preamble)).Preamble->Preamble.getBounds().Size,
701  0u);
702  });
703 }
704 
705 TEST_F(TUSchedulerTests, ASTSignalsSmokeTests) {
706  TUScheduler S(CDB, optsForTest());
707  auto Foo = testPath("foo.cpp");
708  auto Header = testPath("foo.h");
709 
710  FS.Files[Header] = "namespace tar { int foo(); }";
711  const char *Contents = R"cpp(
712  #include "foo.h"
713  namespace ns {
714  int func() {
715  return tar::foo());
716  }
717  } // namespace ns
718  )cpp";
719  // Update the file which results in an empty preamble.
720  S.update(Foo, getInputs(Foo, Contents), WantDiagnostics::Yes);
721  // Wait while the preamble is being built.
722  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
723  Notification TaskRun;
724  S.runWithPreamble(
725  "ASTSignals", Foo, TUScheduler::Stale,
726  [&](Expected<InputsAndPreamble> IP) {
727  ASSERT_FALSE(!IP);
728  std::vector<std::pair<StringRef, int>> NS;
729  for (const auto &P : IP->Signals->RelatedNamespaces)
730  NS.emplace_back(P.getKey(), P.getValue());
731  EXPECT_THAT(NS,
732  UnorderedElementsAre(Pair("ns::", 1), Pair("tar::", 1)));
733 
734  std::vector<std::pair<SymbolID, int>> Sym;
735  for (const auto &P : IP->Signals->ReferencedSymbols)
736  Sym.emplace_back(P.getFirst(), P.getSecond());
737  EXPECT_THAT(Sym, UnorderedElementsAre(Pair(ns("tar").ID, 1),
738  Pair(ns("ns").ID, 1),
739  Pair(func("tar::foo").ID, 1),
740  Pair(func("ns::func").ID, 1)));
741  TaskRun.notify();
742  });
743  TaskRun.wait();
744 }
745 
746 TEST_F(TUSchedulerTests, RunWaitsForPreamble) {
747  // Testing strategy: we update the file and schedule a few preamble reads at
748  // the same time. All reads should get the same non-null preamble.
749  TUScheduler S(CDB, optsForTest());
750  auto Foo = testPath("foo.cpp");
751  auto NonEmptyPreamble = R"cpp(
752  #define FOO 1
753  #define BAR 2
754 
755  int main() {}
756  )cpp";
757  constexpr int ReadsToSchedule = 10;
758  std::mutex PreamblesMut;
759  std::vector<const void *> Preambles(ReadsToSchedule, nullptr);
760  S.update(Foo, getInputs(Foo, NonEmptyPreamble), WantDiagnostics::Auto);
761  for (int I = 0; I < ReadsToSchedule; ++I) {
762  S.runWithPreamble(
763  "test", Foo, TUScheduler::Stale,
764  [I, &PreamblesMut, &Preambles](Expected<InputsAndPreamble> IP) {
765  std::lock_guard<std::mutex> Lock(PreamblesMut);
766  Preambles[I] = cantFail(std::move(IP)).Preamble;
767  });
768  }
769  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
770  // Check all actions got the same non-null preamble.
771  std::lock_guard<std::mutex> Lock(PreamblesMut);
772  ASSERT_NE(Preambles[0], nullptr);
773  ASSERT_THAT(Preambles, Each(Preambles[0]));
774 }
775 
776 TEST_F(TUSchedulerTests, NoopOnEmptyChanges) {
777  TUScheduler S(CDB, optsForTest(), captureDiags());
778 
779  auto Source = testPath("foo.cpp");
780  auto Header = testPath("foo.h");
781 
782  FS.Files[Header] = "int a;";
783  FS.Timestamps[Header] = time_t(0);
784 
785  std::string SourceContents = R"cpp(
786  #include "foo.h"
787  int b = a;
788  )cpp";
789 
790  // Return value indicates if the updated callback was received.
791  auto DoUpdate = [&](std::string Contents) -> bool {
792  std::atomic<bool> Updated(false);
793  Updated = false;
794  updateWithDiags(S, Source, Contents, WantDiagnostics::Yes,
795  [&Updated](std::vector<Diag>) { Updated = true; });
796  bool UpdateFinished = S.blockUntilIdle(timeoutSeconds(10));
797  if (!UpdateFinished)
798  ADD_FAILURE() << "Updated has not finished in one second. Threading bug?";
799  return Updated;
800  };
801 
802  // Test that subsequent updates with the same inputs do not cause rebuilds.
803  ASSERT_TRUE(DoUpdate(SourceContents));
804  ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 1u);
805  ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 1u);
806  ASSERT_FALSE(DoUpdate(SourceContents));
807  ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 1u);
808  ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 1u);
809 
810  // Update to a header should cause a rebuild, though.
811  FS.Timestamps[Header] = time_t(1);
812  ASSERT_TRUE(DoUpdate(SourceContents));
813  ASSERT_FALSE(DoUpdate(SourceContents));
814  ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 2u);
815  ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 2u);
816 
817  // Update to the contents should cause a rebuild.
818  SourceContents += "\nint c = b;";
819  ASSERT_TRUE(DoUpdate(SourceContents));
820  ASSERT_FALSE(DoUpdate(SourceContents));
821  ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 3u);
822  ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 2u);
823 
824  // Update to the compile commands should also cause a rebuild.
825  CDB.ExtraClangFlags.push_back("-DSOMETHING");
826  ASSERT_TRUE(DoUpdate(SourceContents));
827  ASSERT_FALSE(DoUpdate(SourceContents));
828  ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 4u);
829  ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 3u);
830 }
831 
832 // We rebuild if a completely missing header exists, but not if one is added
833 // on a higher-priority include path entry (for performance).
834 // (Previously we wouldn't automatically rebuild when files were added).
835 TEST_F(TUSchedulerTests, MissingHeader) {
836  CDB.ExtraClangFlags.push_back("-I" + testPath("a"));
837  CDB.ExtraClangFlags.push_back("-I" + testPath("b"));
838  // Force both directories to exist so they don't get pruned.
839  FS.Files.try_emplace("a/__unused__");
840  FS.Files.try_emplace("b/__unused__");
841  TUScheduler S(CDB, optsForTest(), captureDiags());
842 
843  auto Source = testPath("foo.cpp");
844  auto HeaderA = testPath("a/foo.h");
845  auto HeaderB = testPath("b/foo.h");
846 
847  auto SourceContents = R"cpp(
848  #include "foo.h"
849  int c = b;
850  )cpp";
851 
852  ParseInputs Inputs = getInputs(Source, SourceContents);
853  std::atomic<size_t> DiagCount(0);
854 
855  // Update the source contents, which should trigger an initial build with
856  // the header file missing.
857  updateWithDiags(
858  S, Source, Inputs, WantDiagnostics::Yes,
859  [&DiagCount](std::vector<Diag> Diags) {
860  ++DiagCount;
861  EXPECT_THAT(Diags,
862  ElementsAre(Field(&Diag::Message, "'foo.h' file not found"),
864  "use of undeclared identifier 'b'")));
865  });
866  S.blockUntilIdle(timeoutSeconds(10));
867 
868  FS.Files[HeaderB] = "int b;";
869  FS.Timestamps[HeaderB] = time_t(1);
870 
871  // The addition of the missing header file triggers a rebuild, no errors.
872  updateWithDiags(S, Source, Inputs, WantDiagnostics::Yes,
873  [&DiagCount](std::vector<Diag> Diags) {
874  ++DiagCount;
875  EXPECT_THAT(Diags, IsEmpty());
876  });
877 
878  // Ensure previous assertions are done before we touch the FS again.
879  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
880  // Add the high-priority header file, which should reintroduce the error.
881  FS.Files[HeaderA] = "int a;";
882  FS.Timestamps[HeaderA] = time_t(1);
883 
884  // This isn't detected: we don't stat a/foo.h to validate the preamble.
885  updateWithDiags(S, Source, Inputs, WantDiagnostics::Yes,
886  [&DiagCount](std::vector<Diag> Diags) {
887  ++DiagCount;
888  ADD_FAILURE()
889  << "Didn't expect new diagnostics when adding a/foo.h";
890  });
891 
892  // Forcing the reload should should cause a rebuild.
893  Inputs.ForceRebuild = true;
894  updateWithDiags(
895  S, Source, Inputs, WantDiagnostics::Yes,
896  [&DiagCount](std::vector<Diag> Diags) {
897  ++DiagCount;
898  ElementsAre(Field(&Diag::Message, "use of undeclared identifier 'b'"));
899  });
900 
901  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
902  EXPECT_EQ(DiagCount, 3U);
903 }
904 
905 TEST_F(TUSchedulerTests, NoChangeDiags) {
906  trace::TestTracer Tracer;
907  TUScheduler S(CDB, optsForTest(), captureDiags());
908 
909  auto FooCpp = testPath("foo.cpp");
910  const auto *Contents = "int a; int b;";
911 
912  EXPECT_THAT(Tracer.takeMetric("ast_access_read", "hit"), SizeIs(0));
913  EXPECT_THAT(Tracer.takeMetric("ast_access_read", "miss"), SizeIs(0));
914  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
915  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
916  updateWithDiags(
917  S, FooCpp, Contents, WantDiagnostics::No,
918  [](std::vector<Diag>) { ADD_FAILURE() << "Should not be called."; });
919  S.runWithAST("touchAST", FooCpp, [](Expected<InputsAndAST> IA) {
920  // Make sure the AST was actually built.
921  cantFail(std::move(IA));
922  });
923  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
924  EXPECT_THAT(Tracer.takeMetric("ast_access_read", "hit"), SizeIs(0));
925  EXPECT_THAT(Tracer.takeMetric("ast_access_read", "miss"), SizeIs(1));
926 
927  // Even though the inputs didn't change and AST can be reused, we need to
928  // report the diagnostics, as they were not reported previously.
929  std::atomic<bool> SeenDiags(false);
930  updateWithDiags(S, FooCpp, Contents, WantDiagnostics::Auto,
931  [&](std::vector<Diag>) { SeenDiags = true; });
932  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
933  ASSERT_TRUE(SeenDiags);
934  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(1));
935  EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
936 
937  // Subsequent request does not get any diagnostics callback because the same
938  // diags have previously been reported and the inputs didn't change.
939  updateWithDiags(
940  S, FooCpp, Contents, WantDiagnostics::Auto,
941  [&](std::vector<Diag>) { ADD_FAILURE() << "Should not be called."; });
942  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
943 }
944 
945 TEST_F(TUSchedulerTests, Run) {
946  for (bool Sync : {false, true}) {
947  auto Opts = optsForTest();
948  if (Sync)
949  Opts.AsyncThreadsCount = 0;
950  TUScheduler S(CDB, Opts);
951  std::atomic<int> Counter(0);
952  S.run("add 1", /*Path=*/"", [&] { ++Counter; });
953  S.run("add 2", /*Path=*/"", [&] { Counter += 2; });
954  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
955  EXPECT_EQ(Counter.load(), 3);
956 
957  Notification TaskRun;
958  Key<int> TestKey;
959  WithContextValue CtxWithKey(TestKey, 10);
960  const char *Path = "somepath";
961  S.run("props context", Path, [&] {
962  EXPECT_EQ(Context::current().getExisting(TestKey), 10);
963  EXPECT_EQ(Path, boundPath());
964  TaskRun.notify();
965  });
966  TaskRun.wait();
967  }
968 }
969 
970 TEST_F(TUSchedulerTests, TUStatus) {
971  class CaptureTUStatus : public ClangdServer::Callbacks {
972  public:
973  void onFileUpdated(PathRef File, const TUStatus &Status) override {
974  auto ASTAction = Status.ASTActivity.K;
975  auto PreambleAction = Status.PreambleActivity;
976  std::lock_guard<std::mutex> Lock(Mutex);
977  // Only push the action if it has changed. Since TUStatus can be published
978  // from either Preamble or AST thread and when one changes the other stays
979  // the same.
980  // Note that this can result in missing some updates when something other
981  // than action kind changes, e.g. when AST is built/reused the action kind
982  // stays as Building.
983  if (ASTActions.empty() || ASTActions.back() != ASTAction)
984  ASTActions.push_back(ASTAction);
985  if (PreambleActions.empty() || PreambleActions.back() != PreambleAction)
986  PreambleActions.push_back(PreambleAction);
987  }
988 
989  std::vector<PreambleAction> preambleStatuses() {
990  std::lock_guard<std::mutex> Lock(Mutex);
991  return PreambleActions;
992  }
993 
994  std::vector<ASTAction::Kind> astStatuses() {
995  std::lock_guard<std::mutex> Lock(Mutex);
996  return ASTActions;
997  }
998 
999  private:
1000  std::mutex Mutex;
1001  std::vector<ASTAction::Kind> ASTActions;
1002  std::vector<PreambleAction> PreambleActions;
1003  } CaptureTUStatus;
1004  MockFS FS;
1005  MockCompilationDatabase CDB;
1006  ClangdServer Server(CDB, FS, ClangdServer::optsForTest(), &CaptureTUStatus);
1007  Annotations Code("int m^ain () {}");
1008 
1009  // We schedule the following tasks in the queue:
1010  // [Update] [GoToDefinition]
1011  Server.addDocument(testPath("foo.cpp"), Code.code(), "1",
1013  ASSERT_TRUE(Server.blockUntilIdleForTest());
1014  Server.locateSymbolAt(testPath("foo.cpp"), Code.point(),
1015  [](Expected<std::vector<LocatedSymbol>> Result) {
1016  ASSERT_TRUE((bool)Result);
1017  });
1018  ASSERT_TRUE(Server.blockUntilIdleForTest());
1019 
1020  EXPECT_THAT(CaptureTUStatus.preambleStatuses(),
1021  ElementsAre(
1022  // PreambleThread starts idle, as the update is first handled
1023  // by ASTWorker.
1025  // Then it starts building first preamble and releases that to
1026  // ASTWorker.
1028  // Then goes idle and stays that way as we don't receive any
1029  // more update requests.
1031  EXPECT_THAT(CaptureTUStatus.astStatuses(),
1032  ElementsAre(
1033  // Starts handling the update action and blocks until the
1034  // first preamble is built.
1036  // Afterwqards it builds an AST for that preamble to publish
1037  // diagnostics.
1039  // Then goes idle.
1041  // Afterwards we start executing go-to-def.
1043  // Then go idle.
1044  ASTAction::Idle));
1045 }
1046 
1047 TEST_F(TUSchedulerTests, CommandLineErrors) {
1048  // We should see errors from command-line parsing inside the main file.
1049  CDB.ExtraClangFlags = {"-fsome-unknown-flag"};
1050 
1051  // (!) 'Ready' must live longer than TUScheduler.
1052  Notification Ready;
1053 
1054  TUScheduler S(CDB, optsForTest(), captureDiags());
1055  std::vector<Diag> Diagnostics;
1056  updateWithDiags(S, testPath("foo.cpp"), "void test() {}",
1057  WantDiagnostics::Yes, [&](std::vector<Diag> D) {
1058  Diagnostics = std::move(D);
1059  Ready.notify();
1060  });
1061  Ready.wait();
1062 
1063  EXPECT_THAT(
1064  Diagnostics,
1065  ElementsAre(AllOf(
1066  Field(&Diag::ID, Eq(diag::err_drv_unknown_argument)),
1067  Field(&Diag::Name, Eq("drv_unknown_argument")),
1068  Field(&Diag::Message, "unknown argument: '-fsome-unknown-flag'"))));
1069 }
1070 
1071 TEST_F(TUSchedulerTests, CommandLineWarnings) {
1072  // We should not see warnings from command-line parsing.
1073  CDB.ExtraClangFlags = {"-Wsome-unknown-warning"};
1074 
1075  // (!) 'Ready' must live longer than TUScheduler.
1076  Notification Ready;
1077 
1078  TUScheduler S(CDB, optsForTest(), captureDiags());
1079  std::vector<Diag> Diagnostics;
1080  updateWithDiags(S, testPath("foo.cpp"), "void test() {}",
1081  WantDiagnostics::Yes, [&](std::vector<Diag> D) {
1082  Diagnostics = std::move(D);
1083  Ready.notify();
1084  });
1085  Ready.wait();
1086 
1087  EXPECT_THAT(Diagnostics, IsEmpty());
1088 }
1089 
1090 TEST(DebouncePolicy, Compute) {
1091  namespace c = std::chrono;
1092  DebouncePolicy::clock::duration History[] = {
1093  c::seconds(0),
1094  c::seconds(5),
1095  c::seconds(10),
1096  c::seconds(20),
1097  };
1098  DebouncePolicy Policy;
1099  Policy.Min = c::seconds(3);
1100  Policy.Max = c::seconds(25);
1101  // Call Policy.compute(History) and return seconds as a float.
1102  auto Compute = [&](llvm::ArrayRef<DebouncePolicy::clock::duration> History) {
1103  return c::duration_cast<c::duration<float, c::seconds::period>>(
1104  Policy.compute(History))
1105  .count();
1106  };
1107  EXPECT_NEAR(10, Compute(History), 0.01) << "(upper) median = 10";
1108  Policy.RebuildRatio = 1.5;
1109  EXPECT_NEAR(15, Compute(History), 0.01) << "median = 10, ratio = 1.5";
1110  Policy.RebuildRatio = 3;
1111  EXPECT_NEAR(25, Compute(History), 0.01) << "constrained by max";
1112  Policy.RebuildRatio = 0;
1113  EXPECT_NEAR(3, Compute(History), 0.01) << "constrained by min";
1114  EXPECT_NEAR(25, Compute({}), 0.01) << "no history -> max";
1115 }
1116 
1117 TEST_F(TUSchedulerTests, AsyncPreambleThread) {
1118  // Blocks preamble thread while building preamble with \p BlockVersion until
1119  // \p N is notified.
1120  class BlockPreambleThread : public ParsingCallbacks {
1121  public:
1122  BlockPreambleThread(llvm::StringRef BlockVersion, Notification &N)
1123  : BlockVersion(BlockVersion), N(N) {}
1124  void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx,
1125  std::shared_ptr<clang::Preprocessor> PP,
1126  const CanonicalIncludes &) override {
1127  if (Version == BlockVersion)
1128  N.wait();
1129  }
1130 
1131  private:
1132  llvm::StringRef BlockVersion;
1133  Notification &N;
1134  };
1135 
1136  static constexpr llvm::StringLiteral InputsV0 = "v0";
1137  static constexpr llvm::StringLiteral InputsV1 = "v1";
1138  Notification Ready;
1139  TUScheduler S(CDB, optsForTest(),
1140  std::make_unique<BlockPreambleThread>(InputsV1, Ready));
1141 
1142  Path File = testPath("foo.cpp");
1143  auto PI = getInputs(File, "");
1144  PI.Version = InputsV0.str();
1145  S.update(File, PI, WantDiagnostics::Auto);
1146  S.blockUntilIdle(timeoutSeconds(10));
1147 
1148  // Block preamble builds.
1149  PI.Version = InputsV1.str();
1150  // Issue second update which will block preamble thread.
1151  S.update(File, PI, WantDiagnostics::Auto);
1152 
1153  Notification RunASTAction;
1154  // Issue an AST read, which shouldn't be blocked and see latest version of the
1155  // file.
1156  S.runWithAST("test", File, [&](Expected<InputsAndAST> AST) {
1157  ASSERT_TRUE(bool(AST));
1158  // Make sure preamble is built with stale inputs, but AST was built using
1159  // new ones.
1160  EXPECT_THAT(AST->AST.preambleVersion(), InputsV0);
1161  EXPECT_THAT(AST->Inputs.Version, InputsV1.str());
1162  RunASTAction.notify();
1163  });
1164  RunASTAction.wait();
1165  Ready.notify();
1166 }
1167 
1168 TEST_F(TUSchedulerTests, OnlyPublishWhenPreambleIsBuilt) {
1169  struct PreamblePublishCounter : public ParsingCallbacks {
1170  PreamblePublishCounter(int &PreamblePublishCount)
1171  : PreamblePublishCount(PreamblePublishCount) {}
1172  void onPreamblePublished(PathRef File) override { ++PreamblePublishCount; }
1173  int &PreamblePublishCount;
1174  };
1175 
1176  int PreamblePublishCount = 0;
1177  TUScheduler S(CDB, optsForTest(),
1178  std::make_unique<PreamblePublishCounter>(PreamblePublishCount));
1179 
1180  Path File = testPath("foo.cpp");
1181  S.update(File, getInputs(File, ""), WantDiagnostics::Auto);
1182  S.blockUntilIdle(timeoutSeconds(10));
1183  EXPECT_EQ(PreamblePublishCount, 1);
1184  // Same contents, no publish.
1185  S.update(File, getInputs(File, ""), WantDiagnostics::Auto);
1186  S.blockUntilIdle(timeoutSeconds(10));
1187  EXPECT_EQ(PreamblePublishCount, 1);
1188  // New contents, should publish.
1189  S.update(File, getInputs(File, "#define FOO"), WantDiagnostics::Auto);
1190  S.blockUntilIdle(timeoutSeconds(10));
1191  EXPECT_EQ(PreamblePublishCount, 2);
1192 }
1193 
1194 // If a header file is missing from the CDB (or inferred using heuristics), and
1195 // it's included by another open file, then we parse it using that files flags.
1196 TEST_F(TUSchedulerTests, IncluderCache) {
1197  static std::string Main = testPath("main.cpp"), Main2 = testPath("main2.cpp"),
1198  Main3 = testPath("main3.cpp"),
1199  NoCmd = testPath("no_cmd.h"),
1200  Unreliable = testPath("unreliable.h"),
1201  OK = testPath("ok.h"),
1202  NotIncluded = testPath("not_included.h");
1203  struct NoHeadersCDB : public GlobalCompilationDatabase {
1204  llvm::Optional<tooling::CompileCommand>
1205  getCompileCommand(PathRef File) const override {
1206  if (File == NoCmd || File == NotIncluded || FailAll)
1207  return llvm::None;
1208  auto Basic = getFallbackCommand(File);
1209  Basic.Heuristic.clear();
1210  if (File == Unreliable) {
1211  Basic.Heuristic = "not reliable";
1212  } else if (File == Main) {
1213  Basic.CommandLine.push_back("-DMAIN");
1214  } else if (File == Main2) {
1215  Basic.CommandLine.push_back("-DMAIN2");
1216  } else if (File == Main3) {
1217  Basic.CommandLine.push_back("-DMAIN3");
1218  }
1219  return Basic;
1220  }
1221 
1222  std::atomic<bool> FailAll{false};
1223  } CDB;
1224  TUScheduler S(CDB, optsForTest());
1225  auto GetFlags = [&](PathRef Header) {
1226  S.update(Header, getInputs(Header, ";"), WantDiagnostics::Yes);
1227  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1228  tooling::CompileCommand Cmd;
1229  S.runWithPreamble("GetFlags", Header, TUScheduler::StaleOrAbsent,
1230  [&](llvm::Expected<InputsAndPreamble> Inputs) {
1231  ASSERT_FALSE(!Inputs) << Inputs.takeError();
1232  Cmd = std::move(Inputs->Command);
1233  });
1234  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1235  return Cmd.CommandLine;
1236  };
1237 
1238  for (const auto &Path : {NoCmd, Unreliable, OK, NotIncluded})
1239  FS.Files[Path] = ";";
1240 
1241  // Initially these files have normal commands from the CDB.
1242  EXPECT_THAT(GetFlags(Main), Contains("-DMAIN")) << "sanity check";
1243  EXPECT_THAT(GetFlags(NoCmd), Not(Contains("-DMAIN"))) << "no includes yet";
1244 
1245  // Now make Main include the others, and some should pick up its flags.
1246  const char *AllIncludes = R"cpp(
1247  #include "no_cmd.h"
1248  #include "ok.h"
1249  #include "unreliable.h"
1250  )cpp";
1251  S.update(Main, getInputs(Main, AllIncludes), WantDiagnostics::Yes);
1252  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1253  EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN"))
1254  << "Included from main file, has no own command";
1255  EXPECT_THAT(GetFlags(Unreliable), Contains("-DMAIN"))
1256  << "Included from main file, own command is heuristic";
1257  EXPECT_THAT(GetFlags(OK), Not(Contains("-DMAIN")))
1258  << "Included from main file, but own command is used";
1259  EXPECT_THAT(GetFlags(NotIncluded), Not(Contains("-DMAIN")))
1260  << "Not included from main file";
1261 
1262  // Open another file - it won't overwrite the associations with Main.
1263  std::string SomeIncludes = R"cpp(
1264  #include "no_cmd.h"
1265  #include "not_included.h"
1266  )cpp";
1267  S.update(Main2, getInputs(Main2, SomeIncludes), WantDiagnostics::Yes);
1268  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1269  EXPECT_THAT(GetFlags(NoCmd),
1270  AllOf(Contains("-DMAIN"), Not(Contains("-DMAIN2"))))
1271  << "mainfile association is stable";
1272  EXPECT_THAT(GetFlags(NotIncluded),
1273  AllOf(Contains("-DMAIN2"), Not(Contains("-DMAIN"))))
1274  << "new headers are associated with new mainfile";
1275 
1276  // Remove includes from main - this marks the associations as invalid but
1277  // doesn't actually remove them until another preamble claims them.
1278  S.update(Main, getInputs(Main, ""), WantDiagnostics::Yes);
1279  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1280  EXPECT_THAT(GetFlags(NoCmd),
1281  AllOf(Contains("-DMAIN"), Not(Contains("-DMAIN2"))))
1282  << "mainfile association not updated yet!";
1283 
1284  // Open yet another file - this time it claims the associations.
1285  S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);
1286  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1287  EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN3"))
1288  << "association invalidated and then claimed by main3";
1289  EXPECT_THAT(GetFlags(Unreliable), Contains("-DMAIN"))
1290  << "association invalidated but not reclaimed";
1291  EXPECT_THAT(GetFlags(NotIncluded), Contains("-DMAIN2"))
1292  << "association still valid";
1293 
1294  // Delete the file from CDB, it should invalidate the associations.
1295  CDB.FailAll = true;
1296  EXPECT_THAT(GetFlags(NoCmd), Not(Contains("-DMAIN3")))
1297  << "association should've been invalidated.";
1298  // Also run update for Main3 to invalidate the preeamble to make sure next
1299  // update populates include cache associations.
1300  S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);
1301  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1302  // Re-add the file and make sure nothing crashes.
1303  CDB.FailAll = false;
1304  S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);
1305  EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1306  EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN3"))
1307  << "association invalidated and then claimed by main3";
1308 }
1309 
1310 TEST_F(TUSchedulerTests, PreservesLastActiveFile) {
1311  for (bool Sync : {false, true}) {
1312  auto Opts = optsForTest();
1313  if (Sync)
1314  Opts.AsyncThreadsCount = 0;
1315  TUScheduler S(CDB, Opts);
1316 
1317  auto CheckNoFileActionsSeesLastActiveFile =
1318  [&](llvm::StringRef LastActiveFile) {
1319  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1320  std::atomic<int> Counter(0);
1321  // We only check for run and runQuick as runWithAST and
1322  // runWithPreamble is always bound to a file.
1323  S.run("run-UsesLastActiveFile", /*Path=*/"", [&] {
1324  ++Counter;
1325  EXPECT_EQ(LastActiveFile, boundPath());
1326  });
1327  S.runQuick("runQuick-UsesLastActiveFile", /*Path=*/"", [&] {
1328  ++Counter;
1329  EXPECT_EQ(LastActiveFile, boundPath());
1330  });
1331  ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1332  EXPECT_EQ(2, Counter.load());
1333  };
1334 
1335  // Check that we see no file initially
1336  CheckNoFileActionsSeesLastActiveFile("");
1337 
1338  // Now check that every action scheduled with a particular file changes the
1339  // LastActiveFile.
1340  auto Path = testPath("run.cc");
1341  S.run(Path, Path, [] {});
1342  CheckNoFileActionsSeesLastActiveFile(Path);
1343 
1344  Path = testPath("runQuick.cc");
1345  S.runQuick(Path, Path, [] {});
1346  CheckNoFileActionsSeesLastActiveFile(Path);
1347 
1348  Path = testPath("runWithAST.cc");
1349  S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1350  S.runWithAST(Path, Path, [](llvm::Expected<InputsAndAST> Inp) {
1351  EXPECT_TRUE(bool(Inp));
1352  });
1353  CheckNoFileActionsSeesLastActiveFile(Path);
1354 
1355  Path = testPath("runWithPreamble.cc");
1356  S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1357  S.runWithPreamble(
1359  [](llvm::Expected<InputsAndPreamble> Inp) { EXPECT_TRUE(bool(Inp)); });
1360  CheckNoFileActionsSeesLastActiveFile(Path);
1361 
1362  Path = testPath("update.cc");
1363  S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1364  CheckNoFileActionsSeesLastActiveFile(Path);
1365 
1366  // An update with the same contents should not change LastActiveFile.
1367  auto LastActive = Path;
1368  Path = testPath("runWithAST.cc");
1369  S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1370  CheckNoFileActionsSeesLastActiveFile(LastActive);
1371  }
1372 }
1373 } // namespace
1374 } // namespace clangd
1375 } // namespace clang
clang::clangd::ParseInputs::Version
std::string Version
Definition: Compiler.h:52
clang::clangd::cancelableTask
std::pair< Context, Canceler > cancelableTask(int Reason)
Defines a new task whose cancellation may be requested.
Definition: Cancellation.cpp:24
clang::clangd::TEST
TEST(BackgroundQueueTest, Priority)
Definition: BackgroundIndexTests.cpp:751
clang::clangd::MATCHER_P2
MATCHER_P2(hasFlag, Flag, Path, "")
Definition: GlobalCompilationDatabaseTests.cpp:451
clang::clangd::timeoutSeconds
Deadline timeoutSeconds(llvm::Optional< double > Seconds)
Makes a deadline from a timeout in seconds. None means wait forever.
Definition: Threading.cpp:105
E
const Expr * E
Definition: AvoidBindCheck.cpp:88
clang::clangd::ParseInputs::ForceRebuild
bool ForceRebuild
Definition: Compiler.h:55
clang::clangd::testPath
std::string testPath(PathRef File, llvm::sys::path::Style Style)
Definition: TestFS.cpp:82
DiagsCallbackKey
static Key< llvm::unique_function< void(PathRef File, std::vector< Diag >)> > DiagsCallbackKey
Definition: TUSchedulerTests.cpp:116
clang::clangd::ParseInputs::Contents
std::string Contents
Definition: Compiler.h:50
clang::clangd::Path
std::string Path
A typedef to represent a file path.
Definition: Path.h:26
Update
llvm::Optional< UpdateType > Update
Definition: TUScheduler.cpp:609
Expected
std::vector< const char * > Expected
Definition: PrintASTTests.cpp:27
Tracer
std::unique_ptr< trace::EventTracer > Tracer
Definition: TraceTests.cpp:164
Path.h
clang::clangd::Diag::Name
std::string Name
Definition: Diagnostics.h:98
clang::clangd::Context::current
static const Context & current()
Returns the context for the current thread, creating it if needed.
Definition: Context.cpp:27
clang::clangd::WantDiagnostics::Yes
@ Yes
Diagnostics
WantDiagnostics Diagnostics
Definition: TUScheduler.cpp:567
clang::clangd::Context::get
const Type * get(const Key< Type > &Key) const
Get data stored for a typed Key.
Definition: Context.h:98
clang::clangd::CompletionItemKind::Field
@ Field
clang::clangd::WantDiagnostics::Auto
@ Auto
Diagnostics must not be generated for this snapshot.
clang::clangd::PreambleData::Preamble
PrecompiledPreamble Preamble
Definition: Preamble.h:55
clang::tidy::utils::fixit::QualifierTarget::Pointee
@ Pointee
clang::clangd::PreambleAction
PreambleAction
Definition: TUScheduler.h:91
Preamble.h
Ctx
Context Ctx
Definition: TUScheduler.cpp:460
clang::clangd::ClangdServer::optsForTest
static Options optsForTest()
Definition: ClangdServer.cpp:135
clang::clangd::TEST_F
TEST_F(BackgroundIndexTest, NoCrashOnErrorFile)
Definition: BackgroundIndexTests.cpp:94
clang::clangd::func
Symbol func(llvm::StringRef Name)
Definition: TestIndex.cpp:60
clang::clangd::ns
Symbol ns(llvm::StringRef Name)
Definition: TestIndex.cpp:72
FS
MockFS FS
Definition: TUSchedulerTests.cpp:171
clang::clangd::TUScheduler::Stale
@ Stale
The preamble may be generated from an older version of the file.
Definition: TUScheduler.h:288
Preamble
const PreambleData & Preamble
Definition: CodeComplete.cpp:1148
Cancellation.h
clang::clangd::TUScheduler::StaleOrAbsent
@ StaleOrAbsent
Besides accepting stale preamble, this also allow preamble to be absent (not ready or failed to build...
Definition: TUScheduler.h:291
clang::clangd::WantDiagnostics
WantDiagnostics
Determines whether diagnostics should be generated for a file snapshot.
Definition: TUScheduler.h:53
ThreadsafeFS.h
Inputs
ParseInputs Inputs
Definition: TUScheduler.cpp:457
Code
std::string Code
Definition: FindTargetTests.cpp:67
Foo
Definition: sample.h:4
ns1::ns2::D
@ D
Definition: CategoricalFeature.h:3
clang::clangd::ParsedAST::version
llvm::StringRef version() const
Returns the version of the ParseInputs this AST was built from.
Definition: ParsedAST.h:111
clang::clangd::DiagBase::Message
std::string Message
Definition: Diagnostics.h:62
GlobalCompilationDatabase.h
clang::clangd::PreambleAction::Idle
@ Idle
clang::clangd::SymbolKind::Key
@ Key
clang::clangd::MockFS::Timestamps
llvm::StringMap< time_t > Timestamps
Definition: TestFS.h:48
TestFS.h
Threading.h
Name
static constexpr llvm::StringLiteral Name
Definition: UppercaseLiteralSuffixCheck.cpp:28
Diagnostics.h
clang::clangd::ASTAction::Idle
@ Idle
Definition: TUScheduler.h:101
Files
llvm::DenseSet< FileID > Files
Definition: IncludeCleaner.cpp:125
EXPECT_ERROR
#define EXPECT_ERROR(expectedValue)
Definition: clangd/unittests/Matchers.h:118
Annotations.h
Bar
Definition: sample.cpp:5
clang::clangd::ASTAction::RunningAction
@ RunningAction
Definition: TUScheduler.h:99
clang::tidy::bugprone::PP
static Preprocessor * PP
Definition: BadSignalToKillThreadCheck.cpp:29
TestTracer.h
Entry
Definition: Modularize.cpp:428
Counter
trace::Metric Counter
Definition: TraceTests.cpp:152
clang::clangd::Context::derive
Context derive(const Key< Type > &Key, typename std::decay< Type >::type Value) const &
Derives a child context It is safe to move or destroy a parent context after calling derive().
Definition: Context.h:119
ID
static char ID
Definition: Logger.cpp:74
TestIndex.h
clang::clangd::PathRef
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:29
C
const Criteria C
Definition: FunctionCognitiveComplexityCheck.cpp:93
clang::clangd::Canceler
std::function< void()> Canceler
A canceller requests cancellation of a task, when called.
Definition: Cancellation.h:70
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
clang::clangd::TUScheduler::NoInvalidation
@ NoInvalidation
The request will run unless explicitly cancelled.
Definition: TUScheduler.h:259
ClangdServer.h
TUScheduler.h
clang::clangd::DiagBase::ID
unsigned ID
Definition: Diagnostics.h:75
CDB
MockCompilationDatabase CDB
Definition: TUSchedulerTests.cpp:172
Diags
CapturedDiags Diags
Definition: ConfigCompileTests.cpp:38
clang::tidy::cppcoreguidelines::toString
static llvm::StringRef toString(SpecialMemberFunctionsCheck::SpecialMemberFunctionKind K)
Definition: SpecialMemberFunctionsCheck.cpp:55
clang::clangd::ASTAction::Building
@ Building
Definition: TUScheduler.h:100
clang::clangd::MockFS::Files
llvm::StringMap< std::string > Files
Definition: TestFS.h:47
clang::clangd::DebouncePolicy::fixed
static DebouncePolicy fixed(clock::duration)
A policy that always returns the same duration, useful for tests.
Definition: TUScheduler.cpp:1732
clang::clangd::CompletionItemKind::Missing
@ Missing
clang::clangd::DocumentHighlightKind::Read
@ Read
clang::clangd::PreambleAction::Building
@ Building
clang::clangd::WantDiagnostics::No
@ No
Diagnostics must be generated for this snapshot.
clang::clangd::MockCompilationDatabase::ExtraClangFlags
std::vector< std::string > ExtraClangFlags
Definition: TestFS.h:70
Matchers.h
clang::clangd::ErrorCode::ContentModified
@ ContentModified
clang::clangd::ParsedAST::preambleVersion
llvm::Optional< llvm::StringRef > preambleVersion() const
Returns the version of the ParseInputs used to build Preamble part of this AST.
Definition: ParsedAST.cpp:634
Context.h
clang::clangd::TUScheduler::getFileBeingProcessedInContext
static llvm::Optional< llvm::StringRef > getFileBeingProcessedInContext()
Definition: TUScheduler.cpp:105
ParsedAST.h
clang::clangd::TUScheduler::InvalidateOnUpdate
@ InvalidateOnUpdate
The request will be implicitly cancelled by a subsequent update().
Definition: TUScheduler.h:264