clang-tools  15.0.0git
TUScheduler.cpp
Go to the documentation of this file.
1 //===--- TUScheduler.cpp -----------------------------------------*-C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // TUScheduler manages a worker per active file. This ASTWorker processes
9 // updates (modifications to file contents) and reads (actions performed on
10 // preamble/AST) to the file.
11 //
12 // Each ASTWorker owns a dedicated thread to process updates and reads to the
13 // relevant file. Any request gets queued in FIFO order to be processed by that
14 // thread.
15 //
16 // An update request replaces current praser inputs to ensure any subsequent
17 // read sees the version of the file they were requested. It will also issue a
18 // build for new inputs.
19 //
20 // ASTWorker processes the file in two parts, a preamble and a main-file
21 // section. A preamble can be reused between multiple versions of the file until
22 // invalidated by a modification to a header, compile commands or modification
23 // to relevant part of the current file. Such a preamble is called compatible.
24 // An update is considered dead if no read was issued for that version and
25 // diagnostics weren't requested by client or could be generated for a later
26 // version of the file. ASTWorker eliminates such requests as they are
27 // redundant.
28 //
29 // In the presence of stale (non-compatible) preambles, ASTWorker won't publish
30 // diagnostics for update requests. Read requests will be served with ASTs build
31 // with stale preambles, unless the read is picky and requires a compatible
32 // preamble. In such cases it will block until new preamble is built.
33 //
34 // ASTWorker owns a PreambleThread for building preambles. If the preamble gets
35 // invalidated by an update request, a new build will be requested on
36 // PreambleThread. Since PreambleThread only receives requests for newer
37 // versions of the file, in case of multiple requests it will only build the
38 // last one and skip requests in between. Unless client force requested
39 // diagnostics(WantDiagnostics::Yes).
40 //
41 // When a new preamble is built, a "golden" AST is immediately built from that
42 // version of the file. This ensures diagnostics get updated even if the queue
43 // is full.
44 //
45 // Some read requests might just need preamble. Since preambles can be read
46 // concurrently, ASTWorker runs these requests on their own thread. These
47 // requests will receive latest build preamble, which might possibly be stale.
48 
49 #include "TUScheduler.h"
50 #include "CompileCommands.h"
51 #include "Compiler.h"
52 #include "Diagnostics.h"
54 #include "ParsedAST.h"
55 #include "Preamble.h"
57 #include "support/Cancellation.h"
58 #include "support/Context.h"
59 #include "support/Logger.h"
60 #include "support/MemoryTree.h"
61 #include "support/Path.h"
63 #include "support/Threading.h"
64 #include "support/Trace.h"
65 #include "clang/Frontend/CompilerInvocation.h"
66 #include "clang/Tooling/CompilationDatabase.h"
67 #include "llvm/ADT/FunctionExtras.h"
68 #include "llvm/ADT/None.h"
69 #include "llvm/ADT/Optional.h"
70 #include "llvm/ADT/STLExtras.h"
71 #include "llvm/ADT/ScopeExit.h"
72 #include "llvm/ADT/SmallVector.h"
73 #include "llvm/ADT/StringExtras.h"
74 #include "llvm/ADT/StringRef.h"
75 #include "llvm/Support/Allocator.h"
76 #include "llvm/Support/Errc.h"
77 #include "llvm/Support/ErrorHandling.h"
78 #include "llvm/Support/FormatVariadic.h"
79 #include "llvm/Support/Path.h"
80 #include "llvm/Support/Threading.h"
81 #include "llvm/Support/raw_ostream.h"
82 #include <algorithm>
83 #include <atomic>
84 #include <chrono>
85 #include <condition_variable>
86 #include <functional>
87 #include <memory>
88 #include <mutex>
89 #include <queue>
90 #include <string>
91 #include <thread>
92 #include <type_traits>
93 #include <utility>
94 #include <vector>
95 
96 namespace clang {
97 namespace clangd {
98 using std::chrono::steady_clock;
99 
100 namespace {
101 // Tracks latency (in seconds) of FS operations done during a preamble build.
102 // build_type allows to split by expected VFS cache state (cold on first
103 // preamble, somewhat warm after that when building first preamble for new file,
104 // likely ~everything cached on preamble rebuild.
105 constexpr trace::Metric
106  PreambleBuildFilesystemLatency("preamble_fs_latency",
107  trace::Metric::Distribution, "build_type");
108 // Tracks latency of FS operations done during a preamble build as a ratio of
109 // preamble build time. build_type is same as above.
110 constexpr trace::Metric PreambleBuildFilesystemLatencyRatio(
111  "preamble_fs_latency_ratio", trace::Metric::Distribution, "build_type");
112 
113 constexpr trace::Metric PreambleBuildSize("preamble_build_size",
115 constexpr trace::Metric PreambleSerializedSize("preamble_serialized_size",
117 
118 void reportPreambleBuild(const PreambleBuildStats &Stats,
119  bool IsFirstPreamble) {
120  auto RecordWithLabel = [&Stats](llvm::StringRef Label) {
121  PreambleBuildFilesystemLatency.record(Stats.FileSystemTime, Label);
122  if (Stats.TotalBuildTime > 0) // Avoid division by zero.
123  PreambleBuildFilesystemLatencyRatio.record(
124  Stats.FileSystemTime / Stats.TotalBuildTime, Label);
125  };
126 
127  static llvm::once_flag OnceFlag;
128  llvm::call_once(OnceFlag, [&] { RecordWithLabel("first_build"); });
129  RecordWithLabel(IsFirstPreamble ? "first_build_for_file" : "rebuild");
130 
131  PreambleBuildSize.record(Stats.BuildSize);
132  PreambleSerializedSize.record(Stats.SerializedSize);
133 }
134 
135 class ASTWorker;
136 } // namespace
137 
139 
140 llvm::Optional<llvm::StringRef> TUScheduler::getFileBeingProcessedInContext() {
141  if (auto *File = Context::current().get(FileBeingProcessed))
142  return llvm::StringRef(*File);
143  return None;
144 }
145 
146 /// An LRU cache of idle ASTs.
147 /// Because we want to limit the overall number of these we retain, the cache
148 /// owns ASTs (and may evict them) while their workers are idle.
149 /// Workers borrow ASTs when active, and return them when done.
151 public:
152  using Key = const ASTWorker *;
153 
154  ASTCache(unsigned MaxRetainedASTs) : MaxRetainedASTs(MaxRetainedASTs) {}
155 
156  /// Returns result of getUsedBytes() for the AST cached by \p K.
157  /// If no AST is cached, 0 is returned.
158  std::size_t getUsedBytes(Key K) {
159  std::lock_guard<std::mutex> Lock(Mut);
160  auto It = findByKey(K);
161  if (It == LRU.end() || !It->second)
162  return 0;
163  return It->second->getUsedBytes();
164  }
165 
166  /// Store the value in the pool, possibly removing the last used AST.
167  /// The value should not be in the pool when this function is called.
168  void put(Key K, std::unique_ptr<ParsedAST> V) {
169  std::unique_lock<std::mutex> Lock(Mut);
170  assert(findByKey(K) == LRU.end());
171 
172  LRU.insert(LRU.begin(), {K, std::move(V)});
173  if (LRU.size() <= MaxRetainedASTs)
174  return;
175  // We're past the limit, remove the last element.
176  std::unique_ptr<ParsedAST> ForCleanup = std::move(LRU.back().second);
177  LRU.pop_back();
178  // Run the expensive destructor outside the lock.
179  Lock.unlock();
180  ForCleanup.reset();
181  }
182 
183  /// Returns the cached value for \p K, or llvm::None if the value is not in
184  /// the cache anymore. If nullptr was cached for \p K, this function will
185  /// return a null unique_ptr wrapped into an optional.
186  /// If \p AccessMetric is set records whether there was a hit or miss.
187  llvm::Optional<std::unique_ptr<ParsedAST>>
188  take(Key K, const trace::Metric *AccessMetric = nullptr) {
189  // Record metric after unlocking the mutex.
190  std::unique_lock<std::mutex> Lock(Mut);
191  auto Existing = findByKey(K);
192  if (Existing == LRU.end()) {
193  if (AccessMetric)
194  AccessMetric->record(1, "miss");
195  return None;
196  }
197  if (AccessMetric)
198  AccessMetric->record(1, "hit");
199  std::unique_ptr<ParsedAST> V = std::move(Existing->second);
200  LRU.erase(Existing);
201  // GCC 4.8 fails to compile `return V;`, as it tries to call the copy
202  // constructor of unique_ptr, so we call the move ctor explicitly to avoid
203  // this miscompile.
204  return llvm::Optional<std::unique_ptr<ParsedAST>>(std::move(V));
205  }
206 
207 private:
208  using KVPair = std::pair<Key, std::unique_ptr<ParsedAST>>;
209 
210  std::vector<KVPair>::iterator findByKey(Key K) {
211  return llvm::find_if(LRU, [K](const KVPair &P) { return P.first == K; });
212  }
213 
214  std::mutex Mut;
215  unsigned MaxRetainedASTs;
216  /// Items sorted in LRU order, i.e. first item is the most recently accessed
217  /// one.
218  std::vector<KVPair> LRU; /* GUARDED_BY(Mut) */
219 };
220 
221 /// A map from header files to an opened "proxy" file that includes them.
222 /// If you open the header, the compile command from the proxy file is used.
223 ///
224 /// This inclusion information could also naturally live in the index, but there
225 /// are advantages to using open files instead:
226 /// - it's easier to achieve a *stable* choice of proxy, which is important
227 /// to avoid invalidating the preamble
228 /// - context-sensitive flags for libraries with multiple configurations
229 /// (e.g. C++ stdlib sensitivity to -std version)
230 /// - predictable behavior, e.g. guarantees that go-to-def landing on a header
231 /// will have a suitable command available
232 /// - fewer scaling problems to solve (project include graphs are big!)
233 ///
234 /// Implementation details:
235 /// - We only record this for mainfiles where the command was trustworthy
236 /// (i.e. not inferred). This avoids a bad inference "infecting" other files.
237 /// - Once we've picked a proxy file for a header, we stick with it until the
238 /// proxy file is invalidated *and* a new candidate proxy file is built.
239 /// Switching proxies is expensive, as the compile flags will (probably)
240 /// change and therefore we'll end up rebuilding the header's preamble.
241 /// - We don't capture the actual compile command, but just the filename we
242 /// should query to get it. This avoids getting out of sync with the CDB.
243 ///
244 /// All methods are threadsafe. In practice, update() comes from preamble
245 /// threads, remove()s mostly from the main thread, and get() from ASTWorker.
246 /// Writes are rare and reads are cheap, so we don't expect much contention.
248  // We should be be a little careful how we store the include graph of open
249  // files, as each can have a large number of transitive headers.
250  // This representation is O(unique transitive source files).
251  llvm::BumpPtrAllocator Arena;
252  struct Association {
253  llvm::StringRef MainFile;
254  // Circular-linked-list of associations with the same mainFile.
255  // Null indicates that the mainfile was removed.
256  Association *Next;
257  };
258  llvm::StringMap<Association, llvm::BumpPtrAllocator &> HeaderToMain;
259  llvm::StringMap<Association *, llvm::BumpPtrAllocator &> MainToFirst;
260  std::atomic<size_t> UsedBytes; // Updated after writes.
261  mutable std::mutex Mu;
262 
263  void invalidate(Association *First) {
264  Association *Current = First;
265  do {
266  Association *Next = Current->Next;
267  Current->Next = nullptr;
268  Current = Next;
269  } while (Current != First);
270  }
271 
272  // Create the circular list and return the head of it.
273  Association *associate(llvm::StringRef MainFile,
274  llvm::ArrayRef<std::string> Headers) {
275  Association *First = nullptr, *Prev = nullptr;
276  for (const std::string &Header : Headers) {
277  auto &Assoc = HeaderToMain[Header];
278  if (Assoc.Next)
279  continue; // Already has a valid association.
280 
281  Assoc.MainFile = MainFile;
282  Assoc.Next = Prev;
283  Prev = &Assoc;
284  if (!First)
285  First = &Assoc;
286  }
287  if (First)
288  First->Next = Prev;
289  return First;
290  }
291 
292  void updateMemoryUsage() {
293  auto StringMapHeap = [](const auto &Map) {
294  // StringMap stores the hashtable on the heap.
295  // It contains pointers to the entries, and a hashcode for each.
296  return Map.getNumBuckets() * (sizeof(void *) + sizeof(unsigned));
297  };
298  size_t Usage = Arena.getTotalMemory() + StringMapHeap(MainToFirst) +
299  StringMapHeap(HeaderToMain) + sizeof(*this);
300  UsedBytes.store(Usage, std::memory_order_release);
301  }
302 
303 public:
304  HeaderIncluderCache() : HeaderToMain(Arena), MainToFirst(Arena) {
305  updateMemoryUsage();
306  }
307 
308  // Associate each header with MainFile (unless already associated).
309  // Headers not in the list will have their associations removed.
310  void update(PathRef MainFile, llvm::ArrayRef<std::string> Headers) {
311  std::lock_guard<std::mutex> Lock(Mu);
312  auto It = MainToFirst.try_emplace(MainFile, nullptr);
313  Association *&First = It.first->second;
314  if (First)
315  invalidate(First);
316  First = associate(It.first->first(), Headers);
317  updateMemoryUsage();
318  }
319 
320  // Mark MainFile as gone.
321  // This will *not* disassociate headers with MainFile immediately, but they
322  // will be eligible for association with other files that get update()d.
324  std::lock_guard<std::mutex> Lock(Mu);
325  Association *&First = MainToFirst[MainFile];
326  if (First) {
327  invalidate(First);
328  First = nullptr;
329  }
330  // MainToFirst entry should stay alive, as Associations might be pointing at
331  // its key.
332  }
333 
334  /// Get the mainfile associated with Header, or the empty string if none.
335  std::string get(PathRef Header) const {
336  std::lock_guard<std::mutex> Lock(Mu);
337  return HeaderToMain.lookup(Header).MainFile.str();
338  }
339 
340  size_t getUsedBytes() const {
341  return UsedBytes.load(std::memory_order_acquire);
342  }
343 };
344 
345 namespace {
346 
347 bool isReliable(const tooling::CompileCommand &Cmd) {
348  return Cmd.Heuristic.empty();
349 }
350 
351 /// Threadsafe manager for updating a TUStatus and emitting it after each
352 /// update.
353 class SynchronizedTUStatus {
354 public:
355  SynchronizedTUStatus(PathRef FileName, ParsingCallbacks &Callbacks)
356  : FileName(FileName), Callbacks(Callbacks) {}
357 
358  void update(llvm::function_ref<void(TUStatus &)> Mutator) {
359  std::lock_guard<std::mutex> Lock(StatusMu);
360  Mutator(Status);
361  emitStatusLocked();
362  }
363 
364  /// Prevents emitting of further updates.
365  void stop() {
366  std::lock_guard<std::mutex> Lock(StatusMu);
367  CanPublish = false;
368  }
369 
370 private:
371  void emitStatusLocked() {
372  if (CanPublish)
373  Callbacks.onFileUpdated(FileName, Status);
374  }
375 
376  const Path FileName;
377 
378  std::mutex StatusMu;
379  TUStatus Status;
380  bool CanPublish = true;
381  ParsingCallbacks &Callbacks;
382 };
383 
384 /// Responsible for building preambles. Whenever the thread is idle and the
385 /// preamble is outdated, it starts to build a fresh preamble from the latest
386 /// inputs. If RunSync is true, preambles are built synchronously in update()
387 /// instead.
388 class PreambleThread {
389 public:
390  PreambleThread(llvm::StringRef FileName, ParsingCallbacks &Callbacks,
391  bool StorePreambleInMemory, bool RunSync,
392  SynchronizedTUStatus &Status,
393  TUScheduler::HeaderIncluderCache &HeaderIncluders,
394  ASTWorker &AW)
395  : FileName(FileName), Callbacks(Callbacks),
396  StoreInMemory(StorePreambleInMemory), RunSync(RunSync), Status(Status),
397  ASTPeer(AW), HeaderIncluders(HeaderIncluders) {}
398 
399  /// It isn't guaranteed that each requested version will be built. If there
400  /// are multiple update requests while building a preamble, only the last one
401  /// will be built.
402  void update(std::unique_ptr<CompilerInvocation> CI, ParseInputs PI,
403  std::vector<Diag> CIDiags, WantDiagnostics WantDiags) {
404  Request Req = {std::move(CI), std::move(PI), std::move(CIDiags), WantDiags,
405  Context::current().clone()};
406  if (RunSync) {
407  build(std::move(Req));
408  Status.update([](TUStatus &Status) {
409  Status.PreambleActivity = PreambleAction::Idle;
410  });
411  return;
412  }
413  {
414  std::unique_lock<std::mutex> Lock(Mutex);
415  // If NextReq was requested with WantDiagnostics::Yes we cannot just drop
416  // that on the floor. Block until we start building it. This won't
417  // dead-lock as we are blocking the caller thread, while builds continue
418  // on preamble thread.
419  ReqCV.wait(Lock, [this] {
420  return !NextReq || NextReq->WantDiags != WantDiagnostics::Yes;
421  });
422  NextReq = std::move(Req);
423  }
424  // Let the worker thread know there's a request, notify_one is safe as there
425  // should be a single worker thread waiting on it.
426  ReqCV.notify_all();
427  }
428 
429  void run() {
430  while (true) {
431  {
432  std::unique_lock<std::mutex> Lock(Mutex);
433  assert(!CurrentReq && "Already processing a request?");
434  // Wait until stop is called or there is a request.
435  ReqCV.wait(Lock, [this] { return NextReq || Done; });
436  if (Done)
437  break;
438  CurrentReq = std::move(*NextReq);
439  NextReq.reset();
440  }
441 
442  {
443  WithContext Guard(std::move(CurrentReq->Ctx));
444  // Note that we don't make use of the ContextProvider here.
445  // Preamble tasks are always scheduled by ASTWorker tasks, and we
446  // reuse the context/config that was created at that level.
447 
448  // Build the preamble and let the waiters know about it.
449  build(std::move(*CurrentReq));
450  }
451  bool IsEmpty = false;
452  {
453  std::lock_guard<std::mutex> Lock(Mutex);
454  CurrentReq.reset();
455  IsEmpty = !NextReq;
456  }
457  if (IsEmpty) {
458  // We don't perform this above, before waiting for a request to make
459  // tests more deterministic. As there can be a race between this thread
460  // and client thread(clangdserver).
461  Status.update([](TUStatus &Status) {
462  Status.PreambleActivity = PreambleAction::Idle;
463  });
464  }
465  ReqCV.notify_all();
466  }
467  dlog("Preamble worker for {0} stopped", FileName);
468  }
469 
470  /// Signals the run loop to exit.
471  void stop() {
472  dlog("Preamble worker for {0} received stop", FileName);
473  {
474  std::lock_guard<std::mutex> Lock(Mutex);
475  Done = true;
476  NextReq.reset();
477  }
478  // Let the worker thread know that it should stop.
479  ReqCV.notify_all();
480  }
481 
482  bool blockUntilIdle(Deadline Timeout) const {
483  std::unique_lock<std::mutex> Lock(Mutex);
484  return wait(Lock, ReqCV, Timeout, [&] { return !NextReq && !CurrentReq; });
485  }
486 
487 private:
488  /// Holds inputs required for building a preamble. CI is guaranteed to be
489  /// non-null.
490  struct Request {
491  std::unique_ptr<CompilerInvocation> CI;
492  ParseInputs Inputs;
493  std::vector<Diag> CIDiags;
495  Context Ctx;
496  };
497 
498  bool isDone() {
499  std::lock_guard<std::mutex> Lock(Mutex);
500  return Done;
501  }
502 
503  /// Builds a preamble for \p Req, might reuse LatestBuild if possible.
504  /// Notifies ASTWorker after build finishes.
505  void build(Request Req);
506 
507  mutable std::mutex Mutex;
508  bool Done = false; /* GUARDED_BY(Mutex) */
509  llvm::Optional<Request> NextReq; /* GUARDED_BY(Mutex) */
510  llvm::Optional<Request> CurrentReq; /* GUARDED_BY(Mutex) */
511  // Signaled whenever a thread populates NextReq or worker thread builds a
512  // Preamble.
513  mutable std::condition_variable ReqCV; /* GUARDED_BY(Mutex) */
514  // Accessed only by preamble thread.
515  std::shared_ptr<const PreambleData> LatestBuild;
516 
517  const Path FileName;
518  ParsingCallbacks &Callbacks;
519  const bool StoreInMemory;
520  const bool RunSync;
521 
522  SynchronizedTUStatus &Status;
523  ASTWorker &ASTPeer;
524  TUScheduler::HeaderIncluderCache &HeaderIncluders;
525 };
526 
527 class ASTWorkerHandle;
528 
529 /// Owns one instance of the AST, schedules updates and reads of it.
530 /// Also responsible for building and providing access to the preamble.
531 /// Each ASTWorker processes the async requests sent to it on a separate
532 /// dedicated thread.
533 /// The ASTWorker that manages the AST is shared by both the processing thread
534 /// and the TUScheduler. The TUScheduler should discard an ASTWorker when
535 /// remove() is called, but its thread may be busy and we don't want to block.
536 /// So the workers are accessed via an ASTWorkerHandle. Destroying the handle
537 /// signals the worker to exit its run loop and gives up shared ownership of the
538 /// worker.
539 class ASTWorker {
540  friend class ASTWorkerHandle;
541  ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
542  TUScheduler::ASTCache &LRUCache,
543  TUScheduler::HeaderIncluderCache &HeaderIncluders,
544  Semaphore &Barrier, bool RunSync, const TUScheduler::Options &Opts,
545  ParsingCallbacks &Callbacks);
546 
547 public:
548  /// Create a new ASTWorker and return a handle to it.
549  /// The processing thread is spawned using \p Tasks. However, when \p Tasks
550  /// is null, all requests will be processed on the calling thread
551  /// synchronously instead. \p Barrier is acquired when processing each
552  /// request, it is used to limit the number of actively running threads.
553  static ASTWorkerHandle
554  create(PathRef FileName, const GlobalCompilationDatabase &CDB,
555  TUScheduler::ASTCache &IdleASTs,
556  TUScheduler::HeaderIncluderCache &HeaderIncluders,
557  AsyncTaskRunner *Tasks, Semaphore &Barrier,
558  const TUScheduler::Options &Opts, ParsingCallbacks &Callbacks);
559  ~ASTWorker();
560 
561  void update(ParseInputs Inputs, WantDiagnostics, bool ContentChanged);
562  void
563  runWithAST(llvm::StringRef Name,
564  llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
566  bool blockUntilIdle(Deadline Timeout) const;
567 
568  std::shared_ptr<const PreambleData> getPossiblyStalePreamble(
569  std::shared_ptr<const ASTSignals> *ASTSignals = nullptr) const;
570 
571  /// Used to inform ASTWorker about a new preamble build by PreambleThread.
572  /// Diagnostics are only published through this callback. This ensures they
573  /// are always for newer versions of the file, as the callback gets called in
574  /// the same order as update requests.
575  void updatePreamble(std::unique_ptr<CompilerInvocation> CI, ParseInputs PI,
576  std::shared_ptr<const PreambleData> Preamble,
577  std::vector<Diag> CIDiags, WantDiagnostics WantDiags);
578 
579  /// Obtain a preamble reflecting all updates so far. Threadsafe.
580  /// It may be delivered immediately, or later on the worker thread.
581  void getCurrentPreamble(
582  llvm::unique_function<void(std::shared_ptr<const PreambleData>)>);
583  /// Returns compile command from the current file inputs.
584  tooling::CompileCommand getCurrentCompileCommand() const;
585 
586  /// Wait for the first build of preamble to finish. Preamble itself can be
587  /// accessed via getPossiblyStalePreamble(). Note that this function will
588  /// return after an unsuccessful build of the preamble too, i.e. result of
589  /// getPossiblyStalePreamble() can be null even after this function returns.
590  void waitForFirstPreamble() const;
591 
592  TUScheduler::FileStats stats() const;
593  bool isASTCached() const;
594 
595 private:
596  // Details of an update request that are relevant to scheduling.
597  struct UpdateType {
598  // Do we want diagnostics from this version?
599  // If Yes, we must always build this version.
600  // If No, we only need to build this version if it's read.
601  // If Auto, we build if it's read or if the debounce expires.
603  // Did the main-file content of the document change?
604  // If so, we're allowed to cancel certain invalidated preceding reads.
606  };
607 
608  /// Publishes diagnostics for \p Inputs. It will build an AST or reuse the
609  /// cached one if applicable. Assumes LatestPreamble is compatible for \p
610  /// Inputs.
611  void generateDiagnostics(std::unique_ptr<CompilerInvocation> Invocation,
612  ParseInputs Inputs, std::vector<Diag> CIDiags);
613 
614  void updateASTSignals(ParsedAST &AST);
615 
616  // Must be called exactly once on processing thread. Will return after
617  // stop() is called on a separate thread and all pending requests are
618  // processed.
619  void run();
620  /// Signal that run() should finish processing pending requests and exit.
621  void stop();
622 
623  /// Adds a new task to the end of the request queue.
624  void startTask(llvm::StringRef Name, llvm::unique_function<void()> Task,
625  llvm::Optional<UpdateType> Update,
627  /// Runs a task synchronously.
628  void runTask(llvm::StringRef Name, llvm::function_ref<void()> Task);
629 
630  /// Determines the next action to perform.
631  /// All actions that should never run are discarded.
632  /// Returns a deadline for the next action. If it's expired, run now.
633  /// scheduleLocked() is called again at the deadline, or if requests arrive.
634  Deadline scheduleLocked();
635  /// Should the first task in the queue be skipped instead of run?
636  bool shouldSkipHeadLocked() const;
637 
638  struct Request {
639  llvm::unique_function<void()> Action;
640  std::string Name;
641  steady_clock::time_point AddTime;
642  Context Ctx;
643  llvm::Optional<Context> QueueCtx;
644  llvm::Optional<UpdateType> Update;
647  };
648 
649  /// Handles retention of ASTs.
650  TUScheduler::ASTCache &IdleASTs;
651  TUScheduler::HeaderIncluderCache &HeaderIncluders;
652  const bool RunSync;
653  /// Time to wait after an update to see whether another update obsoletes it.
654  const DebouncePolicy UpdateDebounce;
655  /// File that ASTWorker is responsible for.
656  const Path FileName;
657  /// Callback to create processing contexts for tasks.
658  const std::function<Context(llvm::StringRef)> ContextProvider;
659  const GlobalCompilationDatabase &CDB;
660  /// Callback invoked when preamble or main file AST is built.
661  ParsingCallbacks &Callbacks;
662 
663  Semaphore &Barrier;
664  /// Whether the 'onMainAST' callback ran for the current FileInputs.
665  bool RanASTCallback = false;
666  /// Guards members used by both TUScheduler and the worker thread.
667  mutable std::mutex Mutex;
668  /// File inputs, currently being used by the worker.
669  /// Writes and reads from unknown threads are locked. Reads from the worker
670  /// thread are not locked, as it's the only writer.
671  ParseInputs FileInputs; /* GUARDED_BY(Mutex) */
672  /// Times of recent AST rebuilds, used for UpdateDebounce computation.
673  llvm::SmallVector<DebouncePolicy::clock::duration>
674  RebuildTimes; /* GUARDED_BY(Mutex) */
675  /// Set to true to signal run() to finish processing.
676  bool Done; /* GUARDED_BY(Mutex) */
677  std::deque<Request> Requests; /* GUARDED_BY(Mutex) */
678  llvm::Optional<Request> CurrentRequest; /* GUARDED_BY(Mutex) */
679  /// Signalled whenever a new request has been scheduled or processing of a
680  /// request has completed.
681  mutable std::condition_variable RequestsCV;
682  std::shared_ptr<const ASTSignals> LatestASTSignals; /* GUARDED_BY(Mutex) */
683  /// Latest build preamble for current TU.
684  /// None means no builds yet, null means there was an error while building.
685  /// Only written by ASTWorker's thread.
686  llvm::Optional<std::shared_ptr<const PreambleData>> LatestPreamble;
687  std::deque<Request> PreambleRequests; /* GUARDED_BY(Mutex) */
688  /// Signaled whenever LatestPreamble changes state or there's a new
689  /// PreambleRequest.
690  mutable std::condition_variable PreambleCV;
691  /// Guards the callback that publishes results of AST-related computations
692  /// (diagnostics) and file statuses.
693  std::mutex PublishMu;
694  // Used to prevent remove document + add document races that lead to
695  // out-of-order callbacks for publishing results of onMainAST callback.
696  //
697  // The lifetime of the old/new ASTWorkers will overlap, but their handles
698  // don't. When the old handle is destroyed, the old worker will stop reporting
699  // any results to the user.
700  bool CanPublishResults = true; /* GUARDED_BY(PublishMu) */
701  std::atomic<unsigned> ASTBuildCount = {0};
702  std::atomic<unsigned> PreambleBuildCount = {0};
703 
704  SynchronizedTUStatus Status;
705  PreambleThread PreamblePeer;
706 };
707 
708 /// A smart-pointer-like class that points to an active ASTWorker.
709 /// In destructor, signals to the underlying ASTWorker that no new requests will
710 /// be sent and the processing loop may exit (after running all pending
711 /// requests).
712 class ASTWorkerHandle {
713  friend class ASTWorker;
714  ASTWorkerHandle(std::shared_ptr<ASTWorker> Worker)
715  : Worker(std::move(Worker)) {
716  assert(this->Worker);
717  }
718 
719 public:
720  ASTWorkerHandle(const ASTWorkerHandle &) = delete;
721  ASTWorkerHandle &operator=(const ASTWorkerHandle &) = delete;
722  ASTWorkerHandle(ASTWorkerHandle &&) = default;
723  ASTWorkerHandle &operator=(ASTWorkerHandle &&) = default;
724 
725  ~ASTWorkerHandle() {
726  if (Worker)
727  Worker->stop();
728  }
729 
730  ASTWorker &operator*() {
731  assert(Worker && "Handle was moved from");
732  return *Worker;
733  }
734 
735  ASTWorker *operator->() {
736  assert(Worker && "Handle was moved from");
737  return Worker.get();
738  }
739 
740  /// Returns an owning reference to the underlying ASTWorker that can outlive
741  /// the ASTWorkerHandle. However, no new requests to an active ASTWorker can
742  /// be schedule via the returned reference, i.e. only reads of the preamble
743  /// are possible.
744  std::shared_ptr<const ASTWorker> lock() { return Worker; }
745 
746 private:
747  std::shared_ptr<ASTWorker> Worker;
748 };
749 
750 ASTWorkerHandle
751 ASTWorker::create(PathRef FileName, const GlobalCompilationDatabase &CDB,
752  TUScheduler::ASTCache &IdleASTs,
753  TUScheduler::HeaderIncluderCache &HeaderIncluders,
754  AsyncTaskRunner *Tasks, Semaphore &Barrier,
755  const TUScheduler::Options &Opts,
756  ParsingCallbacks &Callbacks) {
757  std::shared_ptr<ASTWorker> Worker(
758  new ASTWorker(FileName, CDB, IdleASTs, HeaderIncluders, Barrier,
759  /*RunSync=*/!Tasks, Opts, Callbacks));
760  if (Tasks) {
761  Tasks->runAsync("ASTWorker:" + llvm::sys::path::filename(FileName),
762  [Worker]() { Worker->run(); });
763  Tasks->runAsync("PreambleWorker:" + llvm::sys::path::filename(FileName),
764  [Worker]() { Worker->PreamblePeer.run(); });
765  }
766 
767  return ASTWorkerHandle(std::move(Worker));
768 }
769 
770 ASTWorker::ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
771  TUScheduler::ASTCache &LRUCache,
772  TUScheduler::HeaderIncluderCache &HeaderIncluders,
773  Semaphore &Barrier, bool RunSync,
774  const TUScheduler::Options &Opts,
775  ParsingCallbacks &Callbacks)
776  : IdleASTs(LRUCache), HeaderIncluders(HeaderIncluders), RunSync(RunSync),
777  UpdateDebounce(Opts.UpdateDebounce), FileName(FileName),
778  ContextProvider(Opts.ContextProvider), CDB(CDB), Callbacks(Callbacks),
779  Barrier(Barrier), Done(false), Status(FileName, Callbacks),
780  PreamblePeer(FileName, Callbacks, Opts.StorePreamblesInMemory, RunSync,
781  Status, HeaderIncluders, *this) {
782  // Set a fallback command because compile command can be accessed before
783  // `Inputs` is initialized. Other fields are only used after initialization
784  // from client inputs.
785  FileInputs.CompileCommand = CDB.getFallbackCommand(FileName);
786 }
787 
788 ASTWorker::~ASTWorker() {
789  // Make sure we remove the cached AST, if any.
790  IdleASTs.take(this);
791 #ifndef NDEBUG
792  std::lock_guard<std::mutex> Lock(Mutex);
793  assert(Done && "handle was not destroyed");
794  assert(Requests.empty() && !CurrentRequest &&
795  "unprocessed requests when destroying ASTWorker");
796 #endif
797 }
798 
799 void ASTWorker::update(ParseInputs Inputs, WantDiagnostics WantDiags,
800  bool ContentChanged) {
801  llvm::StringLiteral TaskName = "Update";
802  auto Task = [=]() mutable {
803  // Get the actual command as `Inputs` does not have a command.
804  // FIXME: some build systems like Bazel will take time to preparing
805  // environment to build the file, it would be nice if we could emit a
806  // "PreparingBuild" status to inform users, it is non-trivial given the
807  // current implementation.
808  auto Cmd = CDB.getCompileCommand(FileName);
809  // If we don't have a reliable command for this file, it may be a header.
810  // Try to find a file that includes it, to borrow its command.
811  if (!Cmd || !isReliable(*Cmd)) {
812  std::string ProxyFile = HeaderIncluders.get(FileName);
813  if (!ProxyFile.empty()) {
814  auto ProxyCmd = CDB.getCompileCommand(ProxyFile);
815  if (!ProxyCmd || !isReliable(*ProxyCmd)) {
816  // This command is supposed to be reliable! It's probably gone.
817  HeaderIncluders.remove(ProxyFile);
818  } else {
819  // We have a reliable command for an including file, use it.
820  Cmd = tooling::transferCompileCommand(std::move(*ProxyCmd), FileName);
821  }
822  }
823  }
824  if (Cmd)
825  Inputs.CompileCommand = std::move(*Cmd);
826  else
827  Inputs.CompileCommand = CDB.getFallbackCommand(FileName);
828 
829  bool InputsAreTheSame =
830  std::tie(FileInputs.CompileCommand, FileInputs.Contents) ==
831  std::tie(Inputs.CompileCommand, Inputs.Contents);
832  // Cached AST is invalidated.
833  if (!InputsAreTheSame) {
834  IdleASTs.take(this);
835  RanASTCallback = false;
836  }
837 
838  // Update current inputs so that subsequent reads can see them.
839  {
840  std::lock_guard<std::mutex> Lock(Mutex);
841  FileInputs = Inputs;
842  }
843 
844  log("ASTWorker building file {0} version {1} with command {2}\n[{3}]\n{4}",
845  FileName, Inputs.Version, Inputs.CompileCommand.Heuristic,
846  Inputs.CompileCommand.Directory,
847  printArgv(Inputs.CompileCommand.CommandLine));
848 
849  StoreDiags CompilerInvocationDiagConsumer;
850  std::vector<std::string> CC1Args;
851  std::unique_ptr<CompilerInvocation> Invocation = buildCompilerInvocation(
852  Inputs, CompilerInvocationDiagConsumer, &CC1Args);
853  // Log cc1 args even (especially!) if creating invocation failed.
854  if (!CC1Args.empty())
855  vlog("Driver produced command: cc1 {0}", printArgv(CC1Args));
856  std::vector<Diag> CompilerInvocationDiags =
857  CompilerInvocationDiagConsumer.take();
858  if (!Invocation) {
859  elog("Could not build CompilerInvocation for file {0}", FileName);
860  // Remove the old AST if it's still in cache.
861  IdleASTs.take(this);
862  RanASTCallback = false;
863  // Report the diagnostics we collected when parsing the command line.
864  Callbacks.onFailedAST(FileName, Inputs.Version,
865  std::move(CompilerInvocationDiags),
866  [&](llvm::function_ref<void()> Publish) {
867  // Ensure we only publish results from the worker
868  // if the file was not removed, making sure there
869  // are not race conditions.
870  std::lock_guard<std::mutex> Lock(PublishMu);
871  if (CanPublishResults)
872  Publish();
873  });
874  // Note that this might throw away a stale preamble that might still be
875  // useful, but this is how we communicate a build error.
876  LatestPreamble.emplace();
877  // Make sure anyone waiting for the preamble gets notified it could not be
878  // built.
879  PreambleCV.notify_all();
880  return;
881  }
882 
883  PreamblePeer.update(std::move(Invocation), std::move(Inputs),
884  std::move(CompilerInvocationDiags), WantDiags);
885  std::unique_lock<std::mutex> Lock(Mutex);
886  PreambleCV.wait(Lock, [this] {
887  // Block until we reiceve a preamble request, unless a preamble already
888  // exists, as patching an empty preamble would imply rebuilding it from
889  // scratch.
890  // We block here instead of the consumer to prevent any deadlocks. Since
891  // LatestPreamble is only populated by ASTWorker thread.
892  return LatestPreamble || !PreambleRequests.empty() || Done;
893  });
894  };
895  startTask(TaskName, std::move(Task), UpdateType{WantDiags, ContentChanged},
896  TUScheduler::NoInvalidation);
897 }
898 
899 void ASTWorker::runWithAST(
900  llvm::StringRef Name,
901  llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
902  TUScheduler::ASTActionInvalidation Invalidation) {
903  // Tracks ast cache accesses for read operations.
904  static constexpr trace::Metric ASTAccessForRead(
905  "ast_access_read", trace::Metric::Counter, "result");
906  auto Task = [=, Action = std::move(Action)]() mutable {
907  if (auto Reason = isCancelled())
908  return Action(llvm::make_error<CancelledError>(Reason));
909  llvm::Optional<std::unique_ptr<ParsedAST>> AST =
910  IdleASTs.take(this, &ASTAccessForRead);
911  if (!AST) {
912  StoreDiags CompilerInvocationDiagConsumer;
913  std::unique_ptr<CompilerInvocation> Invocation =
914  buildCompilerInvocation(FileInputs, CompilerInvocationDiagConsumer);
915  // Try rebuilding the AST.
916  vlog("ASTWorker rebuilding evicted AST to run {0}: {1} version {2}", Name,
917  FileName, FileInputs.Version);
918  // FIXME: We might need to build a patched ast once preamble thread starts
919  // running async. Currently getPossiblyStalePreamble below will always
920  // return a compatible preamble as ASTWorker::update blocks.
921  llvm::Optional<ParsedAST> NewAST;
922  if (Invocation) {
923  NewAST = ParsedAST::build(FileName, FileInputs, std::move(Invocation),
924  CompilerInvocationDiagConsumer.take(),
925  getPossiblyStalePreamble());
926  ++ASTBuildCount;
927  }
928  AST = NewAST ? std::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;
929  }
930  // Make sure we put the AST back into the LRU cache.
931  auto _ = llvm::make_scope_exit(
932  [&AST, this]() { IdleASTs.put(this, std::move(*AST)); });
933  // Run the user-provided action.
934  if (!*AST)
935  return Action(error(llvm::errc::invalid_argument, "invalid AST"));
936  vlog("ASTWorker running {0} on version {2} of {1}", Name, FileName,
937  FileInputs.Version);
938  Action(InputsAndAST{FileInputs, **AST});
939  };
940  startTask(Name, std::move(Task), /*Update=*/None, Invalidation);
941 }
942 
943 /// To be called from ThreadCrashReporter's signal handler.
944 static void crashDumpCompileCommand(llvm::raw_ostream &OS,
945  const tooling::CompileCommand &Command) {
946  OS << " Filename: " << Command.Filename << "\n";
947  OS << " Directory: " << Command.Directory << "\n";
948  OS << " Command Line:";
949  for (auto &Arg : Command.CommandLine) {
950  OS << " " << Arg;
951  }
952  OS << "\n";
953 }
954 
955 /// To be called from ThreadCrashReporter's signal handler.
956 static void crashDumpFileContents(llvm::raw_ostream &OS,
957  const std::string &Contents) {
958  // Avoid flooding the terminal with source code by default, but allow clients
959  // to opt in. Use an env var to preserve backwards compatibility of the
960  // command line interface, while allowing it to be set outside the clangd
961  // launch site for more flexibility.
962  if (getenv("CLANGD_CRASH_DUMP_SOURCE")) {
963  OS << " Contents:\n";
964  OS << Contents << "\n";
965  }
966 }
967 
968 /// To be called from ThreadCrashReporter's signal handler.
969 static void crashDumpParseInputs(llvm::raw_ostream &OS,
970  const ParseInputs &FileInputs) {
971  auto &Command = FileInputs.CompileCommand;
972  crashDumpCompileCommand(OS, Command);
973  OS << " Version: " << FileInputs.Version << "\n";
974  crashDumpFileContents(OS, FileInputs.Contents);
975 }
976 
977 void PreambleThread::build(Request Req) {
978  assert(Req.CI && "Got preamble request with null compiler invocation");
979  const ParseInputs &Inputs = Req.Inputs;
980  bool ReusedPreamble = false;
981 
982  Status.update([&](TUStatus &Status) {
983  Status.PreambleActivity = PreambleAction::Building;
984  });
985  auto _ = llvm::make_scope_exit([this, &Req, &ReusedPreamble] {
986  ASTPeer.updatePreamble(std::move(Req.CI), std::move(Req.Inputs),
987  LatestBuild, std::move(Req.CIDiags),
988  std::move(Req.WantDiags));
989  if (!ReusedPreamble)
990  Callbacks.onPreamblePublished(FileName);
991  });
992 
993  if (!LatestBuild || Inputs.ForceRebuild) {
994  vlog("Building first preamble for {0} version {1}", FileName,
995  Inputs.Version);
996  } else if (isPreambleCompatible(*LatestBuild, Inputs, FileName, *Req.CI)) {
997  vlog("Reusing preamble version {0} for version {1} of {2}",
998  LatestBuild->Version, Inputs.Version, FileName);
999  ReusedPreamble = true;
1000  return;
1001  } else {
1002  vlog("Rebuilding invalidated preamble for {0} version {1} (previous was "
1003  "version {2})",
1004  FileName, Inputs.Version, LatestBuild->Version);
1005  }
1006 
1007  ThreadCrashReporter ScopedReporter([&Inputs]() {
1008  llvm::errs() << "Signalled while building preamble\n";
1009  crashDumpParseInputs(llvm::errs(), Inputs);
1010  });
1011 
1012  PreambleBuildStats Stats;
1013  bool IsFirstPreamble = !LatestBuild;
1014  LatestBuild = clang::clangd::buildPreamble(
1015  FileName, *Req.CI, Inputs, StoreInMemory,
1016  [&](ASTContext &Ctx, Preprocessor &PP,
1017  const CanonicalIncludes &CanonIncludes) {
1018  Callbacks.onPreambleAST(FileName, Inputs.Version, *Req.CI, Ctx, PP,
1019  CanonIncludes);
1020  },
1021  &Stats);
1022  if (!LatestBuild)
1023  return;
1024  reportPreambleBuild(Stats, IsFirstPreamble);
1025  if (isReliable(LatestBuild->CompileCommand))
1026  HeaderIncluders.update(FileName, LatestBuild->Includes.allHeaders());
1027 }
1028 
1029 void ASTWorker::updatePreamble(std::unique_ptr<CompilerInvocation> CI,
1030  ParseInputs PI,
1031  std::shared_ptr<const PreambleData> Preamble,
1032  std::vector<Diag> CIDiags,
1034  llvm::StringLiteral TaskName = "Build AST";
1035  // Store preamble and build diagnostics with new preamble if requested.
1036  auto Task = [this, Preamble = std::move(Preamble), CI = std::move(CI),
1037  PI = std::move(PI), CIDiags = std::move(CIDiags),
1038  WantDiags = std::move(WantDiags)]() mutable {
1039  // Update the preamble inside ASTWorker queue to ensure atomicity. As a task
1040  // running inside ASTWorker assumes internals won't change until it
1041  // finishes.
1042  if (!LatestPreamble || Preamble != *LatestPreamble) {
1043  ++PreambleBuildCount;
1044  // Cached AST is no longer valid.
1045  IdleASTs.take(this);
1046  RanASTCallback = false;
1047  std::lock_guard<std::mutex> Lock(Mutex);
1048  // LatestPreamble might be the last reference to old preamble, do not
1049  // trigger destructor while holding the lock.
1050  if (LatestPreamble)
1051  std::swap(*LatestPreamble, Preamble);
1052  else
1053  LatestPreamble = std::move(Preamble);
1054  }
1055  // Notify anyone waiting for a preamble.
1056  PreambleCV.notify_all();
1057  // Give up our ownership to old preamble before starting expensive AST
1058  // build.
1059  Preamble.reset();
1060  // We only need to build the AST if diagnostics were requested.
1061  if (WantDiags == WantDiagnostics::No)
1062  return;
1063  // Report diagnostics with the new preamble to ensure progress. Otherwise
1064  // diagnostics might get stale indefinitely if user keeps invalidating the
1065  // preamble.
1066  generateDiagnostics(std::move(CI), std::move(PI), std::move(CIDiags));
1067  };
1068  if (RunSync) {
1069  runTask(TaskName, Task);
1070  return;
1071  }
1072  {
1073  std::lock_guard<std::mutex> Lock(Mutex);
1074  PreambleRequests.push_back({std::move(Task), std::string(TaskName),
1075  steady_clock::now(), Context::current().clone(),
1076  llvm::None, llvm::None,
1077  TUScheduler::NoInvalidation, nullptr});
1078  }
1079  PreambleCV.notify_all();
1080  RequestsCV.notify_all();
1081 }
1082 
1083 void ASTWorker::updateASTSignals(ParsedAST &AST) {
1084  auto Signals = std::make_shared<const ASTSignals>(ASTSignals::derive(AST));
1085  // Existing readers of ASTSignals will have their copy preserved until the
1086  // read is completed. The last reader deletes the old ASTSignals.
1087  {
1088  std::lock_guard<std::mutex> Lock(Mutex);
1089  std::swap(LatestASTSignals, Signals);
1090  }
1091 }
1092 
1093 void ASTWorker::generateDiagnostics(
1094  std::unique_ptr<CompilerInvocation> Invocation, ParseInputs Inputs,
1095  std::vector<Diag> CIDiags) {
1096  // Tracks ast cache accesses for publishing diags.
1097  static constexpr trace::Metric ASTAccessForDiag(
1098  "ast_access_diag", trace::Metric::Counter, "result");
1099  assert(Invocation);
1100  assert(LatestPreamble);
1101  // No need to rebuild the AST if we won't send the diagnostics.
1102  {
1103  std::lock_guard<std::mutex> Lock(PublishMu);
1104  if (!CanPublishResults)
1105  return;
1106  }
1107  // Used to check whether we can update AST cache.
1108  bool InputsAreLatest =
1109  std::tie(FileInputs.CompileCommand, FileInputs.Contents) ==
1110  std::tie(Inputs.CompileCommand, Inputs.Contents);
1111  // Take a shortcut and don't report the diagnostics, since they should be the
1112  // same. All the clients should handle the lack of OnUpdated() call anyway to
1113  // handle empty result from buildAST.
1114  // FIXME: the AST could actually change if non-preamble includes changed,
1115  // but we choose to ignore it.
1116  if (InputsAreLatest && RanASTCallback)
1117  return;
1118 
1119  // Get the AST for diagnostics, either build it or use the cached one.
1120  std::string TaskName = llvm::formatv("Build AST ({0})", Inputs.Version);
1121  Status.update([&](TUStatus &Status) {
1122  Status.ASTActivity.K = ASTAction::Building;
1123  Status.ASTActivity.Name = std::move(TaskName);
1124  });
1125  // We might be able to reuse the last we've built for a read request.
1126  // FIXME: It might be better to not reuse this AST. That way queued AST builds
1127  // won't be required for diags.
1128  llvm::Optional<std::unique_ptr<ParsedAST>> AST =
1129  IdleASTs.take(this, &ASTAccessForDiag);
1130  if (!AST || !InputsAreLatest) {
1131  auto RebuildStartTime = DebouncePolicy::clock::now();
1132  llvm::Optional<ParsedAST> NewAST = ParsedAST::build(
1133  FileName, Inputs, std::move(Invocation), CIDiags, *LatestPreamble);
1134  auto RebuildDuration = DebouncePolicy::clock::now() - RebuildStartTime;
1135  ++ASTBuildCount;
1136  // Try to record the AST-build time, to inform future update debouncing.
1137  // This is best-effort only: if the lock is held, don't bother.
1138  std::unique_lock<std::mutex> Lock(Mutex, std::try_to_lock);
1139  if (Lock.owns_lock()) {
1140  // Do not let RebuildTimes grow beyond its small-size (i.e.
1141  // capacity).
1142  if (RebuildTimes.size() == RebuildTimes.capacity())
1143  RebuildTimes.erase(RebuildTimes.begin());
1144  RebuildTimes.push_back(RebuildDuration);
1145  Lock.unlock();
1146  }
1147  Status.update([&](TUStatus &Status) {
1148  Status.Details.ReuseAST = false;
1149  Status.Details.BuildFailed = !NewAST;
1150  });
1151  AST = NewAST ? std::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;
1152  } else {
1153  log("Skipping rebuild of the AST for {0}, inputs are the same.", FileName);
1154  Status.update([](TUStatus &Status) {
1155  Status.Details.ReuseAST = true;
1156  Status.Details.BuildFailed = false;
1157  });
1158  }
1159 
1160  // Publish diagnostics.
1161  auto RunPublish = [&](llvm::function_ref<void()> Publish) {
1162  // Ensure we only publish results from the worker if the file was not
1163  // removed, making sure there are not race conditions.
1164  std::lock_guard<std::mutex> Lock(PublishMu);
1165  if (CanPublishResults)
1166  Publish();
1167  };
1168  if (*AST) {
1169  trace::Span Span("Running main AST callback");
1170  Callbacks.onMainAST(FileName, **AST, RunPublish);
1171  updateASTSignals(**AST);
1172  } else {
1173  // Failed to build the AST, at least report diagnostics from the
1174  // command line if there were any.
1175  // FIXME: we might have got more errors while trying to build the
1176  // AST, surface them too.
1177  Callbacks.onFailedAST(FileName, Inputs.Version, CIDiags, RunPublish);
1178  }
1179 
1180  // AST might've been built for an older version of the source, as ASTWorker
1181  // queue raced ahead while we were waiting on the preamble. In that case the
1182  // queue can't reuse the AST.
1183  if (InputsAreLatest) {
1184  RanASTCallback = *AST != nullptr;
1185  IdleASTs.put(this, std::move(*AST));
1186  }
1187 }
1188 
1189 std::shared_ptr<const PreambleData> ASTWorker::getPossiblyStalePreamble(
1190  std::shared_ptr<const ASTSignals> *ASTSignals) const {
1191  std::lock_guard<std::mutex> Lock(Mutex);
1192  if (ASTSignals)
1193  *ASTSignals = LatestASTSignals;
1194  return LatestPreamble ? *LatestPreamble : nullptr;
1195 }
1196 
1197 void ASTWorker::waitForFirstPreamble() const {
1198  std::unique_lock<std::mutex> Lock(Mutex);
1199  PreambleCV.wait(Lock, [this] { return LatestPreamble || Done; });
1200 }
1201 
1202 tooling::CompileCommand ASTWorker::getCurrentCompileCommand() const {
1203  std::unique_lock<std::mutex> Lock(Mutex);
1204  return FileInputs.CompileCommand;
1205 }
1206 
1207 TUScheduler::FileStats ASTWorker::stats() const {
1208  TUScheduler::FileStats Result;
1209  Result.ASTBuilds = ASTBuildCount;
1210  Result.PreambleBuilds = PreambleBuildCount;
1211  // Note that we don't report the size of ASTs currently used for processing
1212  // the in-flight requests. We used this information for debugging purposes
1213  // only, so this should be fine.
1214  Result.UsedBytesAST = IdleASTs.getUsedBytes(this);
1215  if (auto Preamble = getPossiblyStalePreamble())
1216  Result.UsedBytesPreamble = Preamble->Preamble.getSize();
1217  return Result;
1218 }
1219 
1220 bool ASTWorker::isASTCached() const { return IdleASTs.getUsedBytes(this) != 0; }
1221 
1222 void ASTWorker::stop() {
1223  {
1224  std::lock_guard<std::mutex> Lock(PublishMu);
1225  CanPublishResults = false;
1226  }
1227  {
1228  std::lock_guard<std::mutex> Lock(Mutex);
1229  assert(!Done && "stop() called twice");
1230  Done = true;
1231  }
1232  PreamblePeer.stop();
1233  // We are no longer going to build any preambles, let the waiters know that.
1234  PreambleCV.notify_all();
1235  Status.stop();
1236  RequestsCV.notify_all();
1237 }
1238 
1239 void ASTWorker::runTask(llvm::StringRef Name, llvm::function_ref<void()> Task) {
1240  ThreadCrashReporter ScopedReporter([this, Name]() {
1241  llvm::errs() << "Signalled during AST worker action: " << Name << "\n";
1242  crashDumpParseInputs(llvm::errs(), FileInputs);
1243  });
1244  trace::Span Tracer(Name);
1245  WithContext WithProvidedContext(ContextProvider(FileName));
1246  Task();
1247 }
1248 
1249 void ASTWorker::startTask(llvm::StringRef Name,
1250  llvm::unique_function<void()> Task,
1251  llvm::Optional<UpdateType> Update,
1252  TUScheduler::ASTActionInvalidation Invalidation) {
1253  if (RunSync) {
1254  assert(!Done && "running a task after stop()");
1255  runTask(Name, Task);
1256  return;
1257  }
1258 
1259  {
1260  std::lock_guard<std::mutex> Lock(Mutex);
1261  assert(!Done && "running a task after stop()");
1262  // Cancel any requests invalidated by this request.
1263  if (Update && Update->ContentChanged) {
1264  for (auto &R : llvm::reverse(Requests)) {
1265  if (R.InvalidationPolicy == TUScheduler::InvalidateOnUpdate)
1266  R.Invalidate();
1267  if (R.Update && R.Update->ContentChanged)
1268  break; // Older requests were already invalidated by the older update.
1269  }
1270  }
1271 
1272  // Allow this request to be cancelled if invalidated.
1273  Context Ctx = Context::current().derive(FileBeingProcessed, FileName);
1274  Canceler Invalidate = nullptr;
1275  if (Invalidation) {
1276  WithContext WC(std::move(Ctx));
1277  std::tie(Ctx, Invalidate) = cancelableTask(
1278  /*Reason=*/static_cast<int>(ErrorCode::ContentModified));
1279  }
1280  // Trace the time the request spends in the queue, and the requests that
1281  // it's going to wait for.
1282  llvm::Optional<Context> QueueCtx;
1283  if (trace::enabled()) {
1284  // Tracers that follow threads and need strict nesting will see a tiny
1285  // instantaneous event "we're enqueueing", and sometime later it runs.
1286  WithContext WC(Ctx.clone());
1287  trace::Span Tracer("Queued:" + Name);
1288  if (Tracer.Args) {
1289  if (CurrentRequest)
1290  SPAN_ATTACH(Tracer, "CurrentRequest", CurrentRequest->Name);
1291  llvm::json::Array PreambleRequestsNames;
1292  for (const auto &Req : PreambleRequests)
1293  PreambleRequestsNames.push_back(Req.Name);
1294  SPAN_ATTACH(Tracer, "PreambleRequestsNames",
1295  std::move(PreambleRequestsNames));
1296  llvm::json::Array RequestsNames;
1297  for (const auto &Req : Requests)
1298  RequestsNames.push_back(Req.Name);
1299  SPAN_ATTACH(Tracer, "RequestsNames", std::move(RequestsNames));
1300  }
1301  // For tracers that follow contexts, keep the trace span's context alive
1302  // until we dequeue the request, so they see the full duration.
1303  QueueCtx = Context::current().clone();
1304  }
1305  Requests.push_back({std::move(Task), std::string(Name), steady_clock::now(),
1306  std::move(Ctx), std::move(QueueCtx), Update,
1307  Invalidation, std::move(Invalidate)});
1308  }
1309  RequestsCV.notify_all();
1310 }
1311 
1312 void ASTWorker::run() {
1313  while (true) {
1314  {
1315  std::unique_lock<std::mutex> Lock(Mutex);
1316  assert(!CurrentRequest && "A task is already running, multiple workers?");
1317  for (auto Wait = scheduleLocked(); !Wait.expired();
1318  Wait = scheduleLocked()) {
1319  assert(PreambleRequests.empty() &&
1320  "Preamble updates should be scheduled immediately");
1321  if (Done) {
1322  if (Requests.empty())
1323  return;
1324  // Even though Done is set, finish pending requests.
1325  break; // However, skip delays to shutdown fast.
1326  }
1327 
1328  // Tracing: we have a next request, attribute this sleep to it.
1329  llvm::Optional<WithContext> Ctx;
1330  llvm::Optional<trace::Span> Tracer;
1331  if (!Requests.empty()) {
1332  Ctx.emplace(Requests.front().Ctx.clone());
1333  Tracer.emplace("Debounce");
1334  SPAN_ATTACH(*Tracer, "next_request", Requests.front().Name);
1335  if (!(Wait == Deadline::infinity())) {
1336  Status.update([&](TUStatus &Status) {
1337  Status.ASTActivity.K = ASTAction::Queued;
1338  Status.ASTActivity.Name = Requests.front().Name;
1339  });
1340  SPAN_ATTACH(*Tracer, "sleep_ms",
1341  std::chrono::duration_cast<std::chrono::milliseconds>(
1342  Wait.time() - steady_clock::now())
1343  .count());
1344  }
1345  }
1346 
1347  wait(Lock, RequestsCV, Wait);
1348  }
1349  // Any request in ReceivedPreambles is at least as old as the
1350  // Requests.front(), so prefer them first to preserve LSP order.
1351  if (!PreambleRequests.empty()) {
1352  CurrentRequest = std::move(PreambleRequests.front());
1353  PreambleRequests.pop_front();
1354  } else {
1355  CurrentRequest = std::move(Requests.front());
1356  Requests.pop_front();
1357  }
1358  } // unlock Mutex
1359 
1360  // Inform tracing that the request was dequeued.
1361  CurrentRequest->QueueCtx.reset();
1362 
1363  // It is safe to perform reads to CurrentRequest without holding the lock as
1364  // only writer is also this thread.
1365  {
1366  std::unique_lock<Semaphore> Lock(Barrier, std::try_to_lock);
1367  if (!Lock.owns_lock()) {
1368  Status.update([&](TUStatus &Status) {
1369  Status.ASTActivity.K = ASTAction::Queued;
1370  Status.ASTActivity.Name = CurrentRequest->Name;
1371  });
1372  Lock.lock();
1373  }
1374  WithContext Guard(std::move(CurrentRequest->Ctx));
1375  Status.update([&](TUStatus &Status) {
1376  Status.ASTActivity.K = ASTAction::RunningAction;
1377  Status.ASTActivity.Name = CurrentRequest->Name;
1378  });
1379  runTask(CurrentRequest->Name, CurrentRequest->Action);
1380  }
1381 
1382  bool IsEmpty = false;
1383  {
1384  std::lock_guard<std::mutex> Lock(Mutex);
1385  CurrentRequest.reset();
1386  IsEmpty = Requests.empty() && PreambleRequests.empty();
1387  }
1388  if (IsEmpty) {
1389  Status.update([&](TUStatus &Status) {
1390  Status.ASTActivity.K = ASTAction::Idle;
1391  Status.ASTActivity.Name = "";
1392  });
1393  }
1394  RequestsCV.notify_all();
1395  }
1396 }
1397 
1398 Deadline ASTWorker::scheduleLocked() {
1399  // Process new preambles immediately.
1400  if (!PreambleRequests.empty())
1401  return Deadline::zero();
1402  if (Requests.empty())
1403  return Deadline::infinity(); // Wait for new requests.
1404  // Handle cancelled requests first so the rest of the scheduler doesn't.
1405  for (auto I = Requests.begin(), E = Requests.end(); I != E; ++I) {
1406  if (!isCancelled(I->Ctx)) {
1407  // Cancellations after the first read don't affect current scheduling.
1408  if (I->Update == None)
1409  break;
1410  continue;
1411  }
1412  // Cancelled reads are moved to the front of the queue and run immediately.
1413  if (I->Update == None) {
1414  Request R = std::move(*I);
1415  Requests.erase(I);
1416  Requests.push_front(std::move(R));
1417  return Deadline::zero();
1418  }
1419  // Cancelled updates are downgraded to auto-diagnostics, and may be elided.
1420  if (I->Update->Diagnostics == WantDiagnostics::Yes)
1421  I->Update->Diagnostics = WantDiagnostics::Auto;
1422  }
1423 
1424  while (shouldSkipHeadLocked()) {
1425  vlog("ASTWorker skipping {0} for {1}", Requests.front().Name, FileName);
1426  Requests.pop_front();
1427  }
1428  assert(!Requests.empty() && "skipped the whole queue");
1429  // Some updates aren't dead yet, but never end up being used.
1430  // e.g. the first keystroke is live until obsoleted by the second.
1431  // We debounce "maybe-unused" writes, sleeping in case they become dead.
1432  // But don't delay reads (including updates where diagnostics are needed).
1433  for (const auto &R : Requests)
1434  if (R.Update == None || R.Update->Diagnostics == WantDiagnostics::Yes)
1435  return Deadline::zero();
1436  // Front request needs to be debounced, so determine when we're ready.
1437  Deadline D(Requests.front().AddTime + UpdateDebounce.compute(RebuildTimes));
1438  return D;
1439 }
1440 
1441 // Returns true if Requests.front() is a dead update that can be skipped.
1442 bool ASTWorker::shouldSkipHeadLocked() const {
1443  assert(!Requests.empty());
1444  auto Next = Requests.begin();
1445  auto Update = Next->Update;
1446  if (!Update) // Only skip updates.
1447  return false;
1448  ++Next;
1449  // An update is live if its AST might still be read.
1450  // That is, if it's not immediately followed by another update.
1451  if (Next == Requests.end() || !Next->Update)
1452  return false;
1453  // The other way an update can be live is if its diagnostics might be used.
1454  switch (Update->Diagnostics) {
1455  case WantDiagnostics::Yes:
1456  return false; // Always used.
1457  case WantDiagnostics::No:
1458  return true; // Always dead.
1459  case WantDiagnostics::Auto:
1460  // Used unless followed by an update that generates diagnostics.
1461  for (; Next != Requests.end(); ++Next)
1462  if (Next->Update && Next->Update->Diagnostics != WantDiagnostics::No)
1463  return true; // Prefer later diagnostics.
1464  return false;
1465  }
1466  llvm_unreachable("Unknown WantDiagnostics");
1467 }
1468 
1469 bool ASTWorker::blockUntilIdle(Deadline Timeout) const {
1470  auto WaitUntilASTWorkerIsIdle = [&] {
1471  std::unique_lock<std::mutex> Lock(Mutex);
1472  return wait(Lock, RequestsCV, Timeout, [&] {
1473  return PreambleRequests.empty() && Requests.empty() && !CurrentRequest;
1474  });
1475  };
1476  // Make sure ASTWorker has processed all requests, which might issue new
1477  // updates to PreamblePeer.
1478  if (!WaitUntilASTWorkerIsIdle())
1479  return false;
1480  // Now that ASTWorker processed all requests, ensure PreamblePeer has served
1481  // all update requests. This might create new PreambleRequests for the
1482  // ASTWorker.
1483  if (!PreamblePeer.blockUntilIdle(Timeout))
1484  return false;
1485  assert(Requests.empty() &&
1486  "No new normal tasks can be scheduled concurrently with "
1487  "blockUntilIdle(): ASTWorker isn't threadsafe");
1488  // Finally make sure ASTWorker has processed all of the preamble updates.
1489  return WaitUntilASTWorkerIsIdle();
1490 }
1491 
1492 // Render a TUAction to a user-facing string representation.
1493 // TUAction represents clangd-internal states, we don't intend to expose them
1494 // to users (say C++ programmers) directly to avoid confusion, we use terms that
1495 // are familiar by C++ programmers.
1496 std::string renderTUAction(const PreambleAction PA, const ASTAction &AA) {
1497  llvm::SmallVector<std::string, 2> Result;
1498  switch (PA) {
1499  case PreambleAction::Building:
1500  Result.push_back("parsing includes");
1501  break;
1502  case PreambleAction::Idle:
1503  // We handle idle specially below.
1504  break;
1505  }
1506  switch (AA.K) {
1507  case ASTAction::Queued:
1508  Result.push_back("file is queued");
1509  break;
1510  case ASTAction::RunningAction:
1511  Result.push_back("running " + AA.Name);
1512  break;
1513  case ASTAction::Building:
1514  Result.push_back("parsing main file");
1515  break;
1516  case ASTAction::Idle:
1517  // We handle idle specially below.
1518  break;
1519  }
1520  if (Result.empty())
1521  return "idle";
1522  return llvm::join(Result, ", ");
1523 }
1524 
1525 } // namespace
1526 
1528  return llvm::heavyweight_hardware_concurrency().compute_thread_count();
1529 }
1530 
1531 FileStatus TUStatus::render(PathRef File) const {
1532  FileStatus FStatus;
1533  FStatus.uri = URIForFile::canonicalize(File, /*TUPath=*/File);
1534  FStatus.state = renderTUAction(PreambleActivity, ASTActivity);
1535  return FStatus;
1536 }
1537 
1539  /// Latest inputs, passed to TUScheduler::update().
1540  std::string Contents;
1541  ASTWorkerHandle Worker;
1542 };
1543 
1544 TUScheduler::TUScheduler(const GlobalCompilationDatabase &CDB,
1545  const Options &Opts,
1546  std::unique_ptr<ParsingCallbacks> Callbacks)
1547  : CDB(CDB), Opts(Opts),
1548  Callbacks(Callbacks ? std::move(Callbacks)
1549  : std::make_unique<ParsingCallbacks>()),
1550  Barrier(Opts.AsyncThreadsCount), QuickRunBarrier(Opts.AsyncThreadsCount),
1551  IdleASTs(
1552  std::make_unique<ASTCache>(Opts.RetentionPolicy.MaxRetainedASTs)),
1553  HeaderIncluders(std::make_unique<HeaderIncluderCache>()) {
1554  // Avoid null checks everywhere.
1555  if (!Opts.ContextProvider) {
1556  this->Opts.ContextProvider = [](llvm::StringRef) {
1557  return Context::current().clone();
1558  };
1559  }
1560  if (0 < Opts.AsyncThreadsCount) {
1561  PreambleTasks.emplace();
1562  WorkerThreads.emplace();
1563  }
1564 }
1565 
1567  // Notify all workers that they need to stop.
1568  Files.clear();
1569 
1570  // Wait for all in-flight tasks to finish.
1571  if (PreambleTasks)
1572  PreambleTasks->wait();
1573  if (WorkerThreads)
1574  WorkerThreads->wait();
1575 }
1576 
1578  for (auto &File : Files)
1579  if (!File.getValue()->Worker->blockUntilIdle(D))
1580  return false;
1581  if (PreambleTasks)
1582  if (!PreambleTasks->wait(D))
1583  return false;
1584  return true;
1585 }
1586 
1589  std::unique_ptr<FileData> &FD = Files[File];
1590  bool NewFile = FD == nullptr;
1591  bool ContentChanged = false;
1592  if (!FD) {
1593  // Create a new worker to process the AST-related tasks.
1594  ASTWorkerHandle Worker =
1595  ASTWorker::create(File, CDB, *IdleASTs, *HeaderIncluders,
1596  WorkerThreads ? WorkerThreads.getPointer() : nullptr,
1597  Barrier, Opts, *Callbacks);
1598  FD = std::unique_ptr<FileData>(
1599  new FileData{Inputs.Contents, std::move(Worker)});
1600  ContentChanged = true;
1601  } else if (FD->Contents != Inputs.Contents) {
1602  ContentChanged = true;
1603  FD->Contents = Inputs.Contents;
1604  }
1605  FD->Worker->update(std::move(Inputs), WantDiags, ContentChanged);
1606  // There might be synthetic update requests, don't change the LastActiveFile
1607  // in such cases.
1608  if (ContentChanged)
1609  LastActiveFile = File.str();
1610  return NewFile;
1611 }
1612 
1614  bool Removed = Files.erase(File);
1615  if (!Removed)
1616  elog("Trying to remove file from TUScheduler that is not tracked: {0}",
1617  File);
1618  // We don't call HeaderIncluders.remove(File) here.
1619  // If we did, we'd avoid potentially stale header/mainfile associations.
1620  // However, it would mean that closing a mainfile could invalidate the
1621  // preamble of several open headers.
1622 }
1623 
1624 void TUScheduler::run(llvm::StringRef Name, llvm::StringRef Path,
1625  llvm::unique_function<void()> Action) {
1626  runWithSemaphore(Name, Path, std::move(Action), Barrier);
1627 }
1628 
1629 void TUScheduler::runQuick(llvm::StringRef Name, llvm::StringRef Path,
1630  llvm::unique_function<void()> Action) {
1631  // Use QuickRunBarrier to serialize quick tasks: we are ignoring
1632  // the parallelism level set by the user, don't abuse it
1633  runWithSemaphore(Name, Path, std::move(Action), QuickRunBarrier);
1634 }
1635 
1636 void TUScheduler::runWithSemaphore(llvm::StringRef Name, llvm::StringRef Path,
1637  llvm::unique_function<void()> Action,
1638  Semaphore &Sem) {
1639  if (Path.empty())
1640  Path = LastActiveFile;
1641  else
1642  LastActiveFile = Path.str();
1643  if (!PreambleTasks) {
1644  WithContext WithProvidedContext(Opts.ContextProvider(Path));
1645  return Action();
1646  }
1647  PreambleTasks->runAsync(Name, [this, &Sem, Ctx = Context::current().clone(),
1648  Path(Path.str()),
1649  Action = std::move(Action)]() mutable {
1650  std::lock_guard<Semaphore> BarrierLock(Sem);
1651  WithContext WC(std::move(Ctx));
1652  WithContext WithProvidedContext(Opts.ContextProvider(Path));
1653  Action();
1654  });
1655 }
1656 
1658  llvm::StringRef Name, PathRef File,
1659  llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
1660  TUScheduler::ASTActionInvalidation Invalidation) {
1661  auto It = Files.find(File);
1662  if (It == Files.end()) {
1663  Action(llvm::make_error<LSPError>(
1664  "trying to get AST for non-added document", ErrorCode::InvalidParams));
1665  return;
1666  }
1667  LastActiveFile = File.str();
1668 
1669  It->second->Worker->runWithAST(Name, std::move(Action), Invalidation);
1670 }
1671 
1672 void TUScheduler::runWithPreamble(llvm::StringRef Name, PathRef File,
1673  PreambleConsistency Consistency,
1675  auto It = Files.find(File);
1676  if (It == Files.end()) {
1677  Action(llvm::make_error<LSPError>(
1678  "trying to get preamble for non-added document",
1680  return;
1681  }
1682  LastActiveFile = File.str();
1683 
1684  if (!PreambleTasks) {
1686  SPAN_ATTACH(Tracer, "file", File);
1687  std::shared_ptr<const ASTSignals> Signals;
1688  std::shared_ptr<const PreambleData> Preamble =
1689  It->second->Worker->getPossiblyStalePreamble(&Signals);
1690  WithContext WithProvidedContext(Opts.ContextProvider(File));
1691  Action(InputsAndPreamble{It->second->Contents,
1692  It->second->Worker->getCurrentCompileCommand(),
1693  Preamble.get(), Signals.get()});
1694  return;
1695  }
1696 
1697  std::shared_ptr<const ASTWorker> Worker = It->second->Worker.lock();
1698  auto Task = [Worker, Consistency, Name = Name.str(), File = File.str(),
1699  Contents = It->second->Contents,
1700  Command = Worker->getCurrentCompileCommand(),
1702  std::string(File)),
1703  Action = std::move(Action), this]() mutable {
1704  ThreadCrashReporter ScopedReporter([&Name, &Contents, &Command]() {
1705  llvm::errs() << "Signalled during preamble action: " << Name << "\n";
1706  crashDumpCompileCommand(llvm::errs(), Command);
1707  crashDumpFileContents(llvm::errs(), Contents);
1708  });
1709  std::shared_ptr<const PreambleData> Preamble;
1710  if (Consistency == PreambleConsistency::Stale) {
1711  // Wait until the preamble is built for the first time, if preamble
1712  // is required. This avoids extra work of processing the preamble
1713  // headers in parallel multiple times.
1714  Worker->waitForFirstPreamble();
1715  }
1716  std::shared_ptr<const ASTSignals> Signals;
1717  Preamble = Worker->getPossiblyStalePreamble(&Signals);
1718 
1719  std::lock_guard<Semaphore> BarrierLock(Barrier);
1720  WithContext Guard(std::move(Ctx));
1722  SPAN_ATTACH(Tracer, "file", File);
1723  WithContext WithProvidedContext(Opts.ContextProvider(File));
1724  Action(InputsAndPreamble{Contents, Command, Preamble.get(), Signals.get()});
1725  };
1726 
1727  PreambleTasks->runAsync("task:" + llvm::sys::path::filename(File),
1728  std::move(Task));
1729 }
1730 
1731 llvm::StringMap<TUScheduler::FileStats> TUScheduler::fileStats() const {
1732  llvm::StringMap<TUScheduler::FileStats> Result;
1733  for (const auto &PathAndFile : Files)
1734  Result.try_emplace(PathAndFile.first(),
1735  PathAndFile.second->Worker->stats());
1736  return Result;
1737 }
1738 
1739 std::vector<Path> TUScheduler::getFilesWithCachedAST() const {
1740  std::vector<Path> Result;
1741  for (auto &&PathAndFile : Files) {
1742  if (!PathAndFile.second->Worker->isASTCached())
1743  continue;
1744  Result.push_back(std::string(PathAndFile.first()));
1745  }
1746  return Result;
1747 }
1748 
1749 DebouncePolicy::clock::duration
1750 DebouncePolicy::compute(llvm::ArrayRef<clock::duration> History) const {
1751  assert(Min <= Max && "Invalid policy");
1752  if (History.empty())
1753  return Max; // Arbitrary.
1754 
1755  // Base the result on the median rebuild.
1756  // nth_element needs a mutable array, take the chance to bound the data size.
1757  History = History.take_back(15);
1758  llvm::SmallVector<clock::duration, 15> Recent(History.begin(), History.end());
1759  auto *Median = Recent.begin() + Recent.size() / 2;
1760  std::nth_element(Recent.begin(), Median, Recent.end());
1761 
1762  clock::duration Target =
1763  std::chrono::duration_cast<clock::duration>(RebuildRatio * *Median);
1764  if (Target > Max)
1765  return Max;
1766  if (Target < Min)
1767  return Min;
1768  return Target;
1769 }
1770 
1772  DebouncePolicy P;
1773  P.Min = P.Max = T;
1774  return P;
1775 }
1776 
1778  for (const auto &Elem : fileStats()) {
1779  MT.detail(Elem.first())
1780  .child("preamble")
1781  .addUsage(Opts.StorePreamblesInMemory ? Elem.second.UsedBytesPreamble
1782  : 0);
1783  MT.detail(Elem.first()).child("ast").addUsage(Elem.second.UsedBytesAST);
1784  MT.child("header_includer_cache").addUsage(HeaderIncluders->getUsedBytes());
1785  }
1786 }
1787 } // namespace clangd
1788 } // namespace clang
dlog
#define dlog(...)
Definition: Logger.h:101
clang::clangd::buildPreamble
std::shared_ptr< const PreambleData > buildPreamble(PathRef FileName, CompilerInvocation CI, const ParseInputs &Inputs, bool StoreInMemory, PreambleParsedCallback PreambleCallback, PreambleBuildStats *Stats)
Build a preamble for the new inputs unless an old one can be reused.
Definition: Preamble.cpp:466
WantDiags
WantDiagnostics WantDiags
Definition: TUScheduler.cpp:494
clang::clangd::MemoryTree::child
MemoryTree & child(llvm::StringLiteral Name)
No copy of the Name.
Definition: MemoryTree.h:39
clang::clangd::isPreambleCompatible
bool isPreambleCompatible(const PreambleData &Preamble, const ParseInputs &Inputs, PathRef FileName, const CompilerInvocation &CI)
Returns true if Preamble is reusable for Inputs.
Definition: Preamble.cpp:577
clang::clangd::cancelableTask
std::pair< Context, Canceler > cancelableTask(int Reason)
Defines a new task whose cancellation may be requested.
Definition: Cancellation.cpp:24
Label
std::string Label
Definition: InlayHintTests.cpp:46
E
const Expr * E
Definition: AvoidBindCheck.cpp:88
clang::clangd::TUScheduler::runWithPreamble
void runWithPreamble(llvm::StringRef Name, PathRef File, PreambleConsistency Consistency, Callback< InputsAndPreamble > Action)
Schedule an async read of the preamble.
Definition: TUScheduler.cpp:1672
clang::clangd::getDefaultAsyncThreadsCount
unsigned getDefaultAsyncThreadsCount()
Returns a number of a default async threads to use for TUScheduler.
Definition: TUScheduler.cpp:1527
CIDiags
std::vector< Diag > CIDiags
Definition: TUScheduler.cpp:493
clang::clangd::printArgv
std::string printArgv(llvm::ArrayRef< llvm::StringRef > Args)
Definition: CompileCommands.cpp:628
clang::clangd::Path
std::string Path
A typedef to represent a file path.
Definition: Path.h:26
Update
llvm::Optional< UpdateType > Update
Definition: TUScheduler.cpp:644
clang::clangd::TUScheduler::~TUScheduler
~TUScheduler()
Definition: TUScheduler.cpp:1566
Usage
const char Usage[]
Definition: ClangReorderFields.cpp:50
Tracer
std::unique_ptr< trace::EventTracer > Tracer
Definition: TraceTests.cpp:161
Path.h
clang::clangd::Context::current
static const Context & current()
Returns the context for the current thread, creating it if needed.
Definition: Context.cpp:27
clang::clangd::WantDiagnostics::Yes
@ Yes
clang::clangd::TUScheduler::blockUntilIdle
bool blockUntilIdle(Deadline D) const
Wait until there are no scheduled or running tasks.
Definition: TUScheduler.cpp:1577
clang::clangd::Context::clone
Context clone() const
Clone this context object.
Definition: Context.cpp:20
Diagnostics
WantDiagnostics Diagnostics
Definition: TUScheduler.cpp:602
ContentChanged
bool ContentChanged
Definition: TUScheduler.cpp:605
clang::clangd::detail::error
llvm::Error error(std::error_code, std::string &&)
Definition: Logger.cpp:80
CI
std::unique_ptr< CompilerInvocation > CI
Definition: TUScheduler.cpp:491
QueueCtx
llvm::Optional< Context > QueueCtx
Definition: TUScheduler.cpp:643
clang::clangd::PreambleAction
PreambleAction
Definition: TUScheduler.h:90
Preamble.h
clang::clangd::PreambleBuildStats::TotalBuildTime
double TotalBuildTime
Total wall time it took to build preamble, in seconds.
Definition: Preamble.h:83
Ctx
Context Ctx
Definition: TUScheduler.cpp:495
clang::clangd::TUScheduler::run
void run(llvm::StringRef Name, llvm::StringRef Path, llvm::unique_function< void()> Action)
Schedule an async task with no dependencies.
Definition: TUScheduler.cpp:1624
Trace.h
AST
llvm::Optional< ParsedAST > AST
Definition: SymbolCollectorTests.cpp:135
clang::clangd::TUScheduler::getFilesWithCachedAST
std::vector< Path > getFilesWithCachedAST() const
Returns a list of files with ASTs currently stored in memory.
Definition: TUScheduler.cpp:1739
clang::tidy::cppcoreguidelines::join
static std::string join(ArrayRef< SpecialMemberFunctionsCheck::SpecialMemberFunctionKind > SMFS, llvm::StringRef AndOr)
Definition: SpecialMemberFunctionsCheck.cpp:78
clang::clangd::TUScheduler::update
bool update(PathRef File, ParseInputs Inputs, WantDiagnostics WD)
Schedule an update for File.
Definition: TUScheduler.cpp:1587
clang::clangd::GlobalCompilationDatabase
Provides compilation arguments used for parsing C and C++ files.
Definition: GlobalCompilationDatabase.h:34
Action
llvm::unique_function< void()> Action
Definition: TUScheduler.cpp:639
clang::clangd::TUScheduler::Options::ContextProvider
std::function< Context(PathRef)> ContextProvider
Used to create a context that wraps each single operation.
Definition: TUScheduler.h:206
Target
std::string Target
Definition: QueryDriverDatabase.cpp:64
clang::clangd::Semaphore
Limits the number of threads that can acquire the lock at the same time.
Definition: Threading.h:28
Preamble
const PreambleData & Preamble
Definition: CodeComplete.cpp:1204
Cancellation.h
clang::clangd::PreambleBuildStats::SerializedSize
size_t SerializedSize
The serialized size of the preamble.
Definition: Preamble.h:93
clang::clangd::Deadline
A point in time we can wait for.
Definition: Threading.h:45
clang::clangd::WantDiagnostics
WantDiagnostics
Determines whether diagnostics should be generated for a file snapshot.
Definition: TUScheduler.h:52
clang::clangd::canonicalize
static llvm::SmallString< 128 > canonicalize(llvm::StringRef Path)
Definition: FileDistance.cpp:48
clang::clangd::MemoryTree::detail
MemoryTree & detail(llvm::StringRef Name)
Makes a copy of the Name in detailed mode, returns current node otherwise.
Definition: MemoryTree.h:51
Inputs
ParseInputs Inputs
Definition: TUScheduler.cpp:492
clang::clangd::isCancelled
int isCancelled(const Context &Ctx)
If the current context is within a cancelled task, returns the reason.
Definition: Cancellation.cpp:35
clang::clangd::TUScheduler::HeaderIncluderCache::HeaderIncluderCache
HeaderIncluderCache()
Definition: TUScheduler.cpp:304
clang::clangd::MemoryTree::addUsage
void addUsage(size_t Increment)
Increases size of current node by Increment.
Definition: MemoryTree.h:56
clang::clangd::wait
void wait(std::unique_lock< std::mutex > &Lock, std::condition_variable &CV, Deadline D)
Wait once on CV for the specified duration.
Definition: Threading.cpp:120
clang::clangd::ParseInputs
Information required to run clang, e.g. to parse AST or do code completion.
Definition: Compiler.h:46
CanonicalIncludes.h
ns1::ns2::D
@ D
Definition: CategoricalFeature.h:3
clang::clangd::InputsAndPreamble::Contents
llvm::StringRef Contents
Definition: TUScheduler.h:43
clang::clangd::FileStatus::state
std::string state
The human-readable string presents the current state of the file, can be shown in the UI (e....
Definition: Protocol.h:1627
clang::clangd::ErrorCode::InvalidParams
@ InvalidParams
clang::clangd::ParsingCallbacks
Definition: TUScheduler.h:128
clang::clangd::TUScheduler::HeaderIncluderCache::remove
void remove(PathRef MainFile)
Definition: TUScheduler.cpp:323
clang::clangd::MemoryTree
A tree that can be used to represent memory usage of nested components while preserving the hierarchy...
Definition: MemoryTree.h:30
GlobalCompilationDatabase.h
clang::clangd::PreambleAction::Idle
@ Idle
clang::clangd::ThreadCrashReporter
Allows setting per-thread abort/kill signal callbacks, to print additional information about the cras...
Definition: ThreadCrashReporter.h:19
clang::clangd::PreambleBuildStats::FileSystemTime
double FileSystemTime
Time spent in filesystem operations during the build, in seconds.
Definition: Preamble.h:85
clang::clangd::TUScheduler::ASTCache::ASTCache
ASTCache(unsigned MaxRetainedASTs)
Definition: TUScheduler.cpp:154
CompileCommands.h
clang::clangd::trace::enabled
bool enabled()
Returns true if there is an active tracer.
Definition: Trace.cpp:283
Logger.h
Threading.h
clang::clangd::FileStatus::uri
URIForFile uri
The text document's URI.
Definition: Protocol.h:1624
Name
std::string Name
Definition: TUScheduler.cpp:640
clang::clangd::TUScheduler::HeaderIncluderCache::get
std::string get(PathRef Header) const
Get the mainfile associated with Header, or the empty string if none.
Definition: TUScheduler.cpp:335
SPAN_ATTACH
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:164
clang::clangd::TUScheduler::FileData::Contents
std::string Contents
Latest inputs, passed to TUScheduler::update().
Definition: TUScheduler.cpp:1540
clang::clangd::vlog
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:72
clang::clangd::trace::Metric::Distribution
@ Distribution
A distribution of values with a meaningful mean and count.
Definition: Trace.h:52
Diagnostics.h
clang::clangd::TUScheduler::remove
void remove(PathRef File)
Remove File from the list of tracked files and schedule removal of its resources.
Definition: TUScheduler.cpp:1613
clang::clangd::Key
Values in a Context are indexed by typed keys.
Definition: Context.h:40
FileName
StringRef FileName
Definition: KernelNameRestrictionCheck.cpp:46
clang::clangd::TUScheduler::fileStats
llvm::StringMap< FileStats > fileStats() const
Returns resources used for each of the currently open files.
Definition: TUScheduler.cpp:1731
clang::clangd::TUScheduler::ASTCache::getUsedBytes
std::size_t getUsedBytes(Key K)
Returns result of getUsedBytes() for the AST cached by K.
Definition: TUScheduler.cpp:158
clang::clangd::WithContext
WithContext replaces Context::current() with a provided scope.
Definition: Context.h:187
clang::clangd::DebouncePolicy::Min
clock::duration Min
The minimum time that we always debounce for.
Definition: TUScheduler.h:77
clang::tidy::bugprone::PP
static Preprocessor * PP
Definition: BadSignalToKillThreadCheck.cpp:29
clang::clangd::buildCompilerInvocation
std::unique_ptr< CompilerInvocation > buildCompilerInvocation(const ParseInputs &Inputs, clang::DiagnosticConsumer &D, std::vector< std::string > *CC1Args)
Builds compiler invocation that could be used to build AST or preamble.
Definition: Compiler.cpp:86
Counter
trace::Metric Counter
Definition: TraceTests.cpp:149
clang::clangd::TUScheduler::ASTCache
An LRU cache of idle ASTs.
Definition: TUScheduler.cpp:150
clang::clangd::FileBeingProcessed
static clang::clangd::Key< std::string > FileBeingProcessed
Definition: TUScheduler.cpp:138
Compiler.h
clang::clangd::TUScheduler::runQuick
void runQuick(llvm::StringRef Name, llvm::StringRef Path, llvm::unique_function< void()> Action)
Similar to run, except the task is expected to be quick.
Definition: TUScheduler.cpp:1629
AddTime
steady_clock::time_point AddTime
Definition: TUScheduler.cpp:641
clang::clangd::Context::derive
Context derive(const Key< Type > &Key, typename std::decay< Type >::type Value) const &
Derives a child context It is safe to move or destroy a parent context after calling derive().
Definition: Context.h:119
clang::clangd::TUScheduler::profile
void profile(MemoryTree &MT) const
Definition: TUScheduler.cpp:1777
clang::clangd::PathRef
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:29
clang::clangd::TUScheduler::ASTActionInvalidation
ASTActionInvalidation
Defines how a runWithAST action is implicitly cancelled by other actions.
Definition: TUScheduler.h:255
clang::clangd::trace::Metric
Represents measurements of clangd events, e.g.
Definition: Trace.h:38
clang::clangd::InputsAndPreamble
Definition: TUScheduler.h:42
InvalidationPolicy
TUScheduler::ASTActionInvalidation InvalidationPolicy
Definition: TUScheduler.cpp:645
clang::clangd::TUScheduler::HeaderIncluderCache
A map from header files to an opened "proxy" file that includes them.
Definition: TUScheduler.cpp:247
clang::clangd::TUScheduler::Options
Definition: TUScheduler.h:186
clang::clangd::AsyncTaskRunner::runAsync
void runAsync(const llvm::Twine &Name, llvm::unique_function< void()> Action)
Definition: Threading.cpp:80
clang::clangd::TUScheduler::runWithAST
void runWithAST(llvm::StringRef Name, PathRef File, Callback< InputsAndAST > Action, ASTActionInvalidation=NoInvalidation)
Schedule an async read of the AST.
Definition: TUScheduler.cpp:1657
ThreadCrashReporter.h
clang::clangd::TUScheduler::ASTCache::put
void put(Key K, std::unique_ptr< ParsedAST > V)
Store the value in the pool, possibly removing the last used AST.
Definition: TUScheduler.cpp:168
clang::clangd::DebouncePolicy
Clangd may wait after an update to see if another one comes along.
Definition: TUScheduler.h:73
MemoryTree.h
if
if(CLANG_PLUGIN_SUPPORT) export_executable_symbols_for_plugins(clang-tidy) endif() install(PROGRAMS clang-tidy-diff.py DESTINATION "$
Definition: clang-tidy/tool/CMakeLists.txt:59
clang::clangd::Canceler
std::function< void()> Canceler
A canceller requests cancellation of a task, when called.
Definition: Cancellation.h:70
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
OS
llvm::raw_string_ostream OS
Definition: TraceTests.cpp:160
Arena
llvm::BumpPtrAllocator Arena
Definition: Serialization.cpp:212
TUScheduler.h
clang::clangd::FileStatus
Clangd extension: indicates the current state of the file in clangd, sent from server via the textDoc...
Definition: Protocol.h:1622
clang::clangd::DebouncePolicy::RebuildRatio
float RebuildRatio
Target debounce, as a fraction of file rebuild time.
Definition: TUScheduler.h:82
clang::clangd::DebouncePolicy::Max
clock::duration Max
The maximum time we may debounce for.
Definition: TUScheduler.h:79
clang::clangd::TUScheduler::PreambleConsistency
PreambleConsistency
Controls whether preamble reads wait for the preamble to be up-to-date.
Definition: TUScheduler.h:278
clang::clangd::TUScheduler::FileData
Definition: TUScheduler.cpp:1538
clang::clangd::PreambleBuildStats::BuildSize
size_t BuildSize
Estimate of the memory used while building the preamble.
Definition: Preamble.h:90
clang::clangd::TUScheduler::HeaderIncluderCache::update
void update(PathRef MainFile, llvm::ArrayRef< std::string > Headers)
Definition: TUScheduler.cpp:310
clang::clangd::DebouncePolicy::fixed
static DebouncePolicy fixed(clock::duration)
A policy that always returns the same duration, useful for tests.
Definition: TUScheduler.cpp:1771
clang::clangd::Callback
llvm::unique_function< void(llvm::Expected< T >)> Callback
A Callback<T> is a void function that accepts Expected<T>.
Definition: Function.h:28
MainFile
std::string MainFile
Definition: HeadersTests.cpp:134
Invalidate
Canceler Invalidate
Definition: TUScheduler.cpp:646
clang::clangd::TUScheduler::FileData::Worker
ASTWorkerHandle Worker
Definition: TUScheduler.cpp:1541
clang::clangd::TUScheduler::ASTCache::take
llvm::Optional< std::unique_ptr< ParsedAST > > take(Key K, const trace::Metric *AccessMetric=nullptr)
Returns the cached value for K, or llvm::None if the value is not in the cache anymore.
Definition: TUScheduler.cpp:188
clang::clangd::elog
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:61
clang::clangd::detail::log
void log(Logger::Level L, const char *Fmt, Ts &&... Vals)
Definition: Logger.h:47
clang::clangd::DebouncePolicy::compute
clock::duration compute(llvm::ArrayRef< clock::duration > History) const
Compute the time to debounce based on this policy and recent build times.
Definition: TUScheduler.cpp:1750
K
Kind K
Definition: Rename.cpp:436
clang::clangd::TUScheduler::ASTCache::Key
const ASTWorker * Key
Definition: TUScheduler.cpp:152
clang::clangd::TUScheduler::HeaderIncluderCache::getUsedBytes
size_t getUsedBytes() const
Definition: TUScheduler.cpp:340
Context.h
clang::clangd::TUScheduler::getFileBeingProcessedInContext
static llvm::Optional< llvm::StringRef > getFileBeingProcessedInContext()
Definition: TUScheduler.cpp:140
ParsedAST.h
clang::clangd::trace::Span
Records an event whose duration is the lifetime of the Span object.
Definition: Trace.h:143