clang-tools  14.0.0git
TUScheduler.cpp
Go to the documentation of this file.
1 //===--- TUScheduler.cpp -----------------------------------------*-C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // TUScheduler manages a worker per active file. This ASTWorker processes
9 // updates (modifications to file contents) and reads (actions performed on
10 // preamble/AST) to the file.
11 //
12 // Each ASTWorker owns a dedicated thread to process updates and reads to the
13 // relevant file. Any request gets queued in FIFO order to be processed by that
14 // thread.
15 //
16 // An update request replaces current praser inputs to ensure any subsequent
17 // read sees the version of the file they were requested. It will also issue a
18 // build for new inputs.
19 //
20 // ASTWorker processes the file in two parts, a preamble and a main-file
21 // section. A preamble can be reused between multiple versions of the file until
22 // invalidated by a modification to a header, compile commands or modification
23 // to relevant part of the current file. Such a preamble is called compatible.
24 // An update is considered dead if no read was issued for that version and
25 // diagnostics weren't requested by client or could be generated for a later
26 // version of the file. ASTWorker eliminates such requests as they are
27 // redundant.
28 //
29 // In the presence of stale (non-compatible) preambles, ASTWorker won't publish
30 // diagnostics for update requests. Read requests will be served with ASTs build
31 // with stale preambles, unless the read is picky and requires a compatible
32 // preamble. In such cases it will block until new preamble is built.
33 //
34 // ASTWorker owns a PreambleThread for building preambles. If the preamble gets
35 // invalidated by an update request, a new build will be requested on
36 // PreambleThread. Since PreambleThread only receives requests for newer
37 // versions of the file, in case of multiple requests it will only build the
38 // last one and skip requests in between. Unless client force requested
39 // diagnostics(WantDiagnostics::Yes).
40 //
41 // When a new preamble is built, a "golden" AST is immediately built from that
42 // version of the file. This ensures diagnostics get updated even if the queue
43 // is full.
44 //
45 // Some read requests might just need preamble. Since preambles can be read
46 // concurrently, ASTWorker runs these requests on their own thread. These
47 // requests will receive latest build preamble, which might possibly be stale.
48 
49 #include "TUScheduler.h"
50 #include "Compiler.h"
51 #include "Diagnostics.h"
53 #include "ParsedAST.h"
54 #include "Preamble.h"
56 #include "support/Cancellation.h"
57 #include "support/Context.h"
58 #include "support/Logger.h"
59 #include "support/MemoryTree.h"
60 #include "support/Path.h"
62 #include "support/Threading.h"
63 #include "support/Trace.h"
64 #include "clang/Frontend/CompilerInvocation.h"
65 #include "clang/Tooling/CompilationDatabase.h"
66 #include "llvm/ADT/FunctionExtras.h"
67 #include "llvm/ADT/None.h"
68 #include "llvm/ADT/Optional.h"
69 #include "llvm/ADT/STLExtras.h"
70 #include "llvm/ADT/ScopeExit.h"
71 #include "llvm/ADT/SmallVector.h"
72 #include "llvm/ADT/StringExtras.h"
73 #include "llvm/ADT/StringRef.h"
74 #include "llvm/Support/Allocator.h"
75 #include "llvm/Support/Errc.h"
76 #include "llvm/Support/ErrorHandling.h"
77 #include "llvm/Support/FormatVariadic.h"
78 #include "llvm/Support/Path.h"
79 #include "llvm/Support/Threading.h"
80 #include "llvm/Support/raw_ostream.h"
81 #include <algorithm>
82 #include <atomic>
83 #include <chrono>
84 #include <condition_variable>
85 #include <functional>
86 #include <memory>
87 #include <mutex>
88 #include <queue>
89 #include <string>
90 #include <thread>
91 #include <type_traits>
92 #include <utility>
93 #include <vector>
94 
95 namespace clang {
96 namespace clangd {
97 using std::chrono::steady_clock;
98 
99 namespace {
100 class ASTWorker;
101 } // namespace
102 
104 
105 llvm::Optional<llvm::StringRef> TUScheduler::getFileBeingProcessedInContext() {
106  if (auto *File = Context::current().get(kFileBeingProcessed))
107  return llvm::StringRef(*File);
108  return None;
109 }
110 
111 /// An LRU cache of idle ASTs.
112 /// Because we want to limit the overall number of these we retain, the cache
113 /// owns ASTs (and may evict them) while their workers are idle.
114 /// Workers borrow ASTs when active, and return them when done.
116 public:
117  using Key = const ASTWorker *;
118 
119  ASTCache(unsigned MaxRetainedASTs) : MaxRetainedASTs(MaxRetainedASTs) {}
120 
121  /// Returns result of getUsedBytes() for the AST cached by \p K.
122  /// If no AST is cached, 0 is returned.
123  std::size_t getUsedBytes(Key K) {
124  std::lock_guard<std::mutex> Lock(Mut);
125  auto It = findByKey(K);
126  if (It == LRU.end() || !It->second)
127  return 0;
128  return It->second->getUsedBytes();
129  }
130 
131  /// Store the value in the pool, possibly removing the last used AST.
132  /// The value should not be in the pool when this function is called.
133  void put(Key K, std::unique_ptr<ParsedAST> V) {
134  std::unique_lock<std::mutex> Lock(Mut);
135  assert(findByKey(K) == LRU.end());
136 
137  LRU.insert(LRU.begin(), {K, std::move(V)});
138  if (LRU.size() <= MaxRetainedASTs)
139  return;
140  // We're past the limit, remove the last element.
141  std::unique_ptr<ParsedAST> ForCleanup = std::move(LRU.back().second);
142  LRU.pop_back();
143  // Run the expensive destructor outside the lock.
144  Lock.unlock();
145  ForCleanup.reset();
146  }
147 
148  /// Returns the cached value for \p K, or llvm::None if the value is not in
149  /// the cache anymore. If nullptr was cached for \p K, this function will
150  /// return a null unique_ptr wrapped into an optional.
151  /// If \p AccessMetric is set records whether there was a hit or miss.
152  llvm::Optional<std::unique_ptr<ParsedAST>>
153  take(Key K, const trace::Metric *AccessMetric = nullptr) {
154  // Record metric after unlocking the mutex.
155  std::unique_lock<std::mutex> Lock(Mut);
156  auto Existing = findByKey(K);
157  if (Existing == LRU.end()) {
158  if (AccessMetric)
159  AccessMetric->record(1, "miss");
160  return None;
161  }
162  if (AccessMetric)
163  AccessMetric->record(1, "hit");
164  std::unique_ptr<ParsedAST> V = std::move(Existing->second);
165  LRU.erase(Existing);
166  // GCC 4.8 fails to compile `return V;`, as it tries to call the copy
167  // constructor of unique_ptr, so we call the move ctor explicitly to avoid
168  // this miscompile.
169  return llvm::Optional<std::unique_ptr<ParsedAST>>(std::move(V));
170  }
171 
172 private:
173  using KVPair = std::pair<Key, std::unique_ptr<ParsedAST>>;
174 
175  std::vector<KVPair>::iterator findByKey(Key K) {
176  return llvm::find_if(LRU, [K](const KVPair &P) { return P.first == K; });
177  }
178 
179  std::mutex Mut;
180  unsigned MaxRetainedASTs;
181  /// Items sorted in LRU order, i.e. first item is the most recently accessed
182  /// one.
183  std::vector<KVPair> LRU; /* GUARDED_BY(Mut) */
184 };
185 
186 /// A map from header files to an opened "proxy" file that includes them.
187 /// If you open the header, the compile command from the proxy file is used.
188 ///
189 /// This inclusion information could also naturally live in the index, but there
190 /// are advantages to using open files instead:
191 /// - it's easier to achieve a *stable* choice of proxy, which is important
192 /// to avoid invalidating the preamble
193 /// - context-sensitive flags for libraries with multiple configurations
194 /// (e.g. C++ stdlib sensitivity to -std version)
195 /// - predictable behavior, e.g. guarantees that go-to-def landing on a header
196 /// will have a suitable command available
197 /// - fewer scaling problems to solve (project include graphs are big!)
198 ///
199 /// Implementation details:
200 /// - We only record this for mainfiles where the command was trustworthy
201 /// (i.e. not inferred). This avoids a bad inference "infecting" other files.
202 /// - Once we've picked a proxy file for a header, we stick with it until the
203 /// proxy file is invalidated *and* a new candidate proxy file is built.
204 /// Switching proxies is expensive, as the compile flags will (probably)
205 /// change and therefore we'll end up rebuilding the header's preamble.
206 /// - We don't capture the actual compile command, but just the filename we
207 /// should query to get it. This avoids getting out of sync with the CDB.
208 ///
209 /// All methods are threadsafe. In practice, update() comes from preamble
210 /// threads, remove()s mostly from the main thread, and get() from ASTWorker.
211 /// Writes are rare and reads are cheap, so we don't expect much contention.
213  // We should be be a little careful how we store the include graph of open
214  // files, as each can have a large number of transitive headers.
215  // This representation is O(unique transitive source files).
216  llvm::BumpPtrAllocator Arena;
217  struct Association {
218  llvm::StringRef MainFile;
219  // Circular-linked-list of associations with the same mainFile.
220  // Null indicates that the mainfile was removed.
221  Association *Next;
222  };
223  llvm::StringMap<Association, llvm::BumpPtrAllocator &> HeaderToMain;
224  llvm::StringMap<Association *, llvm::BumpPtrAllocator &> MainToFirst;
225  std::atomic<size_t> UsedBytes; // Updated after writes.
226  mutable std::mutex Mu;
227 
228  void invalidate(Association *First) {
229  Association *Current = First;
230  do {
231  Association *Next = Current->Next;
232  Current->Next = nullptr;
233  Current = Next;
234  } while (Current != First);
235  }
236 
237  // Create the circular list and return the head of it.
238  Association *associate(llvm::StringRef MainFile,
239  llvm::ArrayRef<std::string> Headers) {
240  Association *First = nullptr, *Prev = nullptr;
241  for (const std::string &Header : Headers) {
242  auto &Assoc = HeaderToMain[Header];
243  if (Assoc.Next)
244  continue; // Already has a valid association.
245 
246  Assoc.MainFile = MainFile;
247  Assoc.Next = Prev;
248  Prev = &Assoc;
249  if (!First)
250  First = &Assoc;
251  }
252  if (First)
253  First->Next = Prev;
254  return First;
255  }
256 
257  void updateMemoryUsage() {
258  auto StringMapHeap = [](const auto &Map) {
259  // StringMap stores the hashtable on the heap.
260  // It contains pointers to the entries, and a hashcode for each.
261  return Map.getNumBuckets() * (sizeof(void *) + sizeof(unsigned));
262  };
263  size_t Usage = Arena.getTotalMemory() + StringMapHeap(MainToFirst) +
264  StringMapHeap(HeaderToMain) + sizeof(*this);
265  UsedBytes.store(Usage, std::memory_order_release);
266  }
267 
268 public:
269  HeaderIncluderCache() : HeaderToMain(Arena), MainToFirst(Arena) {
270  updateMemoryUsage();
271  }
272 
273  // Associate each header with MainFile (unless already associated).
274  // Headers not in the list will have their associations removed.
275  void update(PathRef MainFile, llvm::ArrayRef<std::string> Headers) {
276  std::lock_guard<std::mutex> Lock(Mu);
277  auto It = MainToFirst.try_emplace(MainFile, nullptr);
278  Association *&First = It.first->second;
279  if (First)
280  invalidate(First);
281  First = associate(It.first->first(), Headers);
282  updateMemoryUsage();
283  }
284 
285  // Mark MainFile as gone.
286  // This will *not* disassociate headers with MainFile immediately, but they
287  // will be eligible for association with other files that get update()d.
289  std::lock_guard<std::mutex> Lock(Mu);
290  Association *&First = MainToFirst[MainFile];
291  if (First) {
292  invalidate(First);
293  First = nullptr;
294  }
295  // MainToFirst entry should stay alive, as Associations might be pointing at
296  // its key.
297  }
298 
299  /// Get the mainfile associated with Header, or the empty string if none.
300  std::string get(PathRef Header) const {
301  std::lock_guard<std::mutex> Lock(Mu);
302  return HeaderToMain.lookup(Header).MainFile.str();
303  }
304 
305  size_t getUsedBytes() const {
306  return UsedBytes.load(std::memory_order_acquire);
307  }
308 };
309 
310 namespace {
311 
312 bool isReliable(const tooling::CompileCommand &Cmd) {
313  return Cmd.Heuristic.empty();
314 }
315 
316 /// Threadsafe manager for updating a TUStatus and emitting it after each
317 /// update.
318 class SynchronizedTUStatus {
319 public:
320  SynchronizedTUStatus(PathRef FileName, ParsingCallbacks &Callbacks)
321  : FileName(FileName), Callbacks(Callbacks) {}
322 
323  void update(llvm::function_ref<void(TUStatus &)> Mutator) {
324  std::lock_guard<std::mutex> Lock(StatusMu);
325  Mutator(Status);
326  emitStatusLocked();
327  }
328 
329  /// Prevents emitting of further updates.
330  void stop() {
331  std::lock_guard<std::mutex> Lock(StatusMu);
332  CanPublish = false;
333  }
334 
335 private:
336  void emitStatusLocked() {
337  if (CanPublish)
338  Callbacks.onFileUpdated(FileName, Status);
339  }
340 
341  const Path FileName;
342 
343  std::mutex StatusMu;
344  TUStatus Status;
345  bool CanPublish = true;
346  ParsingCallbacks &Callbacks;
347 };
348 
349 /// Responsible for building preambles. Whenever the thread is idle and the
350 /// preamble is outdated, it starts to build a fresh preamble from the latest
351 /// inputs. If RunSync is true, preambles are built synchronously in update()
352 /// instead.
353 class PreambleThread {
354 public:
355  PreambleThread(llvm::StringRef FileName, ParsingCallbacks &Callbacks,
356  bool StorePreambleInMemory, bool RunSync,
357  SynchronizedTUStatus &Status,
358  TUScheduler::HeaderIncluderCache &HeaderIncluders,
359  ASTWorker &AW)
360  : FileName(FileName), Callbacks(Callbacks),
361  StoreInMemory(StorePreambleInMemory), RunSync(RunSync), Status(Status),
362  ASTPeer(AW), HeaderIncluders(HeaderIncluders) {}
363 
364  /// It isn't guaranteed that each requested version will be built. If there
365  /// are multiple update requests while building a preamble, only the last one
366  /// will be built.
367  void update(std::unique_ptr<CompilerInvocation> CI, ParseInputs PI,
368  std::vector<Diag> CIDiags, WantDiagnostics WantDiags) {
369  Request Req = {std::move(CI), std::move(PI), std::move(CIDiags), WantDiags,
370  Context::current().clone()};
371  if (RunSync) {
372  build(std::move(Req));
373  Status.update([](TUStatus &Status) {
374  Status.PreambleActivity = PreambleAction::Idle;
375  });
376  return;
377  }
378  {
379  std::unique_lock<std::mutex> Lock(Mutex);
380  // If NextReq was requested with WantDiagnostics::Yes we cannot just drop
381  // that on the floor. Block until we start building it. This won't
382  // dead-lock as we are blocking the caller thread, while builds continue
383  // on preamble thread.
384  ReqCV.wait(Lock, [this] {
385  return !NextReq || NextReq->WantDiags != WantDiagnostics::Yes;
386  });
387  NextReq = std::move(Req);
388  }
389  // Let the worker thread know there's a request, notify_one is safe as there
390  // should be a single worker thread waiting on it.
391  ReqCV.notify_all();
392  }
393 
394  void run() {
395  while (true) {
396  {
397  std::unique_lock<std::mutex> Lock(Mutex);
398  assert(!CurrentReq && "Already processing a request?");
399  // Wait until stop is called or there is a request.
400  ReqCV.wait(Lock, [this] { return NextReq || Done; });
401  if (Done)
402  break;
403  CurrentReq = std::move(*NextReq);
404  NextReq.reset();
405  }
406 
407  {
408  WithContext Guard(std::move(CurrentReq->Ctx));
409  // Note that we don't make use of the ContextProvider here.
410  // Preamble tasks are always scheduled by ASTWorker tasks, and we
411  // reuse the context/config that was created at that level.
412 
413  // Build the preamble and let the waiters know about it.
414  build(std::move(*CurrentReq));
415  }
416  bool IsEmpty = false;
417  {
418  std::lock_guard<std::mutex> Lock(Mutex);
419  CurrentReq.reset();
420  IsEmpty = !NextReq.hasValue();
421  }
422  if (IsEmpty) {
423  // We don't perform this above, before waiting for a request to make
424  // tests more deterministic. As there can be a race between this thread
425  // and client thread(clangdserver).
426  Status.update([](TUStatus &Status) {
427  Status.PreambleActivity = PreambleAction::Idle;
428  });
429  }
430  ReqCV.notify_all();
431  }
432  dlog("Preamble worker for {0} stopped", FileName);
433  }
434 
435  /// Signals the run loop to exit.
436  void stop() {
437  dlog("Preamble worker for {0} received stop", FileName);
438  {
439  std::lock_guard<std::mutex> Lock(Mutex);
440  Done = true;
441  NextReq.reset();
442  }
443  // Let the worker thread know that it should stop.
444  ReqCV.notify_all();
445  }
446 
447  bool blockUntilIdle(Deadline Timeout) const {
448  std::unique_lock<std::mutex> Lock(Mutex);
449  return wait(Lock, ReqCV, Timeout, [&] { return !NextReq && !CurrentReq; });
450  }
451 
452 private:
453  /// Holds inputs required for building a preamble. CI is guaranteed to be
454  /// non-null.
455  struct Request {
456  std::unique_ptr<CompilerInvocation> CI;
457  ParseInputs Inputs;
458  std::vector<Diag> CIDiags;
460  Context Ctx;
461  };
462 
463  bool isDone() {
464  std::lock_guard<std::mutex> Lock(Mutex);
465  return Done;
466  }
467 
468  /// Builds a preamble for \p Req, might reuse LatestBuild if possible.
469  /// Notifies ASTWorker after build finishes.
470  void build(Request Req);
471 
472  mutable std::mutex Mutex;
473  bool Done = false; /* GUARDED_BY(Mutex) */
474  llvm::Optional<Request> NextReq; /* GUARDED_BY(Mutex) */
475  llvm::Optional<Request> CurrentReq; /* GUARDED_BY(Mutex) */
476  // Signaled whenever a thread populates NextReq or worker thread builds a
477  // Preamble.
478  mutable std::condition_variable ReqCV; /* GUARDED_BY(Mutex) */
479  // Accessed only by preamble thread.
480  std::shared_ptr<const PreambleData> LatestBuild;
481 
482  const Path FileName;
483  ParsingCallbacks &Callbacks;
484  const bool StoreInMemory;
485  const bool RunSync;
486 
487  SynchronizedTUStatus &Status;
488  ASTWorker &ASTPeer;
489  TUScheduler::HeaderIncluderCache &HeaderIncluders;
490 };
491 
492 class ASTWorkerHandle;
493 
494 /// Owns one instance of the AST, schedules updates and reads of it.
495 /// Also responsible for building and providing access to the preamble.
496 /// Each ASTWorker processes the async requests sent to it on a separate
497 /// dedicated thread.
498 /// The ASTWorker that manages the AST is shared by both the processing thread
499 /// and the TUScheduler. The TUScheduler should discard an ASTWorker when
500 /// remove() is called, but its thread may be busy and we don't want to block.
501 /// So the workers are accessed via an ASTWorkerHandle. Destroying the handle
502 /// signals the worker to exit its run loop and gives up shared ownership of the
503 /// worker.
504 class ASTWorker {
505  friend class ASTWorkerHandle;
506  ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
507  TUScheduler::ASTCache &LRUCache,
508  TUScheduler::HeaderIncluderCache &HeaderIncluders,
509  Semaphore &Barrier, bool RunSync, const TUScheduler::Options &Opts,
510  ParsingCallbacks &Callbacks);
511 
512 public:
513  /// Create a new ASTWorker and return a handle to it.
514  /// The processing thread is spawned using \p Tasks. However, when \p Tasks
515  /// is null, all requests will be processed on the calling thread
516  /// synchronously instead. \p Barrier is acquired when processing each
517  /// request, it is used to limit the number of actively running threads.
518  static ASTWorkerHandle
519  create(PathRef FileName, const GlobalCompilationDatabase &CDB,
520  TUScheduler::ASTCache &IdleASTs,
521  TUScheduler::HeaderIncluderCache &HeaderIncluders,
522  AsyncTaskRunner *Tasks, Semaphore &Barrier,
523  const TUScheduler::Options &Opts, ParsingCallbacks &Callbacks);
524  ~ASTWorker();
525 
526  void update(ParseInputs Inputs, WantDiagnostics, bool ContentChanged);
527  void
528  runWithAST(llvm::StringRef Name,
529  llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
531  bool blockUntilIdle(Deadline Timeout) const;
532 
533  std::shared_ptr<const PreambleData> getPossiblyStalePreamble(
534  std::shared_ptr<const ASTSignals> *ASTSignals = nullptr) const;
535 
536  /// Used to inform ASTWorker about a new preamble build by PreambleThread.
537  /// Diagnostics are only published through this callback. This ensures they
538  /// are always for newer versions of the file, as the callback gets called in
539  /// the same order as update requests.
540  void updatePreamble(std::unique_ptr<CompilerInvocation> CI, ParseInputs PI,
541  std::shared_ptr<const PreambleData> Preamble,
542  std::vector<Diag> CIDiags, WantDiagnostics WantDiags);
543 
544  /// Obtain a preamble reflecting all updates so far. Threadsafe.
545  /// It may be delivered immediately, or later on the worker thread.
546  void getCurrentPreamble(
547  llvm::unique_function<void(std::shared_ptr<const PreambleData>)>);
548  /// Returns compile command from the current file inputs.
549  tooling::CompileCommand getCurrentCompileCommand() const;
550 
551  /// Wait for the first build of preamble to finish. Preamble itself can be
552  /// accessed via getPossiblyStalePreamble(). Note that this function will
553  /// return after an unsuccessful build of the preamble too, i.e. result of
554  /// getPossiblyStalePreamble() can be null even after this function returns.
555  void waitForFirstPreamble() const;
556 
557  TUScheduler::FileStats stats() const;
558  bool isASTCached() const;
559 
560 private:
561  // Details of an update request that are relevant to scheduling.
562  struct UpdateType {
563  // Do we want diagnostics from this version?
564  // If Yes, we must always build this version.
565  // If No, we only need to build this version if it's read.
566  // If Auto, we build if it's read or if the debounce expires.
568  // Did the main-file content of the document change?
569  // If so, we're allowed to cancel certain invalidated preceding reads.
571  };
572 
573  /// Publishes diagnostics for \p Inputs. It will build an AST or reuse the
574  /// cached one if applicable. Assumes LatestPreamble is compatible for \p
575  /// Inputs.
576  void generateDiagnostics(std::unique_ptr<CompilerInvocation> Invocation,
577  ParseInputs Inputs, std::vector<Diag> CIDiags);
578 
579  void updateASTSignals(ParsedAST &AST);
580 
581  // Must be called exactly once on processing thread. Will return after
582  // stop() is called on a separate thread and all pending requests are
583  // processed.
584  void run();
585  /// Signal that run() should finish processing pending requests and exit.
586  void stop();
587 
588  /// Adds a new task to the end of the request queue.
589  void startTask(llvm::StringRef Name, llvm::unique_function<void()> Task,
590  llvm::Optional<UpdateType> Update,
592  /// Runs a task synchronously.
593  void runTask(llvm::StringRef Name, llvm::function_ref<void()> Task);
594 
595  /// Determines the next action to perform.
596  /// All actions that should never run are discarded.
597  /// Returns a deadline for the next action. If it's expired, run now.
598  /// scheduleLocked() is called again at the deadline, or if requests arrive.
599  Deadline scheduleLocked();
600  /// Should the first task in the queue be skipped instead of run?
601  bool shouldSkipHeadLocked() const;
602 
603  struct Request {
604  llvm::unique_function<void()> Action;
605  std::string Name;
606  steady_clock::time_point AddTime;
607  Context Ctx;
608  llvm::Optional<Context> QueueCtx;
609  llvm::Optional<UpdateType> Update;
612  };
613 
614  /// Handles retention of ASTs.
615  TUScheduler::ASTCache &IdleASTs;
616  TUScheduler::HeaderIncluderCache &HeaderIncluders;
617  const bool RunSync;
618  /// Time to wait after an update to see whether another update obsoletes it.
619  const DebouncePolicy UpdateDebounce;
620  /// File that ASTWorker is responsible for.
621  const Path FileName;
622  /// Callback to create processing contexts for tasks.
623  const std::function<Context(llvm::StringRef)> ContextProvider;
624  const GlobalCompilationDatabase &CDB;
625  /// Callback invoked when preamble or main file AST is built.
626  ParsingCallbacks &Callbacks;
627 
628  Semaphore &Barrier;
629  /// Whether the 'onMainAST' callback ran for the current FileInputs.
630  bool RanASTCallback = false;
631  /// Guards members used by both TUScheduler and the worker thread.
632  mutable std::mutex Mutex;
633  /// File inputs, currently being used by the worker.
634  /// Writes and reads from unknown threads are locked. Reads from the worker
635  /// thread are not locked, as it's the only writer.
636  ParseInputs FileInputs; /* GUARDED_BY(Mutex) */
637  /// Times of recent AST rebuilds, used for UpdateDebounce computation.
638  llvm::SmallVector<DebouncePolicy::clock::duration>
639  RebuildTimes; /* GUARDED_BY(Mutex) */
640  /// Set to true to signal run() to finish processing.
641  bool Done; /* GUARDED_BY(Mutex) */
642  std::deque<Request> Requests; /* GUARDED_BY(Mutex) */
643  llvm::Optional<Request> CurrentRequest; /* GUARDED_BY(Mutex) */
644  /// Signalled whenever a new request has been scheduled or processing of a
645  /// request has completed.
646  mutable std::condition_variable RequestsCV;
647  std::shared_ptr<const ASTSignals> LatestASTSignals; /* GUARDED_BY(Mutex) */
648  /// Latest build preamble for current TU.
649  /// None means no builds yet, null means there was an error while building.
650  /// Only written by ASTWorker's thread.
651  llvm::Optional<std::shared_ptr<const PreambleData>> LatestPreamble;
652  std::deque<Request> PreambleRequests; /* GUARDED_BY(Mutex) */
653  /// Signaled whenever LatestPreamble changes state or there's a new
654  /// PreambleRequest.
655  mutable std::condition_variable PreambleCV;
656  /// Guards the callback that publishes results of AST-related computations
657  /// (diagnostics) and file statuses.
658  std::mutex PublishMu;
659  // Used to prevent remove document + add document races that lead to
660  // out-of-order callbacks for publishing results of onMainAST callback.
661  //
662  // The lifetime of the old/new ASTWorkers will overlap, but their handles
663  // don't. When the old handle is destroyed, the old worker will stop reporting
664  // any results to the user.
665  bool CanPublishResults = true; /* GUARDED_BY(PublishMu) */
666  std::atomic<unsigned> ASTBuildCount = {0};
667  std::atomic<unsigned> PreambleBuildCount = {0};
668 
669  SynchronizedTUStatus Status;
670  PreambleThread PreamblePeer;
671 };
672 
673 /// A smart-pointer-like class that points to an active ASTWorker.
674 /// In destructor, signals to the underlying ASTWorker that no new requests will
675 /// be sent and the processing loop may exit (after running all pending
676 /// requests).
677 class ASTWorkerHandle {
678  friend class ASTWorker;
679  ASTWorkerHandle(std::shared_ptr<ASTWorker> Worker)
680  : Worker(std::move(Worker)) {
681  assert(this->Worker);
682  }
683 
684 public:
685  ASTWorkerHandle(const ASTWorkerHandle &) = delete;
686  ASTWorkerHandle &operator=(const ASTWorkerHandle &) = delete;
687  ASTWorkerHandle(ASTWorkerHandle &&) = default;
688  ASTWorkerHandle &operator=(ASTWorkerHandle &&) = default;
689 
690  ~ASTWorkerHandle() {
691  if (Worker)
692  Worker->stop();
693  }
694 
695  ASTWorker &operator*() {
696  assert(Worker && "Handle was moved from");
697  return *Worker;
698  }
699 
700  ASTWorker *operator->() {
701  assert(Worker && "Handle was moved from");
702  return Worker.get();
703  }
704 
705  /// Returns an owning reference to the underlying ASTWorker that can outlive
706  /// the ASTWorkerHandle. However, no new requests to an active ASTWorker can
707  /// be schedule via the returned reference, i.e. only reads of the preamble
708  /// are possible.
709  std::shared_ptr<const ASTWorker> lock() { return Worker; }
710 
711 private:
712  std::shared_ptr<ASTWorker> Worker;
713 };
714 
715 ASTWorkerHandle
716 ASTWorker::create(PathRef FileName, const GlobalCompilationDatabase &CDB,
717  TUScheduler::ASTCache &IdleASTs,
718  TUScheduler::HeaderIncluderCache &HeaderIncluders,
719  AsyncTaskRunner *Tasks, Semaphore &Barrier,
720  const TUScheduler::Options &Opts,
721  ParsingCallbacks &Callbacks) {
722  std::shared_ptr<ASTWorker> Worker(
723  new ASTWorker(FileName, CDB, IdleASTs, HeaderIncluders, Barrier,
724  /*RunSync=*/!Tasks, Opts, Callbacks));
725  if (Tasks) {
726  Tasks->runAsync("ASTWorker:" + llvm::sys::path::filename(FileName),
727  [Worker]() { Worker->run(); });
728  Tasks->runAsync("PreambleWorker:" + llvm::sys::path::filename(FileName),
729  [Worker]() { Worker->PreamblePeer.run(); });
730  }
731 
732  return ASTWorkerHandle(std::move(Worker));
733 }
734 
735 ASTWorker::ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
736  TUScheduler::ASTCache &LRUCache,
737  TUScheduler::HeaderIncluderCache &HeaderIncluders,
738  Semaphore &Barrier, bool RunSync,
739  const TUScheduler::Options &Opts,
740  ParsingCallbacks &Callbacks)
741  : IdleASTs(LRUCache), HeaderIncluders(HeaderIncluders), RunSync(RunSync),
742  UpdateDebounce(Opts.UpdateDebounce), FileName(FileName),
743  ContextProvider(Opts.ContextProvider), CDB(CDB), Callbacks(Callbacks),
744  Barrier(Barrier), Done(false), Status(FileName, Callbacks),
745  PreamblePeer(FileName, Callbacks, Opts.StorePreamblesInMemory, RunSync,
746  Status, HeaderIncluders, *this) {
747  // Set a fallback command because compile command can be accessed before
748  // `Inputs` is initialized. Other fields are only used after initialization
749  // from client inputs.
750  FileInputs.CompileCommand = CDB.getFallbackCommand(FileName);
751 }
752 
753 ASTWorker::~ASTWorker() {
754  // Make sure we remove the cached AST, if any.
755  IdleASTs.take(this);
756 #ifndef NDEBUG
757  std::lock_guard<std::mutex> Lock(Mutex);
758  assert(Done && "handle was not destroyed");
759  assert(Requests.empty() && !CurrentRequest &&
760  "unprocessed requests when destroying ASTWorker");
761 #endif
762 }
763 
764 void ASTWorker::update(ParseInputs Inputs, WantDiagnostics WantDiags,
765  bool ContentChanged) {
766  std::string TaskName = llvm::formatv("Update ({0})", Inputs.Version);
767  auto Task = [=]() mutable {
768  // Get the actual command as `Inputs` does not have a command.
769  // FIXME: some build systems like Bazel will take time to preparing
770  // environment to build the file, it would be nice if we could emit a
771  // "PreparingBuild" status to inform users, it is non-trivial given the
772  // current implementation.
773  auto Cmd = CDB.getCompileCommand(FileName);
774  // If we don't have a reliable command for this file, it may be a header.
775  // Try to find a file that includes it, to borrow its command.
776  if (!Cmd || !isReliable(*Cmd)) {
777  std::string ProxyFile = HeaderIncluders.get(FileName);
778  if (!ProxyFile.empty()) {
779  auto ProxyCmd = CDB.getCompileCommand(ProxyFile);
780  if (!ProxyCmd || !isReliable(*ProxyCmd)) {
781  // This command is supposed to be reliable! It's probably gone.
782  HeaderIncluders.remove(ProxyFile);
783  } else {
784  // We have a reliable command for an including file, use it.
785  Cmd = tooling::transferCompileCommand(std::move(*ProxyCmd), FileName);
786  }
787  }
788  }
789  if (Cmd)
790  Inputs.CompileCommand = std::move(*Cmd);
791  else
792  Inputs.CompileCommand = CDB.getFallbackCommand(FileName);
793 
794  bool InputsAreTheSame =
795  std::tie(FileInputs.CompileCommand, FileInputs.Contents) ==
796  std::tie(Inputs.CompileCommand, Inputs.Contents);
797  // Cached AST is invalidated.
798  if (!InputsAreTheSame) {
799  IdleASTs.take(this);
800  RanASTCallback = false;
801  }
802 
803  // Update current inputs so that subsequent reads can see them.
804  {
805  std::lock_guard<std::mutex> Lock(Mutex);
806  FileInputs = Inputs;
807  }
808 
809  log("ASTWorker building file {0} version {1} with command {2}\n[{3}]\n{4}",
810  FileName, Inputs.Version, Inputs.CompileCommand.Heuristic,
811  Inputs.CompileCommand.Directory,
812  printArgv(Inputs.CompileCommand.CommandLine));
813 
814  StoreDiags CompilerInvocationDiagConsumer;
815  std::vector<std::string> CC1Args;
816  std::unique_ptr<CompilerInvocation> Invocation = buildCompilerInvocation(
817  Inputs, CompilerInvocationDiagConsumer, &CC1Args);
818  // Log cc1 args even (especially!) if creating invocation failed.
819  if (!CC1Args.empty())
820  vlog("Driver produced command: cc1 {0}", printArgv(CC1Args));
821  std::vector<Diag> CompilerInvocationDiags =
822  CompilerInvocationDiagConsumer.take();
823  if (!Invocation) {
824  elog("Could not build CompilerInvocation for file {0}", FileName);
825  // Remove the old AST if it's still in cache.
826  IdleASTs.take(this);
827  RanASTCallback = false;
828  // Report the diagnostics we collected when parsing the command line.
829  Callbacks.onFailedAST(FileName, Inputs.Version,
830  std::move(CompilerInvocationDiags),
831  [&](llvm::function_ref<void()> Publish) {
832  // Ensure we only publish results from the worker
833  // if the file was not removed, making sure there
834  // are not race conditions.
835  std::lock_guard<std::mutex> Lock(PublishMu);
836  if (CanPublishResults)
837  Publish();
838  });
839  // Note that this might throw away a stale preamble that might still be
840  // useful, but this is how we communicate a build error.
841  LatestPreamble.emplace();
842  // Make sure anyone waiting for the preamble gets notified it could not be
843  // built.
844  PreambleCV.notify_all();
845  return;
846  }
847 
848  PreamblePeer.update(std::move(Invocation), std::move(Inputs),
849  std::move(CompilerInvocationDiags), WantDiags);
850  std::unique_lock<std::mutex> Lock(Mutex);
851  PreambleCV.wait(Lock, [this] {
852  // Block until we reiceve a preamble request, unless a preamble already
853  // exists, as patching an empty preamble would imply rebuilding it from
854  // scratch.
855  // We block here instead of the consumer to prevent any deadlocks. Since
856  // LatestPreamble is only populated by ASTWorker thread.
857  return LatestPreamble || !PreambleRequests.empty() || Done;
858  });
859  };
860  startTask(TaskName, std::move(Task), UpdateType{WantDiags, ContentChanged},
861  TUScheduler::NoInvalidation);
862 }
863 
864 void ASTWorker::runWithAST(
865  llvm::StringRef Name,
866  llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
867  TUScheduler::ASTActionInvalidation Invalidation) {
868  // Tracks ast cache accesses for read operations.
869  static constexpr trace::Metric ASTAccessForRead(
870  "ast_access_read", trace::Metric::Counter, "result");
871  auto Task = [=, Action = std::move(Action)]() mutable {
872  if (auto Reason = isCancelled())
873  return Action(llvm::make_error<CancelledError>(Reason));
874  llvm::Optional<std::unique_ptr<ParsedAST>> AST =
875  IdleASTs.take(this, &ASTAccessForRead);
876  if (!AST) {
877  StoreDiags CompilerInvocationDiagConsumer;
878  std::unique_ptr<CompilerInvocation> Invocation =
879  buildCompilerInvocation(FileInputs, CompilerInvocationDiagConsumer);
880  // Try rebuilding the AST.
881  vlog("ASTWorker rebuilding evicted AST to run {0}: {1} version {2}", Name,
882  FileName, FileInputs.Version);
883  // FIXME: We might need to build a patched ast once preamble thread starts
884  // running async. Currently getPossiblyStalePreamble below will always
885  // return a compatible preamble as ASTWorker::update blocks.
886  llvm::Optional<ParsedAST> NewAST;
887  if (Invocation) {
888  NewAST = ParsedAST::build(FileName, FileInputs, std::move(Invocation),
889  CompilerInvocationDiagConsumer.take(),
890  getPossiblyStalePreamble());
891  ++ASTBuildCount;
892  }
893  AST = NewAST ? std::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;
894  }
895  // Make sure we put the AST back into the LRU cache.
896  auto _ = llvm::make_scope_exit(
897  [&AST, this]() { IdleASTs.put(this, std::move(*AST)); });
898  // Run the user-provided action.
899  if (!*AST)
900  return Action(error(llvm::errc::invalid_argument, "invalid AST"));
901  vlog("ASTWorker running {0} on version {2} of {1}", Name, FileName,
902  FileInputs.Version);
903  Action(InputsAndAST{FileInputs, **AST});
904  };
905  startTask(Name, std::move(Task), /*Update=*/None, Invalidation);
906 }
907 
908 /// To be called from ThreadCrashReporter's signal handler.
909 static void crashDumpCompileCommand(llvm::raw_ostream &OS,
910  const tooling::CompileCommand &Command) {
911  OS << " Filename: " << Command.Filename << "\n";
912  OS << " Directory: " << Command.Directory << "\n";
913  OS << " Command Line:";
914  for (auto &Arg : Command.CommandLine) {
915  OS << " " << Arg;
916  }
917  OS << "\n";
918 }
919 
920 /// To be called from ThreadCrashReporter's signal handler.
921 static void crashDumpFileContents(llvm::raw_ostream &OS,
922  const std::string &Contents) {
923  // Avoid flooding the terminal with source code by default, but allow clients
924  // to opt in. Use an env var to preserve backwards compatibility of the
925  // command line interface, while allowing it to be set outside the clangd
926  // launch site for more flexibility.
927  if (getenv("CLANGD_CRASH_DUMP_SOURCE")) {
928  OS << " Contents:\n";
929  OS << Contents << "\n";
930  }
931 }
932 
933 /// To be called from ThreadCrashReporter's signal handler.
934 static void crashDumpParseInputs(llvm::raw_ostream &OS,
935  const ParseInputs &FileInputs) {
936  auto &Command = FileInputs.CompileCommand;
937  crashDumpCompileCommand(OS, Command);
938  OS << " Version: " << FileInputs.Version << "\n";
939  crashDumpFileContents(OS, FileInputs.Contents);
940 }
941 
942 void PreambleThread::build(Request Req) {
943  assert(Req.CI && "Got preamble request with null compiler invocation");
944  const ParseInputs &Inputs = Req.Inputs;
945  bool ReusedPreamble = false;
946 
947  Status.update([&](TUStatus &Status) {
948  Status.PreambleActivity = PreambleAction::Building;
949  });
950  auto _ = llvm::make_scope_exit([this, &Req, &ReusedPreamble] {
951  ASTPeer.updatePreamble(std::move(Req.CI), std::move(Req.Inputs),
952  LatestBuild, std::move(Req.CIDiags),
953  std::move(Req.WantDiags));
954  if (!ReusedPreamble)
955  Callbacks.onPreamblePublished(FileName);
956  });
957 
958  if (!LatestBuild || Inputs.ForceRebuild) {
959  vlog("Building first preamble for {0} version {1}", FileName,
960  Inputs.Version);
961  } else if (isPreambleCompatible(*LatestBuild, Inputs, FileName, *Req.CI)) {
962  vlog("Reusing preamble version {0} for version {1} of {2}",
963  LatestBuild->Version, Inputs.Version, FileName);
964  ReusedPreamble = true;
965  return;
966  } else {
967  vlog("Rebuilding invalidated preamble for {0} version {1} (previous was "
968  "version {2})",
969  FileName, Inputs.Version, LatestBuild->Version);
970  }
971 
972  ThreadCrashReporter ScopedReporter([&Inputs]() {
973  llvm::errs() << "Signalled while building preamble\n";
974  crashDumpParseInputs(llvm::errs(), Inputs);
975  });
976 
977  LatestBuild = clang::clangd::buildPreamble(
978  FileName, *Req.CI, Inputs, StoreInMemory,
979  [this, Version(Inputs.Version)](ASTContext &Ctx, Preprocessor &PP,
980  const CanonicalIncludes &CanonIncludes) {
981  Callbacks.onPreambleAST(FileName, Version, Ctx, PP, CanonIncludes);
982  });
983  if (LatestBuild && isReliable(LatestBuild->CompileCommand))
984  HeaderIncluders.update(FileName, LatestBuild->Includes.allHeaders());
985 }
986 
987 void ASTWorker::updatePreamble(std::unique_ptr<CompilerInvocation> CI,
988  ParseInputs PI,
989  std::shared_ptr<const PreambleData> Preamble,
990  std::vector<Diag> CIDiags,
992  llvm::StringLiteral TaskName = "Build AST";
993  // Store preamble and build diagnostics with new preamble if requested.
994  auto Task = [this, Preamble = std::move(Preamble), CI = std::move(CI),
995  PI = std::move(PI), CIDiags = std::move(CIDiags),
996  WantDiags = std::move(WantDiags)]() mutable {
997  // Update the preamble inside ASTWorker queue to ensure atomicity. As a task
998  // running inside ASTWorker assumes internals won't change until it
999  // finishes.
1000  if (!LatestPreamble || Preamble != *LatestPreamble) {
1001  ++PreambleBuildCount;
1002  // Cached AST is no longer valid.
1003  IdleASTs.take(this);
1004  RanASTCallback = false;
1005  std::lock_guard<std::mutex> Lock(Mutex);
1006  // LatestPreamble might be the last reference to old preamble, do not
1007  // trigger destructor while holding the lock.
1008  if (LatestPreamble)
1009  std::swap(*LatestPreamble, Preamble);
1010  else
1011  LatestPreamble = std::move(Preamble);
1012  }
1013  // Notify anyone waiting for a preamble.
1014  PreambleCV.notify_all();
1015  // Give up our ownership to old preamble before starting expensive AST
1016  // build.
1017  Preamble.reset();
1018  // We only need to build the AST if diagnostics were requested.
1019  if (WantDiags == WantDiagnostics::No)
1020  return;
1021  // Report diagnostics with the new preamble to ensure progress. Otherwise
1022  // diagnostics might get stale indefinitely if user keeps invalidating the
1023  // preamble.
1024  generateDiagnostics(std::move(CI), std::move(PI), std::move(CIDiags));
1025  };
1026  if (RunSync) {
1027  runTask(TaskName, Task);
1028  return;
1029  }
1030  {
1031  std::lock_guard<std::mutex> Lock(Mutex);
1032  PreambleRequests.push_back({std::move(Task), std::string(TaskName),
1033  steady_clock::now(), Context::current().clone(),
1034  llvm::None, llvm::None,
1035  TUScheduler::NoInvalidation, nullptr});
1036  }
1037  PreambleCV.notify_all();
1038  RequestsCV.notify_all();
1039 }
1040 
1041 void ASTWorker::updateASTSignals(ParsedAST &AST) {
1042  auto Signals = std::make_shared<const ASTSignals>(ASTSignals::derive(AST));
1043  // Existing readers of ASTSignals will have their copy preserved until the
1044  // read is completed. The last reader deletes the old ASTSignals.
1045  {
1046  std::lock_guard<std::mutex> Lock(Mutex);
1047  std::swap(LatestASTSignals, Signals);
1048  }
1049 }
1050 
1051 void ASTWorker::generateDiagnostics(
1052  std::unique_ptr<CompilerInvocation> Invocation, ParseInputs Inputs,
1053  std::vector<Diag> CIDiags) {
1054  // Tracks ast cache accesses for publishing diags.
1055  static constexpr trace::Metric ASTAccessForDiag(
1056  "ast_access_diag", trace::Metric::Counter, "result");
1057  assert(Invocation);
1058  assert(LatestPreamble);
1059  // No need to rebuild the AST if we won't send the diagnostics.
1060  {
1061  std::lock_guard<std::mutex> Lock(PublishMu);
1062  if (!CanPublishResults)
1063  return;
1064  }
1065  // Used to check whether we can update AST cache.
1066  bool InputsAreLatest =
1067  std::tie(FileInputs.CompileCommand, FileInputs.Contents) ==
1068  std::tie(Inputs.CompileCommand, Inputs.Contents);
1069  // Take a shortcut and don't report the diagnostics, since they should be the
1070  // same. All the clients should handle the lack of OnUpdated() call anyway to
1071  // handle empty result from buildAST.
1072  // FIXME: the AST could actually change if non-preamble includes changed,
1073  // but we choose to ignore it.
1074  if (InputsAreLatest && RanASTCallback)
1075  return;
1076 
1077  // Get the AST for diagnostics, either build it or use the cached one.
1078  std::string TaskName = llvm::formatv("Build AST ({0})", Inputs.Version);
1079  Status.update([&](TUStatus &Status) {
1080  Status.ASTActivity.K = ASTAction::Building;
1081  Status.ASTActivity.Name = std::move(TaskName);
1082  });
1083  // We might be able to reuse the last we've built for a read request.
1084  // FIXME: It might be better to not reuse this AST. That way queued AST builds
1085  // won't be required for diags.
1086  llvm::Optional<std::unique_ptr<ParsedAST>> AST =
1087  IdleASTs.take(this, &ASTAccessForDiag);
1088  if (!AST || !InputsAreLatest) {
1089  auto RebuildStartTime = DebouncePolicy::clock::now();
1090  llvm::Optional<ParsedAST> NewAST = ParsedAST::build(
1091  FileName, Inputs, std::move(Invocation), CIDiags, *LatestPreamble);
1092  auto RebuildDuration = DebouncePolicy::clock::now() - RebuildStartTime;
1093  ++ASTBuildCount;
1094  // Try to record the AST-build time, to inform future update debouncing.
1095  // This is best-effort only: if the lock is held, don't bother.
1096  std::unique_lock<std::mutex> Lock(Mutex, std::try_to_lock);
1097  if (Lock.owns_lock()) {
1098  // Do not let RebuildTimes grow beyond its small-size (i.e.
1099  // capacity).
1100  if (RebuildTimes.size() == RebuildTimes.capacity())
1101  RebuildTimes.erase(RebuildTimes.begin());
1102  RebuildTimes.push_back(RebuildDuration);
1103  Lock.unlock();
1104  }
1105  Status.update([&](TUStatus &Status) {
1106  Status.Details.ReuseAST = false;
1107  Status.Details.BuildFailed = !NewAST.hasValue();
1108  });
1109  AST = NewAST ? std::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;
1110  } else {
1111  log("Skipping rebuild of the AST for {0}, inputs are the same.", FileName);
1112  Status.update([](TUStatus &Status) {
1113  Status.Details.ReuseAST = true;
1114  Status.Details.BuildFailed = false;
1115  });
1116  }
1117 
1118  // Publish diagnostics.
1119  auto RunPublish = [&](llvm::function_ref<void()> Publish) {
1120  // Ensure we only publish results from the worker if the file was not
1121  // removed, making sure there are not race conditions.
1122  std::lock_guard<std::mutex> Lock(PublishMu);
1123  if (CanPublishResults)
1124  Publish();
1125  };
1126  if (*AST) {
1127  trace::Span Span("Running main AST callback");
1128  Callbacks.onMainAST(FileName, **AST, RunPublish);
1129  updateASTSignals(**AST);
1130  } else {
1131  // Failed to build the AST, at least report diagnostics from the
1132  // command line if there were any.
1133  // FIXME: we might have got more errors while trying to build the
1134  // AST, surface them too.
1135  Callbacks.onFailedAST(FileName, Inputs.Version, CIDiags, RunPublish);
1136  }
1137 
1138  // AST might've been built for an older version of the source, as ASTWorker
1139  // queue raced ahead while we were waiting on the preamble. In that case the
1140  // queue can't reuse the AST.
1141  if (InputsAreLatest) {
1142  RanASTCallback = *AST != nullptr;
1143  IdleASTs.put(this, std::move(*AST));
1144  }
1145 }
1146 
1147 std::shared_ptr<const PreambleData> ASTWorker::getPossiblyStalePreamble(
1148  std::shared_ptr<const ASTSignals> *ASTSignals) const {
1149  std::lock_guard<std::mutex> Lock(Mutex);
1150  if (ASTSignals)
1151  *ASTSignals = LatestASTSignals;
1152  return LatestPreamble ? *LatestPreamble : nullptr;
1153 }
1154 
1155 void ASTWorker::waitForFirstPreamble() const {
1156  std::unique_lock<std::mutex> Lock(Mutex);
1157  PreambleCV.wait(Lock, [this] { return LatestPreamble.hasValue() || Done; });
1158 }
1159 
1160 tooling::CompileCommand ASTWorker::getCurrentCompileCommand() const {
1161  std::unique_lock<std::mutex> Lock(Mutex);
1162  return FileInputs.CompileCommand;
1163 }
1164 
1165 TUScheduler::FileStats ASTWorker::stats() const {
1166  TUScheduler::FileStats Result;
1167  Result.ASTBuilds = ASTBuildCount;
1168  Result.PreambleBuilds = PreambleBuildCount;
1169  // Note that we don't report the size of ASTs currently used for processing
1170  // the in-flight requests. We used this information for debugging purposes
1171  // only, so this should be fine.
1172  Result.UsedBytesAST = IdleASTs.getUsedBytes(this);
1173  if (auto Preamble = getPossiblyStalePreamble())
1174  Result.UsedBytesPreamble = Preamble->Preamble.getSize();
1175  return Result;
1176 }
1177 
1178 bool ASTWorker::isASTCached() const { return IdleASTs.getUsedBytes(this) != 0; }
1179 
1180 void ASTWorker::stop() {
1181  {
1182  std::lock_guard<std::mutex> Lock(PublishMu);
1183  CanPublishResults = false;
1184  }
1185  {
1186  std::lock_guard<std::mutex> Lock(Mutex);
1187  assert(!Done && "stop() called twice");
1188  Done = true;
1189  }
1190  PreamblePeer.stop();
1191  // We are no longer going to build any preambles, let the waiters know that.
1192  PreambleCV.notify_all();
1193  Status.stop();
1194  RequestsCV.notify_all();
1195 }
1196 
1197 void ASTWorker::runTask(llvm::StringRef Name, llvm::function_ref<void()> Task) {
1198  ThreadCrashReporter ScopedReporter([this, Name]() {
1199  llvm::errs() << "Signalled during AST worker action: " << Name << "\n";
1200  crashDumpParseInputs(llvm::errs(), FileInputs);
1201  });
1202  trace::Span Tracer(Name);
1203  WithContext WithProvidedContext(ContextProvider(FileName));
1204  Task();
1205 }
1206 
1207 void ASTWorker::startTask(llvm::StringRef Name,
1208  llvm::unique_function<void()> Task,
1209  llvm::Optional<UpdateType> Update,
1210  TUScheduler::ASTActionInvalidation Invalidation) {
1211  if (RunSync) {
1212  assert(!Done && "running a task after stop()");
1213  runTask(Name, Task);
1214  return;
1215  }
1216 
1217  {
1218  std::lock_guard<std::mutex> Lock(Mutex);
1219  assert(!Done && "running a task after stop()");
1220  // Cancel any requests invalidated by this request.
1221  if (Update && Update->ContentChanged) {
1222  for (auto &R : llvm::reverse(Requests)) {
1223  if (R.InvalidationPolicy == TUScheduler::InvalidateOnUpdate)
1224  R.Invalidate();
1225  if (R.Update && R.Update->ContentChanged)
1226  break; // Older requests were already invalidated by the older update.
1227  }
1228  }
1229 
1230  // Allow this request to be cancelled if invalidated.
1231  Context Ctx = Context::current().derive(kFileBeingProcessed, FileName);
1232  Canceler Invalidate = nullptr;
1233  if (Invalidation) {
1234  WithContext WC(std::move(Ctx));
1235  std::tie(Ctx, Invalidate) = cancelableTask(
1236  /*Reason=*/static_cast<int>(ErrorCode::ContentModified));
1237  }
1238  // Trace the time the request spends in the queue, and the requests that
1239  // it's going to wait for.
1240  llvm::Optional<Context> QueueCtx;
1241  if (trace::enabled()) {
1242  // Tracers that follow threads and need strict nesting will see a tiny
1243  // instantaneous event "we're enqueueing", and sometime later it runs.
1244  WithContext WC(Ctx.clone());
1245  trace::Span Tracer("Queued:" + Name);
1246  if (Tracer.Args) {
1247  if (CurrentRequest)
1248  SPAN_ATTACH(Tracer, "CurrentRequest", CurrentRequest->Name);
1249  llvm::json::Array PreambleRequestsNames;
1250  for (const auto &Req : PreambleRequests)
1251  PreambleRequestsNames.push_back(Req.Name);
1252  SPAN_ATTACH(Tracer, "PreambleRequestsNames",
1253  std::move(PreambleRequestsNames));
1254  llvm::json::Array RequestsNames;
1255  for (const auto &Req : Requests)
1256  RequestsNames.push_back(Req.Name);
1257  SPAN_ATTACH(Tracer, "RequestsNames", std::move(RequestsNames));
1258  }
1259  // For tracers that follow contexts, keep the trace span's context alive
1260  // until we dequeue the request, so they see the full duration.
1261  QueueCtx = Context::current().clone();
1262  }
1263  Requests.push_back({std::move(Task), std::string(Name), steady_clock::now(),
1264  std::move(Ctx), std::move(QueueCtx), Update,
1265  Invalidation, std::move(Invalidate)});
1266  }
1267  RequestsCV.notify_all();
1268 }
1269 
1270 void ASTWorker::run() {
1271  while (true) {
1272  {
1273  std::unique_lock<std::mutex> Lock(Mutex);
1274  assert(!CurrentRequest && "A task is already running, multiple workers?");
1275  for (auto Wait = scheduleLocked(); !Wait.expired();
1276  Wait = scheduleLocked()) {
1277  assert(PreambleRequests.empty() &&
1278  "Preamble updates should be scheduled immediately");
1279  if (Done) {
1280  if (Requests.empty())
1281  return;
1282  // Even though Done is set, finish pending requests.
1283  break; // However, skip delays to shutdown fast.
1284  }
1285 
1286  // Tracing: we have a next request, attribute this sleep to it.
1287  llvm::Optional<WithContext> Ctx;
1288  llvm::Optional<trace::Span> Tracer;
1289  if (!Requests.empty()) {
1290  Ctx.emplace(Requests.front().Ctx.clone());
1291  Tracer.emplace("Debounce");
1292  SPAN_ATTACH(*Tracer, "next_request", Requests.front().Name);
1293  if (!(Wait == Deadline::infinity())) {
1294  Status.update([&](TUStatus &Status) {
1295  Status.ASTActivity.K = ASTAction::Queued;
1296  Status.ASTActivity.Name = Requests.front().Name;
1297  });
1298  SPAN_ATTACH(*Tracer, "sleep_ms",
1299  std::chrono::duration_cast<std::chrono::milliseconds>(
1300  Wait.time() - steady_clock::now())
1301  .count());
1302  }
1303  }
1304 
1305  wait(Lock, RequestsCV, Wait);
1306  }
1307  // Any request in ReceivedPreambles is at least as old as the
1308  // Requests.front(), so prefer them first to preserve LSP order.
1309  if (!PreambleRequests.empty()) {
1310  CurrentRequest = std::move(PreambleRequests.front());
1311  PreambleRequests.pop_front();
1312  } else {
1313  CurrentRequest = std::move(Requests.front());
1314  Requests.pop_front();
1315  }
1316  } // unlock Mutex
1317 
1318  // Inform tracing that the request was dequeued.
1319  CurrentRequest->QueueCtx.reset();
1320 
1321  // It is safe to perform reads to CurrentRequest without holding the lock as
1322  // only writer is also this thread.
1323  {
1324  std::unique_lock<Semaphore> Lock(Barrier, std::try_to_lock);
1325  if (!Lock.owns_lock()) {
1326  Status.update([&](TUStatus &Status) {
1327  Status.ASTActivity.K = ASTAction::Queued;
1328  Status.ASTActivity.Name = CurrentRequest->Name;
1329  });
1330  Lock.lock();
1331  }
1332  WithContext Guard(std::move(CurrentRequest->Ctx));
1333  Status.update([&](TUStatus &Status) {
1334  Status.ASTActivity.K = ASTAction::RunningAction;
1335  Status.ASTActivity.Name = CurrentRequest->Name;
1336  });
1337  runTask(CurrentRequest->Name, CurrentRequest->Action);
1338  }
1339 
1340  bool IsEmpty = false;
1341  {
1342  std::lock_guard<std::mutex> Lock(Mutex);
1343  CurrentRequest.reset();
1344  IsEmpty = Requests.empty() && PreambleRequests.empty();
1345  }
1346  if (IsEmpty) {
1347  Status.update([&](TUStatus &Status) {
1348  Status.ASTActivity.K = ASTAction::Idle;
1349  Status.ASTActivity.Name = "";
1350  });
1351  }
1352  RequestsCV.notify_all();
1353  }
1354 }
1355 
1356 Deadline ASTWorker::scheduleLocked() {
1357  // Process new preambles immediately.
1358  if (!PreambleRequests.empty())
1359  return Deadline::zero();
1360  if (Requests.empty())
1361  return Deadline::infinity(); // Wait for new requests.
1362  // Handle cancelled requests first so the rest of the scheduler doesn't.
1363  for (auto I = Requests.begin(), E = Requests.end(); I != E; ++I) {
1364  if (!isCancelled(I->Ctx)) {
1365  // Cancellations after the first read don't affect current scheduling.
1366  if (I->Update == None)
1367  break;
1368  continue;
1369  }
1370  // Cancelled reads are moved to the front of the queue and run immediately.
1371  if (I->Update == None) {
1372  Request R = std::move(*I);
1373  Requests.erase(I);
1374  Requests.push_front(std::move(R));
1375  return Deadline::zero();
1376  }
1377  // Cancelled updates are downgraded to auto-diagnostics, and may be elided.
1378  if (I->Update->Diagnostics == WantDiagnostics::Yes)
1379  I->Update->Diagnostics = WantDiagnostics::Auto;
1380  }
1381 
1382  while (shouldSkipHeadLocked()) {
1383  vlog("ASTWorker skipping {0} for {1}", Requests.front().Name, FileName);
1384  Requests.pop_front();
1385  }
1386  assert(!Requests.empty() && "skipped the whole queue");
1387  // Some updates aren't dead yet, but never end up being used.
1388  // e.g. the first keystroke is live until obsoleted by the second.
1389  // We debounce "maybe-unused" writes, sleeping in case they become dead.
1390  // But don't delay reads (including updates where diagnostics are needed).
1391  for (const auto &R : Requests)
1392  if (R.Update == None || R.Update->Diagnostics == WantDiagnostics::Yes)
1393  return Deadline::zero();
1394  // Front request needs to be debounced, so determine when we're ready.
1395  Deadline D(Requests.front().AddTime + UpdateDebounce.compute(RebuildTimes));
1396  return D;
1397 }
1398 
1399 // Returns true if Requests.front() is a dead update that can be skipped.
1400 bool ASTWorker::shouldSkipHeadLocked() const {
1401  assert(!Requests.empty());
1402  auto Next = Requests.begin();
1403  auto Update = Next->Update;
1404  if (!Update) // Only skip updates.
1405  return false;
1406  ++Next;
1407  // An update is live if its AST might still be read.
1408  // That is, if it's not immediately followed by another update.
1409  if (Next == Requests.end() || !Next->Update)
1410  return false;
1411  // The other way an update can be live is if its diagnostics might be used.
1412  switch (Update->Diagnostics) {
1413  case WantDiagnostics::Yes:
1414  return false; // Always used.
1415  case WantDiagnostics::No:
1416  return true; // Always dead.
1417  case WantDiagnostics::Auto:
1418  // Used unless followed by an update that generates diagnostics.
1419  for (; Next != Requests.end(); ++Next)
1420  if (Next->Update && Next->Update->Diagnostics != WantDiagnostics::No)
1421  return true; // Prefer later diagnostics.
1422  return false;
1423  }
1424  llvm_unreachable("Unknown WantDiagnostics");
1425 }
1426 
1427 bool ASTWorker::blockUntilIdle(Deadline Timeout) const {
1428  auto WaitUntilASTWorkerIsIdle = [&] {
1429  std::unique_lock<std::mutex> Lock(Mutex);
1430  return wait(Lock, RequestsCV, Timeout, [&] {
1431  return PreambleRequests.empty() && Requests.empty() && !CurrentRequest;
1432  });
1433  };
1434  // Make sure ASTWorker has processed all requests, which might issue new
1435  // updates to PreamblePeer.
1436  if (!WaitUntilASTWorkerIsIdle())
1437  return false;
1438  // Now that ASTWorker processed all requests, ensure PreamblePeer has served
1439  // all update requests. This might create new PreambleRequests for the
1440  // ASTWorker.
1441  if (!PreamblePeer.blockUntilIdle(Timeout))
1442  return false;
1443  assert(Requests.empty() &&
1444  "No new normal tasks can be scheduled concurrently with "
1445  "blockUntilIdle(): ASTWorker isn't threadsafe");
1446  // Finally make sure ASTWorker has processed all of the preamble updates.
1447  return WaitUntilASTWorkerIsIdle();
1448 }
1449 
1450 // Render a TUAction to a user-facing string representation.
1451 // TUAction represents clangd-internal states, we don't intend to expose them
1452 // to users (say C++ programmers) directly to avoid confusion, we use terms that
1453 // are familiar by C++ programmers.
1454 std::string renderTUAction(const PreambleAction PA, const ASTAction &AA) {
1455  llvm::SmallVector<std::string, 2> Result;
1456  switch (PA) {
1457  case PreambleAction::Building:
1458  Result.push_back("parsing includes");
1459  break;
1460  case PreambleAction::Idle:
1461  // We handle idle specially below.
1462  break;
1463  }
1464  switch (AA.K) {
1465  case ASTAction::Queued:
1466  Result.push_back("file is queued");
1467  break;
1468  case ASTAction::RunningAction:
1469  Result.push_back("running " + AA.Name);
1470  break;
1471  case ASTAction::Building:
1472  Result.push_back("parsing main file");
1473  break;
1474  case ASTAction::Idle:
1475  // We handle idle specially below.
1476  break;
1477  }
1478  if (Result.empty())
1479  return "idle";
1480  return llvm::join(Result, ", ");
1481 }
1482 
1483 } // namespace
1484 
1486  return llvm::heavyweight_hardware_concurrency().compute_thread_count();
1487 }
1488 
1489 FileStatus TUStatus::render(PathRef File) const {
1490  FileStatus FStatus;
1491  FStatus.uri = URIForFile::canonicalize(File, /*TUPath=*/File);
1492  FStatus.state = renderTUAction(PreambleActivity, ASTActivity);
1493  return FStatus;
1494 }
1495 
1497  /// Latest inputs, passed to TUScheduler::update().
1498  std::string Contents;
1499  ASTWorkerHandle Worker;
1500 };
1501 
1502 TUScheduler::TUScheduler(const GlobalCompilationDatabase &CDB,
1503  const Options &Opts,
1504  std::unique_ptr<ParsingCallbacks> Callbacks)
1505  : CDB(CDB), Opts(Opts),
1506  Callbacks(Callbacks ? move(Callbacks)
1507  : std::make_unique<ParsingCallbacks>()),
1508  Barrier(Opts.AsyncThreadsCount), QuickRunBarrier(Opts.AsyncThreadsCount),
1509  IdleASTs(
1510  std::make_unique<ASTCache>(Opts.RetentionPolicy.MaxRetainedASTs)),
1511  HeaderIncluders(std::make_unique<HeaderIncluderCache>()) {
1512  // Avoid null checks everywhere.
1513  if (!Opts.ContextProvider) {
1514  this->Opts.ContextProvider = [](llvm::StringRef) {
1515  return Context::current().clone();
1516  };
1517  }
1518  if (0 < Opts.AsyncThreadsCount) {
1519  PreambleTasks.emplace();
1520  WorkerThreads.emplace();
1521  }
1522 }
1523 
1525  // Notify all workers that they need to stop.
1526  Files.clear();
1527 
1528  // Wait for all in-flight tasks to finish.
1529  if (PreambleTasks)
1530  PreambleTasks->wait();
1531  if (WorkerThreads)
1532  WorkerThreads->wait();
1533 }
1534 
1536  for (auto &File : Files)
1537  if (!File.getValue()->Worker->blockUntilIdle(D))
1538  return false;
1539  if (PreambleTasks)
1540  if (!PreambleTasks->wait(D))
1541  return false;
1542  return true;
1543 }
1544 
1547  std::unique_ptr<FileData> &FD = Files[File];
1548  bool NewFile = FD == nullptr;
1549  bool ContentChanged = false;
1550  if (!FD) {
1551  // Create a new worker to process the AST-related tasks.
1552  ASTWorkerHandle Worker =
1553  ASTWorker::create(File, CDB, *IdleASTs, *HeaderIncluders,
1554  WorkerThreads ? WorkerThreads.getPointer() : nullptr,
1555  Barrier, Opts, *Callbacks);
1556  FD = std::unique_ptr<FileData>(
1557  new FileData{Inputs.Contents, std::move(Worker)});
1558  ContentChanged = true;
1559  } else if (FD->Contents != Inputs.Contents) {
1560  ContentChanged = true;
1561  FD->Contents = Inputs.Contents;
1562  }
1563  FD->Worker->update(std::move(Inputs), WantDiags, ContentChanged);
1564  // There might be synthetic update requests, don't change the LastActiveFile
1565  // in such cases.
1566  if (ContentChanged)
1567  LastActiveFile = File.str();
1568  return NewFile;
1569 }
1570 
1572  bool Removed = Files.erase(File);
1573  if (!Removed)
1574  elog("Trying to remove file from TUScheduler that is not tracked: {0}",
1575  File);
1576  // We don't call HeaderIncluders.remove(File) here.
1577  // If we did, we'd avoid potentially stale header/mainfile associations.
1578  // However, it would mean that closing a mainfile could invalidate the
1579  // preamble of several open headers.
1580 }
1581 
1582 void TUScheduler::run(llvm::StringRef Name, llvm::StringRef Path,
1583  llvm::unique_function<void()> Action) {
1584  runWithSemaphore(Name, Path, std::move(Action), Barrier);
1585 }
1586 
1587 void TUScheduler::runQuick(llvm::StringRef Name, llvm::StringRef Path,
1588  llvm::unique_function<void()> Action) {
1589  // Use QuickRunBarrier to serialize quick tasks: we are ignoring
1590  // the parallelism level set by the user, don't abuse it
1591  runWithSemaphore(Name, Path, std::move(Action), QuickRunBarrier);
1592 }
1593 
1594 void TUScheduler::runWithSemaphore(llvm::StringRef Name, llvm::StringRef Path,
1595  llvm::unique_function<void()> Action,
1596  Semaphore &Sem) {
1597  if (Path.empty())
1598  Path = LastActiveFile;
1599  else
1600  LastActiveFile = Path.str();
1601  if (!PreambleTasks) {
1602  WithContext WithProvidedContext(Opts.ContextProvider(Path));
1603  return Action();
1604  }
1605  PreambleTasks->runAsync(Name, [this, &Sem, Ctx = Context::current().clone(),
1606  Path(Path.str()),
1607  Action = std::move(Action)]() mutable {
1608  std::lock_guard<Semaphore> BarrierLock(Sem);
1609  WithContext WC(std::move(Ctx));
1610  WithContext WithProvidedContext(Opts.ContextProvider(Path));
1611  Action();
1612  });
1613 }
1614 
1616  llvm::StringRef Name, PathRef File,
1617  llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
1618  TUScheduler::ASTActionInvalidation Invalidation) {
1619  auto It = Files.find(File);
1620  if (It == Files.end()) {
1621  Action(llvm::make_error<LSPError>(
1622  "trying to get AST for non-added document", ErrorCode::InvalidParams));
1623  return;
1624  }
1625  LastActiveFile = File.str();
1626 
1627  It->second->Worker->runWithAST(Name, std::move(Action), Invalidation);
1628 }
1629 
1630 void TUScheduler::runWithPreamble(llvm::StringRef Name, PathRef File,
1631  PreambleConsistency Consistency,
1633  auto It = Files.find(File);
1634  if (It == Files.end()) {
1635  Action(llvm::make_error<LSPError>(
1636  "trying to get preamble for non-added document",
1638  return;
1639  }
1640  LastActiveFile = File.str();
1641 
1642  if (!PreambleTasks) {
1644  SPAN_ATTACH(Tracer, "file", File);
1645  std::shared_ptr<const ASTSignals> Signals;
1646  std::shared_ptr<const PreambleData> Preamble =
1647  It->second->Worker->getPossiblyStalePreamble(&Signals);
1648  WithContext WithProvidedContext(Opts.ContextProvider(File));
1649  Action(InputsAndPreamble{It->second->Contents,
1650  It->second->Worker->getCurrentCompileCommand(),
1651  Preamble.get(), Signals.get()});
1652  return;
1653  }
1654 
1655  std::shared_ptr<const ASTWorker> Worker = It->second->Worker.lock();
1656  auto Task = [Worker, Consistency, Name = Name.str(), File = File.str(),
1657  Contents = It->second->Contents,
1658  Command = Worker->getCurrentCompileCommand(),
1660  std::string(File)),
1661  Action = std::move(Action), this]() mutable {
1662  ThreadCrashReporter ScopedReporter([&Name, &Contents, &Command]() {
1663  llvm::errs() << "Signalled during preamble action: " << Name << "\n";
1664  crashDumpCompileCommand(llvm::errs(), Command);
1665  crashDumpFileContents(llvm::errs(), Contents);
1666  });
1667  std::shared_ptr<const PreambleData> Preamble;
1668  if (Consistency == PreambleConsistency::Stale) {
1669  // Wait until the preamble is built for the first time, if preamble
1670  // is required. This avoids extra work of processing the preamble
1671  // headers in parallel multiple times.
1672  Worker->waitForFirstPreamble();
1673  }
1674  std::shared_ptr<const ASTSignals> Signals;
1675  Preamble = Worker->getPossiblyStalePreamble(&Signals);
1676 
1677  std::lock_guard<Semaphore> BarrierLock(Barrier);
1678  WithContext Guard(std::move(Ctx));
1680  SPAN_ATTACH(Tracer, "file", File);
1681  WithContext WithProvidedContext(Opts.ContextProvider(File));
1682  Action(InputsAndPreamble{Contents, Command, Preamble.get(), Signals.get()});
1683  };
1684 
1685  PreambleTasks->runAsync("task:" + llvm::sys::path::filename(File),
1686  std::move(Task));
1687 }
1688 
1689 llvm::StringMap<TUScheduler::FileStats> TUScheduler::fileStats() const {
1690  llvm::StringMap<TUScheduler::FileStats> Result;
1691  for (const auto &PathAndFile : Files)
1692  Result.try_emplace(PathAndFile.first(),
1693  PathAndFile.second->Worker->stats());
1694  return Result;
1695 }
1696 
1697 std::vector<Path> TUScheduler::getFilesWithCachedAST() const {
1698  std::vector<Path> Result;
1699  for (auto &&PathAndFile : Files) {
1700  if (!PathAndFile.second->Worker->isASTCached())
1701  continue;
1702  Result.push_back(std::string(PathAndFile.first()));
1703  }
1704  return Result;
1705 }
1706 
1707 DebouncePolicy::clock::duration
1708 DebouncePolicy::compute(llvm::ArrayRef<clock::duration> History) const {
1709  assert(Min <= Max && "Invalid policy");
1710  if (History.empty())
1711  return Max; // Arbitrary.
1712 
1713  // Base the result on the median rebuild.
1714  // nth_element needs a mutable array, take the chance to bound the data size.
1715  History = History.take_back(15);
1716  llvm::SmallVector<clock::duration, 15> Recent(History.begin(), History.end());
1717  auto Median = Recent.begin() + Recent.size() / 2;
1718  std::nth_element(Recent.begin(), Median, Recent.end());
1719 
1720  clock::duration Target =
1721  std::chrono::duration_cast<clock::duration>(RebuildRatio * *Median);
1722  if (Target > Max)
1723  return Max;
1724  if (Target < Min)
1725  return Min;
1726  return Target;
1727 }
1728 
1730  DebouncePolicy P;
1731  P.Min = P.Max = T;
1732  return P;
1733 }
1734 
1736  for (const auto &Elem : fileStats()) {
1737  MT.detail(Elem.first())
1738  .child("preamble")
1739  .addUsage(Opts.StorePreamblesInMemory ? Elem.second.UsedBytesPreamble
1740  : 0);
1741  MT.detail(Elem.first()).child("ast").addUsage(Elem.second.UsedBytesAST);
1742  MT.child("header_includer_cache").addUsage(HeaderIncluders->getUsedBytes());
1743  }
1744 }
1745 } // namespace clangd
1746 } // namespace clang
dlog
#define dlog(...)
Definition: Logger.h:102
WantDiags
WantDiagnostics WantDiags
Definition: TUScheduler.cpp:459
clang::clangd::MemoryTree::child
MemoryTree & child(llvm::StringLiteral Name)
No copy of the Name.
Definition: MemoryTree.h:41
clang::clangd::isPreambleCompatible
bool isPreambleCompatible(const PreambleData &Preamble, const ParseInputs &Inputs, PathRef FileName, const CompilerInvocation &CI)
Returns true if Preamble is reusable for Inputs.
Definition: Preamble.cpp:413
clang::clangd::cancelableTask
std::pair< Context, Canceler > cancelableTask(int Reason)
Defines a new task whose cancellation may be requested.
Definition: Cancellation.cpp:24
E
const Expr * E
Definition: AvoidBindCheck.cpp:88
clang::clangd::TUScheduler::runWithPreamble
void runWithPreamble(llvm::StringRef Name, PathRef File, PreambleConsistency Consistency, Callback< InputsAndPreamble > Action)
Schedule an async read of the preamble.
Definition: TUScheduler.cpp:1630
clang::clangd::getDefaultAsyncThreadsCount
unsigned getDefaultAsyncThreadsCount()
Returns a number of a default async threads to use for TUScheduler.
Definition: TUScheduler.cpp:1485
CIDiags
std::vector< Diag > CIDiags
Definition: TUScheduler.cpp:458
clang::clangd::printArgv
std::string printArgv(llvm::ArrayRef< llvm::StringRef > Args)
Definition: CompileCommands.cpp:615
clang::clangd::Path
std::string Path
A typedef to represent a file path.
Definition: Path.h:26
Update
llvm::Optional< UpdateType > Update
Definition: TUScheduler.cpp:609
clang::clangd::TUScheduler::~TUScheduler
~TUScheduler()
Definition: TUScheduler.cpp:1524
Usage
const char Usage[]
Definition: ClangReorderFields.cpp:50
Tracer
std::unique_ptr< trace::EventTracer > Tracer
Definition: TraceTests.cpp:164
Path.h
clang::clangd::Context::current
static const Context & current()
Returns the context for the current thread, creating it if needed.
Definition: Context.cpp:27
clang::clangd::WantDiagnostics::Yes
@ Yes
clang::clangd::TUScheduler::blockUntilIdle
bool blockUntilIdle(Deadline D) const
Wait until there are no scheduled or running tasks.
Definition: TUScheduler.cpp:1535
clang::clangd::Context::clone
Context clone() const
Clone this context object.
Definition: Context.cpp:20
Diagnostics
WantDiagnostics Diagnostics
Definition: TUScheduler.cpp:567
ContentChanged
bool ContentChanged
Definition: TUScheduler.cpp:570
clang::clangd::detail::error
llvm::Error error(std::error_code, std::string &&)
Definition: Logger.cpp:80
CI
std::unique_ptr< CompilerInvocation > CI
Definition: TUScheduler.cpp:456
QueueCtx
llvm::Optional< Context > QueueCtx
Definition: TUScheduler.cpp:608
clang::clangd::PreambleAction
PreambleAction
Definition: TUScheduler.h:91
Preamble.h
Ctx
Context Ctx
Definition: TUScheduler.cpp:460
clang::clangd::TUScheduler::run
void run(llvm::StringRef Name, llvm::StringRef Path, llvm::unique_function< void()> Action)
Schedule an async task with no dependencies.
Definition: TUScheduler.cpp:1582
Trace.h
AST
llvm::Optional< ParsedAST > AST
Definition: SymbolCollectorTests.cpp:133
clang::clangd::TUScheduler::getFilesWithCachedAST
std::vector< Path > getFilesWithCachedAST() const
Returns a list of files with ASTs currently stored in memory.
Definition: TUScheduler.cpp:1697
clang::tidy::cppcoreguidelines::join
static std::string join(ArrayRef< SpecialMemberFunctionsCheck::SpecialMemberFunctionKind > SMFS, llvm::StringRef AndOr)
Definition: SpecialMemberFunctionsCheck.cpp:78
clang::clangd::TUScheduler::update
bool update(PathRef File, ParseInputs Inputs, WantDiagnostics WD)
Schedule an update for File.
Definition: TUScheduler.cpp:1545
clang::clangd::GlobalCompilationDatabase
Provides compilation arguments used for parsing C and C++ files.
Definition: GlobalCompilationDatabase.h:36
Action
llvm::unique_function< void()> Action
Definition: TUScheduler.cpp:604
clang::clangd::TUScheduler::Options::ContextProvider
std::function< Context(PathRef)> ContextProvider
Used to create a context that wraps each single operation.
Definition: TUScheduler.h:207
Target
std::string Target
Definition: QueryDriverDatabase.cpp:64
clang::clangd::Semaphore
Limits the number of threads that can acquire the lock at the same time.
Definition: Threading.h:42
Preamble
const PreambleData & Preamble
Definition: CodeComplete.cpp:1193
Cancellation.h
clang::clangd::Deadline
A point in time we can wait for.
Definition: Threading.h:59
clang::clangd::WantDiagnostics
WantDiagnostics
Determines whether diagnostics should be generated for a file snapshot.
Definition: TUScheduler.h:53
clang::clangd::canonicalize
static llvm::SmallString< 128 > canonicalize(llvm::StringRef Path)
Definition: FileDistance.cpp:46
clang::clangd::MemoryTree::detail
MemoryTree & detail(llvm::StringRef Name)
Makes a copy of the Name in detailed mode, returns current node otherwise.
Definition: MemoryTree.h:53
Inputs
ParseInputs Inputs
Definition: TUScheduler.cpp:457
clang::clangd::isCancelled
int isCancelled(const Context &Ctx)
If the current context is within a cancelled task, returns the reason.
Definition: Cancellation.cpp:35
clang::clangd::TUScheduler::HeaderIncluderCache::HeaderIncluderCache
HeaderIncluderCache()
Definition: TUScheduler.cpp:269
clang::clangd::MemoryTree::addUsage
void addUsage(size_t Increment)
Increases size of current node by Increment.
Definition: MemoryTree.h:58
clang::clangd::wait
void wait(std::unique_lock< std::mutex > &Lock, std::condition_variable &CV, Deadline D)
Wait once on CV for the specified duration.
Definition: Threading.cpp:113
clang::clangd::ParseInputs
Information required to run clang, e.g. to parse AST or do code completion.
Definition: Compiler.h:47
CanonicalIncludes.h
ns1::ns2::D
@ D
Definition: CategoricalFeature.h:3
clang::clangd::InputsAndPreamble::Contents
llvm::StringRef Contents
Definition: TUScheduler.h:44
clang::clangd::FileStatus::state
std::string state
The human-readable string presents the current state of the file, can be shown in the UI (e....
Definition: Protocol.h:1594
clang::clangd::ErrorCode::InvalidParams
@ InvalidParams
clang::clangd::ParsingCallbacks
Definition: TUScheduler.h:129
clang::clangd::TUScheduler::HeaderIncluderCache::remove
void remove(PathRef MainFile)
Definition: TUScheduler.cpp:288
clang::clangd::MemoryTree
A tree that can be used to represent memory usage of nested components while preserving the hierarchy...
Definition: MemoryTree.h:32
GlobalCompilationDatabase.h
clang::clangd::PreambleAction::Idle
@ Idle
clang::clangd::ThreadCrashReporter
Allows setting per-thread abort/kill signal callbacks, to print additional information about the cras...
Definition: ThreadCrashReporter.h:19
clang::clangd::TUScheduler::ASTCache::ASTCache
ASTCache(unsigned MaxRetainedASTs)
Definition: TUScheduler.cpp:119
clang::clangd::buildPreamble
std::shared_ptr< const PreambleData > buildPreamble(PathRef FileName, CompilerInvocation CI, const ParseInputs &Inputs, bool StoreInMemory, PreambleParsedCallback PreambleCallback)
Build a preamble for the new inputs unless an old one can be reused.
Definition: Preamble.cpp:322
clang::clangd::trace::enabled
bool enabled()
Returns true if there is an active tracer.
Definition: Trace.cpp:284
Logger.h
Threading.h
if
if(CLANGD_ENABLE_REMOTE) generate_protos(RemoteIndexProto "Index.proto") generate_protos(MonitoringServiceProto "MonitoringService.proto" GRPC) generate_protos(RemoteIndexServiceProto "Service.proto" DEPENDS "Index.proto" GRPC) target_link_libraries(RemoteIndexServiceProto PRIVATE RemoteIndexProto MonitoringServiceProto) include_directories($
Definition: clangd/index/remote/CMakeLists.txt:1
clang::clangd::FileStatus::uri
URIForFile uri
The text document's URI.
Definition: Protocol.h:1591
Name
std::string Name
Definition: TUScheduler.cpp:605
clang::clangd::TUScheduler::HeaderIncluderCache::get
std::string get(PathRef Header) const
Get the mainfile associated with Header, or the empty string if none.
Definition: TUScheduler.cpp:300
SPAN_ATTACH
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:164
clang::clangd::TUScheduler::FileData::Contents
std::string Contents
Latest inputs, passed to TUScheduler::update().
Definition: TUScheduler.cpp:1498
clang::clangd::vlog
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:73
Diagnostics.h
clang::clangd::TUScheduler::remove
void remove(PathRef File)
Remove File from the list of tracked files and schedule removal of its resources.
Definition: TUScheduler.cpp:1571
clang::clangd::Key
Values in a Context are indexed by typed keys.
Definition: Context.h:40
FileName
StringRef FileName
Definition: KernelNameRestrictionCheck.cpp:46
clang::clangd::TUScheduler::fileStats
llvm::StringMap< FileStats > fileStats() const
Returns resources used for each of the currently open files.
Definition: TUScheduler.cpp:1689
clang::clangd::TUScheduler::ASTCache::getUsedBytes
std::size_t getUsedBytes(Key K)
Returns result of getUsedBytes() for the AST cached by K.
Definition: TUScheduler.cpp:123
clang::clangd::WithContext
WithContext replaces Context::current() with a provided scope.
Definition: Context.h:187
clang::clangd::DebouncePolicy::Min
clock::duration Min
The minimum time that we always debounce for.
Definition: TUScheduler.h:78
clang::tidy::bugprone::PP
static Preprocessor * PP
Definition: BadSignalToKillThreadCheck.cpp:29
clang::clangd::buildCompilerInvocation
std::unique_ptr< CompilerInvocation > buildCompilerInvocation(const ParseInputs &Inputs, clang::DiagnosticConsumer &D, std::vector< std::string > *CC1Args)
Builds compiler invocation that could be used to build AST or preamble.
Definition: Compiler.cpp:88
Counter
trace::Metric Counter
Definition: TraceTests.cpp:152
clang::clangd::TUScheduler::ASTCache
An LRU cache of idle ASTs.
Definition: TUScheduler.cpp:115
Compiler.h
clang::clangd::TUScheduler::runQuick
void runQuick(llvm::StringRef Name, llvm::StringRef Path, llvm::unique_function< void()> Action)
Similar to run, except the task is expected to be quick.
Definition: TUScheduler.cpp:1587
AddTime
steady_clock::time_point AddTime
Definition: TUScheduler.cpp:606
clang::clangd::Context::derive
Context derive(const Key< Type > &Key, typename std::decay< Type >::type Value) const &
Derives a child context It is safe to move or destroy a parent context after calling derive().
Definition: Context.h:119
clang::clangd::TUScheduler::profile
void profile(MemoryTree &MT) const
Definition: TUScheduler.cpp:1735
clang::clangd::PathRef
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:29
clang::clangd::TUScheduler::ASTActionInvalidation
ASTActionInvalidation
Defines how a runWithAST action is implicitly cancelled by other actions.
Definition: TUScheduler.h:256
clang::clangd::trace::Metric
Represents measurements of clangd events, e.g.
Definition: Trace.h:38
clang::clangd::InputsAndPreamble
Definition: TUScheduler.h:43
InvalidationPolicy
TUScheduler::ASTActionInvalidation InvalidationPolicy
Definition: TUScheduler.cpp:610
clang::clangd::kFileBeingProcessed
static clang::clangd::Key< std::string > kFileBeingProcessed
Definition: TUScheduler.cpp:103
clang::clangd::TUScheduler::HeaderIncluderCache
A map from header files to an opened "proxy" file that includes them.
Definition: TUScheduler.cpp:212
clang::clangd::TUScheduler::Options
Definition: TUScheduler.h:187
clang::clangd::TUScheduler::runWithAST
void runWithAST(llvm::StringRef Name, PathRef File, Callback< InputsAndAST > Action, ASTActionInvalidation=NoInvalidation)
Schedule an async read of the AST.
Definition: TUScheduler.cpp:1615
ThreadCrashReporter.h
clang::clangd::TUScheduler::ASTCache::put
void put(Key K, std::unique_ptr< ParsedAST > V)
Store the value in the pool, possibly removing the last used AST.
Definition: TUScheduler.cpp:133
clang::clangd::DebouncePolicy
Clangd may wait after an update to see if another one comes along.
Definition: TUScheduler.h:74
MemoryTree.h
clang::clangd::Canceler
std::function< void()> Canceler
A canceller requests cancellation of a task, when called.
Definition: Cancellation.h:70
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
OS
llvm::raw_string_ostream OS
Definition: TraceTests.cpp:163
Arena
llvm::BumpPtrAllocator Arena
Definition: Serialization.cpp:211
TUScheduler.h
clang::clangd::FileStatus
Clangd extension: indicates the current state of the file in clangd, sent from server via the textDoc...
Definition: Protocol.h:1589
clang::clangd::DebouncePolicy::RebuildRatio
float RebuildRatio
Target debounce, as a fraction of file rebuild time.
Definition: TUScheduler.h:83
clang::clangd::DebouncePolicy::Max
clock::duration Max
The maximum time we may debounce for.
Definition: TUScheduler.h:80
clang::clangd::TUScheduler::PreambleConsistency
PreambleConsistency
Controls whether preamble reads wait for the preamble to be up-to-date.
Definition: TUScheduler.h:279
clang::clangd::TUScheduler::FileData
Definition: TUScheduler.cpp:1496
clang::clangd::TUScheduler::HeaderIncluderCache::update
void update(PathRef MainFile, llvm::ArrayRef< std::string > Headers)
Definition: TUScheduler.cpp:275
clang::clangd::DebouncePolicy::fixed
static DebouncePolicy fixed(clock::duration)
A policy that always returns the same duration, useful for tests.
Definition: TUScheduler.cpp:1729
clang::clangd::Callback
llvm::unique_function< void(llvm::Expected< T >)> Callback
A Callback<T> is a void function that accepts Expected<T>.
Definition: Function.h:28
MainFile
std::string MainFile
Definition: HeadersTests.cpp:135
Invalidate
Canceler Invalidate
Definition: TUScheduler.cpp:611
clang::clangd::TUScheduler::FileData::Worker
ASTWorkerHandle Worker
Definition: TUScheduler.cpp:1499
clang::clangd::TUScheduler::ASTCache::take
llvm::Optional< std::unique_ptr< ParsedAST > > take(Key K, const trace::Metric *AccessMetric=nullptr)
Returns the cached value for K, or llvm::None if the value is not in the cache anymore.
Definition: TUScheduler.cpp:153
clang::clangd::elog
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:62
clang::clangd::detail::log
void log(Logger::Level L, const char *Fmt, Ts &&... Vals)
Definition: Logger.h:48
clang::clangd::DebouncePolicy::compute
clock::duration compute(llvm::ArrayRef< clock::duration > History) const
Compute the time to debounce based on this policy and recent build times.
Definition: TUScheduler.cpp:1708
K
Kind K
Definition: Rename.cpp:436
clang::clangd::TUScheduler::ASTCache::Key
const ASTWorker * Key
Definition: TUScheduler.cpp:117
clang::clangd::TUScheduler::HeaderIncluderCache::getUsedBytes
size_t getUsedBytes() const
Definition: TUScheduler.cpp:305
Context.h
clang::clangd::TUScheduler::getFileBeingProcessedInContext
static llvm::Optional< llvm::StringRef > getFileBeingProcessedInContext()
Definition: TUScheduler.cpp:105
ParsedAST.h
clang::clangd::trace::Span
Records an event whose duration is the lifetime of the Span object.
Definition: Trace.h:143