clang-tools  14.0.0git
TUScheduler.cpp
Go to the documentation of this file.
1 //===--- TUScheduler.cpp -----------------------------------------*-C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // TUScheduler manages a worker per active file. This ASTWorker processes
9 // updates (modifications to file contents) and reads (actions performed on
10 // preamble/AST) to the file.
11 //
12 // Each ASTWorker owns a dedicated thread to process updates and reads to the
13 // relevant file. Any request gets queued in FIFO order to be processed by that
14 // thread.
15 //
16 // An update request replaces current praser inputs to ensure any subsequent
17 // read sees the version of the file they were requested. It will also issue a
18 // build for new inputs.
19 //
20 // ASTWorker processes the file in two parts, a preamble and a main-file
21 // section. A preamble can be reused between multiple versions of the file until
22 // invalidated by a modification to a header, compile commands or modification
23 // to relevant part of the current file. Such a preamble is called compatible.
24 // An update is considered dead if no read was issued for that version and
25 // diagnostics weren't requested by client or could be generated for a later
26 // version of the file. ASTWorker eliminates such requests as they are
27 // redundant.
28 //
29 // In the presence of stale (non-compatible) preambles, ASTWorker won't publish
30 // diagnostics for update requests. Read requests will be served with ASTs build
31 // with stale preambles, unless the read is picky and requires a compatible
32 // preamble. In such cases it will block until new preamble is built.
33 //
34 // ASTWorker owns a PreambleThread for building preambles. If the preamble gets
35 // invalidated by an update request, a new build will be requested on
36 // PreambleThread. Since PreambleThread only receives requests for newer
37 // versions of the file, in case of multiple requests it will only build the
38 // last one and skip requests in between. Unless client force requested
39 // diagnostics(WantDiagnostics::Yes).
40 //
41 // When a new preamble is built, a "golden" AST is immediately built from that
42 // version of the file. This ensures diagnostics get updated even if the queue
43 // is full.
44 //
45 // Some read requests might just need preamble. Since preambles can be read
46 // concurrently, ASTWorker runs these requests on their own thread. These
47 // requests will receive latest build preamble, which might possibly be stale.
48 
49 #include "TUScheduler.h"
50 #include "Compiler.h"
51 #include "Diagnostics.h"
53 #include "ParsedAST.h"
54 #include "Preamble.h"
56 #include "support/Cancellation.h"
57 #include "support/Context.h"
58 #include "support/Logger.h"
59 #include "support/MemoryTree.h"
60 #include "support/Path.h"
61 #include "support/Threading.h"
62 #include "support/Trace.h"
63 #include "clang/Frontend/CompilerInvocation.h"
64 #include "clang/Tooling/CompilationDatabase.h"
65 #include "llvm/ADT/FunctionExtras.h"
66 #include "llvm/ADT/None.h"
67 #include "llvm/ADT/Optional.h"
68 #include "llvm/ADT/STLExtras.h"
69 #include "llvm/ADT/ScopeExit.h"
70 #include "llvm/ADT/SmallVector.h"
71 #include "llvm/ADT/StringExtras.h"
72 #include "llvm/ADT/StringRef.h"
73 #include "llvm/Support/Allocator.h"
74 #include "llvm/Support/Errc.h"
75 #include "llvm/Support/ErrorHandling.h"
76 #include "llvm/Support/FormatVariadic.h"
77 #include "llvm/Support/Path.h"
78 #include "llvm/Support/Threading.h"
79 #include <algorithm>
80 #include <atomic>
81 #include <chrono>
82 #include <condition_variable>
83 #include <functional>
84 #include <memory>
85 #include <mutex>
86 #include <queue>
87 #include <string>
88 #include <thread>
89 #include <type_traits>
90 #include <utility>
91 #include <vector>
92 
93 namespace clang {
94 namespace clangd {
95 using std::chrono::steady_clock;
96 
97 namespace {
98 class ASTWorker;
99 } // namespace
100 
102 
103 llvm::Optional<llvm::StringRef> TUScheduler::getFileBeingProcessedInContext() {
104  if (auto *File = Context::current().get(kFileBeingProcessed))
105  return llvm::StringRef(*File);
106  return None;
107 }
108 
109 /// An LRU cache of idle ASTs.
110 /// Because we want to limit the overall number of these we retain, the cache
111 /// owns ASTs (and may evict them) while their workers are idle.
112 /// Workers borrow ASTs when active, and return them when done.
114 public:
115  using Key = const ASTWorker *;
116 
117  ASTCache(unsigned MaxRetainedASTs) : MaxRetainedASTs(MaxRetainedASTs) {}
118 
119  /// Returns result of getUsedBytes() for the AST cached by \p K.
120  /// If no AST is cached, 0 is returned.
121  std::size_t getUsedBytes(Key K) {
122  std::lock_guard<std::mutex> Lock(Mut);
123  auto It = findByKey(K);
124  if (It == LRU.end() || !It->second)
125  return 0;
126  return It->second->getUsedBytes();
127  }
128 
129  /// Store the value in the pool, possibly removing the last used AST.
130  /// The value should not be in the pool when this function is called.
131  void put(Key K, std::unique_ptr<ParsedAST> V) {
132  std::unique_lock<std::mutex> Lock(Mut);
133  assert(findByKey(K) == LRU.end());
134 
135  LRU.insert(LRU.begin(), {K, std::move(V)});
136  if (LRU.size() <= MaxRetainedASTs)
137  return;
138  // We're past the limit, remove the last element.
139  std::unique_ptr<ParsedAST> ForCleanup = std::move(LRU.back().second);
140  LRU.pop_back();
141  // Run the expensive destructor outside the lock.
142  Lock.unlock();
143  ForCleanup.reset();
144  }
145 
146  /// Returns the cached value for \p K, or llvm::None if the value is not in
147  /// the cache anymore. If nullptr was cached for \p K, this function will
148  /// return a null unique_ptr wrapped into an optional.
149  /// If \p AccessMetric is set records whether there was a hit or miss.
150  llvm::Optional<std::unique_ptr<ParsedAST>>
151  take(Key K, const trace::Metric *AccessMetric = nullptr) {
152  // Record metric after unlocking the mutex.
153  std::unique_lock<std::mutex> Lock(Mut);
154  auto Existing = findByKey(K);
155  if (Existing == LRU.end()) {
156  if (AccessMetric)
157  AccessMetric->record(1, "miss");
158  return None;
159  }
160  if (AccessMetric)
161  AccessMetric->record(1, "hit");
162  std::unique_ptr<ParsedAST> V = std::move(Existing->second);
163  LRU.erase(Existing);
164  // GCC 4.8 fails to compile `return V;`, as it tries to call the copy
165  // constructor of unique_ptr, so we call the move ctor explicitly to avoid
166  // this miscompile.
167  return llvm::Optional<std::unique_ptr<ParsedAST>>(std::move(V));
168  }
169 
170 private:
171  using KVPair = std::pair<Key, std::unique_ptr<ParsedAST>>;
172 
173  std::vector<KVPair>::iterator findByKey(Key K) {
174  return llvm::find_if(LRU, [K](const KVPair &P) { return P.first == K; });
175  }
176 
177  std::mutex Mut;
178  unsigned MaxRetainedASTs;
179  /// Items sorted in LRU order, i.e. first item is the most recently accessed
180  /// one.
181  std::vector<KVPair> LRU; /* GUARDED_BY(Mut) */
182 };
183 
184 /// A map from header files to an opened "proxy" file that includes them.
185 /// If you open the header, the compile command from the proxy file is used.
186 ///
187 /// This inclusion information could also naturally live in the index, but there
188 /// are advantages to using open files instead:
189 /// - it's easier to achieve a *stable* choice of proxy, which is important
190 /// to avoid invalidating the preamble
191 /// - context-sensitive flags for libraries with multiple configurations
192 /// (e.g. C++ stdlib sensitivity to -std version)
193 /// - predictable behavior, e.g. guarantees that go-to-def landing on a header
194 /// will have a suitable command available
195 /// - fewer scaling problems to solve (project include graphs are big!)
196 ///
197 /// Implementation details:
198 /// - We only record this for mainfiles where the command was trustworthy
199 /// (i.e. not inferred). This avoids a bad inference "infecting" other files.
200 /// - Once we've picked a proxy file for a header, we stick with it until the
201 /// proxy file is invalidated *and* a new candidate proxy file is built.
202 /// Switching proxies is expensive, as the compile flags will (probably)
203 /// change and therefore we'll end up rebuilding the header's preamble.
204 /// - We don't capture the actual compile command, but just the filename we
205 /// should query to get it. This avoids getting out of sync with the CDB.
206 ///
207 /// All methods are threadsafe. In practice, update() comes from preamble
208 /// threads, remove()s mostly from the main thread, and get() from ASTWorker.
209 /// Writes are rare and reads are cheap, so we don't expect much contention.
211  // We should be be a little careful how we store the include graph of open
212  // files, as each can have a large number of transitive headers.
213  // This representation is O(unique transitive source files).
214  llvm::BumpPtrAllocator Arena;
215  struct Association {
216  llvm::StringRef MainFile;
217  // Circular-linked-list of associations with the same mainFile.
218  // Null indicates that the mainfile was removed.
219  Association *Next;
220  };
221  llvm::StringMap<Association, llvm::BumpPtrAllocator &> HeaderToMain;
222  llvm::StringMap<Association *, llvm::BumpPtrAllocator &> MainToFirst;
223  std::atomic<size_t> UsedBytes; // Updated after writes.
224  mutable std::mutex Mu;
225 
226  void invalidate(Association *First) {
227  Association *Current = First;
228  do {
229  Association *Next = Current->Next;
230  Current->Next = nullptr;
231  Current = Next;
232  } while (Current != First);
233  }
234 
235  // Create the circular list and return the head of it.
236  Association *associate(llvm::StringRef MainFile,
237  llvm::ArrayRef<std::string> Headers) {
238  Association *First = nullptr, *Prev = nullptr;
239  for (const std::string &Header : Headers) {
240  auto &Assoc = HeaderToMain[Header];
241  if (Assoc.Next)
242  continue; // Already has a valid association.
243 
244  Assoc.MainFile = MainFile;
245  Assoc.Next = Prev;
246  Prev = &Assoc;
247  if (!First)
248  First = &Assoc;
249  }
250  if (First)
251  First->Next = Prev;
252  return First;
253  }
254 
255  void updateMemoryUsage() {
256  auto StringMapHeap = [](const auto &Map) {
257  // StringMap stores the hashtable on the heap.
258  // It contains pointers to the entries, and a hashcode for each.
259  return Map.getNumBuckets() * (sizeof(void *) + sizeof(unsigned));
260  };
261  size_t Usage = Arena.getTotalMemory() + StringMapHeap(MainToFirst) +
262  StringMapHeap(HeaderToMain) + sizeof(*this);
263  UsedBytes.store(Usage, std::memory_order_release);
264  }
265 
266 public:
267  HeaderIncluderCache() : HeaderToMain(Arena), MainToFirst(Arena) {
268  updateMemoryUsage();
269  }
270 
271  // Associate each header with MainFile (unless already associated).
272  // Headers not in the list will have their associations removed.
273  void update(PathRef MainFile, llvm::ArrayRef<std::string> Headers) {
274  std::lock_guard<std::mutex> Lock(Mu);
275  auto It = MainToFirst.try_emplace(MainFile, nullptr);
276  Association *&First = It.first->second;
277  if (First)
278  invalidate(First);
279  First = associate(It.first->first(), Headers);
280  updateMemoryUsage();
281  }
282 
283  // Mark MainFile as gone.
284  // This will *not* disassociate headers with MainFile immediately, but they
285  // will be eligible for association with other files that get update()d.
287  std::lock_guard<std::mutex> Lock(Mu);
288  Association *&First = MainToFirst[MainFile];
289  if (First)
290  invalidate(First);
291  }
292 
293  /// Get the mainfile associated with Header, or the empty string if none.
294  std::string get(PathRef Header) const {
295  std::lock_guard<std::mutex> Lock(Mu);
296  return HeaderToMain.lookup(Header).MainFile.str();
297  }
298 
299  size_t getUsedBytes() const {
300  return UsedBytes.load(std::memory_order_acquire);
301  }
302 };
303 
304 namespace {
305 
306 bool isReliable(const tooling::CompileCommand &Cmd) {
307  return Cmd.Heuristic.empty();
308 }
309 
310 /// Threadsafe manager for updating a TUStatus and emitting it after each
311 /// update.
312 class SynchronizedTUStatus {
313 public:
314  SynchronizedTUStatus(PathRef FileName, ParsingCallbacks &Callbacks)
315  : FileName(FileName), Callbacks(Callbacks) {}
316 
317  void update(llvm::function_ref<void(TUStatus &)> Mutator) {
318  std::lock_guard<std::mutex> Lock(StatusMu);
319  Mutator(Status);
320  emitStatusLocked();
321  }
322 
323  /// Prevents emitting of further updates.
324  void stop() {
325  std::lock_guard<std::mutex> Lock(StatusMu);
326  CanPublish = false;
327  }
328 
329 private:
330  void emitStatusLocked() {
331  if (CanPublish)
332  Callbacks.onFileUpdated(FileName, Status);
333  }
334 
335  const Path FileName;
336 
337  std::mutex StatusMu;
338  TUStatus Status;
339  bool CanPublish = true;
340  ParsingCallbacks &Callbacks;
341 };
342 
343 /// Responsible for building preambles. Whenever the thread is idle and the
344 /// preamble is outdated, it starts to build a fresh preamble from the latest
345 /// inputs. If RunSync is true, preambles are built synchronously in update()
346 /// instead.
347 class PreambleThread {
348 public:
349  PreambleThread(llvm::StringRef FileName, ParsingCallbacks &Callbacks,
350  bool StorePreambleInMemory, bool RunSync,
351  SynchronizedTUStatus &Status,
352  TUScheduler::HeaderIncluderCache &HeaderIncluders,
353  ASTWorker &AW)
354  : FileName(FileName), Callbacks(Callbacks),
355  StoreInMemory(StorePreambleInMemory), RunSync(RunSync), Status(Status),
356  ASTPeer(AW), HeaderIncluders(HeaderIncluders) {}
357 
358  /// It isn't guaranteed that each requested version will be built. If there
359  /// are multiple update requests while building a preamble, only the last one
360  /// will be built.
361  void update(std::unique_ptr<CompilerInvocation> CI, ParseInputs PI,
362  std::vector<Diag> CIDiags, WantDiagnostics WantDiags) {
363  Request Req = {std::move(CI), std::move(PI), std::move(CIDiags), WantDiags,
364  Context::current().clone()};
365  if (RunSync) {
366  build(std::move(Req));
367  Status.update([](TUStatus &Status) {
368  Status.PreambleActivity = PreambleAction::Idle;
369  });
370  return;
371  }
372  {
373  std::unique_lock<std::mutex> Lock(Mutex);
374  // If NextReq was requested with WantDiagnostics::Yes we cannot just drop
375  // that on the floor. Block until we start building it. This won't
376  // dead-lock as we are blocking the caller thread, while builds continue
377  // on preamble thread.
378  ReqCV.wait(Lock, [this] {
379  return !NextReq || NextReq->WantDiags != WantDiagnostics::Yes;
380  });
381  NextReq = std::move(Req);
382  }
383  // Let the worker thread know there's a request, notify_one is safe as there
384  // should be a single worker thread waiting on it.
385  ReqCV.notify_all();
386  }
387 
388  void run() {
389  while (true) {
390  {
391  std::unique_lock<std::mutex> Lock(Mutex);
392  assert(!CurrentReq && "Already processing a request?");
393  // Wait until stop is called or there is a request.
394  ReqCV.wait(Lock, [this] { return NextReq || Done; });
395  if (Done)
396  break;
397  CurrentReq = std::move(*NextReq);
398  NextReq.reset();
399  }
400 
401  {
402  WithContext Guard(std::move(CurrentReq->Ctx));
403  // Note that we don't make use of the ContextProvider here.
404  // Preamble tasks are always scheduled by ASTWorker tasks, and we
405  // reuse the context/config that was created at that level.
406 
407  // Build the preamble and let the waiters know about it.
408  build(std::move(*CurrentReq));
409  }
410  bool IsEmpty = false;
411  {
412  std::lock_guard<std::mutex> Lock(Mutex);
413  CurrentReq.reset();
414  IsEmpty = !NextReq.hasValue();
415  }
416  if (IsEmpty) {
417  // We don't perform this above, before waiting for a request to make
418  // tests more deterministic. As there can be a race between this thread
419  // and client thread(clangdserver).
420  Status.update([](TUStatus &Status) {
421  Status.PreambleActivity = PreambleAction::Idle;
422  });
423  }
424  ReqCV.notify_all();
425  }
426  dlog("Preamble worker for {0} stopped", FileName);
427  }
428 
429  /// Signals the run loop to exit.
430  void stop() {
431  dlog("Preamble worker for {0} received stop", FileName);
432  {
433  std::lock_guard<std::mutex> Lock(Mutex);
434  Done = true;
435  NextReq.reset();
436  }
437  // Let the worker thread know that it should stop.
438  ReqCV.notify_all();
439  }
440 
441  bool blockUntilIdle(Deadline Timeout) const {
442  std::unique_lock<std::mutex> Lock(Mutex);
443  return wait(Lock, ReqCV, Timeout, [&] { return !NextReq && !CurrentReq; });
444  }
445 
446 private:
447  /// Holds inputs required for building a preamble. CI is guaranteed to be
448  /// non-null.
449  struct Request {
450  std::unique_ptr<CompilerInvocation> CI;
451  ParseInputs Inputs;
452  std::vector<Diag> CIDiags;
454  Context Ctx;
455  };
456 
457  bool isDone() {
458  std::lock_guard<std::mutex> Lock(Mutex);
459  return Done;
460  }
461 
462  /// Builds a preamble for \p Req, might reuse LatestBuild if possible.
463  /// Notifies ASTWorker after build finishes.
464  void build(Request Req);
465 
466  mutable std::mutex Mutex;
467  bool Done = false; /* GUARDED_BY(Mutex) */
468  llvm::Optional<Request> NextReq; /* GUARDED_BY(Mutex) */
469  llvm::Optional<Request> CurrentReq; /* GUARDED_BY(Mutex) */
470  // Signaled whenever a thread populates NextReq or worker thread builds a
471  // Preamble.
472  mutable std::condition_variable ReqCV; /* GUARDED_BY(Mutex) */
473  // Accessed only by preamble thread.
474  std::shared_ptr<const PreambleData> LatestBuild;
475 
476  const Path FileName;
477  ParsingCallbacks &Callbacks;
478  const bool StoreInMemory;
479  const bool RunSync;
480 
481  SynchronizedTUStatus &Status;
482  ASTWorker &ASTPeer;
483  TUScheduler::HeaderIncluderCache &HeaderIncluders;
484 };
485 
486 class ASTWorkerHandle;
487 
488 /// Owns one instance of the AST, schedules updates and reads of it.
489 /// Also responsible for building and providing access to the preamble.
490 /// Each ASTWorker processes the async requests sent to it on a separate
491 /// dedicated thread.
492 /// The ASTWorker that manages the AST is shared by both the processing thread
493 /// and the TUScheduler. The TUScheduler should discard an ASTWorker when
494 /// remove() is called, but its thread may be busy and we don't want to block.
495 /// So the workers are accessed via an ASTWorkerHandle. Destroying the handle
496 /// signals the worker to exit its run loop and gives up shared ownership of the
497 /// worker.
498 class ASTWorker {
499  friend class ASTWorkerHandle;
500  ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
501  TUScheduler::ASTCache &LRUCache,
502  TUScheduler::HeaderIncluderCache &HeaderIncluders,
503  Semaphore &Barrier, bool RunSync, const TUScheduler::Options &Opts,
504  ParsingCallbacks &Callbacks);
505 
506 public:
507  /// Create a new ASTWorker and return a handle to it.
508  /// The processing thread is spawned using \p Tasks. However, when \p Tasks
509  /// is null, all requests will be processed on the calling thread
510  /// synchronously instead. \p Barrier is acquired when processing each
511  /// request, it is used to limit the number of actively running threads.
512  static ASTWorkerHandle
513  create(PathRef FileName, const GlobalCompilationDatabase &CDB,
514  TUScheduler::ASTCache &IdleASTs,
515  TUScheduler::HeaderIncluderCache &HeaderIncluders,
516  AsyncTaskRunner *Tasks, Semaphore &Barrier,
517  const TUScheduler::Options &Opts, ParsingCallbacks &Callbacks);
518  ~ASTWorker();
519 
520  void update(ParseInputs Inputs, WantDiagnostics, bool ContentChanged);
521  void
522  runWithAST(llvm::StringRef Name,
523  llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
525  bool blockUntilIdle(Deadline Timeout) const;
526 
527  std::shared_ptr<const PreambleData> getPossiblyStalePreamble(
528  std::shared_ptr<const ASTSignals> *ASTSignals = nullptr) const;
529 
530  /// Used to inform ASTWorker about a new preamble build by PreambleThread.
531  /// Diagnostics are only published through this callback. This ensures they
532  /// are always for newer versions of the file, as the callback gets called in
533  /// the same order as update requests.
534  void updatePreamble(std::unique_ptr<CompilerInvocation> CI, ParseInputs PI,
535  std::shared_ptr<const PreambleData> Preamble,
536  std::vector<Diag> CIDiags, WantDiagnostics WantDiags);
537 
538  /// Obtain a preamble reflecting all updates so far. Threadsafe.
539  /// It may be delivered immediately, or later on the worker thread.
540  void getCurrentPreamble(
541  llvm::unique_function<void(std::shared_ptr<const PreambleData>)>);
542  /// Returns compile command from the current file inputs.
543  tooling::CompileCommand getCurrentCompileCommand() const;
544 
545  /// Wait for the first build of preamble to finish. Preamble itself can be
546  /// accessed via getPossiblyStalePreamble(). Note that this function will
547  /// return after an unsuccessful build of the preamble too, i.e. result of
548  /// getPossiblyStalePreamble() can be null even after this function returns.
549  void waitForFirstPreamble() const;
550 
551  TUScheduler::FileStats stats() const;
552  bool isASTCached() const;
553 
554 private:
555  // Details of an update request that are relevant to scheduling.
556  struct UpdateType {
557  // Do we want diagnostics from this version?
558  // If Yes, we must always build this version.
559  // If No, we only need to build this version if it's read.
560  // If Auto, we build if it's read or if the debounce expires.
562  // Did the main-file content of the document change?
563  // If so, we're allowed to cancel certain invalidated preceding reads.
565  };
566 
567  /// Publishes diagnostics for \p Inputs. It will build an AST or reuse the
568  /// cached one if applicable. Assumes LatestPreamble is compatible for \p
569  /// Inputs.
570  void generateDiagnostics(std::unique_ptr<CompilerInvocation> Invocation,
571  ParseInputs Inputs, std::vector<Diag> CIDiags);
572 
573  void updateASTSignals(ParsedAST &AST);
574 
575  // Must be called exactly once on processing thread. Will return after
576  // stop() is called on a separate thread and all pending requests are
577  // processed.
578  void run();
579  /// Signal that run() should finish processing pending requests and exit.
580  void stop();
581 
582  /// Adds a new task to the end of the request queue.
583  void startTask(llvm::StringRef Name, llvm::unique_function<void()> Task,
584  llvm::Optional<UpdateType> Update,
586 
587  /// Determines the next action to perform.
588  /// All actions that should never run are discarded.
589  /// Returns a deadline for the next action. If it's expired, run now.
590  /// scheduleLocked() is called again at the deadline, or if requests arrive.
591  Deadline scheduleLocked();
592  /// Should the first task in the queue be skipped instead of run?
593  bool shouldSkipHeadLocked() const;
594 
595  struct Request {
596  llvm::unique_function<void()> Action;
597  std::string Name;
598  steady_clock::time_point AddTime;
599  Context Ctx;
600  llvm::Optional<Context> QueueCtx;
601  llvm::Optional<UpdateType> Update;
604  };
605 
606  /// Handles retention of ASTs.
607  TUScheduler::ASTCache &IdleASTs;
608  TUScheduler::HeaderIncluderCache &HeaderIncluders;
609  const bool RunSync;
610  /// Time to wait after an update to see whether another update obsoletes it.
611  const DebouncePolicy UpdateDebounce;
612  /// File that ASTWorker is responsible for.
613  const Path FileName;
614  /// Callback to create processing contexts for tasks.
615  const std::function<Context(llvm::StringRef)> ContextProvider;
616  const GlobalCompilationDatabase &CDB;
617  /// Callback invoked when preamble or main file AST is built.
618  ParsingCallbacks &Callbacks;
619 
620  Semaphore &Barrier;
621  /// Whether the 'onMainAST' callback ran for the current FileInputs.
622  bool RanASTCallback = false;
623  /// Guards members used by both TUScheduler and the worker thread.
624  mutable std::mutex Mutex;
625  /// File inputs, currently being used by the worker.
626  /// Writes and reads from unknown threads are locked. Reads from the worker
627  /// thread are not locked, as it's the only writer.
628  ParseInputs FileInputs; /* GUARDED_BY(Mutex) */
629  /// Times of recent AST rebuilds, used for UpdateDebounce computation.
630  llvm::SmallVector<DebouncePolicy::clock::duration>
631  RebuildTimes; /* GUARDED_BY(Mutex) */
632  /// Set to true to signal run() to finish processing.
633  bool Done; /* GUARDED_BY(Mutex) */
634  std::deque<Request> Requests; /* GUARDED_BY(Mutex) */
635  llvm::Optional<Request> CurrentRequest; /* GUARDED_BY(Mutex) */
636  /// Signalled whenever a new request has been scheduled or processing of a
637  /// request has completed.
638  mutable std::condition_variable RequestsCV;
639  std::shared_ptr<const ASTSignals> LatestASTSignals; /* GUARDED_BY(Mutex) */
640  /// Latest build preamble for current TU.
641  /// None means no builds yet, null means there was an error while building.
642  /// Only written by ASTWorker's thread.
643  llvm::Optional<std::shared_ptr<const PreambleData>> LatestPreamble;
644  std::deque<Request> PreambleRequests; /* GUARDED_BY(Mutex) */
645  /// Signaled whenever LatestPreamble changes state or there's a new
646  /// PreambleRequest.
647  mutable std::condition_variable PreambleCV;
648  /// Guards the callback that publishes results of AST-related computations
649  /// (diagnostics) and file statuses.
650  std::mutex PublishMu;
651  // Used to prevent remove document + add document races that lead to
652  // out-of-order callbacks for publishing results of onMainAST callback.
653  //
654  // The lifetime of the old/new ASTWorkers will overlap, but their handles
655  // don't. When the old handle is destroyed, the old worker will stop reporting
656  // any results to the user.
657  bool CanPublishResults = true; /* GUARDED_BY(PublishMu) */
658  std::atomic<unsigned> ASTBuildCount = {0};
659  std::atomic<unsigned> PreambleBuildCount = {0};
660 
661  SynchronizedTUStatus Status;
662  PreambleThread PreamblePeer;
663 };
664 
665 /// A smart-pointer-like class that points to an active ASTWorker.
666 /// In destructor, signals to the underlying ASTWorker that no new requests will
667 /// be sent and the processing loop may exit (after running all pending
668 /// requests).
669 class ASTWorkerHandle {
670  friend class ASTWorker;
671  ASTWorkerHandle(std::shared_ptr<ASTWorker> Worker)
672  : Worker(std::move(Worker)) {
673  assert(this->Worker);
674  }
675 
676 public:
677  ASTWorkerHandle(const ASTWorkerHandle &) = delete;
678  ASTWorkerHandle &operator=(const ASTWorkerHandle &) = delete;
679  ASTWorkerHandle(ASTWorkerHandle &&) = default;
680  ASTWorkerHandle &operator=(ASTWorkerHandle &&) = default;
681 
682  ~ASTWorkerHandle() {
683  if (Worker)
684  Worker->stop();
685  }
686 
687  ASTWorker &operator*() {
688  assert(Worker && "Handle was moved from");
689  return *Worker;
690  }
691 
692  ASTWorker *operator->() {
693  assert(Worker && "Handle was moved from");
694  return Worker.get();
695  }
696 
697  /// Returns an owning reference to the underlying ASTWorker that can outlive
698  /// the ASTWorkerHandle. However, no new requests to an active ASTWorker can
699  /// be schedule via the returned reference, i.e. only reads of the preamble
700  /// are possible.
701  std::shared_ptr<const ASTWorker> lock() { return Worker; }
702 
703 private:
704  std::shared_ptr<ASTWorker> Worker;
705 };
706 
707 ASTWorkerHandle
708 ASTWorker::create(PathRef FileName, const GlobalCompilationDatabase &CDB,
709  TUScheduler::ASTCache &IdleASTs,
710  TUScheduler::HeaderIncluderCache &HeaderIncluders,
711  AsyncTaskRunner *Tasks, Semaphore &Barrier,
712  const TUScheduler::Options &Opts,
713  ParsingCallbacks &Callbacks) {
714  std::shared_ptr<ASTWorker> Worker(
715  new ASTWorker(FileName, CDB, IdleASTs, HeaderIncluders, Barrier,
716  /*RunSync=*/!Tasks, Opts, Callbacks));
717  if (Tasks) {
718  Tasks->runAsync("ASTWorker:" + llvm::sys::path::filename(FileName),
719  [Worker]() { Worker->run(); });
720  Tasks->runAsync("PreambleWorker:" + llvm::sys::path::filename(FileName),
721  [Worker]() { Worker->PreamblePeer.run(); });
722  }
723 
724  return ASTWorkerHandle(std::move(Worker));
725 }
726 
727 ASTWorker::ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
728  TUScheduler::ASTCache &LRUCache,
729  TUScheduler::HeaderIncluderCache &HeaderIncluders,
730  Semaphore &Barrier, bool RunSync,
731  const TUScheduler::Options &Opts,
732  ParsingCallbacks &Callbacks)
733  : IdleASTs(LRUCache), HeaderIncluders(HeaderIncluders), RunSync(RunSync),
734  UpdateDebounce(Opts.UpdateDebounce), FileName(FileName),
735  ContextProvider(Opts.ContextProvider), CDB(CDB), Callbacks(Callbacks),
736  Barrier(Barrier), Done(false), Status(FileName, Callbacks),
737  PreamblePeer(FileName, Callbacks, Opts.StorePreamblesInMemory, RunSync,
738  Status, HeaderIncluders, *this) {
739  // Set a fallback command because compile command can be accessed before
740  // `Inputs` is initialized. Other fields are only used after initialization
741  // from client inputs.
742  FileInputs.CompileCommand = CDB.getFallbackCommand(FileName);
743 }
744 
745 ASTWorker::~ASTWorker() {
746  // Make sure we remove the cached AST, if any.
747  IdleASTs.take(this);
748 #ifndef NDEBUG
749  std::lock_guard<std::mutex> Lock(Mutex);
750  assert(Done && "handle was not destroyed");
751  assert(Requests.empty() && !CurrentRequest &&
752  "unprocessed requests when destroying ASTWorker");
753 #endif
754 }
755 
756 void ASTWorker::update(ParseInputs Inputs, WantDiagnostics WantDiags,
757  bool ContentChanged) {
758  std::string TaskName = llvm::formatv("Update ({0})", Inputs.Version);
759  auto Task = [=]() mutable {
760  // Get the actual command as `Inputs` does not have a command.
761  // FIXME: some build systems like Bazel will take time to preparing
762  // environment to build the file, it would be nice if we could emit a
763  // "PreparingBuild" status to inform users, it is non-trivial given the
764  // current implementation.
765  auto Cmd = CDB.getCompileCommand(FileName);
766  // If we don't have a reliable command for this file, it may be a header.
767  // Try to find a file that includes it, to borrow its command.
768  if (!Cmd || !isReliable(*Cmd)) {
769  std::string ProxyFile = HeaderIncluders.get(FileName);
770  if (!ProxyFile.empty()) {
771  auto ProxyCmd = CDB.getCompileCommand(ProxyFile);
772  if (!ProxyCmd || !isReliable(*ProxyCmd)) {
773  // This command is supposed to be reliable! It's probably gone.
774  HeaderIncluders.remove(ProxyFile);
775  } else {
776  // We have a reliable command for an including file, use it.
777  Cmd = tooling::transferCompileCommand(std::move(*ProxyCmd), FileName);
778  }
779  }
780  }
781  if (Cmd)
782  Inputs.CompileCommand = std::move(*Cmd);
783  else
784  Inputs.CompileCommand = CDB.getFallbackCommand(FileName);
785 
786  bool InputsAreTheSame =
787  std::tie(FileInputs.CompileCommand, FileInputs.Contents) ==
788  std::tie(Inputs.CompileCommand, Inputs.Contents);
789  // Cached AST is invalidated.
790  if (!InputsAreTheSame) {
791  IdleASTs.take(this);
792  RanASTCallback = false;
793  }
794 
795  // Update current inputs so that subsequent reads can see them.
796  {
797  std::lock_guard<std::mutex> Lock(Mutex);
798  FileInputs = Inputs;
799  }
800 
801  log("ASTWorker building file {0} version {1} with command {2}\n[{3}]\n{4}",
802  FileName, Inputs.Version, Inputs.CompileCommand.Heuristic,
803  Inputs.CompileCommand.Directory,
804  printArgv(Inputs.CompileCommand.CommandLine));
805 
806  StoreDiags CompilerInvocationDiagConsumer;
807  std::vector<std::string> CC1Args;
808  std::unique_ptr<CompilerInvocation> Invocation = buildCompilerInvocation(
809  Inputs, CompilerInvocationDiagConsumer, &CC1Args);
810  // Log cc1 args even (especially!) if creating invocation failed.
811  if (!CC1Args.empty())
812  vlog("Driver produced command: cc1 {0}", printArgv(CC1Args));
813  std::vector<Diag> CompilerInvocationDiags =
814  CompilerInvocationDiagConsumer.take();
815  if (!Invocation) {
816  elog("Could not build CompilerInvocation for file {0}", FileName);
817  // Remove the old AST if it's still in cache.
818  IdleASTs.take(this);
819  RanASTCallback = false;
820  // Report the diagnostics we collected when parsing the command line.
821  Callbacks.onFailedAST(FileName, Inputs.Version,
822  std::move(CompilerInvocationDiags),
823  [&](llvm::function_ref<void()> Publish) {
824  // Ensure we only publish results from the worker
825  // if the file was not removed, making sure there
826  // are not race conditions.
827  std::lock_guard<std::mutex> Lock(PublishMu);
828  if (CanPublishResults)
829  Publish();
830  });
831  // Note that this might throw away a stale preamble that might still be
832  // useful, but this is how we communicate a build error.
833  LatestPreamble.emplace();
834  // Make sure anyone waiting for the preamble gets notified it could not be
835  // built.
836  PreambleCV.notify_all();
837  return;
838  }
839 
840  PreamblePeer.update(std::move(Invocation), std::move(Inputs),
841  std::move(CompilerInvocationDiags), WantDiags);
842  std::unique_lock<std::mutex> Lock(Mutex);
843  PreambleCV.wait(Lock, [this] {
844  // Block until we reiceve a preamble request, unless a preamble already
845  // exists, as patching an empty preamble would imply rebuilding it from
846  // scratch.
847  // We block here instead of the consumer to prevent any deadlocks. Since
848  // LatestPreamble is only populated by ASTWorker thread.
849  return LatestPreamble || !PreambleRequests.empty() || Done;
850  });
851  return;
852  };
853  startTask(TaskName, std::move(Task), UpdateType{WantDiags, ContentChanged},
854  TUScheduler::NoInvalidation);
855 }
856 
857 void ASTWorker::runWithAST(
858  llvm::StringRef Name,
859  llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
860  TUScheduler::ASTActionInvalidation Invalidation) {
861  // Tracks ast cache accesses for read operations.
862  static constexpr trace::Metric ASTAccessForRead(
863  "ast_access_read", trace::Metric::Counter, "result");
864  auto Task = [=, Action = std::move(Action)]() mutable {
865  if (auto Reason = isCancelled())
866  return Action(llvm::make_error<CancelledError>(Reason));
867  llvm::Optional<std::unique_ptr<ParsedAST>> AST =
868  IdleASTs.take(this, &ASTAccessForRead);
869  if (!AST) {
870  StoreDiags CompilerInvocationDiagConsumer;
871  std::unique_ptr<CompilerInvocation> Invocation =
872  buildCompilerInvocation(FileInputs, CompilerInvocationDiagConsumer);
873  // Try rebuilding the AST.
874  vlog("ASTWorker rebuilding evicted AST to run {0}: {1} version {2}", Name,
875  FileName, FileInputs.Version);
876  // FIXME: We might need to build a patched ast once preamble thread starts
877  // running async. Currently getPossiblyStalePreamble below will always
878  // return a compatible preamble as ASTWorker::update blocks.
879  llvm::Optional<ParsedAST> NewAST;
880  if (Invocation) {
881  NewAST = ParsedAST::build(FileName, FileInputs, std::move(Invocation),
882  CompilerInvocationDiagConsumer.take(),
883  getPossiblyStalePreamble());
884  ++ASTBuildCount;
885  }
886  AST = NewAST ? std::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;
887  }
888  // Make sure we put the AST back into the LRU cache.
889  auto _ = llvm::make_scope_exit(
890  [&AST, this]() { IdleASTs.put(this, std::move(*AST)); });
891  // Run the user-provided action.
892  if (!*AST)
893  return Action(error(llvm::errc::invalid_argument, "invalid AST"));
894  vlog("ASTWorker running {0} on version {2} of {1}", Name, FileName,
895  FileInputs.Version);
896  Action(InputsAndAST{FileInputs, **AST});
897  };
898  startTask(Name, std::move(Task), /*Update=*/None, Invalidation);
899 }
900 
901 void PreambleThread::build(Request Req) {
902  assert(Req.CI && "Got preamble request with null compiler invocation");
903  const ParseInputs &Inputs = Req.Inputs;
904 
905  Status.update([&](TUStatus &Status) {
906  Status.PreambleActivity = PreambleAction::Building;
907  });
908  auto _ = llvm::make_scope_exit([this, &Req] {
909  ASTPeer.updatePreamble(std::move(Req.CI), std::move(Req.Inputs),
910  LatestBuild, std::move(Req.CIDiags),
911  std::move(Req.WantDiags));
912  Callbacks.onPreamblePublished(FileName);
913  });
914 
915  if (!LatestBuild || Inputs.ForceRebuild) {
916  vlog("Building first preamble for {0} version {1}", FileName,
917  Inputs.Version);
918  } else if (isPreambleCompatible(*LatestBuild, Inputs, FileName, *Req.CI)) {
919  vlog("Reusing preamble version {0} for version {1} of {2}",
920  LatestBuild->Version, Inputs.Version, FileName);
921  return;
922  } else {
923  vlog("Rebuilding invalidated preamble for {0} version {1} (previous was "
924  "version {2})",
925  FileName, Inputs.Version, LatestBuild->Version);
926  }
927 
928  LatestBuild = clang::clangd::buildPreamble(
929  FileName, *Req.CI, Inputs, StoreInMemory,
930  [this, Version(Inputs.Version)](ASTContext &Ctx,
931  std::shared_ptr<clang::Preprocessor> PP,
932  const CanonicalIncludes &CanonIncludes) {
933  Callbacks.onPreambleAST(FileName, Version, Ctx, std::move(PP),
934  CanonIncludes);
935  });
936  if (LatestBuild && isReliable(LatestBuild->CompileCommand))
937  HeaderIncluders.update(FileName, LatestBuild->Includes.allHeaders());
938 }
939 
940 void ASTWorker::updatePreamble(std::unique_ptr<CompilerInvocation> CI,
941  ParseInputs PI,
942  std::shared_ptr<const PreambleData> Preamble,
943  std::vector<Diag> CIDiags,
945  std::string TaskName = llvm::formatv("Build AST for ({0})", PI.Version);
946  // Store preamble and build diagnostics with new preamble if requested.
947  auto Task = [this, Preamble = std::move(Preamble), CI = std::move(CI),
948  PI = std::move(PI), CIDiags = std::move(CIDiags),
949  WantDiags = std::move(WantDiags)]() mutable {
950  // Update the preamble inside ASTWorker queue to ensure atomicity. As a task
951  // running inside ASTWorker assumes internals won't change until it
952  // finishes.
953  if (!LatestPreamble || Preamble != *LatestPreamble) {
954  ++PreambleBuildCount;
955  // Cached AST is no longer valid.
956  IdleASTs.take(this);
957  RanASTCallback = false;
958  std::lock_guard<std::mutex> Lock(Mutex);
959  // LatestPreamble might be the last reference to old preamble, do not
960  // trigger destructor while holding the lock.
961  if (LatestPreamble)
962  std::swap(*LatestPreamble, Preamble);
963  else
964  LatestPreamble = std::move(Preamble);
965  }
966  // Notify anyone waiting for a preamble.
967  PreambleCV.notify_all();
968  // Give up our ownership to old preamble before starting expensive AST
969  // build.
970  Preamble.reset();
971  // We only need to build the AST if diagnostics were requested.
972  if (WantDiags == WantDiagnostics::No)
973  return;
974  // Report diagnostics with the new preamble to ensure progress. Otherwise
975  // diagnostics might get stale indefinitely if user keeps invalidating the
976  // preamble.
977  generateDiagnostics(std::move(CI), std::move(PI), std::move(CIDiags));
978  };
979  if (RunSync) {
980  Task();
981  return;
982  }
983  {
984  std::lock_guard<std::mutex> Lock(Mutex);
985  PreambleRequests.push_back({std::move(Task), std::move(TaskName),
986  steady_clock::now(), Context::current().clone(),
987  llvm::None, llvm::None,
988  TUScheduler::NoInvalidation, nullptr});
989  }
990  PreambleCV.notify_all();
991  RequestsCV.notify_all();
992 }
993 
994 void ASTWorker::updateASTSignals(ParsedAST &AST) {
995  auto Signals = std::make_shared<const ASTSignals>(ASTSignals::derive(AST));
996  // Existing readers of ASTSignals will have their copy preserved until the
997  // read is completed. The last reader deletes the old ASTSignals.
998  {
999  std::lock_guard<std::mutex> Lock(Mutex);
1000  std::swap(LatestASTSignals, Signals);
1001  }
1002 }
1003 
1004 void ASTWorker::generateDiagnostics(
1005  std::unique_ptr<CompilerInvocation> Invocation, ParseInputs Inputs,
1006  std::vector<Diag> CIDiags) {
1007  // Tracks ast cache accesses for publishing diags.
1008  static constexpr trace::Metric ASTAccessForDiag(
1009  "ast_access_diag", trace::Metric::Counter, "result");
1010  assert(Invocation);
1011  assert(LatestPreamble);
1012  // No need to rebuild the AST if we won't send the diagnostics.
1013  {
1014  std::lock_guard<std::mutex> Lock(PublishMu);
1015  if (!CanPublishResults)
1016  return;
1017  }
1018  // Used to check whether we can update AST cache.
1019  bool InputsAreLatest =
1020  std::tie(FileInputs.CompileCommand, FileInputs.Contents) ==
1021  std::tie(Inputs.CompileCommand, Inputs.Contents);
1022  // Take a shortcut and don't report the diagnostics, since they should be the
1023  // same. All the clients should handle the lack of OnUpdated() call anyway to
1024  // handle empty result from buildAST.
1025  // FIXME: the AST could actually change if non-preamble includes changed,
1026  // but we choose to ignore it.
1027  if (InputsAreLatest && RanASTCallback)
1028  return;
1029 
1030  // Get the AST for diagnostics, either build it or use the cached one.
1031  std::string TaskName = llvm::formatv("Build AST ({0})", Inputs.Version);
1032  Status.update([&](TUStatus &Status) {
1033  Status.ASTActivity.K = ASTAction::Building;
1034  Status.ASTActivity.Name = std::move(TaskName);
1035  });
1036  // We might be able to reuse the last we've built for a read request.
1037  // FIXME: It might be better to not reuse this AST. That way queued AST builds
1038  // won't be required for diags.
1039  llvm::Optional<std::unique_ptr<ParsedAST>> AST =
1040  IdleASTs.take(this, &ASTAccessForDiag);
1041  if (!AST || !InputsAreLatest) {
1042  auto RebuildStartTime = DebouncePolicy::clock::now();
1043  llvm::Optional<ParsedAST> NewAST = ParsedAST::build(
1044  FileName, Inputs, std::move(Invocation), CIDiags, *LatestPreamble);
1045  auto RebuildDuration = DebouncePolicy::clock::now() - RebuildStartTime;
1046  ++ASTBuildCount;
1047  // Try to record the AST-build time, to inform future update debouncing.
1048  // This is best-effort only: if the lock is held, don't bother.
1049  std::unique_lock<std::mutex> Lock(Mutex, std::try_to_lock);
1050  if (Lock.owns_lock()) {
1051  // Do not let RebuildTimes grow beyond its small-size (i.e.
1052  // capacity).
1053  if (RebuildTimes.size() == RebuildTimes.capacity())
1054  RebuildTimes.erase(RebuildTimes.begin());
1055  RebuildTimes.push_back(RebuildDuration);
1056  Lock.unlock();
1057  }
1058  Status.update([&](TUStatus &Status) {
1059  Status.Details.ReuseAST = false;
1060  Status.Details.BuildFailed = !NewAST.hasValue();
1061  });
1062  AST = NewAST ? std::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;
1063  } else {
1064  log("Skipping rebuild of the AST for {0}, inputs are the same.", FileName);
1065  Status.update([](TUStatus &Status) {
1066  Status.Details.ReuseAST = true;
1067  Status.Details.BuildFailed = false;
1068  });
1069  }
1070 
1071  // Publish diagnostics.
1072  auto RunPublish = [&](llvm::function_ref<void()> Publish) {
1073  // Ensure we only publish results from the worker if the file was not
1074  // removed, making sure there are not race conditions.
1075  std::lock_guard<std::mutex> Lock(PublishMu);
1076  if (CanPublishResults)
1077  Publish();
1078  };
1079  if (*AST) {
1080  trace::Span Span("Running main AST callback");
1081  Callbacks.onMainAST(FileName, **AST, RunPublish);
1082  updateASTSignals(**AST);
1083  } else {
1084  // Failed to build the AST, at least report diagnostics from the
1085  // command line if there were any.
1086  // FIXME: we might have got more errors while trying to build the
1087  // AST, surface them too.
1088  Callbacks.onFailedAST(FileName, Inputs.Version, CIDiags, RunPublish);
1089  }
1090 
1091  // AST might've been built for an older version of the source, as ASTWorker
1092  // queue raced ahead while we were waiting on the preamble. In that case the
1093  // queue can't reuse the AST.
1094  if (InputsAreLatest) {
1095  RanASTCallback = *AST != nullptr;
1096  IdleASTs.put(this, std::move(*AST));
1097  }
1098 }
1099 
1100 std::shared_ptr<const PreambleData> ASTWorker::getPossiblyStalePreamble(
1101  std::shared_ptr<const ASTSignals> *ASTSignals) const {
1102  std::lock_guard<std::mutex> Lock(Mutex);
1103  if (ASTSignals)
1104  *ASTSignals = LatestASTSignals;
1105  return LatestPreamble ? *LatestPreamble : nullptr;
1106 }
1107 
1108 void ASTWorker::waitForFirstPreamble() const {
1109  std::unique_lock<std::mutex> Lock(Mutex);
1110  PreambleCV.wait(Lock, [this] { return LatestPreamble.hasValue() || Done; });
1111 }
1112 
1113 tooling::CompileCommand ASTWorker::getCurrentCompileCommand() const {
1114  std::unique_lock<std::mutex> Lock(Mutex);
1115  return FileInputs.CompileCommand;
1116 }
1117 
1118 TUScheduler::FileStats ASTWorker::stats() const {
1119  TUScheduler::FileStats Result;
1120  Result.ASTBuilds = ASTBuildCount;
1121  Result.PreambleBuilds = PreambleBuildCount;
1122  // Note that we don't report the size of ASTs currently used for processing
1123  // the in-flight requests. We used this information for debugging purposes
1124  // only, so this should be fine.
1125  Result.UsedBytesAST = IdleASTs.getUsedBytes(this);
1126  if (auto Preamble = getPossiblyStalePreamble())
1127  Result.UsedBytesPreamble = Preamble->Preamble.getSize();
1128  return Result;
1129 }
1130 
1131 bool ASTWorker::isASTCached() const { return IdleASTs.getUsedBytes(this) != 0; }
1132 
1133 void ASTWorker::stop() {
1134  {
1135  std::lock_guard<std::mutex> Lock(PublishMu);
1136  CanPublishResults = false;
1137  }
1138  {
1139  std::lock_guard<std::mutex> Lock(Mutex);
1140  assert(!Done && "stop() called twice");
1141  Done = true;
1142  }
1143  PreamblePeer.stop();
1144  // We are no longer going to build any preambles, let the waiters know that.
1145  PreambleCV.notify_all();
1146  Status.stop();
1147  RequestsCV.notify_all();
1148 }
1149 
1150 void ASTWorker::startTask(llvm::StringRef Name,
1151  llvm::unique_function<void()> Task,
1152  llvm::Optional<UpdateType> Update,
1153  TUScheduler::ASTActionInvalidation Invalidation) {
1154  if (RunSync) {
1155  assert(!Done && "running a task after stop()");
1156  trace::Span Tracer(Name + ":" + llvm::sys::path::filename(FileName));
1157  WithContext WithProvidedContext(ContextProvider(FileName));
1158  Task();
1159  return;
1160  }
1161 
1162  {
1163  std::lock_guard<std::mutex> Lock(Mutex);
1164  assert(!Done && "running a task after stop()");
1165  // Cancel any requests invalidated by this request.
1166  if (Update && Update->ContentChanged) {
1167  for (auto &R : llvm::reverse(Requests)) {
1168  if (R.InvalidationPolicy == TUScheduler::InvalidateOnUpdate)
1169  R.Invalidate();
1170  if (R.Update && R.Update->ContentChanged)
1171  break; // Older requests were already invalidated by the older update.
1172  }
1173  }
1174 
1175  // Allow this request to be cancelled if invalidated.
1176  Context Ctx = Context::current().derive(kFileBeingProcessed, FileName);
1177  Canceler Invalidate = nullptr;
1178  if (Invalidation) {
1179  WithContext WC(std::move(Ctx));
1180  std::tie(Ctx, Invalidate) = cancelableTask(
1181  /*Reason=*/static_cast<int>(ErrorCode::ContentModified));
1182  }
1183  // Trace the time the request spends in the queue, and the requests that
1184  // it's going to wait for.
1185  llvm::Optional<Context> QueueCtx;
1186  if (trace::enabled()) {
1187  // Tracers that follow threads and need strict nesting will see a tiny
1188  // instantaneous event "we're enqueueing", and sometime later it runs.
1189  WithContext WC(Ctx.clone());
1190  trace::Span Tracer("Queued:" + Name);
1191  if (Tracer.Args) {
1192  if (CurrentRequest)
1193  SPAN_ATTACH(Tracer, "CurrentRequest", CurrentRequest->Name);
1194  llvm::json::Array PreambleRequestsNames;
1195  for (const auto &Req : PreambleRequests)
1196  PreambleRequestsNames.push_back(Req.Name);
1197  SPAN_ATTACH(Tracer, "PreambleRequestsNames",
1198  std::move(PreambleRequestsNames));
1199  llvm::json::Array RequestsNames;
1200  for (const auto &Req : Requests)
1201  RequestsNames.push_back(Req.Name);
1202  SPAN_ATTACH(Tracer, "RequestsNames", std::move(RequestsNames));
1203  }
1204  // For tracers that follow contexts, keep the trace span's context alive
1205  // until we dequeue the request, so they see the full duration.
1206  QueueCtx = Context::current().clone();
1207  }
1208  Requests.push_back({std::move(Task), std::string(Name), steady_clock::now(),
1209  std::move(Ctx), std::move(QueueCtx), Update,
1210  Invalidation, std::move(Invalidate)});
1211  }
1212  RequestsCV.notify_all();
1213 }
1214 
1215 void ASTWorker::run() {
1216  while (true) {
1217  {
1218  std::unique_lock<std::mutex> Lock(Mutex);
1219  assert(!CurrentRequest && "A task is already running, multiple workers?");
1220  for (auto Wait = scheduleLocked(); !Wait.expired();
1221  Wait = scheduleLocked()) {
1222  assert(PreambleRequests.empty() &&
1223  "Preamble updates should be scheduled immediately");
1224  if (Done) {
1225  if (Requests.empty())
1226  return;
1227  else // Even though Done is set, finish pending requests.
1228  break; // However, skip delays to shutdown fast.
1229  }
1230 
1231  // Tracing: we have a next request, attribute this sleep to it.
1232  llvm::Optional<WithContext> Ctx;
1233  llvm::Optional<trace::Span> Tracer;
1234  if (!Requests.empty()) {
1235  Ctx.emplace(Requests.front().Ctx.clone());
1236  Tracer.emplace("Debounce");
1237  SPAN_ATTACH(*Tracer, "next_request", Requests.front().Name);
1238  if (!(Wait == Deadline::infinity())) {
1239  Status.update([&](TUStatus &Status) {
1240  Status.ASTActivity.K = ASTAction::Queued;
1241  Status.ASTActivity.Name = Requests.front().Name;
1242  });
1243  SPAN_ATTACH(*Tracer, "sleep_ms",
1244  std::chrono::duration_cast<std::chrono::milliseconds>(
1245  Wait.time() - steady_clock::now())
1246  .count());
1247  }
1248  }
1249 
1250  wait(Lock, RequestsCV, Wait);
1251  }
1252  // Any request in ReceivedPreambles is at least as old as the
1253  // Requests.front(), so prefer them first to preserve LSP order.
1254  if (!PreambleRequests.empty()) {
1255  CurrentRequest = std::move(PreambleRequests.front());
1256  PreambleRequests.pop_front();
1257  } else {
1258  CurrentRequest = std::move(Requests.front());
1259  Requests.pop_front();
1260  }
1261  } // unlock Mutex
1262 
1263  // Inform tracing that the request was dequeued.
1264  CurrentRequest->QueueCtx.reset();
1265 
1266  // It is safe to perform reads to CurrentRequest without holding the lock as
1267  // only writer is also this thread.
1268  {
1269  std::unique_lock<Semaphore> Lock(Barrier, std::try_to_lock);
1270  if (!Lock.owns_lock()) {
1271  Status.update([&](TUStatus &Status) {
1272  Status.ASTActivity.K = ASTAction::Queued;
1273  Status.ASTActivity.Name = CurrentRequest->Name;
1274  });
1275  Lock.lock();
1276  }
1277  WithContext Guard(std::move(CurrentRequest->Ctx));
1278  trace::Span Tracer(CurrentRequest->Name);
1279  Status.update([&](TUStatus &Status) {
1280  Status.ASTActivity.K = ASTAction::RunningAction;
1281  Status.ASTActivity.Name = CurrentRequest->Name;
1282  });
1283  WithContext WithProvidedContext(ContextProvider(FileName));
1284  CurrentRequest->Action();
1285  }
1286 
1287  bool IsEmpty = false;
1288  {
1289  std::lock_guard<std::mutex> Lock(Mutex);
1290  CurrentRequest.reset();
1291  IsEmpty = Requests.empty() && PreambleRequests.empty();
1292  }
1293  if (IsEmpty) {
1294  Status.update([&](TUStatus &Status) {
1295  Status.ASTActivity.K = ASTAction::Idle;
1296  Status.ASTActivity.Name = "";
1297  });
1298  }
1299  RequestsCV.notify_all();
1300  }
1301 }
1302 
1303 Deadline ASTWorker::scheduleLocked() {
1304  // Process new preambles immediately.
1305  if (!PreambleRequests.empty())
1306  return Deadline::zero();
1307  if (Requests.empty())
1308  return Deadline::infinity(); // Wait for new requests.
1309  // Handle cancelled requests first so the rest of the scheduler doesn't.
1310  for (auto I = Requests.begin(), E = Requests.end(); I != E; ++I) {
1311  if (!isCancelled(I->Ctx)) {
1312  // Cancellations after the first read don't affect current scheduling.
1313  if (I->Update == None)
1314  break;
1315  continue;
1316  }
1317  // Cancelled reads are moved to the front of the queue and run immediately.
1318  if (I->Update == None) {
1319  Request R = std::move(*I);
1320  Requests.erase(I);
1321  Requests.push_front(std::move(R));
1322  return Deadline::zero();
1323  }
1324  // Cancelled updates are downgraded to auto-diagnostics, and may be elided.
1325  if (I->Update->Diagnostics == WantDiagnostics::Yes)
1326  I->Update->Diagnostics = WantDiagnostics::Auto;
1327  }
1328 
1329  while (shouldSkipHeadLocked()) {
1330  vlog("ASTWorker skipping {0} for {1}", Requests.front().Name, FileName);
1331  Requests.pop_front();
1332  }
1333  assert(!Requests.empty() && "skipped the whole queue");
1334  // Some updates aren't dead yet, but never end up being used.
1335  // e.g. the first keystroke is live until obsoleted by the second.
1336  // We debounce "maybe-unused" writes, sleeping in case they become dead.
1337  // But don't delay reads (including updates where diagnostics are needed).
1338  for (const auto &R : Requests)
1339  if (R.Update == None || R.Update->Diagnostics == WantDiagnostics::Yes)
1340  return Deadline::zero();
1341  // Front request needs to be debounced, so determine when we're ready.
1342  Deadline D(Requests.front().AddTime + UpdateDebounce.compute(RebuildTimes));
1343  return D;
1344 }
1345 
1346 // Returns true if Requests.front() is a dead update that can be skipped.
1347 bool ASTWorker::shouldSkipHeadLocked() const {
1348  assert(!Requests.empty());
1349  auto Next = Requests.begin();
1350  auto Update = Next->Update;
1351  if (!Update) // Only skip updates.
1352  return false;
1353  ++Next;
1354  // An update is live if its AST might still be read.
1355  // That is, if it's not immediately followed by another update.
1356  if (Next == Requests.end() || !Next->Update)
1357  return false;
1358  // The other way an update can be live is if its diagnostics might be used.
1359  switch (Update->Diagnostics) {
1360  case WantDiagnostics::Yes:
1361  return false; // Always used.
1362  case WantDiagnostics::No:
1363  return true; // Always dead.
1364  case WantDiagnostics::Auto:
1365  // Used unless followed by an update that generates diagnostics.
1366  for (; Next != Requests.end(); ++Next)
1367  if (Next->Update && Next->Update->Diagnostics != WantDiagnostics::No)
1368  return true; // Prefer later diagnostics.
1369  return false;
1370  }
1371  llvm_unreachable("Unknown WantDiagnostics");
1372 }
1373 
1374 bool ASTWorker::blockUntilIdle(Deadline Timeout) const {
1375  auto WaitUntilASTWorkerIsIdle = [&] {
1376  std::unique_lock<std::mutex> Lock(Mutex);
1377  return wait(Lock, RequestsCV, Timeout, [&] {
1378  return PreambleRequests.empty() && Requests.empty() && !CurrentRequest;
1379  });
1380  };
1381  // Make sure ASTWorker has processed all requests, which might issue new
1382  // updates to PreamblePeer.
1383  if (!WaitUntilASTWorkerIsIdle())
1384  return false;
1385  // Now that ASTWorker processed all requests, ensure PreamblePeer has served
1386  // all update requests. This might create new PreambleRequests for the
1387  // ASTWorker.
1388  if (!PreamblePeer.blockUntilIdle(Timeout))
1389  return false;
1390  assert(Requests.empty() &&
1391  "No new normal tasks can be scheduled concurrently with "
1392  "blockUntilIdle(): ASTWorker isn't threadsafe");
1393  // Finally make sure ASTWorker has processed all of the preamble updates.
1394  return WaitUntilASTWorkerIsIdle();
1395 }
1396 
1397 // Render a TUAction to a user-facing string representation.
1398 // TUAction represents clangd-internal states, we don't intend to expose them
1399 // to users (say C++ programmers) directly to avoid confusion, we use terms that
1400 // are familiar by C++ programmers.
1401 std::string renderTUAction(const PreambleAction PA, const ASTAction &AA) {
1402  llvm::SmallVector<std::string, 2> Result;
1403  switch (PA) {
1404  case PreambleAction::Building:
1405  Result.push_back("parsing includes");
1406  break;
1407  case PreambleAction::Idle:
1408  // We handle idle specially below.
1409  break;
1410  }
1411  switch (AA.K) {
1412  case ASTAction::Queued:
1413  Result.push_back("file is queued");
1414  break;
1415  case ASTAction::RunningAction:
1416  Result.push_back("running " + AA.Name);
1417  break;
1418  case ASTAction::Building:
1419  Result.push_back("parsing main file");
1420  break;
1421  case ASTAction::Idle:
1422  // We handle idle specially below.
1423  break;
1424  }
1425  if (Result.empty())
1426  return "idle";
1427  return llvm::join(Result, ", ");
1428 }
1429 
1430 } // namespace
1431 
1433  return llvm::heavyweight_hardware_concurrency().compute_thread_count();
1434 }
1435 
1436 FileStatus TUStatus::render(PathRef File) const {
1437  FileStatus FStatus;
1438  FStatus.uri = URIForFile::canonicalize(File, /*TUPath=*/File);
1439  FStatus.state = renderTUAction(PreambleActivity, ASTActivity);
1440  return FStatus;
1441 }
1442 
1444  /// Latest inputs, passed to TUScheduler::update().
1445  std::string Contents;
1446  ASTWorkerHandle Worker;
1447 };
1448 
1449 TUScheduler::TUScheduler(const GlobalCompilationDatabase &CDB,
1450  const Options &Opts,
1451  std::unique_ptr<ParsingCallbacks> Callbacks)
1452  : CDB(CDB), Opts(Opts),
1453  Callbacks(Callbacks ? move(Callbacks)
1454  : std::make_unique<ParsingCallbacks>()),
1455  Barrier(Opts.AsyncThreadsCount), QuickRunBarrier(Opts.AsyncThreadsCount),
1456  IdleASTs(
1457  std::make_unique<ASTCache>(Opts.RetentionPolicy.MaxRetainedASTs)),
1458  HeaderIncluders(std::make_unique<HeaderIncluderCache>()) {
1459  // Avoid null checks everywhere.
1460  if (!Opts.ContextProvider) {
1461  this->Opts.ContextProvider = [](llvm::StringRef) {
1462  return Context::current().clone();
1463  };
1464  }
1465  if (0 < Opts.AsyncThreadsCount) {
1466  PreambleTasks.emplace();
1467  WorkerThreads.emplace();
1468  }
1469 }
1470 
1472  // Notify all workers that they need to stop.
1473  Files.clear();
1474 
1475  // Wait for all in-flight tasks to finish.
1476  if (PreambleTasks)
1477  PreambleTasks->wait();
1478  if (WorkerThreads)
1479  WorkerThreads->wait();
1480 }
1481 
1483  for (auto &File : Files)
1484  if (!File.getValue()->Worker->blockUntilIdle(D))
1485  return false;
1486  if (PreambleTasks)
1487  if (!PreambleTasks->wait(D))
1488  return false;
1489  return true;
1490 }
1491 
1494  std::unique_ptr<FileData> &FD = Files[File];
1495  bool NewFile = FD == nullptr;
1496  bool ContentChanged = false;
1497  if (!FD) {
1498  // Create a new worker to process the AST-related tasks.
1499  ASTWorkerHandle Worker =
1500  ASTWorker::create(File, CDB, *IdleASTs, *HeaderIncluders,
1501  WorkerThreads ? WorkerThreads.getPointer() : nullptr,
1502  Barrier, Opts, *Callbacks);
1503  FD = std::unique_ptr<FileData>(
1504  new FileData{Inputs.Contents, std::move(Worker)});
1505  ContentChanged = true;
1506  } else if (FD->Contents != Inputs.Contents) {
1507  ContentChanged = true;
1508  FD->Contents = Inputs.Contents;
1509  }
1510  FD->Worker->update(std::move(Inputs), WantDiags, ContentChanged);
1511  // There might be synthetic update requests, don't change the LastActiveFile
1512  // in such cases.
1513  if (ContentChanged)
1514  LastActiveFile = File.str();
1515  return NewFile;
1516 }
1517 
1519  bool Removed = Files.erase(File);
1520  if (!Removed)
1521  elog("Trying to remove file from TUScheduler that is not tracked: {0}",
1522  File);
1523  // We don't call HeaderIncluders.remove(File) here.
1524  // If we did, we'd avoid potentially stale header/mainfile associations.
1525  // However, it would mean that closing a mainfile could invalidate the
1526  // preamble of several open headers.
1527 }
1528 
1529 void TUScheduler::run(llvm::StringRef Name, llvm::StringRef Path,
1530  llvm::unique_function<void()> Action) {
1531  runWithSemaphore(Name, Path, std::move(Action), Barrier);
1532 }
1533 
1534 void TUScheduler::runQuick(llvm::StringRef Name, llvm::StringRef Path,
1535  llvm::unique_function<void()> Action) {
1536  // Use QuickRunBarrier to serialize quick tasks: we are ignoring
1537  // the parallelism level set by the user, don't abuse it
1538  runWithSemaphore(Name, Path, std::move(Action), QuickRunBarrier);
1539 }
1540 
1541 void TUScheduler::runWithSemaphore(llvm::StringRef Name, llvm::StringRef Path,
1542  llvm::unique_function<void()> Action,
1543  Semaphore &Sem) {
1544  if (Path.empty())
1545  Path = LastActiveFile;
1546  else
1547  LastActiveFile = Path.str();
1548  if (!PreambleTasks) {
1549  WithContext WithProvidedContext(Opts.ContextProvider(Path));
1550  return Action();
1551  }
1552  PreambleTasks->runAsync(Name, [this, &Sem, Ctx = Context::current().clone(),
1553  Path(Path.str()),
1554  Action = std::move(Action)]() mutable {
1555  std::lock_guard<Semaphore> BarrierLock(Sem);
1556  WithContext WC(std::move(Ctx));
1557  WithContext WithProvidedContext(Opts.ContextProvider(Path));
1558  Action();
1559  });
1560 }
1561 
1563  llvm::StringRef Name, PathRef File,
1564  llvm::unique_function<void(llvm::Expected<InputsAndAST>)> Action,
1565  TUScheduler::ASTActionInvalidation Invalidation) {
1566  auto It = Files.find(File);
1567  if (It == Files.end()) {
1568  Action(llvm::make_error<LSPError>(
1569  "trying to get AST for non-added document", ErrorCode::InvalidParams));
1570  return;
1571  }
1572  LastActiveFile = File.str();
1573 
1574  It->second->Worker->runWithAST(Name, std::move(Action), Invalidation);
1575 }
1576 
1577 void TUScheduler::runWithPreamble(llvm::StringRef Name, PathRef File,
1578  PreambleConsistency Consistency,
1580  auto It = Files.find(File);
1581  if (It == Files.end()) {
1582  Action(llvm::make_error<LSPError>(
1583  "trying to get preamble for non-added document",
1585  return;
1586  }
1587  LastActiveFile = File.str();
1588 
1589  if (!PreambleTasks) {
1591  SPAN_ATTACH(Tracer, "file", File);
1592  std::shared_ptr<const ASTSignals> Signals;
1593  std::shared_ptr<const PreambleData> Preamble =
1594  It->second->Worker->getPossiblyStalePreamble(&Signals);
1595  WithContext WithProvidedContext(Opts.ContextProvider(File));
1596  Action(InputsAndPreamble{It->second->Contents,
1597  It->second->Worker->getCurrentCompileCommand(),
1598  Preamble.get(), Signals.get()});
1599  return;
1600  }
1601 
1602  std::shared_ptr<const ASTWorker> Worker = It->second->Worker.lock();
1603  auto Task = [Worker, Consistency, Name = Name.str(), File = File.str(),
1604  Contents = It->second->Contents,
1605  Command = Worker->getCurrentCompileCommand(),
1607  std::string(File)),
1608  Action = std::move(Action), this]() mutable {
1609  std::shared_ptr<const PreambleData> Preamble;
1610  if (Consistency == PreambleConsistency::Stale) {
1611  // Wait until the preamble is built for the first time, if preamble
1612  // is required. This avoids extra work of processing the preamble
1613  // headers in parallel multiple times.
1614  Worker->waitForFirstPreamble();
1615  }
1616  std::shared_ptr<const ASTSignals> Signals;
1617  Preamble = Worker->getPossiblyStalePreamble(&Signals);
1618 
1619  std::lock_guard<Semaphore> BarrierLock(Barrier);
1620  WithContext Guard(std::move(Ctx));
1622  SPAN_ATTACH(Tracer, "file", File);
1623  WithContext WithProvidedContext(Opts.ContextProvider(File));
1624  Action(InputsAndPreamble{Contents, Command, Preamble.get(), Signals.get()});
1625  };
1626 
1627  PreambleTasks->runAsync("task:" + llvm::sys::path::filename(File),
1628  std::move(Task));
1629 }
1630 
1631 llvm::StringMap<TUScheduler::FileStats> TUScheduler::fileStats() const {
1632  llvm::StringMap<TUScheduler::FileStats> Result;
1633  for (const auto &PathAndFile : Files)
1634  Result.try_emplace(PathAndFile.first(),
1635  PathAndFile.second->Worker->stats());
1636  return Result;
1637 }
1638 
1639 std::vector<Path> TUScheduler::getFilesWithCachedAST() const {
1640  std::vector<Path> Result;
1641  for (auto &&PathAndFile : Files) {
1642  if (!PathAndFile.second->Worker->isASTCached())
1643  continue;
1644  Result.push_back(std::string(PathAndFile.first()));
1645  }
1646  return Result;
1647 }
1648 
1649 DebouncePolicy::clock::duration
1650 DebouncePolicy::compute(llvm::ArrayRef<clock::duration> History) const {
1651  assert(Min <= Max && "Invalid policy");
1652  if (History.empty())
1653  return Max; // Arbitrary.
1654 
1655  // Base the result on the median rebuild.
1656  // nth_element needs a mutable array, take the chance to bound the data size.
1657  History = History.take_back(15);
1658  llvm::SmallVector<clock::duration, 15> Recent(History.begin(), History.end());
1659  auto Median = Recent.begin() + Recent.size() / 2;
1660  std::nth_element(Recent.begin(), Median, Recent.end());
1661 
1662  clock::duration Target =
1663  std::chrono::duration_cast<clock::duration>(RebuildRatio * *Median);
1664  if (Target > Max)
1665  return Max;
1666  if (Target < Min)
1667  return Min;
1668  return Target;
1669 }
1670 
1672  DebouncePolicy P;
1673  P.Min = P.Max = T;
1674  return P;
1675 }
1676 
1678  for (const auto &Elem : fileStats()) {
1679  MT.detail(Elem.first())
1680  .child("preamble")
1681  .addUsage(Opts.StorePreamblesInMemory ? Elem.second.UsedBytesPreamble
1682  : 0);
1683  MT.detail(Elem.first()).child("ast").addUsage(Elem.second.UsedBytesAST);
1684  MT.child("header_includer_cache").addUsage(HeaderIncluders->getUsedBytes());
1685  }
1686 }
1687 } // namespace clangd
1688 } // namespace clang
dlog
#define dlog(...)
Definition: Logger.h:102
WantDiags
WantDiagnostics WantDiags
Definition: TUScheduler.cpp:453
clang::clangd::MemoryTree::child
MemoryTree & child(llvm::StringLiteral Name)
No copy of the Name.
Definition: MemoryTree.h:41
clang::clangd::isPreambleCompatible
bool isPreambleCompatible(const PreambleData &Preamble, const ParseInputs &Inputs, PathRef FileName, const CompilerInvocation &CI)
Returns true if Preamble is reusable for Inputs.
Definition: Preamble.cpp:403
clang::clangd::cancelableTask
std::pair< Context, Canceler > cancelableTask(int Reason)
Defines a new task whose cancellation may be requested.
Definition: Cancellation.cpp:24
E
const Expr * E
Definition: AvoidBindCheck.cpp:88
clang::clangd::TUScheduler::runWithPreamble
void runWithPreamble(llvm::StringRef Name, PathRef File, PreambleConsistency Consistency, Callback< InputsAndPreamble > Action)
Schedule an async read of the preamble.
Definition: TUScheduler.cpp:1577
clang::clangd::getDefaultAsyncThreadsCount
unsigned getDefaultAsyncThreadsCount()
Returns a number of a default async threads to use for TUScheduler.
Definition: TUScheduler.cpp:1432
CIDiags
std::vector< Diag > CIDiags
Definition: TUScheduler.cpp:452
clang::clangd::printArgv
std::string printArgv(llvm::ArrayRef< llvm::StringRef > Args)
Definition: CompileCommands.cpp:582
clang::clangd::Path
std::string Path
A typedef to represent a file path.
Definition: Path.h:26
Update
llvm::Optional< UpdateType > Update
Definition: TUScheduler.cpp:601
clang::clangd::TUScheduler::~TUScheduler
~TUScheduler()
Definition: TUScheduler.cpp:1471
Usage
const char Usage[]
Definition: ClangReorderFields.cpp:50
Tracer
std::unique_ptr< trace::EventTracer > Tracer
Definition: TraceTests.cpp:164
Path.h
clang::clangd::Context::current
static const Context & current()
Returns the context for the current thread, creating it if needed.
Definition: Context.cpp:27
clang::clangd::WantDiagnostics::Yes
@ Yes
clang::clangd::TUScheduler::blockUntilIdle
bool blockUntilIdle(Deadline D) const
Wait until there are no scheduled or running tasks.
Definition: TUScheduler.cpp:1482
clang::clangd::Context::clone
Context clone() const
Clone this context object.
Definition: Context.cpp:20
Diagnostics
WantDiagnostics Diagnostics
Definition: TUScheduler.cpp:561
ContentChanged
bool ContentChanged
Definition: TUScheduler.cpp:564
clang::clangd::detail::error
llvm::Error error(std::error_code, std::string &&)
Definition: Logger.cpp:80
CI
std::unique_ptr< CompilerInvocation > CI
Definition: TUScheduler.cpp:450
QueueCtx
llvm::Optional< Context > QueueCtx
Definition: TUScheduler.cpp:600
clang::clangd::PreambleAction
PreambleAction
Definition: TUScheduler.h:91
Preamble.h
Ctx
Context Ctx
Definition: TUScheduler.cpp:454
clang::clangd::TUScheduler::run
void run(llvm::StringRef Name, llvm::StringRef Path, llvm::unique_function< void()> Action)
Schedule an async task with no dependencies.
Definition: TUScheduler.cpp:1529
Trace.h
AST
llvm::Optional< ParsedAST > AST
Definition: SymbolCollectorTests.cpp:134
clang::clangd::TUScheduler::getFilesWithCachedAST
std::vector< Path > getFilesWithCachedAST() const
Returns a list of files with ASTs currently stored in memory.
Definition: TUScheduler.cpp:1639
clang::tidy::cppcoreguidelines::join
static std::string join(ArrayRef< SpecialMemberFunctionsCheck::SpecialMemberFunctionKind > SMFS, llvm::StringRef AndOr)
Definition: SpecialMemberFunctionsCheck.cpp:78
clang::clangd::TUScheduler::update
bool update(PathRef File, ParseInputs Inputs, WantDiagnostics WD)
Schedule an update for File.
Definition: TUScheduler.cpp:1492
clang::clangd::GlobalCompilationDatabase
Provides compilation arguments used for parsing C and C++ files.
Definition: GlobalCompilationDatabase.h:36
Action
llvm::unique_function< void()> Action
Definition: TUScheduler.cpp:596
clang::clangd::TUScheduler::Options::ContextProvider
std::function< Context(PathRef)> ContextProvider
Used to create a context that wraps each single operation.
Definition: TUScheduler.h:208
Target
std::string Target
Definition: QueryDriverDatabase.cpp:64
clang::clangd::Semaphore
Limits the number of threads that can acquire the lock at the same time.
Definition: Threading.h:42
Preamble
const PreambleData & Preamble
Definition: CodeComplete.cpp:1104
Cancellation.h
clang::clangd::Deadline
A point in time we can wait for.
Definition: Threading.h:59
clang::clangd::WantDiagnostics
WantDiagnostics
Determines whether diagnostics should be generated for a file snapshot.
Definition: TUScheduler.h:53
clang::clangd::canonicalize
static llvm::SmallString< 128 > canonicalize(llvm::StringRef Path)
Definition: FileDistance.cpp:46
clang::clangd::MemoryTree::detail
MemoryTree & detail(llvm::StringRef Name)
Makes a copy of the Name in detailed mode, returns current node otherwise.
Definition: MemoryTree.h:53
Inputs
ParseInputs Inputs
Definition: TUScheduler.cpp:451
clang::clangd::isCancelled
int isCancelled(const Context &Ctx)
If the current context is within a cancelled task, returns the reason.
Definition: Cancellation.cpp:35
clang::clangd::TUScheduler::HeaderIncluderCache::HeaderIncluderCache
HeaderIncluderCache()
Definition: TUScheduler.cpp:267
clang::clangd::MemoryTree::addUsage
void addUsage(size_t Increment)
Increases size of current node by Increment.
Definition: MemoryTree.h:58
clang::clangd::wait
void wait(std::unique_lock< std::mutex > &Lock, std::condition_variable &CV, Deadline D)
Wait once on CV for the specified duration.
Definition: Threading.cpp:113
clang::clangd::ParseInputs
Information required to run clang, e.g. to parse AST or do code completion.
Definition: Compiler.h:47
CanonicalIncludes.h
ns1::ns2::D
@ D
Definition: CategoricalFeature.h:3
clang::clangd::InputsAndPreamble::Contents
llvm::StringRef Contents
Definition: TUScheduler.h:44
clang::clangd::FileStatus::state
std::string state
The human-readable string presents the current state of the file, can be shown in the UI (e....
Definition: Protocol.h:1571
clang::clangd::ErrorCode::InvalidParams
@ InvalidParams
clang::clangd::ParsingCallbacks
Definition: TUScheduler.h:129
clang::clangd::TUScheduler::HeaderIncluderCache::remove
void remove(PathRef MainFile)
Definition: TUScheduler.cpp:286
clang::clangd::MemoryTree
A tree that can be used to represent memory usage of nested components while preserving the hierarchy...
Definition: MemoryTree.h:32
GlobalCompilationDatabase.h
clang::clangd::PreambleAction::Idle
@ Idle
clang::clangd::TUScheduler::ASTCache::ASTCache
ASTCache(unsigned MaxRetainedASTs)
Definition: TUScheduler.cpp:117
clang::clangd::buildPreamble
std::shared_ptr< const PreambleData > buildPreamble(PathRef FileName, CompilerInvocation CI, const ParseInputs &Inputs, bool StoreInMemory, PreambleParsedCallback PreambleCallback)
Build a preamble for the new inputs unless an old one can be reused.
Definition: Preamble.cpp:318
clang::clangd::trace::enabled
bool enabled()
Returns true if there is an active tracer.
Definition: Trace.cpp:284
Logger.h
Threading.h
clang::clangd::FileStatus::uri
URIForFile uri
The text document's URI.
Definition: Protocol.h:1568
Name
std::string Name
Definition: TUScheduler.cpp:597
clang::clangd::TUScheduler::HeaderIncluderCache::get
std::string get(PathRef Header) const
Get the mainfile associated with Header, or the empty string if none.
Definition: TUScheduler.cpp:294
SPAN_ATTACH
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:164
clang::clangd::TUScheduler::FileData::Contents
std::string Contents
Latest inputs, passed to TUScheduler::update().
Definition: TUScheduler.cpp:1445
clang::clangd::vlog
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:73
Diagnostics.h
clang::clangd::TUScheduler::remove
void remove(PathRef File)
Remove File from the list of tracked files and schedule removal of its resources.
Definition: TUScheduler.cpp:1518
clang::clangd::Key
Values in a Context are indexed by typed keys.
Definition: Context.h:40
FileName
StringRef FileName
Definition: KernelNameRestrictionCheck.cpp:46
clang::clangd::TUScheduler::fileStats
llvm::StringMap< FileStats > fileStats() const
Returns resources used for each of the currently open files.
Definition: TUScheduler.cpp:1631
clang::clangd::TUScheduler::ASTCache::getUsedBytes
std::size_t getUsedBytes(Key K)
Returns result of getUsedBytes() for the AST cached by K.
Definition: TUScheduler.cpp:121
clang::clangd::WithContext
WithContext replaces Context::current() with a provided scope.
Definition: Context.h:187
clang::clangd::DebouncePolicy::Min
clock::duration Min
The minimum time that we always debounce for.
Definition: TUScheduler.h:78
clang::tidy::bugprone::PP
static Preprocessor * PP
Definition: BadSignalToKillThreadCheck.cpp:29
clang::clangd::buildCompilerInvocation
std::unique_ptr< CompilerInvocation > buildCompilerInvocation(const ParseInputs &Inputs, clang::DiagnosticConsumer &D, std::vector< std::string > *CC1Args)
Builds compiler invocation that could be used to build AST or preamble.
Definition: Compiler.cpp:84
Counter
trace::Metric Counter
Definition: TraceTests.cpp:152
clang::clangd::TUScheduler::ASTCache
An LRU cache of idle ASTs.
Definition: TUScheduler.cpp:113
Compiler.h
clang::clangd::TUScheduler::runQuick
void runQuick(llvm::StringRef Name, llvm::StringRef Path, llvm::unique_function< void()> Action)
Similar to run, except the task is expected to be quick.
Definition: TUScheduler.cpp:1534
AddTime
steady_clock::time_point AddTime
Definition: TUScheduler.cpp:598
clang::clangd::Context::derive
Context derive(const Key< Type > &Key, typename std::decay< Type >::type Value) const &
Derives a child context It is safe to move or destroy a parent context after calling derive().
Definition: Context.h:119
clang::clangd::TUScheduler::profile
void profile(MemoryTree &MT) const
Definition: TUScheduler.cpp:1677
clang::clangd::PathRef
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:29
clang::clangd::TUScheduler::ASTActionInvalidation
ASTActionInvalidation
Defines how a runWithAST action is implicitly cancelled by other actions.
Definition: TUScheduler.h:257
clang::clangd::trace::Metric
Represents measurements of clangd events, e.g.
Definition: Trace.h:38
clang::clangd::InputsAndPreamble
Definition: TUScheduler.h:43
InvalidationPolicy
TUScheduler::ASTActionInvalidation InvalidationPolicy
Definition: TUScheduler.cpp:602
clang::clangd::kFileBeingProcessed
static clang::clangd::Key< std::string > kFileBeingProcessed
Definition: TUScheduler.cpp:101
clang::clangd::TUScheduler::HeaderIncluderCache
A map from header files to an opened "proxy" file that includes them.
Definition: TUScheduler.cpp:210
clang::clangd::TUScheduler::Options
Definition: TUScheduler.h:188
clang::clangd::TUScheduler::runWithAST
void runWithAST(llvm::StringRef Name, PathRef File, Callback< InputsAndAST > Action, ASTActionInvalidation=NoInvalidation)
Schedule an async read of the AST.
Definition: TUScheduler.cpp:1562
clang::clangd::TUScheduler::ASTCache::put
void put(Key K, std::unique_ptr< ParsedAST > V)
Store the value in the pool, possibly removing the last used AST.
Definition: TUScheduler.cpp:131
clang::clangd::DebouncePolicy
Clangd may wait after an update to see if another one comes along.
Definition: TUScheduler.h:74
MemoryTree.h
clang::clangd::Canceler
std::function< void()> Canceler
A canceller requests cancellation of a task, when called.
Definition: Cancellation.h:70
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
Arena
llvm::BumpPtrAllocator Arena
Definition: Serialization.cpp:211
TUScheduler.h
clang::clangd::FileStatus
Clangd extension: indicates the current state of the file in clangd, sent from server via the textDoc...
Definition: Protocol.h:1566
clang::clangd::DebouncePolicy::RebuildRatio
float RebuildRatio
Target debounce, as a fraction of file rebuild time.
Definition: TUScheduler.h:83
clang::clangd::DebouncePolicy::Max
clock::duration Max
The maximum time we may debounce for.
Definition: TUScheduler.h:80
clang::clangd::TUScheduler::PreambleConsistency
PreambleConsistency
Controls whether preamble reads wait for the preamble to be up-to-date.
Definition: TUScheduler.h:280
clang::clangd::TUScheduler::FileData
Definition: TUScheduler.cpp:1443
clang::clangd::TUScheduler::HeaderIncluderCache::update
void update(PathRef MainFile, llvm::ArrayRef< std::string > Headers)
Definition: TUScheduler.cpp:273
clang::clangd::DebouncePolicy::fixed
static DebouncePolicy fixed(clock::duration)
A policy that always returns the same duration, useful for tests.
Definition: TUScheduler.cpp:1671
clang::clangd::Callback
llvm::unique_function< void(llvm::Expected< T >)> Callback
A Callback<T> is a void function that accepts Expected<T>.
Definition: Function.h:28
MainFile
std::string MainFile
Definition: HeadersTests.cpp:125
Invalidate
Canceler Invalidate
Definition: TUScheduler.cpp:603
clang::clangd::TUScheduler::FileData::Worker
ASTWorkerHandle Worker
Definition: TUScheduler.cpp:1446
clang::clangd::TUScheduler::ASTCache::take
llvm::Optional< std::unique_ptr< ParsedAST > > take(Key K, const trace::Metric *AccessMetric=nullptr)
Returns the cached value for K, or llvm::None if the value is not in the cache anymore.
Definition: TUScheduler.cpp:151
clang::clangd::elog
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:62
clang::clangd::detail::log
void log(Logger::Level L, const char *Fmt, Ts &&... Vals)
Definition: Logger.h:48
clang::clangd::DebouncePolicy::compute
clock::duration compute(llvm::ArrayRef< clock::duration > History) const
Compute the time to debounce based on this policy and recent build times.
Definition: TUScheduler.cpp:1650
K
Kind K
Definition: Rename.cpp:442
clang::clangd::TUScheduler::ASTCache::Key
const ASTWorker * Key
Definition: TUScheduler.cpp:115
clang::clangd::TUScheduler::HeaderIncluderCache::getUsedBytes
size_t getUsedBytes() const
Definition: TUScheduler.cpp:299
Context.h
clang::clangd::TUScheduler::getFileBeingProcessedInContext
static llvm::Optional< llvm::StringRef > getFileBeingProcessedInContext()
Definition: TUScheduler.cpp:103
ParsedAST.h
clang::clangd::trace::Span
Records an event whose duration is the lifetime of the Span object.
Definition: Trace.h:143