clang-tools  11.0.0git
TUScheduler.h
Go to the documentation of this file.
1 //===--- TUScheduler.h -------------------------------------------*-C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
10 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
11 
12 #include "Compiler.h"
13 #include "Diagnostics.h"
16 #include "support/Function.h"
17 #include "support/Path.h"
18 #include "support/Threading.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/StringMap.h"
22 #include "llvm/ADT/StringRef.h"
23 #include <chrono>
24 
25 namespace clang {
26 namespace clangd {
27 class ParsedAST;
28 struct PreambleData;
29 
30 /// Returns a number of a default async threads to use for TUScheduler.
31 /// Returned value is always >= 1 (i.e. will not cause requests to be processed
32 /// synchronously).
34 
35 struct InputsAndAST {
38 };
39 
41  llvm::StringRef Contents;
42  const tooling::CompileCommand &Command;
43  // This can be nullptr if no preamble is available.
45 };
46 
47 /// Determines whether diagnostics should be generated for a file snapshot.
48 enum class WantDiagnostics {
49  Yes, /// Diagnostics must be generated for this snapshot.
50  No, /// Diagnostics must not be generated for this snapshot.
51  Auto, /// Diagnostics must be generated for this snapshot or a subsequent one,
52  /// within a bounded amount of time.
53 };
54 
55 /// Configuration of the AST retention policy. This only covers retention of
56 /// *idle* ASTs. If queue has operations requiring the AST, they might be
57 /// kept in memory.
59  /// Maximum number of ASTs to be retained in memory when there are no pending
60  /// requests for them.
61  unsigned MaxRetainedASTs = 3;
62 };
63 
64 /// Clangd may wait after an update to see if another one comes along.
65 /// This is so we rebuild once the user stops typing, not when they start.
66 /// Debounce may be disabled/interrupted if we must build this version.
67 /// The debounce time is responsive to user preferences and rebuild time.
68 /// In the future, we could also consider different types of edits.
70  using clock = std::chrono::steady_clock;
71 
72  /// The minimum time that we always debounce for.
73  clock::duration Min = /*zero*/ {};
74  /// The maximum time we may debounce for.
75  clock::duration Max = /*zero*/ {};
76  /// Target debounce, as a fraction of file rebuild time.
77  /// e.g. RebuildRatio = 2, recent builds took 200ms => debounce for 400ms.
78  float RebuildRatio = 1;
79 
80  /// Compute the time to debounce based on this policy and recent build times.
81  clock::duration compute(llvm::ArrayRef<clock::duration> History) const;
82  /// A policy that always returns the same duration, useful for tests.
83  static DebouncePolicy fixed(clock::duration);
84 };
85 
86 enum class PreambleAction {
87  Idle,
88  Building,
89 };
90 
91 struct ASTAction {
92  enum Kind {
93  Queued, // The action is pending in the thread task queue to be run.
94  RunningAction, // Started running actions on the TU.
95  Building, // The AST is being built.
96  Idle, // Indicates the worker thread is idle, and ready to run any upcoming
97  // actions.
98  };
99  ASTAction() = default;
100  ASTAction(Kind K, llvm::StringRef Name) : K(K), Name(Name) {}
102  /// The name of the action currently running, e.g. Update, GoToDef, Hover.
103  /// Empty if we are in the idle state.
104  std::string Name;
105 };
106 
107 // Internal status of the TU in TUScheduler.
108 struct TUStatus {
109  struct BuildDetails {
110  /// Indicates whether clang failed to build the TU.
111  bool BuildFailed = false;
112  /// Indicates whether we reused the prebuilt AST.
113  bool ReuseAST = false;
114  };
115  /// Serialize this to an LSP file status item.
116  FileStatus render(PathRef File) const;
117 
120  /// Stores status of the last build for the translation unit.
122 };
123 
125 public:
126  virtual ~ParsingCallbacks() = default;
127 
128  /// Called on the AST that was built for emitting the preamble. The built AST
129  /// contains only AST nodes from the #include directives at the start of the
130  /// file. AST node in the current file should be observed on onMainAST call.
131  virtual void onPreambleAST(PathRef Path, llvm::StringRef Version,
132  ASTContext &Ctx,
133  std::shared_ptr<clang::Preprocessor> PP,
134  const CanonicalIncludes &) {}
135 
136  /// The argument function is run under the critical section guarding against
137  /// races when closing the files.
138  using PublishFn = llvm::function_ref<void(llvm::function_ref<void()>)>;
139  /// Called on the AST built for the file itself. Note that preamble AST nodes
140  /// are not deserialized and should be processed in the onPreambleAST call
141  /// instead.
142  /// The \p AST always contains all AST nodes for the main file itself, and
143  /// only a portion of the AST nodes deserialized from the preamble. Note that
144  /// some nodes from the preamble may have been deserialized and may also be
145  /// accessed from the main file AST, e.g. redecls of functions from preamble,
146  /// etc. Clients are expected to process only the AST nodes from the main file
147  /// in this callback (obtained via ParsedAST::getLocalTopLevelDecls) to obtain
148  /// optimal performance.
149  ///
150  /// When information about the file (diagnostics, syntax highlighting) is
151  /// published to clients, this should be wrapped in Publish, e.g.
152  /// void onMainAST(...) {
153  /// Highlights = computeHighlights();
154  /// Publish([&] { notifyHighlights(Path, Highlights); });
155  /// }
156  /// This guarantees that clients will see results in the correct sequence if
157  /// the file is concurrently closed and/or reopened. (The lambda passed to
158  /// Publish() may never run in this case).
159  virtual void onMainAST(PathRef Path, ParsedAST &AST, PublishFn Publish) {}
160 
161  /// Called whenever the AST fails to build. \p Diags will have the diagnostics
162  /// that led to failure.
163  virtual void onFailedAST(PathRef Path, llvm::StringRef Version,
164  std::vector<Diag> Diags, PublishFn Publish) {}
165 
166  /// Called whenever the TU status is updated.
167  virtual void onFileUpdated(PathRef File, const TUStatus &Status) {}
168 };
169 
170 /// Handles running tasks for ClangdServer and managing the resources (e.g.,
171 /// preambles and ASTs) for opened files.
172 /// TUScheduler is not thread-safe, only one thread should be providing updates
173 /// and scheduling tasks.
174 /// Callbacks are run on a threadpool and it's appropriate to do slow work in
175 /// them. Each task has a name, used for tracing (should be UpperCamelCase).
176 class TUScheduler {
177 public:
178  struct Options {
179  /// Number of concurrent actions.
180  /// Governs per-file worker threads and threads spawned for other tasks.
181  /// (This does not prevent threads being spawned, but rather blocks them).
182  /// If 0, executes actions synchronously on the calling thread.
183  unsigned AsyncThreadsCount = getDefaultAsyncThreadsCount();
184 
185  /// Cache (large) preamble data in RAM rather than temporary files on disk.
186  bool StorePreamblesInMemory = false;
187 
188  /// Time to wait after an update to see if another one comes along.
189  /// This tries to ensure we rebuild once the user stops typing.
191 
192  /// Determines when to keep idle ASTs in memory for future use.
194 
195  /// Whether to run PreamblePeer asynchronously.
196  /// No-op if AsyncThreadsCount is 0.
197  bool AsyncPreambleBuilds = false;
198 
199  /// Used to create a context that wraps each single operation.
200  /// Typically to inject per-file configuration.
201  /// If the path is empty, context sholud be "generic".
202  std::function<Context(PathRef)> ContextProvider;
203  };
204 
205  TUScheduler(const GlobalCompilationDatabase &CDB, const Options &Opts,
206  std::unique_ptr<ParsingCallbacks> ASTCallbacks = nullptr);
207  ~TUScheduler();
208 
209  struct FileStats {
210  std::size_t UsedBytes = 0;
211  unsigned PreambleBuilds = 0;
212  unsigned ASTBuilds = 0;
213  };
214  /// Returns resources used for each of the currently open files.
215  /// Results are inherently racy as they measure activity of other threads.
216  llvm::StringMap<FileStats> fileStats() const;
217 
218  /// Returns a list of files with ASTs currently stored in memory. This method
219  /// is not very reliable and is only used for test. E.g., the results will not
220  /// contain files that currently run something over their AST.
221  std::vector<Path> getFilesWithCachedAST() const;
222 
223  /// Schedule an update for \p File.
224  /// The compile command in \p Inputs is ignored; worker queries CDB to get
225  /// the actual compile command.
226  /// If diagnostics are requested (Yes), and the context is cancelled
227  /// before they are prepared, they may be skipped if eventual-consistency
228  /// permits it (i.e. WantDiagnostics is downgraded to Auto).
229  /// Returns true if the file was not previously tracked.
230  bool update(PathRef File, ParseInputs Inputs, WantDiagnostics WD);
231 
232  /// Remove \p File from the list of tracked files and schedule removal of its
233  /// resources. Pending diagnostics for closed files may not be delivered, even
234  /// if requested with WantDiags::Auto or WantDiags::Yes.
235  void remove(PathRef File);
236 
237  /// Returns a snapshot of all file buffer contents, per last update().
238  llvm::StringMap<std::string> getAllFileContents() const;
239 
240  /// Schedule an async task with no dependencies.
241  /// Path may be empty (it is used only to set the Context).
242  void run(llvm::StringRef Name, llvm::StringRef Path,
243  llvm::unique_function<void()> Action);
244 
245  /// Defines how a runWithAST action is implicitly cancelled by other actions.
247  /// The request will run unless explicitly cancelled.
249  /// The request will be implicitly cancelled by a subsequent update().
250  /// (Only if the request was not yet cancelled).
251  /// Useful for requests that are generated by clients, without any explicit
252  /// user action. These can otherwise e.g. force every version to be built.
254  };
255 
256  /// Schedule an async read of the AST. \p Action will be called when AST is
257  /// ready. The AST passed to \p Action refers to the version of \p File
258  /// tracked at the time of the call, even if new updates are received before
259  /// \p Action is executed.
260  /// If an error occurs during processing, it is forwarded to the \p Action
261  /// callback.
262  /// If the context is cancelled before the AST is ready, or the invalidation
263  /// policy is triggered, the callback will receive a CancelledError.
264  void runWithAST(llvm::StringRef Name, PathRef File,
266  ASTActionInvalidation = NoInvalidation);
267 
268  /// Controls whether preamble reads wait for the preamble to be up-to-date.
270  /// The preamble may be generated from an older version of the file.
271  /// Reading from locations in the preamble may cause files to be re-read.
272  /// This gives callers two options:
273  /// - validate that the preamble is still valid, and only use it if so
274  /// - accept that the preamble contents may be outdated, and try to avoid
275  /// reading source code from headers.
276  /// This is the fastest option, usually a preamble is available immediately.
278  /// Besides accepting stale preamble, this also allow preamble to be absent
279  /// (not ready or failed to build).
281  };
282 
283  /// Schedule an async read of the preamble.
284  /// If there's no up-to-date preamble, we follow the PreambleConsistency
285  /// policy.
286  /// If an error occurs, it is forwarded to the \p Action callback.
287  /// Context cancellation is ignored and should be handled by the Action.
288  /// (In practice, the Action is almost always executed immediately).
289  void runWithPreamble(llvm::StringRef Name, PathRef File,
290  PreambleConsistency Consistency,
292 
293  /// Wait until there are no scheduled or running tasks.
294  /// Mostly useful for synchronizing tests.
295  bool blockUntilIdle(Deadline D) const;
296 
297 private:
298  /// This class stores per-file data in the Files map.
299  struct FileData;
300 
301 public:
302  /// Responsible for retaining and rebuilding idle ASTs. An implementation is
303  /// an LRU cache.
304  class ASTCache;
305 
306  // The file being built/processed in the current thread. This is a hack in
307  // order to get the file name into the index implementations. Do not depend on
308  // this inside clangd.
309  // FIXME: remove this when there is proper index support via build system
310  // integration.
311  // FIXME: move to ClangdServer via createProcessingContext.
312  static llvm::Optional<llvm::StringRef> getFileBeingProcessedInContext();
313 
314 private:
315  const GlobalCompilationDatabase &CDB;
316  const Options Opts;
317  std::unique_ptr<ParsingCallbacks> Callbacks; // not nullptr
318  Semaphore Barrier;
319  llvm::StringMap<std::unique_ptr<FileData>> Files;
320  std::unique_ptr<ASTCache> IdleASTs;
321  // None when running tasks synchronously and non-None when running tasks
322  // asynchronously.
323  llvm::Optional<AsyncTaskRunner> PreambleTasks;
324  llvm::Optional<AsyncTaskRunner> WorkerThreads;
325 };
326 
327 } // namespace clangd
328 } // namespace clang
329 
330 #endif
PreambleConsistency
Controls whether preamble reads wait for the preamble to be up-to-date.
Definition: TUScheduler.h:269
WantDiagnostics
Determines whether diagnostics should be generated for a file snapshot.
Definition: TUScheduler.h:48
The preamble may be generated from an older version of the file.
Definition: TUScheduler.h:277
Diagnostics must be generated for this snapshot.
const tooling::CompileCommand & Command
Definition: TUScheduler.h:42
virtual void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx, std::shared_ptr< clang::Preprocessor > PP, const CanonicalIncludes &)
Called on the AST that was built for emitting the preamble.
Definition: TUScheduler.h:131
virtual void onMainAST(PathRef Path, ParsedAST &AST, PublishFn Publish)
Called on the AST built for the file itself.
Definition: TUScheduler.h:159
Besides accepting stale preamble, this also allow preamble to be absent (not ready or failed to build...
Definition: TUScheduler.h:280
std::string Name
The name of the action currently running, e.g.
Definition: TUScheduler.h:104
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:23
llvm::unique_function< void(llvm::Expected< T >)> Callback
A Callback<T> is a void function that accepts Expected<T>.
Definition: Function.h:28
Limits the number of threads that can acquire the lock at the same time.
Definition: Threading.h:41
virtual void onFileUpdated(PathRef File, const TUStatus &Status)
Called whenever the TU status is updated.
Definition: TUScheduler.h:167
BuildDetails Details
Stores status of the last build for the translation unit.
Definition: TUScheduler.h:121
virtual void onFailedAST(PathRef Path, llvm::StringRef Version, std::vector< Diag > Diags, PublishFn Publish)
Called whenever the AST fails to build.
Definition: TUScheduler.h:163
Configuration of the AST retention policy.
Definition: TUScheduler.h:58
Provides compilation arguments used for parsing C and C++ files.
Maps a definition location onto an #include file, based on a set of filename rules.
Context Ctx
llvm::unique_function< void()> Action
ASTActionInvalidation
Defines how a runWithAST action is implicitly cancelled by other actions.
Definition: TUScheduler.h:246
std::string Path
A typedef to represent a file path.
Definition: Path.h:20
CapturedDiags Diags
const ParseInputs & Inputs
Definition: TUScheduler.h:36
static constexpr llvm::StringLiteral Name
std::chrono::steady_clock clock
Definition: TUScheduler.h:70
Stores and provides access to parsed AST.
Definition: ParsedAST.h:48
unsigned getDefaultAsyncThreadsCount()
Returns a number of a default async threads to use for TUScheduler.
std::function< Context(PathRef)> ContextProvider
Used to create a context that wraps each single operation.
Definition: TUScheduler.h:202
Information required to run clang, e.g. to parse AST or do code completion.
Definition: Compiler.h:47
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
The request will run unless explicitly cancelled.
Definition: TUScheduler.h:248
DebouncePolicy UpdateDebounce
Time to wait after an update to see if another one comes along.
Definition: TUScheduler.h:190
A point in time we can wait for.
Definition: Threading.h:58
Clangd extension: indicates the current state of the file in clangd, sent from server via the textDoc...
Definition: Protocol.h:1351
Handles running tasks for ClangdServer and managing the resources (e.g., preambles and ASTs) for open...
Definition: TUScheduler.h:176
ASTAction(Kind K, llvm::StringRef Name)
Definition: TUScheduler.h:100
const PreambleData * Preamble
Definition: TUScheduler.h:44
The request will be implicitly cancelled by a subsequent update().
Definition: TUScheduler.h:253
An LRU cache of idle ASTs.
The parsed preamble and associated data.
Definition: Preamble.h:49
Clangd may wait after an update to see if another one comes along.
Definition: TUScheduler.h:69
ASTRetentionPolicy RetentionPolicy
Determines when to keep idle ASTs in memory for future use.
Definition: TUScheduler.h:193
Diagnostics must not be generated for this snapshot.
llvm::function_ref< void(llvm::function_ref< void()>)> PublishFn
The argument function is run under the critical section guarding against races when closing the files...
Definition: TUScheduler.h:138