clang-tools  10.0.0svn
TUScheduler.h
Go to the documentation of this file.
1 //===--- TUScheduler.h -------------------------------------------*-C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
10 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
11 
12 #include "Compiler.h"
13 #include "Diagnostics.h"
14 #include "Function.h"
16 #include "Path.h"
17 #include "Threading.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/StringMap.h"
22 #include "llvm/ADT/StringRef.h"
23 
24 namespace clang {
25 namespace clangd {
26 class ParsedAST;
27 struct PreambleData;
28 
29 /// Returns a number of a default async threads to use for TUScheduler.
30 /// Returned value is always >= 1 (i.e. will not cause requests to be processed
31 /// synchronously).
33 
34 struct InputsAndAST {
37 };
38 
40  llvm::StringRef Contents;
41  const tooling::CompileCommand &Command;
42  // This can be nullptr if no preamble is availble.
44 };
45 
46 /// Determines whether diagnostics should be generated for a file snapshot.
47 enum class WantDiagnostics {
48  Yes, /// Diagnostics must be generated for this snapshot.
49  No, /// Diagnostics must not be generated for this snapshot.
50  Auto, /// Diagnostics must be generated for this snapshot or a subsequent one,
51  /// within a bounded amount of time.
52 };
53 
54 /// Configuration of the AST retention policy. This only covers retention of
55 /// *idle* ASTs. If queue has operations requiring the AST, they might be
56 /// kept in memory.
58  /// Maximum number of ASTs to be retained in memory when there are no pending
59  /// requests for them.
60  unsigned MaxRetainedASTs = 3;
61 };
62 
63 struct TUAction {
64  enum State {
65  Queued, // The TU is pending in the thread task queue to be built.
66  RunningAction, // Starting running actions on the TU.
67  BuildingPreamble, // The preamble of the TU is being built.
68  BuildingFile, // The TU is being built. It is only emitted when building
69  // the AST for diagnostics in write action (update).
70  Idle, // Indicates the worker thread is idle, and ready to run any upcoming
71  // actions.
72  };
73  TUAction(State S, llvm::StringRef Name) : S(S), Name(Name) {}
75  /// The name of the action currently running, e.g. Update, GoToDef, Hover.
76  /// Empty if we are in the idle state.
77  std::string Name;
78 };
79 
80 // Internal status of the TU in TUScheduler.
81 struct TUStatus {
82  struct BuildDetails {
83  /// Indicates whether clang failed to build the TU.
84  bool BuildFailed = false;
85  /// Indicates whether we reused the prebuilt AST.
86  bool ReuseAST = false;
87  };
88  /// Serialize this to an LSP file status item.
89  FileStatus render(PathRef File) const;
90 
93 };
94 
96 public:
97  virtual ~ParsingCallbacks() = default;
98 
99  /// Called on the AST that was built for emitting the preamble. The built AST
100  /// contains only AST nodes from the #include directives at the start of the
101  /// file. AST node in the current file should be observed on onMainAST call.
102  virtual void onPreambleAST(PathRef Path, ASTContext &Ctx,
103  std::shared_ptr<clang::Preprocessor> PP,
104  const CanonicalIncludes &) {}
105 
106  /// The argument function is run under the critical section guarding against
107  /// races when closing the files.
108  using PublishFn = llvm::function_ref<void(llvm::function_ref<void()>)>;
109  /// Called on the AST built for the file itself. Note that preamble AST nodes
110  /// are not deserialized and should be processed in the onPreambleAST call
111  /// instead.
112  /// The \p AST always contains all AST nodes for the main file itself, and
113  /// only a portion of the AST nodes deserialized from the preamble. Note that
114  /// some nodes from the preamble may have been deserialized and may also be
115  /// accessed from the main file AST, e.g. redecls of functions from preamble,
116  /// etc. Clients are expected to process only the AST nodes from the main file
117  /// in this callback (obtained via ParsedAST::getLocalTopLevelDecls) to obtain
118  /// optimal performance.
119  ///
120  /// When information about the file (diagnostics, syntax highlighting) is
121  /// published to clients, this should be wrapped in Publish, e.g.
122  /// void onMainAST(...) {
123  /// Highlights = computeHighlights();
124  /// Publish([&] { notifyHighlights(Path, Highlights); });
125  /// }
126  /// This guarantees that clients will see results in the correct sequence if
127  /// the file is concurrently closed and/or reopened. (The lambda passed to
128  /// Publish() may never run in this case).
129  virtual void onMainAST(PathRef Path, ParsedAST &AST, PublishFn Publish) {}
130 
131  /// Called whenever the AST fails to build. \p Diags will have the diagnostics
132  /// that led to failure.
133  virtual void onFailedAST(PathRef Path, std::vector<Diag> Diags,
134  PublishFn Publish) {}
135 
136  /// Called whenever the TU status is updated.
137  virtual void onFileUpdated(PathRef File, const TUStatus &Status) {}
138 };
139 
140 /// Handles running tasks for ClangdServer and managing the resources (e.g.,
141 /// preambles and ASTs) for opened files.
142 /// TUScheduler is not thread-safe, only one thread should be providing updates
143 /// and scheduling tasks.
144 /// Callbacks are run on a threadpool and it's appropriate to do slow work in
145 /// them. Each task has a name, used for tracing (should be UpperCamelCase).
146 /// FIXME(sammccall): pull out a scheduler options struct.
147 class TUScheduler {
148 public:
149  TUScheduler(const GlobalCompilationDatabase &CDB, unsigned AsyncThreadsCount,
150  bool StorePreamblesInMemory,
151  std::unique_ptr<ParsingCallbacks> ASTCallbacks,
152  std::chrono::steady_clock::duration UpdateDebounce,
153  ASTRetentionPolicy RetentionPolicy);
154  ~TUScheduler();
155 
156  /// Returns estimated memory usage for each of the currently open files.
157  /// The order of results is unspecified.
158  std::vector<std::pair<Path, std::size_t>> getUsedBytesPerFile() const;
159 
160  /// Returns a list of files with ASTs currently stored in memory. This method
161  /// is not very reliable and is only used for test. E.g., the results will not
162  /// contain files that currently run something over their AST.
163  std::vector<Path> getFilesWithCachedAST() const;
164 
165  /// Schedule an update for \p File.
166  /// The compile command in \p Inputs is ignored; worker queries CDB to get
167  /// the actual compile command.
168  /// If diagnostics are requested (Yes), and the context is cancelled
169  /// before they are prepared, they may be skipped if eventual-consistency
170  /// permits it (i.e. WantDiagnostics is downgraded to Auto).
171  /// Returns true if the file was not previously tracked.
172  bool update(PathRef File, ParseInputs Inputs, WantDiagnostics WD);
173 
174  /// Remove \p File from the list of tracked files and schedule removal of its
175  /// resources. Pending diagnostics for closed files may not be delivered, even
176  /// if requested with WantDiags::Auto or WantDiags::Yes.
177  void remove(PathRef File);
178 
179  /// Returns the current contents of the buffer for File, per last update().
180  /// The returned StringRef may be invalidated by any write to TUScheduler.
181  llvm::StringRef getContents(PathRef File) const;
182 
183  /// Schedule an async task with no dependencies.
184  void run(llvm::StringRef Name, llvm::unique_function<void()> Action);
185 
186  /// Schedule an async read of the AST. \p Action will be called when AST is
187  /// ready. The AST passed to \p Action refers to the version of \p File
188  /// tracked at the time of the call, even if new updates are received before
189  /// \p Action is executed.
190  /// If an error occurs during processing, it is forwarded to the \p Action
191  /// callback.
192  /// If the context is cancelled before the AST is ready, the callback will
193  /// receive a CancelledError.
194  void runWithAST(llvm::StringRef Name, PathRef File,
196 
197  /// Controls whether preamble reads wait for the preamble to be up-to-date.
199  /// The preamble is generated from the current version of the file.
200  /// If the content was recently updated, we will wait until we have a
201  /// preamble that reflects that update.
202  /// This is the slowest option, and may be delayed by other tasks.
204  /// The preamble may be generated from an older version of the file.
205  /// Reading from locations in the preamble may cause files to be re-read.
206  /// This gives callers two options:
207  /// - validate that the preamble is still valid, and only use it if so
208  /// - accept that the preamble contents may be outdated, and try to avoid
209  /// reading source code from headers.
210  /// This is the fastest option, usually a preamble is available immediately.
212  /// Besides accepting stale preamble, this also allow preamble to be absent
213  /// (not ready or failed to build).
215  };
216 
217  /// Schedule an async read of the preamble.
218  /// If there's no up-to-date preamble, we follow the PreambleConsistency
219  /// policy.
220  /// If an error occurs, it is forwarded to the \p Action callback.
221  /// Context cancellation is ignored and should be handled by the Action.
222  /// (In practice, the Action is almost always executed immediately).
223  void runWithPreamble(llvm::StringRef Name, PathRef File,
224  PreambleConsistency Consistency,
226 
227  /// Wait until there are no scheduled or running tasks.
228  /// Mostly useful for synchronizing tests.
229  bool blockUntilIdle(Deadline D) const;
230 
231 private:
232  /// This class stores per-file data in the Files map.
233  struct FileData;
234 
235 public:
236  /// Responsible for retaining and rebuilding idle ASTs. An implementation is
237  /// an LRU cache.
238  class ASTCache;
239 
240  // The file being built/processed in the current thread. This is a hack in
241  // order to get the file name into the index implementations. Do not depend on
242  // this inside clangd.
243  // FIXME: remove this when there is proper index support via build system
244  // integration.
245  static llvm::Optional<llvm::StringRef> getFileBeingProcessedInContext();
246 
247 private:
248  const GlobalCompilationDatabase &CDB;
249  const bool StorePreamblesInMemory;
250  std::unique_ptr<ParsingCallbacks> Callbacks; // not nullptr
251  Semaphore Barrier;
252  llvm::StringMap<std::unique_ptr<FileData>> Files;
253  std::unique_ptr<ASTCache> IdleASTs;
254  // None when running tasks synchronously and non-None when running tasks
255  // asynchronously.
256  llvm::Optional<AsyncTaskRunner> PreambleTasks;
257  llvm::Optional<AsyncTaskRunner> WorkerThreads;
258  std::chrono::steady_clock::duration UpdateDebounce;
259 };
260 
261 } // namespace clangd
262 } // namespace clang
263 
264 #endif
PreambleConsistency
Controls whether preamble reads wait for the preamble to be up-to-date.
Definition: TUScheduler.h:198
WantDiagnostics
Determines whether diagnostics should be generated for a file snapshot.
Definition: TUScheduler.h:47
The preamble may be generated from an older version of the file.
Definition: TUScheduler.h:211
Diagnostics must be generated for this snapshot.
const tooling::CompileCommand & Command
Definition: TUScheduler.h:41
virtual void onMainAST(PathRef Path, ParsedAST &AST, PublishFn Publish)
Called on the AST built for the file itself.
Definition: TUScheduler.h:129
Besides accepting stale preamble, this also allow preamble to be absent (not ready or failed to build...
Definition: TUScheduler.h:214
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:23
llvm::unique_function< void(llvm::Expected< T >)> Callback
A Callback<T> is a void function that accepts Expected<T>.
Definition: Function.h:28
Limits the number of threads that can acquire the lock at the same time.
Definition: Threading.h:41
virtual void onFileUpdated(PathRef File, const TUStatus &Status)
Called whenever the TU status is updated.
Definition: TUScheduler.h:137
BuildDetails Details
Definition: TUScheduler.h:92
Configuration of the AST retention policy.
Definition: TUScheduler.h:57
Provides compilation arguments used for parsing C and C++ files.
Maps a definition location onto an #include file, based on a set of filename rules.
virtual void onFailedAST(PathRef Path, std::vector< Diag > Diags, PublishFn Publish)
Called whenever the AST fails to build.
Definition: TUScheduler.h:133
Context Ctx
llvm::unique_function< void()> Action
std::string Path
A typedef to represent a file path.
Definition: Path.h:20
const ParseInputs & Inputs
Definition: TUScheduler.h:35
static constexpr llvm::StringLiteral Name
const Decl * D
Definition: XRefs.cpp:849
virtual void onPreambleAST(PathRef Path, ASTContext &Ctx, std::shared_ptr< clang::Preprocessor > PP, const CanonicalIncludes &)
Called on the AST that was built for emitting the preamble.
Definition: TUScheduler.h:102
Stores and provides access to parsed AST.
Definition: ParsedAST.h:46
unsigned getDefaultAsyncThreadsCount()
Returns a number of a default async threads to use for TUScheduler.
TUAction(State S, llvm::StringRef Name)
Definition: TUScheduler.h:73
Information required to run clang, e.g. to parse AST or do code completion.
Definition: Compiler.h:44
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
std::string Name
The name of the action currently running, e.g.
Definition: TUScheduler.h:77
A point in time we can wait for.
Definition: Threading.h:58
Clangd extension: indicates the current state of the file in clangd, sent from server via the textDoc...
Definition: Protocol.h:1195
Handles running tasks for ClangdServer and managing the resources (e.g., preambles and ASTs) for open...
Definition: TUScheduler.h:147
const PreambleData * Preamble
Definition: TUScheduler.h:43
An LRU cache of idle ASTs.
Definition: TUScheduler.cpp:86
The parsed preamble and associated data.
Definition: Preamble.h:45
Diagnostics must not be generated for this snapshot.
llvm::StringMap< std::string > Files
The preamble is generated from the current version of the file.
Definition: TUScheduler.h:203
llvm::function_ref< void(llvm::function_ref< void()>)> PublishFn
The argument function is run under the critical section guarding against races when closing the files...
Definition: TUScheduler.h:108