clang-tools  16.0.0git
Preamble.cpp
Go to the documentation of this file.
1 //===--- Preamble.cpp - Reusing expensive parts of the AST ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "Preamble.h"
10 #include "Compiler.h"
11 #include "Config.h"
12 #include "Headers.h"
13 #include "SourceCode.h"
14 #include "support/Logger.h"
15 #include "support/ThreadsafeFS.h"
16 #include "support/Trace.h"
17 #include "clang/AST/DeclTemplate.h"
18 #include "clang/Basic/Diagnostic.h"
19 #include "clang/Basic/DiagnosticLex.h"
20 #include "clang/Basic/LangOptions.h"
21 #include "clang/Basic/SourceLocation.h"
22 #include "clang/Basic/SourceManager.h"
23 #include "clang/Basic/TokenKinds.h"
24 #include "clang/Frontend/CompilerInvocation.h"
25 #include "clang/Frontend/FrontendActions.h"
26 #include "clang/Lex/HeaderSearch.h"
27 #include "clang/Lex/Lexer.h"
28 #include "clang/Lex/PPCallbacks.h"
29 #include "clang/Lex/Preprocessor.h"
30 #include "clang/Lex/PreprocessorOptions.h"
31 #include "clang/Tooling/CompilationDatabase.h"
32 #include "llvm/ADT/ArrayRef.h"
33 #include "llvm/ADT/DenseMap.h"
34 #include "llvm/ADT/IntrusiveRefCntPtr.h"
35 #include "llvm/ADT/None.h"
36 #include "llvm/ADT/STLExtras.h"
37 #include "llvm/ADT/SmallString.h"
38 #include "llvm/ADT/StringRef.h"
39 #include "llvm/Support/Error.h"
40 #include "llvm/Support/ErrorHandling.h"
41 #include "llvm/Support/FormatVariadic.h"
42 #include "llvm/Support/MemoryBuffer.h"
43 #include "llvm/Support/Path.h"
44 #include "llvm/Support/VirtualFileSystem.h"
45 #include "llvm/Support/raw_ostream.h"
46 #include <iterator>
47 #include <memory>
48 #include <string>
49 #include <system_error>
50 #include <utility>
51 #include <vector>
52 
53 namespace clang {
54 namespace clangd {
55 namespace {
56 constexpr llvm::StringLiteral PreamblePatchHeaderName = "__preamble_patch__.h";
57 
58 bool compileCommandsAreEqual(const tooling::CompileCommand &LHS,
59  const tooling::CompileCommand &RHS) {
60  // We don't check for Output, it should not matter to clangd.
61  return LHS.Directory == RHS.Directory && LHS.Filename == RHS.Filename &&
62  llvm::makeArrayRef(LHS.CommandLine).equals(RHS.CommandLine);
63 }
64 
65 class CppFilePreambleCallbacks : public PreambleCallbacks {
66 public:
67  CppFilePreambleCallbacks(
68  PathRef File, PreambleParsedCallback ParsedCallback,
69  PreambleBuildStats *Stats, bool ParseForwardingFunctions,
70  std::function<void(CompilerInstance &)> BeforeExecuteCallback)
71  : File(File), ParsedCallback(ParsedCallback), Stats(Stats),
72  ParseForwardingFunctions(ParseForwardingFunctions),
73  BeforeExecuteCallback(std::move(BeforeExecuteCallback)) {}
74 
75  IncludeStructure takeIncludes() { return std::move(Includes); }
76 
77  MainFileMacros takeMacros() { return std::move(Macros); }
78 
79  std::vector<PragmaMark> takeMarks() { return std::move(Marks); }
80 
81  CanonicalIncludes takeCanonicalIncludes() { return std::move(CanonIncludes); }
82 
83  bool isMainFileIncludeGuarded() const { return IsMainFileIncludeGuarded; }
84 
85  void AfterExecute(CompilerInstance &CI) override {
86  if (ParsedCallback) {
87  trace::Span Tracer("Running PreambleCallback");
88  ParsedCallback(CI.getASTContext(), CI.getPreprocessor(), CanonIncludes);
89  }
90 
91  const SourceManager &SM = CI.getSourceManager();
92  const FileEntry *MainFE = SM.getFileEntryForID(SM.getMainFileID());
93  IsMainFileIncludeGuarded =
94  CI.getPreprocessor().getHeaderSearchInfo().isFileMultipleIncludeGuarded(
95  MainFE);
96 
97  if (Stats) {
98  const ASTContext &AST = CI.getASTContext();
99  Stats->BuildSize = AST.getASTAllocatedMemory();
100  Stats->BuildSize += AST.getSideTableAllocatedMemory();
101  Stats->BuildSize += AST.Idents.getAllocator().getTotalMemory();
102  Stats->BuildSize += AST.Selectors.getTotalMemory();
103 
104  Stats->BuildSize += AST.getSourceManager().getContentCacheSize();
105  Stats->BuildSize += AST.getSourceManager().getDataStructureSizes();
106  Stats->BuildSize +=
107  AST.getSourceManager().getMemoryBufferSizes().malloc_bytes;
108 
109  const Preprocessor &PP = CI.getPreprocessor();
110  Stats->BuildSize += PP.getTotalMemory();
111  if (PreprocessingRecord *PRec = PP.getPreprocessingRecord())
112  Stats->BuildSize += PRec->getTotalMemory();
113  Stats->BuildSize += PP.getHeaderSearchInfo().getTotalMemory();
114  }
115  }
116 
117  void BeforeExecute(CompilerInstance &CI) override {
118  CanonIncludes.addSystemHeadersMapping(CI.getLangOpts());
119  LangOpts = &CI.getLangOpts();
120  SourceMgr = &CI.getSourceManager();
121  Includes.collect(CI);
122  if (BeforeExecuteCallback)
123  BeforeExecuteCallback(CI);
124  }
125 
126  std::unique_ptr<PPCallbacks> createPPCallbacks() override {
127  assert(SourceMgr && LangOpts &&
128  "SourceMgr and LangOpts must be set at this point");
129 
130  return std::make_unique<PPChainedCallbacks>(
131  std::make_unique<CollectMainFileMacros>(*SourceMgr, Macros),
133  }
134 
135  CommentHandler *getCommentHandler() override {
136  IWYUHandler = collectIWYUHeaderMaps(&CanonIncludes);
137  return IWYUHandler.get();
138  }
139 
140  static bool isLikelyForwardingFunction(FunctionTemplateDecl *FT) {
141  const auto *FD = FT->getTemplatedDecl();
142  const auto NumParams = FD->getNumParams();
143  // Check whether its last parameter is a parameter pack...
144  if (NumParams > 0) {
145  const auto *LastParam = FD->getParamDecl(NumParams - 1);
146  if (const auto *PET = dyn_cast<PackExpansionType>(LastParam->getType())) {
147  // ... of the type T&&... or T...
148  const auto BaseType = PET->getPattern().getNonReferenceType();
149  if (const auto *TTPT =
150  dyn_cast<TemplateTypeParmType>(BaseType.getTypePtr())) {
151  // ... whose template parameter comes from the function directly
152  if (FT->getTemplateParameters()->getDepth() == TTPT->getDepth()) {
153  return true;
154  }
155  }
156  }
157  }
158  return false;
159  }
160 
161  bool shouldSkipFunctionBody(Decl *D) override {
162  // Usually we don't need to look inside the bodies of header functions
163  // to understand the program. However when forwarding function like
164  // emplace() forward their arguments to some other function, the
165  // interesting overload resolution happens inside the forwarding
166  // function's body. To provide more meaningful diagnostics,
167  // code completion, and parameter hints we should parse (and later
168  // instantiate) the bodies.
169  if (auto *FT = llvm::dyn_cast<clang::FunctionTemplateDecl>(D)) {
170  if (ParseForwardingFunctions) {
171  // Don't skip parsing the body if it looks like a forwarding function
172  if (isLikelyForwardingFunction(FT))
173  return false;
174  } else {
175  // By default, only take care of make_unique
176  // std::make_unique is trivial, and we diagnose bad constructor calls.
177  if (const auto *II = FT->getDeclName().getAsIdentifierInfo()) {
178  if (II->isStr("make_unique") && FT->isInStdNamespace())
179  return false;
180  }
181  }
182  }
183  return true;
184  }
185 
186 private:
187  PathRef File;
188  PreambleParsedCallback ParsedCallback;
189  IncludeStructure Includes;
190  CanonicalIncludes CanonIncludes;
191  MainFileMacros Macros;
192  std::vector<PragmaMark> Marks;
193  bool IsMainFileIncludeGuarded = false;
194  std::unique_ptr<CommentHandler> IWYUHandler = nullptr;
195  const clang::LangOptions *LangOpts = nullptr;
196  const SourceManager *SourceMgr = nullptr;
197  PreambleBuildStats *Stats;
198  bool ParseForwardingFunctions;
199  std::function<void(CompilerInstance &)> BeforeExecuteCallback;
200 };
201 
202 // Represents directives other than includes, where basic textual information is
203 // enough.
204 struct TextualPPDirective {
205  unsigned DirectiveLine;
206  // Full text that's representing the directive, including the `#`.
207  std::string Text;
208  unsigned Offset;
209 
210  bool operator==(const TextualPPDirective &RHS) const {
211  return std::tie(DirectiveLine, Offset, Text) ==
212  std::tie(RHS.DirectiveLine, RHS.Offset, RHS.Text);
213  }
214 };
215 
216 // Formats a PP directive consisting of Prefix (e.g. "#define ") and Body ("X
217 // 10"). The formatting is copied so that the tokens in Body have PresumedLocs
218 // with correct columns and lines.
219 std::string spellDirective(llvm::StringRef Prefix,
220  CharSourceRange DirectiveRange,
221  const LangOptions &LangOpts, const SourceManager &SM,
222  unsigned &DirectiveLine, unsigned &Offset) {
223  std::string SpelledDirective;
224  llvm::raw_string_ostream OS(SpelledDirective);
225  OS << Prefix;
226 
227  // Make sure DirectiveRange is a char range and doesn't contain macro ids.
228  DirectiveRange = SM.getExpansionRange(DirectiveRange);
229  if (DirectiveRange.isTokenRange()) {
230  DirectiveRange.setEnd(
231  Lexer::getLocForEndOfToken(DirectiveRange.getEnd(), 0, SM, LangOpts));
232  }
233 
234  auto DecompLoc = SM.getDecomposedLoc(DirectiveRange.getBegin());
235  DirectiveLine = SM.getLineNumber(DecompLoc.first, DecompLoc.second);
236  Offset = DecompLoc.second;
237  auto TargetColumn = SM.getColumnNumber(DecompLoc.first, DecompLoc.second) - 1;
238 
239  // Pad with spaces before DirectiveRange to make sure it will be on right
240  // column when patched.
241  if (Prefix.size() <= TargetColumn) {
242  // There is enough space for Prefix and space before directive, use it.
243  // We try to squeeze the Prefix into the same line whenever we can, as
244  // putting onto a separate line won't work at the beginning of the file.
245  OS << std::string(TargetColumn - Prefix.size(), ' ');
246  } else {
247  // Prefix was longer than the space we had. We produce e.g.:
248  // #line N-1
249  // #define \
250  // X 10
251  OS << "\\\n" << std::string(TargetColumn, ' ');
252  // Decrement because we put an additional line break before
253  // DirectiveRange.begin().
254  --DirectiveLine;
255  }
256  OS << toSourceCode(SM, DirectiveRange.getAsRange());
257  return OS.str();
258 }
259 
260 // Collects #define directives inside the main file.
261 struct DirectiveCollector : public PPCallbacks {
262  DirectiveCollector(const Preprocessor &PP,
263  std::vector<TextualPPDirective> &TextualDirectives)
264  : LangOpts(PP.getLangOpts()), SM(PP.getSourceManager()),
265  TextualDirectives(TextualDirectives) {}
266 
267  void FileChanged(SourceLocation Loc, FileChangeReason Reason,
268  SrcMgr::CharacteristicKind FileType,
269  FileID PrevFID) override {
270  InMainFile = SM.isWrittenInMainFile(Loc);
271  }
272 
273  void MacroDefined(const Token &MacroNameTok,
274  const MacroDirective *MD) override {
275  if (!InMainFile)
276  return;
277  TextualDirectives.emplace_back();
278  TextualPPDirective &TD = TextualDirectives.back();
279 
280  const auto *MI = MD->getMacroInfo();
281  TD.Text =
282  spellDirective("#define ",
283  CharSourceRange::getTokenRange(
284  MI->getDefinitionLoc(), MI->getDefinitionEndLoc()),
285  LangOpts, SM, TD.DirectiveLine, TD.Offset);
286  }
287 
288 private:
289  bool InMainFile = true;
290  const LangOptions &LangOpts;
291  const SourceManager &SM;
292  std::vector<TextualPPDirective> &TextualDirectives;
293 };
294 
295 struct ScannedPreamble {
296  std::vector<Inclusion> Includes;
297  std::vector<TextualPPDirective> TextualDirectives;
298  PreambleBounds Bounds = {0, false};
299 };
300 
301 /// Scans the preprocessor directives in the preamble section of the file by
302 /// running preprocessor over \p Contents. Returned includes do not contain
303 /// resolved paths. \p Cmd is used to build the compiler invocation, which might
304 /// stat/read files.
305 llvm::Expected<ScannedPreamble>
306 scanPreamble(llvm::StringRef Contents, const tooling::CompileCommand &Cmd) {
307  class EmptyFS : public ThreadsafeFS {
308  private:
309  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> viewImpl() const override {
311  }
312  };
313  EmptyFS FS;
314  // Build and run Preprocessor over the preamble.
315  ParseInputs PI;
316  PI.Contents = Contents.str();
317  PI.TFS = &FS;
318  PI.CompileCommand = Cmd;
319  IgnoringDiagConsumer IgnoreDiags;
321  if (!CI)
322  return error("failed to create compiler invocation");
323  CI->getDiagnosticOpts().IgnoreWarnings = true;
324  auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(Contents);
325  // This means we're scanning (though not preprocessing) the preamble section
326  // twice. However, it's important to precisely follow the preamble bounds used
327  // elsewhere.
328  auto Bounds = ComputePreambleBounds(*CI->getLangOpts(), *ContentsBuffer, 0);
329  auto PreambleContents =
330  llvm::MemoryBuffer::getMemBufferCopy(Contents.substr(0, Bounds.Size));
332  std::move(CI), nullptr, std::move(PreambleContents),
333  // Provide an empty FS to prevent preprocessor from performing IO. This
334  // also implies missing resolved paths for includes.
335  FS.view(llvm::None), IgnoreDiags);
336  if (Clang->getFrontendOpts().Inputs.empty())
337  return error("compiler instance had no inputs");
338  // We are only interested in main file includes.
339  Clang->getPreprocessorOpts().SingleFileParseMode = true;
340  Clang->getPreprocessorOpts().UsePredefines = false;
341  PreprocessOnlyAction Action;
342  if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0]))
343  return error("failed BeginSourceFile");
344  Preprocessor &PP = Clang->getPreprocessor();
345  IncludeStructure Includes;
346  Includes.collect(*Clang);
347  ScannedPreamble SP;
348  SP.Bounds = Bounds;
349  PP.addPPCallbacks(
350  std::make_unique<DirectiveCollector>(PP, SP.TextualDirectives));
351  if (llvm::Error Err = Action.Execute())
352  return std::move(Err);
353  Action.EndSourceFile();
354  SP.Includes = std::move(Includes.MainFileIncludes);
355  return SP;
356 }
357 
358 const char *spellingForIncDirective(tok::PPKeywordKind IncludeDirective) {
359  switch (IncludeDirective) {
360  case tok::pp_include:
361  return "include";
362  case tok::pp_import:
363  return "import";
364  case tok::pp_include_next:
365  return "include_next";
366  default:
367  break;
368  }
369  llvm_unreachable("not an include directive");
370 }
371 
372 // Checks whether \p FileName is a valid spelling of main file.
373 bool isMainFile(llvm::StringRef FileName, const SourceManager &SM) {
374  auto FE = SM.getFileManager().getFile(FileName);
375  return FE && *FE == SM.getFileEntryForID(SM.getMainFileID());
376 }
377 
378 // Accumulating wall time timer. Similar to llvm::Timer, but much cheaper,
379 // it only tracks wall time.
380 // Since this is a generic timer, We may want to move this to support/ if we
381 // find a use case outside of FS time tracking.
382 class WallTimer {
383 public:
384  WallTimer() : TotalTime(std::chrono::steady_clock::duration::zero()) {}
385  // [Re-]Start the timer.
386  void startTimer() { StartTime = std::chrono::steady_clock::now(); }
387  // Stop the timer and update total time.
388  void stopTimer() {
389  TotalTime += std::chrono::steady_clock::now() - StartTime;
390  }
391  // Return total time, in seconds.
392  double getTime() { return std::chrono::duration<double>(TotalTime).count(); }
393 
394 private:
395  std::chrono::steady_clock::duration TotalTime;
396  std::chrono::steady_clock::time_point StartTime;
397 };
398 
399 class WallTimerRegion {
400 public:
401  WallTimerRegion(WallTimer &T) : T(T) { T.startTimer(); }
402  ~WallTimerRegion() { T.stopTimer(); }
403 
404 private:
405  WallTimer &T;
406 };
407 
408 // Used by TimerFS, tracks time spent in status() and getBuffer() calls while
409 // proxying to underlying File implementation.
410 class TimerFile : public llvm::vfs::File {
411 public:
412  TimerFile(WallTimer &Timer, std::unique_ptr<File> InnerFile)
413  : Timer(Timer), InnerFile(std::move(InnerFile)) {}
414 
415  llvm::ErrorOr<llvm::vfs::Status> status() override {
416  WallTimerRegion T(Timer);
417  return InnerFile->status();
418  }
419  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
420  getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
421  bool IsVolatile) override {
422  WallTimerRegion T(Timer);
423  return InnerFile->getBuffer(Name, FileSize, RequiresNullTerminator,
424  IsVolatile);
425  }
426  std::error_code close() override {
427  WallTimerRegion T(Timer);
428  return InnerFile->close();
429  }
430 
431 private:
432  WallTimer &Timer;
433  std::unique_ptr<llvm::vfs::File> InnerFile;
434 };
435 
436 // A wrapper for FileSystems that tracks the amount of time spent in status()
437 // and openFileForRead() calls.
438 class TimerFS : public llvm::vfs::ProxyFileSystem {
439 public:
440  TimerFS(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
441  : ProxyFileSystem(std::move(FS)) {}
442 
443  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
444  openFileForRead(const llvm::Twine &Path) override {
445  WallTimerRegion T(Timer);
446  auto FileOr = getUnderlyingFS().openFileForRead(Path);
447  if (!FileOr)
448  return FileOr;
449  return std::make_unique<TimerFile>(Timer, std::move(FileOr.get()));
450  }
451 
452  llvm::ErrorOr<llvm::vfs::Status> status(const llvm::Twine &Path) override {
453  WallTimerRegion T(Timer);
454  return getUnderlyingFS().status(Path);
455  }
456 
457  double getTime() { return Timer.getTime(); }
458 
459 private:
460  WallTimer Timer;
461 };
462 
463 } // namespace
464 
465 std::shared_ptr<const PreambleData>
466 buildPreamble(PathRef FileName, CompilerInvocation CI,
467  const ParseInputs &Inputs, bool StoreInMemory,
468  PreambleParsedCallback PreambleCallback,
469  PreambleBuildStats *Stats) {
470  // Note that we don't need to copy the input contents, preamble can live
471  // without those.
472  auto ContentsBuffer =
473  llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName);
474  auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), *ContentsBuffer, 0);
475 
476  trace::Span Tracer("BuildPreamble");
477  SPAN_ATTACH(Tracer, "File", FileName);
478  std::vector<std::unique_ptr<FeatureModule::ASTListener>> ASTListeners;
479  if (Inputs.FeatureModules) {
480  for (auto &M : *Inputs.FeatureModules) {
481  if (auto Listener = M.astListeners())
482  ASTListeners.emplace_back(std::move(Listener));
483  }
484  }
485  StoreDiags PreambleDiagnostics;
486  PreambleDiagnostics.setDiagCallback(
487  [&ASTListeners](const clang::Diagnostic &D, clangd::Diag &Diag) {
488  llvm::for_each(ASTListeners,
489  [&](const auto &L) { L->sawDiagnostic(D, Diag); });
490  });
491  llvm::IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine =
492  CompilerInstance::createDiagnostics(&CI.getDiagnosticOpts(),
493  &PreambleDiagnostics, false);
494  const Config &Cfg = Config::current();
495  PreambleDiagnostics.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
496  const clang::Diagnostic &Info) {
497  if (Cfg.Diagnostics.SuppressAll ||
499  *CI.getLangOpts()))
500  return DiagnosticsEngine::Ignored;
501  switch (Info.getID()) {
502  case diag::warn_no_newline_eof:
503  case diag::warn_cxx98_compat_no_newline_eof:
504  case diag::ext_no_newline_eof:
505  // If the preamble doesn't span the whole file, drop the no newline at
506  // eof warnings.
507  return Bounds.Size != ContentsBuffer->getBufferSize()
508  ? DiagnosticsEngine::Level::Ignored
509  : DiagLevel;
510  }
511  return DiagLevel;
512  });
513 
514  // Skip function bodies when building the preamble to speed up building
515  // the preamble and make it smaller.
516  assert(!CI.getFrontendOpts().SkipFunctionBodies);
517  CI.getFrontendOpts().SkipFunctionBodies = true;
518  // We don't want to write comment locations into PCH. They are racy and slow
519  // to read back. We rely on dynamic index for the comments instead.
520  CI.getPreprocessorOpts().WriteCommentListToPCH = false;
521 
522  CppFilePreambleCallbacks CapturedInfo(
523  FileName, PreambleCallback, Stats,
525  [&ASTListeners](CompilerInstance &CI) {
526  for (const auto &L : ASTListeners)
527  L->beforeExecute(CI);
528  });
529  auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
530  llvm::SmallString<32> AbsFileName(FileName);
531  VFS->makeAbsolute(AbsFileName);
532  auto StatCache = std::make_unique<PreambleFileStatusCache>(AbsFileName);
533  auto StatCacheFS = StatCache->getProducingFS(VFS);
534  llvm::IntrusiveRefCntPtr<TimerFS> TimedFS(new TimerFS(StatCacheFS));
535 
536  WallTimer PreambleTimer;
537  PreambleTimer.startTimer();
538  auto BuiltPreamble = PrecompiledPreamble::Build(
539  CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine,
540  Stats ? TimedFS : StatCacheFS, std::make_shared<PCHContainerOperations>(),
541  StoreInMemory, CapturedInfo);
542  PreambleTimer.stopTimer();
543 
544  // When building the AST for the main file, we do want the function
545  // bodies.
546  CI.getFrontendOpts().SkipFunctionBodies = false;
547 
548  if (Stats != nullptr) {
549  Stats->TotalBuildTime = PreambleTimer.getTime();
550  Stats->FileSystemTime = TimedFS->getTime();
551  Stats->SerializedSize = BuiltPreamble ? BuiltPreamble->getSize() : 0;
552  }
553 
554  if (BuiltPreamble) {
555  log("Built preamble of size {0} for file {1} version {2} in {3} seconds",
556  BuiltPreamble->getSize(), FileName, Inputs.Version,
557  PreambleTimer.getTime());
558  std::vector<Diag> Diags = PreambleDiagnostics.take();
559  auto Result = std::make_shared<PreambleData>(std::move(*BuiltPreamble));
560  Result->Version = Inputs.Version;
561  Result->CompileCommand = Inputs.CompileCommand;
562  Result->Diags = std::move(Diags);
563  Result->Includes = CapturedInfo.takeIncludes();
564  Result->Macros = CapturedInfo.takeMacros();
565  Result->Marks = CapturedInfo.takeMarks();
566  Result->CanonIncludes = CapturedInfo.takeCanonicalIncludes();
567  Result->StatCache = std::move(StatCache);
568  Result->MainIsIncludeGuarded = CapturedInfo.isMainFileIncludeGuarded();
569  return Result;
570  }
571 
572  elog("Could not build a preamble for file {0} version {1}: {2}", FileName,
573  Inputs.Version, BuiltPreamble.getError().message());
574  return nullptr;
575 }
576 
577 bool isPreambleCompatible(const PreambleData &Preamble,
579  const CompilerInvocation &CI) {
580  auto ContentsBuffer =
581  llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName);
582  auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), *ContentsBuffer, 0);
583  auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
584  return compileCommandsAreEqual(Inputs.CompileCommand,
586  Preamble.Preamble.CanReuse(CI, *ContentsBuffer, Bounds, *VFS);
587 }
588 
589 void escapeBackslashAndQuotes(llvm::StringRef Text, llvm::raw_ostream &OS) {
590  for (char C : Text) {
591  switch (C) {
592  case '\\':
593  case '"':
594  OS << '\\';
595  break;
596  default:
597  break;
598  }
599  OS << C;
600  }
601 }
602 
603 PreamblePatch PreamblePatch::create(llvm::StringRef FileName,
604  const ParseInputs &Modified,
605  const PreambleData &Baseline,
606  PatchType PatchType) {
607  trace::Span Tracer("CreatePreamblePatch");
608  SPAN_ATTACH(Tracer, "File", FileName);
609  assert(llvm::sys::path::is_absolute(FileName) && "relative FileName!");
610  // First scan preprocessor directives in Baseline and Modified. These will be
611  // used to figure out newly added directives in Modified. Scanning can fail,
612  // the code just bails out and creates an empty patch in such cases, as:
613  // - If scanning for Baseline fails, no knowledge of existing includes hence
614  // patch will contain all the includes in Modified. Leading to rebuild of
615  // whole preamble, which is terribly slow.
616  // - If scanning for Modified fails, cannot figure out newly added ones so
617  // there's nothing to do but generate an empty patch.
618  auto BaselineScan = scanPreamble(
619  // Contents needs to be null-terminated.
620  Baseline.Preamble.getContents().str(), Modified.CompileCommand);
621  if (!BaselineScan) {
622  elog("Failed to scan baseline of {0}: {1}", FileName,
623  BaselineScan.takeError());
624  return PreamblePatch::unmodified(Baseline);
625  }
626  auto ModifiedScan = scanPreamble(Modified.Contents, Modified.CompileCommand);
627  if (!ModifiedScan) {
628  elog("Failed to scan modified contents of {0}: {1}", FileName,
629  ModifiedScan.takeError());
630  return PreamblePatch::unmodified(Baseline);
631  }
632 
633  bool IncludesChanged = BaselineScan->Includes != ModifiedScan->Includes;
634  bool DirectivesChanged =
635  BaselineScan->TextualDirectives != ModifiedScan->TextualDirectives;
636  if ((PatchType == PatchType::MacroDirectives || !IncludesChanged) &&
637  !DirectivesChanged)
638  return PreamblePatch::unmodified(Baseline);
639 
640  PreamblePatch PP;
641  // This shouldn't coincide with any real file name.
642  llvm::SmallString<128> PatchName;
643  llvm::sys::path::append(PatchName, llvm::sys::path::parent_path(FileName),
644  PreamblePatchHeaderName);
645  PP.PatchFileName = PatchName.str().str();
646  PP.ModifiedBounds = ModifiedScan->Bounds;
647 
648  llvm::raw_string_ostream Patch(PP.PatchContents);
649  // Set default filename for subsequent #line directives
650  Patch << "#line 0 \"";
651  // FileName part of a line directive is subject to backslash escaping, which
652  // might lead to problems on windows especially.
654  Patch << "\"\n";
655 
656  if (IncludesChanged && PatchType == PatchType::All) {
657  // We are only interested in newly added includes, record the ones in
658  // Baseline for exclusion.
659  llvm::DenseMap<std::pair<tok::PPKeywordKind, llvm::StringRef>,
660  /*Resolved=*/llvm::StringRef>
661  ExistingIncludes;
662  for (const auto &Inc : Baseline.Includes.MainFileIncludes)
663  ExistingIncludes[{Inc.Directive, Inc.Written}] = Inc.Resolved;
664  // There might be includes coming from disabled regions, record these for
665  // exclusion too. note that we don't have resolved paths for those.
666  for (const auto &Inc : BaselineScan->Includes)
667  ExistingIncludes.try_emplace({Inc.Directive, Inc.Written});
668  // Calculate extra includes that needs to be inserted.
669  for (auto &Inc : ModifiedScan->Includes) {
670  auto It = ExistingIncludes.find({Inc.Directive, Inc.Written});
671  // Include already present in the baseline preamble. Set resolved path and
672  // put into preamble includes.
673  if (It != ExistingIncludes.end()) {
674  Inc.Resolved = It->second.str();
675  PP.PreambleIncludes.push_back(Inc);
676  continue;
677  }
678  // Include is new in the modified preamble. Inject it into the patch and
679  // use #line to set the presumed location to where it is spelled.
680  auto LineCol = offsetToClangLineColumn(Modified.Contents, Inc.HashOffset);
681  Patch << llvm::formatv("#line {0}\n", LineCol.first);
682  Patch << llvm::formatv(
683  "#{0} {1}\n", spellingForIncDirective(Inc.Directive), Inc.Written);
684  }
685  }
686 
687  if (DirectivesChanged) {
688  // We need to patch all the directives, since they are order dependent. e.g:
689  // #define BAR(X) NEW(X) // Newly introduced in Modified
690  // #define BAR(X) OLD(X) // Exists in the Baseline
691  //
692  // If we've patched only the first directive, the macro definition would've
693  // been wrong for the rest of the file, since patch is applied after the
694  // baseline preamble.
695  //
696  // Note that we deliberately ignore conditional directives and undefs to
697  // reduce complexity. The former might cause problems because scanning is
698  // imprecise and might pick directives from disabled regions.
699  for (const auto &TD : ModifiedScan->TextualDirectives) {
700  Patch << "#line " << TD.DirectiveLine << '\n';
701  Patch << TD.Text << '\n';
702  }
703  }
704  dlog("Created preamble patch: {0}", Patch.str());
705  Patch.flush();
706  return PP;
707 }
708 
710  const ParseInputs &Modified,
711  const PreambleData &Baseline) {
712  return create(FileName, Modified, Baseline, PatchType::All);
713 }
714 
716  const ParseInputs &Modified,
717  const PreambleData &Baseline) {
718  return create(FileName, Modified, Baseline, PatchType::MacroDirectives);
719 }
720 
721 void PreamblePatch::apply(CompilerInvocation &CI) const {
722  // No need to map an empty file.
723  if (PatchContents.empty())
724  return;
725  auto &PPOpts = CI.getPreprocessorOpts();
726  auto PatchBuffer =
727  // we copy here to ensure contents are still valid if CI outlives the
728  // PreamblePatch.
729  llvm::MemoryBuffer::getMemBufferCopy(PatchContents, PatchFileName);
730  // CI will take care of the lifetime of the buffer.
731  PPOpts.addRemappedFile(PatchFileName, PatchBuffer.release());
732  // The patch will be parsed after loading the preamble ast and before parsing
733  // the main file.
734  PPOpts.Includes.push_back(PatchFileName);
735 }
736 
737 std::vector<Inclusion> PreamblePatch::preambleIncludes() const {
738  return PreambleIncludes;
739 }
740 
743  PP.PreambleIncludes = Preamble.Includes.MainFileIncludes;
744  PP.ModifiedBounds = Preamble.Preamble.getBounds();
745  return PP;
746 }
747 
748 SourceLocation translatePreamblePatchLocation(SourceLocation Loc,
749  const SourceManager &SM) {
750  auto DefFile = SM.getFileID(Loc);
751  if (auto FE = SM.getFileEntryRefForID(DefFile)) {
752  auto IncludeLoc = SM.getIncludeLoc(DefFile);
753  // Preamble patch is included inside the builtin file.
754  if (IncludeLoc.isValid() && SM.isWrittenInBuiltinFile(IncludeLoc) &&
755  FE->getName().endswith(PreamblePatchHeaderName)) {
756  auto Presumed = SM.getPresumedLoc(Loc);
757  // Check that line directive is pointing at main file.
758  if (Presumed.isValid() && Presumed.getFileID().isInvalid() &&
759  isMainFile(Presumed.getFilename(), SM)) {
760  Loc = SM.translateLineCol(SM.getMainFileID(), Presumed.getLine(),
761  Presumed.getColumn());
762  }
763  }
764  }
765  return Loc;
766 }
767 
768 } // namespace clangd
769 } // namespace clang
dlog
#define dlog(...)
Definition: Logger.h:101
clang::clangd::ParseInputs::Version
std::string Version
Definition: Compiler.h:51
clang::clangd::buildPreamble
std::shared_ptr< const PreambleData > buildPreamble(PathRef FileName, CompilerInvocation CI, const ParseInputs &Inputs, bool StoreInMemory, PreambleParsedCallback PreambleCallback, PreambleBuildStats *Stats)
Build a preamble for the new inputs unless an old one can be reused.
Definition: Preamble.cpp:466
clang::clangd::IncludeStructure::collect
void collect(const CompilerInstance &CI)
Definition: Headers.cpp:235
Loc
SourceLocation Loc
Definition: KernelNameRestrictionCheck.cpp:45
clang::clangd::collectIWYUHeaderMaps
std::unique_ptr< CommentHandler > collectIWYUHeaderMaps(CanonicalIncludes *Includes)
Returns a CommentHandler that parses pragma comment on include files to determine when we should incl...
Definition: CanonicalIncludes.cpp:709
clang::clangd::isPreambleCompatible
bool isPreambleCompatible(const PreambleData &Preamble, const ParseInputs &Inputs, PathRef FileName, const CompilerInvocation &CI)
Returns true if Preamble is reusable for Inputs.
Definition: Preamble.cpp:577
Headers.h
clang::clangd::ParseInputs::FeatureModules
FeatureModuleSet * FeatureModules
Definition: Compiler.h:60
clang::clangd::escapeBackslashAndQuotes
void escapeBackslashAndQuotes(llvm::StringRef Text, llvm::raw_ostream &OS)
Definition: Preamble.cpp:589
clang::clangd::prepareCompilerInstance
std::unique_ptr< CompilerInstance > prepareCompilerInstance(std::unique_ptr< clang::CompilerInvocation > CI, const PrecompiledPreamble *Preamble, std::unique_ptr< llvm::MemoryBuffer > Buffer, llvm::IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, DiagnosticConsumer &DiagsClient)
Definition: Compiler.cpp:120
clang::clangd::ParseInputs::Contents
std::string Contents
Definition: Compiler.h:49
clang::clangd::Path
std::string Path
A typedef to represent a file path.
Definition: Path.h:26
Macros
llvm::DenseSet< FileID > Macros
Definition: IncludeCleaner.cpp:195
clang::doc::MD
static GeneratorRegistry::Add< MDGenerator > MD(MDGenerator::Format, "Generator for MD output.")
Tracer
std::unique_ptr< trace::EventTracer > Tracer
Definition: TraceTests.cpp:161
clang::clangd::error
llvm::Error error(std::error_code EC, const char *Fmt, Ts &&... Vals)
Definition: Logger.h:79
clang::clangd::PreambleData::Preamble
PrecompiledPreamble Preamble
Definition: Preamble.h:55
CI
std::unique_ptr< CompilerInvocation > CI
Definition: TUScheduler.cpp:549
clang::clangd::PreamblePatch::unmodified
static PreamblePatch unmodified(const PreambleData &Preamble)
Preamble is used verbatim.
Definition: Preamble.cpp:741
Preamble.h
clang::clangd::PreambleBuildStats::TotalBuildTime
double TotalBuildTime
Total wall time it took to build preamble, in seconds.
Definition: Preamble.h:83
Text
std::string Text
Definition: HTMLGenerator.cpp:80
Trace.h
clang::clangd::Config
Settings that express user/project preferences and control clangd behavior.
Definition: Config.h:44
clang::clangd::PreamblePatch
Stores information required to parse a TU using a (possibly stale) Baseline preamble.
Definition: Preamble.h:119
clang::clangd::PreambleBuildStats
Timings and statistics from the premble build.
Definition: Preamble.h:81
SourceMgr
llvm::SourceMgr * SourceMgr
Definition: ConfigCompile.cpp:101
clang::clangd::ParseInputs::CompileCommand
tooling::CompileCommand CompileCommand
Definition: Compiler.h:47
clang::clangd::IncludeStructure::MainFileIncludes
std::vector< Inclusion > MainFileIncludes
Definition: Headers.h:168
Preamble
const PreambleData & Preamble
Definition: CodeComplete.cpp:1212
clang::clangd::PreambleBuildStats::SerializedSize
size_t SerializedSize
The serialized size of the preamble.
Definition: Preamble.h:93
M
const google::protobuf::Message & M
Definition: Server.cpp:309
ThreadsafeFS.h
Inputs
ParseInputs Inputs
Definition: TUScheduler.cpp:550
clang::clangd::ParseInputs::Opts
ParseOptions Opts
Definition: Compiler.h:57
clang::clangd::PreamblePatch::apply
void apply(CompilerInvocation &CI) const
Adjusts CI (which compiles the modified inputs) to be used with the baseline preamble.
Definition: Preamble.cpp:721
clang::clangd::ParseInputs
Information required to run clang, e.g. to parse AST or do code completion.
Definition: Compiler.h:46
ns1::ns2::D
@ D
Definition: CategoricalFeature.h:3
Diagnostic
DiagnosticCallback Diagnostic
Definition: ConfigCompile.cpp:100
clang::clangd::ParseInputs::TFS
const ThreadsafeFS * TFS
Definition: Compiler.h:48
Decl
const FunctionDecl * Decl
Definition: AvoidBindCheck.cpp:100
clang::clangd::PreambleData
The parsed preamble and associated data.
Definition: Preamble.h:49
clang::clangd::PreambleBuildStats::FileSystemTime
double FileSystemTime
Time spent in filesystem operations during the build, in seconds.
Definition: Preamble.h:85
clang::clangd::PreamblePatch::createMacroPatch
static PreamblePatch createMacroPatch(llvm::StringRef FileName, const ParseInputs &Modified, const PreambleData &Baseline)
Definition: Preamble.cpp:715
clang::clangd::PreamblePatch::PatchType::MacroDirectives
@ MacroDirectives
clang::clangd::PreamblePatch::PatchType
PatchType
Definition: Preamble.h:121
clang::clangd::Diag
A top-level diagnostic that may have Notes and Fixes.
Definition: Diagnostics.h:94
clang::clangd::Config::Suppress
llvm::StringSet Suppress
Definition: Config.h:95
Logger.h
Bounds
PreambleBounds Bounds
Definition: Preamble.cpp:298
clang::clangd::collectPragmaMarksCallback
std::unique_ptr< PPCallbacks > collectPragmaMarksCallback(const SourceManager &SM, std::vector< PragmaMark > &Out)
Collect all pragma marks from the main file.
Definition: CollectMacros.cpp:56
Patch
const llvm::Optional< PreamblePatch > Patch
Definition: CodeComplete.cpp:1213
SPAN_ATTACH
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:164
clang::clangd::ParseOptions::PreambleParseForwardingFunctions
bool PreambleParseForwardingFunctions
Definition: Compiler.h:42
Name
Token Name
Definition: MacroToEnumCheck.cpp:89
FileName
StringRef FileName
Definition: KernelNameRestrictionCheck.cpp:46
clang::clangd::Config::SuppressAll
bool SuppressAll
Definition: Config.h:94
Offset
unsigned Offset
Definition: Preamble.cpp:208
DirectiveLine
unsigned DirectiveLine
Definition: Preamble.cpp:205
clang::tidy::bugprone::PP
static Preprocessor * PP
Definition: BadSignalToKillThreadCheck.cpp:29
clang::clangd::log
void log(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:67
clang::clangd::PreambleData::CompileCommand
tooling::CompileCommand CompileCommand
Definition: Preamble.h:54
clang::clangd::buildCompilerInvocation
std::unique_ptr< CompilerInvocation > buildCompilerInvocation(const ParseInputs &Inputs, clang::DiagnosticConsumer &D, std::vector< std::string > *CC1Args)
Builds compiler invocation that could be used to build AST or preamble.
Definition: Compiler.cpp:86
SourceCode.h
clang::tidy::getBuffer
static Optional< StringRef > getBuffer(const SourceManager &SrcMgr, FileID File, bool AllowIO)
Definition: NoLintDirectiveHandler.cpp:308
Config.h
Compiler.h
InMemoryFileSystem
llvm::IntrusiveRefCntPtr< llvm::vfs::InMemoryFileSystem > InMemoryFileSystem
Definition: IndexActionTests.cpp:110
clang::clangd::PreamblePatch::createFullPatch
static PreamblePatch createFullPatch(llvm::StringRef FileName, const ParseInputs &Modified, const PreambleData &Baseline)
Builds a patch that contains new PP directives introduced to the preamble section of Modified compare...
Definition: Preamble.cpp:709
Info
FunctionInfo Info
Definition: FunctionSizeCheck.cpp:121
clang::clangd::PathRef
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:29
clang::clangd::ThreadsafeFS::view
llvm::IntrusiveRefCntPtr< llvm::vfs::FileSystem > view(llvm::NoneType CWD) const
Obtain a vfs::FileSystem with an arbitrary initial working directory.
Definition: ThreadsafeFS.h:33
C
const Criteria C
Definition: FunctionCognitiveComplexityCheck.cpp:93
PPCallbacks
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
OS
llvm::raw_string_ostream OS
Definition: TraceTests.cpp:160
clang::clangd::StoreDiags::setDiagCallback
void setDiagCallback(DiagCallback CB)
Invokes a callback every time a diagnostics is completely formed.
Definition: Diagnostics.h:164
clang::clangd::StoreDiags
StoreDiags collects the diagnostics that can later be reported by clangd.
Definition: Diagnostics.h:137
Diags
CapturedDiags Diags
Definition: ConfigCompileTests.cpp:39
LangOpts
const LangOptions * LangOpts
Definition: ExtractFunction.cpp:366
clang::clangd::Config::current
static const Config & current()
Returns the Config of the current Context, or an empty configuration.
Definition: Config.cpp:17
clang::clangd::operator==
bool operator==(const Inclusion &LHS, const Inclusion &RHS)
Definition: Headers.cpp:368
clang::clangd::Config::Diagnostics
struct clang::clangd::Config::@4 Diagnostics
Controls warnings and errors when parsing code.
clang::clangd::StoreDiags::setLevelAdjuster
void setLevelAdjuster(LevelAdjuster Adjuster)
If set, this allows the client of this class to adjust the level of diagnostics, such as promoting wa...
Definition: Diagnostics.h:161
clang::clangd::elog
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:61
clang::clangd::PreambleParsedCallback
std::function< void(ASTContext &, Preprocessor &, const CanonicalIncludes &)> PreambleParsedCallback
Definition: Preamble.h:76
clang::clangd::PreamblePatch::preambleIncludes
std::vector< Inclusion > preambleIncludes() const
Returns #include directives from the Modified preamble that were resolved using the Baseline preamble...
Definition: Preamble.cpp:737
clang::clangd::isBuiltinDiagnosticSuppressed
bool isBuiltinDiagnosticSuppressed(unsigned ID, const llvm::StringSet<> &Suppress, const LangOptions &LangOpts)
Determine whether a (non-clang-tidy) diagnostic is suppressed by config.
Definition: Diagnostics.cpp:885
clang::clangd::toSourceCode
llvm::StringRef toSourceCode(const SourceManager &SM, SourceRange R)
Returns the source code covered by the source range.
Definition: SourceCode.cpp:443
clang::clangd::PreamblePatch::PatchType::All
@ All
clang::clangd::StoreDiags::take
std::vector< Diag > take(const clang::tidy::ClangTidyContext *Tidy=nullptr)
Definition: Diagnostics.cpp:564
IgnoreDiags
IgnoringDiagConsumer IgnoreDiags
Definition: HeadersTests.cpp:137
Clang
std::unique_ptr< CompilerInstance > Clang
Definition: HeadersTests.cpp:138
clang::clangd::translatePreamblePatchLocation
SourceLocation translatePreamblePatchLocation(SourceLocation Loc, const SourceManager &SM)
Translates locations inside preamble patch to their main-file equivalent using presumed locations.
Definition: Preamble.cpp:748
clang::clangd::offsetToClangLineColumn
std::pair< size_t, size_t > offsetToClangLineColumn(llvm::StringRef Code, size_t Offset)
Definition: SourceCode.cpp:478
Action
FieldAction Action
Definition: MemberwiseConstructor.cpp:261
clang::clangd::trace::Span
Records an event whose duration is the lifetime of the Span object.
Definition: Trace.h:143