clang-tools 17.0.0git
Preamble.cpp
Go to the documentation of this file.
1//===--- Preamble.cpp - Reusing expensive parts of the AST ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Preamble.h"
10#include "CollectMacros.h"
11#include "Compiler.h"
12#include "Config.h"
13#include "Diagnostics.h"
14#include "Headers.h"
15#include "Protocol.h"
16#include "SourceCode.h"
17#include "clang-include-cleaner/Record.h"
19#include "support/Logger.h"
20#include "support/Path.h"
22#include "support/Trace.h"
23#include "clang/AST/DeclTemplate.h"
24#include "clang/Basic/Diagnostic.h"
25#include "clang/Basic/DiagnosticLex.h"
26#include "clang/Basic/LangOptions.h"
27#include "clang/Basic/SourceLocation.h"
28#include "clang/Basic/SourceManager.h"
29#include "clang/Basic/TokenKinds.h"
30#include "clang/Frontend/CompilerInvocation.h"
31#include "clang/Frontend/FrontendActions.h"
32#include "clang/Frontend/PrecompiledPreamble.h"
33#include "clang/Lex/HeaderSearch.h"
34#include "clang/Lex/Lexer.h"
35#include "clang/Lex/PPCallbacks.h"
36#include "clang/Lex/Preprocessor.h"
37#include "clang/Lex/PreprocessorOptions.h"
38#include "clang/Tooling/CompilationDatabase.h"
39#include "llvm/ADT/ArrayRef.h"
40#include "llvm/ADT/DenseMap.h"
41#include "llvm/ADT/IntrusiveRefCntPtr.h"
42#include "llvm/ADT/STLExtras.h"
43#include "llvm/ADT/SmallString.h"
44#include "llvm/ADT/SmallVector.h"
45#include "llvm/ADT/StringExtras.h"
46#include "llvm/ADT/StringMap.h"
47#include "llvm/ADT/StringRef.h"
48#include "llvm/Support/Casting.h"
49#include "llvm/Support/Error.h"
50#include "llvm/Support/ErrorHandling.h"
51#include "llvm/Support/FormatVariadic.h"
52#include "llvm/Support/MemoryBuffer.h"
53#include "llvm/Support/Path.h"
54#include "llvm/Support/VirtualFileSystem.h"
55#include "llvm/Support/raw_ostream.h"
56#include <cstddef>
57#include <functional>
58#include <memory>
59#include <optional>
60#include <string>
61#include <system_error>
62#include <tuple>
63#include <utility>
64#include <vector>
65
66namespace clang {
67namespace clangd {
68namespace {
69
70bool compileCommandsAreEqual(const tooling::CompileCommand &LHS,
71 const tooling::CompileCommand &RHS) {
72 // We don't check for Output, it should not matter to clangd.
73 return LHS.Directory == RHS.Directory && LHS.Filename == RHS.Filename &&
74 llvm::ArrayRef(LHS.CommandLine).equals(RHS.CommandLine);
75}
76
77class CppFilePreambleCallbacks : public PreambleCallbacks {
78public:
79 CppFilePreambleCallbacks(
80 PathRef File, PreambleParsedCallback ParsedCallback,
81 PreambleBuildStats *Stats, bool ParseForwardingFunctions,
82 std::function<void(CompilerInstance &)> BeforeExecuteCallback)
83 : File(File), ParsedCallback(ParsedCallback), Stats(Stats),
84 ParseForwardingFunctions(ParseForwardingFunctions),
85 BeforeExecuteCallback(std::move(BeforeExecuteCallback)) {}
86
87 IncludeStructure takeIncludes() { return std::move(Includes); }
88
89 MainFileMacros takeMacros() { return std::move(Macros); }
90
91 std::vector<PragmaMark> takeMarks() { return std::move(Marks); }
92
93 include_cleaner::PragmaIncludes takePragmaIncludes() {
94 return std::move(Pragmas);
95 }
96 CanonicalIncludes takeCanonicalIncludes() { return std::move(CanonIncludes); }
97
98 bool isMainFileIncludeGuarded() const { return IsMainFileIncludeGuarded; }
99
100 void AfterExecute(CompilerInstance &CI) override {
101 if (ParsedCallback) {
102 trace::Span Tracer("Running PreambleCallback");
103 ParsedCallback(CI.getASTContext(), CI.getPreprocessor(), CanonIncludes);
104 }
105
106 const SourceManager &SM = CI.getSourceManager();
107 const FileEntry *MainFE = SM.getFileEntryForID(SM.getMainFileID());
108 IsMainFileIncludeGuarded =
109 CI.getPreprocessor().getHeaderSearchInfo().isFileMultipleIncludeGuarded(
110 MainFE);
111
112 if (Stats) {
113 const ASTContext &AST = CI.getASTContext();
114 Stats->BuildSize = AST.getASTAllocatedMemory();
115 Stats->BuildSize += AST.getSideTableAllocatedMemory();
116 Stats->BuildSize += AST.Idents.getAllocator().getTotalMemory();
117 Stats->BuildSize += AST.Selectors.getTotalMemory();
118
119 Stats->BuildSize += AST.getSourceManager().getContentCacheSize();
120 Stats->BuildSize += AST.getSourceManager().getDataStructureSizes();
121 Stats->BuildSize +=
122 AST.getSourceManager().getMemoryBufferSizes().malloc_bytes;
123
124 const Preprocessor &PP = CI.getPreprocessor();
125 Stats->BuildSize += PP.getTotalMemory();
126 if (PreprocessingRecord *PRec = PP.getPreprocessingRecord())
127 Stats->BuildSize += PRec->getTotalMemory();
128 Stats->BuildSize += PP.getHeaderSearchInfo().getTotalMemory();
129 }
130 }
131
132 void BeforeExecute(CompilerInstance &CI) override {
133 CanonIncludes.addSystemHeadersMapping(CI.getLangOpts());
134 LangOpts = &CI.getLangOpts();
135 SourceMgr = &CI.getSourceManager();
136 PP = &CI.getPreprocessor();
137 Includes.collect(CI);
138 Pragmas.record(CI);
139 if (BeforeExecuteCallback)
140 BeforeExecuteCallback(CI);
141 }
142
143 std::unique_ptr<PPCallbacks> createPPCallbacks() override {
144 assert(SourceMgr && LangOpts && PP &&
145 "SourceMgr, LangOpts and PP must be set at this point");
146
147 return std::make_unique<PPChainedCallbacks>(
148 std::make_unique<CollectMainFileMacros>(*PP, Macros),
149 collectPragmaMarksCallback(*SourceMgr, Marks));
150 }
151
152 CommentHandler *getCommentHandler() override {
153 IWYUHandler = collectIWYUHeaderMaps(&CanonIncludes);
154 return IWYUHandler.get();
155 }
156
157 static bool isLikelyForwardingFunction(FunctionTemplateDecl *FT) {
158 const auto *FD = FT->getTemplatedDecl();
159 const auto NumParams = FD->getNumParams();
160 // Check whether its last parameter is a parameter pack...
161 if (NumParams > 0) {
162 const auto *LastParam = FD->getParamDecl(NumParams - 1);
163 if (const auto *PET = dyn_cast<PackExpansionType>(LastParam->getType())) {
164 // ... of the type T&&... or T...
165 const auto BaseType = PET->getPattern().getNonReferenceType();
166 if (const auto *TTPT =
167 dyn_cast<TemplateTypeParmType>(BaseType.getTypePtr())) {
168 // ... whose template parameter comes from the function directly
169 if (FT->getTemplateParameters()->getDepth() == TTPT->getDepth()) {
170 return true;
171 }
172 }
173 }
174 }
175 return false;
176 }
177
178 bool shouldSkipFunctionBody(Decl *D) override {
179 // Usually we don't need to look inside the bodies of header functions
180 // to understand the program. However when forwarding function like
181 // emplace() forward their arguments to some other function, the
182 // interesting overload resolution happens inside the forwarding
183 // function's body. To provide more meaningful diagnostics,
184 // code completion, and parameter hints we should parse (and later
185 // instantiate) the bodies.
186 if (auto *FT = llvm::dyn_cast<clang::FunctionTemplateDecl>(D)) {
187 if (ParseForwardingFunctions) {
188 // Don't skip parsing the body if it looks like a forwarding function
189 if (isLikelyForwardingFunction(FT))
190 return false;
191 } else {
192 // By default, only take care of make_unique
193 // std::make_unique is trivial, and we diagnose bad constructor calls.
194 if (const auto *II = FT->getDeclName().getAsIdentifierInfo()) {
195 if (II->isStr("make_unique") && FT->isInStdNamespace())
196 return false;
197 }
198 }
199 }
200 return true;
201 }
202
203private:
204 PathRef File;
205 PreambleParsedCallback ParsedCallback;
206 IncludeStructure Includes;
207 CanonicalIncludes CanonIncludes;
208 include_cleaner::PragmaIncludes Pragmas;
209 MainFileMacros Macros;
210 std::vector<PragmaMark> Marks;
211 bool IsMainFileIncludeGuarded = false;
212 std::unique_ptr<CommentHandler> IWYUHandler = nullptr;
213 const clang::LangOptions *LangOpts = nullptr;
214 const SourceManager *SourceMgr = nullptr;
215 const Preprocessor *PP = nullptr;
216 PreambleBuildStats *Stats;
217 bool ParseForwardingFunctions;
218 std::function<void(CompilerInstance &)> BeforeExecuteCallback;
219};
220
221// Represents directives other than includes, where basic textual information is
222// enough.
223struct TextualPPDirective {
225 // Full text that's representing the directive, including the `#`.
226 std::string Text;
227 unsigned Offset;
228 tok::PPKeywordKind Directive = tok::PPKeywordKind::pp_not_keyword;
229 // Name of the macro being defined in the case of a #define directive.
230 std::string MacroName;
231
232 bool operator==(const TextualPPDirective &RHS) const {
233 return std::tie(DirectiveLine, Offset, Text) ==
234 std::tie(RHS.DirectiveLine, RHS.Offset, RHS.Text);
235 }
236};
237
238// Formats a PP directive consisting of Prefix (e.g. "#define ") and Body ("X
239// 10"). The formatting is copied so that the tokens in Body have PresumedLocs
240// with correct columns and lines.
241std::string spellDirective(llvm::StringRef Prefix,
242 CharSourceRange DirectiveRange,
243 const LangOptions &LangOpts, const SourceManager &SM,
244 unsigned &DirectiveLine, unsigned &Offset) {
245 std::string SpelledDirective;
246 llvm::raw_string_ostream OS(SpelledDirective);
247 OS << Prefix;
248
249 // Make sure DirectiveRange is a char range and doesn't contain macro ids.
250 DirectiveRange = SM.getExpansionRange(DirectiveRange);
251 if (DirectiveRange.isTokenRange()) {
252 DirectiveRange.setEnd(
253 Lexer::getLocForEndOfToken(DirectiveRange.getEnd(), 0, SM, LangOpts));
254 }
255
256 auto DecompLoc = SM.getDecomposedLoc(DirectiveRange.getBegin());
257 DirectiveLine = SM.getLineNumber(DecompLoc.first, DecompLoc.second);
258 Offset = DecompLoc.second;
259 auto TargetColumn = SM.getColumnNumber(DecompLoc.first, DecompLoc.second) - 1;
260
261 // Pad with spaces before DirectiveRange to make sure it will be on right
262 // column when patched.
263 if (Prefix.size() <= TargetColumn) {
264 // There is enough space for Prefix and space before directive, use it.
265 // We try to squeeze the Prefix into the same line whenever we can, as
266 // putting onto a separate line won't work at the beginning of the file.
267 OS << std::string(TargetColumn - Prefix.size(), ' ');
268 } else {
269 // Prefix was longer than the space we had. We produce e.g.:
270 // #line N-1
271 // #define \
272 // X 10
273 OS << "\\\n" << std::string(TargetColumn, ' ');
274 // Decrement because we put an additional line break before
275 // DirectiveRange.begin().
277 }
278 OS << toSourceCode(SM, DirectiveRange.getAsRange());
279 return OS.str();
280}
281
282// Collects #define directives inside the main file.
283struct DirectiveCollector : public PPCallbacks {
284 DirectiveCollector(const Preprocessor &PP,
285 std::vector<TextualPPDirective> &TextualDirectives)
286 : LangOpts(PP.getLangOpts()), SM(PP.getSourceManager()),
287 TextualDirectives(TextualDirectives) {}
288
289 void FileChanged(SourceLocation Loc, FileChangeReason Reason,
290 SrcMgr::CharacteristicKind FileType,
291 FileID PrevFID) override {
292 InMainFile = SM.isWrittenInMainFile(Loc);
293 }
294
295 void MacroDefined(const Token &MacroNameTok,
296 const MacroDirective *MD) override {
297 if (!InMainFile)
298 return;
299 TextualDirectives.emplace_back();
300 TextualPPDirective &TD = TextualDirectives.back();
301 TD.Directive = tok::pp_define;
302 TD.MacroName = MacroNameTok.getIdentifierInfo()->getName().str();
303
304 const auto *MI = MD->getMacroInfo();
305 TD.Text =
306 spellDirective("#define ",
307 CharSourceRange::getTokenRange(
308 MI->getDefinitionLoc(), MI->getDefinitionEndLoc()),
309 LangOpts, SM, TD.DirectiveLine, TD.Offset);
310 }
311
312private:
313 bool InMainFile = true;
314 const LangOptions &LangOpts;
315 const SourceManager &SM;
316 std::vector<TextualPPDirective> &TextualDirectives;
317};
318
319struct ScannedPreamble {
320 std::vector<Inclusion> Includes;
321 std::vector<TextualPPDirective> TextualDirectives;
322 // Literal lines of the preamble contents.
323 std::vector<llvm::StringRef> Lines;
324 PreambleBounds Bounds = {0, false};
325 std::vector<PragmaMark> Marks;
326 MainFileMacros Macros;
327};
328
329/// Scans the preprocessor directives in the preamble section of the file by
330/// running preprocessor over \p Contents. Returned includes do not contain
331/// resolved paths. \p Cmd is used to build the compiler invocation, which might
332/// stat/read files.
333llvm::Expected<ScannedPreamble>
334scanPreamble(llvm::StringRef Contents, const tooling::CompileCommand &Cmd) {
335 class EmptyFS : public ThreadsafeFS {
336 private:
337 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> viewImpl() const override {
338 return new llvm::vfs::InMemoryFileSystem;
339 }
340 };
341 EmptyFS FS;
342 // Build and run Preprocessor over the preamble.
343 ParseInputs PI;
344 // Memory buffers below expect null-terminated && non-null strings. So make
345 // sure to always use PI.Contents!
346 PI.Contents = Contents.str();
347 PI.TFS = &FS;
348 PI.CompileCommand = Cmd;
349 IgnoringDiagConsumer IgnoreDiags;
351 if (!CI)
352 return error("failed to create compiler invocation");
353 CI->getDiagnosticOpts().IgnoreWarnings = true;
354 auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(PI.Contents);
355 // This means we're scanning (though not preprocessing) the preamble section
356 // twice. However, it's important to precisely follow the preamble bounds used
357 // elsewhere.
358 auto Bounds = ComputePreambleBounds(*CI->getLangOpts(), *ContentsBuffer, 0);
359 auto PreambleContents = llvm::MemoryBuffer::getMemBufferCopy(
360 llvm::StringRef(PI.Contents).take_front(Bounds.Size));
362 std::move(CI), nullptr, std::move(PreambleContents),
363 // Provide an empty FS to prevent preprocessor from performing IO. This
364 // also implies missing resolved paths for includes.
365 FS.view(std::nullopt), IgnoreDiags);
366 if (Clang->getFrontendOpts().Inputs.empty())
367 return error("compiler instance had no inputs");
368 // We are only interested in main file includes.
369 Clang->getPreprocessorOpts().SingleFileParseMode = true;
370 Clang->getPreprocessorOpts().UsePredefines = false;
371 PreprocessOnlyAction Action;
372 if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0]))
373 return error("failed BeginSourceFile");
374 Preprocessor &PP = Clang->getPreprocessor();
375 const auto &SM = PP.getSourceManager();
376 IncludeStructure Includes;
377 Includes.collect(*Clang);
378 ScannedPreamble SP;
379 SP.Bounds = Bounds;
380 PP.addPPCallbacks(
381 std::make_unique<DirectiveCollector>(PP, SP.TextualDirectives));
382 PP.addPPCallbacks(collectPragmaMarksCallback(SM, SP.Marks));
383 PP.addPPCallbacks(std::make_unique<CollectMainFileMacros>(PP, SP.Macros));
384 if (llvm::Error Err = Action.Execute())
385 return std::move(Err);
386 Action.EndSourceFile();
387 SP.Includes = std::move(Includes.MainFileIncludes);
388 llvm::append_range(SP.Lines, llvm::split(Contents, "\n"));
389 return SP;
390}
391
392const char *spellingForIncDirective(tok::PPKeywordKind IncludeDirective) {
393 switch (IncludeDirective) {
394 case tok::pp_include:
395 return "include";
396 case tok::pp_import:
397 return "import";
398 case tok::pp_include_next:
399 return "include_next";
400 default:
401 break;
402 }
403 llvm_unreachable("not an include directive");
404}
405
406// Accumulating wall time timer. Similar to llvm::Timer, but much cheaper,
407// it only tracks wall time.
408// Since this is a generic timer, We may want to move this to support/ if we
409// find a use case outside of FS time tracking.
410class WallTimer {
411public:
412 WallTimer() : TotalTime(std::chrono::steady_clock::duration::zero()) {}
413 // [Re-]Start the timer.
414 void startTimer() { StartTime = std::chrono::steady_clock::now(); }
415 // Stop the timer and update total time.
416 void stopTimer() {
417 TotalTime += std::chrono::steady_clock::now() - StartTime;
418 }
419 // Return total time, in seconds.
420 double getTime() { return std::chrono::duration<double>(TotalTime).count(); }
421
422private:
423 std::chrono::steady_clock::duration TotalTime;
424 std::chrono::steady_clock::time_point StartTime;
425};
426
427class WallTimerRegion {
428public:
429 WallTimerRegion(WallTimer &T) : T(T) { T.startTimer(); }
430 ~WallTimerRegion() { T.stopTimer(); }
431
432private:
433 WallTimer &T;
434};
435
436// Used by TimerFS, tracks time spent in status() and getBuffer() calls while
437// proxying to underlying File implementation.
438class TimerFile : public llvm::vfs::File {
439public:
440 TimerFile(WallTimer &Timer, std::unique_ptr<File> InnerFile)
441 : Timer(Timer), InnerFile(std::move(InnerFile)) {}
442
443 llvm::ErrorOr<llvm::vfs::Status> status() override {
444 WallTimerRegion T(Timer);
445 return InnerFile->status();
446 }
447 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
448 getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
449 bool IsVolatile) override {
450 WallTimerRegion T(Timer);
451 return InnerFile->getBuffer(Name, FileSize, RequiresNullTerminator,
452 IsVolatile);
453 }
454 std::error_code close() override {
455 WallTimerRegion T(Timer);
456 return InnerFile->close();
457 }
458
459private:
460 WallTimer &Timer;
461 std::unique_ptr<llvm::vfs::File> InnerFile;
462};
463
464// A wrapper for FileSystems that tracks the amount of time spent in status()
465// and openFileForRead() calls.
466class TimerFS : public llvm::vfs::ProxyFileSystem {
467public:
468 TimerFS(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
469 : ProxyFileSystem(std::move(FS)) {}
470
471 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
472 openFileForRead(const llvm::Twine &Path) override {
473 WallTimerRegion T(Timer);
474 auto FileOr = getUnderlyingFS().openFileForRead(Path);
475 if (!FileOr)
476 return FileOr;
477 return std::make_unique<TimerFile>(Timer, std::move(FileOr.get()));
478 }
479
480 llvm::ErrorOr<llvm::vfs::Status> status(const llvm::Twine &Path) override {
481 WallTimerRegion T(Timer);
482 return getUnderlyingFS().status(Path);
483 }
484
485 double getTime() { return Timer.getTime(); }
486
487private:
488 WallTimer Timer;
489};
490
491// Helpers for patching diagnostics between two versions of file contents.
492class DiagPatcher {
493 llvm::ArrayRef<llvm::StringRef> OldLines;
494 llvm::ArrayRef<llvm::StringRef> CurrentLines;
495 llvm::StringMap<llvm::SmallVector<int>> CurrentContentsToLine;
496
497 // Translates a range from old lines to current lines.
498 // Finds the consecutive set of lines that corresponds to the same contents in
499 // old and current, and applies the same translation to the range.
500 // Returns true if translation succeeded.
501 bool translateRange(Range &R) {
502 int OldStart = R.start.line;
503 int OldEnd = R.end.line;
504 assert(OldStart <= OldEnd);
505
506 size_t RangeLen = OldEnd - OldStart + 1;
507 auto RangeContents = OldLines.slice(OldStart).take_front(RangeLen);
508 // Make sure the whole range is covered in old contents.
509 if (RangeContents.size() < RangeLen)
510 return false;
511
512 std::optional<int> Closest;
513 for (int AlternateLine : CurrentContentsToLine.lookup(RangeContents[0])) {
514 // Check if AlternateLine matches all lines in the range.
515 if (RangeContents !=
516 CurrentLines.slice(AlternateLine).take_front(RangeLen))
517 continue;
518 int Delta = AlternateLine - OldStart;
519 if (!Closest.has_value() || abs(Delta) < abs(*Closest))
520 Closest = Delta;
521 }
522 // Couldn't find any viable matches in the current contents.
523 if (!Closest.has_value())
524 return false;
525 R.start.line += *Closest;
526 R.end.line += *Closest;
527 return true;
528 }
529
530 // Translates a Note by patching its range when inside main file. Returns true
531 // on success.
532 bool translateNote(Note &N) {
533 if (!N.InsideMainFile)
534 return true;
535 if (translateRange(N.Range))
536 return true;
537 return false;
538 }
539
540 // Tries to translate all the edit ranges inside the fix. Returns true on
541 // success. On failure fixes might be in an invalid state.
542 bool translateFix(Fix &F) {
543 return llvm::all_of(
544 F.Edits, [this](TextEdit &E) { return translateRange(E.range); });
545 }
546
547public:
548 DiagPatcher(llvm::ArrayRef<llvm::StringRef> OldLines,
549 llvm::ArrayRef<llvm::StringRef> CurrentLines) {
550 this->OldLines = OldLines;
551 this->CurrentLines = CurrentLines;
552 for (int Line = 0, E = CurrentLines.size(); Line != E; ++Line) {
553 llvm::StringRef Contents = CurrentLines[Line];
554 CurrentContentsToLine[Contents].push_back(Line);
555 }
556 }
557 // Translate diagnostic by moving its main range to new location (if inside
558 // the main file). Preserve all the notes and fixes that can be translated to
559 // new contents.
560 // Drops the whole diagnostic if main range can't be patched.
561 std::optional<Diag> translateDiag(const Diag &D) {
562 Range NewRange = D.Range;
563 // Patch range if it's inside main file.
564 if (D.InsideMainFile && !translateRange(NewRange)) {
565 // Drop the diagnostic if we couldn't patch the range.
566 return std::nullopt;
567 }
568
569 Diag NewD = D;
570 NewD.Range = NewRange;
571 // Translate ranges inside notes and fixes too, dropping the ones that are
572 // no longer relevant.
573 llvm::erase_if(NewD.Notes, [this](Note &N) { return !translateNote(N); });
574 llvm::erase_if(NewD.Fixes, [this](Fix &F) { return !translateFix(F); });
575 return NewD;
576 }
577};
578} // namespace
579
580std::shared_ptr<const PreambleData>
581buildPreamble(PathRef FileName, CompilerInvocation CI,
582 const ParseInputs &Inputs, bool StoreInMemory,
583 PreambleParsedCallback PreambleCallback,
584 PreambleBuildStats *Stats) {
585 // Note that we don't need to copy the input contents, preamble can live
586 // without those.
587 auto ContentsBuffer =
588 llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName);
589 auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), *ContentsBuffer, 0);
590
591 trace::Span Tracer("BuildPreamble");
592 SPAN_ATTACH(Tracer, "File", FileName);
593 std::vector<std::unique_ptr<FeatureModule::ASTListener>> ASTListeners;
594 if (Inputs.FeatureModules) {
595 for (auto &M : *Inputs.FeatureModules) {
596 if (auto Listener = M.astListeners())
597 ASTListeners.emplace_back(std::move(Listener));
598 }
599 }
600 StoreDiags PreambleDiagnostics;
601 PreambleDiagnostics.setDiagCallback(
602 [&ASTListeners](const clang::Diagnostic &D, clangd::Diag &Diag) {
603 llvm::for_each(ASTListeners,
604 [&](const auto &L) { L->sawDiagnostic(D, Diag); });
605 });
606 llvm::IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine =
607 CompilerInstance::createDiagnostics(&CI.getDiagnosticOpts(),
608 &PreambleDiagnostics, false);
609 const Config &Cfg = Config::current();
610 PreambleDiagnostics.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
611 const clang::Diagnostic &Info) {
612 if (Cfg.Diagnostics.SuppressAll ||
614 *CI.getLangOpts()))
615 return DiagnosticsEngine::Ignored;
616 switch (Info.getID()) {
617 case diag::warn_no_newline_eof:
618 case diag::warn_cxx98_compat_no_newline_eof:
619 case diag::ext_no_newline_eof:
620 // If the preamble doesn't span the whole file, drop the no newline at
621 // eof warnings.
622 return Bounds.Size != ContentsBuffer->getBufferSize()
623 ? DiagnosticsEngine::Level::Ignored
624 : DiagLevel;
625 }
626 return DiagLevel;
627 });
628
629 // Skip function bodies when building the preamble to speed up building
630 // the preamble and make it smaller.
631 assert(!CI.getFrontendOpts().SkipFunctionBodies);
632 CI.getFrontendOpts().SkipFunctionBodies = true;
633 // We don't want to write comment locations into PCH. They are racy and slow
634 // to read back. We rely on dynamic index for the comments instead.
635 CI.getPreprocessorOpts().WriteCommentListToPCH = false;
636
637 CppFilePreambleCallbacks CapturedInfo(
638 FileName, PreambleCallback, Stats,
640 [&ASTListeners](CompilerInstance &CI) {
641 for (const auto &L : ASTListeners)
642 L->beforeExecute(CI);
643 });
644 auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
645 llvm::SmallString<32> AbsFileName(FileName);
646 VFS->makeAbsolute(AbsFileName);
647 auto StatCache = std::make_unique<PreambleFileStatusCache>(AbsFileName);
648 auto StatCacheFS = StatCache->getProducingFS(VFS);
649 llvm::IntrusiveRefCntPtr<TimerFS> TimedFS(new TimerFS(StatCacheFS));
650
651 WallTimer PreambleTimer;
652 PreambleTimer.startTimer();
653 auto BuiltPreamble = PrecompiledPreamble::Build(
654 CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine,
655 Stats ? TimedFS : StatCacheFS, std::make_shared<PCHContainerOperations>(),
656 StoreInMemory, /*StoragePath=*/StringRef(), CapturedInfo);
657 PreambleTimer.stopTimer();
658
659 // When building the AST for the main file, we do want the function
660 // bodies.
661 CI.getFrontendOpts().SkipFunctionBodies = false;
662
663 if (Stats != nullptr) {
664 Stats->TotalBuildTime = PreambleTimer.getTime();
665 Stats->FileSystemTime = TimedFS->getTime();
666 Stats->SerializedSize = BuiltPreamble ? BuiltPreamble->getSize() : 0;
667 }
668
669 if (BuiltPreamble) {
670 log("Built preamble of size {0} for file {1} version {2} in {3} seconds",
671 BuiltPreamble->getSize(), FileName, Inputs.Version,
672 PreambleTimer.getTime());
673 std::vector<Diag> Diags = PreambleDiagnostics.take();
674 auto Result = std::make_shared<PreambleData>(std::move(*BuiltPreamble));
675 Result->Version = Inputs.Version;
676 Result->CompileCommand = Inputs.CompileCommand;
677 Result->Diags = std::move(Diags);
678 Result->Includes = CapturedInfo.takeIncludes();
679 Result->Pragmas = CapturedInfo.takePragmaIncludes();
680 Result->Macros = CapturedInfo.takeMacros();
681 Result->Marks = CapturedInfo.takeMarks();
682 Result->CanonIncludes = CapturedInfo.takeCanonicalIncludes();
683 Result->StatCache = std::move(StatCache);
684 Result->MainIsIncludeGuarded = CapturedInfo.isMainFileIncludeGuarded();
685 return Result;
686 }
687
688 elog("Could not build a preamble for file {0} version {1}: {2}", FileName,
689 Inputs.Version, BuiltPreamble.getError().message());
690 for (const Diag &D : PreambleDiagnostics.take()) {
691 if (D.Severity < DiagnosticsEngine::Error)
692 continue;
693 // Not an ideal way to show errors, but better than nothing!
694 elog(" error: {0}", D.Message);
695 }
696 return nullptr;
697}
698
700 const ParseInputs &Inputs, PathRef FileName,
701 const CompilerInvocation &CI) {
702 auto ContentsBuffer =
703 llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName);
704 auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), *ContentsBuffer, 0);
705 auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
706 return compileCommandsAreEqual(Inputs.CompileCommand,
707 Preamble.CompileCommand) &&
708 Preamble.Preamble.CanReuse(CI, *ContentsBuffer, Bounds, *VFS);
709}
710
711void escapeBackslashAndQuotes(llvm::StringRef Text, llvm::raw_ostream &OS) {
712 for (char C : Text) {
713 switch (C) {
714 case '\\':
715 case '"':
716 OS << '\\';
717 break;
718 default:
719 break;
720 }
721 OS << C;
722 }
723}
724
725// Translate diagnostics from baseline into modified for the lines that have the
726// same spelling.
727static std::vector<Diag> patchDiags(llvm::ArrayRef<Diag> BaselineDiags,
728 const ScannedPreamble &BaselineScan,
729 const ScannedPreamble &ModifiedScan) {
730 std::vector<Diag> PatchedDiags;
731 if (BaselineDiags.empty())
732 return PatchedDiags;
733 DiagPatcher Patcher(BaselineScan.Lines, ModifiedScan.Lines);
734 for (auto &D : BaselineDiags) {
735 if (auto NewD = Patcher.translateDiag(D))
736 PatchedDiags.emplace_back(std::move(*NewD));
737 }
738 return PatchedDiags;
739}
740
741static std::string getPatchName(llvm::StringRef FileName) {
742 // This shouldn't coincide with any real file name.
743 llvm::SmallString<128> PatchName;
744 llvm::sys::path::append(PatchName, llvm::sys::path::parent_path(FileName),
746 return PatchName.str().str();
747}
748
749PreamblePatch PreamblePatch::create(llvm::StringRef FileName,
750 const ParseInputs &Modified,
751 const PreambleData &Baseline,
752 PatchType PatchType) {
753 trace::Span Tracer("CreatePreamblePatch");
754 SPAN_ATTACH(Tracer, "File", FileName);
755 assert(llvm::sys::path::is_absolute(FileName) && "relative FileName!");
756 // First scan preprocessor directives in Baseline and Modified. These will be
757 // used to figure out newly added directives in Modified. Scanning can fail,
758 // the code just bails out and creates an empty patch in such cases, as:
759 // - If scanning for Baseline fails, no knowledge of existing includes hence
760 // patch will contain all the includes in Modified. Leading to rebuild of
761 // whole preamble, which is terribly slow.
762 // - If scanning for Modified fails, cannot figure out newly added ones so
763 // there's nothing to do but generate an empty patch.
764 auto BaselineScan =
765 scanPreamble(Baseline.Preamble.getContents(), Modified.CompileCommand);
766 if (!BaselineScan) {
767 elog("Failed to scan baseline of {0}: {1}", FileName,
768 BaselineScan.takeError());
769 return PreamblePatch::unmodified(Baseline);
770 }
771 auto ModifiedScan = scanPreamble(Modified.Contents, Modified.CompileCommand);
772 if (!ModifiedScan) {
773 elog("Failed to scan modified contents of {0}: {1}", FileName,
774 ModifiedScan.takeError());
775 return PreamblePatch::unmodified(Baseline);
776 }
777
778 bool IncludesChanged = BaselineScan->Includes != ModifiedScan->Includes;
779 bool DirectivesChanged =
780 BaselineScan->TextualDirectives != ModifiedScan->TextualDirectives;
781 if ((PatchType == PatchType::MacroDirectives || !IncludesChanged) &&
782 !DirectivesChanged)
783 return PreamblePatch::unmodified(Baseline);
784
785 PreamblePatch PP;
786 PP.Baseline = &Baseline;
787 PP.PatchFileName = getPatchName(FileName);
788 PP.ModifiedBounds = ModifiedScan->Bounds;
789
790 llvm::raw_string_ostream Patch(PP.PatchContents);
791 // Set default filename for subsequent #line directives
792 Patch << "#line 0 \"";
793 // FileName part of a line directive is subject to backslash escaping, which
794 // might lead to problems on windows especially.
796 Patch << "\"\n";
797
798 if (IncludesChanged && PatchType == PatchType::All) {
799 // We are only interested in newly added includes, record the ones in
800 // Baseline for exclusion.
801 llvm::DenseMap<std::pair<tok::PPKeywordKind, llvm::StringRef>,
802 const Inclusion *>
803 ExistingIncludes;
804 for (const auto &Inc : Baseline.Includes.MainFileIncludes)
805 ExistingIncludes[{Inc.Directive, Inc.Written}] = &Inc;
806 // There might be includes coming from disabled regions, record these for
807 // exclusion too. note that we don't have resolved paths for those.
808 for (const auto &Inc : BaselineScan->Includes)
809 ExistingIncludes.try_emplace({Inc.Directive, Inc.Written});
810 // Calculate extra includes that needs to be inserted.
811 for (auto &Inc : ModifiedScan->Includes) {
812 auto It = ExistingIncludes.find({Inc.Directive, Inc.Written});
813 // Include already present in the baseline preamble. Set resolved path and
814 // put into preamble includes.
815 if (It != ExistingIncludes.end()) {
816 if (It->second) {
817 // If this header is included in an active region of the baseline
818 // preamble, preserve it.
819 auto &PatchedInc = PP.PreambleIncludes.emplace_back();
820 // Copy everything from existing include, apart from the location,
821 // when it's coming from baseline preamble.
822 PatchedInc = *It->second;
823 PatchedInc.HashLine = Inc.HashLine;
824 PatchedInc.HashOffset = Inc.HashOffset;
825 }
826 continue;
827 }
828 // Include is new in the modified preamble. Inject it into the patch and
829 // use #line to set the presumed location to where it is spelled.
830 auto LineCol = offsetToClangLineColumn(Modified.Contents, Inc.HashOffset);
831 Patch << llvm::formatv("#line {0}\n", LineCol.first);
832 Patch << llvm::formatv(
833 "#{0} {1}\n", spellingForIncDirective(Inc.Directive), Inc.Written);
834 }
835 } else {
836 // Make sure we have the full set of includes available even when we're not
837 // patching. As these are used by features we provide afterwards like hover,
838 // go-to-def or include-cleaner when preamble is stale.
839 PP.PreambleIncludes = Baseline.Includes.MainFileIncludes;
840 }
841
842 if (DirectivesChanged) {
843 // We need to patch all the directives, since they are order dependent. e.g:
844 // #define BAR(X) NEW(X) // Newly introduced in Modified
845 // #define BAR(X) OLD(X) // Exists in the Baseline
846 //
847 // If we've patched only the first directive, the macro definition would've
848 // been wrong for the rest of the file, since patch is applied after the
849 // baseline preamble.
850 //
851 // Note that we deliberately ignore conditional directives and undefs to
852 // reduce complexity. The former might cause problems because scanning is
853 // imprecise and might pick directives from disabled regions.
854 for (const auto &TD : ModifiedScan->TextualDirectives) {
855 // Introduce an #undef directive before #defines to suppress any
856 // re-definition warnings.
857 if (TD.Directive == tok::pp_define)
858 Patch << "#undef " << TD.MacroName << '\n';
859 Patch << "#line " << TD.DirectiveLine << '\n';
860 Patch << TD.Text << '\n';
861 }
862 }
863
864 PP.PatchedDiags = patchDiags(Baseline.Diags, *BaselineScan, *ModifiedScan);
865 PP.PatchedMarks = std::move(ModifiedScan->Marks);
866 PP.PatchedMacros = std::move(ModifiedScan->Macros);
867 dlog("Created preamble patch: {0}", Patch.str());
868 Patch.flush();
869 return PP;
870}
871
873 const ParseInputs &Modified,
874 const PreambleData &Baseline) {
875 return create(FileName, Modified, Baseline, PatchType::All);
876}
877
879 const ParseInputs &Modified,
880 const PreambleData &Baseline) {
881 return create(FileName, Modified, Baseline, PatchType::MacroDirectives);
882}
883
884void PreamblePatch::apply(CompilerInvocation &CI) const {
885 // No need to map an empty file.
886 if (PatchContents.empty())
887 return;
888 auto &PPOpts = CI.getPreprocessorOpts();
889 auto PatchBuffer =
890 // we copy here to ensure contents are still valid if CI outlives the
891 // PreamblePatch.
892 llvm::MemoryBuffer::getMemBufferCopy(PatchContents, PatchFileName);
893 // CI will take care of the lifetime of the buffer.
894 PPOpts.addRemappedFile(PatchFileName, PatchBuffer.release());
895 // The patch will be parsed after loading the preamble ast and before parsing
896 // the main file.
897 PPOpts.Includes.push_back(PatchFileName);
898}
899
900std::vector<Inclusion> PreamblePatch::preambleIncludes() const {
901 return PreambleIncludes;
902}
903
905 PreamblePatch PP;
906 PP.Baseline = &Preamble;
907 PP.PreambleIncludes = Preamble.Includes.MainFileIncludes;
908 PP.ModifiedBounds = Preamble.Preamble.getBounds();
909 PP.PatchedDiags = Preamble.Diags;
910 return PP;
911}
912
914 return PatchContents.empty() ||
916}
917llvm::ArrayRef<PragmaMark> PreamblePatch::marks() const {
918 if (PatchContents.empty())
919 return Baseline->Marks;
920 return PatchedMarks;
921}
922
924 if (PatchContents.empty())
925 return Baseline->Macros;
926 return PatchedMacros;
927}
928
929const FileEntry *PreamblePatch::getPatchEntry(llvm::StringRef MainFilePath,
930 const SourceManager &SM) {
931 auto PatchFilePath = getPatchName(MainFilePath);
932 if (auto File = SM.getFileManager().getFile(PatchFilePath))
933 return *File;
934 return nullptr;
935}
936} // namespace clangd
937} // namespace clang
const Expr * E
const FunctionDecl * Decl
size_t Offset
const std::optional< PreamblePatch > Patch
const Criteria C
unsigned Lines
std::string Text
IgnoringDiagConsumer IgnoreDiags
std::unique_ptr< CompilerInstance > Clang
CharSourceRange Range
SourceRange for the file name.
StringRef FileName
SourceLocation Loc
#define dlog(...)
Definition: Logger.h:101
const MacroDirective * Directive
Token Name
FieldAction Action
std::string MacroName
Definition: Preamble.cpp:230
PreambleBounds Bounds
Definition: Preamble.cpp:324
unsigned DirectiveLine
Definition: Preamble.cpp:224
const google::protobuf::Message & M
Definition: Server.cpp:309
std::unique_ptr< CompilerInvocation > CI
llvm::raw_string_ostream OS
Definition: TraceTests.cpp:160
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:164
std::vector< Inclusion > MainFileIncludes
Definition: Headers.h:174
void collect(const CompilerInstance &CI)
Definition: Headers.cpp:174
Stores information required to parse a TU using a (possibly stale) Baseline preamble.
Definition: Preamble.h:127
bool preserveDiagnostics() const
Whether diagnostics generated using this patch are trustable.
Definition: Preamble.cpp:913
const MainFileMacros & mainFileMacros() const
Definition: Preamble.cpp:923
llvm::ArrayRef< PragmaMark > marks() const
Definition: Preamble.cpp:917
void apply(CompilerInvocation &CI) const
Adjusts CI (which compiles the modified inputs) to be used with the baseline preamble.
Definition: Preamble.cpp:884
static PreamblePatch unmodified(const PreambleData &Preamble)
Preamble is used verbatim.
Definition: Preamble.cpp:904
static PreamblePatch createMacroPatch(llvm::StringRef FileName, const ParseInputs &Modified, const PreambleData &Baseline)
Definition: Preamble.cpp:878
static const FileEntry * getPatchEntry(llvm::StringRef MainFilePath, const SourceManager &SM)
Returns the FileEntry for the preamble patch of MainFilePath in SM, if any.
Definition: Preamble.cpp:929
static PreamblePatch createFullPatch(llvm::StringRef FileName, const ParseInputs &Modified, const PreambleData &Baseline)
Builds a patch that contains new PP directives introduced to the preamble section of Modified compare...
Definition: Preamble.cpp:872
static constexpr llvm::StringLiteral HeaderName
Definition: Preamble.h:172
std::vector< Inclusion > preambleIncludes() const
Returns #include directives from the Modified preamble that were resolved using the Baseline preamble...
Definition: Preamble.cpp:900
StoreDiags collects the diagnostics that can later be reported by clangd.
Definition: Diagnostics.h:138
void setLevelAdjuster(LevelAdjuster Adjuster)
If set, this allows the client of this class to adjust the level of diagnostics, such as promoting wa...
Definition: Diagnostics.h:162
std::vector< Diag > take(const clang::tidy::ClangTidyContext *Tidy=nullptr)
void setDiagCallback(DiagCallback CB)
Invokes a callback every time a diagnostics is completely formed.
Definition: Diagnostics.h:165
llvm::IntrusiveRefCntPtr< llvm::vfs::FileSystem > view(std::nullopt_t CWD) const
Obtain a vfs::FileSystem with an arbitrary initial working directory.
Definition: ThreadsafeFS.h:32
Records an event whose duration is the lifetime of the Span object.
Definition: Trace.h:143
@ Info
An information message.
static std::string getPatchName(llvm::StringRef FileName)
Definition: Preamble.cpp:741
std::string Path
A typedef to represent a file path.
Definition: Path.h:26
std::unique_ptr< CompilerInvocation > buildCompilerInvocation(const ParseInputs &Inputs, clang::DiagnosticConsumer &D, std::vector< std::string > *CC1Args)
Builds compiler invocation that could be used to build AST or preamble.
Definition: Compiler.cpp:95
llvm::Error error(std::error_code EC, const char *Fmt, Ts &&... Vals)
Definition: Logger.h:79
bool operator==(const Inclusion &LHS, const Inclusion &RHS)
Definition: Headers.cpp:316
bool isBuiltinDiagnosticSuppressed(unsigned ID, const llvm::StringSet<> &Suppress, const LangOptions &LangOpts)
Determine whether a (non-clang-tidy) diagnostic is suppressed by config.
std::unique_ptr< PPCallbacks > collectPragmaMarksCallback(const SourceManager &SM, std::vector< PragmaMark > &Out)
Collect all pragma marks from the main file.
bool isPreambleCompatible(const PreambleData &Preamble, const ParseInputs &Inputs, PathRef FileName, const CompilerInvocation &CI)
Returns true if Preamble is reusable for Inputs.
Definition: Preamble.cpp:699
std::shared_ptr< const PreambleData > buildPreamble(PathRef FileName, CompilerInvocation CI, const ParseInputs &Inputs, bool StoreInMemory, PreambleParsedCallback PreambleCallback, PreambleBuildStats *Stats)
Build a preamble for the new inputs unless an old one can be reused.
Definition: Preamble.cpp:581
std::unique_ptr< CommentHandler > collectIWYUHeaderMaps(CanonicalIncludes *Includes)
Returns a CommentHandler that parses pragma comment on include files to determine when we should incl...
std::unique_ptr< CompilerInstance > prepareCompilerInstance(std::unique_ptr< clang::CompilerInvocation > CI, const PrecompiledPreamble *Preamble, std::unique_ptr< llvm::MemoryBuffer > Buffer, llvm::IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, DiagnosticConsumer &DiagsClient)
Definition: Compiler.cpp:129
std::function< void(ASTContext &, Preprocessor &, const CanonicalIncludes &)> PreambleParsedCallback
Definition: Preamble.h:84
void log(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:67
llvm::StringRef toSourceCode(const SourceManager &SM, SourceRange R)
Returns the source code covered by the source range.
Definition: SourceCode.cpp:448
std::pair< size_t, size_t > offsetToClangLineColumn(llvm::StringRef Code, size_t Offset)
Definition: SourceCode.cpp:483
static std::vector< Diag > patchDiags(llvm::ArrayRef< Diag > BaselineDiags, const ScannedPreamble &BaselineScan, const ScannedPreamble &ModifiedScan)
Definition: Preamble.cpp:727
void escapeBackslashAndQuotes(llvm::StringRef Text, llvm::raw_ostream &OS)
Definition: Preamble.cpp:711
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:29
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:61
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Settings that express user/project preferences and control clangd behavior.
Definition: Config.h:44
static const Config & current()
Returns the Config of the current Context, or an empty configuration.
Definition: Config.cpp:17
llvm::StringSet Suppress
Definition: Config.h:99
struct clang::clangd::Config::@4 Diagnostics
Controls warnings and errors when parsing code.
bool AllowStalePreamble
Enable emitting diagnostics using stale preambles.
Definition: Config.h:109
A top-level diagnostic that may have Notes and Fixes.
Definition: Diagnostics.h:98
Information required to run clang, e.g. to parse AST or do code completion.
Definition: Compiler.h:48
tooling::CompileCommand CompileCommand
Definition: Compiler.h:49
const ThreadsafeFS * TFS
Definition: Compiler.h:50
FeatureModuleSet * FeatureModules
Definition: Compiler.h:62
Timings and statistics from the premble build.
Definition: Preamble.h:89
double FileSystemTime
Time spent in filesystem operations during the build, in seconds.
Definition: Preamble.h:93
double TotalBuildTime
Total wall time it took to build preamble, in seconds.
Definition: Preamble.h:91
size_t SerializedSize
The serialized size of the preamble.
Definition: Preamble.h:101
The parsed preamble and associated data.
Definition: Preamble.h:55