clang 19.0.0git
PrecompiledPreamble.cpp
Go to the documentation of this file.
1//===--- PrecompiledPreamble.cpp - Build precompiled preambles --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Helper class to build precompiled preamble.
10//
11//===----------------------------------------------------------------------===//
12
21#include "clang/Lex/Lexer.h"
25#include "llvm/ADT/SmallString.h"
26#include "llvm/ADT/StringSet.h"
27#include "llvm/ADT/iterator_range.h"
28#include "llvm/Config/llvm-config.h"
29#include "llvm/Support/CrashRecoveryContext.h"
30#include "llvm/Support/FileSystem.h"
31#include "llvm/Support/Path.h"
32#include "llvm/Support/Process.h"
33#include "llvm/Support/VirtualFileSystem.h"
34#include <limits>
35#include <mutex>
36#include <utility>
37
38using namespace clang;
39
40namespace {
41
42StringRef getInMemoryPreamblePath() {
43#if defined(LLVM_ON_UNIX)
44 return "/__clang_tmp/___clang_inmemory_preamble___";
45#elif defined(_WIN32)
46 return "C:\\__clang_tmp\\___clang_inmemory_preamble___";
47#else
48#warning "Unknown platform. Defaulting to UNIX-style paths for in-memory PCHs"
49 return "/__clang_tmp/___clang_inmemory_preamble___";
50#endif
51}
52
54createVFSOverlayForPreamblePCH(StringRef PCHFilename,
55 std::unique_ptr<llvm::MemoryBuffer> PCHBuffer,
57 // We want only the PCH file from the real filesystem to be available,
58 // so we create an in-memory VFS with just that and overlay it on top.
60 new llvm::vfs::InMemoryFileSystem());
61 PCHFS->addFile(PCHFilename, 0, std::move(PCHBuffer));
63 new llvm::vfs::OverlayFileSystem(VFS));
64 Overlay->pushOverlay(PCHFS);
65 return Overlay;
66}
67
68class PreambleDependencyCollector : public DependencyCollector {
69public:
70 // We want to collect all dependencies for correctness. Avoiding the real
71 // system dependencies (e.g. stl from /usr/lib) would probably be a good idea,
72 // but there is no way to distinguish between those and the ones that can be
73 // spuriously added by '-isystem' (e.g. to suppress warnings from those
74 // headers).
75 bool needSystemDependencies() override { return true; }
76};
77
78// Collects files whose existence would invalidate the preamble.
79// Collecting *all* of these would make validating it too slow though, so we
80// just find all the candidates for 'file not found' diagnostics.
81//
82// A caveat that may be significant for generated files: we'll omit files under
83// search path entries whose roots don't exist when the preamble is built.
84// These are pruned by InitHeaderSearch and so we don't see the search path.
85// It would be nice to include them but we don't want to duplicate all the rest
86// of the InitHeaderSearch logic to reconstruct them.
87class MissingFileCollector : public PPCallbacks {
88 llvm::StringSet<> &Out;
89 const HeaderSearch &Search;
90 const SourceManager &SM;
91
92public:
93 MissingFileCollector(llvm::StringSet<> &Out, const HeaderSearch &Search,
94 const SourceManager &SM)
95 : Out(Out), Search(Search), SM(SM) {}
96
97 void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
98 StringRef FileName, bool IsAngled,
99 CharSourceRange FilenameRange,
100 OptionalFileEntryRef File, StringRef SearchPath,
101 StringRef RelativePath, const Module *SuggestedModule,
102 bool ModuleImported,
104 // File is std::nullopt if it wasn't found.
105 // (We have some false negatives if PP recovered e.g. <foo> -> "foo")
106 if (File)
107 return;
108
109 // If it's a rare absolute include, we know the full path already.
110 if (llvm::sys::path::is_absolute(FileName)) {
111 Out.insert(FileName);
112 return;
113 }
114
115 // Reconstruct the filenames that would satisfy this directive...
117 auto NotFoundRelativeTo = [&](DirectoryEntryRef DE) {
118 Buf = DE.getName();
119 llvm::sys::path::append(Buf, FileName);
120 llvm::sys::path::remove_dots(Buf, /*remove_dot_dot=*/true);
121 Out.insert(Buf);
122 };
123 // ...relative to the including file.
124 if (!IsAngled) {
125 if (OptionalFileEntryRef IncludingFile =
126 SM.getFileEntryRefForID(SM.getFileID(IncludeTok.getLocation())))
127 if (IncludingFile->getDir())
128 NotFoundRelativeTo(IncludingFile->getDir());
129 }
130 // ...relative to the search paths.
131 for (const auto &Dir : llvm::make_range(
132 IsAngled ? Search.angled_dir_begin() : Search.search_dir_begin(),
133 Search.search_dir_end())) {
134 // No support for frameworks or header maps yet.
135 if (Dir.isNormalDir())
136 NotFoundRelativeTo(*Dir.getDirRef());
137 }
138 }
139};
140
141/// Keeps a track of files to be deleted in destructor.
142class TemporaryFiles {
143public:
144 // A static instance to be used by all clients.
145 static TemporaryFiles &getInstance();
146
147private:
148 // Disallow constructing the class directly.
149 TemporaryFiles() = default;
150 // Disallow copy.
151 TemporaryFiles(const TemporaryFiles &) = delete;
152
153public:
154 ~TemporaryFiles();
155
156 /// Adds \p File to a set of tracked files.
157 void addFile(StringRef File);
158
159 /// Remove \p File from disk and from the set of tracked files.
160 void removeFile(StringRef File);
161
162private:
163 std::mutex Mutex;
164 llvm::StringSet<> Files;
165};
166
167TemporaryFiles &TemporaryFiles::getInstance() {
168 static TemporaryFiles Instance;
169 return Instance;
170}
171
172TemporaryFiles::~TemporaryFiles() {
173 std::lock_guard<std::mutex> Guard(Mutex);
174 for (const auto &File : Files)
175 llvm::sys::fs::remove(File.getKey());
176}
177
178void TemporaryFiles::addFile(StringRef File) {
179 std::lock_guard<std::mutex> Guard(Mutex);
180 auto IsInserted = Files.insert(File).second;
181 (void)IsInserted;
182 assert(IsInserted && "File has already been added");
183}
184
185void TemporaryFiles::removeFile(StringRef File) {
186 std::lock_guard<std::mutex> Guard(Mutex);
187 auto WasPresent = Files.erase(File);
188 (void)WasPresent;
189 assert(WasPresent && "File was not tracked");
190 llvm::sys::fs::remove(File);
191}
192
193// A temp file that would be deleted on destructor call. If destructor is not
194// called for any reason, the file will be deleted at static objects'
195// destruction.
196// An assertion will fire if two TempPCHFiles are created with the same name,
197// so it's not intended to be used outside preamble-handling.
198class TempPCHFile {
199public:
200 // A main method used to construct TempPCHFile.
201 static std::unique_ptr<TempPCHFile> create(StringRef StoragePath) {
202 // FIXME: This is a hack so that we can override the preamble file during
203 // crash-recovery testing, which is the only case where the preamble files
204 // are not necessarily cleaned up.
205 if (const char *TmpFile = ::getenv("CINDEXTEST_PREAMBLE_FILE"))
206 return std::unique_ptr<TempPCHFile>(new TempPCHFile(TmpFile));
207
209 // Using the versions of createTemporaryFile() and
210 // createUniqueFile() with a file descriptor guarantees
211 // that we would never get a race condition in a multi-threaded setting
212 // (i.e., multiple threads getting the same temporary path).
213 int FD;
214 std::error_code EC;
215 if (StoragePath.empty())
216 EC = llvm::sys::fs::createTemporaryFile("preamble", "pch", FD, File);
217 else {
218 llvm::SmallString<128> TempPath = StoragePath;
219 // Use the same filename model as fs::createTemporaryFile().
220 llvm::sys::path::append(TempPath, "preamble-%%%%%%.pch");
221 namespace fs = llvm::sys::fs;
222 // Use the same owner-only file permissions as fs::createTemporaryFile().
223 EC = fs::createUniqueFile(TempPath, FD, File, fs::OF_None,
224 fs::owner_read | fs::owner_write);
225 }
226 if (EC)
227 return nullptr;
228 // We only needed to make sure the file exists, close the file right away.
229 llvm::sys::Process::SafelyCloseFileDescriptor(FD);
230 return std::unique_ptr<TempPCHFile>(new TempPCHFile(File.str().str()));
231 }
232
233 TempPCHFile &operator=(const TempPCHFile &) = delete;
234 TempPCHFile(const TempPCHFile &) = delete;
235 ~TempPCHFile() { TemporaryFiles::getInstance().removeFile(FilePath); };
236
237 /// A path where temporary file is stored.
238 llvm::StringRef getFilePath() const { return FilePath; };
239
240private:
241 TempPCHFile(std::string FilePath) : FilePath(std::move(FilePath)) {
242 TemporaryFiles::getInstance().addFile(this->FilePath);
243 }
244
245 std::string FilePath;
246};
247
248class PrecompilePreambleAction : public ASTFrontendAction {
249public:
250 PrecompilePreambleAction(std::shared_ptr<PCHBuffer> Buffer, bool WritePCHFile,
251 PreambleCallbacks &Callbacks)
252 : Buffer(std::move(Buffer)), WritePCHFile(WritePCHFile),
253 Callbacks(Callbacks) {}
254
255 std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
256 StringRef InFile) override;
257
258 bool hasEmittedPreamblePCH() const { return HasEmittedPreamblePCH; }
259
260 void setEmittedPreamblePCH(ASTWriter &Writer) {
261 if (FileOS) {
262 *FileOS << Buffer->Data;
263 // Make sure it hits disk now.
264 FileOS.reset();
265 }
266
267 this->HasEmittedPreamblePCH = true;
268 Callbacks.AfterPCHEmitted(Writer);
269 }
270
271 bool BeginSourceFileAction(CompilerInstance &CI) override {
272 assert(CI.getLangOpts().CompilingPCH);
274 }
275
276 bool shouldEraseOutputFiles() override { return !hasEmittedPreamblePCH(); }
277 bool hasCodeCompletionSupport() const override { return false; }
278 bool hasASTFileSupport() const override { return false; }
280
281private:
282 friend class PrecompilePreambleConsumer;
283
284 bool HasEmittedPreamblePCH = false;
285 std::shared_ptr<PCHBuffer> Buffer;
286 bool WritePCHFile; // otherwise the PCH is written into the PCHBuffer only.
287 std::unique_ptr<llvm::raw_pwrite_stream> FileOS; // null if in-memory
288 PreambleCallbacks &Callbacks;
289};
290
291class PrecompilePreambleConsumer : public PCHGenerator {
292public:
293 PrecompilePreambleConsumer(PrecompilePreambleAction &Action, Preprocessor &PP,
294 InMemoryModuleCache &ModuleCache,
295 StringRef isysroot,
296 std::shared_ptr<PCHBuffer> Buffer)
297 : PCHGenerator(PP, ModuleCache, "", isysroot, std::move(Buffer),
298 ArrayRef<std::shared_ptr<ModuleFileExtension>>(),
299 /*AllowASTWithErrors=*/true),
300 Action(Action) {}
301
302 bool HandleTopLevelDecl(DeclGroupRef DG) override {
303 Action.Callbacks.HandleTopLevelDecl(DG);
304 return true;
305 }
306
307 void HandleTranslationUnit(ASTContext &Ctx) override {
309 if (!hasEmittedPCH())
310 return;
311 Action.setEmittedPreamblePCH(getWriter());
312 }
313
314 bool shouldSkipFunctionBody(Decl *D) override {
315 return Action.Callbacks.shouldSkipFunctionBody(D);
316 }
317
318private:
319 PrecompilePreambleAction &Action;
320};
321
322std::unique_ptr<ASTConsumer>
323PrecompilePreambleAction::CreateASTConsumer(CompilerInstance &CI,
324 StringRef InFile) {
325 std::string Sysroot;
327 return nullptr;
328
329 if (WritePCHFile) {
330 std::string OutputFile; // unused
331 FileOS = GeneratePCHAction::CreateOutputFile(CI, InFile, OutputFile);
332 if (!FileOS)
333 return nullptr;
334 }
335
337 Sysroot.clear();
338
339 return std::make_unique<PrecompilePreambleConsumer>(
340 *this, CI.getPreprocessor(), CI.getModuleCache(), Sysroot, Buffer);
341}
342
343template <class T> bool moveOnNoError(llvm::ErrorOr<T> Val, T &Output) {
344 if (!Val)
345 return false;
346 Output = std::move(*Val);
347 return true;
348}
349
350} // namespace
351
353 const llvm::MemoryBufferRef &Buffer,
354 unsigned MaxLines) {
355 return Lexer::ComputePreamble(Buffer.getBuffer(), LangOpts, MaxLines);
356}
357
359public:
360 static std::unique_ptr<PCHStorage> file(std::unique_ptr<TempPCHFile> File) {
361 assert(File);
362 std::unique_ptr<PCHStorage> S(new PCHStorage());
363 S->File = std::move(File);
364 return S;
365 }
366 static std::unique_ptr<PCHStorage> inMemory(std::shared_ptr<PCHBuffer> Buf) {
367 std::unique_ptr<PCHStorage> S(new PCHStorage());
368 S->Memory = std::move(Buf);
369 return S;
370 }
371
372 enum class Kind { InMemory, TempFile };
373 Kind getKind() const {
374 if (Memory)
375 return Kind::InMemory;
376 if (File)
377 return Kind::TempFile;
378 llvm_unreachable("Neither Memory nor File?");
379 }
380 llvm::StringRef filePath() const {
381 assert(getKind() == Kind::TempFile);
382 return File->getFilePath();
383 }
384 llvm::StringRef memoryContents() const {
385 assert(getKind() == Kind::InMemory);
386 return StringRef(Memory->Data.data(), Memory->Data.size());
387 }
388
389 // Shrink in-memory buffers to fit.
390 // This incurs a copy, but preambles tend to be long-lived.
391 // Only safe to call once nothing can alias the buffer.
392 void shrink() {
393 if (!Memory)
394 return;
395 Memory->Data = decltype(Memory->Data)(Memory->Data);
396 }
397
398private:
399 PCHStorage() = default;
400 PCHStorage(const PCHStorage &) = delete;
401 PCHStorage &operator=(const PCHStorage &) = delete;
402
403 std::shared_ptr<PCHBuffer> Memory;
404 std::unique_ptr<TempPCHFile> File;
405};
406
411
412llvm::ErrorOr<PrecompiledPreamble> PrecompiledPreamble::Build(
413 const CompilerInvocation &Invocation,
414 const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds,
415 DiagnosticsEngine &Diagnostics,
417 std::shared_ptr<PCHContainerOperations> PCHContainerOps, bool StoreInMemory,
418 StringRef StoragePath, PreambleCallbacks &Callbacks) {
419 assert(VFS && "VFS is null");
420
421 auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
422 FrontendOptions &FrontendOpts = PreambleInvocation->getFrontendOpts();
423 PreprocessorOptions &PreprocessorOpts =
424 PreambleInvocation->getPreprocessorOpts();
425
426 std::shared_ptr<PCHBuffer> Buffer = std::make_shared<PCHBuffer>();
427 std::unique_ptr<PCHStorage> Storage;
428 if (StoreInMemory) {
429 Storage = PCHStorage::inMemory(Buffer);
430 } else {
431 // Create a temporary file for the precompiled preamble. In rare
432 // circumstances, this can fail.
433 std::unique_ptr<TempPCHFile> PreamblePCHFile =
434 TempPCHFile::create(StoragePath);
435 if (!PreamblePCHFile)
437 Storage = PCHStorage::file(std::move(PreamblePCHFile));
438 }
439
440 // Save the preamble text for later; we'll need to compare against it for
441 // subsequent reparses.
442 std::vector<char> PreambleBytes(MainFileBuffer->getBufferStart(),
443 MainFileBuffer->getBufferStart() +
444 Bounds.Size);
445 bool PreambleEndsAtStartOfLine = Bounds.PreambleEndsAtStartOfLine;
446
447 // Tell the compiler invocation to generate a temporary precompiled header.
449 FrontendOpts.OutputFile = std::string(
450 StoreInMemory ? getInMemoryPreamblePath() : Storage->filePath());
451 PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
452 PreprocessorOpts.PrecompiledPreambleBytes.second = false;
453 // Inform preprocessor to record conditional stack when building the preamble.
454 PreprocessorOpts.GeneratePreamble = true;
455
456 // Create the compiler instance to use for building the precompiled preamble.
457 std::unique_ptr<CompilerInstance> Clang(
458 new CompilerInstance(std::move(PCHContainerOps)));
459
460 // Recover resources if we crash before exiting this method.
461 llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> CICleanup(
462 Clang.get());
463
464 Clang->setInvocation(std::move(PreambleInvocation));
465 Clang->setDiagnostics(&Diagnostics);
466
467 // Create the target instance.
468 if (!Clang->createTarget())
470
471 if (Clang->getFrontendOpts().Inputs.size() != 1 ||
472 Clang->getFrontendOpts().Inputs[0].getKind().getFormat() !=
474 Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() ==
477 }
478
479 // Clear out old caches and data.
480 Diagnostics.Reset();
481 ProcessWarningOptions(Diagnostics, Clang->getDiagnosticOpts());
482
483 VFS =
484 createVFSFromCompilerInvocation(Clang->getInvocation(), Diagnostics, VFS);
485
486 // Create a file manager object to provide access to and cache the filesystem.
487 Clang->setFileManager(new FileManager(Clang->getFileSystemOpts(), VFS));
488
489 // Create the source manager.
490 Clang->setSourceManager(
491 new SourceManager(Diagnostics, Clang->getFileManager()));
492
493 auto PreambleDepCollector = std::make_shared<PreambleDependencyCollector>();
494 Clang->addDependencyCollector(PreambleDepCollector);
495
496 Clang->getLangOpts().CompilingPCH = true;
497
498 // Remap the main source file to the preamble buffer.
499 StringRef MainFilePath = FrontendOpts.Inputs[0].getFile();
500 auto PreambleInputBuffer = llvm::MemoryBuffer::getMemBufferCopy(
501 MainFileBuffer->getBuffer().slice(0, Bounds.Size), MainFilePath);
502 if (PreprocessorOpts.RetainRemappedFileBuffers) {
503 // MainFileBuffer will be deleted by unique_ptr after leaving the method.
504 PreprocessorOpts.addRemappedFile(MainFilePath, PreambleInputBuffer.get());
505 } else {
506 // In that case, remapped buffer will be deleted by CompilerInstance on
507 // BeginSourceFile, so we call release() to avoid double deletion.
508 PreprocessorOpts.addRemappedFile(MainFilePath,
509 PreambleInputBuffer.release());
510 }
511
512 auto Act = std::make_unique<PrecompilePreambleAction>(
513 std::move(Buffer),
514 /*WritePCHFile=*/Storage->getKind() == PCHStorage::Kind::TempFile,
515 Callbacks);
516 if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0]))
518
519 // Performed after BeginSourceFile to ensure Clang->Preprocessor can be
520 // referenced in the callback.
521 Callbacks.BeforeExecute(*Clang);
522
523 std::unique_ptr<PPCallbacks> DelegatedPPCallbacks =
524 Callbacks.createPPCallbacks();
525 if (DelegatedPPCallbacks)
526 Clang->getPreprocessor().addPPCallbacks(std::move(DelegatedPPCallbacks));
527 if (auto CommentHandler = Callbacks.getCommentHandler())
528 Clang->getPreprocessor().addCommentHandler(CommentHandler);
529 llvm::StringSet<> MissingFiles;
530 Clang->getPreprocessor().addPPCallbacks(
531 std::make_unique<MissingFileCollector>(
532 MissingFiles, Clang->getPreprocessor().getHeaderSearchInfo(),
533 Clang->getSourceManager()));
534
535 if (llvm::Error Err = Act->Execute())
536 return errorToErrorCode(std::move(Err));
537
538 // Run the callbacks.
539 Callbacks.AfterExecute(*Clang);
540
541 Act->EndSourceFile();
542
543 if (!Act->hasEmittedPreamblePCH())
545 Act.reset(); // Frees the PCH buffer, unless Storage keeps it in memory.
546
547 // Keep track of all of the files that the source manager knows about,
548 // so we can verify whether they have changed or not.
549 llvm::StringMap<PrecompiledPreamble::PreambleFileHash> FilesInPreamble;
550
551 SourceManager &SourceMgr = Clang->getSourceManager();
552 for (auto &Filename : PreambleDepCollector->getDependencies()) {
553 auto MaybeFile = Clang->getFileManager().getOptionalFileRef(Filename);
554 if (!MaybeFile ||
555 MaybeFile == SourceMgr.getFileEntryRefForID(SourceMgr.getMainFileID()))
556 continue;
557 auto File = *MaybeFile;
558 if (time_t ModTime = File.getModificationTime()) {
559 FilesInPreamble[File.getName()] =
560 PrecompiledPreamble::PreambleFileHash::createForFile(File.getSize(),
561 ModTime);
562 } else {
563 llvm::MemoryBufferRef Buffer =
565 FilesInPreamble[File.getName()] =
566 PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(Buffer);
567 }
568 }
569
570 // Shrinking the storage requires extra temporary memory.
571 // Destroying clang first reduces peak memory usage.
572 CICleanup.unregister();
573 Clang.reset();
574 Storage->shrink();
575 return PrecompiledPreamble(
576 std::move(Storage), std::move(PreambleBytes), PreambleEndsAtStartOfLine,
577 std::move(FilesInPreamble), std::move(MissingFiles));
578}
579
581 return PreambleBounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
582}
583
584std::size_t PrecompiledPreamble::getSize() const {
585 switch (Storage->getKind()) {
586 case PCHStorage::Kind::InMemory:
587 return Storage->memoryContents().size();
588 case PCHStorage::Kind::TempFile: {
589 uint64_t Result;
590 if (llvm::sys::fs::file_size(Storage->filePath(), Result))
591 return 0;
592
593 assert(Result <= std::numeric_limits<std::size_t>::max() &&
594 "file size did not fit into size_t");
595 return Result;
596 }
597 }
598 llvm_unreachable("Unhandled storage kind");
599}
600
602 const llvm::MemoryBufferRef &MainFileBuffer,
603 PreambleBounds Bounds,
604 llvm::vfs::FileSystem &VFS) const {
605
606 assert(
607 Bounds.Size <= MainFileBuffer.getBufferSize() &&
608 "Buffer is too large. Bounds were calculated from a different buffer?");
609
610 auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
611 PreprocessorOptions &PreprocessorOpts =
612 PreambleInvocation->getPreprocessorOpts();
613
614 // We've previously computed a preamble. Check whether we have the same
615 // preamble now that we did before, and that there's enough space in
616 // the main-file buffer within the precompiled preamble to fit the
617 // new main file.
618 if (PreambleBytes.size() != Bounds.Size ||
619 PreambleEndsAtStartOfLine != Bounds.PreambleEndsAtStartOfLine ||
620 !std::equal(PreambleBytes.begin(), PreambleBytes.end(),
621 MainFileBuffer.getBuffer().begin()))
622 return false;
623 // The preamble has not changed. We may be able to re-use the precompiled
624 // preamble.
625
626 // Check that none of the files used by the preamble have changed.
627 // First, make a record of those files that have been overridden via
628 // remapping or unsaved_files.
629 std::map<llvm::sys::fs::UniqueID, PreambleFileHash> OverriddenFiles;
630 llvm::StringSet<> OverriddenAbsPaths; // Either by buffers or files.
631 for (const auto &R : PreprocessorOpts.RemappedFiles) {
632 llvm::vfs::Status Status;
633 if (!moveOnNoError(VFS.status(R.second), Status)) {
634 // If we can't stat the file we're remapping to, assume that something
635 // horrible happened.
636 return false;
637 }
638 // If a mapped file was previously missing, then it has changed.
639 llvm::SmallString<128> MappedPath(R.first);
640 if (!VFS.makeAbsolute(MappedPath))
641 OverriddenAbsPaths.insert(MappedPath);
642
643 OverriddenFiles[Status.getUniqueID()] = PreambleFileHash::createForFile(
644 Status.getSize(), llvm::sys::toTimeT(Status.getLastModificationTime()));
645 }
646
647 // OverridenFileBuffers tracks only the files not found in VFS.
648 llvm::StringMap<PreambleFileHash> OverridenFileBuffers;
649 for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) {
650 const PrecompiledPreamble::PreambleFileHash PreambleHash =
651 PreambleFileHash::createForMemoryBuffer(RB.second->getMemBufferRef());
652 llvm::vfs::Status Status;
653 if (moveOnNoError(VFS.status(RB.first), Status))
654 OverriddenFiles[Status.getUniqueID()] = PreambleHash;
655 else
656 OverridenFileBuffers[RB.first] = PreambleHash;
657
658 llvm::SmallString<128> MappedPath(RB.first);
659 if (!VFS.makeAbsolute(MappedPath))
660 OverriddenAbsPaths.insert(MappedPath);
661 }
662
663 // Check whether anything has changed.
664 for (const auto &F : FilesInPreamble) {
665 auto OverridenFileBuffer = OverridenFileBuffers.find(F.first());
666 if (OverridenFileBuffer != OverridenFileBuffers.end()) {
667 // The file's buffer was remapped and the file was not found in VFS.
668 // Check whether it matches up with the previous mapping.
669 if (OverridenFileBuffer->second != F.second)
670 return false;
671 continue;
672 }
673
674 llvm::vfs::Status Status;
675 if (!moveOnNoError(VFS.status(F.first()), Status)) {
676 // If the file's buffer is not remapped and we can't stat it,
677 // assume that something horrible happened.
678 return false;
679 }
680
681 std::map<llvm::sys::fs::UniqueID, PreambleFileHash>::iterator Overridden =
682 OverriddenFiles.find(Status.getUniqueID());
683 if (Overridden != OverriddenFiles.end()) {
684 // This file was remapped; check whether the newly-mapped file
685 // matches up with the previous mapping.
686 if (Overridden->second != F.second)
687 return false;
688 continue;
689 }
690
691 // Neither the file's buffer nor the file itself was remapped;
692 // check whether it has changed on disk.
693 if (Status.getSize() != uint64_t(F.second.Size) ||
694 llvm::sys::toTimeT(Status.getLastModificationTime()) !=
695 F.second.ModTime)
696 return false;
697 }
698 for (const auto &F : MissingFiles) {
699 // A missing file may be "provided" by an override buffer or file.
700 if (OverriddenAbsPaths.count(F.getKey()))
701 return false;
702 // If a file previously recorded as missing exists as a regular file, then
703 // consider the preamble out-of-date.
704 if (auto Status = VFS.status(F.getKey())) {
705 if (Status->isRegularFile())
706 return false;
707 }
708 }
709 return true;
710}
711
714 llvm::MemoryBuffer *MainFileBuffer) const {
715 PreambleBounds Bounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
716 configurePreamble(Bounds, CI, VFS, MainFileBuffer);
717}
718
721 llvm::MemoryBuffer *MainFileBuffer) const {
722 auto Bounds = ComputePreambleBounds(CI.getLangOpts(), *MainFileBuffer, 0);
723 configurePreamble(Bounds, CI, VFS, MainFileBuffer);
724}
725
727 std::unique_ptr<PCHStorage> Storage, std::vector<char> PreambleBytes,
728 bool PreambleEndsAtStartOfLine,
729 llvm::StringMap<PreambleFileHash> FilesInPreamble,
730 llvm::StringSet<> MissingFiles)
731 : Storage(std::move(Storage)), FilesInPreamble(std::move(FilesInPreamble)),
732 MissingFiles(std::move(MissingFiles)),
733 PreambleBytes(std::move(PreambleBytes)),
734 PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {
735 assert(this->Storage != nullptr);
736}
737
738PrecompiledPreamble::PreambleFileHash
739PrecompiledPreamble::PreambleFileHash::createForFile(off_t Size,
740 time_t ModTime) {
741 PreambleFileHash Result;
742 Result.Size = Size;
743 Result.ModTime = ModTime;
744 Result.MD5 = {};
745 return Result;
746}
747
748PrecompiledPreamble::PreambleFileHash
749PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(
750 const llvm::MemoryBufferRef &Buffer) {
751 PreambleFileHash Result;
752 Result.Size = Buffer.getBufferSize();
753 Result.ModTime = 0;
754
755 llvm::MD5 MD5Ctx;
756 MD5Ctx.update(Buffer.getBuffer().data());
757 MD5Ctx.final(Result.MD5);
758
759 return Result;
760}
761
762void PrecompiledPreamble::configurePreamble(
765 llvm::MemoryBuffer *MainFileBuffer) const {
766 assert(VFS);
767
768 auto &PreprocessorOpts = CI.getPreprocessorOpts();
769
770 // Remap main file to point to MainFileBuffer.
771 auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
772 PreprocessorOpts.addRemappedFile(MainFilePath, MainFileBuffer);
773
774 // Configure ImpicitPCHInclude.
775 PreprocessorOpts.PrecompiledPreambleBytes.first = Bounds.Size;
776 PreprocessorOpts.PrecompiledPreambleBytes.second =
778 PreprocessorOpts.DisablePCHOrModuleValidation =
780
781 // Don't bother generating the long version of the predefines buffer.
782 // The preamble is going to overwrite it anyway.
783 PreprocessorOpts.UsePredefines = false;
784
785 setupPreambleStorage(*Storage, PreprocessorOpts, VFS);
786}
787
788void PrecompiledPreamble::setupPreambleStorage(
789 const PCHStorage &Storage, PreprocessorOptions &PreprocessorOpts,
791 if (Storage.getKind() == PCHStorage::Kind::TempFile) {
792 llvm::StringRef PCHPath = Storage.filePath();
793 PreprocessorOpts.ImplicitPCHInclude = PCHPath.str();
794
795 // Make sure we can access the PCH file even if we're using a VFS
797 llvm::vfs::getRealFileSystem();
798 if (VFS == RealFS || VFS->exists(PCHPath))
799 return;
800 auto Buf = RealFS->getBufferForFile(PCHPath);
801 if (!Buf) {
802 // We can't read the file even from RealFS, this is clearly an error,
803 // but we'll just leave the current VFS as is and let clang's code
804 // figure out what to do with missing PCH.
805 return;
806 }
807
808 // We have a slight inconsistency here -- we're using the VFS to
809 // read files, but the PCH was generated in the real file system.
810 VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(*Buf), VFS);
811 } else {
812 assert(Storage.getKind() == PCHStorage::Kind::InMemory);
813 // For in-memory preamble, we have to provide a VFS overlay that makes it
814 // accessible.
815 StringRef PCHPath = getInMemoryPreamblePath();
816 PreprocessorOpts.ImplicitPCHInclude = std::string(PCHPath);
817
818 auto Buf = llvm::MemoryBuffer::getMemBuffer(
819 Storage.memoryContents(), PCHPath, /*RequiresNullTerminator=*/false);
820 VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(Buf), VFS);
821 }
822}
823
828std::unique_ptr<PPCallbacks> PreambleCallbacks::createPPCallbacks() {
829 return nullptr;
830}
832
833static llvm::ManagedStatic<BuildPreambleErrorCategory> BuildPreambleErrCategory;
834
836 return std::error_code(static_cast<int>(Error), *BuildPreambleErrCategory);
837}
838
839const char *BuildPreambleErrorCategory::name() const noexcept {
840 return "build-preamble.error";
841}
842
843std::string BuildPreambleErrorCategory::message(int condition) const {
844 switch (static_cast<BuildPreambleError>(condition)) {
846 return "Could not create temporary file for PCH";
848 return "CreateTargetInfo() return null";
850 return "BeginSourceFile() return an error";
852 return "Could not emit PCH";
854 return "Command line arguments must contain exactly one source file";
855 }
856 llvm_unreachable("unexpected BuildPreambleError");
857}
static bool moveOnNoError(llvm::ErrorOr< T > Val, T &Output)
Definition: ASTUnit.cpp:148
#define SM(sm)
Definition: Cuda.cpp:83
Defines the clang::FileManager interface and associated types.
StringRef Filename
Definition: Format.cpp:2975
llvm::MachO::FileType FileType
Definition: MachO.h:45
static llvm::ManagedStatic< BuildPreambleErrorCategory > BuildPreambleErrCategory
Defines the clang::Preprocessor interface.
static std::unique_ptr< PCHStorage > file(std::unique_ptr< TempPCHFile > File)
static std::unique_ptr< PCHStorage > inMemory(std::shared_ptr< PCHBuffer > Buf)
virtual bool HandleTopLevelDecl(DeclGroupRef D)
HandleTopLevelDecl - Handle the specified top-level declaration.
Definition: ASTConsumer.cpp:18
virtual bool shouldSkipFunctionBody(Decl *D)
This callback is called for each function if the Parser was initialized with SkipFunctionBodies set t...
Definition: ASTConsumer.h:145
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:185
Abstract base class to use for AST consumer-based frontend actions.
Writes an AST file containing the contents of a translation unit.
Definition: ASTWriter.h:90
std::string message(int condition) const override
const char * name() const noexcept override
Represents a character-granular source range.
Abstract base class that describes a handler that will receive source ranges for each of the comments...
CompilerInstance - Helper class for managing a single instance of the Clang compiler.
InMemoryModuleCache & getModuleCache() const
Preprocessor & getPreprocessor() const
Return the current preprocessor.
FrontendOptions & getFrontendOpts()
LangOptions & getLangOpts()
Helper class for holding the data necessary to invoke the compiler.
PreprocessorOptions & getPreprocessorOpts()
LangOptions & getLangOpts()
Mutable getters.
FrontendOptions & getFrontendOpts()
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:86
An interface for collecting the dependencies of a compilation.
Definition: Utils.h:63
virtual bool needSystemDependencies()
Return true if system files should be passed to sawDependency().
Definition: Utils.h:82
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
void Reset(bool soft=false)
Reset the state of the diagnostic object to its initial configuration.
Definition: Diagnostic.cpp:118
A reference to a DirectoryEntry that includes the name of the directory as it was accessed by the Fil...
Implements support for file system lookup, file system caching, and directory search management.
Definition: FileManager.h:53
OptionalFileEntryRef getOptionalFileRef(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Get a FileEntryRef if it exists, without doing anything on error.
Definition: FileManager.h:240
virtual bool shouldEraseOutputFiles()
Callback at the end of processing a single input, to determine if the output files should be erased o...
virtual std::unique_ptr< ASTConsumer > CreateASTConsumer(CompilerInstance &CI, StringRef InFile)=0
Create the AST consumer object for this action, if supported.
virtual bool hasASTFileSupport() const
Does this action support use with AST files?
virtual bool BeginSourceFileAction(CompilerInstance &CI)
Callback at the start of processing a single input.
virtual TranslationUnitKind getTranslationUnitKind()
For AST-based actions, the kind of translation unit we're handling.
virtual bool hasCodeCompletionSupport() const
Does this action support use with code completion?
FrontendOptions - Options for controlling the behavior of the frontend.
std::string OutputFile
The output file, if any.
unsigned RelocatablePCH
When generating PCH files, instruct the AST writer to create relocatable PCH files.
SmallVector< FrontendInputFile, 0 > Inputs
The input files and their types.
frontend::ActionKind ProgramAction
The frontend action to perform.
static std::unique_ptr< llvm::raw_pwrite_stream > CreateOutputFile(CompilerInstance &CI, StringRef InFile, std::string &OutputFile)
Creates file to write the PCH into and returns a stream to write it into.
static bool ComputeASTConsumerArguments(CompilerInstance &CI, std::string &Sysroot)
Compute the AST consumer arguments that will be used to create the PCHGenerator instance returned by ...
Encapsulates the information needed to find the file referenced by a #include or #include_next,...
Definition: HeaderSearch.h:253
ConstSearchDirIterator angled_dir_begin() const
Definition: HeaderSearch.h:877
SearchDirIterator search_dir_end()
Definition: HeaderSearch.h:857
SearchDirIterator search_dir_begin()
Definition: HeaderSearch.h:856
In-memory cache for modules.
Record the location of an inclusion directive, such as an #include or #import statement.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:461
static PreambleBounds ComputePreamble(StringRef Buffer, const LangOptions &LangOpts, unsigned MaxLines=0)
Compute the preamble of the given file.
Definition: Lexer.cpp:637
An abstract superclass that describes a custom extension to the module/precompiled header file format...
Describes a module or submodule.
Definition: Module.h:105
AST and semantic-analysis consumer that generates a precompiled header from the parsed source code.
Definition: ASTWriter.h:906
void HandleTranslationUnit(ASTContext &Ctx) override
HandleTranslationUnit - This method is called when the ASTs for entire translation unit have been par...
Definition: GeneratePCH.cpp:59
bool hasEmittedPCH() const
Definition: ASTWriter.h:949
ASTWriter & getWriter()
Definition: ASTWriter.h:920
This interface provides a way to observe the actions of the preprocessor as it does its thing.
Definition: PPCallbacks.h:35
A set of callbacks to gather useful information while building a preamble.
virtual void AfterPCHEmitted(ASTWriter &Writer)
Called after PCH has been emitted.
virtual void BeforeExecute(CompilerInstance &CI)
Called before FrontendAction::Execute.
virtual CommentHandler * getCommentHandler()
The returned CommentHandler will be added to the preprocessor if not null.
virtual void HandleTopLevelDecl(DeclGroupRef DG)
Called for each TopLevelDecl.
virtual std::unique_ptr< PPCallbacks > createPPCallbacks()
Creates wrapper class for PPCallbacks so we can also process information about includes that are insi...
virtual void AfterExecute(CompilerInstance &CI)
Called after FrontendAction::Execute(), but before FrontendAction::EndSourceFile().
A class holding a PCH and all information to check whether it is valid to reuse the PCH for the subse...
void OverridePreamble(CompilerInvocation &CI, IntrusiveRefCntPtr< llvm::vfs::FileSystem > &VFS, llvm::MemoryBuffer *MainFileBuffer) const
Configure CI to use this preamble.
PrecompiledPreamble & operator=(PrecompiledPreamble &&)
static llvm::ErrorOr< PrecompiledPreamble > Build(const CompilerInvocation &Invocation, const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds, DiagnosticsEngine &Diagnostics, IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, std::shared_ptr< PCHContainerOperations > PCHContainerOps, bool StoreInMemory, StringRef StoragePath, PreambleCallbacks &Callbacks)
Try to build PrecompiledPreamble for Invocation.
bool CanReuse(const CompilerInvocation &Invocation, const llvm::MemoryBufferRef &MainFileBuffer, PreambleBounds Bounds, llvm::vfs::FileSystem &VFS) const
Check whether PrecompiledPreamble can be reused for the new contents(MainFileBuffer) of the main file...
void AddImplicitPreamble(CompilerInvocation &CI, IntrusiveRefCntPtr< llvm::vfs::FileSystem > &VFS, llvm::MemoryBuffer *MainFileBuffer) const
Changes options inside CI to use PCH from this preamble.
std::size_t getSize() const
Returns the size, in bytes, that preamble takes on disk or in memory.
PreambleBounds getBounds() const
PreambleBounds used to build the preamble.
PrecompiledPreamble(PrecompiledPreamble &&)
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
std::vector< std::pair< std::string, std::string > > RemappedFiles
The set of file remappings, which take existing files on the system (the first part of each pair) and...
std::pair< unsigned, bool > PrecompiledPreambleBytes
If non-zero, the implicit PCH include is actually a precompiled preamble that covers this number of b...
bool RetainRemappedFileBuffers
Whether the compiler instance should retain (i.e., not free) the buffers associated with remapped fil...
std::string ImplicitPCHInclude
The implicit PCH included at the start of the translation unit, or empty.
void addRemappedFile(StringRef From, StringRef To)
bool GeneratePreamble
True indicates that a preamble is being generated.
std::vector< std::pair< std::string, llvm::MemoryBuffer * > > RemappedFileBuffers
The set of file-to-buffer remappings, which take existing files on the system (the first part of each...
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:128
Encodes a location in the source.
This class handles loading and caching of source files into memory.
OptionalFileEntryRef getFileEntryRefForID(FileID FID) const
Returns the FileEntryRef for the provided FileID.
FileManager & getFileManager() const
FileID getMainFileID() const
Returns the FileID of the main source file.
llvm::MemoryBufferRef getMemoryBufferForFileOrFake(FileEntryRef File)
Retrieve the memory buffer associated with the given file.
Token - This structure provides full information about a lexed token.
Definition: Token.h:36
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:132
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
Definition: SourceManager.h:81
@ GeneratePCH
Generate pre-compiled header.
std::unique_ptr< DiagnosticConsumer > create(StringRef OutputFile, DiagnosticOptions *Diags, bool MergeChildRecords=false)
Returns a DiagnosticConsumer that serializes diagnostics to a bitcode file.
@ VFS
Remove unused -ivfsoverlay arguments.
The JSON file list parser is used to communicate input to InstallAPI.
void ProcessWarningOptions(DiagnosticsEngine &Diags, const DiagnosticOptions &Opts, bool ReportDiags=true)
ProcessWarningOptions - Initialize the diagnostic client and process the warning options specified on...
Definition: Warnings.cpp:44
std::error_code make_error_code(BuildPreambleError Error)
@ Result
The result type of a method or function.
IntrusiveRefCntPtr< llvm::vfs::FileSystem > createVFSFromCompilerInvocation(const CompilerInvocation &CI, DiagnosticsEngine &Diags)
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:1033
@ TU_Prefix
The translation unit is a prefix to a translation unit, and is not complete.
Definition: LangOptions.h:1039
const FunctionProtoType * T
@ PCH
Disable validation for a precompiled header and the modules it depends on.
PreambleBounds ComputePreambleBounds(const LangOptions &LangOpts, const llvm::MemoryBufferRef &Buffer, unsigned MaxLines)
Runs lexer to compute suggested preamble bounds.
Definition: Format.h:5427
#define true
Definition: stdbool.h:25
Describes the bounds (start, size) of the preamble and a flag required by PreprocessorOptions::Precom...
Definition: Lexer.h:60
unsigned Size
Size of the preamble in bytes.
Definition: Lexer.h:62
bool PreambleEndsAtStartOfLine
Whether the preamble ends at the start of a new line.
Definition: Lexer.h:68