clang 20.0.0git
PrecompiledPreamble.cpp
Go to the documentation of this file.
1//===--- PrecompiledPreamble.cpp - Build precompiled preambles --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Helper class to build precompiled preamble.
10//
11//===----------------------------------------------------------------------===//
12
21#include "clang/Lex/Lexer.h"
25#include "llvm/ADT/SmallString.h"
26#include "llvm/ADT/StringSet.h"
27#include "llvm/ADT/iterator_range.h"
28#include "llvm/Config/llvm-config.h"
29#include "llvm/Support/CrashRecoveryContext.h"
30#include "llvm/Support/FileSystem.h"
31#include "llvm/Support/ManagedStatic.h"
32#include "llvm/Support/Path.h"
33#include "llvm/Support/Process.h"
34#include "llvm/Support/VirtualFileSystem.h"
35#include <limits>
36#include <mutex>
37#include <utility>
38
39using namespace clang;
40
41namespace {
42
43StringRef getInMemoryPreamblePath() {
44#if defined(LLVM_ON_UNIX)
45 return "/__clang_tmp/___clang_inmemory_preamble___";
46#elif defined(_WIN32)
47 return "C:\\__clang_tmp\\___clang_inmemory_preamble___";
48#else
49#warning "Unknown platform. Defaulting to UNIX-style paths for in-memory PCHs"
50 return "/__clang_tmp/___clang_inmemory_preamble___";
51#endif
52}
53
55createVFSOverlayForPreamblePCH(StringRef PCHFilename,
56 std::unique_ptr<llvm::MemoryBuffer> PCHBuffer,
58 // We want only the PCH file from the real filesystem to be available,
59 // so we create an in-memory VFS with just that and overlay it on top.
61 new llvm::vfs::InMemoryFileSystem());
62 PCHFS->addFile(PCHFilename, 0, std::move(PCHBuffer));
64 new llvm::vfs::OverlayFileSystem(VFS));
65 Overlay->pushOverlay(PCHFS);
66 return Overlay;
67}
68
69class PreambleDependencyCollector : public DependencyCollector {
70public:
71 // We want to collect all dependencies for correctness. Avoiding the real
72 // system dependencies (e.g. stl from /usr/lib) would probably be a good idea,
73 // but there is no way to distinguish between those and the ones that can be
74 // spuriously added by '-isystem' (e.g. to suppress warnings from those
75 // headers).
76 bool needSystemDependencies() override { return true; }
77};
78
79// Collects files whose existence would invalidate the preamble.
80// Collecting *all* of these would make validating it too slow though, so we
81// just find all the candidates for 'file not found' diagnostics.
82//
83// A caveat that may be significant for generated files: we'll omit files under
84// search path entries whose roots don't exist when the preamble is built.
85// These are pruned by InitHeaderSearch and so we don't see the search path.
86// It would be nice to include them but we don't want to duplicate all the rest
87// of the InitHeaderSearch logic to reconstruct them.
88class MissingFileCollector : public PPCallbacks {
89 llvm::StringSet<> &Out;
90 const HeaderSearch &Search;
91 const SourceManager &SM;
92
93public:
94 MissingFileCollector(llvm::StringSet<> &Out, const HeaderSearch &Search,
95 const SourceManager &SM)
96 : Out(Out), Search(Search), SM(SM) {}
97
98 void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
99 StringRef FileName, bool IsAngled,
100 CharSourceRange FilenameRange,
101 OptionalFileEntryRef File, StringRef SearchPath,
102 StringRef RelativePath, const Module *SuggestedModule,
103 bool ModuleImported,
105 // File is std::nullopt if it wasn't found.
106 // (We have some false negatives if PP recovered e.g. <foo> -> "foo")
107 if (File)
108 return;
109
110 // If it's a rare absolute include, we know the full path already.
111 if (llvm::sys::path::is_absolute(FileName)) {
112 Out.insert(FileName);
113 return;
114 }
115
116 // Reconstruct the filenames that would satisfy this directive...
118 auto NotFoundRelativeTo = [&](DirectoryEntryRef DE) {
119 Buf = DE.getName();
120 llvm::sys::path::append(Buf, FileName);
121 llvm::sys::path::remove_dots(Buf, /*remove_dot_dot=*/true);
122 Out.insert(Buf);
123 };
124 // ...relative to the including file.
125 if (!IsAngled) {
126 if (OptionalFileEntryRef IncludingFile =
127 SM.getFileEntryRefForID(SM.getFileID(IncludeTok.getLocation())))
128 if (IncludingFile->getDir())
129 NotFoundRelativeTo(IncludingFile->getDir());
130 }
131 // ...relative to the search paths.
132 for (const auto &Dir : llvm::make_range(
133 IsAngled ? Search.angled_dir_begin() : Search.search_dir_begin(),
134 Search.search_dir_end())) {
135 // No support for frameworks or header maps yet.
136 if (Dir.isNormalDir())
137 NotFoundRelativeTo(*Dir.getDirRef());
138 }
139 }
140};
141
142/// Keeps a track of files to be deleted in destructor.
143class TemporaryFiles {
144public:
145 // A static instance to be used by all clients.
146 static TemporaryFiles &getInstance();
147
148private:
149 // Disallow constructing the class directly.
150 TemporaryFiles() = default;
151 // Disallow copy.
152 TemporaryFiles(const TemporaryFiles &) = delete;
153
154public:
155 ~TemporaryFiles();
156
157 /// Adds \p File to a set of tracked files.
158 void addFile(StringRef File);
159
160 /// Remove \p File from disk and from the set of tracked files.
161 void removeFile(StringRef File);
162
163private:
164 std::mutex Mutex;
165 llvm::StringSet<> Files;
166};
167
168TemporaryFiles &TemporaryFiles::getInstance() {
169 static TemporaryFiles Instance;
170 return Instance;
171}
172
173TemporaryFiles::~TemporaryFiles() {
174 std::lock_guard<std::mutex> Guard(Mutex);
175 for (const auto &File : Files)
176 llvm::sys::fs::remove(File.getKey());
177}
178
179void TemporaryFiles::addFile(StringRef File) {
180 std::lock_guard<std::mutex> Guard(Mutex);
181 auto IsInserted = Files.insert(File).second;
182 (void)IsInserted;
183 assert(IsInserted && "File has already been added");
184}
185
186void TemporaryFiles::removeFile(StringRef File) {
187 std::lock_guard<std::mutex> Guard(Mutex);
188 auto WasPresent = Files.erase(File);
189 (void)WasPresent;
190 assert(WasPresent && "File was not tracked");
191 llvm::sys::fs::remove(File);
192}
193
194// A temp file that would be deleted on destructor call. If destructor is not
195// called for any reason, the file will be deleted at static objects'
196// destruction.
197// An assertion will fire if two TempPCHFiles are created with the same name,
198// so it's not intended to be used outside preamble-handling.
199class TempPCHFile {
200public:
201 // A main method used to construct TempPCHFile.
202 static std::unique_ptr<TempPCHFile> create(StringRef StoragePath) {
203 // FIXME: This is a hack so that we can override the preamble file during
204 // crash-recovery testing, which is the only case where the preamble files
205 // are not necessarily cleaned up.
206 if (const char *TmpFile = ::getenv("CINDEXTEST_PREAMBLE_FILE"))
207 return std::unique_ptr<TempPCHFile>(new TempPCHFile(TmpFile));
208
210 // Using the versions of createTemporaryFile() and
211 // createUniqueFile() with a file descriptor guarantees
212 // that we would never get a race condition in a multi-threaded setting
213 // (i.e., multiple threads getting the same temporary path).
214 int FD;
215 std::error_code EC;
216 if (StoragePath.empty())
217 EC = llvm::sys::fs::createTemporaryFile("preamble", "pch", FD, File);
218 else {
219 llvm::SmallString<128> TempPath = StoragePath;
220 // Use the same filename model as fs::createTemporaryFile().
221 llvm::sys::path::append(TempPath, "preamble-%%%%%%.pch");
222 namespace fs = llvm::sys::fs;
223 // Use the same owner-only file permissions as fs::createTemporaryFile().
224 EC = fs::createUniqueFile(TempPath, FD, File, fs::OF_None,
225 fs::owner_read | fs::owner_write);
226 }
227 if (EC)
228 return nullptr;
229 // We only needed to make sure the file exists, close the file right away.
230 llvm::sys::Process::SafelyCloseFileDescriptor(FD);
231 return std::unique_ptr<TempPCHFile>(new TempPCHFile(File.str().str()));
232 }
233
234 TempPCHFile &operator=(const TempPCHFile &) = delete;
235 TempPCHFile(const TempPCHFile &) = delete;
236 ~TempPCHFile() { TemporaryFiles::getInstance().removeFile(FilePath); };
237
238 /// A path where temporary file is stored.
239 llvm::StringRef getFilePath() const { return FilePath; };
240
241private:
242 TempPCHFile(std::string FilePath) : FilePath(std::move(FilePath)) {
243 TemporaryFiles::getInstance().addFile(this->FilePath);
244 }
245
246 std::string FilePath;
247};
248
249class PrecompilePreambleAction : public ASTFrontendAction {
250public:
251 PrecompilePreambleAction(std::shared_ptr<PCHBuffer> Buffer, bool WritePCHFile,
252 PreambleCallbacks &Callbacks)
253 : Buffer(std::move(Buffer)), WritePCHFile(WritePCHFile),
254 Callbacks(Callbacks) {}
255
256 std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
257 StringRef InFile) override;
258
259 bool hasEmittedPreamblePCH() const { return HasEmittedPreamblePCH; }
260
261 void setEmittedPreamblePCH(ASTWriter &Writer) {
262 if (FileOS) {
263 *FileOS << Buffer->Data;
264 // Make sure it hits disk now.
265 FileOS.reset();
266 }
267
268 this->HasEmittedPreamblePCH = true;
269 Callbacks.AfterPCHEmitted(Writer);
270 }
271
272 bool BeginSourceFileAction(CompilerInstance &CI) override {
273 assert(CI.getLangOpts().CompilingPCH);
275 }
276
277 bool shouldEraseOutputFiles() override { return !hasEmittedPreamblePCH(); }
278 bool hasCodeCompletionSupport() const override { return false; }
279 bool hasASTFileSupport() const override { return false; }
281
282private:
283 friend class PrecompilePreambleConsumer;
284
285 bool HasEmittedPreamblePCH = false;
286 std::shared_ptr<PCHBuffer> Buffer;
287 bool WritePCHFile; // otherwise the PCH is written into the PCHBuffer only.
288 std::unique_ptr<llvm::raw_pwrite_stream> FileOS; // null if in-memory
289 PreambleCallbacks &Callbacks;
290};
291
292class PrecompilePreambleConsumer : public PCHGenerator {
293public:
294 PrecompilePreambleConsumer(PrecompilePreambleAction &Action, Preprocessor &PP,
295 InMemoryModuleCache &ModuleCache,
296 StringRef isysroot,
297 std::shared_ptr<PCHBuffer> Buffer)
298 : PCHGenerator(PP, ModuleCache, "", isysroot, std::move(Buffer),
299 ArrayRef<std::shared_ptr<ModuleFileExtension>>(),
300 /*AllowASTWithErrors=*/true),
301 Action(Action) {}
302
303 bool HandleTopLevelDecl(DeclGroupRef DG) override {
304 Action.Callbacks.HandleTopLevelDecl(DG);
305 return true;
306 }
307
308 void HandleTranslationUnit(ASTContext &Ctx) override {
310 if (!hasEmittedPCH())
311 return;
312 Action.setEmittedPreamblePCH(getWriter());
313 }
314
315 bool shouldSkipFunctionBody(Decl *D) override {
316 return Action.Callbacks.shouldSkipFunctionBody(D);
317 }
318
319private:
320 PrecompilePreambleAction &Action;
321};
322
323std::unique_ptr<ASTConsumer>
324PrecompilePreambleAction::CreateASTConsumer(CompilerInstance &CI,
325 StringRef InFile) {
326 std::string Sysroot;
328 return nullptr;
329
330 if (WritePCHFile) {
331 std::string OutputFile; // unused
332 FileOS = GeneratePCHAction::CreateOutputFile(CI, InFile, OutputFile);
333 if (!FileOS)
334 return nullptr;
335 }
336
338 Sysroot.clear();
339
340 return std::make_unique<PrecompilePreambleConsumer>(
341 *this, CI.getPreprocessor(), CI.getModuleCache(), Sysroot, Buffer);
342}
343
344template <class T> bool moveOnNoError(llvm::ErrorOr<T> Val, T &Output) {
345 if (!Val)
346 return false;
347 Output = std::move(*Val);
348 return true;
349}
350
351} // namespace
352
354 const llvm::MemoryBufferRef &Buffer,
355 unsigned MaxLines) {
356 return Lexer::ComputePreamble(Buffer.getBuffer(), LangOpts, MaxLines);
357}
358
360public:
361 static std::unique_ptr<PCHStorage> file(std::unique_ptr<TempPCHFile> File) {
362 assert(File);
363 std::unique_ptr<PCHStorage> S(new PCHStorage());
364 S->File = std::move(File);
365 return S;
366 }
367 static std::unique_ptr<PCHStorage> inMemory(std::shared_ptr<PCHBuffer> Buf) {
368 std::unique_ptr<PCHStorage> S(new PCHStorage());
369 S->Memory = std::move(Buf);
370 return S;
371 }
372
373 enum class Kind { InMemory, TempFile };
374 Kind getKind() const {
375 if (Memory)
376 return Kind::InMemory;
377 if (File)
378 return Kind::TempFile;
379 llvm_unreachable("Neither Memory nor File?");
380 }
381 llvm::StringRef filePath() const {
382 assert(getKind() == Kind::TempFile);
383 return File->getFilePath();
384 }
385 llvm::StringRef memoryContents() const {
386 assert(getKind() == Kind::InMemory);
387 return StringRef(Memory->Data.data(), Memory->Data.size());
388 }
389
390 // Shrink in-memory buffers to fit.
391 // This incurs a copy, but preambles tend to be long-lived.
392 // Only safe to call once nothing can alias the buffer.
393 void shrink() {
394 if (!Memory)
395 return;
396 Memory->Data = decltype(Memory->Data)(Memory->Data);
397 }
398
399private:
400 PCHStorage() = default;
401 PCHStorage(const PCHStorage &) = delete;
402 PCHStorage &operator=(const PCHStorage &) = delete;
403
404 std::shared_ptr<PCHBuffer> Memory;
405 std::unique_ptr<TempPCHFile> File;
406};
407
412
413llvm::ErrorOr<PrecompiledPreamble> PrecompiledPreamble::Build(
414 const CompilerInvocation &Invocation,
415 const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds,
416 DiagnosticsEngine &Diagnostics,
418 std::shared_ptr<PCHContainerOperations> PCHContainerOps, bool StoreInMemory,
419 StringRef StoragePath, PreambleCallbacks &Callbacks) {
420 assert(VFS && "VFS is null");
421
422 auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
423 FrontendOptions &FrontendOpts = PreambleInvocation->getFrontendOpts();
424 PreprocessorOptions &PreprocessorOpts =
425 PreambleInvocation->getPreprocessorOpts();
426
427 std::shared_ptr<PCHBuffer> Buffer = std::make_shared<PCHBuffer>();
428 std::unique_ptr<PCHStorage> Storage;
429 if (StoreInMemory) {
430 Storage = PCHStorage::inMemory(Buffer);
431 } else {
432 // Create a temporary file for the precompiled preamble. In rare
433 // circumstances, this can fail.
434 std::unique_ptr<TempPCHFile> PreamblePCHFile =
435 TempPCHFile::create(StoragePath);
436 if (!PreamblePCHFile)
438 Storage = PCHStorage::file(std::move(PreamblePCHFile));
439 }
440
441 // Save the preamble text for later; we'll need to compare against it for
442 // subsequent reparses.
443 std::vector<char> PreambleBytes(MainFileBuffer->getBufferStart(),
444 MainFileBuffer->getBufferStart() +
445 Bounds.Size);
446 bool PreambleEndsAtStartOfLine = Bounds.PreambleEndsAtStartOfLine;
447
448 // Tell the compiler invocation to generate a temporary precompiled header.
450 FrontendOpts.OutputFile = std::string(
451 StoreInMemory ? getInMemoryPreamblePath() : Storage->filePath());
452 PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
453 PreprocessorOpts.PrecompiledPreambleBytes.second = false;
454 // Inform preprocessor to record conditional stack when building the preamble.
455 PreprocessorOpts.GeneratePreamble = true;
456
457 // Create the compiler instance to use for building the precompiled preamble.
458 std::unique_ptr<CompilerInstance> Clang(
459 new CompilerInstance(std::move(PCHContainerOps)));
460
461 // Recover resources if we crash before exiting this method.
462 llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> CICleanup(
463 Clang.get());
464
465 Clang->setInvocation(std::move(PreambleInvocation));
466 Clang->setDiagnostics(&Diagnostics);
467
468 // Create the target instance.
469 if (!Clang->createTarget())
471
472 if (Clang->getFrontendOpts().Inputs.size() != 1 ||
473 Clang->getFrontendOpts().Inputs[0].getKind().getFormat() !=
475 Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() ==
478 }
479
480 // Clear out old caches and data.
481 Diagnostics.Reset();
482 ProcessWarningOptions(Diagnostics, Clang->getDiagnosticOpts());
483
484 VFS =
485 createVFSFromCompilerInvocation(Clang->getInvocation(), Diagnostics, VFS);
486
487 // Create a file manager object to provide access to and cache the filesystem.
488 Clang->setFileManager(new FileManager(Clang->getFileSystemOpts(), VFS));
489
490 // Create the source manager.
491 Clang->setSourceManager(
492 new SourceManager(Diagnostics, Clang->getFileManager()));
493
494 auto PreambleDepCollector = std::make_shared<PreambleDependencyCollector>();
495 Clang->addDependencyCollector(PreambleDepCollector);
496
497 Clang->getLangOpts().CompilingPCH = true;
498
499 // Remap the main source file to the preamble buffer.
500 StringRef MainFilePath = FrontendOpts.Inputs[0].getFile();
501 auto PreambleInputBuffer = llvm::MemoryBuffer::getMemBufferCopy(
502 MainFileBuffer->getBuffer().slice(0, Bounds.Size), MainFilePath);
503 if (PreprocessorOpts.RetainRemappedFileBuffers) {
504 // MainFileBuffer will be deleted by unique_ptr after leaving the method.
505 PreprocessorOpts.addRemappedFile(MainFilePath, PreambleInputBuffer.get());
506 } else {
507 // In that case, remapped buffer will be deleted by CompilerInstance on
508 // BeginSourceFile, so we call release() to avoid double deletion.
509 PreprocessorOpts.addRemappedFile(MainFilePath,
510 PreambleInputBuffer.release());
511 }
512
513 auto Act = std::make_unique<PrecompilePreambleAction>(
514 std::move(Buffer),
515 /*WritePCHFile=*/Storage->getKind() == PCHStorage::Kind::TempFile,
516 Callbacks);
517 if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0]))
519
520 // Performed after BeginSourceFile to ensure Clang->Preprocessor can be
521 // referenced in the callback.
522 Callbacks.BeforeExecute(*Clang);
523
524 std::unique_ptr<PPCallbacks> DelegatedPPCallbacks =
525 Callbacks.createPPCallbacks();
526 if (DelegatedPPCallbacks)
527 Clang->getPreprocessor().addPPCallbacks(std::move(DelegatedPPCallbacks));
528 if (auto CommentHandler = Callbacks.getCommentHandler())
529 Clang->getPreprocessor().addCommentHandler(CommentHandler);
530 llvm::StringSet<> MissingFiles;
531 Clang->getPreprocessor().addPPCallbacks(
532 std::make_unique<MissingFileCollector>(
533 MissingFiles, Clang->getPreprocessor().getHeaderSearchInfo(),
534 Clang->getSourceManager()));
535
536 if (llvm::Error Err = Act->Execute())
537 return errorToErrorCode(std::move(Err));
538
539 // Run the callbacks.
540 Callbacks.AfterExecute(*Clang);
541
542 Act->EndSourceFile();
543
544 if (!Act->hasEmittedPreamblePCH())
546 Act.reset(); // Frees the PCH buffer, unless Storage keeps it in memory.
547
548 // Keep track of all of the files that the source manager knows about,
549 // so we can verify whether they have changed or not.
550 llvm::StringMap<PrecompiledPreamble::PreambleFileHash> FilesInPreamble;
551
552 SourceManager &SourceMgr = Clang->getSourceManager();
553 for (auto &Filename : PreambleDepCollector->getDependencies()) {
554 auto MaybeFile = Clang->getFileManager().getOptionalFileRef(Filename);
555 if (!MaybeFile ||
556 MaybeFile == SourceMgr.getFileEntryRefForID(SourceMgr.getMainFileID()))
557 continue;
558 auto File = *MaybeFile;
559 if (time_t ModTime = File.getModificationTime()) {
560 FilesInPreamble[File.getName()] =
561 PrecompiledPreamble::PreambleFileHash::createForFile(File.getSize(),
562 ModTime);
563 } else {
564 llvm::MemoryBufferRef Buffer =
566 FilesInPreamble[File.getName()] =
567 PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(Buffer);
568 }
569 }
570
571 // Shrinking the storage requires extra temporary memory.
572 // Destroying clang first reduces peak memory usage.
573 CICleanup.unregister();
574 Clang.reset();
575 Storage->shrink();
576 return PrecompiledPreamble(
577 std::move(Storage), std::move(PreambleBytes), PreambleEndsAtStartOfLine,
578 std::move(FilesInPreamble), std::move(MissingFiles));
579}
580
582 return PreambleBounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
583}
584
585std::size_t PrecompiledPreamble::getSize() const {
586 switch (Storage->getKind()) {
587 case PCHStorage::Kind::InMemory:
588 return Storage->memoryContents().size();
589 case PCHStorage::Kind::TempFile: {
590 uint64_t Result;
591 if (llvm::sys::fs::file_size(Storage->filePath(), Result))
592 return 0;
593
594 assert(Result <= std::numeric_limits<std::size_t>::max() &&
595 "file size did not fit into size_t");
596 return Result;
597 }
598 }
599 llvm_unreachable("Unhandled storage kind");
600}
601
603 const llvm::MemoryBufferRef &MainFileBuffer,
604 PreambleBounds Bounds,
605 llvm::vfs::FileSystem &VFS) const {
606
607 assert(
608 Bounds.Size <= MainFileBuffer.getBufferSize() &&
609 "Buffer is too large. Bounds were calculated from a different buffer?");
610
611 auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
612 PreprocessorOptions &PreprocessorOpts =
613 PreambleInvocation->getPreprocessorOpts();
614
615 // We've previously computed a preamble. Check whether we have the same
616 // preamble now that we did before, and that there's enough space in
617 // the main-file buffer within the precompiled preamble to fit the
618 // new main file.
619 if (PreambleBytes.size() != Bounds.Size ||
620 PreambleEndsAtStartOfLine != Bounds.PreambleEndsAtStartOfLine ||
621 !std::equal(PreambleBytes.begin(), PreambleBytes.end(),
622 MainFileBuffer.getBuffer().begin()))
623 return false;
624 // The preamble has not changed. We may be able to re-use the precompiled
625 // preamble.
626
627 // Check that none of the files used by the preamble have changed.
628 // First, make a record of those files that have been overridden via
629 // remapping or unsaved_files.
630 std::map<llvm::sys::fs::UniqueID, PreambleFileHash> OverriddenFiles;
631 llvm::StringSet<> OverriddenAbsPaths; // Either by buffers or files.
632 for (const auto &R : PreprocessorOpts.RemappedFiles) {
633 llvm::vfs::Status Status;
634 if (!moveOnNoError(VFS.status(R.second), Status)) {
635 // If we can't stat the file we're remapping to, assume that something
636 // horrible happened.
637 return false;
638 }
639 // If a mapped file was previously missing, then it has changed.
640 llvm::SmallString<128> MappedPath(R.first);
641 if (!VFS.makeAbsolute(MappedPath))
642 OverriddenAbsPaths.insert(MappedPath);
643
644 OverriddenFiles[Status.getUniqueID()] = PreambleFileHash::createForFile(
645 Status.getSize(), llvm::sys::toTimeT(Status.getLastModificationTime()));
646 }
647
648 // OverridenFileBuffers tracks only the files not found in VFS.
649 llvm::StringMap<PreambleFileHash> OverridenFileBuffers;
650 for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) {
651 const PrecompiledPreamble::PreambleFileHash PreambleHash =
652 PreambleFileHash::createForMemoryBuffer(RB.second->getMemBufferRef());
653 llvm::vfs::Status Status;
654 if (moveOnNoError(VFS.status(RB.first), Status))
655 OverriddenFiles[Status.getUniqueID()] = PreambleHash;
656 else
657 OverridenFileBuffers[RB.first] = PreambleHash;
658
659 llvm::SmallString<128> MappedPath(RB.first);
660 if (!VFS.makeAbsolute(MappedPath))
661 OverriddenAbsPaths.insert(MappedPath);
662 }
663
664 // Check whether anything has changed.
665 for (const auto &F : FilesInPreamble) {
666 auto OverridenFileBuffer = OverridenFileBuffers.find(F.first());
667 if (OverridenFileBuffer != OverridenFileBuffers.end()) {
668 // The file's buffer was remapped and the file was not found in VFS.
669 // Check whether it matches up with the previous mapping.
670 if (OverridenFileBuffer->second != F.second)
671 return false;
672 continue;
673 }
674
675 llvm::vfs::Status Status;
676 if (!moveOnNoError(VFS.status(F.first()), Status)) {
677 // If the file's buffer is not remapped and we can't stat it,
678 // assume that something horrible happened.
679 return false;
680 }
681
682 std::map<llvm::sys::fs::UniqueID, PreambleFileHash>::iterator Overridden =
683 OverriddenFiles.find(Status.getUniqueID());
684 if (Overridden != OverriddenFiles.end()) {
685 // This file was remapped; check whether the newly-mapped file
686 // matches up with the previous mapping.
687 if (Overridden->second != F.second)
688 return false;
689 continue;
690 }
691
692 // Neither the file's buffer nor the file itself was remapped;
693 // check whether it has changed on disk.
694 if (Status.getSize() != uint64_t(F.second.Size) ||
695 llvm::sys::toTimeT(Status.getLastModificationTime()) !=
696 F.second.ModTime)
697 return false;
698 }
699 for (const auto &F : MissingFiles) {
700 // A missing file may be "provided" by an override buffer or file.
701 if (OverriddenAbsPaths.count(F.getKey()))
702 return false;
703 // If a file previously recorded as missing exists as a regular file, then
704 // consider the preamble out-of-date.
705 if (auto Status = VFS.status(F.getKey())) {
706 if (Status->isRegularFile())
707 return false;
708 }
709 }
710 return true;
711}
712
715 llvm::MemoryBuffer *MainFileBuffer) const {
716 PreambleBounds Bounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
717 configurePreamble(Bounds, CI, VFS, MainFileBuffer);
718}
719
722 llvm::MemoryBuffer *MainFileBuffer) const {
723 auto Bounds = ComputePreambleBounds(CI.getLangOpts(), *MainFileBuffer, 0);
724 configurePreamble(Bounds, CI, VFS, MainFileBuffer);
725}
726
728 std::unique_ptr<PCHStorage> Storage, std::vector<char> PreambleBytes,
729 bool PreambleEndsAtStartOfLine,
730 llvm::StringMap<PreambleFileHash> FilesInPreamble,
731 llvm::StringSet<> MissingFiles)
732 : Storage(std::move(Storage)), FilesInPreamble(std::move(FilesInPreamble)),
733 MissingFiles(std::move(MissingFiles)),
734 PreambleBytes(std::move(PreambleBytes)),
735 PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {
736 assert(this->Storage != nullptr);
737}
738
739PrecompiledPreamble::PreambleFileHash
740PrecompiledPreamble::PreambleFileHash::createForFile(off_t Size,
741 time_t ModTime) {
742 PreambleFileHash Result;
743 Result.Size = Size;
744 Result.ModTime = ModTime;
745 Result.MD5 = {};
746 return Result;
747}
748
749PrecompiledPreamble::PreambleFileHash
750PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(
751 const llvm::MemoryBufferRef &Buffer) {
752 PreambleFileHash Result;
753 Result.Size = Buffer.getBufferSize();
754 Result.ModTime = 0;
755
756 llvm::MD5 MD5Ctx;
757 MD5Ctx.update(Buffer.getBuffer().data());
758 MD5Ctx.final(Result.MD5);
759
760 return Result;
761}
762
763void PrecompiledPreamble::configurePreamble(
766 llvm::MemoryBuffer *MainFileBuffer) const {
767 assert(VFS);
768
769 auto &PreprocessorOpts = CI.getPreprocessorOpts();
770
771 // Remap main file to point to MainFileBuffer.
772 auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
773 PreprocessorOpts.addRemappedFile(MainFilePath, MainFileBuffer);
774
775 // Configure ImpicitPCHInclude.
776 PreprocessorOpts.PrecompiledPreambleBytes.first = Bounds.Size;
777 PreprocessorOpts.PrecompiledPreambleBytes.second =
779 PreprocessorOpts.DisablePCHOrModuleValidation =
781
782 // Don't bother generating the long version of the predefines buffer.
783 // The preamble is going to overwrite it anyway.
784 PreprocessorOpts.UsePredefines = false;
785
786 setupPreambleStorage(*Storage, PreprocessorOpts, VFS);
787}
788
789void PrecompiledPreamble::setupPreambleStorage(
790 const PCHStorage &Storage, PreprocessorOptions &PreprocessorOpts,
792 if (Storage.getKind() == PCHStorage::Kind::TempFile) {
793 llvm::StringRef PCHPath = Storage.filePath();
794 PreprocessorOpts.ImplicitPCHInclude = PCHPath.str();
795
796 // Make sure we can access the PCH file even if we're using a VFS
798 llvm::vfs::getRealFileSystem();
799 if (VFS == RealFS || VFS->exists(PCHPath))
800 return;
801 auto Buf = RealFS->getBufferForFile(PCHPath);
802 if (!Buf) {
803 // We can't read the file even from RealFS, this is clearly an error,
804 // but we'll just leave the current VFS as is and let clang's code
805 // figure out what to do with missing PCH.
806 return;
807 }
808
809 // We have a slight inconsistency here -- we're using the VFS to
810 // read files, but the PCH was generated in the real file system.
811 VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(*Buf), VFS);
812 } else {
813 assert(Storage.getKind() == PCHStorage::Kind::InMemory);
814 // For in-memory preamble, we have to provide a VFS overlay that makes it
815 // accessible.
816 StringRef PCHPath = getInMemoryPreamblePath();
817 PreprocessorOpts.ImplicitPCHInclude = std::string(PCHPath);
818
819 auto Buf = llvm::MemoryBuffer::getMemBuffer(
820 Storage.memoryContents(), PCHPath, /*RequiresNullTerminator=*/false);
821 VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(Buf), VFS);
822 }
823}
824
829std::unique_ptr<PPCallbacks> PreambleCallbacks::createPPCallbacks() {
830 return nullptr;
831}
833
834static llvm::ManagedStatic<BuildPreambleErrorCategory> BuildPreambleErrCategory;
835
837 return std::error_code(static_cast<int>(Error), *BuildPreambleErrCategory);
838}
839
840const char *BuildPreambleErrorCategory::name() const noexcept {
841 return "build-preamble.error";
842}
843
844std::string BuildPreambleErrorCategory::message(int condition) const {
845 switch (static_cast<BuildPreambleError>(condition)) {
847 return "Could not create temporary file for PCH";
849 return "CreateTargetInfo() return null";
851 return "BeginSourceFile() return an error";
853 return "Could not emit PCH";
855 return "Command line arguments must contain exactly one source file";
856 }
857 llvm_unreachable("unexpected BuildPreambleError");
858}
static bool moveOnNoError(llvm::ErrorOr< T > Val, T &Output)
Definition: ASTUnit.cpp:148
#define SM(sm)
Definition: Cuda.cpp:83
const Decl * D
Defines the clang::FileManager interface and associated types.
StringRef Filename
Definition: Format.cpp:3001
llvm::MachO::FileType FileType
Definition: MachO.h:46
static llvm::ManagedStatic< BuildPreambleErrorCategory > BuildPreambleErrCategory
Defines the clang::Preprocessor interface.
static std::unique_ptr< PCHStorage > file(std::unique_ptr< TempPCHFile > File)
static std::unique_ptr< PCHStorage > inMemory(std::shared_ptr< PCHBuffer > Buf)
virtual bool HandleTopLevelDecl(DeclGroupRef D)
HandleTopLevelDecl - Handle the specified top-level declaration.
Definition: ASTConsumer.cpp:18
virtual bool shouldSkipFunctionBody(Decl *D)
This callback is called for each function if the Parser was initialized with SkipFunctionBodies set t...
Definition: ASTConsumer.h:146
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:187
Abstract base class to use for AST consumer-based frontend actions.
Writes an AST file containing the contents of a translation unit.
Definition: ASTWriter.h:89
std::string message(int condition) const override
const char * name() const noexcept override
Represents a character-granular source range.
Abstract base class that describes a handler that will receive source ranges for each of the comments...
CompilerInstance - Helper class for managing a single instance of the Clang compiler.
InMemoryModuleCache & getModuleCache() const
Preprocessor & getPreprocessor() const
Return the current preprocessor.
FrontendOptions & getFrontendOpts()
LangOptions & getLangOpts()
Helper class for holding the data necessary to invoke the compiler.
PreprocessorOptions & getPreprocessorOpts()
LangOptions & getLangOpts()
Mutable getters.
FrontendOptions & getFrontendOpts()
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:86
An interface for collecting the dependencies of a compilation.
Definition: Utils.h:63
virtual bool needSystemDependencies()
Return true if system files should be passed to sawDependency().
Definition: Utils.h:82
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
void Reset(bool soft=false)
Reset the state of the diagnostic object to its initial configuration.
Definition: Diagnostic.cpp:118
A reference to a DirectoryEntry that includes the name of the directory as it was accessed by the Fil...
Implements support for file system lookup, file system caching, and directory search management.
Definition: FileManager.h:53
OptionalFileEntryRef getOptionalFileRef(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Get a FileEntryRef if it exists, without doing anything on error.
Definition: FileManager.h:240
virtual bool shouldEraseOutputFiles()
Callback at the end of processing a single input, to determine if the output files should be erased o...
virtual std::unique_ptr< ASTConsumer > CreateASTConsumer(CompilerInstance &CI, StringRef InFile)=0
Create the AST consumer object for this action, if supported.
virtual bool hasASTFileSupport() const
Does this action support use with AST files?
virtual bool BeginSourceFileAction(CompilerInstance &CI)
Callback at the start of processing a single input.
virtual TranslationUnitKind getTranslationUnitKind()
For AST-based actions, the kind of translation unit we're handling.
virtual bool hasCodeCompletionSupport() const
Does this action support use with code completion?
FrontendOptions - Options for controlling the behavior of the frontend.
std::string OutputFile
The output file, if any.
unsigned RelocatablePCH
When generating PCH files, instruct the AST writer to create relocatable PCH files.
SmallVector< FrontendInputFile, 0 > Inputs
The input files and their types.
frontend::ActionKind ProgramAction
The frontend action to perform.
static std::unique_ptr< llvm::raw_pwrite_stream > CreateOutputFile(CompilerInstance &CI, StringRef InFile, std::string &OutputFile)
Creates file to write the PCH into and returns a stream to write it into.
static bool ComputeASTConsumerArguments(CompilerInstance &CI, std::string &Sysroot)
Compute the AST consumer arguments that will be used to create the PCHGenerator instance returned by ...
Encapsulates the information needed to find the file referenced by a #include or #include_next,...
Definition: HeaderSearch.h:249
ConstSearchDirIterator angled_dir_begin() const
Definition: HeaderSearch.h:873
SearchDirIterator search_dir_end()
Definition: HeaderSearch.h:853
SearchDirIterator search_dir_begin()
Definition: HeaderSearch.h:852
In-memory cache for modules.
Record the location of an inclusion directive, such as an #include or #import statement.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:461
static PreambleBounds ComputePreamble(StringRef Buffer, const LangOptions &LangOpts, unsigned MaxLines=0)
Compute the preamble of the given file.
Definition: Lexer.cpp:637
An abstract superclass that describes a custom extension to the module/precompiled header file format...
Describes a module or submodule.
Definition: Module.h:105
AST and semantic-analysis consumer that generates a precompiled header from the parsed source code.
Definition: ASTWriter.h:920
void HandleTranslationUnit(ASTContext &Ctx) override
HandleTranslationUnit - This method is called when the ASTs for entire translation unit have been par...
Definition: GeneratePCH.cpp:59
bool hasEmittedPCH() const
Definition: ASTWriter.h:964
ASTWriter & getWriter()
Definition: ASTWriter.h:934
This interface provides a way to observe the actions of the preprocessor as it does its thing.
Definition: PPCallbacks.h:36
A set of callbacks to gather useful information while building a preamble.
virtual void AfterPCHEmitted(ASTWriter &Writer)
Called after PCH has been emitted.
virtual void BeforeExecute(CompilerInstance &CI)
Called before FrontendAction::Execute.
virtual CommentHandler * getCommentHandler()
The returned CommentHandler will be added to the preprocessor if not null.
virtual void HandleTopLevelDecl(DeclGroupRef DG)
Called for each TopLevelDecl.
virtual std::unique_ptr< PPCallbacks > createPPCallbacks()
Creates wrapper class for PPCallbacks so we can also process information about includes that are insi...
virtual void AfterExecute(CompilerInstance &CI)
Called after FrontendAction::Execute(), but before FrontendAction::EndSourceFile().
A class holding a PCH and all information to check whether it is valid to reuse the PCH for the subse...
void OverridePreamble(CompilerInvocation &CI, IntrusiveRefCntPtr< llvm::vfs::FileSystem > &VFS, llvm::MemoryBuffer *MainFileBuffer) const
Configure CI to use this preamble.
PrecompiledPreamble & operator=(PrecompiledPreamble &&)
static llvm::ErrorOr< PrecompiledPreamble > Build(const CompilerInvocation &Invocation, const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds, DiagnosticsEngine &Diagnostics, IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, std::shared_ptr< PCHContainerOperations > PCHContainerOps, bool StoreInMemory, StringRef StoragePath, PreambleCallbacks &Callbacks)
Try to build PrecompiledPreamble for Invocation.
bool CanReuse(const CompilerInvocation &Invocation, const llvm::MemoryBufferRef &MainFileBuffer, PreambleBounds Bounds, llvm::vfs::FileSystem &VFS) const
Check whether PrecompiledPreamble can be reused for the new contents(MainFileBuffer) of the main file...
void AddImplicitPreamble(CompilerInvocation &CI, IntrusiveRefCntPtr< llvm::vfs::FileSystem > &VFS, llvm::MemoryBuffer *MainFileBuffer) const
Changes options inside CI to use PCH from this preamble.
std::size_t getSize() const
Returns the size, in bytes, that preamble takes on disk or in memory.
PreambleBounds getBounds() const
PreambleBounds used to build the preamble.
PrecompiledPreamble(PrecompiledPreamble &&)
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
std::vector< std::pair< std::string, std::string > > RemappedFiles
The set of file remappings, which take existing files on the system (the first part of each pair) and...
std::pair< unsigned, bool > PrecompiledPreambleBytes
If non-zero, the implicit PCH include is actually a precompiled preamble that covers this number of b...
bool RetainRemappedFileBuffers
Whether the compiler instance should retain (i.e., not free) the buffers associated with remapped fil...
std::string ImplicitPCHInclude
The implicit PCH included at the start of the translation unit, or empty.
void addRemappedFile(StringRef From, StringRef To)
bool GeneratePreamble
True indicates that a preamble is being generated.
std::vector< std::pair< std::string, llvm::MemoryBuffer * > > RemappedFileBuffers
The set of file-to-buffer remappings, which take existing files on the system (the first part of each...
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:137
Encodes a location in the source.
This class handles loading and caching of source files into memory.
OptionalFileEntryRef getFileEntryRefForID(FileID FID) const
Returns the FileEntryRef for the provided FileID.
FileManager & getFileManager() const
FileID getMainFileID() const
Returns the FileID of the main source file.
llvm::MemoryBufferRef getMemoryBufferForFileOrFake(FileEntryRef File)
Retrieve the memory buffer associated with the given file.
Token - This structure provides full information about a lexed token.
Definition: Token.h:36
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:132
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
Definition: SourceManager.h:81
@ GeneratePCH
Generate pre-compiled header.
std::unique_ptr< DiagnosticConsumer > create(StringRef OutputFile, DiagnosticOptions *Diags, bool MergeChildRecords=false)
Returns a DiagnosticConsumer that serializes diagnostics to a bitcode file.
@ VFS
Remove unused -ivfsoverlay arguments.
The JSON file list parser is used to communicate input to InstallAPI.
void ProcessWarningOptions(DiagnosticsEngine &Diags, const DiagnosticOptions &Opts, bool ReportDiags=true)
ProcessWarningOptions - Initialize the diagnostic client and process the warning options specified on...
Definition: Warnings.cpp:44
std::error_code make_error_code(BuildPreambleError Error)
@ Result
The result type of a method or function.
IntrusiveRefCntPtr< llvm::vfs::FileSystem > createVFSFromCompilerInvocation(const CompilerInvocation &CI, DiagnosticsEngine &Diags)
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:1037
@ TU_Prefix
The translation unit is a prefix to a translation unit, and is not complete.
Definition: LangOptions.h:1043
const FunctionProtoType * T
@ PCH
Disable validation for a precompiled header and the modules it depends on.
PreambleBounds ComputePreambleBounds(const LangOptions &LangOpts, const llvm::MemoryBufferRef &Buffer, unsigned MaxLines)
Runs lexer to compute suggested preamble bounds.
#define true
Definition: stdbool.h:25
Describes the bounds (start, size) of the preamble and a flag required by PreprocessorOptions::Precom...
Definition: Lexer.h:60
unsigned Size
Size of the preamble in bytes.
Definition: Lexer.h:62
bool PreambleEndsAtStartOfLine
Whether the preamble ends at the start of a new line.
Definition: Lexer.h:68