clang  12.0.0git
PrecompiledPreamble.cpp
Go to the documentation of this file.
1 //===--- PrecompiledPreamble.cpp - Build precompiled preambles --*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Helper class to build precompiled preamble.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "clang/AST/DeclObjC.h"
17 #include "clang/Basic/TargetInfo.h"
22 #include "clang/Lex/HeaderSearch.h"
23 #include "clang/Lex/Lexer.h"
24 #include "clang/Lex/Preprocessor.h"
27 #include "llvm/ADT/SmallString.h"
28 #include "llvm/ADT/StringExtras.h"
29 #include "llvm/ADT/StringSet.h"
30 #include "llvm/ADT/iterator_range.h"
31 #include "llvm/Config/llvm-config.h"
32 #include "llvm/Support/CrashRecoveryContext.h"
33 #include "llvm/Support/FileSystem.h"
34 #include "llvm/Support/Path.h"
35 #include "llvm/Support/Process.h"
36 #include "llvm/Support/VirtualFileSystem.h"
37 #include <limits>
38 #include <mutex>
39 #include <utility>
40 
41 using namespace clang;
42 
43 namespace {
44 
45 StringRef getInMemoryPreamblePath() {
46 #if defined(LLVM_ON_UNIX)
47  return "/__clang_tmp/___clang_inmemory_preamble___";
48 #elif defined(_WIN32)
49  return "C:\\__clang_tmp\\___clang_inmemory_preamble___";
50 #else
51 #warning "Unknown platform. Defaulting to UNIX-style paths for in-memory PCHs"
52  return "/__clang_tmp/___clang_inmemory_preamble___";
53 #endif
54 }
55 
57 createVFSOverlayForPreamblePCH(StringRef PCHFilename,
58  std::unique_ptr<llvm::MemoryBuffer> PCHBuffer,
60  // We want only the PCH file from the real filesystem to be available,
61  // so we create an in-memory VFS with just that and overlay it on top.
63  new llvm::vfs::InMemoryFileSystem());
64  PCHFS->addFile(PCHFilename, 0, std::move(PCHBuffer));
66  new llvm::vfs::OverlayFileSystem(VFS));
67  Overlay->pushOverlay(PCHFS);
68  return Overlay;
69 }
70 
71 class PreambleDependencyCollector : public DependencyCollector {
72 public:
73  // We want to collect all dependencies for correctness. Avoiding the real
74  // system dependencies (e.g. stl from /usr/lib) would probably be a good idea,
75  // but there is no way to distinguish between those and the ones that can be
76  // spuriously added by '-isystem' (e.g. to suppress warnings from those
77  // headers).
78  bool needSystemDependencies() override { return true; }
79 };
80 
81 // Collects files whose existence would invalidate the preamble.
82 // Collecting *all* of these would make validating it too slow though, so we
83 // just find all the candidates for 'file not found' diagnostics.
84 //
85 // A caveat that may be significant for generated files: we'll omit files under
86 // search path entries whose roots don't exist when the preamble is built.
87 // These are pruned by InitHeaderSearch and so we don't see the search path.
88 // It would be nice to include them but we don't want to duplicate all the rest
89 // of the InitHeaderSearch logic to reconstruct them.
90 class MissingFileCollector : public PPCallbacks {
91  llvm::StringSet<> &Out;
92  const HeaderSearch &Search;
93  const SourceManager &SM;
94 
95 public:
96  MissingFileCollector(llvm::StringSet<> &Out, const HeaderSearch &Search,
97  const SourceManager &SM)
98  : Out(Out), Search(Search), SM(SM) {}
99 
100  void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
101  StringRef FileName, bool IsAngled,
102  CharSourceRange FilenameRange, const FileEntry *File,
103  StringRef SearchPath, StringRef RelativePath,
104  const Module *Imported,
105  SrcMgr::CharacteristicKind FileType) override {
106  // File is null if it wasn't found.
107  // (We have some false negatives if PP recovered e.g. <foo> -> "foo")
108  if (File != nullptr)
109  return;
110 
111  // If it's a rare absolute include, we know the full path already.
112  if (llvm::sys::path::is_absolute(FileName)) {
113  Out.insert(FileName);
114  return;
115  }
116 
117  // Reconstruct the filenames that would satisfy this directive...
119  auto NotFoundRelativeTo = [&](const DirectoryEntry *DE) {
120  Buf = DE->getName();
121  llvm::sys::path::append(Buf, FileName);
122  llvm::sys::path::remove_dots(Buf, /*remove_dot_dot=*/true);
123  Out.insert(Buf);
124  };
125  // ...relative to the including file.
126  if (!IsAngled) {
127  if (const FileEntry *IncludingFile =
128  SM.getFileEntryForID(SM.getFileID(IncludeTok.getLocation())))
129  if (IncludingFile->getDir())
130  NotFoundRelativeTo(IncludingFile->getDir());
131  }
132  // ...relative to the search paths.
133  for (const auto &Dir : llvm::make_range(
134  IsAngled ? Search.angled_dir_begin() : Search.search_dir_begin(),
135  Search.search_dir_end())) {
136  // No support for frameworks or header maps yet.
137  if (Dir.isNormalDir())
138  NotFoundRelativeTo(Dir.getDir());
139  }
140  }
141 };
142 
143 /// Keeps a track of files to be deleted in destructor.
144 class TemporaryFiles {
145 public:
146  // A static instance to be used by all clients.
147  static TemporaryFiles &getInstance();
148 
149 private:
150  // Disallow constructing the class directly.
151  TemporaryFiles() = default;
152  // Disallow copy.
153  TemporaryFiles(const TemporaryFiles &) = delete;
154 
155 public:
156  ~TemporaryFiles();
157 
158  /// Adds \p File to a set of tracked files.
159  void addFile(StringRef File);
160 
161  /// Remove \p File from disk and from the set of tracked files.
162  void removeFile(StringRef File);
163 
164 private:
165  std::mutex Mutex;
166  llvm::StringSet<> Files;
167 };
168 
169 TemporaryFiles &TemporaryFiles::getInstance() {
170  static TemporaryFiles Instance;
171  return Instance;
172 }
173 
174 TemporaryFiles::~TemporaryFiles() {
175  std::lock_guard<std::mutex> Guard(Mutex);
176  for (const auto &File : Files)
177  llvm::sys::fs::remove(File.getKey());
178 }
179 
180 void TemporaryFiles::addFile(StringRef File) {
181  std::lock_guard<std::mutex> Guard(Mutex);
182  auto IsInserted = Files.insert(File).second;
183  (void)IsInserted;
184  assert(IsInserted && "File has already been added");
185 }
186 
187 void TemporaryFiles::removeFile(StringRef File) {
188  std::lock_guard<std::mutex> Guard(Mutex);
189  auto WasPresent = Files.erase(File);
190  (void)WasPresent;
191  assert(WasPresent && "File was not tracked");
192  llvm::sys::fs::remove(File);
193 }
194 
195 class PrecompilePreambleAction : public ASTFrontendAction {
196 public:
197  PrecompilePreambleAction(std::string *InMemStorage,
198  PreambleCallbacks &Callbacks)
199  : InMemStorage(InMemStorage), Callbacks(Callbacks) {}
200 
201  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
202  StringRef InFile) override;
203 
204  bool hasEmittedPreamblePCH() const { return HasEmittedPreamblePCH; }
205 
206  void setEmittedPreamblePCH(ASTWriter &Writer) {
207  this->HasEmittedPreamblePCH = true;
208  Callbacks.AfterPCHEmitted(Writer);
209  }
210 
211  bool BeginSourceFileAction(CompilerInstance &CI) override {
212  assert(CI.getLangOpts().CompilingPCH);
214  }
215 
216  bool shouldEraseOutputFiles() override { return !hasEmittedPreamblePCH(); }
217  bool hasCodeCompletionSupport() const override { return false; }
218  bool hasASTFileSupport() const override { return false; }
219  TranslationUnitKind getTranslationUnitKind() override { return TU_Prefix; }
220 
221 private:
222  friend class PrecompilePreambleConsumer;
223 
224  bool HasEmittedPreamblePCH = false;
225  std::string *InMemStorage;
226  PreambleCallbacks &Callbacks;
227 };
228 
229 class PrecompilePreambleConsumer : public PCHGenerator {
230 public:
231  PrecompilePreambleConsumer(PrecompilePreambleAction &Action,
232  const Preprocessor &PP,
233  InMemoryModuleCache &ModuleCache,
234  StringRef isysroot,
235  std::unique_ptr<raw_ostream> Out)
236  : PCHGenerator(PP, ModuleCache, "", isysroot,
237  std::make_shared<PCHBuffer>(),
238  ArrayRef<std::shared_ptr<ModuleFileExtension>>(),
239  /*AllowASTWithErrors=*/true),
240  Action(Action), Out(std::move(Out)) {}
241 
242  bool HandleTopLevelDecl(DeclGroupRef DG) override {
243  Action.Callbacks.HandleTopLevelDecl(DG);
244  return true;
245  }
246 
247  void HandleTranslationUnit(ASTContext &Ctx) override {
249  if (!hasEmittedPCH())
250  return;
251 
252  // Write the generated bitstream to "Out".
253  *Out << getPCH();
254  // Make sure it hits disk now.
255  Out->flush();
256  // Free the buffer.
258  getPCH() = std::move(Empty);
259 
260  Action.setEmittedPreamblePCH(getWriter());
261  }
262 
263  bool shouldSkipFunctionBody(Decl *D) override {
264  return Action.Callbacks.shouldSkipFunctionBody(D);
265  }
266 
267 private:
268  PrecompilePreambleAction &Action;
269  std::unique_ptr<raw_ostream> Out;
270 };
271 
272 std::unique_ptr<ASTConsumer>
273 PrecompilePreambleAction::CreateASTConsumer(CompilerInstance &CI,
274  StringRef InFile) {
275  std::string Sysroot;
277  return nullptr;
278 
279  std::unique_ptr<llvm::raw_ostream> OS;
280  if (InMemStorage) {
281  OS = std::make_unique<llvm::raw_string_ostream>(*InMemStorage);
282  } else {
283  std::string OutputFile;
284  OS = GeneratePCHAction::CreateOutputFile(CI, InFile, OutputFile);
285  }
286  if (!OS)
287  return nullptr;
288 
290  Sysroot.clear();
291 
292  return std::make_unique<PrecompilePreambleConsumer>(
293  *this, CI.getPreprocessor(), CI.getModuleCache(), Sysroot, std::move(OS));
294 }
295 
296 template <class T> bool moveOnNoError(llvm::ErrorOr<T> Val, T &Output) {
297  if (!Val)
298  return false;
299  Output = std::move(*Val);
300  return true;
301 }
302 
303 } // namespace
304 
306  const llvm::MemoryBufferRef &Buffer,
307  unsigned MaxLines) {
308  return Lexer::ComputePreamble(Buffer.getBuffer(), LangOpts, MaxLines);
309 }
310 
311 llvm::ErrorOr<PrecompiledPreamble> PrecompiledPreamble::Build(
312  const CompilerInvocation &Invocation,
313  const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds,
314  DiagnosticsEngine &Diagnostics,
316  std::shared_ptr<PCHContainerOperations> PCHContainerOps, bool StoreInMemory,
317  PreambleCallbacks &Callbacks) {
318  assert(VFS && "VFS is null");
319 
320  auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
321  FrontendOptions &FrontendOpts = PreambleInvocation->getFrontendOpts();
322  PreprocessorOptions &PreprocessorOpts =
323  PreambleInvocation->getPreprocessorOpts();
324 
326  if (!StoreInMemory) {
327  // Create a temporary file for the precompiled preamble. In rare
328  // circumstances, this can fail.
329  llvm::ErrorOr<PrecompiledPreamble::TempPCHFile> PreamblePCHFile =
330  PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile();
331  if (!PreamblePCHFile)
333  TempFile = std::move(*PreamblePCHFile);
334  }
335 
336  PCHStorage Storage = StoreInMemory ? PCHStorage(InMemoryPreamble())
337  : PCHStorage(std::move(*TempFile));
338 
339  // Save the preamble text for later; we'll need to compare against it for
340  // subsequent reparses.
341  std::vector<char> PreambleBytes(MainFileBuffer->getBufferStart(),
342  MainFileBuffer->getBufferStart() +
343  Bounds.Size);
344  bool PreambleEndsAtStartOfLine = Bounds.PreambleEndsAtStartOfLine;
345 
346  // Tell the compiler invocation to generate a temporary precompiled header.
347  FrontendOpts.ProgramAction = frontend::GeneratePCH;
348  FrontendOpts.OutputFile =
349  std::string(StoreInMemory ? getInMemoryPreamblePath()
350  : Storage.asFile().getFilePath());
351  PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
352  PreprocessorOpts.PrecompiledPreambleBytes.second = false;
353  // Inform preprocessor to record conditional stack when building the preamble.
354  PreprocessorOpts.GeneratePreamble = true;
355 
356  // Create the compiler instance to use for building the precompiled preamble.
357  std::unique_ptr<CompilerInstance> Clang(
358  new CompilerInstance(std::move(PCHContainerOps)));
359 
360  // Recover resources if we crash before exiting this method.
361  llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> CICleanup(
362  Clang.get());
363 
364  Clang->setInvocation(std::move(PreambleInvocation));
365  Clang->setDiagnostics(&Diagnostics);
366 
367  // Create the target instance.
368  Clang->setTarget(TargetInfo::CreateTargetInfo(
369  Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
370  if (!Clang->hasTarget())
372 
373  // Inform the target of the language options.
374  //
375  // FIXME: We shouldn't need to do this, the target should be immutable once
376  // created. This complexity should be lifted elsewhere.
377  Clang->getTarget().adjust(Clang->getLangOpts());
378 
379  if (Clang->getFrontendOpts().Inputs.size() != 1 ||
380  Clang->getFrontendOpts().Inputs[0].getKind().getFormat() !=
382  Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() ==
385  }
386 
387  // Clear out old caches and data.
388  Diagnostics.Reset();
389  ProcessWarningOptions(Diagnostics, Clang->getDiagnosticOpts());
390 
391  VFS =
392  createVFSFromCompilerInvocation(Clang->getInvocation(), Diagnostics, VFS);
393 
394  // Create a file manager object to provide access to and cache the filesystem.
395  Clang->setFileManager(new FileManager(Clang->getFileSystemOpts(), VFS));
396 
397  // Create the source manager.
398  Clang->setSourceManager(
399  new SourceManager(Diagnostics, Clang->getFileManager()));
400 
401  auto PreambleDepCollector = std::make_shared<PreambleDependencyCollector>();
402  Clang->addDependencyCollector(PreambleDepCollector);
403 
404  Clang->getLangOpts().CompilingPCH = true;
405 
406  // Remap the main source file to the preamble buffer.
407  StringRef MainFilePath = FrontendOpts.Inputs[0].getFile();
408  auto PreambleInputBuffer = llvm::MemoryBuffer::getMemBufferCopy(
409  MainFileBuffer->getBuffer().slice(0, Bounds.Size), MainFilePath);
410  if (PreprocessorOpts.RetainRemappedFileBuffers) {
411  // MainFileBuffer will be deleted by unique_ptr after leaving the method.
412  PreprocessorOpts.addRemappedFile(MainFilePath, PreambleInputBuffer.get());
413  } else {
414  // In that case, remapped buffer will be deleted by CompilerInstance on
415  // BeginSourceFile, so we call release() to avoid double deletion.
416  PreprocessorOpts.addRemappedFile(MainFilePath,
417  PreambleInputBuffer.release());
418  }
419 
420  std::unique_ptr<PrecompilePreambleAction> Act;
421  Act.reset(new PrecompilePreambleAction(
422  StoreInMemory ? &Storage.asMemory().Data : nullptr, Callbacks));
423  Callbacks.BeforeExecute(*Clang);
424  if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0]))
426 
427  std::unique_ptr<PPCallbacks> DelegatedPPCallbacks =
428  Callbacks.createPPCallbacks();
429  if (DelegatedPPCallbacks)
430  Clang->getPreprocessor().addPPCallbacks(std::move(DelegatedPPCallbacks));
431  if (auto CommentHandler = Callbacks.getCommentHandler())
432  Clang->getPreprocessor().addCommentHandler(CommentHandler);
433  llvm::StringSet<> MissingFiles;
434  Clang->getPreprocessor().addPPCallbacks(
435  std::make_unique<MissingFileCollector>(
436  MissingFiles, Clang->getPreprocessor().getHeaderSearchInfo(),
437  Clang->getSourceManager()));
438 
439  if (llvm::Error Err = Act->Execute())
440  return errorToErrorCode(std::move(Err));
441 
442  // Run the callbacks.
443  Callbacks.AfterExecute(*Clang);
444 
445  Act->EndSourceFile();
446 
447  if (!Act->hasEmittedPreamblePCH())
449 
450  // Keep track of all of the files that the source manager knows about,
451  // so we can verify whether they have changed or not.
452  llvm::StringMap<PrecompiledPreamble::PreambleFileHash> FilesInPreamble;
453 
454  SourceManager &SourceMgr = Clang->getSourceManager();
455  for (auto &Filename : PreambleDepCollector->getDependencies()) {
456  auto FileOrErr = Clang->getFileManager().getFile(Filename);
457  if (!FileOrErr ||
458  *FileOrErr == SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()))
459  continue;
460  auto File = *FileOrErr;
461  if (time_t ModTime = File->getModificationTime()) {
462  FilesInPreamble[File->getName()] =
463  PrecompiledPreamble::PreambleFileHash::createForFile(File->getSize(),
464  ModTime);
465  } else {
466  llvm::MemoryBufferRef Buffer =
468  FilesInPreamble[File->getName()] =
469  PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(Buffer);
470  }
471  }
472 
473  return PrecompiledPreamble(
474  std::move(Storage), std::move(PreambleBytes), PreambleEndsAtStartOfLine,
475  std::move(FilesInPreamble), std::move(MissingFiles));
476 }
477 
479  return PreambleBounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
480 }
481 
483  switch (Storage.getKind()) {
484  case PCHStorage::Kind::Empty:
485  assert(false && "Calling getSize() on invalid PrecompiledPreamble. "
486  "Was it std::moved?");
487  return 0;
488  case PCHStorage::Kind::InMemory:
489  return Storage.asMemory().Data.size();
490  case PCHStorage::Kind::TempFile: {
491  uint64_t Result;
492  if (llvm::sys::fs::file_size(Storage.asFile().getFilePath(), Result))
493  return 0;
494 
496  "file size did not fit into size_t");
497  return Result;
498  }
499  }
500  llvm_unreachable("Unhandled storage kind");
501 }
502 
504  const llvm::MemoryBuffer *MainFileBuffer,
505  PreambleBounds Bounds,
506  llvm::vfs::FileSystem *VFS) const {
507 
508  assert(
509  Bounds.Size <= MainFileBuffer->getBufferSize() &&
510  "Buffer is too large. Bounds were calculated from a different buffer?");
511 
512  auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
513  PreprocessorOptions &PreprocessorOpts =
514  PreambleInvocation->getPreprocessorOpts();
515 
516  // We've previously computed a preamble. Check whether we have the same
517  // preamble now that we did before, and that there's enough space in
518  // the main-file buffer within the precompiled preamble to fit the
519  // new main file.
520  if (PreambleBytes.size() != Bounds.Size ||
521  PreambleEndsAtStartOfLine != Bounds.PreambleEndsAtStartOfLine ||
522  !std::equal(PreambleBytes.begin(), PreambleBytes.end(),
523  MainFileBuffer->getBuffer().begin()))
524  return false;
525  // The preamble has not changed. We may be able to re-use the precompiled
526  // preamble.
527 
528  // Check that none of the files used by the preamble have changed.
529  // First, make a record of those files that have been overridden via
530  // remapping or unsaved_files.
531  std::map<llvm::sys::fs::UniqueID, PreambleFileHash> OverriddenFiles;
532  llvm::StringSet<> OverriddenAbsPaths; // Either by buffers or files.
533  for (const auto &R : PreprocessorOpts.RemappedFiles) {
534  llvm::vfs::Status Status;
535  if (!moveOnNoError(VFS->status(R.second), Status)) {
536  // If we can't stat the file we're remapping to, assume that something
537  // horrible happened.
538  return false;
539  }
540  // If a mapped file was previously missing, then it has changed.
541  llvm::SmallString<128> MappedPath(R.first);
542  if (!VFS->makeAbsolute(MappedPath))
543  OverriddenAbsPaths.insert(MappedPath);
544 
545  OverriddenFiles[Status.getUniqueID()] = PreambleFileHash::createForFile(
546  Status.getSize(), llvm::sys::toTimeT(Status.getLastModificationTime()));
547  }
548 
549  // OverridenFileBuffers tracks only the files not found in VFS.
550  llvm::StringMap<PreambleFileHash> OverridenFileBuffers;
551  for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) {
552  const PrecompiledPreamble::PreambleFileHash PreambleHash =
553  PreambleFileHash::createForMemoryBuffer(RB.second->getMemBufferRef());
554  llvm::vfs::Status Status;
555  if (moveOnNoError(VFS->status(RB.first), Status))
556  OverriddenFiles[Status.getUniqueID()] = PreambleHash;
557  else
558  OverridenFileBuffers[RB.first] = PreambleHash;
559 
560  llvm::SmallString<128> MappedPath(RB.first);
561  if (!VFS->makeAbsolute(MappedPath))
562  OverriddenAbsPaths.insert(MappedPath);
563  }
564 
565  // Check whether anything has changed.
566  for (const auto &F : FilesInPreamble) {
567  auto OverridenFileBuffer = OverridenFileBuffers.find(F.first());
568  if (OverridenFileBuffer != OverridenFileBuffers.end()) {
569  // The file's buffer was remapped and the file was not found in VFS.
570  // Check whether it matches up with the previous mapping.
571  if (OverridenFileBuffer->second != F.second)
572  return false;
573  continue;
574  }
575 
576  llvm::vfs::Status Status;
577  if (!moveOnNoError(VFS->status(F.first()), Status)) {
578  // If the file's buffer is not remapped and we can't stat it,
579  // assume that something horrible happened.
580  return false;
581  }
582 
583  std::map<llvm::sys::fs::UniqueID, PreambleFileHash>::iterator Overridden =
584  OverriddenFiles.find(Status.getUniqueID());
585  if (Overridden != OverriddenFiles.end()) {
586  // This file was remapped; check whether the newly-mapped file
587  // matches up with the previous mapping.
588  if (Overridden->second != F.second)
589  return false;
590  continue;
591  }
592 
593  // Neither the file's buffer nor the file itself was remapped;
594  // check whether it has changed on disk.
595  if (Status.getSize() != uint64_t(F.second.Size) ||
596  llvm::sys::toTimeT(Status.getLastModificationTime()) !=
597  F.second.ModTime)
598  return false;
599  }
600  for (const auto &F : MissingFiles) {
601  // A missing file may be "provided" by an override buffer or file.
602  if (OverriddenAbsPaths.count(F.getKey()))
603  return false;
604  // If a file previously recorded as missing exists as a regular file, then
605  // consider the preamble out-of-date.
606  if (auto Status = VFS->status(F.getKey())) {
607  if (Status->isRegularFile())
608  return false;
609  }
610  }
611  return true;
612 }
613 
616  llvm::MemoryBuffer *MainFileBuffer) const {
617  PreambleBounds Bounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
618  configurePreamble(Bounds, CI, VFS, MainFileBuffer);
619 }
620 
623  llvm::MemoryBuffer *MainFileBuffer) const {
624  auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), *MainFileBuffer, 0);
625  configurePreamble(Bounds, CI, VFS, MainFileBuffer);
626 }
627 
629  PCHStorage Storage, std::vector<char> PreambleBytes,
630  bool PreambleEndsAtStartOfLine,
631  llvm::StringMap<PreambleFileHash> FilesInPreamble,
632  llvm::StringSet<> MissingFiles)
633  : Storage(std::move(Storage)), FilesInPreamble(std::move(FilesInPreamble)),
634  MissingFiles(std::move(MissingFiles)),
635  PreambleBytes(std::move(PreambleBytes)),
636  PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {
637  assert(this->Storage.getKind() != PCHStorage::Kind::Empty);
638 }
639 
640 llvm::ErrorOr<PrecompiledPreamble::TempPCHFile>
641 PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile() {
642  // FIXME: This is a hack so that we can override the preamble file during
643  // crash-recovery testing, which is the only case where the preamble files
644  // are not necessarily cleaned up.
645  if (const char *TmpFile = ::getenv("CINDEXTEST_PREAMBLE_FILE"))
646  return TempPCHFile(TmpFile);
647 
649  // Using a version of createTemporaryFile with a file descriptor guarantees
650  // that we would never get a race condition in a multi-threaded setting
651  // (i.e., multiple threads getting the same temporary path).
652  int FD;
653  auto EC = llvm::sys::fs::createTemporaryFile("preamble", "pch", FD, File);
654  if (EC)
655  return EC;
656  // We only needed to make sure the file exists, close the file right away.
657  llvm::sys::Process::SafelyCloseFileDescriptor(FD);
658  return TempPCHFile(std::string(std::move(File).str()));
659 }
660 
661 PrecompiledPreamble::TempPCHFile::TempPCHFile(std::string FilePath)
662  : FilePath(std::move(FilePath)) {
663  TemporaryFiles::getInstance().addFile(*this->FilePath);
664 }
665 
666 PrecompiledPreamble::TempPCHFile::TempPCHFile(TempPCHFile &&Other) {
667  FilePath = std::move(Other.FilePath);
668  Other.FilePath = None;
669 }
670 
671 PrecompiledPreamble::TempPCHFile &PrecompiledPreamble::TempPCHFile::
672 operator=(TempPCHFile &&Other) {
673  RemoveFileIfPresent();
674 
675  FilePath = std::move(Other.FilePath);
676  Other.FilePath = None;
677  return *this;
678 }
679 
680 PrecompiledPreamble::TempPCHFile::~TempPCHFile() { RemoveFileIfPresent(); }
681 
682 void PrecompiledPreamble::TempPCHFile::RemoveFileIfPresent() {
683  if (FilePath) {
684  TemporaryFiles::getInstance().removeFile(*FilePath);
685  FilePath = None;
686  }
687 }
688 
689 llvm::StringRef PrecompiledPreamble::TempPCHFile::getFilePath() const {
690  assert(FilePath && "TempPCHFile doesn't have a FilePath. Had it been moved?");
691  return *FilePath;
692 }
693 
694 PrecompiledPreamble::PCHStorage::PCHStorage(TempPCHFile File)
695  : StorageKind(Kind::TempFile) {
696  new (&asFile()) TempPCHFile(std::move(File));
697 }
698 
699 PrecompiledPreamble::PCHStorage::PCHStorage(InMemoryPreamble Memory)
700  : StorageKind(Kind::InMemory) {
701  new (&asMemory()) InMemoryPreamble(std::move(Memory));
702 }
703 
704 PrecompiledPreamble::PCHStorage::PCHStorage(PCHStorage &&Other) : PCHStorage() {
705  *this = std::move(Other);
706 }
707 
708 PrecompiledPreamble::PCHStorage &PrecompiledPreamble::PCHStorage::
709 operator=(PCHStorage &&Other) {
710  destroy();
711 
712  StorageKind = Other.StorageKind;
713  switch (StorageKind) {
714  case Kind::Empty:
715  // do nothing;
716  break;
717  case Kind::TempFile:
718  new (&asFile()) TempPCHFile(std::move(Other.asFile()));
719  break;
720  case Kind::InMemory:
721  new (&asMemory()) InMemoryPreamble(std::move(Other.asMemory()));
722  break;
723  }
724 
725  Other.setEmpty();
726  return *this;
727 }
728 
729 PrecompiledPreamble::PCHStorage::~PCHStorage() { destroy(); }
730 
731 PrecompiledPreamble::PCHStorage::Kind
733  return StorageKind;
734 }
735 
736 PrecompiledPreamble::TempPCHFile &PrecompiledPreamble::PCHStorage::asFile() {
737  assert(getKind() == Kind::TempFile);
738  return *reinterpret_cast<TempPCHFile *>(&Storage);
739 }
740 
741 const PrecompiledPreamble::TempPCHFile &
742 PrecompiledPreamble::PCHStorage::asFile() const {
743  return const_cast<PCHStorage *>(this)->asFile();
744 }
745 
746 PrecompiledPreamble::InMemoryPreamble &
747 PrecompiledPreamble::PCHStorage::asMemory() {
748  assert(getKind() == Kind::InMemory);
749  return *reinterpret_cast<InMemoryPreamble *>(&Storage);
750 }
751 
752 const PrecompiledPreamble::InMemoryPreamble &
753 PrecompiledPreamble::PCHStorage::asMemory() const {
754  return const_cast<PCHStorage *>(this)->asMemory();
755 }
756 
757 void PrecompiledPreamble::PCHStorage::destroy() {
758  switch (StorageKind) {
759  case Kind::Empty:
760  return;
761  case Kind::TempFile:
762  asFile().~TempPCHFile();
763  return;
764  case Kind::InMemory:
765  asMemory().~InMemoryPreamble();
766  return;
767  }
768 }
769 
770 void PrecompiledPreamble::PCHStorage::setEmpty() {
771  destroy();
772  StorageKind = Kind::Empty;
773 }
774 
775 PrecompiledPreamble::PreambleFileHash
776 PrecompiledPreamble::PreambleFileHash::createForFile(off_t Size,
777  time_t ModTime) {
778  PreambleFileHash Result;
779  Result.Size = Size;
780  Result.ModTime = ModTime;
781  Result.MD5 = {};
782  return Result;
783 }
784 
785 PrecompiledPreamble::PreambleFileHash
786 PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(
787  const llvm::MemoryBufferRef &Buffer) {
788  PreambleFileHash Result;
789  Result.Size = Buffer.getBufferSize();
790  Result.ModTime = 0;
791 
792  llvm::MD5 MD5Ctx;
793  MD5Ctx.update(Buffer.getBuffer().data());
794  MD5Ctx.final(Result.MD5);
795 
796  return Result;
797 }
798 
799 void PrecompiledPreamble::configurePreamble(
802  llvm::MemoryBuffer *MainFileBuffer) const {
803  assert(VFS);
804 
805  auto &PreprocessorOpts = CI.getPreprocessorOpts();
806 
807  // Remap main file to point to MainFileBuffer.
808  auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
809  PreprocessorOpts.addRemappedFile(MainFilePath, MainFileBuffer);
810 
811  // Configure ImpicitPCHInclude.
812  PreprocessorOpts.PrecompiledPreambleBytes.first = Bounds.Size;
813  PreprocessorOpts.PrecompiledPreambleBytes.second =
815  PreprocessorOpts.DisablePCHValidation = true;
816 
817  setupPreambleStorage(Storage, PreprocessorOpts, VFS);
818 }
819 
820 void PrecompiledPreamble::setupPreambleStorage(
821  const PCHStorage &Storage, PreprocessorOptions &PreprocessorOpts,
823  if (Storage.getKind() == PCHStorage::Kind::TempFile) {
824  const TempPCHFile &PCHFile = Storage.asFile();
825  PreprocessorOpts.ImplicitPCHInclude = std::string(PCHFile.getFilePath());
826 
827  // Make sure we can access the PCH file even if we're using a VFS
829  llvm::vfs::getRealFileSystem();
830  auto PCHPath = PCHFile.getFilePath();
831  if (VFS == RealFS || VFS->exists(PCHPath))
832  return;
833  auto Buf = RealFS->getBufferForFile(PCHPath);
834  if (!Buf) {
835  // We can't read the file even from RealFS, this is clearly an error,
836  // but we'll just leave the current VFS as is and let clang's code
837  // figure out what to do with missing PCH.
838  return;
839  }
840 
841  // We have a slight inconsistency here -- we're using the VFS to
842  // read files, but the PCH was generated in the real file system.
843  VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(*Buf), VFS);
844  } else {
845  assert(Storage.getKind() == PCHStorage::Kind::InMemory);
846  // For in-memory preamble, we have to provide a VFS overlay that makes it
847  // accessible.
848  StringRef PCHPath = getInMemoryPreamblePath();
849  PreprocessorOpts.ImplicitPCHInclude = std::string(PCHPath);
850 
851  auto Buf = llvm::MemoryBuffer::getMemBuffer(Storage.asMemory().Data);
852  VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(Buf), VFS);
853  }
854 }
855 
860 std::unique_ptr<PPCallbacks> PreambleCallbacks::createPPCallbacks() {
861  return nullptr;
862 }
864 
865 static llvm::ManagedStatic<BuildPreambleErrorCategory> BuildPreambleErrCategory;
866 
868  return std::error_code(static_cast<int>(Error), *BuildPreambleErrCategory);
869 }
870 
871 const char *BuildPreambleErrorCategory::name() const noexcept {
872  return "build-preamble.error";
873 }
874 
875 std::string BuildPreambleErrorCategory::message(int condition) const {
876  switch (static_cast<BuildPreambleError>(condition)) {
878  return "Could not create temporary file for PCH";
880  return "CreateTargetInfo() return null";
882  return "BeginSourceFile() return an error";
884  return "Could not emit PCH";
886  return "Command line arguments must contain exactly one source file";
887  }
888  llvm_unreachable("unexpected BuildPreambleError");
889 }
std::size_t getSize() const
Returns the size, in bytes, that preamble takes on disk or in memory.
std::string OutputFile
The output file, if any.
Describes the bounds (start, size) of the preamble and a flag required by PreprocessorOptions::Precom...
Definition: Lexer.h:58
LangOptions & getLangOpts()
Implements support for file system lookup, file system caching, and directory search management.
Definition: FileManager.h:53
Defines the clang::FileManager interface and associated types.
bool CanReuse(const CompilerInvocation &Invocation, const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds, llvm::vfs::FileSystem *VFS) const
Check whether PrecompiledPreamble can be reused for the new contents(MainFileBuffer) of the main file...
An interface for collecting the dependencies of a compilation.
Definition: Utils.h:76
__DEVICE__ int max(int __a, int __b)
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:89
The translation unit is a prefix to a translation unit, and is not complete.
Definition: LangOptions.h:646
InMemoryModuleCache & getModuleCache() const
llvm::MemoryBufferRef getMemoryBufferForFileOrFake(const FileEntry *File)
Retrieve the memory buffer associated with the given file.
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
void addRemappedFile(StringRef From, StringRef To)
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
Definition: SourceManager.h:78
This interface provides a way to observe the actions of the preprocessor as it does its thing.
Definition: PPCallbacks.h:35
FileManager & getFileManager() const
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:186
void HandleTranslationUnit(ASTContext &Ctx) override
HandleTranslationUnit - This method is called when the ASTs for entire translation unit have been par...
Definition: GeneratePCH.cpp:41
Definition: Format.h:3142
static bool moveOnNoError(llvm::ErrorOr< T > Val, T &Output)
Definition: ASTUnit.cpp:149
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
__SIZE_TYPE__ size_t
The unsigned integer type of the result of the sizeof operator.
Definition: opencl-c-base.h:55
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:57
Describes a module or submodule.
Definition: Module.h:96
unsigned RelocatablePCH
When generating PCH files, instruct the AST writer to create relocatable PCH files.
static bool ComputeASTConsumerArguments(CompilerInstance &CI, std::string &Sysroot)
Compute the AST consumer arguments that will be used to create the PCHGenerator instance returned by ...
static llvm::ManagedStatic< BuildPreambleErrorCategory > BuildPreambleErrCategory
std::error_code make_error_code(BuildPreambleError Error)
FrontendOptions & getFrontendOpts()
search_dir_iterator search_dir_begin() const
Definition: HeaderSearch.h:720
SmallVector< FrontendInputFile, 0 > Inputs
The input files and their types.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:191
search_dir_iterator search_dir_end() const
Definition: HeaderSearch.h:721
llvm::Error Error
bool RetainRemappedFileBuffers
Whether the compiler instance should retain (i.e., not free) the buffers associated with remapped fil...
void OverridePreamble(CompilerInvocation &CI, IntrusiveRefCntPtr< llvm::vfs::FileSystem > &VFS, llvm::MemoryBuffer *MainFileBuffer) const
Configure CI to use this preamble.
void Reset()
Reset the state of the diagnostic object to its initial configuration.
Definition: Diagnostic.cpp:133
std::vector< std::pair< std::string, llvm::MemoryBuffer * > > RemappedFileBuffers
The set of file-to-buffer remappings, which take existing files on the system (the first part of each...
Encapsulates the information needed to find the file referenced by a #include or #include_next,...
Definition: HeaderSearch.h:158
FrontendOptions & getFrontendOpts()
bool PreambleEndsAtStartOfLine
Whether the preamble ends at the start of a new line.
Definition: Lexer.h:66
A set of callbacks to gather useful information while building a preamble.
StringRef Filename
Definition: Format.cpp:2079
const char * name() const noexcept override
PreambleBounds getBounds() const
PreambleBounds used to build the preamble.
void AddImplicitPreamble(CompilerInvocation &CI, IntrusiveRefCntPtr< llvm::vfs::FileSystem > &VFS, llvm::MemoryBuffer *MainFileBuffer) const
Changes options inside CI to use PCH from this preamble.
Represents a character-granular source range.
static PreambleBounds ComputePreamble(StringRef Buffer, const LangOptions &LangOpts, unsigned MaxLines=0)
Compute the preamble of the given file.
Definition: Lexer.cpp:585
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:126
Defines the clang::Preprocessor interface.
ASTEdit remove(RangeSelector S)
Removes the source selected by S.
Record the location of an inclusion directive, such as an #include or #import statement.
static TargetInfo * CreateTargetInfo(DiagnosticsEngine &Diags, const std::shared_ptr< TargetOptions > &Opts)
Construct a target for the given options.
Definition: Targets.cpp:645
bool GeneratePreamble
True indicates that a preamble is being generated.
#define SM(sm)
Definition: Cuda.cpp:62
The result type of a method or function.
PreambleBounds ComputePreambleBounds(const LangOptions &LangOpts, const llvm::MemoryBufferRef &Buffer, unsigned MaxLines)
Runs lexer to compute suggested preamble bounds.
In-memory cache for modules.
Kind
CompilerInstance - Helper class for managing a single instance of the Clang compiler.
Encodes a location in the source.
virtual CommentHandler * getCommentHandler()
The returned CommentHandler will be added to the preprocessor if not null.
std::string ImplicitPCHInclude
The implicit PCH included at the start of the translation unit, or empty.
Cached information about one file (either on disk or in the virtual file system).
Definition: FileEntry.h:331
void ProcessWarningOptions(DiagnosticsEngine &Diags, const DiagnosticOptions &Opts, bool ReportDiags=true)
ProcessWarningOptions - Initialize the diagnostic client and process the warning options specified on...
Definition: Warnings.cpp:44
Abstract base class to use for AST consumer-based frontend actions.
virtual void BeforeExecute(CompilerInstance &CI)
Called before FrontendAction::BeginSourceFile.
virtual void AfterPCHEmitted(ASTWriter &Writer)
Called after PCH has been emitted.
PrecompiledPreamble(PrecompiledPreamble &&)=default
constexpr XRayInstrMask None
Definition: XRayInstr.h:38
std::string message(int condition) const override
Dataflow Directional Tag Classes.
PreprocessorOptions & getPreprocessorOpts()
frontend::ActionKind ProgramAction
The frontend action to perform.
IntrusiveRefCntPtr< llvm::vfs::FileSystem > createVFSFromCompilerInvocation(const CompilerInvocation &CI, DiagnosticsEngine &Diags)
FileID getMainFileID() const
Returns the FileID of the main source file.
Helper class for holding the data necessary to invoke the compiler.
FrontendOptions - Options for controlling the behavior of the frontend.
virtual void AfterExecute(CompilerInstance &CI)
Called after FrontendAction::Execute(), but before FrontendAction::EndSourceFile().
static llvm::ErrorOr< PrecompiledPreamble > Build(const CompilerInvocation &Invocation, const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds, DiagnosticsEngine &Diagnostics, IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, std::shared_ptr< PCHContainerOperations > PCHContainerOps, bool StoreInMemory, PreambleCallbacks &Callbacks)
Try to build PrecompiledPreamble for Invocation.
llvm::ErrorOr< const FileEntry * > getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
Indicates that the tracking object is a descendant of a referenced-counted OSObject,...
unsigned Size
Size of the preamble in bytes.
Definition: Lexer.h:60
search_dir_iterator angled_dir_begin() const
Definition: HeaderSearch.h:732
Cached information about one directory (either on disk or in the virtual file system).
Generate pre-compiled header.
Preprocessor & getPreprocessor() const
Return the current preprocessor.
virtual bool BeginSourceFileAction(CompilerInstance &CI)
Callback at the start of processing a single input.
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:640
std::vector< std::pair< std::string, std::string > > RemappedFiles
The set of file remappings, which take existing files on the system (the first part of each pair) and...
Writes an AST file containing the contents of a translation unit.
Definition: ASTWriter.h:99
std::pair< unsigned, bool > PrecompiledPreambleBytes
If non-zero, the implicit PCH include is actually a precompiled preamble that covers this number of b...
Defines the clang::TargetInfo interface.
static Decl::Kind getKind(const Decl *D)
Definition: DeclBase.cpp:997
LLVM IR: we accept this so that we can run the optimizer on it, and compile it to assembly or object ...
Abstract base class that describes a handler that will receive source ranges for each of the comments...
An abstract superclass that describes a custom extension to the module/precompiled header file format...
AST and semantic-analysis consumer that generates a precompiled header from the parsed source code.
Definition: ASTWriter.h:767
static std::unique_ptr< llvm::raw_pwrite_stream > CreateOutputFile(CompilerInstance &CI, StringRef InFile, std::string &OutputFile)
Creates file to write the PCH into and returns a stream to write it into.
#define true
Definition: stdbool.h:16
virtual void HandleTopLevelDecl(DeclGroupRef DG)
Called for each TopLevelDecl.
This class handles loading and caching of source files into memory.
virtual std::unique_ptr< PPCallbacks > createPPCallbacks()
Creates wrapper class for PPCallbacks so we can also process information about includes that are insi...
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:129