clang  12.0.0git
PrecompiledPreamble.cpp
Go to the documentation of this file.
1 //===--- PrecompiledPreamble.cpp - Build precompiled preambles --*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Helper class to build precompiled preamble.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "clang/AST/DeclObjC.h"
17 #include "clang/Basic/TargetInfo.h"
22 #include "clang/Lex/HeaderSearch.h"
23 #include "clang/Lex/Lexer.h"
24 #include "clang/Lex/Preprocessor.h"
27 #include "llvm/ADT/SmallString.h"
28 #include "llvm/ADT/StringExtras.h"
29 #include "llvm/ADT/StringSet.h"
30 #include "llvm/ADT/iterator_range.h"
31 #include "llvm/Config/llvm-config.h"
32 #include "llvm/Support/CrashRecoveryContext.h"
33 #include "llvm/Support/FileSystem.h"
34 #include "llvm/Support/Path.h"
35 #include "llvm/Support/Process.h"
36 #include "llvm/Support/VirtualFileSystem.h"
37 #include <limits>
38 #include <mutex>
39 #include <utility>
40 
41 using namespace clang;
42 
43 namespace {
44 
45 StringRef getInMemoryPreamblePath() {
46 #if defined(LLVM_ON_UNIX)
47  return "/__clang_tmp/___clang_inmemory_preamble___";
48 #elif defined(_WIN32)
49  return "C:\\__clang_tmp\\___clang_inmemory_preamble___";
50 #else
51 #warning "Unknown platform. Defaulting to UNIX-style paths for in-memory PCHs"
52  return "/__clang_tmp/___clang_inmemory_preamble___";
53 #endif
54 }
55 
57 createVFSOverlayForPreamblePCH(StringRef PCHFilename,
58  std::unique_ptr<llvm::MemoryBuffer> PCHBuffer,
60  // We want only the PCH file from the real filesystem to be available,
61  // so we create an in-memory VFS with just that and overlay it on top.
63  new llvm::vfs::InMemoryFileSystem());
64  PCHFS->addFile(PCHFilename, 0, std::move(PCHBuffer));
66  new llvm::vfs::OverlayFileSystem(VFS));
67  Overlay->pushOverlay(PCHFS);
68  return Overlay;
69 }
70 
71 class PreambleDependencyCollector : public DependencyCollector {
72 public:
73  // We want to collect all dependencies for correctness. Avoiding the real
74  // system dependencies (e.g. stl from /usr/lib) would probably be a good idea,
75  // but there is no way to distinguish between those and the ones that can be
76  // spuriously added by '-isystem' (e.g. to suppress warnings from those
77  // headers).
78  bool needSystemDependencies() override { return true; }
79 };
80 
81 // Collects files whose existence would invalidate the preamble.
82 // Collecting *all* of these would make validating it too slow though, so we
83 // just find all the candidates for 'file not found' diagnostics.
84 //
85 // A caveat that may be significant for generated files: we'll omit files under
86 // search path entries whose roots don't exist when the preamble is built.
87 // These are pruned by InitHeaderSearch and so we don't see the search path.
88 // It would be nice to include them but we don't want to duplicate all the rest
89 // of the InitHeaderSearch logic to reconstruct them.
90 class MissingFileCollector : public PPCallbacks {
91  llvm::StringSet<> &Out;
92  const HeaderSearch &Search;
93  const SourceManager &SM;
94 
95 public:
96  MissingFileCollector(llvm::StringSet<> &Out, const HeaderSearch &Search,
97  const SourceManager &SM)
98  : Out(Out), Search(Search), SM(SM) {}
99 
100  void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
101  StringRef FileName, bool IsAngled,
102  CharSourceRange FilenameRange, const FileEntry *File,
103  StringRef SearchPath, StringRef RelativePath,
104  const Module *Imported,
105  SrcMgr::CharacteristicKind FileType) override {
106  // File is null if it wasn't found.
107  // (We have some false negatives if PP recovered e.g. <foo> -> "foo")
108  if (File != nullptr)
109  return;
110 
111  // If it's a rare absolute include, we know the full path already.
112  if (llvm::sys::path::is_absolute(FileName)) {
113  Out.insert(FileName);
114  return;
115  }
116 
117  // Reconstruct the filenames that would satisfy this directive...
119  auto NotFoundRelativeTo = [&](const DirectoryEntry *DE) {
120  Buf = DE->getName();
121  llvm::sys::path::append(Buf, FileName);
122  llvm::sys::path::remove_dots(Buf, /*remove_dot_dot=*/true);
123  Out.insert(Buf);
124  };
125  // ...relative to the including file.
126  if (!IsAngled) {
127  if (const FileEntry *IncludingFile =
128  SM.getFileEntryForID(SM.getFileID(IncludeTok.getLocation())))
129  if (IncludingFile->getDir())
130  NotFoundRelativeTo(IncludingFile->getDir());
131  }
132  // ...relative to the search paths.
133  for (const auto &Dir : llvm::make_range(
134  IsAngled ? Search.angled_dir_begin() : Search.search_dir_begin(),
135  Search.search_dir_end())) {
136  // No support for frameworks or header maps yet.
137  if (Dir.isNormalDir())
138  NotFoundRelativeTo(Dir.getDir());
139  }
140  }
141 };
142 
143 /// Keeps a track of files to be deleted in destructor.
144 class TemporaryFiles {
145 public:
146  // A static instance to be used by all clients.
147  static TemporaryFiles &getInstance();
148 
149 private:
150  // Disallow constructing the class directly.
151  TemporaryFiles() = default;
152  // Disallow copy.
153  TemporaryFiles(const TemporaryFiles &) = delete;
154 
155 public:
156  ~TemporaryFiles();
157 
158  /// Adds \p File to a set of tracked files.
159  void addFile(StringRef File);
160 
161  /// Remove \p File from disk and from the set of tracked files.
162  void removeFile(StringRef File);
163 
164 private:
165  std::mutex Mutex;
166  llvm::StringSet<> Files;
167 };
168 
169 TemporaryFiles &TemporaryFiles::getInstance() {
170  static TemporaryFiles Instance;
171  return Instance;
172 }
173 
174 TemporaryFiles::~TemporaryFiles() {
175  std::lock_guard<std::mutex> Guard(Mutex);
176  for (const auto &File : Files)
177  llvm::sys::fs::remove(File.getKey());
178 }
179 
180 void TemporaryFiles::addFile(StringRef File) {
181  std::lock_guard<std::mutex> Guard(Mutex);
182  auto IsInserted = Files.insert(File).second;
183  (void)IsInserted;
184  assert(IsInserted && "File has already been added");
185 }
186 
187 void TemporaryFiles::removeFile(StringRef File) {
188  std::lock_guard<std::mutex> Guard(Mutex);
189  auto WasPresent = Files.erase(File);
190  (void)WasPresent;
191  assert(WasPresent && "File was not tracked");
192  llvm::sys::fs::remove(File);
193 }
194 
195 class PrecompilePreambleAction : public ASTFrontendAction {
196 public:
197  PrecompilePreambleAction(std::string *InMemStorage,
198  PreambleCallbacks &Callbacks)
199  : InMemStorage(InMemStorage), Callbacks(Callbacks) {}
200 
201  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
202  StringRef InFile) override;
203 
204  bool hasEmittedPreamblePCH() const { return HasEmittedPreamblePCH; }
205 
206  void setEmittedPreamblePCH(ASTWriter &Writer) {
207  this->HasEmittedPreamblePCH = true;
208  Callbacks.AfterPCHEmitted(Writer);
209  }
210 
211  bool shouldEraseOutputFiles() override { return !hasEmittedPreamblePCH(); }
212  bool hasCodeCompletionSupport() const override { return false; }
213  bool hasASTFileSupport() const override { return false; }
214  TranslationUnitKind getTranslationUnitKind() override { return TU_Prefix; }
215 
216 private:
217  friend class PrecompilePreambleConsumer;
218 
219  bool HasEmittedPreamblePCH = false;
220  std::string *InMemStorage;
221  PreambleCallbacks &Callbacks;
222 };
223 
224 class PrecompilePreambleConsumer : public PCHGenerator {
225 public:
226  PrecompilePreambleConsumer(PrecompilePreambleAction &Action,
227  const Preprocessor &PP,
228  InMemoryModuleCache &ModuleCache,
229  StringRef isysroot,
230  std::unique_ptr<raw_ostream> Out)
231  : PCHGenerator(PP, ModuleCache, "", isysroot,
232  std::make_shared<PCHBuffer>(),
233  ArrayRef<std::shared_ptr<ModuleFileExtension>>(),
234  /*AllowASTWithErrors=*/true),
235  Action(Action), Out(std::move(Out)) {}
236 
237  bool HandleTopLevelDecl(DeclGroupRef DG) override {
238  Action.Callbacks.HandleTopLevelDecl(DG);
239  return true;
240  }
241 
242  void HandleTranslationUnit(ASTContext &Ctx) override {
244  if (!hasEmittedPCH())
245  return;
246 
247  // Write the generated bitstream to "Out".
248  *Out << getPCH();
249  // Make sure it hits disk now.
250  Out->flush();
251  // Free the buffer.
253  getPCH() = std::move(Empty);
254 
255  Action.setEmittedPreamblePCH(getWriter());
256  }
257 
258  bool shouldSkipFunctionBody(Decl *D) override {
259  return Action.Callbacks.shouldSkipFunctionBody(D);
260  }
261 
262 private:
263  PrecompilePreambleAction &Action;
264  std::unique_ptr<raw_ostream> Out;
265 };
266 
267 std::unique_ptr<ASTConsumer>
268 PrecompilePreambleAction::CreateASTConsumer(CompilerInstance &CI,
269  StringRef InFile) {
270  std::string Sysroot;
272  return nullptr;
273 
274  std::unique_ptr<llvm::raw_ostream> OS;
275  if (InMemStorage) {
276  OS = std::make_unique<llvm::raw_string_ostream>(*InMemStorage);
277  } else {
278  std::string OutputFile;
279  OS = GeneratePCHAction::CreateOutputFile(CI, InFile, OutputFile);
280  }
281  if (!OS)
282  return nullptr;
283 
285  Sysroot.clear();
286 
287  return std::make_unique<PrecompilePreambleConsumer>(
288  *this, CI.getPreprocessor(), CI.getModuleCache(), Sysroot, std::move(OS));
289 }
290 
291 template <class T> bool moveOnNoError(llvm::ErrorOr<T> Val, T &Output) {
292  if (!Val)
293  return false;
294  Output = std::move(*Val);
295  return true;
296 }
297 
298 } // namespace
299 
301  const llvm::MemoryBuffer *Buffer,
302  unsigned MaxLines) {
303  return Lexer::ComputePreamble(Buffer->getBuffer(), LangOpts, MaxLines);
304 }
305 
306 llvm::ErrorOr<PrecompiledPreamble> PrecompiledPreamble::Build(
307  const CompilerInvocation &Invocation,
308  const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds,
309  DiagnosticsEngine &Diagnostics,
311  std::shared_ptr<PCHContainerOperations> PCHContainerOps, bool StoreInMemory,
312  PreambleCallbacks &Callbacks) {
313  assert(VFS && "VFS is null");
314 
315  auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
316  FrontendOptions &FrontendOpts = PreambleInvocation->getFrontendOpts();
317  PreprocessorOptions &PreprocessorOpts =
318  PreambleInvocation->getPreprocessorOpts();
319 
321  if (!StoreInMemory) {
322  // Create a temporary file for the precompiled preamble. In rare
323  // circumstances, this can fail.
324  llvm::ErrorOr<PrecompiledPreamble::TempPCHFile> PreamblePCHFile =
325  PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile();
326  if (!PreamblePCHFile)
328  TempFile = std::move(*PreamblePCHFile);
329  }
330 
331  PCHStorage Storage = StoreInMemory ? PCHStorage(InMemoryPreamble())
332  : PCHStorage(std::move(*TempFile));
333 
334  // Save the preamble text for later; we'll need to compare against it for
335  // subsequent reparses.
336  std::vector<char> PreambleBytes(MainFileBuffer->getBufferStart(),
337  MainFileBuffer->getBufferStart() +
338  Bounds.Size);
339  bool PreambleEndsAtStartOfLine = Bounds.PreambleEndsAtStartOfLine;
340 
341  // Tell the compiler invocation to generate a temporary precompiled header.
342  FrontendOpts.ProgramAction = frontend::GeneratePCH;
343  FrontendOpts.OutputFile =
344  std::string(StoreInMemory ? getInMemoryPreamblePath()
345  : Storage.asFile().getFilePath());
346  PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
347  PreprocessorOpts.PrecompiledPreambleBytes.second = false;
348  // Inform preprocessor to record conditional stack when building the preamble.
349  PreprocessorOpts.GeneratePreamble = true;
350 
351  // Create the compiler instance to use for building the precompiled preamble.
352  std::unique_ptr<CompilerInstance> Clang(
353  new CompilerInstance(std::move(PCHContainerOps)));
354 
355  // Recover resources if we crash before exiting this method.
356  llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> CICleanup(
357  Clang.get());
358 
359  Clang->setInvocation(std::move(PreambleInvocation));
360  Clang->setDiagnostics(&Diagnostics);
361 
362  // Create the target instance.
363  Clang->setTarget(TargetInfo::CreateTargetInfo(
364  Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
365  if (!Clang->hasTarget())
367 
368  // Inform the target of the language options.
369  //
370  // FIXME: We shouldn't need to do this, the target should be immutable once
371  // created. This complexity should be lifted elsewhere.
372  Clang->getTarget().adjust(Clang->getLangOpts());
373 
374  if (Clang->getFrontendOpts().Inputs.size() != 1 ||
375  Clang->getFrontendOpts().Inputs[0].getKind().getFormat() !=
377  Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() ==
380  }
381 
382  // Clear out old caches and data.
383  Diagnostics.Reset();
384  ProcessWarningOptions(Diagnostics, Clang->getDiagnosticOpts());
385 
386  VFS =
387  createVFSFromCompilerInvocation(Clang->getInvocation(), Diagnostics, VFS);
388 
389  // Create a file manager object to provide access to and cache the filesystem.
390  Clang->setFileManager(new FileManager(Clang->getFileSystemOpts(), VFS));
391 
392  // Create the source manager.
393  Clang->setSourceManager(
394  new SourceManager(Diagnostics, Clang->getFileManager()));
395 
396  auto PreambleDepCollector = std::make_shared<PreambleDependencyCollector>();
397  Clang->addDependencyCollector(PreambleDepCollector);
398 
399  // Remap the main source file to the preamble buffer.
400  StringRef MainFilePath = FrontendOpts.Inputs[0].getFile();
401  auto PreambleInputBuffer = llvm::MemoryBuffer::getMemBufferCopy(
402  MainFileBuffer->getBuffer().slice(0, Bounds.Size), MainFilePath);
403  if (PreprocessorOpts.RetainRemappedFileBuffers) {
404  // MainFileBuffer will be deleted by unique_ptr after leaving the method.
405  PreprocessorOpts.addRemappedFile(MainFilePath, PreambleInputBuffer.get());
406  } else {
407  // In that case, remapped buffer will be deleted by CompilerInstance on
408  // BeginSourceFile, so we call release() to avoid double deletion.
409  PreprocessorOpts.addRemappedFile(MainFilePath,
410  PreambleInputBuffer.release());
411  }
412 
413  std::unique_ptr<PrecompilePreambleAction> Act;
414  Act.reset(new PrecompilePreambleAction(
415  StoreInMemory ? &Storage.asMemory().Data : nullptr, Callbacks));
416  Callbacks.BeforeExecute(*Clang);
417  if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0]))
419 
420  std::unique_ptr<PPCallbacks> DelegatedPPCallbacks =
421  Callbacks.createPPCallbacks();
422  if (DelegatedPPCallbacks)
423  Clang->getPreprocessor().addPPCallbacks(std::move(DelegatedPPCallbacks));
424  if (auto CommentHandler = Callbacks.getCommentHandler())
425  Clang->getPreprocessor().addCommentHandler(CommentHandler);
426  llvm::StringSet<> MissingFiles;
427  Clang->getPreprocessor().addPPCallbacks(
428  std::make_unique<MissingFileCollector>(
429  MissingFiles, Clang->getPreprocessor().getHeaderSearchInfo(),
430  Clang->getSourceManager()));
431 
432  if (llvm::Error Err = Act->Execute())
433  return errorToErrorCode(std::move(Err));
434 
435  // Run the callbacks.
436  Callbacks.AfterExecute(*Clang);
437 
438  Act->EndSourceFile();
439 
440  if (!Act->hasEmittedPreamblePCH())
442 
443  // Keep track of all of the files that the source manager knows about,
444  // so we can verify whether they have changed or not.
445  llvm::StringMap<PrecompiledPreamble::PreambleFileHash> FilesInPreamble;
446 
447  SourceManager &SourceMgr = Clang->getSourceManager();
448  for (auto &Filename : PreambleDepCollector->getDependencies()) {
449  auto FileOrErr = Clang->getFileManager().getFile(Filename);
450  if (!FileOrErr ||
451  *FileOrErr == SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()))
452  continue;
453  auto File = *FileOrErr;
454  if (time_t ModTime = File->getModificationTime()) {
455  FilesInPreamble[File->getName()] =
456  PrecompiledPreamble::PreambleFileHash::createForFile(File->getSize(),
457  ModTime);
458  } else {
459  const llvm::MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File);
460  FilesInPreamble[File->getName()] =
461  PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(Buffer);
462  }
463  }
464 
465  return PrecompiledPreamble(
466  std::move(Storage), std::move(PreambleBytes), PreambleEndsAtStartOfLine,
467  std::move(FilesInPreamble), std::move(MissingFiles));
468 }
469 
471  return PreambleBounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
472 }
473 
475  switch (Storage.getKind()) {
476  case PCHStorage::Kind::Empty:
477  assert(false && "Calling getSize() on invalid PrecompiledPreamble. "
478  "Was it std::moved?");
479  return 0;
480  case PCHStorage::Kind::InMemory:
481  return Storage.asMemory().Data.size();
482  case PCHStorage::Kind::TempFile: {
483  uint64_t Result;
484  if (llvm::sys::fs::file_size(Storage.asFile().getFilePath(), Result))
485  return 0;
486 
487  assert(Result <= std::numeric_limits<std::size_t>::max() &&
488  "file size did not fit into size_t");
489  return Result;
490  }
491  }
492  llvm_unreachable("Unhandled storage kind");
493 }
494 
496  const llvm::MemoryBuffer *MainFileBuffer,
497  PreambleBounds Bounds,
498  llvm::vfs::FileSystem *VFS) const {
499 
500  assert(
501  Bounds.Size <= MainFileBuffer->getBufferSize() &&
502  "Buffer is too large. Bounds were calculated from a different buffer?");
503 
504  auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
505  PreprocessorOptions &PreprocessorOpts =
506  PreambleInvocation->getPreprocessorOpts();
507 
508  // We've previously computed a preamble. Check whether we have the same
509  // preamble now that we did before, and that there's enough space in
510  // the main-file buffer within the precompiled preamble to fit the
511  // new main file.
512  if (PreambleBytes.size() != Bounds.Size ||
513  PreambleEndsAtStartOfLine != Bounds.PreambleEndsAtStartOfLine ||
514  !std::equal(PreambleBytes.begin(), PreambleBytes.end(),
515  MainFileBuffer->getBuffer().begin()))
516  return false;
517  // The preamble has not changed. We may be able to re-use the precompiled
518  // preamble.
519 
520  // Check that none of the files used by the preamble have changed.
521  // First, make a record of those files that have been overridden via
522  // remapping or unsaved_files.
523  std::map<llvm::sys::fs::UniqueID, PreambleFileHash> OverriddenFiles;
524  llvm::StringSet<> OverriddenAbsPaths; // Either by buffers or files.
525  for (const auto &R : PreprocessorOpts.RemappedFiles) {
526  llvm::vfs::Status Status;
527  if (!moveOnNoError(VFS->status(R.second), Status)) {
528  // If we can't stat the file we're remapping to, assume that something
529  // horrible happened.
530  return false;
531  }
532  // If a mapped file was previously missing, then it has changed.
533  llvm::SmallString<128> MappedPath(R.first);
534  if (!VFS->makeAbsolute(MappedPath))
535  OverriddenAbsPaths.insert(MappedPath);
536 
537  OverriddenFiles[Status.getUniqueID()] = PreambleFileHash::createForFile(
538  Status.getSize(), llvm::sys::toTimeT(Status.getLastModificationTime()));
539  }
540 
541  // OverridenFileBuffers tracks only the files not found in VFS.
542  llvm::StringMap<PreambleFileHash> OverridenFileBuffers;
543  for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) {
544  const PrecompiledPreamble::PreambleFileHash PreambleHash =
545  PreambleFileHash::createForMemoryBuffer(RB.second);
546  llvm::vfs::Status Status;
547  if (moveOnNoError(VFS->status(RB.first), Status))
548  OverriddenFiles[Status.getUniqueID()] = PreambleHash;
549  else
550  OverridenFileBuffers[RB.first] = PreambleHash;
551 
552  llvm::SmallString<128> MappedPath(RB.first);
553  if (!VFS->makeAbsolute(MappedPath))
554  OverriddenAbsPaths.insert(MappedPath);
555  }
556 
557  // Check whether anything has changed.
558  for (const auto &F : FilesInPreamble) {
559  auto OverridenFileBuffer = OverridenFileBuffers.find(F.first());
560  if (OverridenFileBuffer != OverridenFileBuffers.end()) {
561  // The file's buffer was remapped and the file was not found in VFS.
562  // Check whether it matches up with the previous mapping.
563  if (OverridenFileBuffer->second != F.second)
564  return false;
565  continue;
566  }
567 
568  llvm::vfs::Status Status;
569  if (!moveOnNoError(VFS->status(F.first()), Status)) {
570  // If the file's buffer is not remapped and we can't stat it,
571  // assume that something horrible happened.
572  return false;
573  }
574 
575  std::map<llvm::sys::fs::UniqueID, PreambleFileHash>::iterator Overridden =
576  OverriddenFiles.find(Status.getUniqueID());
577  if (Overridden != OverriddenFiles.end()) {
578  // This file was remapped; check whether the newly-mapped file
579  // matches up with the previous mapping.
580  if (Overridden->second != F.second)
581  return false;
582  continue;
583  }
584 
585  // Neither the file's buffer nor the file itself was remapped;
586  // check whether it has changed on disk.
587  if (Status.getSize() != uint64_t(F.second.Size) ||
588  llvm::sys::toTimeT(Status.getLastModificationTime()) !=
589  F.second.ModTime)
590  return false;
591  }
592  for (const auto &F : MissingFiles) {
593  // A missing file may be "provided" by an override buffer or file.
594  if (OverriddenAbsPaths.count(F.getKey()))
595  return false;
596  // If a file previously recorded as missing exists as a regular file, then
597  // consider the preamble out-of-date.
598  if (auto Status = VFS->status(F.getKey())) {
599  if (Status->isRegularFile())
600  return false;
601  }
602  }
603  return true;
604 }
605 
608  llvm::MemoryBuffer *MainFileBuffer) const {
609  PreambleBounds Bounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
610  configurePreamble(Bounds, CI, VFS, MainFileBuffer);
611 }
612 
615  llvm::MemoryBuffer *MainFileBuffer) const {
616  auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), MainFileBuffer, 0);
617  configurePreamble(Bounds, CI, VFS, MainFileBuffer);
618 }
619 
621  PCHStorage Storage, std::vector<char> PreambleBytes,
622  bool PreambleEndsAtStartOfLine,
623  llvm::StringMap<PreambleFileHash> FilesInPreamble,
624  llvm::StringSet<> MissingFiles)
625  : Storage(std::move(Storage)), FilesInPreamble(std::move(FilesInPreamble)),
626  MissingFiles(std::move(MissingFiles)),
627  PreambleBytes(std::move(PreambleBytes)),
628  PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {
629  assert(this->Storage.getKind() != PCHStorage::Kind::Empty);
630 }
631 
632 llvm::ErrorOr<PrecompiledPreamble::TempPCHFile>
633 PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile() {
634  // FIXME: This is a hack so that we can override the preamble file during
635  // crash-recovery testing, which is the only case where the preamble files
636  // are not necessarily cleaned up.
637  if (const char *TmpFile = ::getenv("CINDEXTEST_PREAMBLE_FILE"))
638  return TempPCHFile(TmpFile);
639 
641  // Using a version of createTemporaryFile with a file descriptor guarantees
642  // that we would never get a race condition in a multi-threaded setting
643  // (i.e., multiple threads getting the same temporary path).
644  int FD;
645  auto EC = llvm::sys::fs::createTemporaryFile("preamble", "pch", FD, File);
646  if (EC)
647  return EC;
648  // We only needed to make sure the file exists, close the file right away.
649  llvm::sys::Process::SafelyCloseFileDescriptor(FD);
650  return TempPCHFile(std::string(std::move(File).str()));
651 }
652 
653 PrecompiledPreamble::TempPCHFile::TempPCHFile(std::string FilePath)
654  : FilePath(std::move(FilePath)) {
655  TemporaryFiles::getInstance().addFile(*this->FilePath);
656 }
657 
658 PrecompiledPreamble::TempPCHFile::TempPCHFile(TempPCHFile &&Other) {
659  FilePath = std::move(Other.FilePath);
660  Other.FilePath = None;
661 }
662 
663 PrecompiledPreamble::TempPCHFile &PrecompiledPreamble::TempPCHFile::
664 operator=(TempPCHFile &&Other) {
665  RemoveFileIfPresent();
666 
667  FilePath = std::move(Other.FilePath);
668  Other.FilePath = None;
669  return *this;
670 }
671 
672 PrecompiledPreamble::TempPCHFile::~TempPCHFile() { RemoveFileIfPresent(); }
673 
674 void PrecompiledPreamble::TempPCHFile::RemoveFileIfPresent() {
675  if (FilePath) {
676  TemporaryFiles::getInstance().removeFile(*FilePath);
677  FilePath = None;
678  }
679 }
680 
681 llvm::StringRef PrecompiledPreamble::TempPCHFile::getFilePath() const {
682  assert(FilePath && "TempPCHFile doesn't have a FilePath. Had it been moved?");
683  return *FilePath;
684 }
685 
686 PrecompiledPreamble::PCHStorage::PCHStorage(TempPCHFile File)
687  : StorageKind(Kind::TempFile) {
688  new (&asFile()) TempPCHFile(std::move(File));
689 }
690 
691 PrecompiledPreamble::PCHStorage::PCHStorage(InMemoryPreamble Memory)
692  : StorageKind(Kind::InMemory) {
693  new (&asMemory()) InMemoryPreamble(std::move(Memory));
694 }
695 
696 PrecompiledPreamble::PCHStorage::PCHStorage(PCHStorage &&Other) : PCHStorage() {
697  *this = std::move(Other);
698 }
699 
700 PrecompiledPreamble::PCHStorage &PrecompiledPreamble::PCHStorage::
701 operator=(PCHStorage &&Other) {
702  destroy();
703 
704  StorageKind = Other.StorageKind;
705  switch (StorageKind) {
706  case Kind::Empty:
707  // do nothing;
708  break;
709  case Kind::TempFile:
710  new (&asFile()) TempPCHFile(std::move(Other.asFile()));
711  break;
712  case Kind::InMemory:
713  new (&asMemory()) InMemoryPreamble(std::move(Other.asMemory()));
714  break;
715  }
716 
717  Other.setEmpty();
718  return *this;
719 }
720 
721 PrecompiledPreamble::PCHStorage::~PCHStorage() { destroy(); }
722 
723 PrecompiledPreamble::PCHStorage::Kind
725  return StorageKind;
726 }
727 
728 PrecompiledPreamble::TempPCHFile &PrecompiledPreamble::PCHStorage::asFile() {
729  assert(getKind() == Kind::TempFile);
730  return *reinterpret_cast<TempPCHFile *>(Storage.buffer);
731 }
732 
733 const PrecompiledPreamble::TempPCHFile &
734 PrecompiledPreamble::PCHStorage::asFile() const {
735  return const_cast<PCHStorage *>(this)->asFile();
736 }
737 
738 PrecompiledPreamble::InMemoryPreamble &
739 PrecompiledPreamble::PCHStorage::asMemory() {
740  assert(getKind() == Kind::InMemory);
741  return *reinterpret_cast<InMemoryPreamble *>(Storage.buffer);
742 }
743 
744 const PrecompiledPreamble::InMemoryPreamble &
745 PrecompiledPreamble::PCHStorage::asMemory() const {
746  return const_cast<PCHStorage *>(this)->asMemory();
747 }
748 
749 void PrecompiledPreamble::PCHStorage::destroy() {
750  switch (StorageKind) {
751  case Kind::Empty:
752  return;
753  case Kind::TempFile:
754  asFile().~TempPCHFile();
755  return;
756  case Kind::InMemory:
757  asMemory().~InMemoryPreamble();
758  return;
759  }
760 }
761 
762 void PrecompiledPreamble::PCHStorage::setEmpty() {
763  destroy();
764  StorageKind = Kind::Empty;
765 }
766 
767 PrecompiledPreamble::PreambleFileHash
768 PrecompiledPreamble::PreambleFileHash::createForFile(off_t Size,
769  time_t ModTime) {
770  PreambleFileHash Result;
771  Result.Size = Size;
772  Result.ModTime = ModTime;
773  Result.MD5 = {};
774  return Result;
775 }
776 
777 PrecompiledPreamble::PreambleFileHash
778 PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(
779  const llvm::MemoryBuffer *Buffer) {
780  PreambleFileHash Result;
781  Result.Size = Buffer->getBufferSize();
782  Result.ModTime = 0;
783 
784  llvm::MD5 MD5Ctx;
785  MD5Ctx.update(Buffer->getBuffer().data());
786  MD5Ctx.final(Result.MD5);
787 
788  return Result;
789 }
790 
791 void PrecompiledPreamble::configurePreamble(
794  llvm::MemoryBuffer *MainFileBuffer) const {
795  assert(VFS);
796 
797  auto &PreprocessorOpts = CI.getPreprocessorOpts();
798 
799  // Remap main file to point to MainFileBuffer.
800  auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
801  PreprocessorOpts.addRemappedFile(MainFilePath, MainFileBuffer);
802 
803  // Configure ImpicitPCHInclude.
804  PreprocessorOpts.PrecompiledPreambleBytes.first = Bounds.Size;
805  PreprocessorOpts.PrecompiledPreambleBytes.second =
807  PreprocessorOpts.DisablePCHValidation = true;
808 
809  setupPreambleStorage(Storage, PreprocessorOpts, VFS);
810 }
811 
812 void PrecompiledPreamble::setupPreambleStorage(
813  const PCHStorage &Storage, PreprocessorOptions &PreprocessorOpts,
815  if (Storage.getKind() == PCHStorage::Kind::TempFile) {
816  const TempPCHFile &PCHFile = Storage.asFile();
817  PreprocessorOpts.ImplicitPCHInclude = std::string(PCHFile.getFilePath());
818 
819  // Make sure we can access the PCH file even if we're using a VFS
821  llvm::vfs::getRealFileSystem();
822  auto PCHPath = PCHFile.getFilePath();
823  if (VFS == RealFS || VFS->exists(PCHPath))
824  return;
825  auto Buf = RealFS->getBufferForFile(PCHPath);
826  if (!Buf) {
827  // We can't read the file even from RealFS, this is clearly an error,
828  // but we'll just leave the current VFS as is and let clang's code
829  // figure out what to do with missing PCH.
830  return;
831  }
832 
833  // We have a slight inconsistency here -- we're using the VFS to
834  // read files, but the PCH was generated in the real file system.
835  VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(*Buf), VFS);
836  } else {
837  assert(Storage.getKind() == PCHStorage::Kind::InMemory);
838  // For in-memory preamble, we have to provide a VFS overlay that makes it
839  // accessible.
840  StringRef PCHPath = getInMemoryPreamblePath();
841  PreprocessorOpts.ImplicitPCHInclude = std::string(PCHPath);
842 
843  auto Buf = llvm::MemoryBuffer::getMemBuffer(Storage.asMemory().Data);
844  VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(Buf), VFS);
845  }
846 }
847 
852 std::unique_ptr<PPCallbacks> PreambleCallbacks::createPPCallbacks() {
853  return nullptr;
854 }
856 
857 static llvm::ManagedStatic<BuildPreambleErrorCategory> BuildPreambleErrCategory;
858 
860  return std::error_code(static_cast<int>(Error), *BuildPreambleErrCategory);
861 }
862 
863 const char *BuildPreambleErrorCategory::name() const noexcept {
864  return "build-preamble.error";
865 }
866 
867 std::string BuildPreambleErrorCategory::message(int condition) const {
868  switch (static_cast<BuildPreambleError>(condition)) {
870  return "Could not create temporary file for PCH";
872  return "CreateTargetInfo() return null";
874  return "BeginSourceFile() return an error";
876  return "Could not emit PCH";
878  return "Command line arguments must contain exactly one source file";
879  }
880  llvm_unreachable("unexpected BuildPreambleError");
881 }
std::size_t getSize() const
Returns the size, in bytes, that preamble takes on disk or in memory.
Describes the bounds (start, size) of the preamble and a flag required by PreprocessorOptions::Precom...
Definition: Lexer.h:58
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:172
Defines the clang::FileManager interface and associated types.
bool CanReuse(const CompilerInvocation &Invocation, const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds, llvm::vfs::FileSystem *VFS) const
Check whether PrecompiledPreamble can be reused for the new contents(MainFileBuffer) of the main file...
An interface for collecting the dependencies of a compilation.
Definition: Utils.h:76
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:89
The translation unit is a prefix to a translation unit, and is not complete.
Definition: LangOptions.h:547
InMemoryModuleCache & getModuleCache() const
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
void addRemappedFile(StringRef From, StringRef To)
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
Definition: SourceManager.h:79
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:35
FileManager & getFileManager() const
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:174
void HandleTranslationUnit(ASTContext &Ctx) override
HandleTranslationUnit - This method is called when the ASTs for entire translation unit have been par...
Definition: GeneratePCH.cpp:41
Definition: Format.h:2679
static bool moveOnNoError(llvm::ErrorOr< T > Val, T &Output)
Definition: ASTUnit.cpp:148
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
__SIZE_TYPE__ size_t
The unsigned integer type of the result of the sizeof operator.
Definition: opencl-c-base.h:40
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:54
Describes a module or submodule.
Definition: Module.h:87
unsigned RelocatablePCH
When generating PCH files, instruct the AST writer to create relocatable PCH files.
static bool ComputeASTConsumerArguments(CompilerInstance &CI, std::string &Sysroot)
Compute the AST consumer arguments that will be used to create the PCHGenerator instance returned by ...
static llvm::ManagedStatic< BuildPreambleErrorCategory > BuildPreambleErrCategory
std::error_code make_error_code(BuildPreambleError Error)
FrontendOptions & getFrontendOpts()
search_dir_iterator search_dir_begin() const
Definition: HeaderSearch.h:686
SmallVector< FrontendInputFile, 0 > Inputs
The input files and their types.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:153
search_dir_iterator search_dir_end() const
Definition: HeaderSearch.h:687
bool RetainRemappedFileBuffers
Whether the compiler instance should retain (i.e., not free) the buffers associated with remapped fil...
void OverridePreamble(CompilerInvocation &CI, IntrusiveRefCntPtr< llvm::vfs::FileSystem > &VFS, llvm::MemoryBuffer *MainFileBuffer) const
Configure CI to use this preamble.
void Reset()
Reset the state of the diagnostic object to its initial configuration.
Definition: Diagnostic.cpp:127
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:158
FrontendOptions & getFrontendOpts()
bool PreambleEndsAtStartOfLine
Whether the preamble ends at the start of a new line.
Definition: Lexer.h:66
A set of callbacks to gather useful information while building a preamble.
PreambleBounds ComputePreambleBounds(const LangOptions &LangOpts, const llvm::MemoryBuffer *Buffer, unsigned MaxLines)
Runs lexer to compute suggested preamble bounds.
StringRef Filename
Definition: Format.cpp:2018
const char * name() const noexcept override
PreambleBounds getBounds() const
PreambleBounds used to build the preamble.
void AddImplicitPreamble(CompilerInvocation &CI, IntrusiveRefCntPtr< llvm::vfs::FileSystem > &VFS, llvm::MemoryBuffer *MainFileBuffer) const
Changes options inside CI to use PCH from this preamble.
Represents a character-granular source range.
static PreambleBounds ComputePreamble(StringRef Buffer, const LangOptions &LangOpts, unsigned MaxLines=0)
Compute the preamble of the given file.
Definition: Lexer.cpp:583
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:126
Defines the clang::Preprocessor interface.
ASTEdit remove(RangeSelector S)
Removes the source selected by S.
Definition: RewriteRule.cpp:93
Record the location of an inclusion directive, such as an #include or #import statement.
static TargetInfo * CreateTargetInfo(DiagnosticsEngine &Diags, const std::shared_ptr< TargetOptions > &Opts)
Construct a target for the given options.
Definition: Targets.cpp:629
bool GeneratePreamble
True indicates that a preamble is being generated.
#define SM(sm)
Definition: Cuda.cpp:62
The result type of a method or function.
In-memory cache for modules.
CompilerInstance - Helper class for managing a single instance of the Clang compiler.
Encodes a location in the source.
virtual CommentHandler * getCommentHandler()
The returned CommentHandler will be added to the preprocessor if not null.
std::string ImplicitPCHInclude
The implicit PCH included at the start of the translation unit, or empty.
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:79
void ProcessWarningOptions(DiagnosticsEngine &Diags, const DiagnosticOptions &Opts, bool ReportDiags=true)
ProcessWarningOptions - Initialize the diagnostic client and process the warning options specified on...
Definition: Warnings.cpp:44
Abstract base class to use for AST consumer-based frontend actions.
virtual void BeforeExecute(CompilerInstance &CI)
Called before FrontendAction::BeginSourceFile.
virtual void AfterPCHEmitted(ASTWriter &Writer)
Called after PCH has been emitted.
PrecompiledPreamble(PrecompiledPreamble &&)=default
std::string message(int condition) const override
Dataflow Directional Tag Classes.
A class holding a PCH and all information to check whether it is valid to reuse the PCH for the subse...
PreprocessorOptions & getPreprocessorOpts()
IntrusiveRefCntPtr< llvm::vfs::FileSystem > createVFSFromCompilerInvocation(const CompilerInvocation &CI, DiagnosticsEngine &Diags)
const llvm::MemoryBuffer * getMemoryBufferForFile(const FileEntry *File, bool *Invalid=nullptr)
Retrieve the memory buffer associated with the given file.
FileID getMainFileID() const
Returns the FileID of the main source file.
Helper class for holding the data necessary to invoke the compiler.
FrontendOptions - Options for controlling the behavior of the frontend.
virtual void AfterExecute(CompilerInstance &CI)
Called after FrontendAction::Execute(), but before FrontendAction::EndSourceFile().
static llvm::ErrorOr< PrecompiledPreamble > Build(const CompilerInvocation &Invocation, const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds, DiagnosticsEngine &Diagnostics, IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, std::shared_ptr< PCHContainerOperations > PCHContainerOps, bool StoreInMemory, PreambleCallbacks &Callbacks)
Try to build PrecompiledPreamble for Invocation.
llvm::ErrorOr< const FileEntry * > getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
Indicates that the tracking object is a descendant of a referenced-counted OSObject, used in the Darwin kernel.
unsigned Size
Size of the preamble in bytes.
Definition: Lexer.h:60
search_dir_iterator angled_dir_begin() const
Definition: HeaderSearch.h:698
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:46
__DEVICE__ int max(int __a, int __b)
Generate pre-compiled header.
Preprocessor & getPreprocessor() const
Return the current preprocessor.
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:541
Writes an AST file containing the contents of a translation unit.
Definition: ASTWriter.h:98
std::pair< unsigned, bool > PrecompiledPreambleBytes
If non-zero, the implicit PCH include is actually a precompiled preamble that covers this number of b...
Defines the clang::TargetInfo interface.
static Decl::Kind getKind(const Decl *D)
Definition: DeclBase.cpp:996
LLVM IR: we accept this so that we can run the optimizer on it, and compile it to assembly or object ...
Abstract base class that describes a handler that will receive source ranges for each of the comments...
An abstract superclass that describes a custom extension to the module/precompiled header file format...
AST and semantic-analysis consumer that generates a precompiled header from the parsed source code...
Definition: ASTWriter.h:757
static std::unique_ptr< llvm::raw_pwrite_stream > CreateOutputFile(CompilerInstance &CI, StringRef InFile, std::string &OutputFile)
Creates file to write the PCH into and returns a stream to write it into.
#define true
Definition: stdbool.h:16
virtual void HandleTopLevelDecl(DeclGroupRef DG)
Called for each TopLevelDecl.
This class handles loading and caching of source files into memory.
virtual std::unique_ptr< PPCallbacks > createPPCallbacks()
Creates wrapper class for PPCallbacks so we can also process information about includes that are insi...
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:128