27 #include "llvm/ADT/SmallString.h"
28 #include "llvm/ADT/StringExtras.h"
29 #include "llvm/ADT/StringSet.h"
30 #include "llvm/ADT/iterator_range.h"
31 #include "llvm/Config/llvm-config.h"
32 #include "llvm/Support/CrashRecoveryContext.h"
33 #include "llvm/Support/FileSystem.h"
34 #include "llvm/Support/Path.h"
35 #include "llvm/Support/Process.h"
36 #include "llvm/Support/VirtualFileSystem.h"
41 using namespace clang;
45 StringRef getInMemoryPreamblePath() {
46 #if defined(LLVM_ON_UNIX)
47 return "/__clang_tmp/___clang_inmemory_preamble___";
49 return "C:\\__clang_tmp\\___clang_inmemory_preamble___";
51 #warning "Unknown platform. Defaulting to UNIX-style paths for in-memory PCHs"
52 return "/__clang_tmp/___clang_inmemory_preamble___";
57 createVFSOverlayForPreamblePCH(StringRef PCHFilename,
58 std::unique_ptr<llvm::MemoryBuffer>
PCHBuffer,
63 new llvm::vfs::InMemoryFileSystem());
64 PCHFS->addFile(PCHFilename, 0, std::move(
PCHBuffer));
66 new llvm::vfs::OverlayFileSystem(VFS));
67 Overlay->pushOverlay(PCHFS);
78 bool needSystemDependencies()
override {
return true; }
91 llvm::StringSet<> &Out;
96 MissingFileCollector(llvm::StringSet<> &Out,
const HeaderSearch &Search,
98 : Out(Out), Search(Search),
SM(
SM) {}
101 StringRef FileName,
bool IsAngled,
103 StringRef SearchPath, StringRef RelativePath,
112 if (llvm::sys::path::is_absolute(FileName)) {
113 Out.insert(FileName);
121 llvm::sys::path::append(Buf, FileName);
122 llvm::sys::path::remove_dots(Buf,
true);
129 if (IncludingFile->getDir())
130 NotFoundRelativeTo(IncludingFile->getDir());
133 for (
const auto &Dir : llvm::make_range(
137 if (Dir.isNormalDir())
138 NotFoundRelativeTo(Dir.getDir());
144 class TemporaryFiles {
147 static TemporaryFiles &getInstance();
151 TemporaryFiles() =
default;
153 TemporaryFiles(
const TemporaryFiles &) =
delete;
159 void addFile(StringRef File);
162 void removeFile(StringRef File);
166 llvm::StringSet<> Files;
169 TemporaryFiles &TemporaryFiles::getInstance() {
170 static TemporaryFiles Instance;
174 TemporaryFiles::~TemporaryFiles() {
175 std::lock_guard<std::mutex> Guard(Mutex);
176 for (
const auto &File : Files)
180 void TemporaryFiles::addFile(StringRef File) {
181 std::lock_guard<std::mutex> Guard(Mutex);
182 auto IsInserted = Files.insert(File).second;
184 assert(IsInserted &&
"File has already been added");
187 void TemporaryFiles::removeFile(StringRef File) {
188 std::lock_guard<std::mutex> Guard(Mutex);
189 auto WasPresent = Files.erase(File);
191 assert(WasPresent &&
"File was not tracked");
197 PrecompilePreambleAction(
std::string *InMemStorage,
199 : InMemStorage(InMemStorage), Callbacks(Callbacks) {}
202 StringRef InFile)
override;
204 bool hasEmittedPreamblePCH()
const {
return HasEmittedPreamblePCH; }
206 void setEmittedPreamblePCH(
ASTWriter &Writer) {
207 this->HasEmittedPreamblePCH =
true;
208 Callbacks.AfterPCHEmitted(Writer);
216 bool shouldEraseOutputFiles()
override {
return !hasEmittedPreamblePCH(); }
217 bool hasCodeCompletionSupport()
const override {
return false; }
218 bool hasASTFileSupport()
const override {
return false; }
222 friend class PrecompilePreambleConsumer;
224 bool HasEmittedPreamblePCH =
false;
229 class PrecompilePreambleConsumer :
public PCHGenerator {
231 PrecompilePreambleConsumer(PrecompilePreambleAction &Action,
235 std::unique_ptr<raw_ostream> Out)
240 Action(Action), Out(
std::move(Out)) {}
243 Action.Callbacks.HandleTopLevelDecl(DG);
247 void HandleTranslationUnit(
ASTContext &Ctx)
override {
249 if (!hasEmittedPCH())
258 getPCH() = std::move(Empty);
260 Action.setEmittedPreamblePCH(getWriter());
263 bool shouldSkipFunctionBody(
Decl *D)
override {
264 return Action.Callbacks.shouldSkipFunctionBody(D);
268 PrecompilePreambleAction &Action;
269 std::unique_ptr<raw_ostream> Out;
272 std::unique_ptr<ASTConsumer>
279 std::unique_ptr<llvm::raw_ostream>
OS;
281 OS = std::make_unique<llvm::raw_string_ostream>(*InMemStorage);
292 return std::make_unique<PrecompilePreambleConsumer>(
296 template <
class T>
bool moveOnNoError(llvm::ErrorOr<T> Val, T &Output) {
299 Output = std::move(*Val);
306 const llvm::MemoryBufferRef &Buffer,
316 std::shared_ptr<PCHContainerOperations> PCHContainerOps,
bool StoreInMemory,
318 assert(VFS &&
"VFS is null");
320 auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
323 PreambleInvocation->getPreprocessorOpts();
326 if (!StoreInMemory) {
329 llvm::ErrorOr<PrecompiledPreamble::TempPCHFile> PreamblePCHFile =
330 PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile();
331 if (!PreamblePCHFile)
333 TempFile = std::move(*PreamblePCHFile);
336 PCHStorage Storage = StoreInMemory ? PCHStorage(InMemoryPreamble())
337 : PCHStorage(std::move(*TempFile));
341 std::vector<char> PreambleBytes(MainFileBuffer->getBufferStart(),
342 MainFileBuffer->getBufferStart() +
349 std::string(StoreInMemory ? getInMemoryPreamblePath()
350 : Storage.asFile().getFilePath());
357 std::unique_ptr<CompilerInstance> Clang(
361 llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> CICleanup(
364 Clang->setInvocation(std::move(PreambleInvocation));
365 Clang->setDiagnostics(&Diagnostics);
368 if (!Clang->createTarget())
371 if (Clang->getFrontendOpts().Inputs.size() != 1 ||
372 Clang->getFrontendOpts().Inputs[0].getKind().getFormat() !=
374 Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() ==
387 Clang->setFileManager(
new FileManager(Clang->getFileSystemOpts(), VFS));
390 Clang->setSourceManager(
393 auto PreambleDepCollector = std::make_shared<PreambleDependencyCollector>();
394 Clang->addDependencyCollector(PreambleDepCollector);
396 Clang->getLangOpts().CompilingPCH =
true;
399 StringRef MainFilePath = FrontendOpts.
Inputs[0].getFile();
400 auto PreambleInputBuffer = llvm::MemoryBuffer::getMemBufferCopy(
401 MainFileBuffer->getBuffer().slice(0, Bounds.
Size), MainFilePath);
404 PreprocessorOpts.
addRemappedFile(MainFilePath, PreambleInputBuffer.get());
409 PreambleInputBuffer.release());
412 std::unique_ptr<PrecompilePreambleAction> Act;
413 Act.reset(
new PrecompilePreambleAction(
414 StoreInMemory ? &Storage.asMemory().Data :
nullptr, Callbacks));
416 if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0]))
419 std::unique_ptr<PPCallbacks> DelegatedPPCallbacks =
421 if (DelegatedPPCallbacks)
422 Clang->getPreprocessor().addPPCallbacks(std::move(DelegatedPPCallbacks));
425 llvm::StringSet<> MissingFiles;
426 Clang->getPreprocessor().addPPCallbacks(
427 std::make_unique<MissingFileCollector>(
428 MissingFiles, Clang->getPreprocessor().getHeaderSearchInfo(),
429 Clang->getSourceManager()));
431 if (llvm::Error Err = Act->Execute())
432 return errorToErrorCode(std::move(Err));
437 Act->EndSourceFile();
439 if (!Act->hasEmittedPreamblePCH())
444 llvm::StringMap<PrecompiledPreamble::PreambleFileHash> FilesInPreamble;
447 for (
auto &
Filename : PreambleDepCollector->getDependencies()) {
452 auto File = *FileOrErr;
453 if (time_t ModTime =
File->getModificationTime()) {
454 FilesInPreamble[
File->getName()] =
455 PrecompiledPreamble::PreambleFileHash::createForFile(
File->getSize(),
458 llvm::MemoryBufferRef Buffer =
460 FilesInPreamble[
File->getName()] =
461 PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(Buffer);
466 std::move(Storage), std::move(PreambleBytes), PreambleEndsAtStartOfLine,
467 std::move(FilesInPreamble), std::move(MissingFiles));
471 return PreambleBounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
475 switch (Storage.getKind()) {
476 case PCHStorage::Kind::Empty:
477 assert(
false &&
"Calling getSize() on invalid PrecompiledPreamble. "
478 "Was it std::moved?");
480 case PCHStorage::Kind::InMemory:
481 return Storage.asMemory().Data.size();
482 case PCHStorage::Kind::TempFile: {
484 if (llvm::sys::fs::file_size(Storage.asFile().getFilePath(), Result))
488 "file size did not fit into size_t");
492 llvm_unreachable(
"Unhandled storage kind");
496 const llvm::MemoryBufferRef &MainFileBuffer,
498 llvm::vfs::FileSystem &VFS)
const {
501 Bounds.
Size <= MainFileBuffer.getBufferSize() &&
502 "Buffer is too large. Bounds were calculated from a different buffer?");
504 auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
506 PreambleInvocation->getPreprocessorOpts();
512 if (PreambleBytes.size() != Bounds.
Size ||
514 !std::equal(PreambleBytes.begin(), PreambleBytes.end(),
515 MainFileBuffer.getBuffer().begin()))
523 std::map<llvm::sys::fs::UniqueID, PreambleFileHash> OverriddenFiles;
524 llvm::StringSet<> OverriddenAbsPaths;
526 llvm::vfs::Status Status;
534 if (!VFS.makeAbsolute(MappedPath))
535 OverriddenAbsPaths.insert(MappedPath);
537 OverriddenFiles[Status.getUniqueID()] = PreambleFileHash::createForFile(
538 Status.getSize(), llvm::sys::toTimeT(Status.getLastModificationTime()));
542 llvm::StringMap<PreambleFileHash> OverridenFileBuffers;
544 const PrecompiledPreamble::PreambleFileHash PreambleHash =
545 PreambleFileHash::createForMemoryBuffer(RB.second->getMemBufferRef());
546 llvm::vfs::Status Status;
548 OverriddenFiles[Status.getUniqueID()] = PreambleHash;
550 OverridenFileBuffers[RB.first] = PreambleHash;
553 if (!VFS.makeAbsolute(MappedPath))
554 OverriddenAbsPaths.insert(MappedPath);
558 for (
const auto &F : FilesInPreamble) {
559 auto OverridenFileBuffer = OverridenFileBuffers.find(F.first());
560 if (OverridenFileBuffer != OverridenFileBuffers.end()) {
563 if (OverridenFileBuffer->second != F.second)
568 llvm::vfs::Status Status;
575 std::map<llvm::sys::fs::UniqueID, PreambleFileHash>::iterator Overridden =
576 OverriddenFiles.find(Status.getUniqueID());
577 if (Overridden != OverriddenFiles.end()) {
580 if (Overridden->second != F.second)
587 if (Status.getSize() != uint64_t(F.second.Size) ||
588 llvm::sys::toTimeT(Status.getLastModificationTime()) !=
592 for (
const auto &F : MissingFiles) {
594 if (OverriddenAbsPaths.count(F.getKey()))
598 if (
auto Status = VFS.status(F.getKey())) {
599 if (Status->isRegularFile())
608 llvm::MemoryBuffer *MainFileBuffer)
const {
609 PreambleBounds Bounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
610 configurePreamble(Bounds, CI, VFS, MainFileBuffer);
615 llvm::MemoryBuffer *MainFileBuffer)
const {
617 configurePreamble(Bounds, CI, VFS, MainFileBuffer);
621 PCHStorage Storage, std::vector<char> PreambleBytes,
622 bool PreambleEndsAtStartOfLine,
623 llvm::StringMap<PreambleFileHash> FilesInPreamble,
624 llvm::StringSet<> MissingFiles)
625 : Storage(
std::move(Storage)), FilesInPreamble(
std::move(FilesInPreamble)),
626 MissingFiles(
std::move(MissingFiles)),
627 PreambleBytes(
std::move(PreambleBytes)),
628 PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {
629 assert(this->Storage.getKind() != PCHStorage::Kind::Empty);
632 llvm::ErrorOr<PrecompiledPreamble::TempPCHFile>
633 PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile() {
637 if (
const char *TmpFile = ::getenv(
"CINDEXTEST_PREAMBLE_FILE"))
638 return TempPCHFile(TmpFile);
645 auto EC = llvm::sys::fs::createTemporaryFile(
"preamble",
"pch", FD, File);
649 llvm::sys::Process::SafelyCloseFileDescriptor(FD);
650 return TempPCHFile(
std::string(std::move(File).str()));
653 PrecompiledPreamble::TempPCHFile::TempPCHFile(
std::string FilePath)
654 : FilePath(
std::move(FilePath)) {
655 TemporaryFiles::getInstance().addFile(*this->FilePath);
658 PrecompiledPreamble::TempPCHFile::TempPCHFile(TempPCHFile &&Other) {
659 FilePath = std::move(Other.FilePath);
660 Other.FilePath =
None;
663 PrecompiledPreamble::TempPCHFile &PrecompiledPreamble::TempPCHFile::
664 operator=(TempPCHFile &&Other) {
665 RemoveFileIfPresent();
667 FilePath = std::move(Other.FilePath);
668 Other.FilePath =
None;
672 PrecompiledPreamble::TempPCHFile::~TempPCHFile() { RemoveFileIfPresent(); }
674 void PrecompiledPreamble::TempPCHFile::RemoveFileIfPresent() {
676 TemporaryFiles::getInstance().removeFile(*FilePath);
681 llvm::StringRef PrecompiledPreamble::TempPCHFile::getFilePath()
const {
682 assert(FilePath &&
"TempPCHFile doesn't have a FilePath. Had it been moved?");
686 PrecompiledPreamble::PCHStorage::PCHStorage(TempPCHFile File)
687 : StorageKind(
Kind::TempFile) {
688 new (&asFile()) TempPCHFile(std::move(File));
691 PrecompiledPreamble::PCHStorage::PCHStorage(InMemoryPreamble Memory)
692 : StorageKind(
Kind::InMemory) {
693 new (&asMemory()) InMemoryPreamble(std::move(Memory));
696 PrecompiledPreamble::PCHStorage::PCHStorage(PCHStorage &&Other) : PCHStorage() {
697 *
this = std::move(Other);
700 PrecompiledPreamble::PCHStorage &PrecompiledPreamble::PCHStorage::
701 operator=(PCHStorage &&Other) {
704 StorageKind = Other.StorageKind;
705 switch (StorageKind) {
710 new (&asFile()) TempPCHFile(std::move(Other.asFile()));
713 new (&asMemory()) InMemoryPreamble(std::move(Other.asMemory()));
721 PrecompiledPreamble::PCHStorage::~PCHStorage() { destroy(); }
723 PrecompiledPreamble::PCHStorage::Kind
728 PrecompiledPreamble::TempPCHFile &PrecompiledPreamble::PCHStorage::asFile() {
729 assert(
getKind() == Kind::TempFile);
730 return *
reinterpret_cast<TempPCHFile *
>(&Storage);
733 const PrecompiledPreamble::TempPCHFile &
734 PrecompiledPreamble::PCHStorage::asFile()
const {
735 return const_cast<PCHStorage *
>(
this)->asFile();
738 PrecompiledPreamble::InMemoryPreamble &
739 PrecompiledPreamble::PCHStorage::asMemory() {
740 assert(
getKind() == Kind::InMemory);
741 return *
reinterpret_cast<InMemoryPreamble *
>(&Storage);
744 const PrecompiledPreamble::InMemoryPreamble &
745 PrecompiledPreamble::PCHStorage::asMemory()
const {
746 return const_cast<PCHStorage *
>(
this)->asMemory();
749 void PrecompiledPreamble::PCHStorage::destroy() {
750 switch (StorageKind) {
754 asFile().~TempPCHFile();
757 asMemory().~InMemoryPreamble();
762 void PrecompiledPreamble::PCHStorage::setEmpty() {
764 StorageKind = Kind::Empty;
767 PrecompiledPreamble::PreambleFileHash
768 PrecompiledPreamble::PreambleFileHash::createForFile(off_t Size,
770 PreambleFileHash Result;
772 Result.ModTime = ModTime;
777 PrecompiledPreamble::PreambleFileHash
778 PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(
779 const llvm::MemoryBufferRef &Buffer) {
780 PreambleFileHash Result;
781 Result.Size = Buffer.getBufferSize();
785 MD5Ctx.update(Buffer.getBuffer().data());
786 MD5Ctx.final(Result.MD5);
791 void PrecompiledPreamble::configurePreamble(
794 llvm::MemoryBuffer *MainFileBuffer)
const {
801 PreprocessorOpts.addRemappedFile(MainFilePath, MainFileBuffer);
804 PreprocessorOpts.PrecompiledPreambleBytes.first = Bounds.
Size;
805 PreprocessorOpts.PrecompiledPreambleBytes.second =
807 PreprocessorOpts.DisablePCHOrModuleValidation =
810 setupPreambleStorage(Storage, PreprocessorOpts, VFS);
813 void PrecompiledPreamble::setupPreambleStorage(
816 if (Storage.getKind() == PCHStorage::Kind::TempFile) {
817 const TempPCHFile &PCHFile = Storage.asFile();
822 llvm::vfs::getRealFileSystem();
823 auto PCHPath = PCHFile.getFilePath();
824 if (VFS == RealFS || VFS->exists(PCHPath))
826 auto Buf = RealFS->getBufferForFile(PCHPath);
836 VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(*Buf), VFS);
838 assert(Storage.getKind() == PCHStorage::Kind::InMemory);
841 StringRef PCHPath = getInMemoryPreamblePath();
844 auto Buf = llvm::MemoryBuffer::getMemBuffer(Storage.asMemory().Data);
845 VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(Buf), VFS);
865 return "build-preamble.error";
871 return "Could not create temporary file for PCH";
873 return "CreateTargetInfo() return null";
875 return "BeginSourceFile() return an error";
877 return "Could not emit PCH";
879 return "Command line arguments must contain exactly one source file";
881 llvm_unreachable(
"unexpected BuildPreambleError");