clang  10.0.0svn
ASTReader.cpp
Go to the documentation of this file.
1 //===- ASTReader.cpp - AST File Reader ------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the ASTReader class, which reads AST files.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "ASTCommon.h"
15 #include "ASTReaderInternals.h"
16 #include "clang/AST/ASTConsumer.h"
17 #include "clang/AST/ASTContext.h"
20 #include "clang/AST/Decl.h"
21 #include "clang/AST/DeclBase.h"
22 #include "clang/AST/DeclCXX.h"
23 #include "clang/AST/DeclFriend.h"
24 #include "clang/AST/DeclGroup.h"
25 #include "clang/AST/DeclObjC.h"
26 #include "clang/AST/DeclTemplate.h"
28 #include "clang/AST/Expr.h"
29 #include "clang/AST/ExprCXX.h"
32 #include "clang/AST/ODRHash.h"
34 #include "clang/AST/TemplateBase.h"
35 #include "clang/AST/TemplateName.h"
36 #include "clang/AST/Type.h"
37 #include "clang/AST/TypeLoc.h"
41 #include "clang/Basic/Diagnostic.h"
47 #include "clang/Basic/LLVM.h"
49 #include "clang/Basic/Module.h"
53 #include "clang/Basic/Sanitizers.h"
57 #include "clang/Basic/Specifiers.h"
58 #include "clang/Basic/TargetInfo.h"
60 #include "clang/Basic/TokenKinds.h"
61 #include "clang/Basic/Version.h"
62 #include "clang/Lex/HeaderSearch.h"
64 #include "clang/Lex/MacroInfo.h"
65 #include "clang/Lex/ModuleMap.h"
67 #include "clang/Lex/Preprocessor.h"
69 #include "clang/Lex/Token.h"
71 #include "clang/Sema/Scope.h"
72 #include "clang/Sema/Sema.h"
73 #include "clang/Sema/Weak.h"
84 #include "llvm/ADT/APFloat.h"
85 #include "llvm/ADT/APInt.h"
86 #include "llvm/ADT/APSInt.h"
87 #include "llvm/ADT/ArrayRef.h"
88 #include "llvm/ADT/DenseMap.h"
89 #include "llvm/ADT/FoldingSet.h"
90 #include "llvm/ADT/Hashing.h"
91 #include "llvm/ADT/IntrusiveRefCntPtr.h"
92 #include "llvm/ADT/None.h"
93 #include "llvm/ADT/Optional.h"
94 #include "llvm/ADT/STLExtras.h"
95 #include "llvm/ADT/ScopeExit.h"
96 #include "llvm/ADT/SmallPtrSet.h"
97 #include "llvm/ADT/SmallString.h"
98 #include "llvm/ADT/SmallVector.h"
99 #include "llvm/ADT/StringExtras.h"
100 #include "llvm/ADT/StringMap.h"
101 #include "llvm/ADT/StringRef.h"
102 #include "llvm/ADT/Triple.h"
103 #include "llvm/ADT/iterator_range.h"
104 #include "llvm/Bitstream/BitstreamReader.h"
105 #include "llvm/Support/Casting.h"
106 #include "llvm/Support/Compiler.h"
107 #include "llvm/Support/Compression.h"
108 #include "llvm/Support/DJB.h"
109 #include "llvm/Support/Endian.h"
110 #include "llvm/Support/Error.h"
111 #include "llvm/Support/ErrorHandling.h"
112 #include "llvm/Support/FileSystem.h"
113 #include "llvm/Support/MemoryBuffer.h"
114 #include "llvm/Support/Path.h"
115 #include "llvm/Support/SaveAndRestore.h"
116 #include "llvm/Support/Timer.h"
117 #include "llvm/Support/VersionTuple.h"
118 #include "llvm/Support/raw_ostream.h"
119 #include <algorithm>
120 #include <cassert>
121 #include <cstddef>
122 #include <cstdint>
123 #include <cstdio>
124 #include <ctime>
125 #include <iterator>
126 #include <limits>
127 #include <map>
128 #include <memory>
129 #include <string>
130 #include <system_error>
131 #include <tuple>
132 #include <utility>
133 #include <vector>
134 
135 using namespace clang;
136 using namespace clang::serialization;
137 using namespace clang::serialization::reader;
138 using llvm::BitstreamCursor;
139 
140 //===----------------------------------------------------------------------===//
141 // ChainedASTReaderListener implementation
142 //===----------------------------------------------------------------------===//
143 
144 bool
146  return First->ReadFullVersionInformation(FullVersion) ||
147  Second->ReadFullVersionInformation(FullVersion);
148 }
149 
150 void ChainedASTReaderListener::ReadModuleName(StringRef ModuleName) {
151  First->ReadModuleName(ModuleName);
152  Second->ReadModuleName(ModuleName);
153 }
154 
155 void ChainedASTReaderListener::ReadModuleMapFile(StringRef ModuleMapPath) {
156  First->ReadModuleMapFile(ModuleMapPath);
157  Second->ReadModuleMapFile(ModuleMapPath);
158 }
159 
160 bool
162  bool Complain,
163  bool AllowCompatibleDifferences) {
164  return First->ReadLanguageOptions(LangOpts, Complain,
165  AllowCompatibleDifferences) ||
166  Second->ReadLanguageOptions(LangOpts, Complain,
167  AllowCompatibleDifferences);
168 }
169 
171  const TargetOptions &TargetOpts, bool Complain,
172  bool AllowCompatibleDifferences) {
173  return First->ReadTargetOptions(TargetOpts, Complain,
174  AllowCompatibleDifferences) ||
175  Second->ReadTargetOptions(TargetOpts, Complain,
176  AllowCompatibleDifferences);
177 }
178 
180  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts, bool Complain) {
181  return First->ReadDiagnosticOptions(DiagOpts, Complain) ||
182  Second->ReadDiagnosticOptions(DiagOpts, Complain);
183 }
184 
185 bool
187  bool Complain) {
188  return First->ReadFileSystemOptions(FSOpts, Complain) ||
189  Second->ReadFileSystemOptions(FSOpts, Complain);
190 }
191 
193  const HeaderSearchOptions &HSOpts, StringRef SpecificModuleCachePath,
194  bool Complain) {
195  return First->ReadHeaderSearchOptions(HSOpts, SpecificModuleCachePath,
196  Complain) ||
197  Second->ReadHeaderSearchOptions(HSOpts, SpecificModuleCachePath,
198  Complain);
199 }
200 
202  const PreprocessorOptions &PPOpts, bool Complain,
203  std::string &SuggestedPredefines) {
204  return First->ReadPreprocessorOptions(PPOpts, Complain,
205  SuggestedPredefines) ||
206  Second->ReadPreprocessorOptions(PPOpts, Complain, SuggestedPredefines);
207 }
208 
210  unsigned Value) {
211  First->ReadCounter(M, Value);
212  Second->ReadCounter(M, Value);
213 }
214 
216  return First->needsInputFileVisitation() ||
217  Second->needsInputFileVisitation();
218 }
219 
221  return First->needsSystemInputFileVisitation() ||
222  Second->needsSystemInputFileVisitation();
223 }
224 
226  ModuleKind Kind) {
227  First->visitModuleFile(Filename, Kind);
228  Second->visitModuleFile(Filename, Kind);
229 }
230 
232  bool isSystem,
233  bool isOverridden,
234  bool isExplicitModule) {
235  bool Continue = false;
236  if (First->needsInputFileVisitation() &&
237  (!isSystem || First->needsSystemInputFileVisitation()))
238  Continue |= First->visitInputFile(Filename, isSystem, isOverridden,
239  isExplicitModule);
240  if (Second->needsInputFileVisitation() &&
241  (!isSystem || Second->needsSystemInputFileVisitation()))
242  Continue |= Second->visitInputFile(Filename, isSystem, isOverridden,
243  isExplicitModule);
244  return Continue;
245 }
246 
248  const ModuleFileExtensionMetadata &Metadata) {
249  First->readModuleFileExtension(Metadata);
250  Second->readModuleFileExtension(Metadata);
251 }
252 
253 //===----------------------------------------------------------------------===//
254 // PCH validator implementation
255 //===----------------------------------------------------------------------===//
256 
258 
259 /// Compare the given set of language options against an existing set of
260 /// language options.
261 ///
262 /// \param Diags If non-NULL, diagnostics will be emitted via this engine.
263 /// \param AllowCompatibleDifferences If true, differences between compatible
264 /// language options will be permitted.
265 ///
266 /// \returns true if the languagae options mis-match, false otherwise.
267 static bool checkLanguageOptions(const LangOptions &LangOpts,
268  const LangOptions &ExistingLangOpts,
269  DiagnosticsEngine *Diags,
270  bool AllowCompatibleDifferences = true) {
271 #define LANGOPT(Name, Bits, Default, Description) \
272  if (ExistingLangOpts.Name != LangOpts.Name) { \
273  if (Diags) \
274  Diags->Report(diag::err_pch_langopt_mismatch) \
275  << Description << LangOpts.Name << ExistingLangOpts.Name; \
276  return true; \
277  }
278 
279 #define VALUE_LANGOPT(Name, Bits, Default, Description) \
280  if (ExistingLangOpts.Name != LangOpts.Name) { \
281  if (Diags) \
282  Diags->Report(diag::err_pch_langopt_value_mismatch) \
283  << Description; \
284  return true; \
285  }
286 
287 #define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
288  if (ExistingLangOpts.get##Name() != LangOpts.get##Name()) { \
289  if (Diags) \
290  Diags->Report(diag::err_pch_langopt_value_mismatch) \
291  << Description; \
292  return true; \
293  }
294 
295 #define COMPATIBLE_LANGOPT(Name, Bits, Default, Description) \
296  if (!AllowCompatibleDifferences) \
297  LANGOPT(Name, Bits, Default, Description)
298 
299 #define COMPATIBLE_ENUM_LANGOPT(Name, Bits, Default, Description) \
300  if (!AllowCompatibleDifferences) \
301  ENUM_LANGOPT(Name, Bits, Default, Description)
302 
303 #define COMPATIBLE_VALUE_LANGOPT(Name, Bits, Default, Description) \
304  if (!AllowCompatibleDifferences) \
305  VALUE_LANGOPT(Name, Bits, Default, Description)
306 
307 #define BENIGN_LANGOPT(Name, Bits, Default, Description)
308 #define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description)
309 #define BENIGN_VALUE_LANGOPT(Name, Type, Bits, Default, Description)
310 #include "clang/Basic/LangOptions.def"
311 
312  if (ExistingLangOpts.ModuleFeatures != LangOpts.ModuleFeatures) {
313  if (Diags)
314  Diags->Report(diag::err_pch_langopt_value_mismatch) << "module features";
315  return true;
316  }
317 
318  if (ExistingLangOpts.ObjCRuntime != LangOpts.ObjCRuntime) {
319  if (Diags)
320  Diags->Report(diag::err_pch_langopt_value_mismatch)
321  << "target Objective-C runtime";
322  return true;
323  }
324 
325  if (ExistingLangOpts.CommentOpts.BlockCommandNames !=
326  LangOpts.CommentOpts.BlockCommandNames) {
327  if (Diags)
328  Diags->Report(diag::err_pch_langopt_value_mismatch)
329  << "block command names";
330  return true;
331  }
332 
333  // Sanitizer feature mismatches are treated as compatible differences. If
334  // compatible differences aren't allowed, we still only want to check for
335  // mismatches of non-modular sanitizers (the only ones which can affect AST
336  // generation).
337  if (!AllowCompatibleDifferences) {
338  SanitizerMask ModularSanitizers = getPPTransparentSanitizers();
339  SanitizerSet ExistingSanitizers = ExistingLangOpts.Sanitize;
340  SanitizerSet ImportedSanitizers = LangOpts.Sanitize;
341  ExistingSanitizers.clear(ModularSanitizers);
342  ImportedSanitizers.clear(ModularSanitizers);
343  if (ExistingSanitizers.Mask != ImportedSanitizers.Mask) {
344  const std::string Flag = "-fsanitize=";
345  if (Diags) {
346 #define SANITIZER(NAME, ID) \
347  { \
348  bool InExistingModule = ExistingSanitizers.has(SanitizerKind::ID); \
349  bool InImportedModule = ImportedSanitizers.has(SanitizerKind::ID); \
350  if (InExistingModule != InImportedModule) \
351  Diags->Report(diag::err_pch_targetopt_feature_mismatch) \
352  << InExistingModule << (Flag + NAME); \
353  }
354 #include "clang/Basic/Sanitizers.def"
355  }
356  return true;
357  }
358  }
359 
360  return false;
361 }
362 
363 /// Compare the given set of target options against an existing set of
364 /// target options.
365 ///
366 /// \param Diags If non-NULL, diagnostics will be emitted via this engine.
367 ///
368 /// \returns true if the target options mis-match, false otherwise.
369 static bool checkTargetOptions(const TargetOptions &TargetOpts,
370  const TargetOptions &ExistingTargetOpts,
371  DiagnosticsEngine *Diags,
372  bool AllowCompatibleDifferences = true) {
373 #define CHECK_TARGET_OPT(Field, Name) \
374  if (TargetOpts.Field != ExistingTargetOpts.Field) { \
375  if (Diags) \
376  Diags->Report(diag::err_pch_targetopt_mismatch) \
377  << Name << TargetOpts.Field << ExistingTargetOpts.Field; \
378  return true; \
379  }
380 
381  // The triple and ABI must match exactly.
382  CHECK_TARGET_OPT(Triple, "target");
383  CHECK_TARGET_OPT(ABI, "target ABI");
384 
385  // We can tolerate different CPUs in many cases, notably when one CPU
386  // supports a strict superset of another. When allowing compatible
387  // differences skip this check.
388  if (!AllowCompatibleDifferences)
389  CHECK_TARGET_OPT(CPU, "target CPU");
390 
391 #undef CHECK_TARGET_OPT
392 
393  // Compare feature sets.
394  SmallVector<StringRef, 4> ExistingFeatures(
395  ExistingTargetOpts.FeaturesAsWritten.begin(),
396  ExistingTargetOpts.FeaturesAsWritten.end());
397  SmallVector<StringRef, 4> ReadFeatures(TargetOpts.FeaturesAsWritten.begin(),
398  TargetOpts.FeaturesAsWritten.end());
399  llvm::sort(ExistingFeatures);
400  llvm::sort(ReadFeatures);
401 
402  // We compute the set difference in both directions explicitly so that we can
403  // diagnose the differences differently.
404  SmallVector<StringRef, 4> UnmatchedExistingFeatures, UnmatchedReadFeatures;
405  std::set_difference(
406  ExistingFeatures.begin(), ExistingFeatures.end(), ReadFeatures.begin(),
407  ReadFeatures.end(), std::back_inserter(UnmatchedExistingFeatures));
408  std::set_difference(ReadFeatures.begin(), ReadFeatures.end(),
409  ExistingFeatures.begin(), ExistingFeatures.end(),
410  std::back_inserter(UnmatchedReadFeatures));
411 
412  // If we are allowing compatible differences and the read feature set is
413  // a strict subset of the existing feature set, there is nothing to diagnose.
414  if (AllowCompatibleDifferences && UnmatchedReadFeatures.empty())
415  return false;
416 
417  if (Diags) {
418  for (StringRef Feature : UnmatchedReadFeatures)
419  Diags->Report(diag::err_pch_targetopt_feature_mismatch)
420  << /* is-existing-feature */ false << Feature;
421  for (StringRef Feature : UnmatchedExistingFeatures)
422  Diags->Report(diag::err_pch_targetopt_feature_mismatch)
423  << /* is-existing-feature */ true << Feature;
424  }
425 
426  return !UnmatchedReadFeatures.empty() || !UnmatchedExistingFeatures.empty();
427 }
428 
429 bool
431  bool Complain,
432  bool AllowCompatibleDifferences) {
433  const LangOptions &ExistingLangOpts = PP.getLangOpts();
434  return checkLanguageOptions(LangOpts, ExistingLangOpts,
435  Complain ? &Reader.Diags : nullptr,
436  AllowCompatibleDifferences);
437 }
438 
440  bool Complain,
441  bool AllowCompatibleDifferences) {
442  const TargetOptions &ExistingTargetOpts = PP.getTargetInfo().getTargetOpts();
443  return checkTargetOptions(TargetOpts, ExistingTargetOpts,
444  Complain ? &Reader.Diags : nullptr,
445  AllowCompatibleDifferences);
446 }
447 
448 namespace {
449 
450 using MacroDefinitionsMap =
451  llvm::StringMap<std::pair<StringRef, bool /*IsUndef*/>>;
452 using DeclsMap = llvm::DenseMap<DeclarationName, SmallVector<NamedDecl *, 8>>;
453 
454 } // namespace
455 
457  DiagnosticsEngine &Diags,
458  bool Complain) {
460 
461  // Check current mappings for new -Werror mappings, and the stored mappings
462  // for cases that were explicitly mapped to *not* be errors that are now
463  // errors because of options like -Werror.
464  DiagnosticsEngine *MappingSources[] = { &Diags, &StoredDiags };
465 
466  for (DiagnosticsEngine *MappingSource : MappingSources) {
467  for (auto DiagIDMappingPair : MappingSource->getDiagnosticMappings()) {
468  diag::kind DiagID = DiagIDMappingPair.first;
469  Level CurLevel = Diags.getDiagnosticLevel(DiagID, SourceLocation());
470  if (CurLevel < DiagnosticsEngine::Error)
471  continue; // not significant
472  Level StoredLevel =
473  StoredDiags.getDiagnosticLevel(DiagID, SourceLocation());
474  if (StoredLevel < DiagnosticsEngine::Error) {
475  if (Complain)
476  Diags.Report(diag::err_pch_diagopt_mismatch) << "-Werror=" +
477  Diags.getDiagnosticIDs()->getWarningOptionForDiag(DiagID).str();
478  return true;
479  }
480  }
481  }
482 
483  return false;
484 }
485 
488  if (Ext == diag::Severity::Warning && Diags.getWarningsAsErrors())
489  return true;
490  return Ext >= diag::Severity::Error;
491 }
492 
494  DiagnosticsEngine &Diags,
495  bool IsSystem, bool Complain) {
496  // Top-level options
497  if (IsSystem) {
498  if (Diags.getSuppressSystemWarnings())
499  return false;
500  // If -Wsystem-headers was not enabled before, be conservative
501  if (StoredDiags.getSuppressSystemWarnings()) {
502  if (Complain)
503  Diags.Report(diag::err_pch_diagopt_mismatch) << "-Wsystem-headers";
504  return true;
505  }
506  }
507 
508  if (Diags.getWarningsAsErrors() && !StoredDiags.getWarningsAsErrors()) {
509  if (Complain)
510  Diags.Report(diag::err_pch_diagopt_mismatch) << "-Werror";
511  return true;
512  }
513 
514  if (Diags.getWarningsAsErrors() && Diags.getEnableAllWarnings() &&
515  !StoredDiags.getEnableAllWarnings()) {
516  if (Complain)
517  Diags.Report(diag::err_pch_diagopt_mismatch) << "-Weverything -Werror";
518  return true;
519  }
520 
521  if (isExtHandlingFromDiagsError(Diags) &&
522  !isExtHandlingFromDiagsError(StoredDiags)) {
523  if (Complain)
524  Diags.Report(diag::err_pch_diagopt_mismatch) << "-pedantic-errors";
525  return true;
526  }
527 
528  return checkDiagnosticGroupMappings(StoredDiags, Diags, Complain);
529 }
530 
531 /// Return the top import module if it is implicit, nullptr otherwise.
533  Preprocessor &PP) {
534  // If the original import came from a file explicitly generated by the user,
535  // don't check the diagnostic mappings.
536  // FIXME: currently this is approximated by checking whether this is not a
537  // module import of an implicitly-loaded module file.
538  // Note: ModuleMgr.rbegin() may not be the current module, but it must be in
539  // the transitive closure of its imports, since unrelated modules cannot be
540  // imported until after this module finishes validation.
541  ModuleFile *TopImport = &*ModuleMgr.rbegin();
542  while (!TopImport->ImportedBy.empty())
543  TopImport = TopImport->ImportedBy[0];
544  if (TopImport->Kind != MK_ImplicitModule)
545  return nullptr;
546 
547  StringRef ModuleName = TopImport->ModuleName;
548  assert(!ModuleName.empty() && "diagnostic options read before module name");
549 
550  Module *M = PP.getHeaderSearchInfo().lookupModule(ModuleName);
551  assert(M && "missing module");
552  return M;
553 }
554 
556  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts, bool Complain) {
557  DiagnosticsEngine &ExistingDiags = PP.getDiagnostics();
558  IntrusiveRefCntPtr<DiagnosticIDs> DiagIDs(ExistingDiags.getDiagnosticIDs());
560  new DiagnosticsEngine(DiagIDs, DiagOpts.get()));
561  // This should never fail, because we would have processed these options
562  // before writing them to an ASTFile.
563  ProcessWarningOptions(*Diags, *DiagOpts, /*Report*/false);
564 
565  ModuleManager &ModuleMgr = Reader.getModuleManager();
566  assert(ModuleMgr.size() >= 1 && "what ASTFile is this then");
567 
568  Module *TopM = getTopImportImplicitModule(ModuleMgr, PP);
569  if (!TopM)
570  return false;
571 
572  // FIXME: if the diagnostics are incompatible, save a DiagnosticOptions that
573  // contains the union of their flags.
574  return checkDiagnosticMappings(*Diags, ExistingDiags, TopM->IsSystem,
575  Complain);
576 }
577 
578 /// Collect the macro definitions provided by the given preprocessor
579 /// options.
580 static void
582  MacroDefinitionsMap &Macros,
583  SmallVectorImpl<StringRef> *MacroNames = nullptr) {
584  for (unsigned I = 0, N = PPOpts.Macros.size(); I != N; ++I) {
585  StringRef Macro = PPOpts.Macros[I].first;
586  bool IsUndef = PPOpts.Macros[I].second;
587 
588  std::pair<StringRef, StringRef> MacroPair = Macro.split('=');
589  StringRef MacroName = MacroPair.first;
590  StringRef MacroBody = MacroPair.second;
591 
592  // For an #undef'd macro, we only care about the name.
593  if (IsUndef) {
594  if (MacroNames && !Macros.count(MacroName))
595  MacroNames->push_back(MacroName);
596 
597  Macros[MacroName] = std::make_pair("", true);
598  continue;
599  }
600 
601  // For a #define'd macro, figure out the actual definition.
602  if (MacroName.size() == Macro.size())
603  MacroBody = "1";
604  else {
605  // Note: GCC drops anything following an end-of-line character.
606  StringRef::size_type End = MacroBody.find_first_of("\n\r");
607  MacroBody = MacroBody.substr(0, End);
608  }
609 
610  if (MacroNames && !Macros.count(MacroName))
611  MacroNames->push_back(MacroName);
612  Macros[MacroName] = std::make_pair(MacroBody, false);
613  }
614 }
615 
616 /// Check the preprocessor options deserialized from the control block
617 /// against the preprocessor options in an existing preprocessor.
618 ///
619 /// \param Diags If non-null, produce diagnostics for any mismatches incurred.
620 /// \param Validate If true, validate preprocessor options. If false, allow
621 /// macros defined by \p ExistingPPOpts to override those defined by
622 /// \p PPOpts in SuggestedPredefines.
624  const PreprocessorOptions &ExistingPPOpts,
625  DiagnosticsEngine *Diags,
626  FileManager &FileMgr,
627  std::string &SuggestedPredefines,
628  const LangOptions &LangOpts,
629  bool Validate = true) {
630  // Check macro definitions.
631  MacroDefinitionsMap ASTFileMacros;
632  collectMacroDefinitions(PPOpts, ASTFileMacros);
633  MacroDefinitionsMap ExistingMacros;
634  SmallVector<StringRef, 4> ExistingMacroNames;
635  collectMacroDefinitions(ExistingPPOpts, ExistingMacros, &ExistingMacroNames);
636 
637  for (unsigned I = 0, N = ExistingMacroNames.size(); I != N; ++I) {
638  // Dig out the macro definition in the existing preprocessor options.
639  StringRef MacroName = ExistingMacroNames[I];
640  std::pair<StringRef, bool> Existing = ExistingMacros[MacroName];
641 
642  // Check whether we know anything about this macro name or not.
643  llvm::StringMap<std::pair<StringRef, bool /*IsUndef*/>>::iterator Known =
644  ASTFileMacros.find(MacroName);
645  if (!Validate || Known == ASTFileMacros.end()) {
646  // FIXME: Check whether this identifier was referenced anywhere in the
647  // AST file. If so, we should reject the AST file. Unfortunately, this
648  // information isn't in the control block. What shall we do about it?
649 
650  if (Existing.second) {
651  SuggestedPredefines += "#undef ";
652  SuggestedPredefines += MacroName.str();
653  SuggestedPredefines += '\n';
654  } else {
655  SuggestedPredefines += "#define ";
656  SuggestedPredefines += MacroName.str();
657  SuggestedPredefines += ' ';
658  SuggestedPredefines += Existing.first.str();
659  SuggestedPredefines += '\n';
660  }
661  continue;
662  }
663 
664  // If the macro was defined in one but undef'd in the other, we have a
665  // conflict.
666  if (Existing.second != Known->second.second) {
667  if (Diags) {
668  Diags->Report(diag::err_pch_macro_def_undef)
669  << MacroName << Known->second.second;
670  }
671  return true;
672  }
673 
674  // If the macro was #undef'd in both, or if the macro bodies are identical,
675  // it's fine.
676  if (Existing.second || Existing.first == Known->second.first)
677  continue;
678 
679  // The macro bodies differ; complain.
680  if (Diags) {
681  Diags->Report(diag::err_pch_macro_def_conflict)
682  << MacroName << Known->second.first << Existing.first;
683  }
684  return true;
685  }
686 
687  // Check whether we're using predefines.
688  if (PPOpts.UsePredefines != ExistingPPOpts.UsePredefines && Validate) {
689  if (Diags) {
690  Diags->Report(diag::err_pch_undef) << ExistingPPOpts.UsePredefines;
691  }
692  return true;
693  }
694 
695  // Detailed record is important since it is used for the module cache hash.
696  if (LangOpts.Modules &&
697  PPOpts.DetailedRecord != ExistingPPOpts.DetailedRecord && Validate) {
698  if (Diags) {
699  Diags->Report(diag::err_pch_pp_detailed_record) << PPOpts.DetailedRecord;
700  }
701  return true;
702  }
703 
704  // Compute the #include and #include_macros lines we need.
705  for (unsigned I = 0, N = ExistingPPOpts.Includes.size(); I != N; ++I) {
706  StringRef File = ExistingPPOpts.Includes[I];
707 
708  if (!ExistingPPOpts.ImplicitPCHInclude.empty() &&
709  !ExistingPPOpts.PCHThroughHeader.empty()) {
710  // In case the through header is an include, we must add all the includes
711  // to the predefines so the start point can be determined.
712  SuggestedPredefines += "#include \"";
713  SuggestedPredefines += File;
714  SuggestedPredefines += "\"\n";
715  continue;
716  }
717 
718  if (File == ExistingPPOpts.ImplicitPCHInclude)
719  continue;
720 
721  if (std::find(PPOpts.Includes.begin(), PPOpts.Includes.end(), File)
722  != PPOpts.Includes.end())
723  continue;
724 
725  SuggestedPredefines += "#include \"";
726  SuggestedPredefines += File;
727  SuggestedPredefines += "\"\n";
728  }
729 
730  for (unsigned I = 0, N = ExistingPPOpts.MacroIncludes.size(); I != N; ++I) {
731  StringRef File = ExistingPPOpts.MacroIncludes[I];
732  if (std::find(PPOpts.MacroIncludes.begin(), PPOpts.MacroIncludes.end(),
733  File)
734  != PPOpts.MacroIncludes.end())
735  continue;
736 
737  SuggestedPredefines += "#__include_macros \"";
738  SuggestedPredefines += File;
739  SuggestedPredefines += "\"\n##\n";
740  }
741 
742  return false;
743 }
744 
746  bool Complain,
747  std::string &SuggestedPredefines) {
748  const PreprocessorOptions &ExistingPPOpts = PP.getPreprocessorOpts();
749 
750  return checkPreprocessorOptions(PPOpts, ExistingPPOpts,
751  Complain? &Reader.Diags : nullptr,
752  PP.getFileManager(),
753  SuggestedPredefines,
754  PP.getLangOpts());
755 }
756 
758  const PreprocessorOptions &PPOpts,
759  bool Complain,
760  std::string &SuggestedPredefines) {
761  return checkPreprocessorOptions(PPOpts,
762  PP.getPreprocessorOpts(),
763  nullptr,
764  PP.getFileManager(),
765  SuggestedPredefines,
766  PP.getLangOpts(),
767  false);
768 }
769 
770 /// Check the header search options deserialized from the control block
771 /// against the header search options in an existing preprocessor.
772 ///
773 /// \param Diags If non-null, produce diagnostics for any mismatches incurred.
775  StringRef SpecificModuleCachePath,
776  StringRef ExistingModuleCachePath,
777  DiagnosticsEngine *Diags,
778  const LangOptions &LangOpts) {
779  if (LangOpts.Modules) {
780  if (SpecificModuleCachePath != ExistingModuleCachePath) {
781  if (Diags)
782  Diags->Report(diag::err_pch_modulecache_mismatch)
783  << SpecificModuleCachePath << ExistingModuleCachePath;
784  return true;
785  }
786  }
787 
788  return false;
789 }
790 
792  StringRef SpecificModuleCachePath,
793  bool Complain) {
794  return checkHeaderSearchOptions(HSOpts, SpecificModuleCachePath,
795  PP.getHeaderSearchInfo().getModuleCachePath(),
796  Complain ? &Reader.Diags : nullptr,
797  PP.getLangOpts());
798 }
799 
800 void PCHValidator::ReadCounter(const ModuleFile &M, unsigned Value) {
801  PP.setCounterValue(Value);
802 }
803 
804 //===----------------------------------------------------------------------===//
805 // AST reader implementation
806 //===----------------------------------------------------------------------===//
807 
809  bool TakeOwnership) {
810  DeserializationListener = Listener;
811  OwnsDeserializationListener = TakeOwnership;
812 }
813 
815  return serialization::ComputeHash(Sel);
816 }
817 
818 std::pair<unsigned, unsigned>
820  using namespace llvm::support;
821 
822  unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
823  unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
824  return std::make_pair(KeyLen, DataLen);
825 }
826 
828 ASTSelectorLookupTrait::ReadKey(const unsigned char* d, unsigned) {
829  using namespace llvm::support;
830 
831  SelectorTable &SelTable = Reader.getContext().Selectors;
832  unsigned N = endian::readNext<uint16_t, little, unaligned>(d);
833  IdentifierInfo *FirstII = Reader.getLocalIdentifier(
834  F, endian::readNext<uint32_t, little, unaligned>(d));
835  if (N == 0)
836  return SelTable.getNullarySelector(FirstII);
837  else if (N == 1)
838  return SelTable.getUnarySelector(FirstII);
839 
841  Args.push_back(FirstII);
842  for (unsigned I = 1; I != N; ++I)
843  Args.push_back(Reader.getLocalIdentifier(
844  F, endian::readNext<uint32_t, little, unaligned>(d)));
845 
846  return SelTable.getSelector(N, Args.data());
847 }
848 
851  unsigned DataLen) {
852  using namespace llvm::support;
853 
854  data_type Result;
855 
856  Result.ID = Reader.getGlobalSelectorID(
857  F, endian::readNext<uint32_t, little, unaligned>(d));
858  unsigned FullInstanceBits = endian::readNext<uint16_t, little, unaligned>(d);
859  unsigned FullFactoryBits = endian::readNext<uint16_t, little, unaligned>(d);
860  Result.InstanceBits = FullInstanceBits & 0x3;
861  Result.InstanceHasMoreThanOneDecl = (FullInstanceBits >> 2) & 0x1;
862  Result.FactoryBits = FullFactoryBits & 0x3;
863  Result.FactoryHasMoreThanOneDecl = (FullFactoryBits >> 2) & 0x1;
864  unsigned NumInstanceMethods = FullInstanceBits >> 3;
865  unsigned NumFactoryMethods = FullFactoryBits >> 3;
866 
867  // Load instance methods
868  for (unsigned I = 0; I != NumInstanceMethods; ++I) {
869  if (ObjCMethodDecl *Method = Reader.GetLocalDeclAs<ObjCMethodDecl>(
870  F, endian::readNext<uint32_t, little, unaligned>(d)))
871  Result.Instance.push_back(Method);
872  }
873 
874  // Load factory methods
875  for (unsigned I = 0; I != NumFactoryMethods; ++I) {
876  if (ObjCMethodDecl *Method = Reader.GetLocalDeclAs<ObjCMethodDecl>(
877  F, endian::readNext<uint32_t, little, unaligned>(d)))
878  Result.Factory.push_back(Method);
879  }
880 
881  return Result;
882 }
883 
885  return llvm::djbHash(a);
886 }
887 
888 std::pair<unsigned, unsigned>
890  using namespace llvm::support;
891 
892  unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
893  unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
894  return std::make_pair(KeyLen, DataLen);
895 }
896 
898 ASTIdentifierLookupTraitBase::ReadKey(const unsigned char* d, unsigned n) {
899  assert(n >= 2 && d[n-1] == '\0');
900  return StringRef((const char*) d, n-1);
901 }
902 
903 /// Whether the given identifier is "interesting".
905  bool IsModule) {
906  return II.hadMacroDefinition() ||
907  II.isPoisoned() ||
908  (IsModule ? II.hasRevertedBuiltin() : II.getObjCOrBuiltinID()) ||
910  (!(IsModule && Reader.getPreprocessor().getLangOpts().CPlusPlus) &&
911  II.getFETokenInfo());
912 }
913 
914 static bool readBit(unsigned &Bits) {
915  bool Value = Bits & 0x1;
916  Bits >>= 1;
917  return Value;
918 }
919 
921  using namespace llvm::support;
922 
923  unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
924  return Reader.getGlobalIdentifierID(F, RawID >> 1);
925 }
926 
927 static void markIdentifierFromAST(ASTReader &Reader, IdentifierInfo &II) {
928  if (!II.isFromAST()) {
929  II.setIsFromAST();
930  bool IsModule = Reader.getPreprocessor().getCurrentModule() != nullptr;
931  if (isInterestingIdentifier(Reader, II, IsModule))
933  }
934 }
935 
937  const unsigned char* d,
938  unsigned DataLen) {
939  using namespace llvm::support;
940 
941  unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
942  bool IsInteresting = RawID & 0x01;
943 
944  // Wipe out the "is interesting" bit.
945  RawID = RawID >> 1;
946 
947  // Build the IdentifierInfo and link the identifier ID with it.
948  IdentifierInfo *II = KnownII;
949  if (!II) {
950  II = &Reader.getIdentifierTable().getOwn(k);
951  KnownII = II;
952  }
953  markIdentifierFromAST(Reader, *II);
954  Reader.markIdentifierUpToDate(II);
955 
956  IdentID ID = Reader.getGlobalIdentifierID(F, RawID);
957  if (!IsInteresting) {
958  // For uninteresting identifiers, there's nothing else to do. Just notify
959  // the reader that we've finished loading this identifier.
960  Reader.SetIdentifierInfo(ID, II);
961  return II;
962  }
963 
964  unsigned ObjCOrBuiltinID = endian::readNext<uint16_t, little, unaligned>(d);
965  unsigned Bits = endian::readNext<uint16_t, little, unaligned>(d);
966  bool CPlusPlusOperatorKeyword = readBit(Bits);
967  bool HasRevertedTokenIDToIdentifier = readBit(Bits);
968  bool HasRevertedBuiltin = readBit(Bits);
969  bool Poisoned = readBit(Bits);
970  bool ExtensionToken = readBit(Bits);
971  bool HadMacroDefinition = readBit(Bits);
972 
973  assert(Bits == 0 && "Extra bits in the identifier?");
974  DataLen -= 8;
975 
976  // Set or check the various bits in the IdentifierInfo structure.
977  // Token IDs are read-only.
978  if (HasRevertedTokenIDToIdentifier && II->getTokenID() != tok::identifier)
980  if (!F.isModule())
981  II->setObjCOrBuiltinID(ObjCOrBuiltinID);
982  else if (HasRevertedBuiltin && II->getBuiltinID()) {
983  II->revertBuiltin();
984  assert((II->hasRevertedBuiltin() ||
985  II->getObjCOrBuiltinID() == ObjCOrBuiltinID) &&
986  "Incorrect ObjC keyword or builtin ID");
987  }
988  assert(II->isExtensionToken() == ExtensionToken &&
989  "Incorrect extension token flag");
990  (void)ExtensionToken;
991  if (Poisoned)
992  II->setIsPoisoned(true);
993  assert(II->isCPlusPlusOperatorKeyword() == CPlusPlusOperatorKeyword &&
994  "Incorrect C++ operator keyword flag");
995  (void)CPlusPlusOperatorKeyword;
996 
997  // If this identifier is a macro, deserialize the macro
998  // definition.
999  if (HadMacroDefinition) {
1000  uint32_t MacroDirectivesOffset =
1001  endian::readNext<uint32_t, little, unaligned>(d);
1002  DataLen -= 4;
1003 
1004  Reader.addPendingMacro(II, &F, MacroDirectivesOffset);
1005  }
1006 
1007  Reader.SetIdentifierInfo(ID, II);
1008 
1009  // Read all of the declarations visible at global scope with this
1010  // name.
1011  if (DataLen > 0) {
1012  SmallVector<uint32_t, 4> DeclIDs;
1013  for (; DataLen > 0; DataLen -= 4)
1014  DeclIDs.push_back(Reader.getGlobalDeclID(
1015  F, endian::readNext<uint32_t, little, unaligned>(d)));
1016  Reader.SetGloballyVisibleDecls(II, DeclIDs);
1017  }
1018 
1019  return II;
1020 }
1021 
1023  : Kind(Name.getNameKind()) {
1024  switch (Kind) {
1026  Data = (uint64_t)Name.getAsIdentifierInfo();
1027  break;
1031  Data = (uint64_t)Name.getObjCSelector().getAsOpaquePtr();
1032  break;
1034  Data = Name.getCXXOverloadedOperator();
1035  break;
1037  Data = (uint64_t)Name.getCXXLiteralIdentifier();
1038  break;
1040  Data = (uint64_t)Name.getCXXDeductionGuideTemplate()
1042  break;
1047  Data = 0;
1048  break;
1049  }
1050 }
1051 
1053  llvm::FoldingSetNodeID ID;
1054  ID.AddInteger(Kind);
1055 
1056  switch (Kind) {
1060  ID.AddString(((IdentifierInfo*)Data)->getName());
1061  break;
1065  ID.AddInteger(serialization::ComputeHash(Selector(Data)));
1066  break;
1068  ID.AddInteger((OverloadedOperatorKind)Data);
1069  break;
1074  break;
1075  }
1076 
1077  return ID.ComputeHash();
1078 }
1079 
1080 ModuleFile *
1081 ASTDeclContextNameLookupTrait::ReadFileRef(const unsigned char *&d) {
1082  using namespace llvm::support;
1083 
1084  uint32_t ModuleFileID = endian::readNext<uint32_t, little, unaligned>(d);
1085  return Reader.getLocalModuleFile(F, ModuleFileID);
1086 }
1087 
1088 std::pair<unsigned, unsigned>
1089 ASTDeclContextNameLookupTrait::ReadKeyDataLength(const unsigned char *&d) {
1090  using namespace llvm::support;
1091 
1092  unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
1093  unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
1094  return std::make_pair(KeyLen, DataLen);
1095 }
1096 
1098 ASTDeclContextNameLookupTrait::ReadKey(const unsigned char *d, unsigned) {
1099  using namespace llvm::support;
1100 
1101  auto Kind = (DeclarationName::NameKind)*d++;
1102  uint64_t Data;
1103  switch (Kind) {
1107  Data = (uint64_t)Reader.getLocalIdentifier(
1108  F, endian::readNext<uint32_t, little, unaligned>(d));
1109  break;
1113  Data =
1114  (uint64_t)Reader.getLocalSelector(
1115  F, endian::readNext<uint32_t, little, unaligned>(
1116  d)).getAsOpaquePtr();
1117  break;
1119  Data = *d++; // OverloadedOperatorKind
1120  break;
1125  Data = 0;
1126  break;
1127  }
1128 
1129  return DeclarationNameKey(Kind, Data);
1130 }
1131 
1132 void ASTDeclContextNameLookupTrait::ReadDataInto(internal_key_type,
1133  const unsigned char *d,
1134  unsigned DataLen,
1135  data_type_builder &Val) {
1136  using namespace llvm::support;
1137 
1138  for (unsigned NumDecls = DataLen / 4; NumDecls; --NumDecls) {
1139  uint32_t LocalID = endian::readNext<uint32_t, little, unaligned>(d);
1140  Val.insert(Reader.getGlobalDeclID(F, LocalID));
1141  }
1142 }
1143 
1144 bool ASTReader::ReadLexicalDeclContextStorage(ModuleFile &M,
1145  BitstreamCursor &Cursor,
1146  uint64_t Offset,
1147  DeclContext *DC) {
1148  assert(Offset != 0);
1149 
1150  SavedStreamPosition SavedPosition(Cursor);
1151  if (llvm::Error Err = Cursor.JumpToBit(Offset)) {
1152  Error(std::move(Err));
1153  return true;
1154  }
1155 
1156  RecordData Record;
1157  StringRef Blob;
1158  Expected<unsigned> MaybeCode = Cursor.ReadCode();
1159  if (!MaybeCode) {
1160  Error(MaybeCode.takeError());
1161  return true;
1162  }
1163  unsigned Code = MaybeCode.get();
1164 
1165  Expected<unsigned> MaybeRecCode = Cursor.readRecord(Code, Record, &Blob);
1166  if (!MaybeRecCode) {
1167  Error(MaybeRecCode.takeError());
1168  return true;
1169  }
1170  unsigned RecCode = MaybeRecCode.get();
1171  if (RecCode != DECL_CONTEXT_LEXICAL) {
1172  Error("Expected lexical block");
1173  return true;
1174  }
1175 
1176  assert(!isa<TranslationUnitDecl>(DC) &&
1177  "expected a TU_UPDATE_LEXICAL record for TU");
1178  // If we are handling a C++ class template instantiation, we can see multiple
1179  // lexical updates for the same record. It's important that we select only one
1180  // of them, so that field numbering works properly. Just pick the first one we
1181  // see.
1182  auto &Lex = LexicalDecls[DC];
1183  if (!Lex.first) {
1184  Lex = std::make_pair(
1185  &M, llvm::makeArrayRef(
1186  reinterpret_cast<const llvm::support::unaligned_uint32_t *>(
1187  Blob.data()),
1188  Blob.size() / 4));
1189  }
1190  DC->setHasExternalLexicalStorage(true);
1191  return false;
1192 }
1193 
1194 bool ASTReader::ReadVisibleDeclContextStorage(ModuleFile &M,
1195  BitstreamCursor &Cursor,
1196  uint64_t Offset,
1197  DeclID ID) {
1198  assert(Offset != 0);
1199 
1200  SavedStreamPosition SavedPosition(Cursor);
1201  if (llvm::Error Err = Cursor.JumpToBit(Offset)) {
1202  Error(std::move(Err));
1203  return true;
1204  }
1205 
1206  RecordData Record;
1207  StringRef Blob;
1208  Expected<unsigned> MaybeCode = Cursor.ReadCode();
1209  if (!MaybeCode) {
1210  Error(MaybeCode.takeError());
1211  return true;
1212  }
1213  unsigned Code = MaybeCode.get();
1214 
1215  Expected<unsigned> MaybeRecCode = Cursor.readRecord(Code, Record, &Blob);
1216  if (!MaybeRecCode) {
1217  Error(MaybeRecCode.takeError());
1218  return true;
1219  }
1220  unsigned RecCode = MaybeRecCode.get();
1221  if (RecCode != DECL_CONTEXT_VISIBLE) {
1222  Error("Expected visible lookup table block");
1223  return true;
1224  }
1225 
1226  // We can't safely determine the primary context yet, so delay attaching the
1227  // lookup table until we're done with recursive deserialization.
1228  auto *Data = (const unsigned char*)Blob.data();
1229  PendingVisibleUpdates[ID].push_back(PendingVisibleUpdate{&M, Data});
1230  return false;
1231 }
1232 
1233 void ASTReader::Error(StringRef Msg) const {
1234  Error(diag::err_fe_pch_malformed, Msg);
1235  if (PP.getLangOpts().Modules && !Diags.isDiagnosticInFlight() &&
1236  !PP.getHeaderSearchInfo().getModuleCachePath().empty()) {
1237  Diag(diag::note_module_cache_path)
1238  << PP.getHeaderSearchInfo().getModuleCachePath();
1239  }
1240 }
1241 
1242 void ASTReader::Error(unsigned DiagID,
1243  StringRef Arg1, StringRef Arg2) const {
1244  if (Diags.isDiagnosticInFlight())
1245  Diags.SetDelayedDiagnostic(DiagID, Arg1, Arg2);
1246  else
1247  Diag(DiagID) << Arg1 << Arg2;
1248 }
1249 
1250 void ASTReader::Error(llvm::Error &&Err) const {
1251  Error(toString(std::move(Err)));
1252 }
1253 
1254 //===----------------------------------------------------------------------===//
1255 // Source Manager Deserialization
1256 //===----------------------------------------------------------------------===//
1257 
1258 /// Read the line table in the source manager block.
1259 /// \returns true if there was an error.
1260 bool ASTReader::ParseLineTable(ModuleFile &F,
1261  const RecordData &Record) {
1262  unsigned Idx = 0;
1263  LineTableInfo &LineTable = SourceMgr.getLineTable();
1264 
1265  // Parse the file names
1266  std::map<int, int> FileIDs;
1267  FileIDs[-1] = -1; // For unspecified filenames.
1268  for (unsigned I = 0; Record[Idx]; ++I) {
1269  // Extract the file name
1270  auto Filename = ReadPath(F, Record, Idx);
1271  FileIDs[I] = LineTable.getLineTableFilenameID(Filename);
1272  }
1273  ++Idx;
1274 
1275  // Parse the line entries
1276  std::vector<LineEntry> Entries;
1277  while (Idx < Record.size()) {
1278  int FID = Record[Idx++];
1279  assert(FID >= 0 && "Serialized line entries for non-local file.");
1280  // Remap FileID from 1-based old view.
1281  FID += F.SLocEntryBaseID - 1;
1282 
1283  // Extract the line entries
1284  unsigned NumEntries = Record[Idx++];
1285  assert(NumEntries && "no line entries for file ID");
1286  Entries.clear();
1287  Entries.reserve(NumEntries);
1288  for (unsigned I = 0; I != NumEntries; ++I) {
1289  unsigned FileOffset = Record[Idx++];
1290  unsigned LineNo = Record[Idx++];
1291  int FilenameID = FileIDs[Record[Idx++]];
1293  = (SrcMgr::CharacteristicKind)Record[Idx++];
1294  unsigned IncludeOffset = Record[Idx++];
1295  Entries.push_back(LineEntry::get(FileOffset, LineNo, FilenameID,
1296  FileKind, IncludeOffset));
1297  }
1298  LineTable.AddEntry(FileID::get(FID), Entries);
1299  }
1300 
1301  return false;
1302 }
1303 
1304 /// Read a source manager block
1305 bool ASTReader::ReadSourceManagerBlock(ModuleFile &F) {
1306  using namespace SrcMgr;
1307 
1308  BitstreamCursor &SLocEntryCursor = F.SLocEntryCursor;
1309 
1310  // Set the source-location entry cursor to the current position in
1311  // the stream. This cursor will be used to read the contents of the
1312  // source manager block initially, and then lazily read
1313  // source-location entries as needed.
1314  SLocEntryCursor = F.Stream;
1315 
1316  // The stream itself is going to skip over the source manager block.
1317  if (llvm::Error Err = F.Stream.SkipBlock()) {
1318  Error(std::move(Err));
1319  return true;
1320  }
1321 
1322  // Enter the source manager block.
1323  if (llvm::Error Err =
1324  SLocEntryCursor.EnterSubBlock(SOURCE_MANAGER_BLOCK_ID)) {
1325  Error(std::move(Err));
1326  return true;
1327  }
1328 
1329  RecordData Record;
1330  while (true) {
1332  SLocEntryCursor.advanceSkippingSubblocks();
1333  if (!MaybeE) {
1334  Error(MaybeE.takeError());
1335  return true;
1336  }
1337  llvm::BitstreamEntry E = MaybeE.get();
1338 
1339  switch (E.Kind) {
1340  case llvm::BitstreamEntry::SubBlock: // Handled for us already.
1342  Error("malformed block record in AST file");
1343  return true;
1344  case llvm::BitstreamEntry::EndBlock:
1345  return false;
1346  case llvm::BitstreamEntry::Record:
1347  // The interesting case.
1348  break;
1349  }
1350 
1351  // Read a record.
1352  Record.clear();
1353  StringRef Blob;
1354  Expected<unsigned> MaybeRecord =
1355  SLocEntryCursor.readRecord(E.ID, Record, &Blob);
1356  if (!MaybeRecord) {
1357  Error(MaybeRecord.takeError());
1358  return true;
1359  }
1360  switch (MaybeRecord.get()) {
1361  default: // Default behavior: ignore.
1362  break;
1363 
1364  case SM_SLOC_FILE_ENTRY:
1365  case SM_SLOC_BUFFER_ENTRY:
1367  // Once we hit one of the source location entries, we're done.
1368  return false;
1369  }
1370  }
1371 }
1372 
1373 /// If a header file is not found at the path that we expect it to be
1374 /// and the PCH file was moved from its original location, try to resolve the
1375 /// file by assuming that header+PCH were moved together and the header is in
1376 /// the same place relative to the PCH.
1377 static std::string
1379  const std::string &OriginalDir,
1380  const std::string &CurrDir) {
1381  assert(OriginalDir != CurrDir &&
1382  "No point trying to resolve the file if the PCH dir didn't change");
1383 
1384  using namespace llvm::sys;
1385 
1386  SmallString<128> filePath(Filename);
1387  fs::make_absolute(filePath);
1388  assert(path::is_absolute(OriginalDir));
1389  SmallString<128> currPCHPath(CurrDir);
1390 
1391  path::const_iterator fileDirI = path::begin(path::parent_path(filePath)),
1392  fileDirE = path::end(path::parent_path(filePath));
1393  path::const_iterator origDirI = path::begin(OriginalDir),
1394  origDirE = path::end(OriginalDir);
1395  // Skip the common path components from filePath and OriginalDir.
1396  while (fileDirI != fileDirE && origDirI != origDirE &&
1397  *fileDirI == *origDirI) {
1398  ++fileDirI;
1399  ++origDirI;
1400  }
1401  for (; origDirI != origDirE; ++origDirI)
1402  path::append(currPCHPath, "..");
1403  path::append(currPCHPath, fileDirI, fileDirE);
1404  path::append(currPCHPath, path::filename(Filename));
1405  return currPCHPath.str();
1406 }
1407 
1409  if (ID == 0)
1410  return false;
1411 
1412  if (unsigned(-ID) - 2 >= getTotalNumSLocs() || ID > 0) {
1413  Error("source location entry ID out-of-range for AST file");
1414  return true;
1415  }
1416 
1417  // Local helper to read the (possibly-compressed) buffer data following the
1418  // entry record.
1419  auto ReadBuffer = [this](
1420  BitstreamCursor &SLocEntryCursor,
1421  StringRef Name) -> std::unique_ptr<llvm::MemoryBuffer> {
1422  RecordData Record;
1423  StringRef Blob;
1424  Expected<unsigned> MaybeCode = SLocEntryCursor.ReadCode();
1425  if (!MaybeCode) {
1426  Error(MaybeCode.takeError());
1427  return nullptr;
1428  }
1429  unsigned Code = MaybeCode.get();
1430 
1431  Expected<unsigned> MaybeRecCode =
1432  SLocEntryCursor.readRecord(Code, Record, &Blob);
1433  if (!MaybeRecCode) {
1434  Error(MaybeRecCode.takeError());
1435  return nullptr;
1436  }
1437  unsigned RecCode = MaybeRecCode.get();
1438 
1439  if (RecCode == SM_SLOC_BUFFER_BLOB_COMPRESSED) {
1440  if (!llvm::zlib::isAvailable()) {
1441  Error("zlib is not available");
1442  return nullptr;
1443  }
1444  SmallString<0> Uncompressed;
1445  if (llvm::Error E =
1446  llvm::zlib::uncompress(Blob, Uncompressed, Record[0])) {
1447  Error("could not decompress embedded file contents: " +
1448  llvm::toString(std::move(E)));
1449  return nullptr;
1450  }
1451  return llvm::MemoryBuffer::getMemBufferCopy(Uncompressed, Name);
1452  } else if (RecCode == SM_SLOC_BUFFER_BLOB) {
1453  return llvm::MemoryBuffer::getMemBuffer(Blob.drop_back(1), Name, true);
1454  } else {
1455  Error("AST record has invalid code");
1456  return nullptr;
1457  }
1458  };
1459 
1460  ModuleFile *F = GlobalSLocEntryMap.find(-ID)->second;
1461  if (llvm::Error Err = F->SLocEntryCursor.JumpToBit(
1462  F->SLocEntryOffsets[ID - F->SLocEntryBaseID])) {
1463  Error(std::move(Err));
1464  return true;
1465  }
1466 
1467  BitstreamCursor &SLocEntryCursor = F->SLocEntryCursor;
1468  unsigned BaseOffset = F->SLocEntryBaseOffset;
1469 
1470  ++NumSLocEntriesRead;
1471  Expected<llvm::BitstreamEntry> MaybeEntry = SLocEntryCursor.advance();
1472  if (!MaybeEntry) {
1473  Error(MaybeEntry.takeError());
1474  return true;
1475  }
1476  llvm::BitstreamEntry Entry = MaybeEntry.get();
1477 
1478  if (Entry.Kind != llvm::BitstreamEntry::Record) {
1479  Error("incorrectly-formatted source location entry in AST file");
1480  return true;
1481  }
1482 
1483  RecordData Record;
1484  StringRef Blob;
1485  Expected<unsigned> MaybeSLOC =
1486  SLocEntryCursor.readRecord(Entry.ID, Record, &Blob);
1487  if (!MaybeSLOC) {
1488  Error(MaybeSLOC.takeError());
1489  return true;
1490  }
1491  switch (MaybeSLOC.get()) {
1492  default:
1493  Error("incorrectly-formatted source location entry in AST file");
1494  return true;
1495 
1496  case SM_SLOC_FILE_ENTRY: {
1497  // We will detect whether a file changed and return 'Failure' for it, but
1498  // we will also try to fail gracefully by setting up the SLocEntry.
1499  unsigned InputID = Record[4];
1500  InputFile IF = getInputFile(*F, InputID);
1501  const FileEntry *File = IF.getFile();
1502  bool OverriddenBuffer = IF.isOverridden();
1503 
1504  // Note that we only check if a File was returned. If it was out-of-date
1505  // we have complained but we will continue creating a FileID to recover
1506  // gracefully.
1507  if (!File)
1508  return true;
1509 
1510  SourceLocation IncludeLoc = ReadSourceLocation(*F, Record[1]);
1511  if (IncludeLoc.isInvalid() && F->Kind != MK_MainFile) {
1512  // This is the module's main file.
1513  IncludeLoc = getImportLocation(F);
1514  }
1516  FileCharacter = (SrcMgr::CharacteristicKind)Record[2];
1517  // FIXME: The FileID should be created from the FileEntryRef.
1518  FileID FID = SourceMgr.createFileID(File, IncludeLoc, FileCharacter,
1519  ID, BaseOffset + Record[0]);
1520  SrcMgr::FileInfo &FileInfo =
1521  const_cast<SrcMgr::FileInfo&>(SourceMgr.getSLocEntry(FID).getFile());
1522  FileInfo.NumCreatedFIDs = Record[5];
1523  if (Record[3])
1524  FileInfo.setHasLineDirectives();
1525 
1526  const DeclID *FirstDecl = F->FileSortedDecls + Record[6];
1527  unsigned NumFileDecls = Record[7];
1528  if (NumFileDecls && ContextObj) {
1529  assert(F->FileSortedDecls && "FILE_SORTED_DECLS not encountered yet ?");
1530  FileDeclIDs[FID] = FileDeclsInfo(F, llvm::makeArrayRef(FirstDecl,
1531  NumFileDecls));
1532  }
1533 
1534  const SrcMgr::ContentCache *ContentCache
1535  = SourceMgr.getOrCreateContentCache(File, isSystem(FileCharacter));
1536  if (OverriddenBuffer && !ContentCache->BufferOverridden &&
1537  ContentCache->ContentsEntry == ContentCache->OrigEntry &&
1538  !ContentCache->getRawBuffer()) {
1539  auto Buffer = ReadBuffer(SLocEntryCursor, File->getName());
1540  if (!Buffer)
1541  return true;
1542  SourceMgr.overrideFileContents(File, std::move(Buffer));
1543  }
1544 
1545  break;
1546  }
1547 
1548  case SM_SLOC_BUFFER_ENTRY: {
1549  const char *Name = Blob.data();
1550  unsigned Offset = Record[0];
1552  FileCharacter = (SrcMgr::CharacteristicKind)Record[2];
1553  SourceLocation IncludeLoc = ReadSourceLocation(*F, Record[1]);
1554  if (IncludeLoc.isInvalid() && F->isModule()) {
1555  IncludeLoc = getImportLocation(F);
1556  }
1557 
1558  auto Buffer = ReadBuffer(SLocEntryCursor, Name);
1559  if (!Buffer)
1560  return true;
1561  SourceMgr.createFileID(std::move(Buffer), FileCharacter, ID,
1562  BaseOffset + Offset, IncludeLoc);
1563  break;
1564  }
1565 
1566  case SM_SLOC_EXPANSION_ENTRY: {
1567  SourceLocation SpellingLoc = ReadSourceLocation(*F, Record[1]);
1568  SourceMgr.createExpansionLoc(SpellingLoc,
1569  ReadSourceLocation(*F, Record[2]),
1570  ReadSourceLocation(*F, Record[3]),
1571  Record[5],
1572  Record[4],
1573  ID,
1574  BaseOffset + Record[0]);
1575  break;
1576  }
1577  }
1578 
1579  return false;
1580 }
1581 
1582 std::pair<SourceLocation, StringRef> ASTReader::getModuleImportLoc(int ID) {
1583  if (ID == 0)
1584  return std::make_pair(SourceLocation(), "");
1585 
1586  if (unsigned(-ID) - 2 >= getTotalNumSLocs() || ID > 0) {
1587  Error("source location entry ID out-of-range for AST file");
1588  return std::make_pair(SourceLocation(), "");
1589  }
1590 
1591  // Find which module file this entry lands in.
1592  ModuleFile *M = GlobalSLocEntryMap.find(-ID)->second;
1593  if (!M->isModule())
1594  return std::make_pair(SourceLocation(), "");
1595 
1596  // FIXME: Can we map this down to a particular submodule? That would be
1597  // ideal.
1598  return std::make_pair(M->ImportLoc, StringRef(M->ModuleName));
1599 }
1600 
1601 /// Find the location where the module F is imported.
1602 SourceLocation ASTReader::getImportLocation(ModuleFile *F) {
1603  if (F->ImportLoc.isValid())
1604  return F->ImportLoc;
1605 
1606  // Otherwise we have a PCH. It's considered to be "imported" at the first
1607  // location of its includer.
1608  if (F->ImportedBy.empty() || !F->ImportedBy[0]) {
1609  // Main file is the importer.
1610  assert(SourceMgr.getMainFileID().isValid() && "missing main file");
1611  return SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());
1612  }
1613  return F->ImportedBy[0]->FirstLoc;
1614 }
1615 
1616 /// Enter a subblock of the specified BlockID with the specified cursor. Read
1617 /// the abbreviations that are at the top of the block and then leave the cursor
1618 /// pointing into the block.
1619 bool ASTReader::ReadBlockAbbrevs(BitstreamCursor &Cursor, unsigned BlockID) {
1620  if (llvm::Error Err = Cursor.EnterSubBlock(BlockID)) {
1621  // FIXME this drops errors on the floor.
1622  consumeError(std::move(Err));
1623  return true;
1624  }
1625 
1626  while (true) {
1627  uint64_t Offset = Cursor.GetCurrentBitNo();
1628  Expected<unsigned> MaybeCode = Cursor.ReadCode();
1629  if (!MaybeCode) {
1630  // FIXME this drops errors on the floor.
1631  consumeError(MaybeCode.takeError());
1632  return true;
1633  }
1634  unsigned Code = MaybeCode.get();
1635 
1636  // We expect all abbrevs to be at the start of the block.
1637  if (Code != llvm::bitc::DEFINE_ABBREV) {
1638  if (llvm::Error Err = Cursor.JumpToBit(Offset)) {
1639  // FIXME this drops errors on the floor.
1640  consumeError(std::move(Err));
1641  return true;
1642  }
1643  return false;
1644  }
1645  if (llvm::Error Err = Cursor.ReadAbbrevRecord()) {
1646  // FIXME this drops errors on the floor.
1647  consumeError(std::move(Err));
1648  return true;
1649  }
1650  }
1651 }
1652 
1654  unsigned &Idx) {
1655  Token Tok;
1656  Tok.startToken();
1657  Tok.setLocation(ReadSourceLocation(F, Record, Idx));
1658  Tok.setLength(Record[Idx++]);
1659  if (IdentifierInfo *II = getLocalIdentifier(F, Record[Idx++]))
1660  Tok.setIdentifierInfo(II);
1661  Tok.setKind((tok::TokenKind)Record[Idx++]);
1662  Tok.setFlag((Token::TokenFlags)Record[Idx++]);
1663  return Tok;
1664 }
1665 
1667  BitstreamCursor &Stream = F.MacroCursor;
1668 
1669  // Keep track of where we are in the stream, then jump back there
1670  // after reading this macro.
1671  SavedStreamPosition SavedPosition(Stream);
1672 
1673  if (llvm::Error Err = Stream.JumpToBit(Offset)) {
1674  // FIXME this drops errors on the floor.
1675  consumeError(std::move(Err));
1676  return nullptr;
1677  }
1678  RecordData Record;
1680  MacroInfo *Macro = nullptr;
1681 
1682  while (true) {
1683  // Advance to the next record, but if we get to the end of the block, don't
1684  // pop it (removing all the abbreviations from the cursor) since we want to
1685  // be able to reseek within the block and read entries.
1686  unsigned Flags = BitstreamCursor::AF_DontPopBlockAtEnd;
1687  Expected<llvm::BitstreamEntry> MaybeEntry =
1688  Stream.advanceSkippingSubblocks(Flags);
1689  if (!MaybeEntry) {
1690  Error(MaybeEntry.takeError());
1691  return Macro;
1692  }
1693  llvm::BitstreamEntry Entry = MaybeEntry.get();
1694 
1695  switch (Entry.Kind) {
1696  case llvm::BitstreamEntry::SubBlock: // Handled for us already.
1698  Error("malformed block record in AST file");
1699  return Macro;
1700  case llvm::BitstreamEntry::EndBlock:
1701  return Macro;
1702  case llvm::BitstreamEntry::Record:
1703  // The interesting case.
1704  break;
1705  }
1706 
1707  // Read a record.
1708  Record.clear();
1709  PreprocessorRecordTypes RecType;
1710  if (Expected<unsigned> MaybeRecType = Stream.readRecord(Entry.ID, Record))
1711  RecType = (PreprocessorRecordTypes)MaybeRecType.get();
1712  else {
1713  Error(MaybeRecType.takeError());
1714  return Macro;
1715  }
1716  switch (RecType) {
1717  case PP_MODULE_MACRO:
1719  return Macro;
1720 
1721  case PP_MACRO_OBJECT_LIKE:
1722  case PP_MACRO_FUNCTION_LIKE: {
1723  // If we already have a macro, that means that we've hit the end
1724  // of the definition of the macro we were looking for. We're
1725  // done.
1726  if (Macro)
1727  return Macro;
1728 
1729  unsigned NextIndex = 1; // Skip identifier ID.
1730  SourceLocation Loc = ReadSourceLocation(F, Record, NextIndex);
1731  MacroInfo *MI = PP.AllocateMacroInfo(Loc);
1732  MI->setDefinitionEndLoc(ReadSourceLocation(F, Record, NextIndex));
1733  MI->setIsUsed(Record[NextIndex++]);
1734  MI->setUsedForHeaderGuard(Record[NextIndex++]);
1735 
1736  if (RecType == PP_MACRO_FUNCTION_LIKE) {
1737  // Decode function-like macro info.
1738  bool isC99VarArgs = Record[NextIndex++];
1739  bool isGNUVarArgs = Record[NextIndex++];
1740  bool hasCommaPasting = Record[NextIndex++];
1741  MacroParams.clear();
1742  unsigned NumArgs = Record[NextIndex++];
1743  for (unsigned i = 0; i != NumArgs; ++i)
1744  MacroParams.push_back(getLocalIdentifier(F, Record[NextIndex++]));
1745 
1746  // Install function-like macro info.
1747  MI->setIsFunctionLike();
1748  if (isC99VarArgs) MI->setIsC99Varargs();
1749  if (isGNUVarArgs) MI->setIsGNUVarargs();
1750  if (hasCommaPasting) MI->setHasCommaPasting();
1751  MI->setParameterList(MacroParams, PP.getPreprocessorAllocator());
1752  }
1753 
1754  // Remember that we saw this macro last so that we add the tokens that
1755  // form its body to it.
1756  Macro = MI;
1757 
1758  if (NextIndex + 1 == Record.size() && PP.getPreprocessingRecord() &&
1759  Record[NextIndex]) {
1760  // We have a macro definition. Register the association
1762  GlobalID = getGlobalPreprocessedEntityID(F, Record[NextIndex]);
1763  PreprocessingRecord &PPRec = *PP.getPreprocessingRecord();
1764  PreprocessingRecord::PPEntityID PPID =
1765  PPRec.getPPEntityID(GlobalID - 1, /*isLoaded=*/true);
1766  MacroDefinitionRecord *PPDef = cast_or_null<MacroDefinitionRecord>(
1767  PPRec.getPreprocessedEntity(PPID));
1768  if (PPDef)
1769  PPRec.RegisterMacroDefinition(Macro, PPDef);
1770  }
1771 
1772  ++NumMacrosRead;
1773  break;
1774  }
1775 
1776  case PP_TOKEN: {
1777  // If we see a TOKEN before a PP_MACRO_*, then the file is
1778  // erroneous, just pretend we didn't see this.
1779  if (!Macro) break;
1780 
1781  unsigned Idx = 0;
1782  Token Tok = ReadToken(F, Record, Idx);
1783  Macro->AddTokenToBody(Tok);
1784  break;
1785  }
1786  }
1787  }
1788 }
1789 
1792  unsigned LocalID) const {
1793  if (!M.ModuleOffsetMap.empty())
1794  ReadModuleOffsetMap(M);
1795 
1798  assert(I != M.PreprocessedEntityRemap.end()
1799  && "Invalid index into preprocessed entity index remap");
1800 
1801  return LocalID + I->second;
1802 }
1803 
1805  return llvm::hash_combine(ikey.Size, ikey.ModTime);
1806 }
1807 
1809 HeaderFileInfoTrait::GetInternalKey(const FileEntry *FE) {
1810  internal_key_type ikey = {FE->getSize(),
1811  M.HasTimestamps ? FE->getModificationTime() : 0,
1812  FE->getName(), /*Imported*/ false};
1813  return ikey;
1814 }
1815 
1816 bool HeaderFileInfoTrait::EqualKey(internal_key_ref a, internal_key_ref b) {
1817  if (a.Size != b.Size || (a.ModTime && b.ModTime && a.ModTime != b.ModTime))
1818  return false;
1819 
1820  if (llvm::sys::path::is_absolute(a.Filename) && a.Filename == b.Filename)
1821  return true;
1822 
1823  // Determine whether the actual files are equivalent.
1824  FileManager &FileMgr = Reader.getFileManager();
1825  auto GetFile = [&](const internal_key_type &Key) -> const FileEntry* {
1826  if (!Key.Imported) {
1827  if (auto File = FileMgr.getFile(Key.Filename))
1828  return *File;
1829  return nullptr;
1830  }
1831 
1832  std::string Resolved = Key.Filename;
1833  Reader.ResolveImportedPath(M, Resolved);
1834  if (auto File = FileMgr.getFile(Resolved))
1835  return *File;
1836  return nullptr;
1837  };
1838 
1839  const FileEntry *FEA = GetFile(a);
1840  const FileEntry *FEB = GetFile(b);
1841  return FEA && FEA == FEB;
1842 }
1843 
1844 std::pair<unsigned, unsigned>
1845 HeaderFileInfoTrait::ReadKeyDataLength(const unsigned char*& d) {
1846  using namespace llvm::support;
1847 
1848  unsigned KeyLen = (unsigned) endian::readNext<uint16_t, little, unaligned>(d);
1849  unsigned DataLen = (unsigned) *d++;
1850  return std::make_pair(KeyLen, DataLen);
1851 }
1852 
1854 HeaderFileInfoTrait::ReadKey(const unsigned char *d, unsigned) {
1855  using namespace llvm::support;
1856 
1857  internal_key_type ikey;
1858  ikey.Size = off_t(endian::readNext<uint64_t, little, unaligned>(d));
1859  ikey.ModTime = time_t(endian::readNext<uint64_t, little, unaligned>(d));
1860  ikey.Filename = (const char *)d;
1861  ikey.Imported = true;
1862  return ikey;
1863 }
1864 
1866 HeaderFileInfoTrait::ReadData(internal_key_ref key, const unsigned char *d,
1867  unsigned DataLen) {
1868  using namespace llvm::support;
1869 
1870  const unsigned char *End = d + DataLen;
1871  HeaderFileInfo HFI;
1872  unsigned Flags = *d++;
1873  // FIXME: Refactor with mergeHeaderFileInfo in HeaderSearch.cpp.
1874  HFI.isImport |= (Flags >> 5) & 0x01;
1875  HFI.isPragmaOnce |= (Flags >> 4) & 0x01;
1876  HFI.DirInfo = (Flags >> 1) & 0x07;
1877  HFI.IndexHeaderMapHeader = Flags & 0x01;
1878  // FIXME: Find a better way to handle this. Maybe just store a
1879  // "has been included" flag?
1880  HFI.NumIncludes = std::max(endian::readNext<uint16_t, little, unaligned>(d),
1881  HFI.NumIncludes);
1882  HFI.ControllingMacroID = Reader.getGlobalIdentifierID(
1883  M, endian::readNext<uint32_t, little, unaligned>(d));
1884  if (unsigned FrameworkOffset =
1885  endian::readNext<uint32_t, little, unaligned>(d)) {
1886  // The framework offset is 1 greater than the actual offset,
1887  // since 0 is used as an indicator for "no framework name".
1888  StringRef FrameworkName(FrameworkStrings + FrameworkOffset - 1);
1889  HFI.Framework = HS->getUniqueFrameworkName(FrameworkName);
1890  }
1891 
1892  assert((End - d) % 4 == 0 &&
1893  "Wrong data length in HeaderFileInfo deserialization");
1894  while (d != End) {
1895  uint32_t LocalSMID = endian::readNext<uint32_t, little, unaligned>(d);
1896  auto HeaderRole = static_cast<ModuleMap::ModuleHeaderRole>(LocalSMID & 3);
1897  LocalSMID >>= 2;
1898 
1899  // This header is part of a module. Associate it with the module to enable
1900  // implicit module import.
1901  SubmoduleID GlobalSMID = Reader.getGlobalSubmoduleID(M, LocalSMID);
1902  Module *Mod = Reader.getSubmodule(GlobalSMID);
1903  FileManager &FileMgr = Reader.getFileManager();
1904  ModuleMap &ModMap =
1905  Reader.getPreprocessor().getHeaderSearchInfo().getModuleMap();
1906 
1907  std::string Filename = key.Filename;
1908  if (key.Imported)
1909  Reader.ResolveImportedPath(M, Filename);
1910  // FIXME: This is not always the right filename-as-written, but we're not
1911  // going to use this information to rebuild the module, so it doesn't make
1912  // a lot of difference.
1913  Module::Header H = { key.Filename, *FileMgr.getFile(Filename) };
1914  ModMap.addHeader(Mod, H, HeaderRole, /*Imported*/true);
1915  HFI.isModuleHeader |= !(HeaderRole & ModuleMap::TextualHeader);
1916  }
1917 
1918  // This HeaderFileInfo was externally loaded.
1919  HFI.External = true;
1920  HFI.IsValid = true;
1921  return HFI;
1922 }
1923 
1925  ModuleFile *M,
1926  uint64_t MacroDirectivesOffset) {
1927  assert(NumCurrentElementsDeserializing > 0 &&"Missing deserialization guard");
1928  PendingMacroIDs[II].push_back(PendingMacroInfo(M, MacroDirectivesOffset));
1929 }
1930 
1932  // Note that we are loading defined macros.
1933  Deserializing Macros(this);
1934 
1935  for (ModuleFile &I : llvm::reverse(ModuleMgr)) {
1936  BitstreamCursor &MacroCursor = I.MacroCursor;
1937 
1938  // If there was no preprocessor block, skip this file.
1939  if (MacroCursor.getBitcodeBytes().empty())
1940  continue;
1941 
1942  BitstreamCursor Cursor = MacroCursor;
1943  if (llvm::Error Err = Cursor.JumpToBit(I.MacroStartOffset)) {
1944  Error(std::move(Err));
1945  return;
1946  }
1947 
1948  RecordData Record;
1949  while (true) {
1950  Expected<llvm::BitstreamEntry> MaybeE = Cursor.advanceSkippingSubblocks();
1951  if (!MaybeE) {
1952  Error(MaybeE.takeError());
1953  return;
1954  }
1955  llvm::BitstreamEntry E = MaybeE.get();
1956 
1957  switch (E.Kind) {
1958  case llvm::BitstreamEntry::SubBlock: // Handled for us already.
1960  Error("malformed block record in AST file");
1961  return;
1962  case llvm::BitstreamEntry::EndBlock:
1963  goto NextCursor;
1964 
1965  case llvm::BitstreamEntry::Record: {
1966  Record.clear();
1967  Expected<unsigned> MaybeRecord = Cursor.readRecord(E.ID, Record);
1968  if (!MaybeRecord) {
1969  Error(MaybeRecord.takeError());
1970  return;
1971  }
1972  switch (MaybeRecord.get()) {
1973  default: // Default behavior: ignore.
1974  break;
1975 
1976  case PP_MACRO_OBJECT_LIKE:
1977  case PP_MACRO_FUNCTION_LIKE: {
1978  IdentifierInfo *II = getLocalIdentifier(I, Record[0]);
1979  if (II->isOutOfDate())
1980  updateOutOfDateIdentifier(*II);
1981  break;
1982  }
1983 
1984  case PP_TOKEN:
1985  // Ignore tokens.
1986  break;
1987  }
1988  break;
1989  }
1990  }
1991  }
1992  NextCursor: ;
1993  }
1994 }
1995 
1996 namespace {
1997 
1998  /// Visitor class used to look up identifirs in an AST file.
1999  class IdentifierLookupVisitor {
2000  StringRef Name;
2001  unsigned NameHash;
2002  unsigned PriorGeneration;
2003  unsigned &NumIdentifierLookups;
2004  unsigned &NumIdentifierLookupHits;
2005  IdentifierInfo *Found = nullptr;
2006 
2007  public:
2008  IdentifierLookupVisitor(StringRef Name, unsigned PriorGeneration,
2009  unsigned &NumIdentifierLookups,
2010  unsigned &NumIdentifierLookupHits)
2011  : Name(Name), NameHash(ASTIdentifierLookupTrait::ComputeHash(Name)),
2012  PriorGeneration(PriorGeneration),
2013  NumIdentifierLookups(NumIdentifierLookups),
2014  NumIdentifierLookupHits(NumIdentifierLookupHits) {}
2015 
2016  bool operator()(ModuleFile &M) {
2017  // If we've already searched this module file, skip it now.
2018  if (M.Generation <= PriorGeneration)
2019  return true;
2020 
2021  ASTIdentifierLookupTable *IdTable
2023  if (!IdTable)
2024  return false;
2025 
2026  ASTIdentifierLookupTrait Trait(IdTable->getInfoObj().getReader(), M,
2027  Found);
2028  ++NumIdentifierLookups;
2029  ASTIdentifierLookupTable::iterator Pos =
2030  IdTable->find_hashed(Name, NameHash, &Trait);
2031  if (Pos == IdTable->end())
2032  return false;
2033 
2034  // Dereferencing the iterator has the effect of building the
2035  // IdentifierInfo node and populating it with the various
2036  // declarations it needs.
2037  ++NumIdentifierLookupHits;
2038  Found = *Pos;
2039  return true;
2040  }
2041 
2042  // Retrieve the identifier info found within the module
2043  // files.
2044  IdentifierInfo *getIdentifierInfo() const { return Found; }
2045  };
2046 
2047 } // namespace
2048 
2050  // Note that we are loading an identifier.
2051  Deserializing AnIdentifier(this);
2052 
2053  unsigned PriorGeneration = 0;
2054  if (getContext().getLangOpts().Modules)
2055  PriorGeneration = IdentifierGeneration[&II];
2056 
2057  // If there is a global index, look there first to determine which modules
2058  // provably do not have any results for this identifier.
2060  GlobalModuleIndex::HitSet *HitsPtr = nullptr;
2061  if (!loadGlobalIndex()) {
2062  if (GlobalIndex->lookupIdentifier(II.getName(), Hits)) {
2063  HitsPtr = &Hits;
2064  }
2065  }
2066 
2067  IdentifierLookupVisitor Visitor(II.getName(), PriorGeneration,
2068  NumIdentifierLookups,
2069  NumIdentifierLookupHits);
2070  ModuleMgr.visit(Visitor, HitsPtr);
2071  markIdentifierUpToDate(&II);
2072 }
2073 
2075  if (!II)
2076  return;
2077 
2078  II->setOutOfDate(false);
2079 
2080  // Update the generation for this identifier.
2081  if (getContext().getLangOpts().Modules)
2082  IdentifierGeneration[II] = getGeneration();
2083 }
2084 
2086  const PendingMacroInfo &PMInfo) {
2087  ModuleFile &M = *PMInfo.M;
2088 
2089  BitstreamCursor &Cursor = M.MacroCursor;
2090  SavedStreamPosition SavedPosition(Cursor);
2091  if (llvm::Error Err = Cursor.JumpToBit(PMInfo.MacroDirectivesOffset)) {
2092  Error(std::move(Err));
2093  return;
2094  }
2095 
2096  struct ModuleMacroRecord {
2097  SubmoduleID SubModID;
2098  MacroInfo *MI;
2099  SmallVector<SubmoduleID, 8> Overrides;
2100  };
2102 
2103  // We expect to see a sequence of PP_MODULE_MACRO records listing exported
2104  // macros, followed by a PP_MACRO_DIRECTIVE_HISTORY record with the complete
2105  // macro histroy.
2106  RecordData Record;
2107  while (true) {
2108  Expected<llvm::BitstreamEntry> MaybeEntry =
2109  Cursor.advance(BitstreamCursor::AF_DontPopBlockAtEnd);
2110  if (!MaybeEntry) {
2111  Error(MaybeEntry.takeError());
2112  return;
2113  }
2114  llvm::BitstreamEntry Entry = MaybeEntry.get();
2115 
2116  if (Entry.Kind != llvm::BitstreamEntry::Record) {
2117  Error("malformed block record in AST file");
2118  return;
2119  }
2120 
2121  Record.clear();
2122  Expected<unsigned> MaybePP = Cursor.readRecord(Entry.ID, Record);
2123  if (!MaybePP) {
2124  Error(MaybePP.takeError());
2125  return;
2126  }
2127  switch ((PreprocessorRecordTypes)MaybePP.get()) {
2129  break;
2130 
2131  case PP_MODULE_MACRO: {
2132  ModuleMacros.push_back(ModuleMacroRecord());
2133  auto &Info = ModuleMacros.back();
2134  Info.SubModID = getGlobalSubmoduleID(M, Record[0]);
2135  Info.MI = getMacro(getGlobalMacroID(M, Record[1]));
2136  for (int I = 2, N = Record.size(); I != N; ++I)
2137  Info.Overrides.push_back(getGlobalSubmoduleID(M, Record[I]));
2138  continue;
2139  }
2140 
2141  default:
2142  Error("malformed block record in AST file");
2143  return;
2144  }
2145 
2146  // We found the macro directive history; that's the last record
2147  // for this macro.
2148  break;
2149  }
2150 
2151  // Module macros are listed in reverse dependency order.
2152  {
2153  std::reverse(ModuleMacros.begin(), ModuleMacros.end());
2155  for (auto &MMR : ModuleMacros) {
2156  Overrides.clear();
2157  for (unsigned ModID : MMR.Overrides) {
2158  Module *Mod = getSubmodule(ModID);
2159  auto *Macro = PP.getModuleMacro(Mod, II);
2160  assert(Macro && "missing definition for overridden macro");
2161  Overrides.push_back(Macro);
2162  }
2163 
2164  bool Inserted = false;
2165  Module *Owner = getSubmodule(MMR.SubModID);
2166  PP.addModuleMacro(Owner, II, MMR.MI, Overrides, Inserted);
2167  }
2168  }
2169 
2170  // Don't read the directive history for a module; we don't have anywhere
2171  // to put it.
2172  if (M.isModule())
2173  return;
2174 
2175  // Deserialize the macro directives history in reverse source-order.
2176  MacroDirective *Latest = nullptr, *Earliest = nullptr;
2177  unsigned Idx = 0, N = Record.size();
2178  while (Idx < N) {
2179  MacroDirective *MD = nullptr;
2180  SourceLocation Loc = ReadSourceLocation(M, Record, Idx);
2181  MacroDirective::Kind K = (MacroDirective::Kind)Record[Idx++];
2182  switch (K) {
2184  MacroInfo *MI = getMacro(getGlobalMacroID(M, Record[Idx++]));
2185  MD = PP.AllocateDefMacroDirective(MI, Loc);
2186  break;
2187  }
2189  MD = PP.AllocateUndefMacroDirective(Loc);
2190  break;
2192  bool isPublic = Record[Idx++];
2193  MD = PP.AllocateVisibilityMacroDirective(Loc, isPublic);
2194  break;
2195  }
2196 
2197  if (!Latest)
2198  Latest = MD;
2199  if (Earliest)
2200  Earliest->setPrevious(MD);
2201  Earliest = MD;
2202  }
2203 
2204  if (Latest)
2205  PP.setLoadedMacroDirective(II, Earliest, Latest);
2206 }
2207 
2208 ASTReader::InputFileInfo
2209 ASTReader::readInputFileInfo(ModuleFile &F, unsigned ID) {
2210  // Go find this input file.
2211  BitstreamCursor &Cursor = F.InputFilesCursor;
2212  SavedStreamPosition SavedPosition(Cursor);
2213  if (llvm::Error Err = Cursor.JumpToBit(F.InputFileOffsets[ID - 1])) {
2214  // FIXME this drops errors on the floor.
2215  consumeError(std::move(Err));
2216  }
2217 
2218  Expected<unsigned> MaybeCode = Cursor.ReadCode();
2219  if (!MaybeCode) {
2220  // FIXME this drops errors on the floor.
2221  consumeError(MaybeCode.takeError());
2222  }
2223  unsigned Code = MaybeCode.get();
2224  RecordData Record;
2225  StringRef Blob;
2226 
2227  if (Expected<unsigned> Maybe = Cursor.readRecord(Code, Record, &Blob))
2228  assert(static_cast<InputFileRecordTypes>(Maybe.get()) == INPUT_FILE &&
2229  "invalid record type for input file");
2230  else {
2231  // FIXME this drops errors on the floor.
2232  consumeError(Maybe.takeError());
2233  }
2234 
2235  assert(Record[0] == ID && "Bogus stored ID or offset");
2236  InputFileInfo R;
2237  R.StoredSize = static_cast<off_t>(Record[1]);
2238  R.StoredTime = static_cast<time_t>(Record[2]);
2239  R.Overridden = static_cast<bool>(Record[3]);
2240  R.Transient = static_cast<bool>(Record[4]);
2241  R.TopLevelModuleMap = static_cast<bool>(Record[5]);
2242  R.Filename = Blob;
2243  ResolveImportedPath(F, R.Filename);
2244  return R;
2245 }
2246 
2247 static unsigned moduleKindForDiagnostic(ModuleKind Kind);
2248 InputFile ASTReader::getInputFile(ModuleFile &F, unsigned ID, bool Complain) {
2249  // If this ID is bogus, just return an empty input file.
2250  if (ID == 0 || ID > F.InputFilesLoaded.size())
2251  return InputFile();
2252 
2253  // If we've already loaded this input file, return it.
2254  if (F.InputFilesLoaded[ID-1].getFile())
2255  return F.InputFilesLoaded[ID-1];
2256 
2257  if (F.InputFilesLoaded[ID-1].isNotFound())
2258  return InputFile();
2259 
2260  // Go find this input file.
2261  BitstreamCursor &Cursor = F.InputFilesCursor;
2262  SavedStreamPosition SavedPosition(Cursor);
2263  if (llvm::Error Err = Cursor.JumpToBit(F.InputFileOffsets[ID - 1])) {
2264  // FIXME this drops errors on the floor.
2265  consumeError(std::move(Err));
2266  }
2267 
2268  InputFileInfo FI = readInputFileInfo(F, ID);
2269  off_t StoredSize = FI.StoredSize;
2270  time_t StoredTime = FI.StoredTime;
2271  bool Overridden = FI.Overridden;
2272  bool Transient = FI.Transient;
2273  StringRef Filename = FI.Filename;
2274 
2275  const FileEntry *File = nullptr;
2276  if (auto FE = FileMgr.getFile(Filename, /*OpenFile=*/false))
2277  File = *FE;
2278 
2279  // If we didn't find the file, resolve it relative to the
2280  // original directory from which this AST file was created.
2281  if (File == nullptr && !F.OriginalDir.empty() && !F.BaseDirectory.empty() &&
2282  F.OriginalDir != F.BaseDirectory) {
2283  std::string Resolved = resolveFileRelativeToOriginalDir(
2284  Filename, F.OriginalDir, F.BaseDirectory);
2285  if (!Resolved.empty())
2286  if (auto FE = FileMgr.getFile(Resolved))
2287  File = *FE;
2288  }
2289 
2290  // For an overridden file, create a virtual file with the stored
2291  // size/timestamp.
2292  if ((Overridden || Transient) && File == nullptr)
2293  File = FileMgr.getVirtualFile(Filename, StoredSize, StoredTime);
2294 
2295  if (File == nullptr) {
2296  if (Complain) {
2297  std::string ErrorStr = "could not find file '";
2298  ErrorStr += Filename;
2299  ErrorStr += "' referenced by AST file '";
2300  ErrorStr += F.FileName;
2301  ErrorStr += "'";
2302  Error(ErrorStr);
2303  }
2304  // Record that we didn't find the file.
2306  return InputFile();
2307  }
2308 
2309  // Check if there was a request to override the contents of the file
2310  // that was part of the precompiled header. Overriding such a file
2311  // can lead to problems when lexing using the source locations from the
2312  // PCH.
2313  SourceManager &SM = getSourceManager();
2314  // FIXME: Reject if the overrides are different.
2315  if ((!Overridden && !Transient) && SM.isFileOverridden(File)) {
2316  if (Complain)
2317  Error(diag::err_fe_pch_file_overridden, Filename);
2318  // After emitting the diagnostic, recover by disabling the override so
2319  // that the original file will be used.
2320  //
2321  // FIXME: This recovery is just as broken as the original state; there may
2322  // be another precompiled module that's using the overridden contents, or
2323  // we might be half way through parsing it. Instead, we should treat the
2324  // overridden contents as belonging to a separate FileEntry.
2325  SM.disableFileContentsOverride(File);
2326  // The FileEntry is a virtual file entry with the size of the contents
2327  // that would override the original contents. Set it to the original's
2328  // size/time.
2329  FileMgr.modifyFileEntry(const_cast<FileEntry*>(File),
2330  StoredSize, StoredTime);
2331  }
2332 
2333  bool IsOutOfDate = false;
2334 
2335  // For an overridden file, there is nothing to validate.
2336  if (!Overridden && //
2337  (StoredSize != File->getSize() ||
2338  (StoredTime && StoredTime != File->getModificationTime() &&
2339  !DisableValidation)
2340  )) {
2341  if (Complain) {
2342  // Build a list of the PCH imports that got us here (in reverse).
2343  SmallVector<ModuleFile *, 4> ImportStack(1, &F);
2344  while (!ImportStack.back()->ImportedBy.empty())
2345  ImportStack.push_back(ImportStack.back()->ImportedBy[0]);
2346 
2347  // The top-level PCH is stale.
2348  StringRef TopLevelPCHName(ImportStack.back()->FileName);
2349  unsigned DiagnosticKind = moduleKindForDiagnostic(ImportStack.back()->Kind);
2350  if (DiagnosticKind == 0)
2351  Error(diag::err_fe_pch_file_modified, Filename, TopLevelPCHName);
2352  else if (DiagnosticKind == 1)
2353  Error(diag::err_fe_module_file_modified, Filename, TopLevelPCHName);
2354  else
2355  Error(diag::err_fe_ast_file_modified, Filename, TopLevelPCHName);
2356 
2357  // Print the import stack.
2358  if (ImportStack.size() > 1 && !Diags.isDiagnosticInFlight()) {
2359  Diag(diag::note_pch_required_by)
2360  << Filename << ImportStack[0]->FileName;
2361  for (unsigned I = 1; I < ImportStack.size(); ++I)
2362  Diag(diag::note_pch_required_by)
2363  << ImportStack[I-1]->FileName << ImportStack[I]->FileName;
2364  }
2365 
2366  if (!Diags.isDiagnosticInFlight())
2367  Diag(diag::note_pch_rebuild_required) << TopLevelPCHName;
2368  }
2369 
2370  IsOutOfDate = true;
2371  }
2372  // FIXME: If the file is overridden and we've already opened it,
2373  // issue an error (or split it into a separate FileEntry).
2374 
2375  InputFile IF = InputFile(File, Overridden || Transient, IsOutOfDate);
2376 
2377  // Note that we've loaded this input file.
2378  F.InputFilesLoaded[ID-1] = IF;
2379  return IF;
2380 }
2381 
2382 /// If we are loading a relocatable PCH or module file, and the filename
2383 /// is not an absolute path, add the system or module root to the beginning of
2384 /// the file name.
2386  // Resolve relative to the base directory, if we have one.
2387  if (!M.BaseDirectory.empty())
2388  return ResolveImportedPath(Filename, M.BaseDirectory);
2389 }
2390 
2391 void ASTReader::ResolveImportedPath(std::string &Filename, StringRef Prefix) {
2392  if (Filename.empty() || llvm::sys::path::is_absolute(Filename))
2393  return;
2394 
2395  SmallString<128> Buffer;
2396  llvm::sys::path::append(Buffer, Prefix, Filename);
2397  Filename.assign(Buffer.begin(), Buffer.end());
2398 }
2399 
2400 static bool isDiagnosedResult(ASTReader::ASTReadResult ARR, unsigned Caps) {
2401  switch (ARR) {
2402  case ASTReader::Failure: return true;
2403  case ASTReader::Missing: return !(Caps & ASTReader::ARR_Missing);
2404  case ASTReader::OutOfDate: return !(Caps & ASTReader::ARR_OutOfDate);
2407  return !(Caps & ASTReader::ARR_ConfigurationMismatch);
2408  case ASTReader::HadErrors: return true;
2409  case ASTReader::Success: return false;
2410  }
2411 
2412  llvm_unreachable("unknown ASTReadResult");
2413 }
2414 
2415 ASTReader::ASTReadResult ASTReader::ReadOptionsBlock(
2416  BitstreamCursor &Stream, unsigned ClientLoadCapabilities,
2417  bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener,
2418  std::string &SuggestedPredefines) {
2419  if (llvm::Error Err = Stream.EnterSubBlock(OPTIONS_BLOCK_ID)) {
2420  // FIXME this drops errors on the floor.
2421  consumeError(std::move(Err));
2422  return Failure;
2423  }
2424 
2425  // Read all of the records in the options block.
2426  RecordData Record;
2427  ASTReadResult Result = Success;
2428  while (true) {
2429  Expected<llvm::BitstreamEntry> MaybeEntry = Stream.advance();
2430  if (!MaybeEntry) {
2431  // FIXME this drops errors on the floor.
2432  consumeError(MaybeEntry.takeError());
2433  return Failure;
2434  }
2435  llvm::BitstreamEntry Entry = MaybeEntry.get();
2436 
2437  switch (Entry.Kind) {
2439  case llvm::BitstreamEntry::SubBlock:
2440  return Failure;
2441 
2442  case llvm::BitstreamEntry::EndBlock:
2443  return Result;
2444 
2445  case llvm::BitstreamEntry::Record:
2446  // The interesting case.
2447  break;
2448  }
2449 
2450  // Read and process a record.
2451  Record.clear();
2452  Expected<unsigned> MaybeRecordType = Stream.readRecord(Entry.ID, Record);
2453  if (!MaybeRecordType) {
2454  // FIXME this drops errors on the floor.
2455  consumeError(MaybeRecordType.takeError());
2456  return Failure;
2457  }
2458  switch ((OptionsRecordTypes)MaybeRecordType.get()) {
2459  case LANGUAGE_OPTIONS: {
2460  bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch) == 0;
2461  if (ParseLanguageOptions(Record, Complain, Listener,
2462  AllowCompatibleConfigurationMismatch))
2463  Result = ConfigurationMismatch;
2464  break;
2465  }
2466 
2467  case TARGET_OPTIONS: {
2468  bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch) == 0;
2469  if (ParseTargetOptions(Record, Complain, Listener,
2470  AllowCompatibleConfigurationMismatch))
2471  Result = ConfigurationMismatch;
2472  break;
2473  }
2474 
2475  case FILE_SYSTEM_OPTIONS: {
2476  bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch) == 0;
2477  if (!AllowCompatibleConfigurationMismatch &&
2478  ParseFileSystemOptions(Record, Complain, Listener))
2479  Result = ConfigurationMismatch;
2480  break;
2481  }
2482 
2483  case HEADER_SEARCH_OPTIONS: {
2484  bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch) == 0;
2485  if (!AllowCompatibleConfigurationMismatch &&
2486  ParseHeaderSearchOptions(Record, Complain, Listener))
2487  Result = ConfigurationMismatch;
2488  break;
2489  }
2490 
2491  case PREPROCESSOR_OPTIONS:
2492  bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch) == 0;
2493  if (!AllowCompatibleConfigurationMismatch &&
2494  ParsePreprocessorOptions(Record, Complain, Listener,
2495  SuggestedPredefines))
2496  Result = ConfigurationMismatch;
2497  break;
2498  }
2499  }
2500 }
2501 
2503 ASTReader::ReadControlBlock(ModuleFile &F,
2505  const ModuleFile *ImportedBy,
2506  unsigned ClientLoadCapabilities) {
2507  BitstreamCursor &Stream = F.Stream;
2508  ASTReadResult Result = Success;
2509 
2510  if (llvm::Error Err = Stream.EnterSubBlock(CONTROL_BLOCK_ID)) {
2511  Error(std::move(Err));
2512  return Failure;
2513  }
2514 
2515  // Lambda to read the unhashed control block the first time it's called.
2516  //
2517  // For PCM files, the unhashed control block cannot be read until after the
2518  // MODULE_NAME record. However, PCH files have no MODULE_NAME, and yet still
2519  // need to look ahead before reading the IMPORTS record. For consistency,
2520  // this block is always read somehow (see BitstreamEntry::EndBlock).
2521  bool HasReadUnhashedControlBlock = false;
2522  auto readUnhashedControlBlockOnce = [&]() {
2523  if (!HasReadUnhashedControlBlock) {
2524  HasReadUnhashedControlBlock = true;
2525  if (ASTReadResult Result =
2526  readUnhashedControlBlock(F, ImportedBy, ClientLoadCapabilities))
2527  return Result;
2528  }
2529  return Success;
2530  };
2531 
2532  // Read all of the records and blocks in the control block.
2533  RecordData Record;
2534  unsigned NumInputs = 0;
2535  unsigned NumUserInputs = 0;
2536  StringRef BaseDirectoryAsWritten;
2537  while (true) {
2538  Expected<llvm::BitstreamEntry> MaybeEntry = Stream.advance();
2539  if (!MaybeEntry) {
2540  Error(MaybeEntry.takeError());
2541  return Failure;
2542  }
2543  llvm::BitstreamEntry Entry = MaybeEntry.get();
2544 
2545  switch (Entry.Kind) {
2547  Error("malformed block record in AST file");
2548  return Failure;
2549  case llvm::BitstreamEntry::EndBlock: {
2550  // Validate the module before returning. This call catches an AST with
2551  // no module name and no imports.
2552  if (ASTReadResult Result = readUnhashedControlBlockOnce())
2553  return Result;
2554 
2555  // Validate input files.
2556  const HeaderSearchOptions &HSOpts =
2557  PP.getHeaderSearchInfo().getHeaderSearchOpts();
2558 
2559  // All user input files reside at the index range [0, NumUserInputs), and
2560  // system input files reside at [NumUserInputs, NumInputs). For explicitly
2561  // loaded module files, ignore missing inputs.
2562  if (!DisableValidation && F.Kind != MK_ExplicitModule &&
2563  F.Kind != MK_PrebuiltModule) {
2564  bool Complain = (ClientLoadCapabilities & ARR_OutOfDate) == 0;
2565 
2566  // If we are reading a module, we will create a verification timestamp,
2567  // so we verify all input files. Otherwise, verify only user input
2568  // files.
2569 
2570  unsigned N = NumUserInputs;
2571  if (ValidateSystemInputs ||
2574  F.Kind == MK_ImplicitModule))
2575  N = NumInputs;
2576 
2577  for (unsigned I = 0; I < N; ++I) {
2578  InputFile IF = getInputFile(F, I+1, Complain);
2579  if (!IF.getFile() || IF.isOutOfDate())
2580  return OutOfDate;
2581  }
2582  }
2583 
2584  if (Listener)
2585  Listener->visitModuleFile(F.FileName, F.Kind);
2586 
2587  if (Listener && Listener->needsInputFileVisitation()) {
2588  unsigned N = Listener->needsSystemInputFileVisitation() ? NumInputs
2589  : NumUserInputs;
2590  for (unsigned I = 0; I < N; ++I) {
2591  bool IsSystem = I >= NumUserInputs;
2592  InputFileInfo FI = readInputFileInfo(F, I+1);
2593  Listener->visitInputFile(FI.Filename, IsSystem, FI.Overridden,
2594  F.Kind == MK_ExplicitModule ||
2595  F.Kind == MK_PrebuiltModule);
2596  }
2597  }
2598 
2599  return Result;
2600  }
2601 
2602  case llvm::BitstreamEntry::SubBlock:
2603  switch (Entry.ID) {
2604  case INPUT_FILES_BLOCK_ID:
2605  F.InputFilesCursor = Stream;
2606  if (llvm::Error Err = Stream.SkipBlock()) {
2607  Error(std::move(Err));
2608  return Failure;
2609  }
2610  if (ReadBlockAbbrevs(F.InputFilesCursor, INPUT_FILES_BLOCK_ID)) {
2611  Error("malformed block record in AST file");
2612  return Failure;
2613  }
2614  continue;
2615 
2616  case OPTIONS_BLOCK_ID:
2617  // If we're reading the first module for this group, check its options
2618  // are compatible with ours. For modules it imports, no further checking
2619  // is required, because we checked them when we built it.
2620  if (Listener && !ImportedBy) {
2621  // Should we allow the configuration of the module file to differ from
2622  // the configuration of the current translation unit in a compatible
2623  // way?
2624  //
2625  // FIXME: Allow this for files explicitly specified with -include-pch.
2626  bool AllowCompatibleConfigurationMismatch =
2628 
2629  Result = ReadOptionsBlock(Stream, ClientLoadCapabilities,
2630  AllowCompatibleConfigurationMismatch,
2631  *Listener, SuggestedPredefines);
2632  if (Result == Failure) {
2633  Error("malformed block record in AST file");
2634  return Result;
2635  }
2636 
2637  if (DisableValidation ||
2638  (AllowConfigurationMismatch && Result == ConfigurationMismatch))
2639  Result = Success;
2640 
2641  // If we can't load the module, exit early since we likely
2642  // will rebuild the module anyway. The stream may be in the
2643  // middle of a block.
2644  if (Result != Success)
2645  return Result;
2646  } else if (llvm::Error Err = Stream.SkipBlock()) {
2647  Error(std::move(Err));
2648  return Failure;
2649  }
2650  continue;
2651 
2652  default:
2653  if (llvm::Error Err = Stream.SkipBlock()) {
2654  Error(std::move(Err));
2655  return Failure;
2656  }
2657  continue;
2658  }
2659 
2660  case llvm::BitstreamEntry::Record:
2661  // The interesting case.
2662  break;
2663  }
2664 
2665  // Read and process a record.
2666  Record.clear();
2667  StringRef Blob;
2668  Expected<unsigned> MaybeRecordType =
2669  Stream.readRecord(Entry.ID, Record, &Blob);
2670  if (!MaybeRecordType) {
2671  Error(MaybeRecordType.takeError());
2672  return Failure;
2673  }
2674  switch ((ControlRecordTypes)MaybeRecordType.get()) {
2675  case METADATA: {
2676  if (Record[0] != VERSION_MAJOR && !DisableValidation) {
2677  if ((ClientLoadCapabilities & ARR_VersionMismatch) == 0)
2678  Diag(Record[0] < VERSION_MAJOR? diag::err_pch_version_too_old
2679  : diag::err_pch_version_too_new);
2680  return VersionMismatch;
2681  }
2682 
2683  bool hasErrors = Record[7];
2684  if (hasErrors && !DisableValidation && !AllowASTWithCompilerErrors) {
2685  Diag(diag::err_pch_with_compiler_errors);
2686  return HadErrors;
2687  }
2688  if (hasErrors) {
2689  Diags.ErrorOccurred = true;
2690  Diags.UncompilableErrorOccurred = true;
2691  Diags.UnrecoverableErrorOccurred = true;
2692  }
2693 
2694  F.RelocatablePCH = Record[4];
2695  // Relative paths in a relocatable PCH are relative to our sysroot.
2696  if (F.RelocatablePCH)
2697  F.BaseDirectory = isysroot.empty() ? "/" : isysroot;
2698 
2699  F.HasTimestamps = Record[5];
2700 
2701  F.PCHHasObjectFile = Record[6];
2702 
2703  const std::string &CurBranch = getClangFullRepositoryVersion();
2704  StringRef ASTBranch = Blob;
2705  if (StringRef(CurBranch) != ASTBranch && !DisableValidation) {
2706  if ((ClientLoadCapabilities & ARR_VersionMismatch) == 0)
2707  Diag(diag::err_pch_different_branch) << ASTBranch << CurBranch;
2708  return VersionMismatch;
2709  }
2710  break;
2711  }
2712 
2713  case IMPORTS: {
2714  // Validate the AST before processing any imports (otherwise, untangling
2715  // them can be error-prone and expensive). A module will have a name and
2716  // will already have been validated, but this catches the PCH case.
2717  if (ASTReadResult Result = readUnhashedControlBlockOnce())
2718  return Result;
2719 
2720  // Load each of the imported PCH files.
2721  unsigned Idx = 0, N = Record.size();
2722  while (Idx < N) {
2723  // Read information about the AST file.
2724  ModuleKind ImportedKind = (ModuleKind)Record[Idx++];
2725  // The import location will be the local one for now; we will adjust
2726  // all import locations of module imports after the global source
2727  // location info are setup, in ReadAST.
2728  SourceLocation ImportLoc =
2729  ReadUntranslatedSourceLocation(Record[Idx++]);
2730  off_t StoredSize = (off_t)Record[Idx++];
2731  time_t StoredModTime = (time_t)Record[Idx++];
2732  ASTFileSignature StoredSignature = {
2733  {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
2734  (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
2735  (uint32_t)Record[Idx++]}}};
2736 
2737  std::string ImportedName = ReadString(Record, Idx);
2738  std::string ImportedFile;
2739 
2740  // For prebuilt and explicit modules first consult the file map for
2741  // an override. Note that here we don't search prebuilt module
2742  // directories, only the explicit name to file mappings. Also, we will
2743  // still verify the size/signature making sure it is essentially the
2744  // same file but perhaps in a different location.
2745  if (ImportedKind == MK_PrebuiltModule || ImportedKind == MK_ExplicitModule)
2746  ImportedFile = PP.getHeaderSearchInfo().getPrebuiltModuleFileName(
2747  ImportedName, /*FileMapOnly*/ true);
2748 
2749  if (ImportedFile.empty())
2750  // Use BaseDirectoryAsWritten to ensure we use the same path in the
2751  // ModuleCache as when writing.
2752  ImportedFile = ReadPath(BaseDirectoryAsWritten, Record, Idx);
2753  else
2754  SkipPath(Record, Idx);
2755 
2756  // If our client can't cope with us being out of date, we can't cope with
2757  // our dependency being missing.
2758  unsigned Capabilities = ClientLoadCapabilities;
2759  if ((ClientLoadCapabilities & ARR_OutOfDate) == 0)
2760  Capabilities &= ~ARR_Missing;
2761 
2762  // Load the AST file.
2763  auto Result = ReadASTCore(ImportedFile, ImportedKind, ImportLoc, &F,
2764  Loaded, StoredSize, StoredModTime,
2765  StoredSignature, Capabilities);
2766 
2767  // If we diagnosed a problem, produce a backtrace.
2768  if (isDiagnosedResult(Result, Capabilities))
2769  Diag(diag::note_module_file_imported_by)
2770  << F.FileName << !F.ModuleName.empty() << F.ModuleName;
2771 
2772  switch (Result) {
2773  case Failure: return Failure;
2774  // If we have to ignore the dependency, we'll have to ignore this too.
2775  case Missing:
2776  case OutOfDate: return OutOfDate;
2777  case VersionMismatch: return VersionMismatch;
2778  case ConfigurationMismatch: return ConfigurationMismatch;
2779  case HadErrors: return HadErrors;
2780  case Success: break;
2781  }
2782  }
2783  break;
2784  }
2785 
2786  case ORIGINAL_FILE:
2787  F.OriginalSourceFileID = FileID::get(Record[0]);
2790  ResolveImportedPath(F, F.OriginalSourceFileName);
2791  break;
2792 
2793  case ORIGINAL_FILE_ID:
2794  F.OriginalSourceFileID = FileID::get(Record[0]);
2795  break;
2796 
2797  case ORIGINAL_PCH_DIR:
2798  F.OriginalDir = Blob;
2799  break;
2800 
2801  case MODULE_NAME:
2802  F.ModuleName = Blob;
2803  Diag(diag::remark_module_import)
2804  << F.ModuleName << F.FileName << (ImportedBy ? true : false)
2805  << (ImportedBy ? StringRef(ImportedBy->ModuleName) : StringRef());
2806  if (Listener)
2807  Listener->ReadModuleName(F.ModuleName);
2808 
2809  // Validate the AST as soon as we have a name so we can exit early on
2810  // failure.
2811  if (ASTReadResult Result = readUnhashedControlBlockOnce())
2812  return Result;
2813 
2814  break;
2815 
2816  case MODULE_DIRECTORY: {
2817  // Save the BaseDirectory as written in the PCM for computing the module
2818  // filename for the ModuleCache.
2819  BaseDirectoryAsWritten = Blob;
2820  assert(!F.ModuleName.empty() &&
2821  "MODULE_DIRECTORY found before MODULE_NAME");
2822  // If we've already loaded a module map file covering this module, we may
2823  // have a better path for it (relative to the current build).
2824  Module *M = PP.getHeaderSearchInfo().lookupModule(
2825  F.ModuleName, /*AllowSearch*/ true,
2826  /*AllowExtraModuleMapSearch*/ true);
2827  if (M && M->Directory) {
2828  // If we're implicitly loading a module, the base directory can't
2829  // change between the build and use.
2830  // Don't emit module relocation error if we have -fno-validate-pch
2831  if (!PP.getPreprocessorOpts().DisablePCHValidation &&
2833  auto BuildDir = PP.getFileManager().getDirectory(Blob);
2834  if (!BuildDir || *BuildDir != M->Directory) {
2835  if ((ClientLoadCapabilities & ARR_OutOfDate) == 0)
2836  Diag(diag::err_imported_module_relocated)
2837  << F.ModuleName << Blob << M->Directory->getName();
2838  return OutOfDate;
2839  }
2840  }
2841  F.BaseDirectory = M->Directory->getName();
2842  } else {
2843  F.BaseDirectory = Blob;
2844  }
2845  break;
2846  }
2847 
2848  case MODULE_MAP_FILE:
2849  if (ASTReadResult Result =
2850  ReadModuleMapFileBlock(Record, F, ImportedBy, ClientLoadCapabilities))
2851  return Result;
2852  break;
2853 
2854  case INPUT_FILE_OFFSETS:
2855  NumInputs = Record[0];
2856  NumUserInputs = Record[1];
2857  F.InputFileOffsets =
2858  (const llvm::support::unaligned_uint64_t *)Blob.data();
2859  F.InputFilesLoaded.resize(NumInputs);
2860  F.NumUserInputFiles = NumUserInputs;
2861  break;
2862  }
2863  }
2864 }
2865 
2867 ASTReader::ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
2868  BitstreamCursor &Stream = F.Stream;
2869 
2870  if (llvm::Error Err = Stream.EnterSubBlock(AST_BLOCK_ID)) {
2871  Error(std::move(Err));
2872  return Failure;
2873  }
2874 
2875  // Read all of the records and blocks for the AST file.
2876  RecordData Record;
2877  while (true) {
2878  Expected<llvm::BitstreamEntry> MaybeEntry = Stream.advance();
2879  if (!MaybeEntry) {
2880  Error(MaybeEntry.takeError());
2881  return Failure;
2882  }
2883  llvm::BitstreamEntry Entry = MaybeEntry.get();
2884 
2885  switch (Entry.Kind) {
2887  Error("error at end of module block in AST file");
2888  return Failure;
2889  case llvm::BitstreamEntry::EndBlock:
2890  // Outside of C++, we do not store a lookup map for the translation unit.
2891  // Instead, mark it as needing a lookup map to be built if this module
2892  // contains any declarations lexically within it (which it always does!).
2893  // This usually has no cost, since we very rarely need the lookup map for
2894  // the translation unit outside C++.
2895  if (ASTContext *Ctx = ContextObj) {
2896  DeclContext *DC = Ctx->getTranslationUnitDecl();
2897  if (DC->hasExternalLexicalStorage() && !Ctx->getLangOpts().CPlusPlus)
2899  }
2900 
2901  return Success;
2902  case llvm::BitstreamEntry::SubBlock:
2903  switch (Entry.ID) {
2904  case DECLTYPES_BLOCK_ID:
2905  // We lazily load the decls block, but we want to set up the
2906  // DeclsCursor cursor to point into it. Clone our current bitcode
2907  // cursor to it, enter the block and read the abbrevs in that block.
2908  // With the main cursor, we just skip over it.
2909  F.DeclsCursor = Stream;
2910  if (llvm::Error Err = Stream.SkipBlock()) {
2911  Error(std::move(Err));
2912  return Failure;
2913  }
2914  if (ReadBlockAbbrevs(F.DeclsCursor, DECLTYPES_BLOCK_ID)) {
2915  Error("malformed block record in AST file");
2916  return Failure;
2917  }
2918  break;
2919 
2920  case PREPROCESSOR_BLOCK_ID:
2921  F.MacroCursor = Stream;
2922  if (!PP.getExternalSource())
2923  PP.setExternalSource(this);
2924 
2925  if (llvm::Error Err = Stream.SkipBlock()) {
2926  Error(std::move(Err));
2927  return Failure;
2928  }
2929  if (ReadBlockAbbrevs(F.MacroCursor, PREPROCESSOR_BLOCK_ID)) {
2930  Error("malformed block record in AST file");
2931  return Failure;
2932  }
2933  F.MacroStartOffset = F.MacroCursor.GetCurrentBitNo();
2934  break;
2935 
2937  F.PreprocessorDetailCursor = Stream;
2938 
2939  if (llvm::Error Err = Stream.SkipBlock()) {
2940  Error(std::move(Err));
2941  return Failure;
2942  }
2943  if (ReadBlockAbbrevs(F.PreprocessorDetailCursor,
2945  Error("malformed preprocessor detail record in AST file");
2946  return Failure;
2947  }
2949  = F.PreprocessorDetailCursor.GetCurrentBitNo();
2950 
2951  if (!PP.getPreprocessingRecord())
2952  PP.createPreprocessingRecord();
2953  if (!PP.getPreprocessingRecord()->getExternalSource())
2954  PP.getPreprocessingRecord()->SetExternalSource(*this);
2955  break;
2956 
2958  if (ReadSourceManagerBlock(F))
2959  return Failure;
2960  break;
2961 
2962  case SUBMODULE_BLOCK_ID:
2963  if (ASTReadResult Result =
2964  ReadSubmoduleBlock(F, ClientLoadCapabilities))
2965  return Result;
2966  break;
2967 
2968  case COMMENTS_BLOCK_ID: {
2969  BitstreamCursor C = Stream;
2970 
2971  if (llvm::Error Err = Stream.SkipBlock()) {
2972  Error(std::move(Err));
2973  return Failure;
2974  }
2975  if (ReadBlockAbbrevs(C, COMMENTS_BLOCK_ID)) {
2976  Error("malformed comments block in AST file");
2977  return Failure;
2978  }
2979  CommentsCursors.push_back(std::make_pair(C, &F));
2980  break;
2981  }
2982 
2983  default:
2984  if (llvm::Error Err = Stream.SkipBlock()) {
2985  Error(std::move(Err));
2986  return Failure;
2987  }
2988  break;
2989  }
2990  continue;
2991 
2992  case llvm::BitstreamEntry::Record:
2993  // The interesting case.
2994  break;
2995  }
2996 
2997  // Read and process a record.
2998  Record.clear();
2999  StringRef Blob;
3000  Expected<unsigned> MaybeRecordType =
3001  Stream.readRecord(Entry.ID, Record, &Blob);
3002  if (!MaybeRecordType) {
3003  Error(MaybeRecordType.takeError());
3004  return Failure;
3005  }
3006  ASTRecordTypes RecordType = (ASTRecordTypes)MaybeRecordType.get();
3007 
3008  // If we're not loading an AST context, we don't care about most records.
3009  if (!ContextObj) {
3010  switch (RecordType) {
3011  case IDENTIFIER_TABLE:
3012  case IDENTIFIER_OFFSET:
3014  case STATISTICS:
3015  case PP_CONDITIONAL_STACK:
3016  case PP_COUNTER_VALUE:
3018  case MODULE_OFFSET_MAP:
3021  case PPD_ENTITIES_OFFSETS:
3022  case HEADER_SEARCH_TABLE:
3023  case IMPORTED_MODULES:
3024  case MACRO_OFFSET:
3025  break;
3026  default:
3027  continue;
3028  }
3029  }
3030 
3031  switch (RecordType) {
3032  default: // Default behavior: ignore.
3033  break;
3034 
3035  case TYPE_OFFSET: {
3036  if (F.LocalNumTypes != 0) {
3037  Error("duplicate TYPE_OFFSET record in AST file");
3038  return Failure;
3039  }
3040  F.TypeOffsets = (const uint32_t *)Blob.data();
3041  F.LocalNumTypes = Record[0];
3042  unsigned LocalBaseTypeIndex = Record[1];
3043  F.BaseTypeIndex = getTotalNumTypes();
3044 
3045  if (F.LocalNumTypes > 0) {
3046  // Introduce the global -> local mapping for types within this module.
3047  GlobalTypeMap.insert(std::make_pair(getTotalNumTypes(), &F));
3048 
3049  // Introduce the local -> global mapping for types within this module.
3051  std::make_pair(LocalBaseTypeIndex,
3052  F.BaseTypeIndex - LocalBaseTypeIndex));
3053 
3054  TypesLoaded.resize(TypesLoaded.size() + F.LocalNumTypes);
3055  }
3056  break;
3057  }
3058 
3059  case DECL_OFFSET: {
3060  if (F.LocalNumDecls != 0) {
3061  Error("duplicate DECL_OFFSET record in AST file");
3062  return Failure;
3063  }
3064  F.DeclOffsets = (const DeclOffset *)Blob.data();
3065  F.LocalNumDecls = Record[0];
3066  unsigned LocalBaseDeclID = Record[1];
3067  F.BaseDeclID = getTotalNumDecls();
3068 
3069  if (F.LocalNumDecls > 0) {
3070  // Introduce the global -> local mapping for declarations within this
3071  // module.
3072  GlobalDeclMap.insert(
3073  std::make_pair(getTotalNumDecls() + NUM_PREDEF_DECL_IDS, &F));
3074 
3075  // Introduce the local -> global mapping for declarations within this
3076  // module.
3078  std::make_pair(LocalBaseDeclID, F.BaseDeclID - LocalBaseDeclID));
3079 
3080  // Introduce the global -> local mapping for declarations within this
3081  // module.
3082  F.GlobalToLocalDeclIDs[&F] = LocalBaseDeclID;
3083 
3084  DeclsLoaded.resize(DeclsLoaded.size() + F.LocalNumDecls);
3085  }
3086  break;
3087  }
3088 
3089  case TU_UPDATE_LEXICAL: {
3090  DeclContext *TU = ContextObj->getTranslationUnitDecl();
3091  LexicalContents Contents(
3092  reinterpret_cast<const llvm::support::unaligned_uint32_t *>(
3093  Blob.data()),
3094  static_cast<unsigned int>(Blob.size() / 4));
3095  TULexicalDecls.push_back(std::make_pair(&F, Contents));
3096  TU->setHasExternalLexicalStorage(true);
3097  break;
3098  }
3099 
3100  case UPDATE_VISIBLE: {
3101  unsigned Idx = 0;
3102  serialization::DeclID ID = ReadDeclID(F, Record, Idx);
3103  auto *Data = (const unsigned char*)Blob.data();
3104  PendingVisibleUpdates[ID].push_back(PendingVisibleUpdate{&F, Data});
3105  // If we've already loaded the decl, perform the updates when we finish
3106  // loading this block.
3107  if (Decl *D = GetExistingDecl(ID))
3108  PendingUpdateRecords.push_back(
3109  PendingUpdateRecord(ID, D, /*JustLoaded=*/false));
3110  break;
3111  }
3112 
3113  case IDENTIFIER_TABLE:
3114  F.IdentifierTableData = Blob.data();
3115  if (Record[0]) {
3117  (const unsigned char *)F.IdentifierTableData + Record[0],
3118  (const unsigned char *)F.IdentifierTableData + sizeof(uint32_t),
3119  (const unsigned char *)F.IdentifierTableData,
3120  ASTIdentifierLookupTrait(*this, F));
3121 
3122  PP.getIdentifierTable().setExternalIdentifierLookup(this);
3123  }
3124  break;
3125 
3126  case IDENTIFIER_OFFSET: {
3127  if (F.LocalNumIdentifiers != 0) {
3128  Error("duplicate IDENTIFIER_OFFSET record in AST file");
3129  return Failure;
3130  }
3131  F.IdentifierOffsets = (const uint32_t *)Blob.data();
3132  F.LocalNumIdentifiers = Record[0];
3133  unsigned LocalBaseIdentifierID = Record[1];
3134  F.BaseIdentifierID = getTotalNumIdentifiers();
3135 
3136  if (F.LocalNumIdentifiers > 0) {
3137  // Introduce the global -> local mapping for identifiers within this
3138  // module.
3139  GlobalIdentifierMap.insert(std::make_pair(getTotalNumIdentifiers() + 1,
3140  &F));
3141 
3142  // Introduce the local -> global mapping for identifiers within this
3143  // module.
3145  std::make_pair(LocalBaseIdentifierID,
3146  F.BaseIdentifierID - LocalBaseIdentifierID));
3147 
3148  IdentifiersLoaded.resize(IdentifiersLoaded.size()
3149  + F.LocalNumIdentifiers);
3150  }
3151  break;
3152  }
3153 
3155  F.PreloadIdentifierOffsets.assign(Record.begin(), Record.end());
3156  break;
3157 
3159  // FIXME: Skip reading this record if our ASTConsumer doesn't care
3160  // about "interesting" decls (for instance, if we're building a module).
3161  for (unsigned I = 0, N = Record.size(); I != N; ++I)
3162  EagerlyDeserializedDecls.push_back(getGlobalDeclID(F, Record[I]));
3163  break;
3164 
3165  case MODULAR_CODEGEN_DECLS:
3166  // FIXME: Skip reading this record if our ASTConsumer doesn't care about
3167  // them (ie: if we're not codegenerating this module).
3168  if (F.Kind == MK_MainFile)
3169  for (unsigned I = 0, N = Record.size(); I != N; ++I)
3170  EagerlyDeserializedDecls.push_back(getGlobalDeclID(F, Record[I]));
3171  break;
3172 
3173  case SPECIAL_TYPES:
3174  if (SpecialTypes.empty()) {
3175  for (unsigned I = 0, N = Record.size(); I != N; ++I)
3176  SpecialTypes.push_back(getGlobalTypeID(F, Record[I]));
3177  break;
3178  }
3179 
3180  if (SpecialTypes.size() != Record.size()) {
3181  Error("invalid special-types record");
3182  return Failure;
3183  }
3184 
3185  for (unsigned I = 0, N = Record.size(); I != N; ++I) {
3186  serialization::TypeID ID = getGlobalTypeID(F, Record[I]);
3187  if (!SpecialTypes[I])
3188  SpecialTypes[I] = ID;
3189  // FIXME: If ID && SpecialTypes[I] != ID, do we need a separate
3190  // merge step?
3191  }
3192  break;
3193 
3194  case STATISTICS:
3195  TotalNumStatements += Record[0];
3196  TotalNumMacros += Record[1];
3197  TotalLexicalDeclContexts += Record[2];
3198  TotalVisibleDeclContexts += Record[3];
3199  break;
3200 
3202  for (unsigned I = 0, N = Record.size(); I != N; ++I)
3203  UnusedFileScopedDecls.push_back(getGlobalDeclID(F, Record[I]));
3204  break;
3205 
3206  case DELEGATING_CTORS:
3207  for (unsigned I = 0, N = Record.size(); I != N; ++I)
3208  DelegatingCtorDecls.push_back(getGlobalDeclID(F, Record[I]));
3209  break;
3210 
3212  if (Record.size() % 4 != 0) {
3213  Error("invalid weak identifiers record");
3214  return Failure;
3215  }
3216 
3217  // FIXME: Ignore weak undeclared identifiers from non-original PCH
3218  // files. This isn't the way to do it :)
3219  WeakUndeclaredIdentifiers.clear();
3220 
3221  // Translate the weak, undeclared identifiers into global IDs.
3222  for (unsigned I = 0, N = Record.size(); I < N; /* in loop */) {
3223  WeakUndeclaredIdentifiers.push_back(
3224  getGlobalIdentifierID(F, Record[I++]));
3225  WeakUndeclaredIdentifiers.push_back(
3226  getGlobalIdentifierID(F, Record[I++]));
3227  WeakUndeclaredIdentifiers.push_back(
3228  ReadSourceLocation(F, Record, I).getRawEncoding());
3229  WeakUndeclaredIdentifiers.push_back(Record[I++]);
3230  }
3231  break;
3232 
3233  case SELECTOR_OFFSETS: {
3234  F.SelectorOffsets = (const uint32_t *)Blob.data();
3235  F.LocalNumSelectors = Record[0];
3236  unsigned LocalBaseSelectorID = Record[1];
3237  F.BaseSelectorID = getTotalNumSelectors();
3238 
3239  if (F.LocalNumSelectors > 0) {
3240  // Introduce the global -> local mapping for selectors within this
3241  // module.
3242  GlobalSelectorMap.insert(std::make_pair(getTotalNumSelectors()+1, &F));
3243 
3244  // Introduce the local -> global mapping for selectors within this
3245  // module.
3247  std::make_pair(LocalBaseSelectorID,
3248  F.BaseSelectorID - LocalBaseSelectorID));
3249 
3250  SelectorsLoaded.resize(SelectorsLoaded.size() + F.LocalNumSelectors);
3251  }
3252  break;
3253  }
3254 
3255  case METHOD_POOL:
3256  F.SelectorLookupTableData = (const unsigned char *)Blob.data();
3257  if (Record[0])
3260  F.SelectorLookupTableData + Record[0],
3262  ASTSelectorLookupTrait(*this, F));
3263  TotalNumMethodPoolEntries += Record[1];
3264  break;
3265 
3267  if (!Record.empty()) {
3268  for (unsigned Idx = 0, N = Record.size() - 1; Idx < N; /* in loop */) {
3269  ReferencedSelectorsData.push_back(getGlobalSelectorID(F,
3270  Record[Idx++]));
3271  ReferencedSelectorsData.push_back(ReadSourceLocation(F, Record, Idx).
3272  getRawEncoding());
3273  }
3274  }
3275  break;
3276 
3277  case PP_CONDITIONAL_STACK:
3278  if (!Record.empty()) {
3279  unsigned Idx = 0, End = Record.size() - 1;
3280  bool ReachedEOFWhileSkipping = Record[Idx++];
3282  if (ReachedEOFWhileSkipping) {
3283  SourceLocation HashToken = ReadSourceLocation(F, Record, Idx);
3284  SourceLocation IfTokenLoc = ReadSourceLocation(F, Record, Idx);
3285  bool FoundNonSkipPortion = Record[Idx++];
3286  bool FoundElse = Record[Idx++];
3287  SourceLocation ElseLoc = ReadSourceLocation(F, Record, Idx);
3288  SkipInfo.emplace(HashToken, IfTokenLoc, FoundNonSkipPortion,
3289  FoundElse, ElseLoc);
3290  }
3291  SmallVector<PPConditionalInfo, 4> ConditionalStack;
3292  while (Idx < End) {
3293  auto Loc = ReadSourceLocation(F, Record, Idx);
3294  bool WasSkipping = Record[Idx++];
3295  bool FoundNonSkip = Record[Idx++];
3296  bool FoundElse = Record[Idx++];
3297  ConditionalStack.push_back(
3298  {Loc, WasSkipping, FoundNonSkip, FoundElse});
3299  }
3300  PP.setReplayablePreambleConditionalStack(ConditionalStack, SkipInfo);
3301  }
3302  break;
3303 
3304  case PP_COUNTER_VALUE:
3305  if (!Record.empty() && Listener)
3306  Listener->ReadCounter(F, Record[0]);
3307  break;
3308 
3309  case FILE_SORTED_DECLS:
3310  F.FileSortedDecls = (const DeclID *)Blob.data();
3311  F.NumFileSortedDecls = Record[0];
3312  break;
3313 
3314  case SOURCE_LOCATION_OFFSETS: {
3315  F.SLocEntryOffsets = (const uint32_t *)Blob.data();
3316  F.LocalNumSLocEntries = Record[0];
3317  unsigned SLocSpaceSize = Record[1];
3318  std::tie(F.SLocEntryBaseID, F.SLocEntryBaseOffset) =
3319  SourceMgr.AllocateLoadedSLocEntries(F.LocalNumSLocEntries,
3320  SLocSpaceSize);
3321  if (!F.SLocEntryBaseID) {
3322  Error("ran out of source locations");
3323  break;
3324  }
3325  // Make our entry in the range map. BaseID is negative and growing, so
3326  // we invert it. Because we invert it, though, we need the other end of
3327  // the range.
3328  unsigned RangeStart =
3329  unsigned(-F.SLocEntryBaseID) - F.LocalNumSLocEntries + 1;
3330  GlobalSLocEntryMap.insert(std::make_pair(RangeStart, &F));
3332 
3333  // SLocEntryBaseOffset is lower than MaxLoadedOffset and decreasing.
3334  assert((F.SLocEntryBaseOffset & (1U << 31U)) == 0);
3335  GlobalSLocOffsetMap.insert(
3336  std::make_pair(SourceManager::MaxLoadedOffset - F.SLocEntryBaseOffset
3337  - SLocSpaceSize,&F));
3338 
3339  // Initialize the remapping table.
3340  // Invalid stays invalid.
3341  F.SLocRemap.insertOrReplace(std::make_pair(0U, 0));
3342  // This module. Base was 2 when being compiled.
3343  F.SLocRemap.insertOrReplace(std::make_pair(2U,
3344  static_cast<int>(F.SLocEntryBaseOffset - 2)));
3345 
3346  TotalNumSLocEntries += F.LocalNumSLocEntries;
3347  break;
3348  }
3349 
3350  case MODULE_OFFSET_MAP:
3351  F.ModuleOffsetMap = Blob;
3352  break;
3353 
3355  if (ParseLineTable(F, Record))
3356  return Failure;
3357  break;
3358 
3359  case SOURCE_LOCATION_PRELOADS: {
3360  // Need to transform from the local view (1-based IDs) to the global view,
3361  // which is based off F.SLocEntryBaseID.
3362  if (!F.PreloadSLocEntries.empty()) {
3363  Error("Multiple SOURCE_LOCATION_PRELOADS records in AST file");
3364  return Failure;
3365  }
3366 
3367  F.PreloadSLocEntries.swap(Record);
3368  break;
3369  }
3370 
3371  case EXT_VECTOR_DECLS:
3372  for (unsigned I = 0, N = Record.size(); I != N; ++I)
3373  ExtVectorDecls.push_back(getGlobalDeclID(F, Record[I]));
3374  break;
3375 
3376  case VTABLE_USES:
3377  if (Record.size() % 3 != 0) {
3378  Error("Invalid VTABLE_USES record");
3379  return Failure;
3380  }
3381 
3382  // Later tables overwrite earlier ones.
3383  // FIXME: Modules will have some trouble with this. This is clearly not
3384  // the right way to do this.
3385  VTableUses.clear();
3386 
3387  for (unsigned Idx = 0, N = Record.size(); Idx != N; /* In loop */) {
3388  VTableUses.push_back(getGlobalDeclID(F, Record[Idx++]));
3389  VTableUses.push_back(
3390  ReadSourceLocation(F, Record, Idx).getRawEncoding());
3391  VTableUses.push_back(Record[Idx++]);
3392  }
3393  break;
3394 
3396  if (PendingInstantiations.size() % 2 != 0) {
3397  Error("Invalid existing PendingInstantiations");
3398  return Failure;
3399  }
3400 
3401  if (Record.size() % 2 != 0) {
3402  Error("Invalid PENDING_IMPLICIT_INSTANTIATIONS block");
3403  return Failure;
3404  }
3405 
3406  for (unsigned I = 0, N = Record.size(); I != N; /* in loop */) {
3407  PendingInstantiations.push_back(getGlobalDeclID(F, Record[I++]));
3408  PendingInstantiations.push_back(
3409  ReadSourceLocation(F, Record, I).getRawEncoding());
3410  }
3411  break;
3412 
3413  case SEMA_DECL_REFS:
3414  if (Record.size() != 3) {
3415  Error("Invalid SEMA_DECL_REFS block");
3416  return Failure;
3417  }
3418  for (unsigned I = 0, N = Record.size(); I != N; ++I)
3419  SemaDeclRefs.push_back(getGlobalDeclID(F, Record[I]));
3420  break;
3421 
3422  case PPD_ENTITIES_OFFSETS: {
3423  F.PreprocessedEntityOffsets = (const PPEntityOffset *)Blob.data();
3424  assert(Blob.size() % sizeof(PPEntityOffset) == 0);
3425  F.NumPreprocessedEntities = Blob.size() / sizeof(PPEntityOffset);
3426 
3427  unsigned LocalBasePreprocessedEntityID = Record[0];
3428 
3429  unsigned StartingID;
3430  if (!PP.getPreprocessingRecord())
3431  PP.createPreprocessingRecord();
3432  if (!PP.getPreprocessingRecord()->getExternalSource())
3433  PP.getPreprocessingRecord()->SetExternalSource(*this);
3434  StartingID
3435  = PP.getPreprocessingRecord()
3436  ->allocateLoadedEntities(F.NumPreprocessedEntities);
3437  F.BasePreprocessedEntityID = StartingID;
3438 
3439  if (F.NumPreprocessedEntities > 0) {
3440  // Introduce the global -> local mapping for preprocessed entities in
3441  // this module.
3442  GlobalPreprocessedEntityMap.insert(std::make_pair(StartingID, &F));
3443 
3444  // Introduce the local -> global mapping for preprocessed entities in
3445  // this module.
3447  std::make_pair(LocalBasePreprocessedEntityID,
3448  F.BasePreprocessedEntityID - LocalBasePreprocessedEntityID));
3449  }
3450 
3451  break;
3452  }
3453 
3454  case PPD_SKIPPED_RANGES: {
3455  F.PreprocessedSkippedRangeOffsets = (const PPSkippedRange*)Blob.data();
3456  assert(Blob.size() % sizeof(PPSkippedRange) == 0);
3457  F.NumPreprocessedSkippedRanges = Blob.size() / sizeof(PPSkippedRange);
3458 
3459  if (!PP.getPreprocessingRecord())
3460  PP.createPreprocessingRecord();
3461  if (!PP.getPreprocessingRecord()->getExternalSource())
3462  PP.getPreprocessingRecord()->SetExternalSource(*this);
3463  F.BasePreprocessedSkippedRangeID = PP.getPreprocessingRecord()
3464  ->allocateSkippedRanges(F.NumPreprocessedSkippedRanges);
3465 
3466  if (F.NumPreprocessedSkippedRanges > 0)
3467  GlobalSkippedRangeMap.insert(
3468  std::make_pair(F.BasePreprocessedSkippedRangeID, &F));
3469  break;
3470  }
3471 
3472  case DECL_UPDATE_OFFSETS:
3473  if (Record.size() % 2 != 0) {
3474  Error("invalid DECL_UPDATE_OFFSETS block in AST file");
3475  return Failure;
3476  }
3477  for (unsigned I = 0, N = Record.size(); I != N; I += 2) {
3478  GlobalDeclID ID = getGlobalDeclID(F, Record[I]);
3479  DeclUpdateOffsets[ID].push_back(std::make_pair(&F, Record[I + 1]));
3480 
3481  // If we've already loaded the decl, perform the updates when we finish
3482  // loading this block.
3483  if (Decl *D = GetExistingDecl(ID))
3484  PendingUpdateRecords.push_back(
3485  PendingUpdateRecord(ID, D, /*JustLoaded=*/false));
3486  }
3487  break;
3488 
3489  case OBJC_CATEGORIES_MAP:
3490  if (F.LocalNumObjCCategoriesInMap != 0) {
3491  Error("duplicate OBJC_CATEGORIES_MAP record in AST file");
3492  return Failure;
3493  }
3494 
3495  F.LocalNumObjCCategoriesInMap = Record[0];
3496  F.ObjCCategoriesMap = (const ObjCCategoriesInfo *)Blob.data();
3497  break;
3498 
3499  case OBJC_CATEGORIES:
3500  F.ObjCCategories.swap(Record);
3501  break;
3502 
3504  // Later tables overwrite earlier ones.
3505  // FIXME: Modules will have trouble with this.
3506  CUDASpecialDeclRefs.clear();
3507  for (unsigned I = 0, N = Record.size(); I != N; ++I)
3508  CUDASpecialDeclRefs.push_back(getGlobalDeclID(F, Record[I]));
3509  break;
3510 
3511  case HEADER_SEARCH_TABLE:
3512  F.HeaderFileInfoTableData = Blob.data();
3513  F.LocalNumHeaderFileInfos = Record[1];
3514  if (Record[0]) {
3517  (const unsigned char *)F.HeaderFileInfoTableData + Record[0],
3518  (const unsigned char *)F.HeaderFileInfoTableData,
3519  HeaderFileInfoTrait(*this, F,
3520  &PP.getHeaderSearchInfo(),
3521  Blob.data() + Record[2]));
3522 
3523  PP.getHeaderSearchInfo().SetExternalSource(this);
3524  if (!PP.getHeaderSearchInfo().getExternalLookup())
3525  PP.getHeaderSearchInfo().SetExternalLookup(this);
3526  }
3527  break;
3528 
3529  case FP_PRAGMA_OPTIONS:
3530  // Later tables overwrite earlier ones.
3531  FPPragmaOptions.swap(Record);
3532  break;
3533 
3534  case OPENCL_EXTENSIONS:
3535  for (unsigned I = 0, E = Record.size(); I != E; ) {
3536  auto Name = ReadString(Record, I);
3537  auto &Opt = OpenCLExtensions.OptMap[Name];
3538  Opt.Supported = Record[I++] != 0;
3539  Opt.Enabled = Record[I++] != 0;
3540  Opt.Avail = Record[I++];
3541  Opt.Core = Record[I++];
3542  }
3543  break;
3544 
3546  for (unsigned I = 0, E = Record.size(); I != E;) {
3547  auto TypeID = static_cast<::TypeID>(Record[I++]);
3548  auto *Type = GetType(TypeID).getTypePtr();
3549  auto NumExt = static_cast<unsigned>(Record[I++]);
3550  for (unsigned II = 0; II != NumExt; ++II) {
3551  auto Ext = ReadString(Record, I);
3552  OpenCLTypeExtMap[Type].insert(Ext);
3553  }
3554  }
3555  break;
3556 
3558  for (unsigned I = 0, E = Record.size(); I != E;) {
3559  auto DeclID = static_cast<::DeclID>(Record[I++]);
3560  auto *Decl = GetDecl(DeclID);
3561  auto NumExt = static_cast<unsigned>(Record[I++]);
3562  for (unsigned II = 0; II != NumExt; ++II) {
3563  auto Ext = ReadString(Record, I);
3564  OpenCLDeclExtMap[Decl].insert(Ext);
3565  }
3566  }
3567  break;
3568 
3569  case TENTATIVE_DEFINITIONS:
3570  for (unsigned I = 0, N = Record.size(); I != N; ++I)
3571  TentativeDefinitions.push_back(getGlobalDeclID(F, Record[I]));
3572  break;
3573 
3574  case KNOWN_NAMESPACES:
3575  for (unsigned I = 0, N = Record.size(); I != N; ++I)
3576  KnownNamespaces.push_back(getGlobalDeclID(F, Record[I]));
3577  break;
3578 
3579  case UNDEFINED_BUT_USED:
3580  if (UndefinedButUsed.size() % 2 != 0) {
3581  Error("Invalid existing UndefinedButUsed");
3582  return Failure;
3583  }
3584 
3585  if (Record.size() % 2 != 0) {
3586  Error("invalid undefined-but-used record");
3587  return Failure;
3588  }
3589  for (unsigned I = 0, N = Record.size(); I != N; /* in loop */) {
3590  UndefinedButUsed.push_back(getGlobalDeclID(F, Record[I++]));
3591  UndefinedButUsed.push_back(
3592  ReadSourceLocation(F, Record, I).getRawEncoding());
3593  }
3594  break;
3595 
3597  for (unsigned I = 0, N = Record.size(); I != N;) {
3598  DelayedDeleteExprs.push_back(getGlobalDeclID(F, Record[I++]));
3599  const uint64_t Count = Record[I++];
3600  DelayedDeleteExprs.push_back(Count);
3601  for (uint64_t C = 0; C < Count; ++C) {
3602  DelayedDeleteExprs.push_back(ReadSourceLocation(F, Record, I).getRawEncoding());
3603  bool IsArrayForm = Record[I++] == 1;
3604  DelayedDeleteExprs.push_back(IsArrayForm);
3605  }
3606  }
3607  break;
3608 
3609  case IMPORTED_MODULES:
3610  if (!F.isModule()) {
3611  // If we aren't loading a module (which has its own exports), make
3612  // all of the imported modules visible.
3613  // FIXME: Deal with macros-only imports.
3614  for (unsigned I = 0, N = Record.size(); I != N; /**/) {
3615  unsigned GlobalID = getGlobalSubmoduleID(F, Record[I++]);
3616  SourceLocation Loc = ReadSourceLocation(F, Record, I);
3617  if (GlobalID) {
3618  ImportedModules.push_back(ImportedSubmodule(GlobalID, Loc));
3619  if (DeserializationListener)
3620  DeserializationListener->ModuleImportRead(GlobalID, Loc);
3621  }
3622  }
3623  }
3624  break;
3625 
3626  case MACRO_OFFSET: {
3627  if (F.LocalNumMacros != 0) {
3628  Error("duplicate MACRO_OFFSET record in AST file");
3629  return Failure;
3630  }
3631  F.MacroOffsets = (const uint32_t *)Blob.data();
3632  F.LocalNumMacros = Record[0];
3633  unsigned LocalBaseMacroID = Record[1];
3634  F.BaseMacroID = getTotalNumMacros();
3635 
3636  if (F.LocalNumMacros > 0) {
3637  // Introduce the global -> local mapping for macros within this module.
3638  GlobalMacroMap.insert(std::make_pair(getTotalNumMacros() + 1, &F));
3639 
3640  // Introduce the local -> global mapping for macros within this module.
3642  std::make_pair(LocalBaseMacroID,
3643  F.BaseMacroID - LocalBaseMacroID));
3644 
3645  MacrosLoaded.resize(MacrosLoaded.size() + F.LocalNumMacros);
3646  }
3647  break;
3648  }
3649 
3650  case LATE_PARSED_TEMPLATE:
3651  LateParsedTemplates.append(Record.begin(), Record.end());
3652  break;
3653 
3655  if (Record.size() != 1) {
3656  Error("invalid pragma optimize record");
3657  return Failure;
3658  }
3659  OptimizeOffPragmaLocation = ReadSourceLocation(F, Record[0]);
3660  break;
3661 
3663  if (Record.size() != 1) {
3664  Error("invalid pragma ms_struct record");
3665  return Failure;
3666  }
3667  PragmaMSStructState = Record[0];
3668  break;
3669 
3671  if (Record.size() != 2) {
3672  Error("invalid pragma ms_struct record");
3673  return Failure;
3674  }
3675  PragmaMSPointersToMembersState = Record[0];
3676  PointersToMembersPragmaLocation = ReadSourceLocation(F, Record[1]);
3677  break;
3678 
3680  for (unsigned I = 0, N = Record.size(); I != N; ++I)
3681  UnusedLocalTypedefNameCandidates.push_back(
3682  getGlobalDeclID(F, Record[I]));
3683  break;
3684 
3686  if (Record.size() != 1) {
3687  Error("invalid cuda pragma options record");
3688  return Failure;
3689  }
3690  ForceCUDAHostDeviceDepth = Record[0];
3691  break;
3692 
3693  case PACK_PRAGMA_OPTIONS: {
3694  if (Record.size() < 3) {
3695  Error("invalid pragma pack record");
3696  return Failure;
3697  }
3698  PragmaPackCurrentValue = Record[0];
3699  PragmaPackCurrentLocation = ReadSourceLocation(F, Record[1]);
3700  unsigned NumStackEntries = Record[2];
3701  unsigned Idx = 3;
3702  // Reset the stack when importing a new module.
3703  PragmaPackStack.clear();
3704  for (unsigned I = 0; I < NumStackEntries; ++I) {
3705  PragmaPackStackEntry Entry;
3706  Entry.Value = Record[Idx++];
3707  Entry.Location = ReadSourceLocation(F, Record[Idx++]);
3708  Entry.PushLocation = ReadSourceLocation(F, Record[Idx++]);
3709  PragmaPackStrings.push_back(ReadString(Record, Idx));
3710  Entry.SlotLabel = PragmaPackStrings.back();
3711  PragmaPackStack.push_back(Entry);
3712  }
3713  break;
3714  }
3715  }
3716  }
3717 }
3718 
3719 void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const {
3720  assert(!F.ModuleOffsetMap.empty() && "no module offset map to read");
3721 
3722  // Additional remapping information.
3723  const unsigned char *Data = (const unsigned char*)F.ModuleOffsetMap.data();
3724  const unsigned char *DataEnd = Data + F.ModuleOffsetMap.size();
3725  F.ModuleOffsetMap = StringRef();
3726 
3727  // If we see this entry before SOURCE_LOCATION_OFFSETS, add placeholders.
3728  if (F.SLocRemap.find(0) == F.SLocRemap.end()) {
3729  F.SLocRemap.insert(std::make_pair(0U, 0));
3730  F.SLocRemap.insert(std::make_pair(2U, 1));
3731  }
3732 
3733  // Continuous range maps we may be updating in our module.
3734  using RemapBuilder = ContinuousRangeMap<uint32_t, int, 2>::Builder;
3735  RemapBuilder SLocRemap(F.SLocRemap);
3736  RemapBuilder IdentifierRemap(F.IdentifierRemap);
3737  RemapBuilder MacroRemap(F.MacroRemap);
3738  RemapBuilder PreprocessedEntityRemap(F.PreprocessedEntityRemap);
3739  RemapBuilder SubmoduleRemap(F.SubmoduleRemap);
3740  RemapBuilder SelectorRemap(F.SelectorRemap);
3741  RemapBuilder DeclRemap(F.DeclRemap);
3742  RemapBuilder TypeRemap(F.TypeRemap);
3743 
3744  while (Data < DataEnd) {
3745  // FIXME: Looking up dependency modules by filename is horrible. Let's
3746  // start fixing this with prebuilt and explicit modules and see how it
3747  // goes...
3748  using namespace llvm::support;
3749  ModuleKind Kind = static_cast<ModuleKind>(
3750  endian::readNext<uint8_t, little, unaligned>(Data));
3751  uint16_t Len = endian::readNext<uint16_t, little, unaligned>(Data);
3752  StringRef Name = StringRef((const char*)Data, Len);
3753  Data += Len;
3754  ModuleFile *OM = (Kind == MK_PrebuiltModule || Kind == MK_ExplicitModule
3755  ? ModuleMgr.lookupByModuleName(Name)
3756  : ModuleMgr.lookupByFileName(Name));
3757  if (!OM) {
3758  std::string Msg =
3759  "SourceLocation remap refers to unknown module, cannot find ";
3760  Msg.append(Name);
3761  Error(Msg);
3762  return;
3763  }
3764 
3765  uint32_t SLocOffset =
3766  endian::readNext<uint32_t, little, unaligned>(Data);
3767  uint32_t IdentifierIDOffset =
3768  endian::readNext<uint32_t, little, unaligned>(Data);
3769  uint32_t MacroIDOffset =
3770  endian::readNext<uint32_t, little, unaligned>(Data);
3771  uint32_t PreprocessedEntityIDOffset =
3772  endian::readNext<uint32_t, little, unaligned>(Data);
3773  uint32_t SubmoduleIDOffset =
3774  endian::readNext<uint32_t, little, unaligned>(Data);
3775  uint32_t SelectorIDOffset =
3776  endian::readNext<uint32_t, little, unaligned>(Data);
3777  uint32_t DeclIDOffset =
3778  endian::readNext<uint32_t, little, unaligned>(Data);
3779  uint32_t TypeIndexOffset =
3780  endian::readNext<uint32_t, little, unaligned>(Data);
3781 
3782  uint32_t None = std::numeric_limits<uint32_t>::max();
3783 
3784  auto mapOffset = [&](uint32_t Offset, uint32_t BaseOffset,
3785  RemapBuilder &Remap) {
3786  if (Offset != None)
3787  Remap.insert(std::make_pair(Offset,
3788  static_cast<int>(BaseOffset - Offset)));
3789  };
3790  mapOffset(SLocOffset, OM->SLocEntryBaseOffset, SLocRemap);
3791  mapOffset(IdentifierIDOffset, OM->BaseIdentifierID, IdentifierRemap);
3792  mapOffset(MacroIDOffset, OM->BaseMacroID, MacroRemap);
3793  mapOffset(PreprocessedEntityIDOffset, OM->BasePreprocessedEntityID,
3794  PreprocessedEntityRemap);
3795  mapOffset(SubmoduleIDOffset, OM->BaseSubmoduleID, SubmoduleRemap);
3796  mapOffset(SelectorIDOffset, OM->BaseSelectorID, SelectorRemap);
3797  mapOffset(DeclIDOffset, OM->BaseDeclID, DeclRemap);
3798  mapOffset(TypeIndexOffset, OM->BaseTypeIndex, TypeRemap);
3799 
3800  // Global -> local mappings.
3801  F.GlobalToLocalDeclIDs[OM] = DeclIDOffset;
3802  }
3803 }
3804 
3806 ASTReader::ReadModuleMapFileBlock(RecordData &Record, ModuleFile &F,
3807  const ModuleFile *ImportedBy,
3808  unsigned ClientLoadCapabilities) {
3809  unsigned Idx = 0;
3810  F.ModuleMapPath = ReadPath(F, Record, Idx);
3811 
3812  // Try to resolve ModuleName in the current header search context and
3813  // verify that it is found in the same module map file as we saved. If the
3814  // top-level AST file is a main file, skip this check because there is no
3815  // usable header search context.
3816  assert(!F.ModuleName.empty() &&
3817  "MODULE_NAME should come before MODULE_MAP_FILE");
3818  if (F.Kind == MK_ImplicitModule && ModuleMgr.begin()->Kind != MK_MainFile) {
3819  // An implicitly-loaded module file should have its module listed in some
3820  // module map file that we've already loaded.
3821  Module *M = PP.getHeaderSearchInfo().lookupModule(F.ModuleName);
3822  auto &Map = PP.getHeaderSearchInfo().getModuleMap();
3823  const FileEntry *ModMap = M ? Map.getModuleMapFileForUniquing(M) : nullptr;
3824  // Don't emit module relocation error if we have -fno-validate-pch
3825  if (!PP.getPreprocessorOpts().DisablePCHValidation && !ModMap) {
3826  assert(ImportedBy && "top-level import should be verified");
3827  if ((ClientLoadCapabilities & ARR_OutOfDate) == 0) {
3828  if (auto *ASTFE = M ? M->getASTFile() : nullptr) {
3829  // This module was defined by an imported (explicit) module.
3830  Diag(diag::err_module_file_conflict) << F.ModuleName << F.FileName
3831  << ASTFE->getName();
3832  } else {
3833  // This module was built with a different module map.
3834  Diag(diag::err_imported_module_not_found)
3835  << F.ModuleName << F.FileName << ImportedBy->FileName
3836  << F.ModuleMapPath;
3837  // In case it was imported by a PCH, there's a chance the user is
3838  // just missing to include the search path to the directory containing
3839  // the modulemap.
3840  if (ImportedBy->Kind == MK_PCH)
3841  Diag(diag::note_imported_by_pch_module_not_found)
3842  << llvm::sys::path::parent_path(F.ModuleMapPath);
3843  }
3844  }
3845  return OutOfDate;
3846  }
3847 
3848  assert(M->Name == F.ModuleName && "found module with different name");
3849 
3850  // Check the primary module map file.
3851  auto StoredModMap = FileMgr.getFile(F.ModuleMapPath);
3852  if (!StoredModMap || *StoredModMap != ModMap) {
3853  assert(ModMap && "found module is missing module map file");
3854  assert(ImportedBy && "top-level import should be verified");
3855  if ((ClientLoadCapabilities & ARR_OutOfDate) == 0)
3856  Diag(diag::err_imported_module_modmap_changed)
3857  << F.ModuleName << ImportedBy->FileName
3858  << ModMap->getName() << F.ModuleMapPath;
3859  return OutOfDate;
3860  }
3861 
3862  llvm::SmallPtrSet<const FileEntry *, 1> AdditionalStoredMaps;
3863  for (unsigned I = 0, N = Record[Idx++]; I < N; ++I) {
3864  // FIXME: we should use input files rather than storing names.
3865  std::string Filename = ReadPath(F, Record, Idx);
3866  auto F = FileMgr.getFile(Filename, false, false);
3867  if (!F) {
3868  if ((ClientLoadCapabilities & ARR_OutOfDate) == 0)
3869  Error("could not find file '" + Filename +"' referenced by AST file");
3870  return OutOfDate;
3871  }
3872  AdditionalStoredMaps.insert(*F);
3873  }
3874 
3875  // Check any additional module map files (e.g. module.private.modulemap)
3876  // that are not in the pcm.
3877  if (auto *AdditionalModuleMaps = Map.getAdditionalModuleMapFiles(M)) {
3878  for (const FileEntry *ModMap : *AdditionalModuleMaps) {
3879  // Remove files that match
3880  // Note: SmallPtrSet::erase is really remove
3881  if (!AdditionalStoredMaps.erase(ModMap)) {
3882  if ((ClientLoadCapabilities & ARR_OutOfDate) == 0)
3883  Diag(diag::err_module_different_modmap)
3884  << F.ModuleName << /*new*/0 << ModMap->getName();
3885  return OutOfDate;
3886  }
3887  }
3888  }
3889 
3890  // Check any additional module map files that are in the pcm, but not
3891  // found in header search. Cases that match are already removed.
3892  for (const FileEntry *ModMap : AdditionalStoredMaps) {
3893  if ((ClientLoadCapabilities & ARR_OutOfDate) == 0)
3894  Diag(diag::err_module_different_modmap)
3895  << F.ModuleName << /*not new*/1 << ModMap->getName();
3896  return OutOfDate;
3897  }
3898  }
3899 
3900  if (Listener)
3901  Listener->ReadModuleMapFile(F.ModuleMapPath);
3902  return Success;
3903 }
3904 
3905 /// Move the given method to the back of the global list of methods.
3907  // Find the entry for this selector in the method pool.
3908  Sema::GlobalMethodPool::iterator Known
3909  = S.MethodPool.find(Method->getSelector());
3910  if (Known == S.MethodPool.end())
3911  return;
3912 
3913  // Retrieve the appropriate method list.
3914  ObjCMethodList &Start = Method->isInstanceMethod()? Known->second.first
3915  : Known->second.second;
3916  bool Found = false;
3917  for (ObjCMethodList *List = &Start; List; List = List->getNext()) {
3918  if (!Found) {
3919  if (List->getMethod() == Method) {
3920  Found = true;
3921  } else {
3922  // Keep searching.
3923  continue;
3924  }
3925  }
3926 
3927  if (List->getNext())
3928  List->setMethod(List->getNext()->getMethod());
3929  else
3930  List->setMethod(Method);
3931  }
3932 }
3933 
3934 void ASTReader::makeNamesVisible(const HiddenNames &Names, Module *Owner) {
3935  assert(Owner->NameVisibility != Module::Hidden && "nothing to make visible?");
3936  for (Decl *D : Names) {
3937  bool wasHidden = D->isHidden();
3938  D->setVisibleDespiteOwningModule();
3939 
3940  if (wasHidden && SemaObj) {
3941  if (ObjCMethodDecl *Method = dyn_cast<ObjCMethodDecl>(D)) {
3942  moveMethodToBackOfGlobalList(*SemaObj, Method);
3943  }
3944  }
3945  }
3946 }
3947 
3949  Module::NameVisibilityKind NameVisibility,
3950  SourceLocation ImportLoc) {
3951  llvm::SmallPtrSet<Module *, 4> Visited;
3953  Stack.push_back(Mod);
3954  while (!Stack.empty()) {
3955  Mod = Stack.pop_back_val();
3956 
3957  if (NameVisibility <= Mod->NameVisibility) {
3958  // This module already has this level of visibility (or greater), so
3959  // there is nothing more to do.
3960  continue;
3961  }
3962 
3963  if (!Mod->isAvailable()) {
3964  // Modules that aren't available cannot be made visible.
3965  continue;
3966  }
3967 
3968  // Update the module's name visibility.
3969  Mod->NameVisibility = NameVisibility;
3970 
3971  // If we've already deserialized any names from this module,
3972  // mark them as visible.
3973  HiddenNamesMapType::iterator Hidden = HiddenNamesMap.find(Mod);
3974  if (Hidden != HiddenNamesMap.end()) {
3975  auto HiddenNames = std::move(*Hidden);
3976  HiddenNamesMap.erase(Hidden);
3977  makeNamesVisible(HiddenNames.second, HiddenNames.first);
3978  assert(HiddenNamesMap.find(Mod) == HiddenNamesMap.end() &&
3979  "making names visible added hidden names");
3980  }
3981 
3982  // Push any exported modules onto the stack to be marked as visible.
3983  SmallVector<Module *, 16> Exports;
3984  Mod->getExportedModules(Exports);
3986  I = Exports.begin(), E = Exports.end(); I != E; ++I) {
3987  Module *Exported = *I;
3988  if (Visited.insert(Exported).second)
3989  Stack.push_back(Exported);
3990  }
3991  }
3992 }
3993 
3994 /// We've merged the definition \p MergedDef into the existing definition
3995 /// \p Def. Ensure that \p Def is made visible whenever \p MergedDef is made
3996 /// visible.
3998  NamedDecl *MergedDef) {
3999  if (Def->isHidden()) {
4000  // If MergedDef is visible or becomes visible, make the definition visible.
4001  if (!MergedDef->isHidden())
4003  else {
4004  getContext().mergeDefinitionIntoModule(
4005  Def, MergedDef->getImportedOwningModule(),
4006  /*NotifyListeners*/ false);
4007  PendingMergedDefinitionsToDeduplicate.insert(Def);
4008  }
4009  }
4010 }
4011 
4013  if (GlobalIndex)
4014  return false;
4015 
4016  if (TriedLoadingGlobalIndex || !UseGlobalIndex ||
4017  !PP.getLangOpts().Modules)
4018  return true;
4019 
4020  // Try to load the global index.
4021  TriedLoadingGlobalIndex = true;
4022  StringRef ModuleCachePath
4023  = getPreprocessor().getHeaderSearchInfo().getModuleCachePath();
4024  std::pair<GlobalModuleIndex *, llvm::Error> Result =
4025  GlobalModuleIndex::readIndex(ModuleCachePath);
4026  if (llvm::Error Err = std::move(Result.second)) {
4027  assert(!Result.first);
4028  consumeError(std::move(Err)); // FIXME this drops errors on the floor.
4029  return true;
4030  }
4031 
4032  GlobalIndex.reset(Result.first);
4033  ModuleMgr.setGlobalIndex(GlobalIndex.get());
4034  return false;
4035 }
4036 
4038  return PP.getLangOpts().Modules && UseGlobalIndex &&
4039  !hasGlobalIndex() && TriedLoadingGlobalIndex;
4040 }
4041 
4043  // Overwrite the timestamp file contents so that file's mtime changes.
4044  std::string TimestampFilename = MF.getTimestampFilename();
4045  std::error_code EC;
4046  llvm::raw_fd_ostream OS(TimestampFilename, EC, llvm::sys::fs::OF_Text);
4047  if (EC)
4048  return;
4049  OS << "Timestamp file\n";
4050  OS.close();
4051  OS.clear_error(); // Avoid triggering a fatal error.
4052 }
4053 
4054 /// Given a cursor at the start of an AST file, scan ahead and drop the
4055 /// cursor into the start of the given block ID, returning false on success and
4056 /// true on failure.
4057 static bool SkipCursorToBlock(BitstreamCursor &Cursor, unsigned BlockID) {
4058  while (true) {
4059  Expected<llvm::BitstreamEntry> MaybeEntry = Cursor.advance();
4060  if (!MaybeEntry) {
4061  // FIXME this drops errors on the floor.
4062  consumeError(MaybeEntry.takeError());
4063  return true;
4064  }
4065  llvm::BitstreamEntry Entry = MaybeEntry.get();
4066 
4067  switch (Entry.Kind) {
4069  case llvm::BitstreamEntry::EndBlock:
4070  return true;
4071 
4072  case llvm::BitstreamEntry::Record:
4073  // Ignore top-level records.
4074  if (Expected<unsigned> Skipped = Cursor.skipRecord(Entry.ID))
4075  break;
4076  else {
4077  // FIXME this drops errors on the floor.
4078  consumeError(Skipped.takeError());
4079  return true;
4080  }
4081 
4082  case llvm::BitstreamEntry::SubBlock:
4083  if (Entry.ID == BlockID) {
4084  if (llvm::Error Err = Cursor.EnterSubBlock(BlockID)) {
4085  // FIXME this drops the error on the floor.
4086  consumeError(std::move(Err));
4087  return true;
4088  }
4089  // Found it!
4090  return false;
4091  }
4092 
4093  if (llvm::Error Err = Cursor.SkipBlock()) {
4094  // FIXME this drops the error on the floor.
4095  consumeError(std::move(Err));
4096  return true;
4097  }
4098  }
4099  }
4100 }
4101 
4103  ModuleKind Type,
4104  SourceLocation ImportLoc,
4105  unsigned ClientLoadCapabilities,
4108  SetCurImportLocRAII(CurrentImportLoc, ImportLoc);
4109 
4110  // Defer any pending actions until we get to the end of reading the AST file.
4111  Deserializing AnASTFile(this);
4112 
4113  // Bump the generation number.
4114  unsigned PreviousGeneration = 0;
4115  if (ContextObj)
4116  PreviousGeneration = incrementGeneration(*ContextObj);
4117 
4118  unsigned NumModules = ModuleMgr.size();
4120  switch (ASTReadResult ReadResult =
4121  ReadASTCore(FileName, Type, ImportLoc,
4122  /*ImportedBy=*/nullptr, Loaded, 0, 0,
4123  ASTFileSignature(), ClientLoadCapabilities)) {
4124  case Failure:
4125  case Missing:
4126  case OutOfDate:
4127  case VersionMismatch:
4128  case ConfigurationMismatch:
4129  case HadErrors: {
4130  llvm::SmallPtrSet<ModuleFile *, 4> LoadedSet;
4131  for (const ImportedModule &IM : Loaded)
4132  LoadedSet.insert(IM.Mod);
4133 
4134  ModuleMgr.removeModules(ModuleMgr.begin() + NumModules, LoadedSet,
4135  PP.getLangOpts().Modules
4136  ? &PP.getHeaderSearchInfo().getModuleMap()
4137  : nullptr);
4138 
4139  // If we find that any modules are unusable, the global index is going
4140  // to be out-of-date. Just remove it.
4141  GlobalIndex.reset();
4142  ModuleMgr.setGlobalIndex(nullptr);
4143  return ReadResult;
4144  }
4145  case Success:
4146  break;
4147  }
4148 
4149  // Here comes stuff that we only do once the entire chain is loaded.
4150 
4151  // Load the AST blocks of all of the modules that we loaded.
4152  for (SmallVectorImpl<ImportedModule>::iterator M = Loaded.begin(),
4153  MEnd = Loaded.end();
4154  M != MEnd; ++M) {
4155  ModuleFile &F = *M->Mod;
4156 
4157  // Read the AST block.
4158  if (ASTReadResult Result = ReadASTBlock(F, ClientLoadCapabilities))
4159  return Result;
4160 
4161  // Read the extension blocks.
4163  if (ASTReadResult Result = ReadExtensionBlock(F))
4164  return Result;
4165  }
4166 
4167  // Once read, set the ModuleFile bit base offset and update the size in
4168  // bits of all files we've seen.
4169  F.GlobalBitOffset = TotalModulesSizeInBits;
4170  TotalModulesSizeInBits += F.SizeInBits;
4171  GlobalBitOffsetsMap.insert(std::make_pair(F.GlobalBitOffset, &F));
4172 
4173  // Preload SLocEntries.
4174  for (unsigned I = 0, N = F.PreloadSLocEntries.size(); I != N; ++I) {
4175  int Index = int(F.PreloadSLocEntries[I] - 1) + F.SLocEntryBaseID;
4176  // Load it through the SourceManager and don't call ReadSLocEntry()
4177  // directly because the entry may have already been loaded in which case
4178  // calling ReadSLocEntry() directly would trigger an assertion in
4179  // SourceManager.
4180  SourceMgr.getLoadedSLocEntryByID(Index);
4181  }
4182 
4183  // Map the original source file ID into the ID space of the current
4184  // compilation.
4185  if (F.OriginalSourceFileID.isValid()) {
4186  F.OriginalSourceFileID = FileID::get(
4187  F.SLocEntryBaseID + F.OriginalSourceFileID.getOpaqueValue() - 1);
4188  }
4189 
4190  // Preload all the pending interesting identifiers by marking them out of
4191  // date.
4192  for (auto Offset : F.PreloadIdentifierOffsets) {
4193  const unsigned char *Data = reinterpret_cast<const unsigned char *>(
4195 
4196  ASTIdentifierLookupTrait Trait(*this, F);
4197  auto KeyDataLen = Trait.ReadKeyDataLength(Data);
4198  auto Key = Trait.ReadKey(Data, KeyDataLen.first);
4199  auto &II = PP.getIdentifierTable().getOwn(Key);
4200  II.setOutOfDate(true);
4201 
4202  // Mark this identifier as being from an AST file so that we can track
4203  // whether we need to serialize it.
4204  markIdentifierFromAST(*this, II);
4205 
4206  // Associate the ID with the identifier so that the writer can reuse it.
4207  auto ID = Trait.ReadIdentifierID(Data + KeyDataLen.first);
4208  SetIdentifierInfo(ID, &II);
4209  }
4210  }
4211 
4212  // Setup the import locations and notify the module manager that we've
4213  // committed to these module files.
4214  for (SmallVectorImpl<ImportedModule>::iterator M = Loaded.begin(),
4215  MEnd = Loaded.end();
4216  M != MEnd; ++M) {
4217  ModuleFile &F = *M->Mod;
4218 
4219  ModuleMgr.moduleFileAccepted(&F);
4220 
4221  // Set the import location.
4222  F.DirectImportLoc = ImportLoc;
4223  // FIXME: We assume that locations from PCH / preamble do not need
4224  // any translation.
4225  if (!M->ImportedBy)
4226  F.ImportLoc = M->ImportLoc;
4227  else
4228  F.ImportLoc = TranslateSourceLocation(*M->ImportedBy, M->ImportLoc);
4229  }
4230 
4231  if (!PP.getLangOpts().CPlusPlus ||
4232  (Type != MK_ImplicitModule && Type != MK_ExplicitModule &&
4233  Type != MK_PrebuiltModule)) {
4234  // Mark all of the identifiers in the identifier table as being out of date,
4235  // so that various accessors know to check the loaded modules when the
4236  // identifier is used.
4237  //
4238  // For C++ modules, we don't need information on many identifiers (just
4239  // those that provide macros or are poisoned), so we mark all of
4240  // the interesting ones via PreloadIdentifierOffsets.
4241  for (IdentifierTable::iterator Id = PP.getIdentifierTable().begin(),
4242  IdEnd = PP.getIdentifierTable().end();
4243  Id != IdEnd; ++Id)
4244  Id->second->setOutOfDate(true);
4245  }
4246  // Mark selectors as out of date.
4247  for (auto Sel : SelectorGeneration)
4248  SelectorOutOfDate[Sel.first] = true;
4249 
4250  // Resolve any unresolved module exports.
4251  for (unsigned I = 0, N = UnresolvedModuleRefs.size(); I != N; ++I) {
4252  UnresolvedModuleRef &Unresolved = UnresolvedModuleRefs[I];
4253  SubmoduleID GlobalID = getGlobalSubmoduleID(*Unresolved.File,Unresolved.ID);
4254  Module *ResolvedMod = getSubmodule(GlobalID);
4255 
4256  switch (Unresolved.Kind) {
4257  case UnresolvedModuleRef::Conflict:
4258  if (ResolvedMod) {
4259  Module::Conflict Conflict;
4260  Conflict.Other = ResolvedMod;
4261  Conflict.Message = Unresolved.String.str();
4262  Unresolved.Mod->Conflicts.push_back(Conflict);
4263  }
4264  continue;
4265 
4266  case UnresolvedModuleRef::Import:
4267  if (ResolvedMod)
4268  Unresolved.Mod->Imports.insert(ResolvedMod);
4269  continue;
4270 
4271  case UnresolvedModuleRef::Export:
4272  if (ResolvedMod || Unresolved.IsWildcard)
4273  Unresolved.Mod->Exports.push_back(
4274  Module::ExportDecl(ResolvedMod, Unresolved.IsWildcard));
4275  continue;
4276  }
4277  }
4278  UnresolvedModuleRefs.clear();
4279 
4280  if (Imported)
4281  Imported->append(ImportedModules.begin(),
4282  ImportedModules.end());
4283 
4284  // FIXME: How do we load the 'use'd modules? They may not be submodules.
4285  // Might be unnecessary as use declarations are only used to build the
4286  // module itself.
4287 
4288  if (ContextObj)
4289  InitializeContext();
4290 
4291  if (SemaObj)
4292  UpdateSema();
4293 
4294  if (DeserializationListener)
4295  DeserializationListener->ReaderInitialized(this);
4296 
4297  ModuleFile &PrimaryModule = ModuleMgr.getPrimaryModule();
4298  if (PrimaryModule.OriginalSourceFileID.isValid()) {
4299  // If this AST file is a precompiled preamble, then set the
4300  // preamble file ID of the source manager to the file source file
4301  // from which the preamble was built.
4302  if (Type == MK_Preamble) {
4303  SourceMgr.setPreambleFileID(PrimaryModule.OriginalSourceFileID);
4304  } else if (Type == MK_MainFile) {
4305  SourceMgr.setMainFileID(PrimaryModule.OriginalSourceFileID);
4306  }
4307  }
4308 
4309  // For any Objective-C class definitions we have already loaded, make sure
4310  // that we load any additional categories.
4311  if (ContextObj) {
4312  for (unsigned I = 0, N = ObjCClassesLoaded.size(); I != N; ++I) {
4313  loadObjCCategories(ObjCClassesLoaded[I]->getGlobalID(),
4314  ObjCClassesLoaded[I],
4315  PreviousGeneration);
4316  }
4317  }
4318 
4319  if (PP.getHeaderSearchInfo()
4320  .getHeaderSearchOpts()
4321  .ModulesValidateOncePerBuildSession) {
4322  // Now we are certain that the module and all modules it depends on are
4323  // up to date. Create or update timestamp files for modules that are
4324  // located in the module cache (not for PCH files that could be anywhere
4325  // in the filesystem).
4326  for (unsigned I = 0, N = Loaded.size(); I != N; ++I) {
4327  ImportedModule &M = Loaded[I];
4328  if (M.Mod->Kind == MK_ImplicitModule) {
4329  updateModuleTimestamp(*M.Mod);
4330  }
4331  }
4332  }
4333 
4334  return Success;
4335 }
4336 
4337 static ASTFileSignature readASTFileSignature(StringRef PCH);
4338 
4339 /// Whether \p Stream doesn't start with the AST/PCH file magic number 'CPCH'.
4340 static llvm::Error doesntStartWithASTFileMagic(BitstreamCursor &Stream) {
4341  // FIXME checking magic headers is done in other places such as
4342  // SerializedDiagnosticReader and GlobalModuleIndex, but error handling isn't
4343  // always done the same. Unify it all with a helper.
4344  if (!Stream.canSkipToPos(4))
4345  return llvm::createStringError(std::errc::illegal_byte_sequence,
4346  "file too small to contain AST file magic");
4347  for (unsigned C : {'C', 'P', 'C', 'H'})
4348  if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = Stream.Read(8)) {
4349  if (Res.get() != C)
4350  return llvm::createStringError(
4351  std::errc::illegal_byte_sequence,
4352  "file doesn't start with AST file magic");
4353  } else
4354  return Res.takeError();
4355  return llvm::Error::success();
4356 }
4357 
4359  switch (Kind) {
4360  case MK_PCH:
4361  return 0; // PCH
4362  case MK_ImplicitModule:
4363  case MK_ExplicitModule:
4364  case MK_PrebuiltModule:
4365  return 1; // module
4366  case MK_MainFile:
4367  case MK_Preamble:
4368  return 2; // main source file
4369  }
4370  llvm_unreachable("unknown module kind");
4371 }
4372 
4374 ASTReader::ReadASTCore(StringRef FileName,
4375  ModuleKind Type,
4376  SourceLocation ImportLoc,
4377  ModuleFile *ImportedBy,
4379  off_t ExpectedSize, time_t ExpectedModTime,
4380  ASTFileSignature ExpectedSignature,
4381  unsigned ClientLoadCapabilities) {
4382  ModuleFile *M;
4383  std::string ErrorStr;
4385  = ModuleMgr.addModule(FileName, Type, ImportLoc, ImportedBy,
4386  getGeneration(), ExpectedSize, ExpectedModTime,
4387  ExpectedSignature, readASTFileSignature,
4388  M, ErrorStr);
4389 
4390  switch (AddResult) {
4392  Diag(diag::remark_module_import)
4393  << M->ModuleName << M->FileName << (ImportedBy ? true : false)
4394  << (ImportedBy ? StringRef(ImportedBy->ModuleName) : StringRef());
4395  return Success;
4396 
4398  // Load module file below.
4399  break;
4400 
4402  // The module file was missing; if the client can handle that, return
4403  // it.
4404  if (ClientLoadCapabilities & ARR_Missing)
4405  return Missing;
4406 
4407  // Otherwise, return an error.
4408  Diag(diag::err_module_file_not_found) << moduleKindForDiagnostic(Type)
4409  << FileName << !ErrorStr.empty()
4410  << ErrorStr;
4411  return Failure;
4412 
4414  // We couldn't load the module file because it is out-of-date. If the
4415  // client can handle out-of-date, return it.
4416  if (ClientLoadCapabilities & ARR_OutOfDate)
4417  return OutOfDate;
4418 
4419  // Otherwise, return an error.
4420  Diag(diag::err_module_file_out_of_date) << moduleKindForDiagnostic(Type)
4421  << FileName << !ErrorStr.empty()
4422  << ErrorStr;
4423  return Failure;
4424  }
4425 
4426  assert(M && "Missing module file");
4427 
4428  bool ShouldFinalizePCM = false;
4429  auto FinalizeOrDropPCM = llvm::make_scope_exit([&]() {
4430  auto &MC = getModuleManager().getModuleCache();
4431  if (ShouldFinalizePCM)
4432  MC.finalizePCM(FileName);
4433  else
4434  MC.tryToDropPCM(FileName);
4435  });
4436  ModuleFile &F = *M;
4437  BitstreamCursor &Stream = F.Stream;
4438  Stream = BitstreamCursor(PCHContainerRdr.ExtractPCH(*F.Buffer));
4439  F.SizeInBits = F.Buffer->getBufferSize() * 8;
4440 
4441  // Sniff for the signature.
4442  if (llvm::Error Err = doesntStartWithASTFileMagic(Stream)) {
4443  Diag(diag::err_module_file_invalid)
4444  << moduleKindForDiagnostic(Type) << FileName << std::move(Err);
4445  return Failure;
4446  }
4447 
4448  // This is used for compatibility with older PCH formats.
4449  bool HaveReadControlBlock = false;
4450  while (true) {
4451  Expected<llvm::BitstreamEntry> MaybeEntry = Stream.advance();
4452  if (!MaybeEntry) {
4453  Error(MaybeEntry.takeError());
4454  return Failure;
4455  }
4456  llvm::BitstreamEntry Entry = MaybeEntry.get();
4457 
4458  switch (Entry.Kind) {
4460  case llvm::BitstreamEntry::Record:
4461  case llvm::BitstreamEntry::EndBlock:
4462  Error("invalid record at top-level of AST file");
4463  return Failure;
4464 
4465  case llvm::BitstreamEntry::SubBlock:
4466  break;
4467  }
4468 
4469  switch (Entry.ID) {
4470  case CONTROL_BLOCK_ID:
4471  HaveReadControlBlock = true;
4472  switch (ReadControlBlock(F, Loaded, ImportedBy, ClientLoadCapabilities)) {
4473  case Success:
4474  // Check that we didn't try to load a non-module AST file as a module.
4475  //
4476  // FIXME: Should we also perform the converse check? Loading a module as
4477  // a PCH file sort of works, but it's a bit wonky.
4478  if ((Type == MK_ImplicitModule || Type == MK_ExplicitModule ||
4479  Type == MK_PrebuiltModule) &&
4480  F.ModuleName.empty()) {
4481  auto Result = (Type == MK_ImplicitModule) ? OutOfDate : Failure;
4482  if (Result != OutOfDate ||
4483  (ClientLoadCapabilities & ARR_OutOfDate) == 0)
4484  Diag(diag::err_module_file_not_module) << FileName;
4485  return Result;
4486  }
4487  break;
4488 
4489  case Failure: return Failure;
4490  case Missing: return Missing;
4491  case OutOfDate: return OutOfDate;
4492  case VersionMismatch: return VersionMismatch;
4493  case ConfigurationMismatch: return ConfigurationMismatch;
4494  case HadErrors: return HadErrors;
4495  }
4496  break;
4497 
4498  case AST_BLOCK_ID:
4499  if (!HaveReadControlBlock) {
4500  if ((ClientLoadCapabilities & ARR_VersionMismatch) == 0)
4501  Diag(diag::err_pch_version_too_old);
4502  return VersionMismatch;
4503  }
4504 
4505  // Record that we've loaded this module.
4506  Loaded.push_back(ImportedModule(M, ImportedBy, ImportLoc));
4507  ShouldFinalizePCM = true;
4508  return Success;
4509 
4511  // This block is handled using look-ahead during ReadControlBlock. We
4512  // shouldn't get here!
4513  Error("malformed block record in AST file");
4514  return Failure;
4515 
4516  default:
4517  if (llvm::Error Err = Stream.SkipBlock()) {
4518  Error(std::move(Err));
4519  return Failure;
4520  }
4521  break;
4522  }
4523  }
4524 
4525  llvm_unreachable("unexpected break; expected return");
4526 }
4527 
4529 ASTReader::readUnhashedControlBlock(ModuleFile &F, bool WasImportedBy,
4530  unsigned ClientLoadCapabilities) {
4531  const HeaderSearchOptions &HSOpts =
4532  PP.getHeaderSearchInfo().getHeaderSearchOpts();
4533  bool AllowCompatibleConfigurationMismatch =
4535 
4536  ASTReadResult Result = readUnhashedControlBlockImpl(
4537  &F, F.Data, ClientLoadCapabilities, AllowCompatibleConfigurationMismatch,
4538  Listener.get(),
4539  WasImportedBy ? false : HSOpts.ModulesValidateDiagnosticOptions);
4540 
4541  // If F was directly imported by another module, it's implicitly validated by
4542  // the importing module.
4543  if (DisableValidation || WasImportedBy ||
4544  (AllowConfigurationMismatch && Result == ConfigurationMismatch))
4545  return Success;
4546 
4547  if (Result == Failure) {
4548  Error("malformed block record in AST file");
4549  return Failure;
4550  }
4551 
4552  if (Result == OutOfDate && F.Kind == MK_ImplicitModule) {
4553  // If this module has already been finalized in the ModuleCache, we're stuck
4554  // with it; we can only load a single version of each module.
4555  //
4556  // This can happen when a module is imported in two contexts: in one, as a
4557  // user module; in another, as a system module (due to an import from
4558  // another module marked with the [system] flag). It usually indicates a
4559  // bug in the module map: this module should also be marked with [system].
4560  //
4561  // If -Wno-system-headers (the default), and the first import is as a
4562  // system module, then validation will fail during the as-user import,
4563  // since -Werror flags won't have been validated. However, it's reasonable
4564  // to treat this consistently as a system module.
4565  //
4566  // If -Wsystem-headers, the PCM on disk was built with
4567  // -Wno-system-headers, and the first import is as a user module, then
4568  // validation will fail during the as-system import since the PCM on disk
4569  // doesn't guarantee that -Werror was respected. However, the -Werror
4570  // flags were checked during the initial as-user import.
4571  if (getModuleManager().getModuleCache().isPCMFinal(F.FileName)) {
4572  Diag(diag::warn_module_system_bit_conflict) << F.FileName;
4573  return Success;
4574  }
4575  }
4576 
4577  return Result;
4578 }
4579 
4580 ASTReader::ASTReadResult ASTReader::readUnhashedControlBlockImpl(
4581  ModuleFile *F, llvm::StringRef StreamData, unsigned ClientLoadCapabilities,
4582  bool AllowCompatibleConfigurationMismatch, ASTReaderListener *Listener,
4583  bool ValidateDiagnosticOptions) {
4584  // Initialize a stream.
4585  BitstreamCursor Stream(StreamData);
4586 
4587  // Sniff for the signature.
4588  if (llvm::Error Err = doesntStartWithASTFileMagic(Stream)) {
4589  // FIXME this drops the error on the floor.
4590  consumeError(std::move(Err));
4591  return Failure;
4592  }
4593 
4594  // Scan for the UNHASHED_CONTROL_BLOCK_ID block.
4596  return Failure;
4597 
4598  // Read all of the records in the options block.
4599  RecordData Record;
4600  ASTReadResult Result = Success;
4601  while (true) {
4602  Expected<llvm::BitstreamEntry> MaybeEntry = Stream.advance();
4603  if (!MaybeEntry) {
4604  // FIXME this drops the error on the floor.
4605  consumeError(MaybeEntry.takeError());
4606  return Failure;
4607  }
4608  llvm::BitstreamEntry Entry = MaybeEntry.get();
4609 
4610  switch (Entry.Kind) {
4612  case llvm::BitstreamEntry::SubBlock:
4613  return Failure;
4614 
4615  case llvm::BitstreamEntry::EndBlock:
4616  return Result;
4617 
4618  case llvm::BitstreamEntry::Record:
4619  // The interesting case.
4620  break;
4621  }
4622 
4623  // Read and process a record.
4624  Record.clear();
4625  Expected<unsigned> MaybeRecordType = Stream.readRecord(Entry.ID, Record);
4626  if (!MaybeRecordType) {
4627  // FIXME this drops the error.
4628  return Failure;
4629  }
4630  switch ((UnhashedControlBlockRecordTypes)MaybeRecordType.get()) {
4631  case SIGNATURE:
4632  if (F)
4633  std::copy(Record.begin(), Record.end(), F->Signature.data());
4634  break;
4635  case DIAGNOSTIC_OPTIONS: {
4636  bool Complain = (ClientLoadCapabilities & ARR_OutOfDate) == 0;
4637  if (Listener && ValidateDiagnosticOptions &&
4638  !AllowCompatibleConfigurationMismatch &&
4639  ParseDiagnosticOptions(Record, Complain, *Listener))
4640  Result = OutOfDate; // Don't return early. Read the signature.
4641  break;
4642  }
4643  case DIAG_PRAGMA_MAPPINGS:
4644  if (!F)
4645  break;
4646  if (F->PragmaDiagMappings.empty())
4647  F->PragmaDiagMappings.swap(Record);
4648  else
4649  F->PragmaDiagMappings.insert(F->PragmaDiagMappings.end(),
4650  Record.begin(), Record.end());
4651  break;
4652  }
4653  }
4654 }
4655 
4656 /// Parse a record and blob containing module file extension metadata.
4658  const SmallVectorImpl<uint64_t> &Record,
4659  StringRef Blob,
4660  ModuleFileExtensionMetadata &Metadata) {
4661  if (Record.size() < 4) return true;
4662 
4663  Metadata.MajorVersion = Record[0];
4664  Metadata.MinorVersion = Record[1];
4665 
4666  unsigned BlockNameLen = Record[2];
4667  unsigned UserInfoLen = Record[3];
4668 
4669  if (BlockNameLen + UserInfoLen > Blob.size()) return true;
4670 
4671  Metadata.BlockName = std::string(Blob.data(), Blob.data() + BlockNameLen);
4672  Metadata.UserInfo = std::string(Blob.data() + BlockNameLen,
4673  Blob.data() + BlockNameLen + UserInfoLen);
4674  return false;
4675 }
4676 
4677 ASTReader::ASTReadResult ASTReader::ReadExtensionBlock(ModuleFile &F) {
4678  BitstreamCursor &Stream = F.Stream;
4679 
4680  RecordData Record;
4681  while (true) {
4682  Expected<llvm::BitstreamEntry> MaybeEntry = Stream.advance();
4683  if (!MaybeEntry) {
4684  Error(MaybeEntry.takeError());
4685  return Failure;
4686  }
4687  llvm::BitstreamEntry Entry = MaybeEntry.get();
4688 
4689  switch (Entry.Kind) {
4690  case llvm::BitstreamEntry::SubBlock:
4691  if (llvm::Error Err = Stream.SkipBlock()) {
4692  Error(std::move(Err));
4693  return Failure;
4694  }
4695  continue;
4696 
4697  case llvm::BitstreamEntry::EndBlock:
4698  return Success;
4699 
4701  return HadErrors;
4702 
4703  case llvm::BitstreamEntry::Record:
4704  break;
4705  }
4706 
4707  Record.clear();
4708  StringRef Blob;
4709  Expected<unsigned> MaybeRecCode =
4710  Stream.readRecord(Entry.ID, Record, &Blob);
4711  if (!MaybeRecCode) {
4712  Error(MaybeRecCode.takeError());
4713  return Failure;
4714  }
4715  switch (MaybeRecCode.get()) {
4716  case EXTENSION_METADATA: {
4717  ModuleFileExtensionMetadata Metadata;
4718  if (parseModuleFileExtensionMetadata(Record, Blob, Metadata))
4719  return Failure;
4720 
4721  // Find a module file extension with this block name.
4722  auto Known = ModuleFileExtensions.find(Metadata.BlockName);
4723  if (Known == ModuleFileExtensions.end()) break;
4724 
4725  // Form a reader.
4726  if (auto Reader = Known->second->createExtensionReader(Metadata, *this,
4727  F, Stream)) {
4728  F.ExtensionReaders.push_back(std::move(Reader));
4729  }
4730 
4731  break;
4732  }
4733  }
4734  }
4735 
4736  return Success;
4737 }
4738 
4740  assert(ContextObj && "no context to initialize");
4741  ASTContext &Context = *ContextObj;
4742 
4743  // If there's a listener, notify them that we "read" the translation unit.
4744  if (DeserializationListener)
4745  DeserializationListener->DeclRead(PREDEF_DECL_TRANSLATION_UNIT_ID,
4746  Context.getTranslationUnitDecl());
4747 
4748  // FIXME: Find a better way to deal with collisions between these
4749  // built-in types. Right now, we just ignore the problem.
4750 
4751  // Load the special types.
4752  if (SpecialTypes.size() >= NumSpecialTypeIDs) {
4753  if (unsigned String = SpecialTypes[SPECIAL_TYPE_CF_CONSTANT_STRING]) {
4754  if (!Context.CFConstantStringTypeDecl)
4755  Context.setCFConstantStringType(GetType(String));
4756  }
4757 
4758  if (unsigned File = SpecialTypes[SPECIAL_TYPE_FILE]) {
4759  QualType FileType = GetType(File);
4760  if (FileType.isNull()) {
4761  Error("FILE type is NULL");
4762  return;
4763  }
4764 
4765  if (!Context.FILEDecl) {
4766  if (const TypedefType *Typedef = FileType->getAs<TypedefType>())
4767  Context.setFILEDecl(Typedef->getDecl());
4768  else {
4769  const TagType *Tag = FileType->getAs<TagType>();
4770  if (!Tag) {
4771  Error("Invalid FILE type in AST file");
4772  return;
4773  }
4774  Context.setFILEDecl(Tag->getDecl());
4775  }
4776  }
4777  }
4778 
4779  if (unsigned Jmp_buf = SpecialTypes[SPECIAL_TYPE_JMP_BUF]) {
4780  QualType Jmp_bufType = GetType(Jmp_buf);
4781  if (Jmp_bufType.isNull()) {
4782  Error("jmp_buf type is NULL");
4783  return;
4784  }
4785 
4786  if (!Context.jmp_bufDecl) {
4787  if (const TypedefType *Typedef = Jmp_bufType->getAs<TypedefType>())
4788  Context.setjmp_bufDecl(Typedef->getDecl());
4789  else {
4790  const TagType *Tag = Jmp_bufType->getAs<TagType>();
4791  if (!Tag) {
4792  Error("Invalid jmp_buf type in AST file");
4793  return;
4794  }
4795  Context.setjmp_bufDecl(Tag->getDecl());
4796  }
4797  }
4798  }
4799 
4800  if (unsigned Sigjmp_buf = SpecialTypes[SPECIAL_TYPE_SIGJMP_BUF]) {
4801  QualType Sigjmp_bufType = GetType(Sigjmp_buf);
4802  if (Sigjmp_bufType.isNull()) {
4803  Error("sigjmp_buf type is NULL");
4804  return;
4805  }
4806 
4807  if (!Context.sigjmp_bufDecl) {
4808  if (const TypedefType *Typedef = Sigjmp_bufType->getAs<TypedefType>())
4809  Context.setsigjmp_bufDecl(Typedef->getDecl());
4810  else {
4811  const TagType *Tag = Sigjmp_bufType->getAs<TagType>();
4812  assert(Tag && "Invalid sigjmp_buf type in AST file");
4813  Context.setsigjmp_bufDecl(Tag->getDecl());
4814  }
4815  }
4816  }
4817 
4818  if (unsigned ObjCIdRedef
4819  = SpecialTypes[SPECIAL_TYPE_OBJC_ID_REDEFINITION]) {
4820  if (Context.ObjCIdRedefinitionType.isNull())
4821  Context.ObjCIdRedefinitionType = GetType(ObjCIdRedef);
4822  }
4823 
4824  if (unsigned ObjCClassRedef
4825  = SpecialTypes[SPECIAL_TYPE_OBJC_CLASS_REDEFINITION]) {
4826  if (Context.ObjCClassRedefinitionType.isNull())
4827  Context.ObjCClassRedefinitionType = GetType(ObjCClassRedef);
4828  }
4829 
4830  if (unsigned ObjCSelRedef
4831  = SpecialTypes[SPECIAL_TYPE_OBJC_SEL_REDEFINITION]) {
4832  if (Context.ObjCSelRedefinitionType.isNull())
4833  Context.ObjCSelRedefinitionType = GetType(ObjCSelRedef);
4834  }
4835 
4836  if (unsigned Ucontext_t = SpecialTypes[SPECIAL_TYPE_UCONTEXT_T]) {
4837  QualType Ucontext_tType = GetType(Ucontext_t);
4838  if (Ucontext_tType.isNull()) {
4839  Error("ucontext_t type is NULL");
4840  return;
4841  }
4842 
4843  if (!Context.ucontext_tDecl) {
4844  if (const TypedefType *Typedef = Ucontext_tType->getAs<TypedefType>())
4845  Context.setucontext_tDecl(Typedef->getDecl());
4846  else {
4847  const TagType *Tag = Ucontext_tType->getAs<TagType>();
4848  assert(Tag && "Invalid ucontext_t type in AST file");
4849  Context.setucontext_tDecl(Tag->getDecl());
4850  }
4851  }
4852  }
4853  }
4854 
4855  ReadPragmaDiagnosticMappings(Context.getDiagnostics());
4856 
4857  // If there were any CUDA special declarations, deserialize them.
4858  if (!CUDASpecialDeclRefs.empty()) {
4859  assert(CUDASpecialDeclRefs.size() == 1 && "More decl refs than expected!");
4860  Context.setcudaConfigureCallDecl(
4861  cast<FunctionDecl>(GetDecl(CUDASpecialDeclRefs[0])));
4862  }
4863 
4864  // Re-export any modules that were imported by a non-module AST file.
4865  // FIXME: This does not make macro-only imports visible again.
4866  for (auto &Import : ImportedModules) {
4867  if (Module *Imported = getSubmodule(Import.ID)) {
4868  makeModuleVisible(Imported, Module::AllVisible,
4869  /*ImportLoc=*/Import.ImportLoc);
4870  if (Import.ImportLoc.isValid())
4871  PP.makeModuleVisible(Imported, Import.ImportLoc);
4872  // FIXME: should we tell Sema to make the module visible too?
4873  }
4874  }
4875  ImportedModules.clear();
4876 }
4877 
4879  // Nothing to do for now.
4880 }
4881 
4882 /// Reads and return the signature record from \p PCH's control block, or
4883 /// else returns 0.
4885  BitstreamCursor Stream(PCH);
4886  if (llvm::Error Err = doesntStartWithASTFileMagic(Stream)) {
4887  // FIXME this drops the error on the floor.
4888  consumeError(std::move(Err));
4889  return ASTFileSignature();
4890  }
4891 
4892  // Scan for the UNHASHED_CONTROL_BLOCK_ID block.
4894  return ASTFileSignature();
4895 
4896  // Scan for SIGNATURE inside the diagnostic options block.
4897  ASTReader::RecordData Record;
4898  while (true) {
4899  Expected<llvm::BitstreamEntry> MaybeEntry =
4900  Stream.advanceSkippingSubblocks();
4901  if (!MaybeEntry) {
4902  // FIXME this drops the error on the floor.
4903  consumeError(MaybeEntry.takeError());
4904  return ASTFileSignature();
4905  }
4906  llvm::BitstreamEntry Entry = MaybeEntry.get();
4907 
4908  if (Entry.Kind != llvm::BitstreamEntry::Record)
4909  return ASTFileSignature();
4910 
4911  Record.clear();
4912  StringRef Blob;
4913  Expected<unsigned> MaybeRecord = Stream.readRecord(Entry.ID, Record, &Blob);
4914  if (!MaybeRecord) {
4915  // FIXME this drops the error on the floor.
4916  consumeError(MaybeRecord.takeError());
4917  return ASTFileSignature();
4918  }
4919  if (SIGNATURE == MaybeRecord.get())
4920  return {{{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
4921  (uint32_t)Record[3], (uint32_t)Record[4]}}};
4922  }
4923 }
4924 
4925 /// Retrieve the name of the original source file name
4926 /// directly from the AST file, without actually loading the AST
4927 /// file.
4929  const std::string &ASTFileName, FileManager &FileMgr,
4930  const PCHContainerReader &PCHContainerRdr, DiagnosticsEngine &Diags) {
4931  // Open the AST file.
4932  auto Buffer = FileMgr.getBufferForFile(ASTFileName);
4933  if (!Buffer) {
4934  Diags.Report(diag::err_fe_unable_to_read_pch_file)
4935  << ASTFileName << Buffer.getError().message();
4936  return std::string();
4937  }
4938 
4939  // Initialize the stream
4940  BitstreamCursor Stream(PCHContainerRdr.ExtractPCH(**Buffer));
4941 
4942  // Sniff for the signature.
4943  if (llvm::Error Err = doesntStartWithASTFileMagic(Stream)) {
4944  Diags.Report(diag::err_fe_not_a_pch_file) << ASTFileName << std::move(Err);
4945  return std::string();
4946  }
4947 
4948  // Scan for the CONTROL_BLOCK_ID block.
4949  if (SkipCursorToBlock(Stream, CONTROL_BLOCK_ID)) {
4950  Diags.Report(diag::err_fe_pch_malformed_block) << ASTFileName;
4951  return std::string();
4952  }
4953 
4954  // Scan for ORIGINAL_FILE inside the control block.
4955  RecordData Record;
4956  while (true) {
4957  Expected<llvm::BitstreamEntry> MaybeEntry =
4958  Stream.advanceSkippingSubblocks();
4959  if (!MaybeEntry) {
4960  // FIXME this drops errors on the floor.
4961  consumeError(MaybeEntry.takeError());
4962  return std::string();
4963  }
4964  llvm::BitstreamEntry Entry = MaybeEntry.get();
4965 
4966  if (Entry.Kind == llvm::BitstreamEntry::EndBlock)
4967  return std::string();
4968 
4969  if (Entry.Kind != llvm::BitstreamEntry::Record) {
4970  Diags.Report(diag::err_fe_pch_malformed_block) << ASTFileName;
4971  return std::string();
4972  }
4973 
4974  Record.clear();
4975  StringRef Blob;
4976  Expected<unsigned> MaybeRecord = Stream.readRecord(Entry.ID, Record, &Blob);
4977  if (!MaybeRecord) {
4978  // FIXME this drops the errors on the floor.
4979  consumeError(MaybeRecord.takeError());
4980  return std::string();
4981  }
4982  if (ORIGINAL_FILE == MaybeRecord.get())
4983  return Blob.str();
4984  }
4985 }
4986 
4987 namespace {
4988 
4989  class SimplePCHValidator : public ASTReaderListener {
4990  const LangOptions &ExistingLangOpts;
4991  const TargetOptions &ExistingTargetOpts;
4992  const PreprocessorOptions &ExistingPPOpts;
4993  std::string ExistingModuleCachePath;
4994  FileManager &FileMgr;
4995 
4996  public:
4997  SimplePCHValidator(const LangOptions &ExistingLangOpts,
4998  const TargetOptions &ExistingTargetOpts,
4999  const PreprocessorOptions &ExistingPPOpts,
5000  StringRef ExistingModuleCachePath,
5001  FileManager &FileMgr)
5002  : ExistingLangOpts(ExistingLangOpts),
5003  ExistingTargetOpts(ExistingTargetOpts),
5004  ExistingPPOpts(ExistingPPOpts),
5005  ExistingModuleCachePath(ExistingModuleCachePath),
5006  FileMgr(FileMgr) {}
5007 
5008  bool ReadLanguageOptions(const LangOptions &LangOpts, bool Complain,
5009  bool AllowCompatibleDifferences) override {
5010  return checkLanguageOptions(ExistingLangOpts, LangOpts, nullptr,
5011  AllowCompatibleDifferences);
5012  }
5013 
5014  bool ReadTargetOptions(const TargetOptions &TargetOpts, bool Complain,
5015  bool AllowCompatibleDifferences) override {
5016  return checkTargetOptions(ExistingTargetOpts, TargetOpts, nullptr,
5017  AllowCompatibleDifferences);
5018  }
5019 
5020  bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
5021  StringRef SpecificModuleCachePath,
5022  bool Complain) override {
5023  return checkHeaderSearchOptions(HSOpts, SpecificModuleCachePath,
5024  ExistingModuleCachePath,
5025  nullptr, ExistingLangOpts);
5026  }
5027 
5028  bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts,
5029  bool Complain,
5030  std::string &SuggestedPredefines) override {
5031  return checkPreprocessorOptions(ExistingPPOpts, PPOpts, nullptr, FileMgr,
5032  SuggestedPredefines, ExistingLangOpts);
5033  }
5034  };
5035 
5036 } // namespace
5037 
5039  StringRef Filename, FileManager &FileMgr,
5040  const PCHContainerReader &PCHContainerRdr,
5041  bool FindModuleFileExtensions,
5042  ASTReaderListener &Listener, bool ValidateDiagnosticOptions) {
5043  // Open the AST file.
5044  // FIXME: This allows use of the VFS; we do not allow use of the
5045  // VFS when actually loading a module.
5046  auto Buffer = FileMgr.getBufferForFile(Filename);
5047  if (!Buffer) {
5048  return true;
5049  }
5050 
5051  // Initialize the stream
5052  StringRef Bytes = PCHContainerRdr.ExtractPCH(**Buffer);
5053  BitstreamCursor Stream(Bytes);
5054 
5055  // Sniff for the signature.
5056  if (llvm::Error Err = doesntStartWithASTFileMagic(Stream)) {
5057  consumeError(std::move(Err)); // FIXME this drops errors on the floor.
5058  return true;
5059  }
5060 
5061  // Scan for the CONTROL_BLOCK_ID block.
5062  if (SkipCursorToBlock(Stream, CONTROL_BLOCK_ID))
5063  return true;
5064 
5065  bool NeedsInputFiles = Listener.needsInputFileVisitation();
5066  bool NeedsSystemInputFiles = Listener.needsSystemInputFileVisitation();
5067  bool NeedsImports = Listener.needsImportVisitation();
5068  BitstreamCursor InputFilesCursor;
5069 
5070  RecordData Record;
5071  std::string ModuleDir;
5072  bool DoneWithControlBlock = false;
5073  while (!DoneWithControlBlock) {
5074  Expected<llvm::BitstreamEntry> MaybeEntry = Stream.advance();
5075  if (!MaybeEntry) {
5076  // FIXME this drops the error on the floor.
5077  consumeError(MaybeEntry.takeError());
5078  return true;
5079  }
5080  llvm::BitstreamEntry Entry = MaybeEntry.get();
5081 
5082  switch (Entry.Kind) {
5083  case llvm::BitstreamEntry::SubBlock: {
5084  switch (Entry.ID) {
5085  case OPTIONS_BLOCK_ID: {
5086  std::string IgnoredSuggestedPredefines;
5087  if (ReadOptionsBlock(Stream, ARR_ConfigurationMismatch | ARR_OutOfDate,
5088  /*AllowCompatibleConfigurationMismatch*/ false,
5089  Listener, IgnoredSuggestedPredefines) != Success)
5090  return true;
5091  break;
5092  }
5093 
5094  case INPUT_FILES_BLOCK_ID:
5095  InputFilesCursor = Stream;
5096  if (llvm::Error Err = Stream.SkipBlock()) {
5097  // FIXME this drops the error on the floor.
5098  consumeError(std::move(Err));
5099  return true;
5100  }
5101  if (NeedsInputFiles &&
5102  ReadBlockAbbrevs(InputFilesCursor, INPUT_FILES_BLOCK_ID))
5103  return true;
5104  break;
5105 
5106  default:
5107  if (llvm::Error Err = Stream.SkipBlock()) {
5108  // FIXME this drops the error on the floor.
5109  consumeError(std::move(Err));
5110  return true;
5111  }
5112  break;
5113  }
5114 
5115  continue;
5116  }
5117 
5118  case llvm::BitstreamEntry::EndBlock:
5119  DoneWithControlBlock = true;
5120  break;
5121 
5123  return true;
5124 
5125  case llvm::BitstreamEntry::Record:
5126  break;
5127  }
5128 
5129  if (DoneWithControlBlock) break;
5130 
5131  Record.clear();
5132  StringRef Blob;
5133  Expected<unsigned> MaybeRecCode =
5134  Stream.readRecord(Entry.ID, Record, &Blob);
5135  if (!MaybeRecCode) {
5136  // FIXME this drops the error.
5137  return Failure;
5138  }
5139  switch ((ControlRecordTypes)MaybeRecCode.get()) {
5140  case METADATA:
5141  if (Record[0] != VERSION_MAJOR)
5142  return true;
5143  if (Listener.ReadFullVersionInformation(Blob))
5144  return true;
5145  break;
5146  case MODULE_NAME:
5147  Listener.ReadModuleName(Blob);
5148  break;
5149  case MODULE_DIRECTORY:
5150  ModuleDir = Blob;
5151  break;
5152  case MODULE_MAP_FILE: {
5153  unsigned Idx = 0;
5154  auto Path = ReadString(Record, Idx);
5155  ResolveImportedPath(Path, ModuleDir);
5156  Listener.ReadModuleMapFile(Path);
5157  break;
5158  }
5159  case INPUT_FILE_OFFSETS: {
5160  if (!NeedsInputFiles)
5161  break;
5162 
5163  unsigned NumInputFiles = Record[0];
5164  unsigned NumUserFiles = Record[1];
5165  const llvm::support::unaligned_uint64_t *InputFileOffs =
5166  (const llvm::support::unaligned_uint64_t *)Blob.data();
5167  for (unsigned I = 0; I != NumInputFiles; ++I) {
5168  // Go find this input file.
5169  bool isSystemFile = I >= NumUserFiles;
5170 
5171  if (isSystemFile && !NeedsSystemInputFiles)
5172  break; // the rest are system input files
5173 
5174  BitstreamCursor &Cursor = InputFilesCursor;
5175  SavedStreamPosition SavedPosition(Cursor);
5176  if (llvm::Error Err = Cursor.JumpToBit(InputFileOffs[I])) {
5177  // FIXME this drops errors on the floor.
5178  consumeError(std::move(Err));
5179  }
5180 
5181  Expected<unsigned> MaybeCode = Cursor.ReadCode();
5182  if (!MaybeCode) {
5183  // FIXME this drops errors on the floor.
5184  consumeError(MaybeCode.takeError());
5185  }
5186  unsigned Code = MaybeCode.get();
5187 
5188  RecordData Record;
5189  StringRef Blob;
5190  bool shouldContinue = false;
5191  Expected<unsigned> MaybeRecordType =
5192  Cursor.readRecord(Code, Record, &Blob);
5193  if (!MaybeRecordType) {
5194  // FIXME this drops errors on the floor.
5195  consumeError(MaybeRecordType.takeError());
5196  }
5197  switch ((InputFileRecordTypes)MaybeRecordType.get()) {
5198  case INPUT_FILE:
5199  bool Overridden = static_cast<bool>(Record[3]);
5200  std::string Filename = Blob;
5201  ResolveImportedPath(Filename, ModuleDir);
5202  shouldContinue = Listener.visitInputFile(
5203  Filename, isSystemFile, Overridden, /*IsExplicitModule*/false);
5204  break;
5205  }
5206  if (!shouldContinue)
5207  break;
5208  }
5209  break;
5210  }
5211 
5212  case IMPORTS: {
5213  if (!NeedsImports)
5214  break;
5215 
5216  unsigned Idx = 0, N = Record.size();
5217  while (Idx < N) {
5218  // Read information about the AST file.
5219  Idx += 1+1+1+1+5; // Kind, ImportLoc, Size, ModTime, Signature
5220  std::string ModuleName = ReadString(Record, Idx);
5221  std::string Filename = ReadString(Record, Idx);
5222  ResolveImportedPath(Filename, ModuleDir);
5223  Listener.visitImport(ModuleName, Filename);
5224  }
5225  break;
5226  }
5227 
5228  default:
5229  // No other validation to perform.
5230  break;
5231  }
5232  }
5233 
5234  // Look for module file extension blocks, if requested.
5235  if (FindModuleFileExtensions) {
5236  BitstreamCursor SavedStream = Stream;
5237  while (!SkipCursorToBlock(Stream, EXTENSION_BLOCK_ID)) {
5238  bool DoneWithExtensionBlock = false;
5239  while (!DoneWithExtensionBlock) {
5240  Expected<llvm::BitstreamEntry> MaybeEntry = Stream.advance();
5241  if (!MaybeEntry) {
5242  // FIXME this drops the error.
5243  return true;
5244  }
5245  llvm::BitstreamEntry Entry = MaybeEntry.get();
5246 
5247  switch (Entry.Kind) {
5248  case llvm::BitstreamEntry::SubBlock:
5249  if (llvm::Error Err = Stream.SkipBlock()) {
5250  // FIXME this drops the error on the floor.
5251  consumeError(std::move(Err));
5252  return true;
5253  }
5254  continue;
5255 
5256  case llvm::BitstreamEntry::EndBlock:
5257  DoneWithExtensionBlock = true;
5258  continue;
5259 
5261  return true;
5262 
5263  case llvm::BitstreamEntry::Record:
5264  break;
5265  }
5266 
5267  Record.clear();
5268  StringRef Blob;
5269  Expected<unsigned> MaybeRecCode =
5270  Stream.readRecord(Entry.ID, Record, &Blob);
5271  if (!MaybeRecCode) {
5272  // FIXME this drops the error.
5273  return true;
5274  }
5275  switch (MaybeRecCode.get()) {
5276  case EXTENSION_METADATA: {
5277  ModuleFileExtensionMetadata Metadata;
5278  if (parseModuleFileExtensionMetadata(Record, Blob, Metadata))
5279  return true;
5280 
5281  Listener.readModuleFileExtension(Metadata);
5282  break;
5283  }
5284  }
5285  }
5286  }
5287  Stream = SavedStream;
5288  }
5289 
5290  // Scan for the UNHASHED_CONTROL_BLOCK_ID block.
5291  if (readUnhashedControlBlockImpl(
5292  nullptr, Bytes, ARR_ConfigurationMismatch | ARR_OutOfDate,
5293  /*AllowCompatibleConfigurationMismatch*/ false, &Listener,
5294  ValidateDiagnosticOptions) != Success)
5295  return true;
5296 
5297  return false;
5298 }
5299 
5301  const PCHContainerReader &PCHContainerRdr,
5302  const LangOptions &LangOpts,
5303  const TargetOptions &TargetOpts,
5304  const PreprocessorOptions &PPOpts,
5305  StringRef ExistingModuleCachePath) {
5306  SimplePCHValidator validator(LangOpts, TargetOpts, PPOpts,
5307  ExistingModuleCachePath, FileMgr);
5308  return !readASTFileControlBlock(Filename, FileMgr, PCHContainerRdr,
5309  /*FindModuleFileExtensions=*/false,
5310  validator,
5311  /*ValidateDiagnosticOptions=*/true);
5312 }
5313 
5315 ASTReader::ReadSubmoduleBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
5316  // Enter the submodule block.
5317  if (llvm::Error Err = F.Stream.EnterSubBlock(SUBMODULE_BLOCK_ID)) {
5318  Error(std::move(Err));
5319  return Failure;
5320  }
5321 
5322  ModuleMap &ModMap = PP.getHeaderSearchInfo().getModuleMap();
5323  bool First = true;
5324  Module *CurrentModule = nullptr;
5325  RecordData Record;
5326  while (true) {
5327  Expected<llvm::BitstreamEntry> MaybeEntry =
5328  F.Stream.advanceSkippingSubblocks();
5329  if (!MaybeEntry) {
5330  Error(MaybeEntry.takeError());
5331  return Failure;
5332  }
5333  llvm::BitstreamEntry Entry = MaybeEntry.get();
5334 
5335  switch (Entry.Kind) {
5336  case llvm::BitstreamEntry::SubBlock: // Handled for us already.
5338  Error("malformed block record in AST file");
5339  return Failure;
5340  case llvm::BitstreamEntry::EndBlock:
5341  return Success;
5342  case llvm::BitstreamEntry::Record:
5343  // The interesting case.
5344  break;
5345  }
5346 
5347  // Read a record.
5348  StringRef Blob;
5349  Record.clear();
5350  Expected<unsigned> MaybeKind = F.Stream.readRecord(Entry.ID, Record, &Blob);
5351  if (!MaybeKind) {
5352  Error(MaybeKind.takeError());
5353  return Failure;
5354  }
5355  unsigned Kind = MaybeKind.get();
5356 
5357  if ((Kind == SUBMODULE_METADATA) != First) {
5358  Error("submodule metadata record should be at beginning of block");
5359  return Failure;
5360  }
5361  First = false;
5362 
5363  // Submodule information is only valid if we have a current module.
5364  // FIXME: Should we error on these cases?
5365  if (!CurrentModule && Kind != SUBMODULE_METADATA &&
5366  Kind != SUBMODULE_DEFINITION)
5367  continue;
5368 
5369  switch (Kind) {
5370  default: // Default behavior: ignore.
5371  break;
5372 
5373  case SUBMODULE_DEFINITION: {
5374  if (Record.size() < 12) {
5375  Error("malformed module definition");
5376  return Failure;
5377  }
5378 
5379  StringRef Name = Blob;
5380  unsigned Idx = 0;
5381  SubmoduleID GlobalID = getGlobalSubmoduleID(F, Record[Idx++]);
5382  SubmoduleID Parent = getGlobalSubmoduleID(F, Record[Idx++]);
5383  Module::ModuleKind Kind = (Module::ModuleKind)Record[Idx++];
5384  bool IsFramework = Record[Idx++];
5385  bool IsExplicit = Record[Idx++];
5386  bool IsSystem = Record[Idx++];
5387  bool IsExternC = Record[Idx++];
5388  bool InferSubmodules = Record[Idx++];
5389  bool InferExplicitSubmodules = Record[Idx++];
5390  bool InferExportWildcard = Record[Idx++];
5391  bool ConfigMacrosExhaustive = Record[Idx++];
5392  bool ModuleMapIsPrivate = Record[Idx++];
5393 
5394  Module *ParentModule = nullptr;
5395  if (Parent)
5396  ParentModule = getSubmodule(Parent);
5397 
5398  // Retrieve this (sub)module from the module map, creating it if
5399  // necessary.
5400  CurrentModule =
5401  ModMap.findOrCreateModule(Name, ParentModule, IsFramework, IsExplicit)
5402  .first;
5403 
5404  // FIXME: set the definition loc for CurrentModule, or call
5405  // ModMap.setInferredModuleAllowedBy()
5406 
5407  SubmoduleID GlobalIndex = GlobalID - NUM_PREDEF_SUBMODULE_IDS;
5408  if (GlobalIndex >= SubmodulesLoaded.size() ||
5409  SubmodulesLoaded[GlobalIndex]) {
5410  Error("too many submodules");
5411  return Failure;
5412  }
5413 
5414  if (!ParentModule) {
5415  if (const FileEntry *CurFile = CurrentModule->getASTFile()) {
5416  // Don't emit module relocation error if we have -fno-validate-pch
5417  if (!PP.getPreprocessorOpts().DisablePCHValidation &&
5418  CurFile != F.File) {
5419  if (!Diags.isDiagnosticInFlight()) {
5420  Diag(diag::err_module_file_conflict)
5421  << CurrentModule->getTopLevelModuleName()
5422  << CurFile->getName()
5423  << F.File->getName();
5424  }
5425  return Failure;
5426  }
5427  }
5428 
5429  CurrentModule->setASTFile(F.File);
5430  CurrentModule->PresumedModuleMapFile = F.ModuleMapPath;
5431  }
5432 
5433  CurrentModule->Kind = Kind;
5434  CurrentModule->Signature = F.Signature;
5435  CurrentModule->IsFromModuleFile = true;
5436  CurrentModule->IsSystem = IsSystem || CurrentModule->IsSystem;
5437  CurrentModule->IsExternC = IsExternC;
5438  CurrentModule->InferSubmodules = InferSubmodules;
5439  CurrentModule->InferExplicitSubmodules = InferExplicitSubmodules;
5440  CurrentModule->InferExportWildcard = InferExportWildcard;
5441  CurrentModule->ConfigMacrosExhaustive = ConfigMacrosExhaustive;
5442  CurrentModule->ModuleMapIsPrivate = ModuleMapIsPrivate;
5443  if (DeserializationListener)
5444  DeserializationListener->ModuleRead(GlobalID, CurrentModule);
5445 
5446  SubmodulesLoaded[GlobalIndex] = CurrentModule;
5447 
5448  // Clear out data that will be replaced by what is in the module file.
5449  CurrentModule->LinkLibraries.clear();
5450  CurrentModule->ConfigMacros.clear();
5451  CurrentModule->UnresolvedConflicts.clear();
5452  CurrentModule->Conflicts.clear();
5453 
5454  // The module is available unless it's missing a requirement; relevant
5455  // requirements will be (re-)added by SUBMODULE_REQUIRES records.
5456  // Missing headers that were present when the module was built do not
5457  // make it unavailable -- if we got this far, this must be an explicitly
5458  // imported module file.
5459  CurrentModule->Requirements.clear();
5460  CurrentModule->MissingHeaders.clear();
5461  CurrentModule->IsMissingRequirement =
5462  ParentModule && ParentModule->IsMissingRequirement;
5463  CurrentModule->IsAvailable = !CurrentModule->IsMissingRequirement;
5464  break;
5465  }
5466 
5468  std::string Filename = Blob;
5469  ResolveImportedPath(F, Filename);
5470  if (auto Umbrella = PP.getFileManager().getFile(Filename)) {
5471  if (!CurrentModule->getUmbrellaHeader())
5472  ModMap.setUmbrellaHeader(CurrentModule, *Umbrella, Blob);
5473  else if (CurrentModule->getUmbrellaHeader().Entry != *Umbrella) {
5474  if ((ClientLoadCapabilities & ARR_OutOfDate) == 0)
5475  Error("mismatched umbrella headers in submodule");
5476  return OutOfDate;
5477  }
5478  }
5479  break;
5480  }
5481 
5482  case SUBMODULE_HEADER:
5485  // We lazily associate headers with their modules via the HeaderInfo table.
5486  // FIXME: Re-evaluate this section; maybe only store InputFile IDs instead
5487  // of complete filenames or remove it entirely.
5488  break;
5489 
5492  // FIXME: Textual headers are not marked in the HeaderInfo table. Load
5493  // them here.
5494  break;
5495 
5496  case SUBMODULE_TOPHEADER:
5497  CurrentModule->addTopHeaderFilename(Blob);
5498  break;
5499 
5500  case SUBMODULE_UMBRELLA_DIR: {
5501  std::string Dirname = Blob;
5502  ResolveImportedPath(F, Dirname);
5503  if (auto Umbrella = PP.getFileManager().getDirectory(Dirname)) {
5504  if (!CurrentModule->getUmbrellaDir())
5505  ModMap.setUmbrellaDir(CurrentModule, *Umbrella, Blob);
5506  else if (CurrentModule->getUmbrellaDir().Entry != *Umbrella) {
5507  if ((ClientLoadCapabilities & ARR_OutOfDate) == 0)
5508  Error("mismatched umbrella directories in submodule");
5509  return OutOfDate;
5510  }
5511  }
5512  break;
5513  }
5514 
5515  case SUBMODULE_METADATA: {
5516  F.BaseSubmoduleID = getTotalNumSubmodules();
5517  F.LocalNumSubmodules = Record[0];
5518  unsigned LocalBaseSubmoduleID = Record[1];
5519  if (F.LocalNumSubmodules > 0) {
5520  // Introduce the global -> local mapping for submodules within this
5521  // module.
5522  GlobalSubmoduleMap.insert(std::make_pair(getTotalNumSubmodules()+1,&F));
5523 
5524  // Introduce the local -> global mapping for submodules within this
5525  // module.
5527  std::make_pair(LocalBaseSubmoduleID,
5528  F.BaseSubmoduleID - LocalBaseSubmoduleID));
5529 
5530  SubmodulesLoaded.resize(SubmodulesLoaded.size() + F.LocalNumSubmodules);
5531  }
5532  break;
5533  }
5534 
5535  case SUBMODULE_IMPORTS:
5536  for (unsigned Idx = 0; Idx != Record.size(); ++Idx) {
5537  UnresolvedModuleRef Unresolved;
5538  Unresolved.File = &F;
5539  Unresolved.Mod = CurrentModule;
5540  Unresolved.ID = Record[Idx];
5541  Unresolved.Kind = UnresolvedModuleRef::Import;
5542  Unresolved.IsWildcard = false;
5543  UnresolvedModuleRefs.push_back(Unresolved);
5544  }
5545  break;
5546 
5547  case SUBMODULE_EXPORTS:
5548  for (unsigned Idx = 0; Idx + 1 < Record.size(); Idx += 2) {
5549  UnresolvedModuleRef Unresolved;
5550  Unresolved.File = &F;
5551  Unresolved.Mod = CurrentModule;
5552  Unresolved.ID = Record[Idx];
5553  Unresolved.Kind = UnresolvedModuleRef::Export;
5554  Unresolved.IsWildcard = Record[Idx + 1];
5555  UnresolvedModuleRefs.push_back(Unresolved);
5556  }
5557 
5558  // Once we've loaded the set of exports, there's no reason to keep
5559  // the parsed, unresolved exports around.
5560  CurrentModule->UnresolvedExports.clear();
5561  break;
5562 
5563  case SUBMODULE_REQUIRES:
5564  CurrentModule->addRequirement(Blob, Record[0], PP.getLangOpts(),
5565  PP.getTargetInfo());
5566  break;
5567 
5569  ModMap.resolveLinkAsDependencies(CurrentModule);
5570  CurrentModule->LinkLibraries.push_back(
5571  Module::LinkLibrary(Blob, Record[0]));
5572  break;
5573 
5575  CurrentModule->ConfigMacros.push_back(Blob.str());
5576  break;
5577 
5578  case SUBMODULE_CONFLICT: {
5579  UnresolvedModuleRef Unresolved;
5580  Unresolved.File = &F;
5581  Unresolved.Mod = CurrentModule;
5582  Unresolved.ID = Record[0];
5583  Unresolved.Kind = UnresolvedModuleRef::Conflict;
5584  Unresolved.IsWildcard = false;
5585  Unresolved.String = Blob;
5586  UnresolvedModuleRefs.push_back(Unresolved);
5587  break;
5588  }
5589 
5590  case SUBMODULE_INITIALIZERS: {
5591  if (!ContextObj)
5592  break;
5594  for (auto &ID : Record)
5595  Inits.push_back(getGlobalDeclID(F, ID));
5596  ContextObj->addLazyModuleInitializers(CurrentModule, Inits);
5597  break;
5598  }
5599 
5600  case SUBMODULE_EXPORT_AS:
5601  CurrentModule->ExportAsModule = Blob.str();
5602  ModMap.addLinkAsDependency(CurrentModule);
5603  break;
5604  }
5605  }
5606 }
5607 
5608 /// Parse the record that corresponds to a LangOptions data
5609 /// structure.
5610 ///
5611 /// This routine parses the language options from the AST file and then gives
5612 /// them to the AST listener if one is set.
5613 ///
5614 /// \returns true if the listener deems the file unacceptable, false otherwise.
5615 bool ASTReader::ParseLanguageOptions(const RecordData &Record,
5616  bool Complain,
5617  ASTReaderListener &Listener,
5618  bool AllowCompatibleDifferences) {
5619  LangOptions LangOpts;
5620  unsigned Idx = 0;
5621 #define LANGOPT(Name, Bits, Default, Description) \
5622  LangOpts.Name = Record[Idx++];
5623 #define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
5624  LangOpts.set##Name(static_cast<LangOptions::Type>(Record[Idx++]));
5625 #include "clang/Basic/LangOptions.def"
5626 #define SANITIZER(NAME, ID) \
5627  LangOpts.Sanitize.set(SanitizerKind::ID, Record[Idx++]);
5628 #include "clang/Basic/Sanitizers.def"
5629 
5630  for (unsigned N = Record[Idx++]; N; --N)
5631  LangOpts.ModuleFeatures.push_back(ReadString(Record, Idx));
5632 
5633  ObjCRuntime::Kind runtimeKind = (ObjCRuntime::Kind) Record[Idx++];
5634  VersionTuple runtimeVersion = ReadVersionTuple(Record, Idx);
5635  LangOpts.ObjCRuntime = ObjCRuntime(runtimeKind, runtimeVersion);
5636 
5637  LangOpts.CurrentModule = ReadString(Record, Idx);
5638 
5639  // Comment options.
5640  for (unsigned N = Record[Idx++]; N; --N) {
5641  LangOpts.CommentOpts.BlockCommandNames.push_back(
5642  ReadString(Record, Idx));
5643  }
5644  LangOpts.CommentOpts.ParseAllComments = Record[Idx++];
5645 
5646  // OpenMP offloading options.
5647  for (unsigned N = Record[Idx++]; N; --N) {
5648  LangOpts.OMPTargetTriples.push_back(llvm::Triple(ReadString(Record, Idx)));
5649  }
5650 
5651  LangOpts.OMPHostIRFile = ReadString(Record, Idx);
5652 
5653  return Listener.ReadLanguageOptions(LangOpts, Complain,
5654  AllowCompatibleDifferences);
5655 }
5656 
5657 bool ASTReader::ParseTargetOptions(const RecordData &Record, bool Complain,
5658  ASTReaderListener &Listener,
5659  bool AllowCompatibleDifferences) {
5660  unsigned Idx = 0;
5661  TargetOptions TargetOpts;
5662  TargetOpts.Triple = ReadString(Record, Idx);
5663  TargetOpts.CPU = ReadString(Record, Idx);
5664  TargetOpts.ABI = ReadString(Record, Idx);
5665  for (unsigned N = Record[Idx++]; N; --N) {
5666  TargetOpts.FeaturesAsWritten.push_back(ReadString(Record, Idx));
5667  }
5668  for (unsigned N = Record[Idx++]; N; --N) {
5669  TargetOpts.Features.push_back(ReadString(Record, Idx));
5670  }
5671 
5672  return Listener.ReadTargetOptions(TargetOpts, Complain,
5673  AllowCompatibleDifferences);
5674 }
5675 
5676 bool ASTReader::ParseDiagnosticOptions(const RecordData &Record, bool Complain,
5677  ASTReaderListener &Listener) {
5679  unsigned Idx = 0;
5680 #define DIAGOPT(Name, Bits, Default) DiagOpts->Name = Record[Idx++];
5681 #define ENUM_DIAGOPT(Name, Type, Bits, Default) \
5682  DiagOpts->set##Name(static_cast<Type>(Record[Idx++]));
5683 #include "clang/Basic/DiagnosticOptions.def"
5684 
5685  for (unsigned N = Record[Idx++]; N; --N)
5686  DiagOpts->Warnings.push_back(ReadString(Record, Idx));
5687  for (unsigned N = Record[Idx++]; N; --N)
5688  DiagOpts->Remarks.push_back(ReadString(Record, Idx));
5689 
5690  return Listener.ReadDiagnosticOptions(DiagOpts, Complain);
5691 }
5692 
5693 bool ASTReader::ParseFileSystemOptions(const RecordData &Record, bool Complain,
5694  ASTReaderListener &Listener) {
5695  FileSystemOptions FSOpts;
5696  unsigned Idx = 0;
5697  FSOpts.WorkingDir = ReadString(Record, Idx);
5698  return Listener.ReadFileSystemOptions(FSOpts, Complain);
5699 }
5700 
5701 bool ASTReader::ParseHeaderSearchOptions(const RecordData &Record,
5702  bool Complain,
5703  ASTReaderListener &Listener) {
5704  HeaderSearchOptions HSOpts;
5705  unsigned Idx = 0;
5706  HSOpts.Sysroot = ReadString(Record, Idx);
5707 
5708  // Include entries.
5709  for (unsigned N = Record[Idx++]; N; --N) {
5710  std::string Path = ReadString(Record, Idx);
5712  = static_cast<frontend::IncludeDirGroup>(Record[Idx++]);
5713  bool IsFramework = Record[Idx++];
5714  bool IgnoreSysRoot = Record[Idx++];
5715  HSOpts.UserEntries.emplace_back(std::move(Path), Group, IsFramework,
5716  IgnoreSysRoot);
5717  }
5718 
5719  // System header prefixes.
5720  for (unsigned N = Record[Idx++]; N; --N) {
5721  std::string Prefix = ReadString(Record, Idx);
5722  bool IsSystemHeader = Record[Idx++];
5723  HSOpts.SystemHeaderPrefixes.emplace_back(std::move(Prefix), IsSystemHeader);
5724  }
5725 
5726  HSOpts.ResourceDir = ReadString(Record, Idx);
5727  HSOpts.ModuleCachePath = ReadString(Record, Idx);
5728  HSOpts.ModuleUserBuildPath = ReadString(Record, Idx);
5729  HSOpts.DisableModuleHash = Record[Idx++];
5730  HSOpts.ImplicitModuleMaps = Record[Idx++];
5731  HSOpts.ModuleMapFileHomeIsCwd = Record[Idx++];
5732  HSOpts.UseBuiltinIncludes = Record[Idx++];
5733  HSOpts.UseStandardSystemIncludes = Record[Idx++];
5734  HSOpts.UseStandardCXXIncludes = Record[Idx++];
5735  HSOpts.UseLibcxx = Record[Idx++];
5736  std::string SpecificModuleCachePath = ReadString(Record, Idx);
5737 
5738  return Listener.ReadHeaderSearchOptions(HSOpts, SpecificModuleCachePath,
5739  Complain);
5740 }
5741 
5742 bool ASTReader::ParsePreprocessorOptions(const RecordData &Record,
5743  bool Complain,
5744  ASTReaderListener &Listener,
5745  std::string &SuggestedPredefines) {
5746  PreprocessorOptions PPOpts;
5747  unsigned Idx = 0;
5748 
5749  // Macro definitions/undefs
5750  for (unsigned N = Record[Idx++]; N; --N) {
5751  std::string Macro = ReadString(Record, Idx);
5752  bool IsUndef = Record[Idx++];
5753  PPOpts.Macros.push_back(std::make_pair(Macro, IsUndef));
5754  }
5755 
5756  // Includes
5757  for (unsigned N = Record[Idx++]; N; --N) {
5758  PPOpts.Includes.push_back(ReadString(Record, Idx));
5759  }
5760 
5761  // Macro Includes
5762  for (unsigned N = Record[Idx++]; N; --N) {
5763  PPOpts.MacroIncludes.push_back(ReadString(Record, Idx));
5764  }
5765 
5766  PPOpts.UsePredefines = Record[Idx++];
5767  PPOpts.DetailedRecord = Record[Idx++];
5768  PPOpts.ImplicitPCHInclude = ReadString(Record, Idx);
5769  PPOpts.ObjCXXARCStandardLibrary =
5770  static_cast<ObjCXXARCStandardLibraryKind>(Record[Idx++]);
5771  SuggestedPredefines.clear();
5772  return Listener.ReadPreprocessorOptions(PPOpts, Complain,
5773  SuggestedPredefines);
5774 }
5775 
5776 std::pair<ModuleFile *, unsigned>
5777 ASTReader::getModulePreprocessedEntity(unsigned GlobalIndex) {
5779  I = GlobalPreprocessedEntityMap.find(GlobalIndex);
5780  assert(I != GlobalPreprocessedEntityMap.end() &&
5781  "Corrupted global preprocessed entity map");
5782  ModuleFile *M = I->second;
5783  unsigned LocalIndex = GlobalIndex - M->BasePreprocessedEntityID;
5784  return std::make_pair(M, LocalIndex);
5785 }
5786 
5787 llvm::iterator_range<PreprocessingRecord::iterator>
5788 ASTReader::getModulePreprocessedEntities(ModuleFile &Mod) const {
5789  if (PreprocessingRecord *PPRec = PP.getPreprocessingRecord())
5790  return PPRec->getIteratorsForLoadedRange(Mod.BasePreprocessedEntityID,
5792 
5793  return llvm::make_range(PreprocessingRecord::iterator(),
5795 }
5796 
5797 llvm::iterator_range<ASTReader::ModuleDeclIterator>
5799  return llvm::make_range(
5800  ModuleDeclIterator(this, &Mod, Mod.FileSortedDecls),
5801  ModuleDeclIterator(this, &Mod,
5802  Mod.FileSortedDecls + Mod.NumFileSortedDecls));
5803 }
5804 
5806  auto I = GlobalSkippedRangeMap.find(GlobalIndex);
5807  assert(I != GlobalSkippedRangeMap.end() &&
5808  "Corrupted global skipped range map");
5809  ModuleFile *M = I->second;
5810  unsigned LocalIndex = GlobalIndex - M->BasePreprocessedSkippedRangeID;
5811  assert(LocalIndex < M->NumPreprocessedSkippedRanges);
5812  PPSkippedRange RawRange = M->PreprocessedSkippedRangeOffsets[LocalIndex];
5813  SourceRange Range(TranslateSourceLocation(*M, RawRange.getBegin()),
5814  TranslateSourceLocation(*M, RawRange.getEnd()));
5815  assert(Range.isValid());
5816  return Range;
5817 }
5818 
5820  PreprocessedEntityID PPID = Index+1;
5821  std::pair<ModuleFile *, unsigned> PPInfo = getModulePreprocessedEntity(Index);
5822  ModuleFile &M = *PPInfo.first;
5823  unsigned LocalIndex = PPInfo.second;
5824  const PPEntityOffset &PPOffs = M.PreprocessedEntityOffsets[LocalIndex];
5825 
5826  if (!PP.getPreprocessingRecord()) {
5827  Error("no preprocessing record");
5828  return nullptr;
5829  }
5830 
5832  if (llvm::Error Err =
5833  M.PreprocessorDetailCursor.JumpToBit(PPOffs.BitOffset)) {
5834  Error(std::move(Err));
5835  return nullptr;
5836  }
5837 
5838  Expected<llvm::BitstreamEntry> MaybeEntry =
5839  M.PreprocessorDetailCursor.advance(BitstreamCursor::AF_DontPopBlockAtEnd);
5840  if (!MaybeEntry) {
5841  Error(MaybeEntry.takeError());
5842  return nullptr;
5843  }
5844  llvm::BitstreamEntry Entry = MaybeEntry.get();
5845 
5846  if (Entry.Kind != llvm::BitstreamEntry::Record)
5847  return nullptr;
5848 
5849  // Read the record.
5850  SourceRange Range(TranslateSourceLocation(M, PPOffs.getBegin()),
5851  TranslateSourceLocation(M, PPOffs.getEnd()));
5852  PreprocessingRecord &PPRec = *PP.getPreprocessingRecord();
5853  StringRef Blob;
5854  RecordData Record;
5855  Expected<unsigned> MaybeRecType =
5856  M.PreprocessorDetailCursor.readRecord(Entry.ID, Record, &Blob);
5857  if (!MaybeRecType) {
5858  Error(MaybeRecType.takeError());
5859  return nullptr;
5860  }
5861  switch ((PreprocessorDetailRecordTypes)MaybeRecType.get()) {
5862  case PPD_MACRO_EXPANSION: {
5863  bool isBuiltin = Record[0];
5864  IdentifierInfo *Name = nullptr;
5865  MacroDefinitionRecord *Def = nullptr;
5866  if (isBuiltin)
5867  Name = getLocalIdentifier(M, Record[1]);
5868  else {
5869  PreprocessedEntityID GlobalID =
5870  getGlobalPreprocessedEntityID(M, Record[1]);
5871  Def = cast<MacroDefinitionRecord>(
5872  PPRec.getLoadedPreprocessedEntity(GlobalID - 1));
5873  }
5874 
5875  MacroExpansion *ME;
5876  if (isBuiltin)
5877  ME = new (PPRec) MacroExpansion(Name, Range);
5878  else
5879  ME = new (PPRec) MacroExpansion(Def, Range);
5880 
5881  return ME;
5882  }
5883 
5884  case PPD_MACRO_DEFINITION: {
5885  // Decode the identifier info and then check again; if the macro is
5886  // still defined and associated with the identifier,
5887  IdentifierInfo *II = getLocalIdentifier(M, Record[0]);
5888  MacroDefinitionRecord *MD = new (PPRec) MacroDefinitionRecord(II, Range);
5889 
5890  if (DeserializationListener)
5891  DeserializationListener->MacroDefinitionRead(PPID, MD);
5892 
5893  return MD;
5894  }
5895 
5896  case PPD_INCLUSION_DIRECTIVE: {
5897  const char *FullFileNameStart = Blob.data() + Record[0];
5898  StringRef FullFileName(FullFileNameStart, Blob.size() - Record[0]);
5899  const FileEntry *File = nullptr;
5900  if (!FullFileName.empty())
5901  if (auto FE = PP.getFileManager().getFile(FullFileName))
5902  File = *FE;
5903 
5904  // FIXME: Stable encoding
5906  = static_cast<InclusionDirective::InclusionKind>(Record[2]);
5908  = new (PPRec) InclusionDirective(PPRec, Kind,
5909  StringRef(Blob.data(), Record[0]),
5910  Record[1], Record[3],
5911  File,
5912  Range);
5913  return ID;
5914  }
5915  }
5916 
5917  llvm_unreachable("Invalid PreprocessorDetailRecordTypes");
5918 }
5919 
5920 /// Find the next module that contains entities and return the ID
5921 /// of the first entry.
5922 ///
5923 /// \param SLocMapI points at a chunk of a module that contains no
5924 /// preprocessed entities or the entities it contains are not the ones we are
5925 /// looking for.
5926 PreprocessedEntityID ASTReader::findNextPreprocessedEntity(
5927  GlobalSLocOffsetMapType::const_iterator SLocMapI) const {
5928  ++SLocMapI;
5929  for (GlobalSLocOffsetMapType::const_iterator
5930  EndI = GlobalSLocOffsetMap.end(); SLocMapI != EndI; ++SLocMapI) {
5931  ModuleFile &M = *SLocMapI->second;
5933  return M.BasePreprocessedEntityID;
5934  }
5935 
5936  return getTotalNumPreprocessedEntities();
5937 }
5938 
5939 namespace {
5940 
5941 struct PPEntityComp {
5942  const ASTReader &Reader;
5943  ModuleFile &M;
5944 
5945  PPEntityComp(const ASTReader &Reader, ModuleFile &M) : Reader(Reader), M(M) {}
5946 
5947  bool operator()(const PPEntityOffset &L, const PPEntityOffset &R) const {
5948  SourceLocation LHS = getLoc(L);
5949  SourceLocation RHS = getLoc(R);
5950  return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS);
5951  }
5952 
5953  bool operator()(const PPEntityOffset &L, SourceLocation RHS) const {
5954  SourceLocation LHS = getLoc(L);
5955  return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS);
5956  }
5957 
5958  bool operator()(SourceLocation LHS, const PPEntityOffset &R) const {
5959  SourceLocation RHS = getLoc(R);
5960  return Reader.getSourceManager().isBeforeInTranslationUnit(LHS, RHS);
5961  }
5962 
5963  SourceLocation getLoc(const PPEntityOffset &PPE) const {
5964  return Reader.TranslateSourceLocation(M, PPE.getBegin());
5965  }
5966 };
5967 
5968 } // namespace
5969 
5970 PreprocessedEntityID ASTReader::findPreprocessedEntity(SourceLocation Loc,
5971  bool EndsAfter) const {
5972  if (SourceMgr.isLocalSourceLocation(Loc))
5973  return getTotalNumPreprocessedEntities();
5974 
5975  GlobalSLocOffsetMapType::const_iterator SLocMapI = GlobalSLocOffsetMap.find(
5976  SourceManager::MaxLoadedOffset - Loc.getOffset() - 1);
5977  assert(SLocMapI != GlobalSLocOffsetMap.end() &&
5978  "Corrupted global sloc offset map");
5979 
5980  if (SLocMapI->second->NumPreprocessedEntities == 0)
5981  return findNextPreprocessedEntity(SLocMapI);
5982 
5983  ModuleFile &M = *SLocMapI->second;
5984 
5985  using pp_iterator = const PPEntityOffset *;
5986 
5987  pp_iterator pp_begin = M.PreprocessedEntityOffsets;
5988  pp_iterator pp_end = pp_begin + M.NumPreprocessedEntities;
5989 
5990  size_t Count = M.NumPreprocessedEntities;
5991  size_t Half;
5992  pp_iterator First = pp_begin;
5993  pp_iterator PPI;
5994 
5995  if (EndsAfter) {
5996  PPI = std::upper_bound(pp_begin, pp_end, Loc,
5997  PPEntityComp(*this, M));
5998  } else {
5999  // Do a binary search manually instead of using std::lower_bound because
6000  // The end locations of entities may be unordered (when a macro expansion
6001  // is inside another macro argument), but for this case it is not important
6002  // whether we get the first macro expansion or its containing macro.
6003  while (Count > 0) {
6004  Half = Count / 2;
6005  PPI = First;
6006  std::advance(PPI, Half);
6007  if (SourceMgr.isBeforeInTranslationUnit(
6008  TranslateSourceLocation(M, PPI->getEnd()), Loc)) {
6009  First = PPI;
6010  ++First;
6011  Count = Count - Half - 1;
6012  } else
6013  Count = Half;
6014  }
6015  }
6016 
6017  if (PPI == pp_end)
6018  return findNextPreprocessedEntity(SLocMapI);
6019 
6020  return M.BasePreprocessedEntityID + (PPI - pp_begin);
6021 }
6022 
6023 /// Returns a pair of [Begin, End) indices of preallocated
6024 /// preprocessed entities that \arg Range encompasses.
6025 std::pair<unsigned, unsigned>
6027  if (Range.isInvalid())
6028  return std::make_pair(0,0);
6029  assert(!SourceMgr.isBeforeInTranslationUnit(Range.getEnd(),Range.getBegin()));
6030 
6031  PreprocessedEntityID BeginID =
6032  findPreprocessedEntity(Range.getBegin(), false);
6033  PreprocessedEntityID EndID = findPreprocessedEntity(Range.getEnd(), true);
6034  return std::make_pair(BeginID, EndID);
6035 }
6036 
6037 /// Optionally returns true or false if the preallocated preprocessed
6038 /// entity with index \arg Index came from file \arg FID.
6040  FileID FID) {
6041  if (FID.isInvalid())
6042  return false;
6043 
6044  std::pair<ModuleFile *, unsigned> PPInfo = getModulePreprocessedEntity(Index);
6045  ModuleFile &M = *PPInfo.first;
6046  unsigned LocalIndex = PPInfo.second;
6047  const PPEntityOffset &PPOffs = M.PreprocessedEntityOffsets[LocalIndex];
6048 
6049  SourceLocation Loc = TranslateSourceLocation(M, PPOffs.getBegin());
6050  if (Loc.isInvalid())
6051  return false;
6052 
6053  if (SourceMgr.isInFileID(SourceMgr.getFileLoc(Loc), FID))
6054  return true;
6055  else
6056  return false;
6057 }
6058 
6059 namespace {
6060 
6061  /// Visitor used to search for information about a header file.
6062  class HeaderFileInfoVisitor {
6063  const FileEntry *FE;
6065 
6066  public:
6067  explicit HeaderFileInfoVisitor(const FileEntry *FE) : FE(FE) {}
6068 
6069  bool operator()(ModuleFile &M) {
6071  = static_cast<