clang  6.0.0svn
GlobalModuleIndex.cpp
Go to the documentation of this file.
1 //===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the GlobalModuleIndex class.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "ASTReaderInternals.h"
17 #include "clang/Lex/HeaderSearch.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/MapVector.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/ADT/StringExtras.h"
25 #include "llvm/Bitcode/BitstreamReader.h"
26 #include "llvm/Bitcode/BitstreamWriter.h"
27 #include "llvm/Support/FileSystem.h"
28 #include "llvm/Support/LockFileManager.h"
29 #include "llvm/Support/MemoryBuffer.h"
30 #include "llvm/Support/OnDiskHashTable.h"
31 #include "llvm/Support/Path.h"
32 #include <cstdio>
33 using namespace clang;
34 using namespace serialization;
35 
36 //----------------------------------------------------------------------------//
37 // Shared constants
38 //----------------------------------------------------------------------------//
39 namespace {
40  enum {
41  /// \brief The block containing the index.
42  GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
43  };
44 
45  /// \brief Describes the record types in the index.
47  /// \brief Contains version information and potentially other metadata,
48  /// used to determine if we can read this global index file.
49  INDEX_METADATA,
50  /// \brief Describes a module, including its file name and dependencies.
51  MODULE,
52  /// \brief The index for identifiers.
53  IDENTIFIER_INDEX
54  };
55 }
56 
57 /// \brief The name of the global index file.
58 static const char * const IndexFileName = "modules.idx";
59 
60 /// \brief The global index file version.
61 static const unsigned CurrentVersion = 1;
62 
63 //----------------------------------------------------------------------------//
64 // Global module index reader.
65 //----------------------------------------------------------------------------//
66 
67 namespace {
68 
69 /// \brief Trait used to read the identifier index from the on-disk hash
70 /// table.
71 class IdentifierIndexReaderTrait {
72 public:
73  typedef StringRef external_key_type;
74  typedef StringRef internal_key_type;
75  typedef SmallVector<unsigned, 2> data_type;
76  typedef unsigned hash_value_type;
77  typedef unsigned offset_type;
78 
79  static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
80  return a == b;
81  }
82 
83  static hash_value_type ComputeHash(const internal_key_type& a) {
84  return llvm::HashString(a);
85  }
86 
87  static std::pair<unsigned, unsigned>
88  ReadKeyDataLength(const unsigned char*& d) {
89  using namespace llvm::support;
90  unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
91  unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
92  return std::make_pair(KeyLen, DataLen);
93  }
94 
95  static const internal_key_type&
96  GetInternalKey(const external_key_type& x) { return x; }
97 
98  static const external_key_type&
99  GetExternalKey(const internal_key_type& x) { return x; }
100 
101  static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
102  return StringRef((const char *)d, n);
103  }
104 
105  static data_type ReadData(const internal_key_type& k,
106  const unsigned char* d,
107  unsigned DataLen) {
108  using namespace llvm::support;
109 
110  data_type Result;
111  while (DataLen > 0) {
112  unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
113  Result.push_back(ID);
114  DataLen -= 4;
115  }
116 
117  return Result;
118  }
119 };
120 
122  IdentifierIndexTable;
123 
124 }
125 
126 GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer,
127  llvm::BitstreamCursor Cursor)
128  : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
129  NumIdentifierLookupHits() {
130  // Read the global index.
131  bool InGlobalIndexBlock = false;
132  bool Done = false;
133  while (!Done) {
134  llvm::BitstreamEntry Entry = Cursor.advance();
135 
136  switch (Entry.Kind) {
138  return;
139 
140  case llvm::BitstreamEntry::EndBlock:
141  if (InGlobalIndexBlock) {
142  InGlobalIndexBlock = false;
143  Done = true;
144  continue;
145  }
146  return;
147 
148 
149  case llvm::BitstreamEntry::Record:
150  // Entries in the global index block are handled below.
151  if (InGlobalIndexBlock)
152  break;
153 
154  return;
155 
156  case llvm::BitstreamEntry::SubBlock:
157  if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
158  if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
159  return;
160 
161  InGlobalIndexBlock = true;
162  } else if (Cursor.SkipBlock()) {
163  return;
164  }
165  continue;
166  }
167 
169  StringRef Blob;
170  switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) {
171  case INDEX_METADATA:
172  // Make sure that the version matches.
173  if (Record.size() < 1 || Record[0] != CurrentVersion)
174  return;
175  break;
176 
177  case MODULE: {
178  unsigned Idx = 0;
179  unsigned ID = Record[Idx++];
180 
181  // Make room for this module's information.
182  if (ID == Modules.size())
183  Modules.push_back(ModuleInfo());
184  else
185  Modules.resize(ID + 1);
186 
187  // Size/modification time for this module file at the time the
188  // global index was built.
189  Modules[ID].Size = Record[Idx++];
190  Modules[ID].ModTime = Record[Idx++];
191 
192  // File name.
193  unsigned NameLen = Record[Idx++];
194  Modules[ID].FileName.assign(Record.begin() + Idx,
195  Record.begin() + Idx + NameLen);
196  Idx += NameLen;
197 
198  // Dependencies
199  unsigned NumDeps = Record[Idx++];
200  Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
201  Record.begin() + Idx,
202  Record.begin() + Idx + NumDeps);
203  Idx += NumDeps;
204 
205  // Make sure we're at the end of the record.
206  assert(Idx == Record.size() && "More module info?");
207 
208  // Record this module as an unresolved module.
209  // FIXME: this doesn't work correctly for module names containing path
210  // separators.
211  StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
212  // Remove the -<hash of ModuleMapPath>
213  ModuleName = ModuleName.rsplit('-').first;
214  UnresolvedModules[ModuleName] = ID;
215  break;
216  }
217 
218  case IDENTIFIER_INDEX:
219  // Wire up the identifier index.
220  if (Record[0]) {
221  IdentifierIndex = IdentifierIndexTable::Create(
222  (const unsigned char *)Blob.data() + Record[0],
223  (const unsigned char *)Blob.data() + sizeof(uint32_t),
224  (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
225  }
226  break;
227  }
228  }
229 }
230 
232  delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
233 }
234 
235 std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
237  // Load the index file, if it's there.
238  llvm::SmallString<128> IndexPath;
239  IndexPath += Path;
240  llvm::sys::path::append(IndexPath, IndexFileName);
241 
242  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
243  llvm::MemoryBuffer::getFile(IndexPath.c_str());
244  if (!BufferOrErr)
245  return std::make_pair(nullptr, EC_NotFound);
246  std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
247 
248  /// \brief The main bitstream cursor for the main block.
249  llvm::BitstreamCursor Cursor(*Buffer);
250 
251  // Sniff for the signature.
252  if (Cursor.Read(8) != 'B' ||
253  Cursor.Read(8) != 'C' ||
254  Cursor.Read(8) != 'G' ||
255  Cursor.Read(8) != 'I') {
256  return std::make_pair(nullptr, EC_IOError);
257  }
258 
259  return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor),
260  EC_None);
261 }
262 
263 void
265  ModuleFiles.clear();
266  for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
267  if (ModuleFile *MF = Modules[I].File)
268  ModuleFiles.push_back(MF);
269  }
270 }
271 
273  ModuleFile *File,
274  SmallVectorImpl<ModuleFile *> &Dependencies) {
275  // Look for information about this module file.
276  llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
277  = ModulesByFile.find(File);
278  if (Known == ModulesByFile.end())
279  return;
280 
281  // Record dependencies.
282  Dependencies.clear();
283  ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
284  for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
285  if (ModuleFile *MF = Modules[I].File)
286  Dependencies.push_back(MF);
287  }
288 }
289 
290 bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
291  Hits.clear();
292 
293  // If there's no identifier index, there is nothing we can do.
294  if (!IdentifierIndex)
295  return false;
296 
297  // Look into the identifier index.
298  ++NumIdentifierLookups;
299  IdentifierIndexTable &Table
300  = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
301  IdentifierIndexTable::iterator Known = Table.find(Name);
302  if (Known == Table.end()) {
303  return true;
304  }
305 
306  SmallVector<unsigned, 2> ModuleIDs = *Known;
307  for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
308  if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
309  Hits.insert(MF);
310  }
311 
312  ++NumIdentifierLookupHits;
313  return true;
314 }
315 
317  // Look for the module in the global module index based on the module name.
318  StringRef Name = File->ModuleName;
319  llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
320  if (Known == UnresolvedModules.end()) {
321  return true;
322  }
323 
324  // Rectify this module with the global module index.
325  ModuleInfo &Info = Modules[Known->second];
326 
327  // If the size and modification time match what we expected, record this
328  // module file.
329  bool Failed = true;
330  if (File->File->getSize() == Info.Size &&
331  File->File->getModificationTime() == Info.ModTime) {
332  Info.File = File;
333  ModulesByFile[File] = Known->second;
334 
335  Failed = false;
336  }
337 
338  // One way or another, we have resolved this module file.
339  UnresolvedModules.erase(Known);
340  return Failed;
341 }
342 
344  std::fprintf(stderr, "*** Global Module Index Statistics:\n");
345  if (NumIdentifierLookups) {
346  fprintf(stderr, " %u / %u identifier lookups succeeded (%f%%)\n",
347  NumIdentifierLookupHits, NumIdentifierLookups,
348  (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
349  }
350  std::fprintf(stderr, "\n");
351 }
352 
353 LLVM_DUMP_METHOD void GlobalModuleIndex::dump() {
354  llvm::errs() << "*** Global Module Index Dump:\n";
355  llvm::errs() << "Module files:\n";
356  for (auto &MI : Modules) {
357  llvm::errs() << "** " << MI.FileName << "\n";
358  if (MI.File)
359  MI.File->dump();
360  else
361  llvm::errs() << "\n";
362  }
363  llvm::errs() << "\n";
364 }
365 
366 //----------------------------------------------------------------------------//
367 // Global module index writer.
368 //----------------------------------------------------------------------------//
369 
370 namespace {
371  /// \brief Provides information about a specific module file.
372  struct ModuleFileInfo {
373  /// \brief The numberic ID for this module file.
374  unsigned ID;
375 
376  /// \brief The set of modules on which this module depends. Each entry is
377  /// a module ID.
378  SmallVector<unsigned, 4> Dependencies;
379  ASTFileSignature Signature;
380  };
381 
382  struct ImportedModuleFileInfo {
383  off_t StoredSize;
384  time_t StoredModTime;
385  ASTFileSignature StoredSignature;
386  ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig)
387  : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
388  };
389 
390  /// \brief Builder that generates the global module index file.
391  class GlobalModuleIndexBuilder {
392  FileManager &FileMgr;
393  const PCHContainerReader &PCHContainerRdr;
394 
395  /// Mapping from files to module file information.
396  typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
397 
398  /// Information about each of the known module files.
399  ModuleFilesMap ModuleFiles;
400 
401  /// \brief Mapping from the imported module file to the imported
402  /// information.
403  typedef std::multimap<const FileEntry *, ImportedModuleFileInfo>
404  ImportedModuleFilesMap;
405 
406  /// \brief Information about each importing of a module file.
407  ImportedModuleFilesMap ImportedModuleFiles;
408 
409  /// \brief Mapping from identifiers to the list of module file IDs that
410  /// consider this identifier to be interesting.
411  typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
412 
413  /// \brief A mapping from all interesting identifiers to the set of module
414  /// files in which those identifiers are considered interesting.
415  InterestingIdentifierMap InterestingIdentifiers;
416 
417  /// \brief Write the block-info block for the global module index file.
418  void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
419 
420  /// \brief Retrieve the module file information for the given file.
421  ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
422  llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
423  = ModuleFiles.find(File);
424  if (Known != ModuleFiles.end())
425  return Known->second;
426 
427  unsigned NewID = ModuleFiles.size();
428  ModuleFileInfo &Info = ModuleFiles[File];
429  Info.ID = NewID;
430  return Info;
431  }
432 
433  public:
434  explicit GlobalModuleIndexBuilder(
435  FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr)
436  : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
437 
438  /// \brief Load the contents of the given module file into the builder.
439  ///
440  /// \returns true if an error occurred, false otherwise.
441  bool loadModuleFile(const FileEntry *File);
442 
443  /// \brief Write the index to the given bitstream.
444  /// \returns true if an error occurred, false otherwise.
445  bool writeIndex(llvm::BitstreamWriter &Stream);
446  };
447 }
448 
449 static void emitBlockID(unsigned ID, const char *Name,
450  llvm::BitstreamWriter &Stream,
451  SmallVectorImpl<uint64_t> &Record) {
452  Record.clear();
453  Record.push_back(ID);
454  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
455 
456  // Emit the block name if present.
457  if (!Name || Name[0] == 0) return;
458  Record.clear();
459  while (*Name)
460  Record.push_back(*Name++);
461  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
462 }
463 
464 static void emitRecordID(unsigned ID, const char *Name,
465  llvm::BitstreamWriter &Stream,
466  SmallVectorImpl<uint64_t> &Record) {
467  Record.clear();
468  Record.push_back(ID);
469  while (*Name)
470  Record.push_back(*Name++);
471  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
472 }
473 
474 void
475 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
477  Stream.EnterBlockInfoBlock();
478 
479 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
480 #define RECORD(X) emitRecordID(X, #X, Stream, Record)
481  BLOCK(GLOBAL_INDEX_BLOCK);
482  RECORD(INDEX_METADATA);
483  RECORD(MODULE);
484  RECORD(IDENTIFIER_INDEX);
485 #undef RECORD
486 #undef BLOCK
487 
488  Stream.ExitBlock();
489 }
490 
491 namespace {
492  class InterestingASTIdentifierLookupTrait
494 
495  public:
496  /// \brief The identifier and whether it is "interesting".
497  typedef std::pair<StringRef, bool> data_type;
498 
499  data_type ReadData(const internal_key_type& k,
500  const unsigned char* d,
501  unsigned DataLen) {
502  // The first bit indicates whether this identifier is interesting.
503  // That's all we care about.
504  using namespace llvm::support;
505  unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
506  bool IsInteresting = RawID & 0x01;
507  return std::make_pair(k, IsInteresting);
508  }
509  };
510 }
511 
512 bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
513  // Open the module file.
514 
515  auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true);
516  if (!Buffer) {
517  return true;
518  }
519 
520  // Initialize the input stream
521  llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer));
522 
523  // Sniff for the signature.
524  if (InStream.Read(8) != 'C' ||
525  InStream.Read(8) != 'P' ||
526  InStream.Read(8) != 'C' ||
527  InStream.Read(8) != 'H') {
528  return true;
529  }
530 
531  // Record this module file and assign it a unique ID (if it doesn't have
532  // one already).
533  unsigned ID = getModuleFileInfo(File).ID;
534 
535  // Search for the blocks and records we care about.
536  enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other;
537  bool Done = false;
538  while (!Done) {
539  llvm::BitstreamEntry Entry = InStream.advance();
540  switch (Entry.Kind) {
542  Done = true;
543  continue;
544 
545  case llvm::BitstreamEntry::Record:
546  // In the 'other' state, just skip the record. We don't care.
547  if (State == Other) {
548  InStream.skipRecord(Entry.ID);
549  continue;
550  }
551 
552  // Handle potentially-interesting records below.
553  break;
554 
555  case llvm::BitstreamEntry::SubBlock:
556  if (Entry.ID == CONTROL_BLOCK_ID) {
557  if (InStream.EnterSubBlock(CONTROL_BLOCK_ID))
558  return true;
559 
560  // Found the control block.
561  State = ControlBlock;
562  continue;
563  }
564 
565  if (Entry.ID == AST_BLOCK_ID) {
566  if (InStream.EnterSubBlock(AST_BLOCK_ID))
567  return true;
568 
569  // Found the AST block.
570  State = ASTBlock;
571  continue;
572  }
573 
574  if (Entry.ID == UNHASHED_CONTROL_BLOCK_ID) {
575  if (InStream.EnterSubBlock(UNHASHED_CONTROL_BLOCK_ID))
576  return true;
577 
578  // Found the Diagnostic Options block.
579  State = DiagnosticOptionsBlock;
580  continue;
581  }
582 
583  if (InStream.SkipBlock())
584  return true;
585 
586  continue;
587 
588  case llvm::BitstreamEntry::EndBlock:
589  State = Other;
590  continue;
591  }
592 
593  // Read the given record.
595  StringRef Blob;
596  unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);
597 
598  // Handle module dependencies.
599  if (State == ControlBlock && Code == IMPORTS) {
600  // Load each of the imported PCH files.
601  unsigned Idx = 0, N = Record.size();
602  while (Idx < N) {
603  // Read information about the AST file.
604 
605  // Skip the imported kind
606  ++Idx;
607 
608  // Skip the import location
609  ++Idx;
610 
611  // Load stored size/modification time.
612  off_t StoredSize = (off_t)Record[Idx++];
613  time_t StoredModTime = (time_t)Record[Idx++];
614 
615  // Skip the stored signature.
616  // FIXME: we could read the signature out of the import and validate it.
617  ASTFileSignature StoredSignature = {
618  {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
619  (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
620  (uint32_t)Record[Idx++]}}};
621 
622  // Skip the module name (currently this is only used for prebuilt
623  // modules while here we are only dealing with cached).
624  Idx += Record[Idx] + 1;
625 
626  // Retrieve the imported file name.
627  unsigned Length = Record[Idx++];
628  SmallString<128> ImportedFile(Record.begin() + Idx,
629  Record.begin() + Idx + Length);
630  Idx += Length;
631 
632  // Find the imported module file.
633  const FileEntry *DependsOnFile
634  = FileMgr.getFile(ImportedFile, /*openFile=*/false,
635  /*cacheFailure=*/false);
636 
637  if (!DependsOnFile)
638  return true;
639 
640  // Save the information in ImportedModuleFileInfo so we can verify after
641  // loading all pcms.
642  ImportedModuleFiles.insert(std::make_pair(
643  DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime,
644  StoredSignature)));
645 
646  // Record the dependency.
647  unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
648  getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
649  }
650 
651  continue;
652  }
653 
654  // Handle the identifier table
655  if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
657  InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
658  std::unique_ptr<InterestingIdentifierTable> Table(
660  (const unsigned char *)Blob.data() + Record[0],
661  (const unsigned char *)Blob.data() + sizeof(uint32_t),
662  (const unsigned char *)Blob.data()));
663  for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
664  DEnd = Table->data_end();
665  D != DEnd; ++D) {
666  std::pair<StringRef, bool> Ident = *D;
667  if (Ident.second)
668  InterestingIdentifiers[Ident.first].push_back(ID);
669  else
670  (void)InterestingIdentifiers[Ident.first];
671  }
672  }
673 
674  // Get Signature.
675  if (State == DiagnosticOptionsBlock && Code == SIGNATURE)
676  getModuleFileInfo(File).Signature = {
677  {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
678  (uint32_t)Record[3], (uint32_t)Record[4]}}};
679 
680  // We don't care about this record.
681  }
682 
683  return false;
684 }
685 
686 namespace {
687 
688 /// \brief Trait used to generate the identifier index as an on-disk hash
689 /// table.
690 class IdentifierIndexWriterTrait {
691 public:
692  typedef StringRef key_type;
693  typedef StringRef key_type_ref;
694  typedef SmallVector<unsigned, 2> data_type;
695  typedef const SmallVector<unsigned, 2> &data_type_ref;
696  typedef unsigned hash_value_type;
697  typedef unsigned offset_type;
698 
699  static hash_value_type ComputeHash(key_type_ref Key) {
700  return llvm::HashString(Key);
701  }
702 
703  std::pair<unsigned,unsigned>
704  EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
705  using namespace llvm::support;
706  endian::Writer<little> LE(Out);
707  unsigned KeyLen = Key.size();
708  unsigned DataLen = Data.size() * 4;
709  LE.write<uint16_t>(KeyLen);
710  LE.write<uint16_t>(DataLen);
711  return std::make_pair(KeyLen, DataLen);
712  }
713 
714  void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
715  Out.write(Key.data(), KeyLen);
716  }
717 
718  void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
719  unsigned DataLen) {
720  using namespace llvm::support;
721  for (unsigned I = 0, N = Data.size(); I != N; ++I)
722  endian::Writer<little>(Out).write<uint32_t>(Data[I]);
723  }
724 };
725 
726 }
727 
728 bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
729  for (auto MapEntry : ImportedModuleFiles) {
730  auto *File = MapEntry.first;
731  ImportedModuleFileInfo &Info = MapEntry.second;
732  if (getModuleFileInfo(File).Signature) {
733  if (getModuleFileInfo(File).Signature != Info.StoredSignature)
734  // Verify Signature.
735  return true;
736  } else if (Info.StoredSize != File->getSize() ||
737  Info.StoredModTime != File->getModificationTime())
738  // Verify Size and ModTime.
739  return true;
740  }
741 
742  using namespace llvm;
743 
744  // Emit the file header.
745  Stream.Emit((unsigned)'B', 8);
746  Stream.Emit((unsigned)'C', 8);
747  Stream.Emit((unsigned)'G', 8);
748  Stream.Emit((unsigned)'I', 8);
749 
750  // Write the block-info block, which describes the records in this bitcode
751  // file.
752  emitBlockInfoBlock(Stream);
753 
754  Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
755 
756  // Write the metadata.
758  Record.push_back(CurrentVersion);
759  Stream.EmitRecord(INDEX_METADATA, Record);
760 
761  // Write the set of known module files.
762  for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
763  MEnd = ModuleFiles.end();
764  M != MEnd; ++M) {
765  Record.clear();
766  Record.push_back(M->second.ID);
767  Record.push_back(M->first->getSize());
768  Record.push_back(M->first->getModificationTime());
769 
770  // File name
771  StringRef Name(M->first->getName());
772  Record.push_back(Name.size());
773  Record.append(Name.begin(), Name.end());
774 
775  // Dependencies
776  Record.push_back(M->second.Dependencies.size());
777  Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
778  Stream.EmitRecord(MODULE, Record);
779  }
780 
781  // Write the identifier -> module file mapping.
782  {
783  llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
784  IdentifierIndexWriterTrait Trait;
785 
786  // Populate the hash table.
787  for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
788  IEnd = InterestingIdentifiers.end();
789  I != IEnd; ++I) {
790  Generator.insert(I->first(), I->second, Trait);
791  }
792 
793  // Create the on-disk hash table in a buffer.
795  uint32_t BucketOffset;
796  {
797  using namespace llvm::support;
798  llvm::raw_svector_ostream Out(IdentifierTable);
799  // Make sure that no bucket is at offset 0
800  endian::Writer<little>(Out).write<uint32_t>(0);
801  BucketOffset = Generator.Emit(Out, Trait);
802  }
803 
804  // Create a blob abbreviation
805  auto Abbrev = std::make_shared<BitCodeAbbrev>();
806  Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
807  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
808  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
809  unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
810 
811  // Write the identifier table
812  uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
813  Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
814  }
815 
816  Stream.ExitBlock();
817  return false;
818 }
819 
822  const PCHContainerReader &PCHContainerRdr,
823  StringRef Path) {
824  llvm::SmallString<128> IndexPath;
825  IndexPath += Path;
826  llvm::sys::path::append(IndexPath, IndexFileName);
827 
828  // Coordinate building the global index file with other processes that might
829  // try to do the same.
830  llvm::LockFileManager Locked(IndexPath);
831  switch (Locked) {
832  case llvm::LockFileManager::LFS_Error:
833  return EC_IOError;
834 
835  case llvm::LockFileManager::LFS_Owned:
836  // We're responsible for building the index ourselves. Do so below.
837  break;
838 
839  case llvm::LockFileManager::LFS_Shared:
840  // Someone else is responsible for building the index. We don't care
841  // when they finish, so we're done.
842  return EC_Building;
843  }
844 
845  // The module index builder.
846  GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
847 
848  // Load each of the module files.
849  std::error_code EC;
850  for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
851  D != DEnd && !EC;
852  D.increment(EC)) {
853  // If this isn't a module file, we don't care.
854  if (llvm::sys::path::extension(D->path()) != ".pcm") {
855  // ... unless it's a .pcm.lock file, which indicates that someone is
856  // in the process of rebuilding a module. They'll rebuild the index
857  // at the end of that translation unit, so we don't have to.
858  if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
859  return EC_Building;
860 
861  continue;
862  }
863 
864  // If we can't find the module file, skip it.
865  const FileEntry *ModuleFile = FileMgr.getFile(D->path());
866  if (!ModuleFile)
867  continue;
868 
869  // Load this module file.
870  if (Builder.loadModuleFile(ModuleFile))
871  return EC_IOError;
872  }
873 
874  // The output buffer, into which the global index will be written.
875  SmallVector<char, 16> OutputBuffer;
876  {
877  llvm::BitstreamWriter OutputStream(OutputBuffer);
878  if (Builder.writeIndex(OutputStream))
879  return EC_IOError;
880  }
881 
882  // Write the global index file to a temporary file.
883  llvm::SmallString<128> IndexTmpPath;
884  int TmpFD;
885  if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
886  IndexTmpPath))
887  return EC_IOError;
888 
889  // Open the temporary global index file for output.
890  llvm::raw_fd_ostream Out(TmpFD, true);
891  if (Out.has_error())
892  return EC_IOError;
893 
894  // Write the index.
895  Out.write(OutputBuffer.data(), OutputBuffer.size());
896  Out.close();
897  if (Out.has_error())
898  return EC_IOError;
899 
900  // Remove the old index file. It isn't relevant any more.
901  llvm::sys::fs::remove(IndexPath);
902 
903  // Rename the newly-written index file to the proper name.
904  if (llvm::sys::fs::rename(IndexTmpPath, IndexPath)) {
905  // Rename failed; just remove the
906  llvm::sys::fs::remove(IndexTmpPath);
907  return EC_IOError;
908  }
909 
910  // We're done.
911  return EC_None;
912 }
913 
914 namespace {
915  class GlobalIndexIdentifierIterator : public IdentifierIterator {
916  /// \brief The current position within the identifier lookup table.
917  IdentifierIndexTable::key_iterator Current;
918 
919  /// \brief The end position within the identifier lookup table.
920  IdentifierIndexTable::key_iterator End;
921 
922  public:
923  explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
924  Current = Idx.key_begin();
925  End = Idx.key_end();
926  }
927 
928  StringRef Next() override {
929  if (Current == End)
930  return StringRef();
931 
932  StringRef Result = *Current;
933  ++Current;
934  return Result;
935  }
936  };
937 }
938 
940  IdentifierIndexTable &Table =
941  *static_cast<IdentifierIndexTable *>(IdentifierIndex);
942  return new GlobalIndexIdentifierIterator(Table);
943 }
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:116
Defines the clang::FileManager interface and associated types.
time_t getModificationTime() const
Definition: FileManager.h:91
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:26
std::string ModuleName
The name of the module.
Definition: Module.h:118
static const unsigned CurrentVersion
The global index file version.
A block with unhashed content.
Definition: ASTBitCodes.h:278
void printStats()
Print statistics to standard error.
static void emitRecordID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
LineState State
void getModuleDependencies(ModuleFile *File, SmallVectorImpl< ModuleFile *> &Dependencies)
Retrieve the set of module files on which the given module file directly depends. ...
llvm::SmallPtrSet< ModuleFile *, 4 > HitSet
A set of module files in which we found a result.
The signature of a module, which is a hash of the AST content.
Definition: Module.h:55
ErrorCode
An error code returned when trying to read an index.
This abstract interface provides operations for unwrapping containers for serialized ASTs (precompile...
static ErrorCode writeIndex(FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, StringRef Path)
Write a global index into the given.
const FileEntry * getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
void getKnownModules(SmallVectorImpl< ModuleFile *> &ModuleFiles)
Retrieve the set of modules that have up-to-date indexes.
void dump()
Print debugging view to standard error.
SourceLocation End
static void emitBlockID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
IdentifierIterator * createIdentifierIterator() const
Returns an iterator for identifiers stored in the index table.
Implements an efficient mapping from strings to IdentifierInfo nodes.
Base class for the trait describing the on-disk hash table for the identifiers in an AST file...
Information about a module that has been loaded by the ASTReader.
Definition: Module.h:100
An iterator that walks over all of the known identifiers in the lookup table.
static const char *const IndexFileName
The name of the global index file.
Record code for the identifier table.
Definition: ASTBitCodes.h:423
IndexRecordTypes
Describes the record types in the index.
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
The AST block, which acts as a container around the full AST block.
Definition: ASTBitCodes.h:230
A global index for a set of module files, providing information about the identifiers within those mo...
unsigned ComputeHash(Selector Sel)
Definition: ASTCommon.cpp:167
Record code for the signature that identifiers this AST file.
Definition: ASTBitCodes.h:345
Record code for the list of other AST files imported by this AST file.
Definition: ASTBitCodes.h:289
Dataflow Directional Tag Classes.
static std::pair< GlobalModuleIndex *, ErrorCode > readIndex(StringRef Path)
Read a global index file for the given directory.
off_t getSize() const
Definition: FileManager.h:87
#define BLOCK(X)
Dump information about a module file.
#define RECORD(X)
The control block, which contains all of the information that needs to be validated prior to committi...
Definition: ASTBitCodes.h:256
const FileEntry * File
The file entry for the module file.
Definition: Module.h:153
bool lookupIdentifier(StringRef Name, HitSet &Hits)
Look for all of the module files with information about the given identifier, e.g., a global function, variable, or type with that name.
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool loadedModuleFile(ModuleFile *File)
Note that the given module file has been loaded.