clang  9.0.0svn
GlobalModuleIndex.cpp
Go to the documentation of this file.
1 //===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the GlobalModuleIndex class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 
14 #include "ASTReaderInternals.h"
16 #include "clang/Lex/HeaderSearch.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/MapVector.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/Bitcode/BitstreamReader.h"
25 #include "llvm/Bitcode/BitstreamWriter.h"
26 #include "llvm/Support/DJB.h"
27 #include "llvm/Support/FileSystem.h"
28 #include "llvm/Support/LockFileManager.h"
29 #include "llvm/Support/MemoryBuffer.h"
30 #include "llvm/Support/OnDiskHashTable.h"
31 #include "llvm/Support/Path.h"
32 #include "llvm/Support/TimeProfiler.h"
33 #include <cstdio>
34 using namespace clang;
35 using namespace serialization;
36 
37 //----------------------------------------------------------------------------//
38 // Shared constants
39 //----------------------------------------------------------------------------//
40 namespace {
41  enum {
42  /// The block containing the index.
43  GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
44  };
45 
46  /// Describes the record types in the index.
48  /// Contains version information and potentially other metadata,
49  /// used to determine if we can read this global index file.
50  INDEX_METADATA,
51  /// Describes a module, including its file name and dependencies.
52  MODULE,
53  /// The index for identifiers.
54  IDENTIFIER_INDEX
55  };
56 }
57 
58 /// The name of the global index file.
59 static const char * const IndexFileName = "modules.idx";
60 
61 /// The global index file version.
62 static const unsigned CurrentVersion = 1;
63 
64 //----------------------------------------------------------------------------//
65 // Global module index reader.
66 //----------------------------------------------------------------------------//
67 
68 namespace {
69 
70 /// Trait used to read the identifier index from the on-disk hash
71 /// table.
72 class IdentifierIndexReaderTrait {
73 public:
74  typedef StringRef external_key_type;
75  typedef StringRef internal_key_type;
76  typedef SmallVector<unsigned, 2> data_type;
77  typedef unsigned hash_value_type;
78  typedef unsigned offset_type;
79 
80  static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
81  return a == b;
82  }
83 
84  static hash_value_type ComputeHash(const internal_key_type& a) {
85  return llvm::djbHash(a);
86  }
87 
88  static std::pair<unsigned, unsigned>
89  ReadKeyDataLength(const unsigned char*& d) {
90  using namespace llvm::support;
91  unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
92  unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
93  return std::make_pair(KeyLen, DataLen);
94  }
95 
96  static const internal_key_type&
97  GetInternalKey(const external_key_type& x) { return x; }
98 
99  static const external_key_type&
100  GetExternalKey(const internal_key_type& x) { return x; }
101 
102  static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
103  return StringRef((const char *)d, n);
104  }
105 
106  static data_type ReadData(const internal_key_type& k,
107  const unsigned char* d,
108  unsigned DataLen) {
109  using namespace llvm::support;
110 
111  data_type Result;
112  while (DataLen > 0) {
113  unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
114  Result.push_back(ID);
115  DataLen -= 4;
116  }
117 
118  return Result;
119  }
120 };
121 
122 typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
123  IdentifierIndexTable;
124 
125 }
126 
127 GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer,
128  llvm::BitstreamCursor Cursor)
129  : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
130  NumIdentifierLookupHits() {
131  llvm::TimeTraceScope TimeScope("Module LoadIndex", StringRef(""));
132  // Read the global index.
133  bool InGlobalIndexBlock = false;
134  bool Done = false;
135  while (!Done) {
136  llvm::BitstreamEntry Entry = Cursor.advance();
137 
138  switch (Entry.Kind) {
140  return;
141 
142  case llvm::BitstreamEntry::EndBlock:
143  if (InGlobalIndexBlock) {
144  InGlobalIndexBlock = false;
145  Done = true;
146  continue;
147  }
148  return;
149 
150 
151  case llvm::BitstreamEntry::Record:
152  // Entries in the global index block are handled below.
153  if (InGlobalIndexBlock)
154  break;
155 
156  return;
157 
158  case llvm::BitstreamEntry::SubBlock:
159  if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
160  if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
161  return;
162 
163  InGlobalIndexBlock = true;
164  } else if (Cursor.SkipBlock()) {
165  return;
166  }
167  continue;
168  }
169 
171  StringRef Blob;
172  switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) {
173  case INDEX_METADATA:
174  // Make sure that the version matches.
175  if (Record.size() < 1 || Record[0] != CurrentVersion)
176  return;
177  break;
178 
179  case MODULE: {
180  unsigned Idx = 0;
181  unsigned ID = Record[Idx++];
182 
183  // Make room for this module's information.
184  if (ID == Modules.size())
185  Modules.push_back(ModuleInfo());
186  else
187  Modules.resize(ID + 1);
188 
189  // Size/modification time for this module file at the time the
190  // global index was built.
191  Modules[ID].Size = Record[Idx++];
192  Modules[ID].ModTime = Record[Idx++];
193 
194  // File name.
195  unsigned NameLen = Record[Idx++];
196  Modules[ID].FileName.assign(Record.begin() + Idx,
197  Record.begin() + Idx + NameLen);
198  Idx += NameLen;
199 
200  // Dependencies
201  unsigned NumDeps = Record[Idx++];
202  Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
203  Record.begin() + Idx,
204  Record.begin() + Idx + NumDeps);
205  Idx += NumDeps;
206 
207  // Make sure we're at the end of the record.
208  assert(Idx == Record.size() && "More module info?");
209 
210  // Record this module as an unresolved module.
211  // FIXME: this doesn't work correctly for module names containing path
212  // separators.
213  StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
214  // Remove the -<hash of ModuleMapPath>
215  ModuleName = ModuleName.rsplit('-').first;
216  UnresolvedModules[ModuleName] = ID;
217  break;
218  }
219 
220  case IDENTIFIER_INDEX:
221  // Wire up the identifier index.
222  if (Record[0]) {
223  IdentifierIndex = IdentifierIndexTable::Create(
224  (const unsigned char *)Blob.data() + Record[0],
225  (const unsigned char *)Blob.data() + sizeof(uint32_t),
226  (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
227  }
228  break;
229  }
230  }
231 }
232 
234  delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
235 }
236 
237 std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
239  // Load the index file, if it's there.
240  llvm::SmallString<128> IndexPath;
241  IndexPath += Path;
242  llvm::sys::path::append(IndexPath, IndexFileName);
243 
244  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
245  llvm::MemoryBuffer::getFile(IndexPath.c_str());
246  if (!BufferOrErr)
247  return std::make_pair(nullptr, EC_NotFound);
248  std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
249 
250  /// The main bitstream cursor for the main block.
251  llvm::BitstreamCursor Cursor(*Buffer);
252 
253  // Sniff for the signature.
254  if (Cursor.Read(8) != 'B' ||
255  Cursor.Read(8) != 'C' ||
256  Cursor.Read(8) != 'G' ||
257  Cursor.Read(8) != 'I') {
258  return std::make_pair(nullptr, EC_IOError);
259  }
260 
261  return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor),
262  EC_None);
263 }
264 
265 void
267  ModuleFiles.clear();
268  for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
269  if (ModuleFile *MF = Modules[I].File)
270  ModuleFiles.push_back(MF);
271  }
272 }
273 
275  ModuleFile *File,
276  SmallVectorImpl<ModuleFile *> &Dependencies) {
277  // Look for information about this module file.
278  llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
279  = ModulesByFile.find(File);
280  if (Known == ModulesByFile.end())
281  return;
282 
283  // Record dependencies.
284  Dependencies.clear();
285  ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
286  for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
287  if (ModuleFile *MF = Modules[I].File)
288  Dependencies.push_back(MF);
289  }
290 }
291 
292 bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
293  Hits.clear();
294 
295  // If there's no identifier index, there is nothing we can do.
296  if (!IdentifierIndex)
297  return false;
298 
299  // Look into the identifier index.
300  ++NumIdentifierLookups;
301  IdentifierIndexTable &Table
302  = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
303  IdentifierIndexTable::iterator Known = Table.find(Name);
304  if (Known == Table.end()) {
305  return true;
306  }
307 
308  SmallVector<unsigned, 2> ModuleIDs = *Known;
309  for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
310  if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
311  Hits.insert(MF);
312  }
313 
314  ++NumIdentifierLookupHits;
315  return true;
316 }
317 
319  // Look for the module in the global module index based on the module name.
320  StringRef Name = File->ModuleName;
321  llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
322  if (Known == UnresolvedModules.end()) {
323  return true;
324  }
325 
326  // Rectify this module with the global module index.
327  ModuleInfo &Info = Modules[Known->second];
328 
329  // If the size and modification time match what we expected, record this
330  // module file.
331  bool Failed = true;
332  if (File->File->getSize() == Info.Size &&
333  File->File->getModificationTime() == Info.ModTime) {
334  Info.File = File;
335  ModulesByFile[File] = Known->second;
336 
337  Failed = false;
338  }
339 
340  // One way or another, we have resolved this module file.
341  UnresolvedModules.erase(Known);
342  return Failed;
343 }
344 
346  std::fprintf(stderr, "*** Global Module Index Statistics:\n");
347  if (NumIdentifierLookups) {
348  fprintf(stderr, " %u / %u identifier lookups succeeded (%f%%)\n",
349  NumIdentifierLookupHits, NumIdentifierLookups,
350  (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
351  }
352  std::fprintf(stderr, "\n");
353 }
354 
355 LLVM_DUMP_METHOD void GlobalModuleIndex::dump() {
356  llvm::errs() << "*** Global Module Index Dump:\n";
357  llvm::errs() << "Module files:\n";
358  for (auto &MI : Modules) {
359  llvm::errs() << "** " << MI.FileName << "\n";
360  if (MI.File)
361  MI.File->dump();
362  else
363  llvm::errs() << "\n";
364  }
365  llvm::errs() << "\n";
366 }
367 
368 //----------------------------------------------------------------------------//
369 // Global module index writer.
370 //----------------------------------------------------------------------------//
371 
372 namespace {
373  /// Provides information about a specific module file.
374  struct ModuleFileInfo {
375  /// The numberic ID for this module file.
376  unsigned ID;
377 
378  /// The set of modules on which this module depends. Each entry is
379  /// a module ID.
380  SmallVector<unsigned, 4> Dependencies;
381  ASTFileSignature Signature;
382  };
383 
384  struct ImportedModuleFileInfo {
385  off_t StoredSize;
386  time_t StoredModTime;
387  ASTFileSignature StoredSignature;
388  ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig)
389  : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
390  };
391 
392  /// Builder that generates the global module index file.
393  class GlobalModuleIndexBuilder {
394  FileManager &FileMgr;
395  const PCHContainerReader &PCHContainerRdr;
396 
397  /// Mapping from files to module file information.
398  typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
399 
400  /// Information about each of the known module files.
401  ModuleFilesMap ModuleFiles;
402 
403  /// Mapping from the imported module file to the imported
404  /// information.
405  typedef std::multimap<const FileEntry *, ImportedModuleFileInfo>
406  ImportedModuleFilesMap;
407 
408  /// Information about each importing of a module file.
409  ImportedModuleFilesMap ImportedModuleFiles;
410 
411  /// Mapping from identifiers to the list of module file IDs that
412  /// consider this identifier to be interesting.
413  typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
414 
415  /// A mapping from all interesting identifiers to the set of module
416  /// files in which those identifiers are considered interesting.
417  InterestingIdentifierMap InterestingIdentifiers;
418 
419  /// Write the block-info block for the global module index file.
420  void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
421 
422  /// Retrieve the module file information for the given file.
423  ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
424  llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
425  = ModuleFiles.find(File);
426  if (Known != ModuleFiles.end())
427  return Known->second;
428 
429  unsigned NewID = ModuleFiles.size();
430  ModuleFileInfo &Info = ModuleFiles[File];
431  Info.ID = NewID;
432  return Info;
433  }
434 
435  public:
436  explicit GlobalModuleIndexBuilder(
437  FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr)
438  : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
439 
440  /// Load the contents of the given module file into the builder.
441  ///
442  /// \returns true if an error occurred, false otherwise.
443  bool loadModuleFile(const FileEntry *File);
444 
445  /// Write the index to the given bitstream.
446  /// \returns true if an error occurred, false otherwise.
447  bool writeIndex(llvm::BitstreamWriter &Stream);
448  };
449 }
450 
451 static void emitBlockID(unsigned ID, const char *Name,
452  llvm::BitstreamWriter &Stream,
453  SmallVectorImpl<uint64_t> &Record) {
454  Record.clear();
455  Record.push_back(ID);
456  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
457 
458  // Emit the block name if present.
459  if (!Name || Name[0] == 0) return;
460  Record.clear();
461  while (*Name)
462  Record.push_back(*Name++);
463  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
464 }
465 
466 static void emitRecordID(unsigned ID, const char *Name,
467  llvm::BitstreamWriter &Stream,
468  SmallVectorImpl<uint64_t> &Record) {
469  Record.clear();
470  Record.push_back(ID);
471  while (*Name)
472  Record.push_back(*Name++);
473  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
474 }
475 
476 void
477 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
479  Stream.EnterBlockInfoBlock();
480 
481 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
482 #define RECORD(X) emitRecordID(X, #X, Stream, Record)
483  BLOCK(GLOBAL_INDEX_BLOCK);
484  RECORD(INDEX_METADATA);
485  RECORD(MODULE);
486  RECORD(IDENTIFIER_INDEX);
487 #undef RECORD
488 #undef BLOCK
489 
490  Stream.ExitBlock();
491 }
492 
493 namespace {
494  class InterestingASTIdentifierLookupTrait
496 
497  public:
498  /// The identifier and whether it is "interesting".
499  typedef std::pair<StringRef, bool> data_type;
500 
501  data_type ReadData(const internal_key_type& k,
502  const unsigned char* d,
503  unsigned DataLen) {
504  // The first bit indicates whether this identifier is interesting.
505  // That's all we care about.
506  using namespace llvm::support;
507  unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
508  bool IsInteresting = RawID & 0x01;
509  return std::make_pair(k, IsInteresting);
510  }
511  };
512 }
513 
514 bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
515  // Open the module file.
516 
517  auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true);
518  if (!Buffer) {
519  return true;
520  }
521 
522  // Initialize the input stream
523  llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer));
524 
525  // Sniff for the signature.
526  if (InStream.Read(8) != 'C' ||
527  InStream.Read(8) != 'P' ||
528  InStream.Read(8) != 'C' ||
529  InStream.Read(8) != 'H') {
530  return true;
531  }
532 
533  // Record this module file and assign it a unique ID (if it doesn't have
534  // one already).
535  unsigned ID = getModuleFileInfo(File).ID;
536 
537  // Search for the blocks and records we care about.
538  enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other;
539  bool Done = false;
540  while (!Done) {
541  llvm::BitstreamEntry Entry = InStream.advance();
542  switch (Entry.Kind) {
544  Done = true;
545  continue;
546 
547  case llvm::BitstreamEntry::Record:
548  // In the 'other' state, just skip the record. We don't care.
549  if (State == Other) {
550  InStream.skipRecord(Entry.ID);
551  continue;
552  }
553 
554  // Handle potentially-interesting records below.
555  break;
556 
557  case llvm::BitstreamEntry::SubBlock:
558  if (Entry.ID == CONTROL_BLOCK_ID) {
559  if (InStream.EnterSubBlock(CONTROL_BLOCK_ID))
560  return true;
561 
562  // Found the control block.
563  State = ControlBlock;
564  continue;
565  }
566 
567  if (Entry.ID == AST_BLOCK_ID) {
568  if (InStream.EnterSubBlock(AST_BLOCK_ID))
569  return true;
570 
571  // Found the AST block.
572  State = ASTBlock;
573  continue;
574  }
575 
576  if (Entry.ID == UNHASHED_CONTROL_BLOCK_ID) {
577  if (InStream.EnterSubBlock(UNHASHED_CONTROL_BLOCK_ID))
578  return true;
579 
580  // Found the Diagnostic Options block.
581  State = DiagnosticOptionsBlock;
582  continue;
583  }
584 
585  if (InStream.SkipBlock())
586  return true;
587 
588  continue;
589 
590  case llvm::BitstreamEntry::EndBlock:
591  State = Other;
592  continue;
593  }
594 
595  // Read the given record.
597  StringRef Blob;
598  unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);
599 
600  // Handle module dependencies.
601  if (State == ControlBlock && Code == IMPORTS) {
602  // Load each of the imported PCH files.
603  unsigned Idx = 0, N = Record.size();
604  while (Idx < N) {
605  // Read information about the AST file.
606 
607  // Skip the imported kind
608  ++Idx;
609 
610  // Skip the import location
611  ++Idx;
612 
613  // Load stored size/modification time.
614  off_t StoredSize = (off_t)Record[Idx++];
615  time_t StoredModTime = (time_t)Record[Idx++];
616 
617  // Skip the stored signature.
618  // FIXME: we could read the signature out of the import and validate it.
619  ASTFileSignature StoredSignature = {
620  {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
621  (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
622  (uint32_t)Record[Idx++]}}};
623 
624  // Skip the module name (currently this is only used for prebuilt
625  // modules while here we are only dealing with cached).
626  Idx += Record[Idx] + 1;
627 
628  // Retrieve the imported file name.
629  unsigned Length = Record[Idx++];
630  SmallString<128> ImportedFile(Record.begin() + Idx,
631  Record.begin() + Idx + Length);
632  Idx += Length;
633 
634  // Find the imported module file.
635  const FileEntry *DependsOnFile
636  = FileMgr.getFile(ImportedFile, /*openFile=*/false,
637  /*cacheFailure=*/false);
638 
639  if (!DependsOnFile)
640  return true;
641 
642  // Save the information in ImportedModuleFileInfo so we can verify after
643  // loading all pcms.
644  ImportedModuleFiles.insert(std::make_pair(
645  DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime,
646  StoredSignature)));
647 
648  // Record the dependency.
649  unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
650  getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
651  }
652 
653  continue;
654  }
655 
656  // Handle the identifier table
657  if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
658  typedef llvm::OnDiskIterableChainedHashTable<
659  InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
660  std::unique_ptr<InterestingIdentifierTable> Table(
662  (const unsigned char *)Blob.data() + Record[0],
663  (const unsigned char *)Blob.data() + sizeof(uint32_t),
664  (const unsigned char *)Blob.data()));
665  for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
666  DEnd = Table->data_end();
667  D != DEnd; ++D) {
668  std::pair<StringRef, bool> Ident = *D;
669  if (Ident.second)
670  InterestingIdentifiers[Ident.first].push_back(ID);
671  else
672  (void)InterestingIdentifiers[Ident.first];
673  }
674  }
675 
676  // Get Signature.
677  if (State == DiagnosticOptionsBlock && Code == SIGNATURE)
678  getModuleFileInfo(File).Signature = {
679  {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
680  (uint32_t)Record[3], (uint32_t)Record[4]}}};
681 
682  // We don't care about this record.
683  }
684 
685  return false;
686 }
687 
688 namespace {
689 
690 /// Trait used to generate the identifier index as an on-disk hash
691 /// table.
692 class IdentifierIndexWriterTrait {
693 public:
694  typedef StringRef key_type;
695  typedef StringRef key_type_ref;
696  typedef SmallVector<unsigned, 2> data_type;
697  typedef const SmallVector<unsigned, 2> &data_type_ref;
698  typedef unsigned hash_value_type;
699  typedef unsigned offset_type;
700 
701  static hash_value_type ComputeHash(key_type_ref Key) {
702  return llvm::djbHash(Key);
703  }
704 
705  std::pair<unsigned,unsigned>
706  EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
707  using namespace llvm::support;
708  endian::Writer LE(Out, little);
709  unsigned KeyLen = Key.size();
710  unsigned DataLen = Data.size() * 4;
711  LE.write<uint16_t>(KeyLen);
712  LE.write<uint16_t>(DataLen);
713  return std::make_pair(KeyLen, DataLen);
714  }
715 
716  void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
717  Out.write(Key.data(), KeyLen);
718  }
719 
720  void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
721  unsigned DataLen) {
722  using namespace llvm::support;
723  for (unsigned I = 0, N = Data.size(); I != N; ++I)
724  endian::write<uint32_t>(Out, Data[I], little);
725  }
726 };
727 
728 }
729 
730 bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
731  for (auto MapEntry : ImportedModuleFiles) {
732  auto *File = MapEntry.first;
733  ImportedModuleFileInfo &Info = MapEntry.second;
734  if (getModuleFileInfo(File).Signature) {
735  if (getModuleFileInfo(File).Signature != Info.StoredSignature)
736  // Verify Signature.
737  return true;
738  } else if (Info.StoredSize != File->getSize() ||
739  Info.StoredModTime != File->getModificationTime())
740  // Verify Size and ModTime.
741  return true;
742  }
743 
744  using namespace llvm;
745  llvm::TimeTraceScope TimeScope("Module WriteIndex", StringRef(""));
746 
747  // Emit the file header.
748  Stream.Emit((unsigned)'B', 8);
749  Stream.Emit((unsigned)'C', 8);
750  Stream.Emit((unsigned)'G', 8);
751  Stream.Emit((unsigned)'I', 8);
752 
753  // Write the block-info block, which describes the records in this bitcode
754  // file.
755  emitBlockInfoBlock(Stream);
756 
757  Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
758 
759  // Write the metadata.
761  Record.push_back(CurrentVersion);
762  Stream.EmitRecord(INDEX_METADATA, Record);
763 
764  // Write the set of known module files.
765  for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
766  MEnd = ModuleFiles.end();
767  M != MEnd; ++M) {
768  Record.clear();
769  Record.push_back(M->second.ID);
770  Record.push_back(M->first->getSize());
771  Record.push_back(M->first->getModificationTime());
772 
773  // File name
774  StringRef Name(M->first->getName());
775  Record.push_back(Name.size());
776  Record.append(Name.begin(), Name.end());
777 
778  // Dependencies
779  Record.push_back(M->second.Dependencies.size());
780  Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
781  Stream.EmitRecord(MODULE, Record);
782  }
783 
784  // Write the identifier -> module file mapping.
785  {
786  llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
787  IdentifierIndexWriterTrait Trait;
788 
789  // Populate the hash table.
790  for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
791  IEnd = InterestingIdentifiers.end();
792  I != IEnd; ++I) {
793  Generator.insert(I->first(), I->second, Trait);
794  }
795 
796  // Create the on-disk hash table in a buffer.
798  uint32_t BucketOffset;
799  {
800  using namespace llvm::support;
801  llvm::raw_svector_ostream Out(IdentifierTable);
802  // Make sure that no bucket is at offset 0
803  endian::write<uint32_t>(Out, 0, little);
804  BucketOffset = Generator.Emit(Out, Trait);
805  }
806 
807  // Create a blob abbreviation
808  auto Abbrev = std::make_shared<BitCodeAbbrev>();
809  Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
810  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
811  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
812  unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
813 
814  // Write the identifier table
815  uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
816  Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
817  }
818 
819  Stream.ExitBlock();
820  return false;
821 }
822 
825  const PCHContainerReader &PCHContainerRdr,
826  StringRef Path) {
827  llvm::SmallString<128> IndexPath;
828  IndexPath += Path;
829  llvm::sys::path::append(IndexPath, IndexFileName);
830 
831  // Coordinate building the global index file with other processes that might
832  // try to do the same.
833  llvm::LockFileManager Locked(IndexPath);
834  switch (Locked) {
835  case llvm::LockFileManager::LFS_Error:
836  return EC_IOError;
837 
838  case llvm::LockFileManager::LFS_Owned:
839  // We're responsible for building the index ourselves. Do so below.
840  break;
841 
842  case llvm::LockFileManager::LFS_Shared:
843  // Someone else is responsible for building the index. We don't care
844  // when they finish, so we're done.
845  return EC_Building;
846  }
847 
848  // The module index builder.
849  GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
850 
851  // Load each of the module files.
852  std::error_code EC;
853  for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
854  D != DEnd && !EC;
855  D.increment(EC)) {
856  // If this isn't a module file, we don't care.
857  if (llvm::sys::path::extension(D->path()) != ".pcm") {
858  // ... unless it's a .pcm.lock file, which indicates that someone is
859  // in the process of rebuilding a module. They'll rebuild the index
860  // at the end of that translation unit, so we don't have to.
861  if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
862  return EC_Building;
863 
864  continue;
865  }
866 
867  // If we can't find the module file, skip it.
868  const FileEntry *ModuleFile = FileMgr.getFile(D->path());
869  if (!ModuleFile)
870  continue;
871 
872  // Load this module file.
873  if (Builder.loadModuleFile(ModuleFile))
874  return EC_IOError;
875  }
876 
877  // The output buffer, into which the global index will be written.
878  SmallVector<char, 16> OutputBuffer;
879  {
880  llvm::BitstreamWriter OutputStream(OutputBuffer);
881  if (Builder.writeIndex(OutputStream))
882  return EC_IOError;
883  }
884 
885  // Write the global index file to a temporary file.
886  llvm::SmallString<128> IndexTmpPath;
887  int TmpFD;
888  if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
889  IndexTmpPath))
890  return EC_IOError;
891 
892  // Open the temporary global index file for output.
893  llvm::raw_fd_ostream Out(TmpFD, true);
894  if (Out.has_error())
895  return EC_IOError;
896 
897  // Write the index.
898  Out.write(OutputBuffer.data(), OutputBuffer.size());
899  Out.close();
900  if (Out.has_error())
901  return EC_IOError;
902 
903  // Remove the old index file. It isn't relevant any more.
904  llvm::sys::fs::remove(IndexPath);
905 
906  // Rename the newly-written index file to the proper name.
907  if (llvm::sys::fs::rename(IndexTmpPath, IndexPath)) {
908  // Rename failed; just remove the
909  llvm::sys::fs::remove(IndexTmpPath);
910  return EC_IOError;
911  }
912 
913  // We're done.
914  return EC_None;
915 }
916 
917 namespace {
918  class GlobalIndexIdentifierIterator : public IdentifierIterator {
919  /// The current position within the identifier lookup table.
920  IdentifierIndexTable::key_iterator Current;
921 
922  /// The end position within the identifier lookup table.
923  IdentifierIndexTable::key_iterator End;
924 
925  public:
926  explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
927  Current = Idx.key_begin();
928  End = Idx.key_end();
929  }
930 
931  StringRef Next() override {
932  if (Current == End)
933  return StringRef();
934 
935  StringRef Result = *Current;
936  ++Current;
937  return Result;
938  }
939  };
940 }
941 
943  IdentifierIndexTable &Table =
944  *static_cast<IdentifierIndexTable *>(IdentifierIndex);
945  return new GlobalIndexIdentifierIterator(Table);
946 }
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:116
Defines the clang::FileManager interface and associated types.
time_t getModificationTime() const
Definition: FileManager.h:89
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:29
std::string ModuleName
The name of the module.
Definition: Module.h:125
static const unsigned CurrentVersion
The global index file version.
A block with unhashed content.
Definition: ASTBitCodes.h:296
void printStats()
Print statistics to standard error.
static void emitRecordID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
LineState State
llvm::SmallPtrSet< ModuleFile *, 4 > HitSet
A set of module files in which we found a result.
static std::pair< GlobalModuleIndex *, ErrorCode > readIndex(llvm::StringRef Path)
Read a global index file for the given directory.
The signature of a module, which is a hash of the AST content.
Definition: Module.h:54
ErrorCode
An error code returned when trying to read an index.
This abstract interface provides operations for unwrapping containers for serialized ASTs (precompile...
const FileEntry * getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
bool lookupIdentifier(llvm::StringRef Name, HitSet &Hits)
Look for all of the module files with information about the given identifier, e.g., a global function, variable, or type with that name.
void dump()
Print debugging view to standard error.
SourceLocation End
static void emitBlockID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
IdentifierIterator * createIdentifierIterator() const
Returns an iterator for identifiers stored in the index table.
Implements an efficient mapping from strings to IdentifierInfo nodes.
Base class for the trait describing the on-disk hash table for the identifiers in an AST file...
Information about a module that has been loaded by the ASTReader.
Definition: Module.h:107
An iterator that walks over all of the known identifiers in the lookup table.
static const char *const IndexFileName
The name of the global index file.
Record code for the identifier table.
Definition: ASTBitCodes.h:441
IndexRecordTypes
Describes the record types in the index.
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
void getKnownModules(llvm::SmallVectorImpl< ModuleFile *> &ModuleFiles)
Retrieve the set of modules that have up-to-date indexes.
The AST block, which acts as a container around the full AST block.
Definition: ASTBitCodes.h:248
A global index for a set of module files, providing information about the identifiers within those mo...
unsigned ComputeHash(Selector Sel)
Definition: ASTCommon.cpp:246
Record code for the signature that identifiers this AST file.
Definition: ASTBitCodes.h:363
Record code for the list of other AST files imported by this AST file.
Definition: ASTBitCodes.h:307
Dataflow Directional Tag Classes.
off_t getSize() const
Definition: FileManager.h:86
#define BLOCK(X)
void getModuleDependencies(ModuleFile *File, llvm::SmallVectorImpl< ModuleFile *> &Dependencies)
Retrieve the set of module files on which the given module file directly depends. ...
Dump information about a module file.
#define RECORD(X)
static ErrorCode writeIndex(FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, llvm::StringRef Path)
Write a global index into the given.
The control block, which contains all of the information that needs to be validated prior to committi...
Definition: ASTBitCodes.h:274
const FileEntry * File
The file entry for the module file.
Definition: Module.h:163
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool loadedModuleFile(ModuleFile *File)
Note that the given module file has been loaded.