clang  10.0.0svn
GlobalModuleIndex.cpp
Go to the documentation of this file.
1 //===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the GlobalModuleIndex class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "ASTReaderInternals.h"
15 #include "clang/Lex/HeaderSearch.h"
20 #include "llvm/ADT/DenseMap.h"
21 #include "llvm/ADT/MapVector.h"
22 #include "llvm/ADT/SmallString.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/Bitstream/BitstreamReader.h"
25 #include "llvm/Bitstream/BitstreamWriter.h"
26 #include "llvm/Support/DJB.h"
27 #include "llvm/Support/FileSystem.h"
28 #include "llvm/Support/FileUtilities.h"
29 #include "llvm/Support/LockFileManager.h"
30 #include "llvm/Support/MemoryBuffer.h"
31 #include "llvm/Support/OnDiskHashTable.h"
32 #include "llvm/Support/Path.h"
33 #include "llvm/Support/TimeProfiler.h"
34 #include <cstdio>
35 using namespace clang;
36 using namespace serialization;
37 
38 //----------------------------------------------------------------------------//
39 // Shared constants
40 //----------------------------------------------------------------------------//
41 namespace {
42  enum {
43  /// The block containing the index.
44  GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
45  };
46 
47  /// Describes the record types in the index.
49  /// Contains version information and potentially other metadata,
50  /// used to determine if we can read this global index file.
51  INDEX_METADATA,
52  /// Describes a module, including its file name and dependencies.
53  MODULE,
54  /// The index for identifiers.
55  IDENTIFIER_INDEX
56  };
57 }
58 
59 /// The name of the global index file.
60 static const char * const IndexFileName = "modules.idx";
61 
62 /// The global index file version.
63 static const unsigned CurrentVersion = 1;
64 
65 //----------------------------------------------------------------------------//
66 // Global module index reader.
67 //----------------------------------------------------------------------------//
68 
69 namespace {
70 
71 /// Trait used to read the identifier index from the on-disk hash
72 /// table.
73 class IdentifierIndexReaderTrait {
74 public:
75  typedef StringRef external_key_type;
76  typedef StringRef internal_key_type;
77  typedef SmallVector<unsigned, 2> data_type;
78  typedef unsigned hash_value_type;
79  typedef unsigned offset_type;
80 
81  static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
82  return a == b;
83  }
84 
85  static hash_value_type ComputeHash(const internal_key_type& a) {
86  return llvm::djbHash(a);
87  }
88 
89  static std::pair<unsigned, unsigned>
90  ReadKeyDataLength(const unsigned char*& d) {
91  using namespace llvm::support;
92  unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
93  unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
94  return std::make_pair(KeyLen, DataLen);
95  }
96 
97  static const internal_key_type&
98  GetInternalKey(const external_key_type& x) { return x; }
99 
100  static const external_key_type&
101  GetExternalKey(const internal_key_type& x) { return x; }
102 
103  static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
104  return StringRef((const char *)d, n);
105  }
106 
107  static data_type ReadData(const internal_key_type& k,
108  const unsigned char* d,
109  unsigned DataLen) {
110  using namespace llvm::support;
111 
112  data_type Result;
113  while (DataLen > 0) {
114  unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
115  Result.push_back(ID);
116  DataLen -= 4;
117  }
118 
119  return Result;
120  }
121 };
122 
123 typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
124  IdentifierIndexTable;
125 
126 }
127 
128 GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer,
129  llvm::BitstreamCursor Cursor)
130  : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
131  NumIdentifierLookupHits() {
132  auto Fail = [&Buffer](llvm::Error &&Err) {
133  report_fatal_error("Module index '" + Buffer->getBufferIdentifier() +
134  "' failed: " + toString(std::move(Err)));
135  };
136 
137  llvm::TimeTraceScope TimeScope("Module LoadIndex", StringRef(""));
138  // Read the global index.
139  bool InGlobalIndexBlock = false;
140  bool Done = false;
141  while (!Done) {
142  llvm::BitstreamEntry Entry;
143  if (Expected<llvm::BitstreamEntry> Res = Cursor.advance())
144  Entry = Res.get();
145  else
146  Fail(Res.takeError());
147 
148  switch (Entry.Kind) {
150  return;
151 
152  case llvm::BitstreamEntry::EndBlock:
153  if (InGlobalIndexBlock) {
154  InGlobalIndexBlock = false;
155  Done = true;
156  continue;
157  }
158  return;
159 
160 
161  case llvm::BitstreamEntry::Record:
162  // Entries in the global index block are handled below.
163  if (InGlobalIndexBlock)
164  break;
165 
166  return;
167 
168  case llvm::BitstreamEntry::SubBlock:
169  if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
170  if (llvm::Error Err = Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
171  Fail(std::move(Err));
172  InGlobalIndexBlock = true;
173  } else if (llvm::Error Err = Cursor.SkipBlock())
174  Fail(std::move(Err));
175  continue;
176  }
177 
179  StringRef Blob;
180  Expected<unsigned> MaybeIndexRecord =
181  Cursor.readRecord(Entry.ID, Record, &Blob);
182  if (!MaybeIndexRecord)
183  Fail(MaybeIndexRecord.takeError());
184  IndexRecordTypes IndexRecord =
185  static_cast<IndexRecordTypes>(MaybeIndexRecord.get());
186  switch (IndexRecord) {
187  case INDEX_METADATA:
188  // Make sure that the version matches.
189  if (Record.size() < 1 || Record[0] != CurrentVersion)
190  return;
191  break;
192 
193  case MODULE: {
194  unsigned Idx = 0;
195  unsigned ID = Record[Idx++];
196 
197  // Make room for this module's information.
198  if (ID == Modules.size())
199  Modules.push_back(ModuleInfo());
200  else
201  Modules.resize(ID + 1);
202 
203  // Size/modification time for this module file at the time the
204  // global index was built.
205  Modules[ID].Size = Record[Idx++];
206  Modules[ID].ModTime = Record[Idx++];
207 
208  // File name.
209  unsigned NameLen = Record[Idx++];
210  Modules[ID].FileName.assign(Record.begin() + Idx,
211  Record.begin() + Idx + NameLen);
212  Idx += NameLen;
213 
214  // Dependencies
215  unsigned NumDeps = Record[Idx++];
216  Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
217  Record.begin() + Idx,
218  Record.begin() + Idx + NumDeps);
219  Idx += NumDeps;
220 
221  // Make sure we're at the end of the record.
222  assert(Idx == Record.size() && "More module info?");
223 
224  // Record this module as an unresolved module.
225  // FIXME: this doesn't work correctly for module names containing path
226  // separators.
227  StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
228  // Remove the -<hash of ModuleMapPath>
229  ModuleName = ModuleName.rsplit('-').first;
230  UnresolvedModules[ModuleName] = ID;
231  break;
232  }
233 
234  case IDENTIFIER_INDEX:
235  // Wire up the identifier index.
236  if (Record[0]) {
237  IdentifierIndex = IdentifierIndexTable::Create(
238  (const unsigned char *)Blob.data() + Record[0],
239  (const unsigned char *)Blob.data() + sizeof(uint32_t),
240  (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
241  }
242  break;
243  }
244  }
245 }
246 
248  delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
249 }
250 
251 std::pair<GlobalModuleIndex *, llvm::Error>
253  // Load the index file, if it's there.
254  llvm::SmallString<128> IndexPath;
255  IndexPath += Path;
256  llvm::sys::path::append(IndexPath, IndexFileName);
257 
258  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
259  llvm::MemoryBuffer::getFile(IndexPath.c_str());
260  if (!BufferOrErr)
261  return std::make_pair(nullptr,
262  llvm::errorCodeToError(BufferOrErr.getError()));
263  std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
264 
265  /// The main bitstream cursor for the main block.
266  llvm::BitstreamCursor Cursor(*Buffer);
267 
268  // Sniff for the signature.
269  for (unsigned char C : {'B', 'C', 'G', 'I'}) {
270  if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = Cursor.Read(8)) {
271  if (Res.get() != C)
272  return std::make_pair(
273  nullptr, llvm::createStringError(std::errc::illegal_byte_sequence,
274  "expected signature BCGI"));
275  } else
276  return std::make_pair(nullptr, Res.takeError());
277  }
278 
279  return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor),
280  llvm::Error::success());
281 }
282 
283 void
285  ModuleFiles.clear();
286  for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
287  if (ModuleFile *MF = Modules[I].File)
288  ModuleFiles.push_back(MF);
289  }
290 }
291 
293  ModuleFile *File,
294  SmallVectorImpl<ModuleFile *> &Dependencies) {
295  // Look for information about this module file.
296  llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
297  = ModulesByFile.find(File);
298  if (Known == ModulesByFile.end())
299  return;
300 
301  // Record dependencies.
302  Dependencies.clear();
303  ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
304  for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
305  if (ModuleFile *MF = Modules[I].File)
306  Dependencies.push_back(MF);
307  }
308 }
309 
310 bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
311  Hits.clear();
312 
313  // If there's no identifier index, there is nothing we can do.
314  if (!IdentifierIndex)
315  return false;
316 
317  // Look into the identifier index.
318  ++NumIdentifierLookups;
319  IdentifierIndexTable &Table
320  = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
321  IdentifierIndexTable::iterator Known = Table.find(Name);
322  if (Known == Table.end()) {
323  return true;
324  }
325 
326  SmallVector<unsigned, 2> ModuleIDs = *Known;
327  for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
328  if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
329  Hits.insert(MF);
330  }
331 
332  ++NumIdentifierLookupHits;
333  return true;
334 }
335 
337  // Look for the module in the global module index based on the module name.
338  StringRef Name = File->ModuleName;
339  llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
340  if (Known == UnresolvedModules.end()) {
341  return true;
342  }
343 
344  // Rectify this module with the global module index.
345  ModuleInfo &Info = Modules[Known->second];
346 
347  // If the size and modification time match what we expected, record this
348  // module file.
349  bool Failed = true;
350  if (File->File->getSize() == Info.Size &&
351  File->File->getModificationTime() == Info.ModTime) {
352  Info.File = File;
353  ModulesByFile[File] = Known->second;
354 
355  Failed = false;
356  }
357 
358  // One way or another, we have resolved this module file.
359  UnresolvedModules.erase(Known);
360  return Failed;
361 }
362 
364  std::fprintf(stderr, "*** Global Module Index Statistics:\n");
365  if (NumIdentifierLookups) {
366  fprintf(stderr, " %u / %u identifier lookups succeeded (%f%%)\n",
367  NumIdentifierLookupHits, NumIdentifierLookups,
368  (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
369  }
370  std::fprintf(stderr, "\n");
371 }
372 
373 LLVM_DUMP_METHOD void GlobalModuleIndex::dump() {
374  llvm::errs() << "*** Global Module Index Dump:\n";
375  llvm::errs() << "Module files:\n";
376  for (auto &MI : Modules) {
377  llvm::errs() << "** " << MI.FileName << "\n";
378  if (MI.File)
379  MI.File->dump();
380  else
381  llvm::errs() << "\n";
382  }
383  llvm::errs() << "\n";
384 }
385 
386 //----------------------------------------------------------------------------//
387 // Global module index writer.
388 //----------------------------------------------------------------------------//
389 
390 namespace {
391  /// Provides information about a specific module file.
392  struct ModuleFileInfo {
393  /// The numberic ID for this module file.
394  unsigned ID;
395 
396  /// The set of modules on which this module depends. Each entry is
397  /// a module ID.
398  SmallVector<unsigned, 4> Dependencies;
399  ASTFileSignature Signature;
400  };
401 
402  struct ImportedModuleFileInfo {
403  off_t StoredSize;
404  time_t StoredModTime;
405  ASTFileSignature StoredSignature;
406  ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig)
407  : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
408  };
409 
410  /// Builder that generates the global module index file.
411  class GlobalModuleIndexBuilder {
412  FileManager &FileMgr;
413  const PCHContainerReader &PCHContainerRdr;
414 
415  /// Mapping from files to module file information.
416  typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
417 
418  /// Information about each of the known module files.
419  ModuleFilesMap ModuleFiles;
420 
421  /// Mapping from the imported module file to the imported
422  /// information.
423  typedef std::multimap<const FileEntry *, ImportedModuleFileInfo>
424  ImportedModuleFilesMap;
425 
426  /// Information about each importing of a module file.
427  ImportedModuleFilesMap ImportedModuleFiles;
428 
429  /// Mapping from identifiers to the list of module file IDs that
430  /// consider this identifier to be interesting.
431  typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
432 
433  /// A mapping from all interesting identifiers to the set of module
434  /// files in which those identifiers are considered interesting.
435  InterestingIdentifierMap InterestingIdentifiers;
436 
437  /// Write the block-info block for the global module index file.
438  void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
439 
440  /// Retrieve the module file information for the given file.
441  ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
442  llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
443  = ModuleFiles.find(File);
444  if (Known != ModuleFiles.end())
445  return Known->second;
446 
447  unsigned NewID = ModuleFiles.size();
448  ModuleFileInfo &Info = ModuleFiles[File];
449  Info.ID = NewID;
450  return Info;
451  }
452 
453  public:
454  explicit GlobalModuleIndexBuilder(
455  FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr)
456  : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
457 
458  /// Load the contents of the given module file into the builder.
459  llvm::Error loadModuleFile(const FileEntry *File);
460 
461  /// Write the index to the given bitstream.
462  /// \returns true if an error occurred, false otherwise.
463  bool writeIndex(llvm::BitstreamWriter &Stream);
464  };
465 }
466 
467 static void emitBlockID(unsigned ID, const char *Name,
468  llvm::BitstreamWriter &Stream,
469  SmallVectorImpl<uint64_t> &Record) {
470  Record.clear();
471  Record.push_back(ID);
472  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
473 
474  // Emit the block name if present.
475  if (!Name || Name[0] == 0) return;
476  Record.clear();
477  while (*Name)
478  Record.push_back(*Name++);
479  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
480 }
481 
482 static void emitRecordID(unsigned ID, const char *Name,
483  llvm::BitstreamWriter &Stream,
484  SmallVectorImpl<uint64_t> &Record) {
485  Record.clear();
486  Record.push_back(ID);
487  while (*Name)
488  Record.push_back(*Name++);
489  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
490 }
491 
492 void
493 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
495  Stream.EnterBlockInfoBlock();
496 
497 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
498 #define RECORD(X) emitRecordID(X, #X, Stream, Record)
499  BLOCK(GLOBAL_INDEX_BLOCK);
500  RECORD(INDEX_METADATA);
501  RECORD(MODULE);
502  RECORD(IDENTIFIER_INDEX);
503 #undef RECORD
504 #undef BLOCK
505 
506  Stream.ExitBlock();
507 }
508 
509 namespace {
510  class InterestingASTIdentifierLookupTrait
512 
513  public:
514  /// The identifier and whether it is "interesting".
515  typedef std::pair<StringRef, bool> data_type;
516 
517  data_type ReadData(const internal_key_type& k,
518  const unsigned char* d,
519  unsigned DataLen) {
520  // The first bit indicates whether this identifier is interesting.
521  // That's all we care about.
522  using namespace llvm::support;
523  unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
524  bool IsInteresting = RawID & 0x01;
525  return std::make_pair(k, IsInteresting);
526  }
527  };
528 }
529 
530 llvm::Error GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
531  // Open the module file.
532 
533  auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true);
534  if (!Buffer)
535  return llvm::createStringError(Buffer.getError(),
536  "failed getting buffer for module file");
537 
538  // Initialize the input stream
539  llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer));
540 
541  // Sniff for the signature.
542  for (unsigned char C : {'C', 'P', 'C', 'H'})
543  if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = InStream.Read(8)) {
544  if (Res.get() != C)
545  return llvm::createStringError(std::errc::illegal_byte_sequence,
546  "expected signature CPCH");
547  } else
548  return Res.takeError();
549 
550  // Record this module file and assign it a unique ID (if it doesn't have
551  // one already).
552  unsigned ID = getModuleFileInfo(File).ID;
553 
554  // Search for the blocks and records we care about.
555  enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other;
556  bool Done = false;
557  while (!Done) {
558  Expected<llvm::BitstreamEntry> MaybeEntry = InStream.advance();
559  if (!MaybeEntry)
560  return MaybeEntry.takeError();
561  llvm::BitstreamEntry Entry = MaybeEntry.get();
562 
563  switch (Entry.Kind) {
565  Done = true;
566  continue;
567 
568  case llvm::BitstreamEntry::Record:
569  // In the 'other' state, just skip the record. We don't care.
570  if (State == Other) {
571  if (llvm::Expected<unsigned> Skipped = InStream.skipRecord(Entry.ID))
572  continue;
573  else
574  return Skipped.takeError();
575  }
576 
577  // Handle potentially-interesting records below.
578  break;
579 
580  case llvm::BitstreamEntry::SubBlock:
581  if (Entry.ID == CONTROL_BLOCK_ID) {
582  if (llvm::Error Err = InStream.EnterSubBlock(CONTROL_BLOCK_ID))
583  return Err;
584 
585  // Found the control block.
586  State = ControlBlock;
587  continue;
588  }
589 
590  if (Entry.ID == AST_BLOCK_ID) {
591  if (llvm::Error Err = InStream.EnterSubBlock(AST_BLOCK_ID))
592  return Err;
593 
594  // Found the AST block.
595  State = ASTBlock;
596  continue;
597  }
598 
599  if (Entry.ID == UNHASHED_CONTROL_BLOCK_ID) {
600  if (llvm::Error Err = InStream.EnterSubBlock(UNHASHED_CONTROL_BLOCK_ID))
601  return Err;
602 
603  // Found the Diagnostic Options block.
604  State = DiagnosticOptionsBlock;
605  continue;
606  }
607 
608  if (llvm::Error Err = InStream.SkipBlock())
609  return Err;
610 
611  continue;
612 
613  case llvm::BitstreamEntry::EndBlock:
614  State = Other;
615  continue;
616  }
617 
618  // Read the given record.
620  StringRef Blob;
621  Expected<unsigned> MaybeCode = InStream.readRecord(Entry.ID, Record, &Blob);
622  if (!MaybeCode)
623  return MaybeCode.takeError();
624  unsigned Code = MaybeCode.get();
625 
626  // Handle module dependencies.
627  if (State == ControlBlock && Code == IMPORTS) {
628  // Load each of the imported PCH files.
629  unsigned Idx = 0, N = Record.size();
630  while (Idx < N) {
631  // Read information about the AST file.
632 
633  // Skip the imported kind
634  ++Idx;
635 
636  // Skip the import location
637  ++Idx;
638 
639  // Load stored size/modification time.
640  off_t StoredSize = (off_t)Record[Idx++];
641  time_t StoredModTime = (time_t)Record[Idx++];
642 
643  // Skip the stored signature.
644  // FIXME: we could read the signature out of the import and validate it.
645  ASTFileSignature StoredSignature = {
646  {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
647  (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
648  (uint32_t)Record[Idx++]}}};
649 
650  // Skip the module name (currently this is only used for prebuilt
651  // modules while here we are only dealing with cached).
652  Idx += Record[Idx] + 1;
653 
654  // Retrieve the imported file name.
655  unsigned Length = Record[Idx++];
656  SmallString<128> ImportedFile(Record.begin() + Idx,
657  Record.begin() + Idx + Length);
658  Idx += Length;
659 
660  // Find the imported module file.
661  auto DependsOnFile
662  = FileMgr.getFile(ImportedFile, /*OpenFile=*/false,
663  /*CacheFailure=*/false);
664 
665  if (!DependsOnFile)
666  return llvm::createStringError(std::errc::bad_file_descriptor,
667  "imported file \"%s\" not found",
668  ImportedFile.c_str());
669 
670  // Save the information in ImportedModuleFileInfo so we can verify after
671  // loading all pcms.
672  ImportedModuleFiles.insert(std::make_pair(
673  *DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime,
674  StoredSignature)));
675 
676  // Record the dependency.
677  unsigned DependsOnID = getModuleFileInfo(*DependsOnFile).ID;
678  getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
679  }
680 
681  continue;
682  }
683 
684  // Handle the identifier table
685  if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
686  typedef llvm::OnDiskIterableChainedHashTable<
687  InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
688  std::unique_ptr<InterestingIdentifierTable> Table(
690  (const unsigned char *)Blob.data() + Record[0],
691  (const unsigned char *)Blob.data() + sizeof(uint32_t),
692  (const unsigned char *)Blob.data()));
693  for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
694  DEnd = Table->data_end();
695  D != DEnd; ++D) {
696  std::pair<StringRef, bool> Ident = *D;
697  if (Ident.second)
698  InterestingIdentifiers[Ident.first].push_back(ID);
699  else
700  (void)InterestingIdentifiers[Ident.first];
701  }
702  }
703 
704  // Get Signature.
705  if (State == DiagnosticOptionsBlock && Code == SIGNATURE)
706  getModuleFileInfo(File).Signature = {
707  {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
708  (uint32_t)Record[3], (uint32_t)Record[4]}}};
709 
710  // We don't care about this record.
711  }
712 
713  return llvm::Error::success();
714 }
715 
716 namespace {
717 
718 /// Trait used to generate the identifier index as an on-disk hash
719 /// table.
720 class IdentifierIndexWriterTrait {
721 public:
722  typedef StringRef key_type;
723  typedef StringRef key_type_ref;
724  typedef SmallVector<unsigned, 2> data_type;
725  typedef const SmallVector<unsigned, 2> &data_type_ref;
726  typedef unsigned hash_value_type;
727  typedef unsigned offset_type;
728 
729  static hash_value_type ComputeHash(key_type_ref Key) {
730  return llvm::djbHash(Key);
731  }
732 
733  std::pair<unsigned,unsigned>
734  EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
735  using namespace llvm::support;
736  endian::Writer LE(Out, little);
737  unsigned KeyLen = Key.size();
738  unsigned DataLen = Data.size() * 4;
739  LE.write<uint16_t>(KeyLen);
740  LE.write<uint16_t>(DataLen);
741  return std::make_pair(KeyLen, DataLen);
742  }
743 
744  void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
745  Out.write(Key.data(), KeyLen);
746  }
747 
748  void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
749  unsigned DataLen) {
750  using namespace llvm::support;
751  for (unsigned I = 0, N = Data.size(); I != N; ++I)
752  endian::write<uint32_t>(Out, Data[I], little);
753  }
754 };
755 
756 }
757 
758 bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
759  for (auto MapEntry : ImportedModuleFiles) {
760  auto *File = MapEntry.first;
761  ImportedModuleFileInfo &Info = MapEntry.second;
762  if (getModuleFileInfo(File).Signature) {
763  if (getModuleFileInfo(File).Signature != Info.StoredSignature)
764  // Verify Signature.
765  return true;
766  } else if (Info.StoredSize != File->getSize() ||
767  Info.StoredModTime != File->getModificationTime())
768  // Verify Size and ModTime.
769  return true;
770  }
771 
772  using namespace llvm;
773  llvm::TimeTraceScope TimeScope("Module WriteIndex", StringRef(""));
774 
775  // Emit the file header.
776  Stream.Emit((unsigned)'B', 8);
777  Stream.Emit((unsigned)'C', 8);
778  Stream.Emit((unsigned)'G', 8);
779  Stream.Emit((unsigned)'I', 8);
780 
781  // Write the block-info block, which describes the records in this bitcode
782  // file.
783  emitBlockInfoBlock(Stream);
784 
785  Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
786 
787  // Write the metadata.
789  Record.push_back(CurrentVersion);
790  Stream.EmitRecord(INDEX_METADATA, Record);
791 
792  // Write the set of known module files.
793  for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
794  MEnd = ModuleFiles.end();
795  M != MEnd; ++M) {
796  Record.clear();
797  Record.push_back(M->second.ID);
798  Record.push_back(M->first->getSize());
799  Record.push_back(M->first->getModificationTime());
800 
801  // File name
802  StringRef Name(M->first->getName());
803  Record.push_back(Name.size());
804  Record.append(Name.begin(), Name.end());
805 
806  // Dependencies
807  Record.push_back(M->second.Dependencies.size());
808  Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
809  Stream.EmitRecord(MODULE, Record);
810  }
811 
812  // Write the identifier -> module file mapping.
813  {
814  llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
815  IdentifierIndexWriterTrait Trait;
816 
817  // Populate the hash table.
818  for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
819  IEnd = InterestingIdentifiers.end();
820  I != IEnd; ++I) {
821  Generator.insert(I->first(), I->second, Trait);
822  }
823 
824  // Create the on-disk hash table in a buffer.
826  uint32_t BucketOffset;
827  {
828  using namespace llvm::support;
829  llvm::raw_svector_ostream Out(IdentifierTable);
830  // Make sure that no bucket is at offset 0
831  endian::write<uint32_t>(Out, 0, little);
832  BucketOffset = Generator.Emit(Out, Trait);
833  }
834 
835  // Create a blob abbreviation
836  auto Abbrev = std::make_shared<BitCodeAbbrev>();
837  Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
838  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
839  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
840  unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
841 
842  // Write the identifier table
843  uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
844  Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
845  }
846 
847  Stream.ExitBlock();
848  return false;
849 }
850 
853  const PCHContainerReader &PCHContainerRdr,
854  StringRef Path) {
855  llvm::SmallString<128> IndexPath;
856  IndexPath += Path;
857  llvm::sys::path::append(IndexPath, IndexFileName);
858 
859  // Coordinate building the global index file with other processes that might
860  // try to do the same.
861  llvm::LockFileManager Locked(IndexPath);
862  switch (Locked) {
863  case llvm::LockFileManager::LFS_Error:
864  return llvm::createStringError(std::errc::io_error, "LFS error");
865 
866  case llvm::LockFileManager::LFS_Owned:
867  // We're responsible for building the index ourselves. Do so below.
868  break;
869 
870  case llvm::LockFileManager::LFS_Shared:
871  // Someone else is responsible for building the index. We don't care
872  // when they finish, so we're done.
873  return llvm::createStringError(std::errc::device_or_resource_busy,
874  "someone else is building the index");
875  }
876 
877  // The module index builder.
878  GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
879 
880  // Load each of the module files.
881  std::error_code EC;
882  for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
883  D != DEnd && !EC;
884  D.increment(EC)) {
885  // If this isn't a module file, we don't care.
886  if (llvm::sys::path::extension(D->path()) != ".pcm") {
887  // ... unless it's a .pcm.lock file, which indicates that someone is
888  // in the process of rebuilding a module. They'll rebuild the index
889  // at the end of that translation unit, so we don't have to.
890  if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
891  return llvm::createStringError(std::errc::device_or_resource_busy,
892  "someone else is building the index");
893 
894  continue;
895  }
896 
897  // If we can't find the module file, skip it.
898  auto ModuleFile = FileMgr.getFile(D->path());
899  if (!ModuleFile)
900  continue;
901 
902  // Load this module file.
903  if (llvm::Error Err = Builder.loadModuleFile(*ModuleFile))
904  return Err;
905  }
906 
907  // The output buffer, into which the global index will be written.
908  SmallVector<char, 16> OutputBuffer;
909  {
910  llvm::BitstreamWriter OutputStream(OutputBuffer);
911  if (Builder.writeIndex(OutputStream))
912  return llvm::createStringError(std::errc::io_error,
913  "failed writing index");
914  }
915 
916  return llvm::writeFileAtomically(
917  (IndexPath + "-%%%%%%%%").str(), IndexPath,
918  llvm::StringRef(OutputBuffer.data(), OutputBuffer.size()));
919 }
920 
921 namespace {
922  class GlobalIndexIdentifierIterator : public IdentifierIterator {
923  /// The current position within the identifier lookup table.
924  IdentifierIndexTable::key_iterator Current;
925 
926  /// The end position within the identifier lookup table.
927  IdentifierIndexTable::key_iterator End;
928 
929  public:
930  explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
931  Current = Idx.key_begin();
932  End = Idx.key_end();
933  }
934 
935  StringRef Next() override {
936  if (Current == End)
937  return StringRef();
938 
939  StringRef Result = *Current;
940  ++Current;
941  return Result;
942  }
943  };
944 }
945 
947  IdentifierIndexTable &Table =
948  *static_cast<IdentifierIndexTable *>(IdentifierIndex);
949  return new GlobalIndexIdentifierIterator(Table);
950 }
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:171
Defines the clang::FileManager interface and associated types.
time_t getModificationTime() const
Definition: FileManager.h:108
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
Definition: Dominators.h:30
std::string ModuleName
The name of the module.
Definition: Module.h:125
static const unsigned CurrentVersion
The global index file version.
A block with unhashed content.
Definition: ASTBitCodes.h:296
void printStats()
Print statistics to standard error.
static void emitRecordID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
LineState State
static llvm::Error writeIndex(FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, llvm::StringRef Path)
Write a global index into the given.
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
llvm::SmallPtrSet< ModuleFile *, 4 > HitSet
A set of module files in which we found a result.
The signature of a module, which is a hash of the AST content.
Definition: Module.h:54
llvm::Error Error
This abstract interface provides operations for unwrapping containers for serialized ASTs (precompile...
bool lookupIdentifier(llvm::StringRef Name, HitSet &Hits)
Look for all of the module files with information about the given identifier, e.g., a global function, variable, or type with that name.
void dump()
Print debugging view to standard error.
SourceLocation End
static void emitBlockID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
IdentifierIterator * createIdentifierIterator() const
Returns an iterator for identifiers stored in the index table.
Implements an efficient mapping from strings to IdentifierInfo nodes.
Base class for the trait describing the on-disk hash table for the identifiers in an AST file...
Information about a module that has been loaded by the ASTReader.
Definition: Module.h:107
An iterator that walks over all of the known identifiers in the lookup table.
static const char *const IndexFileName
The name of the global index file.
Record code for the identifier table.
Definition: ASTBitCodes.h:444
IndexRecordTypes
Describes the record types in the index.
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:78
static std::pair< GlobalModuleIndex *, llvm::Error > readIndex(llvm::StringRef Path)
Read a global index file for the given directory.
void getKnownModules(llvm::SmallVectorImpl< ModuleFile *> &ModuleFiles)
Retrieve the set of modules that have up-to-date indexes.
The AST block, which acts as a container around the full AST block.
Definition: ASTBitCodes.h:248
A global index for a set of module files, providing information about the identifiers within those mo...
unsigned ComputeHash(Selector Sel)
Definition: ASTCommon.cpp:251
Record code for the signature that identifiers this AST file.
Definition: ASTBitCodes.h:363
Record code for the list of other AST files imported by this AST file.
Definition: ASTBitCodes.h:307
Dataflow Directional Tag Classes.
off_t getSize() const
Definition: FileManager.h:105
#define BLOCK(X)
llvm::ErrorOr< const FileEntry * > getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
void getModuleDependencies(ModuleFile *File, llvm::SmallVectorImpl< ModuleFile *> &Dependencies)
Retrieve the set of module files on which the given module file directly depends. ...
bool LE(InterpState &S, CodePtr OpPC)
Definition: Interp.h:240
Dump information about a module file.
#define RECORD(X)
The control block, which contains all of the information that needs to be validated prior to committi...
Definition: ASTBitCodes.h:274
const FileEntry * File
The file entry for the module file.
Definition: Module.h:163
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool loadedModuleFile(ModuleFile *File)
Note that the given module file has been loaded.