clang  8.0.0svn
CacheTokens.cpp
Go to the documentation of this file.
1 //===--- CacheTokens.cpp - Caching of lexer tokens for PTH support --------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This provides a possible implementation of PTH support for Clang that is
11 // based on caching lexed tokens and identifiers.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "clang/Basic/Diagnostic.h"
20 #include "clang/Frontend/Utils.h"
21 #include "clang/Lex/Lexer.h"
22 #include "clang/Lex/PTHManager.h"
23 #include "clang/Lex/Preprocessor.h"
24 #include "llvm/ADT/StringMap.h"
25 #include "llvm/Support/DJB.h"
26 #include "llvm/Support/EndianStream.h"
27 #include "llvm/Support/FileSystem.h"
28 #include "llvm/Support/MemoryBuffer.h"
29 #include "llvm/Support/OnDiskHashTable.h"
30 #include "llvm/Support/Path.h"
31 
32 // FIXME: put this somewhere else?
33 #ifndef S_ISDIR
34 #define S_ISDIR(x) (((x)&_S_IFDIR)!=0)
35 #endif
36 
37 using namespace clang;
38 
39 //===----------------------------------------------------------------------===//
40 // PTH-specific stuff.
41 //===----------------------------------------------------------------------===//
42 
43 typedef uint32_t Offset;
44 
45 namespace {
46 class PTHEntry {
47  Offset TokenData, PPCondData;
48 
49 public:
50  PTHEntry() {}
51 
52  PTHEntry(Offset td, Offset ppcd)
53  : TokenData(td), PPCondData(ppcd) {}
54 
55  Offset getTokenOffset() const { return TokenData; }
56  Offset getPPCondTableOffset() const { return PPCondData; }
57 };
58 
59 
60 class PTHEntryKeyVariant {
61  union {
62  const FileEntry *FE;
63  // FIXME: Use "StringRef Path;" when MSVC 2013 is dropped.
64  const char *PathPtr;
65  };
66  size_t PathSize;
67  enum { IsFE = 0x1, IsDE = 0x2, IsNoExist = 0x0 } Kind;
68  FileData *Data;
69 
70 public:
71  PTHEntryKeyVariant(const FileEntry *fe) : FE(fe), Kind(IsFE), Data(nullptr) {}
72 
73  PTHEntryKeyVariant(FileData *Data, StringRef Path)
74  : PathPtr(Path.data()), PathSize(Path.size()), Kind(IsDE),
75  Data(new FileData(*Data)) {}
76 
77  explicit PTHEntryKeyVariant(StringRef Path)
78  : PathPtr(Path.data()), PathSize(Path.size()), Kind(IsNoExist),
79  Data(nullptr) {}
80 
81  bool isFile() const { return Kind == IsFE; }
82 
83  StringRef getString() const {
84  return Kind == IsFE ? FE->getName() : StringRef(PathPtr, PathSize);
85  }
86 
87  unsigned getKind() const { return (unsigned) Kind; }
88 
89  void EmitData(raw_ostream& Out) {
90  using namespace llvm::support;
91  endian::Writer LE(Out, little);
92  switch (Kind) {
93  case IsFE: {
94  // Emit stat information.
95  llvm::sys::fs::UniqueID UID = FE->getUniqueID();
96  LE.write<uint64_t>(UID.getFile());
97  LE.write<uint64_t>(UID.getDevice());
98  LE.write<uint64_t>(FE->getModificationTime());
99  LE.write<uint64_t>(FE->getSize());
100  } break;
101  case IsDE:
102  // Emit stat information.
103  LE.write<uint64_t>(Data->UniqueID.getFile());
104  LE.write<uint64_t>(Data->UniqueID.getDevice());
105  LE.write<uint64_t>(Data->ModTime);
106  LE.write<uint64_t>(Data->Size);
107  delete Data;
108  break;
109  default:
110  break;
111  }
112  }
113 
114  unsigned getRepresentationLength() const {
115  return Kind == IsNoExist ? 0 : 4 * 8;
116  }
117 };
118 
119 class FileEntryPTHEntryInfo {
120 public:
121  typedef PTHEntryKeyVariant key_type;
122  typedef key_type key_type_ref;
123 
124  typedef PTHEntry data_type;
125  typedef const PTHEntry& data_type_ref;
126 
127  typedef unsigned hash_value_type;
128  typedef unsigned offset_type;
129 
130  static hash_value_type ComputeHash(PTHEntryKeyVariant V) {
131  return llvm::djbHash(V.getString());
132  }
133 
134  static std::pair<unsigned,unsigned>
135  EmitKeyDataLength(raw_ostream& Out, PTHEntryKeyVariant V,
136  const PTHEntry& E) {
137  using namespace llvm::support;
138  endian::Writer LE(Out, little);
139 
140  unsigned n = V.getString().size() + 1 + 1;
141  LE.write<uint16_t>(n);
142 
143  unsigned m = V.getRepresentationLength() + (V.isFile() ? 4 + 4 : 0);
144  LE.write<uint8_t>(m);
145 
146  return std::make_pair(n, m);
147  }
148 
149  static void EmitKey(raw_ostream& Out, PTHEntryKeyVariant V, unsigned n){
150  using namespace llvm::support;
151  // Emit the entry kind.
152  Out << char(V.getKind());
153  // Emit the string.
154  Out.write(V.getString().data(), n - 1);
155  }
156 
157  static void EmitData(raw_ostream& Out, PTHEntryKeyVariant V,
158  const PTHEntry& E, unsigned) {
159  using namespace llvm::support;
160  endian::Writer LE(Out, little);
161 
162  // For file entries emit the offsets into the PTH file for token data
163  // and the preprocessor blocks table.
164  if (V.isFile()) {
165  LE.write<uint32_t>(E.getTokenOffset());
166  LE.write<uint32_t>(E.getPPCondTableOffset());
167  }
168 
169  // Emit any other data associated with the key (i.e., stat information).
170  V.EmitData(Out);
171  }
172 };
173 
174 class OffsetOpt {
175  bool valid;
176  Offset off;
177 public:
178  OffsetOpt() : valid(false) {}
179  bool hasOffset() const { return valid; }
180  Offset getOffset() const { assert(valid); return off; }
181  void setOffset(Offset o) { off = o; valid = true; }
182 };
183 } // end anonymous namespace
184 
185 typedef llvm::OnDiskChainedHashTableGenerator<FileEntryPTHEntryInfo> PTHMap;
186 
187 namespace {
188 class PTHWriter {
189  typedef llvm::DenseMap<const IdentifierInfo*,uint32_t> IDMap;
190  typedef llvm::StringMap<OffsetOpt, llvm::BumpPtrAllocator> CachedStrsTy;
191 
192  raw_pwrite_stream &Out;
193  Preprocessor& PP;
194  IDMap IM;
195  std::vector<llvm::StringMapEntry<OffsetOpt>*> StrEntries;
196  PTHMap PM;
197  CachedStrsTy CachedStrs;
198  uint32_t idcount;
199  Offset CurStrOffset;
200 
201  //// Get the persistent id for the given IdentifierInfo*.
202  uint32_t ResolveID(const IdentifierInfo* II);
203 
204  /// Emit a token to the PTH file.
205  void EmitToken(const Token& T);
206 
207  void Emit8(uint32_t V) {
208  Out << char(V);
209  }
210 
211  void Emit16(uint32_t V) {
212  using namespace llvm::support;
213  endian::write<uint16_t>(Out, V, little);
214  }
215 
216  void Emit32(uint32_t V) {
217  using namespace llvm::support;
218  endian::write<uint32_t>(Out, V, little);
219  }
220 
221  void EmitBuf(const char *Ptr, unsigned NumBytes) {
222  Out.write(Ptr, NumBytes);
223  }
224 
225  void EmitString(StringRef V) {
226  using namespace llvm::support;
227  endian::write<uint16_t>(Out, V.size(), little);
228  EmitBuf(V.data(), V.size());
229  }
230 
231  /// EmitIdentifierTable - Emits two tables to the PTH file. The first is
232  /// a hashtable mapping from identifier strings to persistent IDs.
233  /// The second is a straight table mapping from persistent IDs to string data
234  /// (the keys of the first table).
235  std::pair<Offset, Offset> EmitIdentifierTable();
236 
237  /// EmitFileTable - Emit a table mapping from file name strings to PTH
238  /// token data.
239  Offset EmitFileTable() { return PM.Emit(Out); }
240 
241  PTHEntry LexTokens(Lexer& L);
242  Offset EmitCachedSpellings();
243 
244 public:
245  PTHWriter(raw_pwrite_stream &out, Preprocessor &pp)
246  : Out(out), PP(pp), idcount(0), CurStrOffset(0) {}
247 
248  PTHMap &getPM() { return PM; }
249  void GeneratePTH(StringRef MainFile);
250 };
251 } // end anonymous namespace
252 
253 uint32_t PTHWriter::ResolveID(const IdentifierInfo* II) {
254  // Null IdentifierInfo's map to the persistent ID 0.
255  if (!II)
256  return 0;
257 
258  IDMap::iterator I = IM.find(II);
259  if (I != IM.end())
260  return I->second; // We've already added 1.
261 
262  IM[II] = ++idcount; // Pre-increment since '0' is reserved for NULL.
263  return idcount;
264 }
265 
266 void PTHWriter::EmitToken(const Token& T) {
267  // Emit the token kind, flags, and length.
268  Emit32(((uint32_t) T.getKind()) | ((((uint32_t) T.getFlags())) << 8)|
269  (((uint32_t) T.getLength()) << 16));
270 
271  if (!T.isLiteral()) {
272  Emit32(ResolveID(T.getIdentifierInfo()));
273  } else {
274  // We cache *un-cleaned* spellings. This gives us 100% fidelity with the
275  // source code.
276  StringRef s(T.getLiteralData(), T.getLength());
277 
278  // Get the string entry.
279  auto &E = *CachedStrs.insert(std::make_pair(s, OffsetOpt())).first;
280 
281  // If this is a new string entry, bump the PTH offset.
282  if (!E.second.hasOffset()) {
283  E.second.setOffset(CurStrOffset);
284  StrEntries.push_back(&E);
285  CurStrOffset += s.size() + 1;
286  }
287 
288  // Emit the relative offset into the PTH file for the spelling string.
289  Emit32(E.second.getOffset());
290  }
291 
292  // Emit the offset into the original source file of this token so that we
293  // can reconstruct its SourceLocation.
294  Emit32(PP.getSourceManager().getFileOffset(T.getLocation()));
295 }
296 
297 PTHEntry PTHWriter::LexTokens(Lexer& L) {
298  // Pad 0's so that we emit tokens to a 4-byte alignment.
299  // This speed up reading them back in.
300  using namespace llvm::support;
301  endian::Writer LE(Out, little);
302  uint32_t TokenOff = Out.tell();
303  for (uint64_t N = llvm::OffsetToAlignment(TokenOff, 4); N; --N, ++TokenOff)
304  LE.write<uint8_t>(0);
305 
306  // Keep track of matching '#if' ... '#endif'.
307  typedef std::vector<std::pair<Offset, unsigned> > PPCondTable;
308  PPCondTable PPCond;
309  std::vector<unsigned> PPStartCond;
310  bool ParsingPreprocessorDirective = false;
311  Token Tok;
312 
313  do {
314  L.LexFromRawLexer(Tok);
315  NextToken:
316 
317  if ((Tok.isAtStartOfLine() || Tok.is(tok::eof)) &&
318  ParsingPreprocessorDirective) {
319  // Insert an eod token into the token cache. It has the same
320  // position as the next token that is not on the same line as the
321  // preprocessor directive. Observe that we continue processing
322  // 'Tok' when we exit this branch.
323  Token Tmp = Tok;
324  Tmp.setKind(tok::eod);
326  Tmp.setIdentifierInfo(nullptr);
327  EmitToken(Tmp);
328  ParsingPreprocessorDirective = false;
329  }
330 
331  if (Tok.is(tok::raw_identifier)) {
332  PP.LookUpIdentifierInfo(Tok);
333  EmitToken(Tok);
334  continue;
335  }
336 
337  if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) {
338  // Special processing for #include. Store the '#' token and lex
339  // the next token.
340  assert(!ParsingPreprocessorDirective);
341  Offset HashOff = (Offset) Out.tell();
342 
343  // Get the next token.
344  Token NextTok;
345  L.LexFromRawLexer(NextTok);
346 
347  // If we see the start of line, then we had a null directive "#". In
348  // this case, discard both tokens.
349  if (NextTok.isAtStartOfLine())
350  goto NextToken;
351 
352  // The token is the start of a directive. Emit it.
353  EmitToken(Tok);
354  Tok = NextTok;
355 
356  // Did we see 'include'/'import'/'include_next'?
357  if (Tok.isNot(tok::raw_identifier)) {
358  EmitToken(Tok);
359  continue;
360  }
361 
362  IdentifierInfo* II = PP.LookUpIdentifierInfo(Tok);
364 
365  ParsingPreprocessorDirective = true;
366 
367  switch (K) {
368  case tok::pp_not_keyword:
369  // Invalid directives "#foo" can occur in #if 0 blocks etc, just pass
370  // them through.
371  default:
372  break;
373 
374  case tok::pp_include:
375  case tok::pp_import:
376  case tok::pp_include_next: {
377  // Save the 'include' token.
378  EmitToken(Tok);
379  // Lex the next token as an include string.
381  L.LexIncludeFilename(Tok);
383  assert(!Tok.isAtStartOfLine());
384  if (Tok.is(tok::raw_identifier))
385  PP.LookUpIdentifierInfo(Tok);
386 
387  break;
388  }
389  case tok::pp_if:
390  case tok::pp_ifdef:
391  case tok::pp_ifndef: {
392  // Add an entry for '#if' and friends. We initially set the target
393  // index to 0. This will get backpatched when we hit #endif.
394  PPStartCond.push_back(PPCond.size());
395  PPCond.push_back(std::make_pair(HashOff, 0U));
396  break;
397  }
398  case tok::pp_endif: {
399  // Add an entry for '#endif'. We set the target table index to itself.
400  // This will later be set to zero when emitting to the PTH file. We
401  // use 0 for uninitialized indices because that is easier to debug.
402  unsigned index = PPCond.size();
403  // Backpatch the opening '#if' entry.
404  assert(!PPStartCond.empty());
405  assert(PPCond.size() > PPStartCond.back());
406  assert(PPCond[PPStartCond.back()].second == 0);
407  PPCond[PPStartCond.back()].second = index;
408  PPStartCond.pop_back();
409  // Add the new entry to PPCond.
410  PPCond.push_back(std::make_pair(HashOff, index));
411  EmitToken(Tok);
412 
413  // Some files have gibberish on the same line as '#endif'.
414  // Discard these tokens.
415  do
416  L.LexFromRawLexer(Tok);
417  while (Tok.isNot(tok::eof) && !Tok.isAtStartOfLine());
418  // We have the next token in hand.
419  // Don't immediately lex the next one.
420  goto NextToken;
421  }
422  case tok::pp_elif:
423  case tok::pp_else: {
424  // Add an entry for #elif or #else.
425  // This serves as both a closing and opening of a conditional block.
426  // This means that its entry will get backpatched later.
427  unsigned index = PPCond.size();
428  // Backpatch the previous '#if' entry.
429  assert(!PPStartCond.empty());
430  assert(PPCond.size() > PPStartCond.back());
431  assert(PPCond[PPStartCond.back()].second == 0);
432  PPCond[PPStartCond.back()].second = index;
433  PPStartCond.pop_back();
434  // Now add '#elif' as a new block opening.
435  PPCond.push_back(std::make_pair(HashOff, 0U));
436  PPStartCond.push_back(index);
437  break;
438  }
439  }
440  }
441 
442  EmitToken(Tok);
443  }
444  while (Tok.isNot(tok::eof));
445 
446  assert(PPStartCond.empty() && "Error: imblanced preprocessor conditionals.");
447 
448  // Next write out PPCond.
449  Offset PPCondOff = (Offset) Out.tell();
450 
451  // Write out the size of PPCond so that clients can identifer empty tables.
452  Emit32(PPCond.size());
453 
454  for (unsigned i = 0, e = PPCond.size(); i!=e; ++i) {
455  Emit32(PPCond[i].first - TokenOff);
456  uint32_t x = PPCond[i].second;
457  assert(x != 0 && "PPCond entry not backpatched.");
458  // Emit zero for #endifs. This allows us to do checking when
459  // we read the PTH file back in.
460  Emit32(x == i ? 0 : x);
461  }
462 
463  return PTHEntry(TokenOff, PPCondOff);
464 }
465 
466 Offset PTHWriter::EmitCachedSpellings() {
467  // Write each cached strings to the PTH file.
468  Offset SpellingsOff = Out.tell();
469 
470  for (std::vector<llvm::StringMapEntry<OffsetOpt>*>::iterator
471  I = StrEntries.begin(), E = StrEntries.end(); I!=E; ++I)
472  EmitBuf((*I)->getKeyData(), (*I)->getKeyLength()+1 /*nul included*/);
473 
474  return SpellingsOff;
475 }
476 
477 static uint32_t swap32le(uint32_t X) {
478  return llvm::support::endian::byte_swap<uint32_t, llvm::support::little>(X);
479 }
480 
481 static void pwrite32le(raw_pwrite_stream &OS, uint32_t Val, uint64_t &Off) {
482  uint32_t LEVal = swap32le(Val);
483  OS.pwrite(reinterpret_cast<const char *>(&LEVal), 4, Off);
484  Off += 4;
485 }
486 
487 void PTHWriter::GeneratePTH(StringRef MainFile) {
488  // Generate the prologue.
489  Out << "cfe-pth" << '\0';
490  Emit32(PTHManager::Version);
491 
492  // Leave 4 words for the prologue.
493  Offset PrologueOffset = Out.tell();
494  for (unsigned i = 0; i < 4; ++i)
495  Emit32(0);
496 
497  // Write the name of the MainFile.
498  if (!MainFile.empty()) {
499  EmitString(MainFile);
500  } else {
501  // String with 0 bytes.
502  Emit16(0);
503  }
504  Emit8(0);
505 
506  // Iterate over all the files in SourceManager. Create a lexer
507  // for each file and cache the tokens.
508  SourceManager &SM = PP.getSourceManager();
509  const LangOptions &LOpts = PP.getLangOpts();
510 
512  E = SM.fileinfo_end(); I != E; ++I) {
513  const SrcMgr::ContentCache &C = *I->second;
514  const FileEntry *FE = C.OrigEntry;
515 
516  // FIXME: Handle files with non-absolute paths.
517  if (llvm::sys::path::is_relative(FE->getName()))
518  continue;
519 
520  const llvm::MemoryBuffer *B = C.getBuffer(PP.getDiagnostics(), SM);
521  if (!B) continue;
522 
524  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
525  Lexer L(FID, FromFile, SM, LOpts);
526  PM.insert(FE, LexTokens(L));
527  }
528 
529  // Write out the identifier table.
530  const std::pair<Offset,Offset> &IdTableOff = EmitIdentifierTable();
531 
532  // Write out the cached strings table.
533  Offset SpellingOff = EmitCachedSpellings();
534 
535  // Write out the file table.
536  Offset FileTableOff = EmitFileTable();
537 
538  // Finally, write the prologue.
539  uint64_t Off = PrologueOffset;
540  pwrite32le(Out, IdTableOff.first, Off);
541  pwrite32le(Out, IdTableOff.second, Off);
542  pwrite32le(Out, FileTableOff, Off);
543  pwrite32le(Out, SpellingOff, Off);
544 }
545 
546 namespace {
547 /// StatListener - A simple "interpose" object used to monitor stat calls
548 /// invoked by FileManager while processing the original sources used
549 /// as input to PTH generation. StatListener populates the PTHWriter's
550 /// file map with stat information for directories as well as negative stats.
551 /// Stat information for files are populated elsewhere.
552 class StatListener : public FileSystemStatCache {
553  PTHMap &PM;
554 public:
555  StatListener(PTHMap &pm) : PM(pm) {}
556  ~StatListener() override {}
557 
558  LookupResult getStat(StringRef Path, FileData &Data, bool isFile,
559  std::unique_ptr<llvm::vfs::File> *F,
560  llvm::vfs::FileSystem &FS) override {
561  LookupResult Result = statChained(Path, Data, isFile, F, FS);
562 
563  if (Result == CacheMissing) // Failed 'stat'.
564  PM.insert(PTHEntryKeyVariant(Path), PTHEntry());
565  else if (Data.IsDirectory) {
566  // Only cache directories with absolute paths.
567  if (llvm::sys::path::is_relative(Path))
568  return Result;
569 
570  PM.insert(PTHEntryKeyVariant(&Data, Path), PTHEntry());
571  }
572 
573  return Result;
574  }
575 };
576 } // end anonymous namespace
577 
578 void clang::CacheTokens(Preprocessor &PP, raw_pwrite_stream *OS) {
579  // Get the name of the main file.
580  const SourceManager &SrcMgr = PP.getSourceManager();
581  const FileEntry *MainFile = SrcMgr.getFileEntryForID(SrcMgr.getMainFileID());
582  SmallString<128> MainFilePath(MainFile->getName());
583 
584  llvm::sys::fs::make_absolute(MainFilePath);
585 
586  // Create the PTHWriter.
587  PTHWriter PW(*OS, PP);
588 
589  // Install the 'stat' system call listener in the FileManager.
590  auto StatCacheOwner = llvm::make_unique<StatListener>(PW.getPM());
591  StatListener *StatCache = StatCacheOwner.get();
592  PP.getFileManager().addStatCache(std::move(StatCacheOwner),
593  /*AtBeginning=*/true);
594 
595  // Lex through the entire file. This will populate SourceManager with
596  // all of the header information.
597  Token Tok;
598  PP.EnterMainSourceFile();
599  do { PP.Lex(Tok); } while (Tok.isNot(tok::eof));
600 
601  // Generate the PTH file.
602  PP.getFileManager().removeStatCache(StatCache);
603  PW.GeneratePTH(MainFilePath.str());
604 }
605 
606 //===----------------------------------------------------------------------===//
607 
608 namespace {
609 class PTHIdKey {
610 public:
611  const IdentifierInfo* II;
612  uint32_t FileOffset;
613 };
614 
615 class PTHIdentifierTableTrait {
616 public:
617  typedef PTHIdKey* key_type;
618  typedef key_type key_type_ref;
619 
620  typedef uint32_t data_type;
621  typedef data_type data_type_ref;
622 
623  typedef unsigned hash_value_type;
624  typedef unsigned offset_type;
625 
626  static hash_value_type ComputeHash(PTHIdKey* key) {
627  return llvm::djbHash(key->II->getName());
628  }
629 
630  static std::pair<unsigned,unsigned>
631  EmitKeyDataLength(raw_ostream& Out, const PTHIdKey* key, uint32_t) {
632  using namespace llvm::support;
633  unsigned n = key->II->getLength() + 1;
634  endian::write<uint16_t>(Out, n, little);
635  return std::make_pair(n, sizeof(uint32_t));
636  }
637 
638  static void EmitKey(raw_ostream& Out, PTHIdKey* key, unsigned n) {
639  // Record the location of the key data. This is used when generating
640  // the mapping from persistent IDs to strings.
641  key->FileOffset = Out.tell();
642  Out.write(key->II->getNameStart(), n);
643  }
644 
645  static void EmitData(raw_ostream& Out, PTHIdKey*, uint32_t pID,
646  unsigned) {
647  using namespace llvm::support;
648  endian::write<uint32_t>(Out, pID, little);
649  }
650 };
651 } // end anonymous namespace
652 
653 /// EmitIdentifierTable - Emits two tables to the PTH file. The first is
654 /// a hashtable mapping from identifier strings to persistent IDs. The second
655 /// is a straight table mapping from persistent IDs to string data (the
656 /// keys of the first table).
657 ///
658 std::pair<Offset,Offset> PTHWriter::EmitIdentifierTable() {
659  // Build two maps:
660  // (1) an inverse map from persistent IDs -> (IdentifierInfo*,Offset)
661  // (2) a map from (IdentifierInfo*, Offset)* -> persistent IDs
662 
663  // Note that we use 'calloc', so all the bytes are 0.
664  PTHIdKey *IIDMap = static_cast<PTHIdKey*>(
665  llvm::safe_calloc(idcount, sizeof(PTHIdKey)));
666 
667  // Create the hashtable.
668  llvm::OnDiskChainedHashTableGenerator<PTHIdentifierTableTrait> IIOffMap;
669 
670  // Generate mapping from persistent IDs -> IdentifierInfo*.
671  for (IDMap::iterator I = IM.begin(), E = IM.end(); I != E; ++I) {
672  // Decrement by 1 because we are using a vector for the lookup and
673  // 0 is reserved for NULL.
674  assert(I->second > 0);
675  assert(I->second-1 < idcount);
676  unsigned idx = I->second-1;
677 
678  // Store the mapping from persistent ID to IdentifierInfo*
679  IIDMap[idx].II = I->first;
680 
681  // Store the reverse mapping in a hashtable.
682  IIOffMap.insert(&IIDMap[idx], I->second);
683  }
684 
685  // Write out the inverse map first. This causes the PCIDKey entries to
686  // record PTH file offsets for the string data. This is used to write
687  // the second table.
688  Offset StringTableOffset = IIOffMap.Emit(Out);
689 
690  // Now emit the table mapping from persistent IDs to PTH file offsets.
691  Offset IDOff = Out.tell();
692  Emit32(idcount); // Emit the number of identifiers.
693  for (unsigned i = 0 ; i < idcount; ++i)
694  Emit32(IIDMap[i].FileOffset);
695 
696  // Finally, release the inverse map.
697  free(IIDMap);
698 
699  return std::make_pair(IDOff, StringTableOffset);
700 }
const FileEntry * OrigEntry
Reference to the file entry representing this ContentCache.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:77
Defines the clang::FileManager interface and associated types.
bool LexFromRawLexer(Token &Result)
LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...
Definition: Lexer.h:195
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
Defines the SourceManager interface.
FileManager & getFileManager() const
Definition: Preprocessor.h:830
fileinfo_iterator fileinfo_end() const
Defines the FileSystemStatCache interface.
tok::TokenKind getKind() const
Definition: Token.h:90
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:113
One of these records is kept for each identifier that is lexed.
One instance of this struct is kept for every file loaded or used.
Definition: SourceManager.h:95
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
static void pwrite32le(raw_pwrite_stream &OS, uint32_t Val, uint64_t &Off)
llvm::OnDiskChainedHashTableGenerator< FileEntryPTHEntryInfo > PTHMap
void setKind(tok::TokenKind K)
Definition: Token.h:91
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:50
Abstract interface for introducing a FileManager cache for &#39;stat&#39; system calls, which is used by prec...
Represents the results of name lookup.
Definition: Lookup.h:47
uint32_t Offset
Definition: CacheTokens.cpp:43
const FormatToken & Tok
Defines the Diagnostic-related interfaces.
void setParsingPreprocessorDirective(bool f)
Inform the lexer whether or not we are currently lexing a preprocessor directive. ...
void CacheTokens(Preprocessor &PP, raw_pwrite_stream *OS)
Cache tokens for use with PCH. Note that this requires a seekable stream.
llvm::MemoryBuffer * getBuffer(DiagnosticsEngine &Diag, const SourceManager &SM, SourceLocation Loc=SourceLocation(), bool *Invalid=nullptr) const
Returns the memory buffer for the associated content.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, unsigned LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
Defines the clang::Preprocessor interface.
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a &#39;#&#39; at the beginning of the line...
Definition: TokenKinds.h:33
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
fileinfo_iterator fileinfo_begin() const
The result type of a method or function.
const SourceManager & SM
Definition: Format.cpp:1490
SourceManager & getSourceManager() const
Definition: Preprocessor.h:831
static uint32_t swap32le(uint32_t X)
Kind
llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
void addStatCache(std::unique_ptr< FileSystemStatCache > statCache, bool AtBeginning=false)
Installs the provided FileSystemStatCache object within the FileManager.
Definition: FileManager.cpp:66
Encodes a location in the source.
llvm::sys::fs::UniqueID UniqueID
StringRef getName() const
Definition: FileManager.h:85
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:177
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:60
void setIdentifierInfo(IdentifierInfo *II)
Definition: Token.h:186
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:266
llvm::DenseMap< const FileEntry *, SrcMgr::ContentCache * >::const_iterator fileinfo_iterator
unsigned ComputeHash(Selector Sel)
Definition: ASTCommon.cpp:247
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:96
Dataflow Directional Tag Classes.
void removeStatCache(FileSystemStatCache *statCache)
Removes the specified FileSystemStatCache object from the manager.
Definition: FileManager.cpp:82
FileID getMainFileID() const
Returns the FileID of the main source file.
tok::PPKeywordKind getPPKeywordID() const
Return the preprocessor keyword ID for this identifier.
unsigned getLength() const
Definition: Token.h:127
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:215
raw_ostream & EmitString(raw_ostream &o, StringRef s)
Definition: PlistSupport.h:79
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
Definition: SemaDecl.cpp:13849
static Decl::Kind getKind(const Decl *D)
Definition: DeclBase.cpp:954
Generate pre-tokenized header.
unsigned getFlags() const
Return the internal represtation of the flags.
Definition: Token.h:252
void clearFlag(TokenFlags Flag)
Unset the specified flag.
Definition: Token.h:244
void LexIncludeFilename(Token &FilenameTok)
After the preprocessor has parsed a #include, lex and (potentially) macro expand the filename...
This class handles loading and caching of source files into memory.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:127