clang  7.0.0svn
CacheTokens.cpp
Go to the documentation of this file.
1 //===--- CacheTokens.cpp - Caching of lexer tokens for PTH support --------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This provides a possible implementation of PTH support for Clang that is
11 // based on caching lexed tokens and identifiers.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "clang/Basic/Diagnostic.h"
20 #include "clang/Frontend/Utils.h"
21 #include "clang/Lex/Lexer.h"
22 #include "clang/Lex/PTHManager.h"
23 #include "clang/Lex/Preprocessor.h"
24 #include "llvm/ADT/StringMap.h"
25 #include "llvm/Support/DJB.h"
26 #include "llvm/Support/EndianStream.h"
27 #include "llvm/Support/FileSystem.h"
28 #include "llvm/Support/MemoryBuffer.h"
29 #include "llvm/Support/OnDiskHashTable.h"
30 #include "llvm/Support/Path.h"
31 
32 // FIXME: put this somewhere else?
33 #ifndef S_ISDIR
34 #define S_ISDIR(x) (((x)&_S_IFDIR)!=0)
35 #endif
36 
37 using namespace clang;
38 
39 //===----------------------------------------------------------------------===//
40 // PTH-specific stuff.
41 //===----------------------------------------------------------------------===//
42 
43 typedef uint32_t Offset;
44 
45 namespace {
46 class PTHEntry {
47  Offset TokenData, PPCondData;
48 
49 public:
50  PTHEntry() {}
51 
52  PTHEntry(Offset td, Offset ppcd)
53  : TokenData(td), PPCondData(ppcd) {}
54 
55  Offset getTokenOffset() const { return TokenData; }
56  Offset getPPCondTableOffset() const { return PPCondData; }
57 };
58 
59 
60 class PTHEntryKeyVariant {
61  union {
62  const FileEntry *FE;
63  // FIXME: Use "StringRef Path;" when MSVC 2013 is dropped.
64  const char *PathPtr;
65  };
66  size_t PathSize;
67  enum { IsFE = 0x1, IsDE = 0x2, IsNoExist = 0x0 } Kind;
68  FileData *Data;
69 
70 public:
71  PTHEntryKeyVariant(const FileEntry *fe) : FE(fe), Kind(IsFE), Data(nullptr) {}
72 
73  PTHEntryKeyVariant(FileData *Data, StringRef Path)
74  : PathPtr(Path.data()), PathSize(Path.size()), Kind(IsDE),
75  Data(new FileData(*Data)) {}
76 
77  explicit PTHEntryKeyVariant(StringRef Path)
78  : PathPtr(Path.data()), PathSize(Path.size()), Kind(IsNoExist),
79  Data(nullptr) {}
80 
81  bool isFile() const { return Kind == IsFE; }
82 
83  StringRef getString() const {
84  return Kind == IsFE ? FE->getName() : StringRef(PathPtr, PathSize);
85  }
86 
87  unsigned getKind() const { return (unsigned) Kind; }
88 
89  void EmitData(raw_ostream& Out) {
90  using namespace llvm::support;
91  endian::Writer LE(Out, little);
92  switch (Kind) {
93  case IsFE: {
94  // Emit stat information.
95  llvm::sys::fs::UniqueID UID = FE->getUniqueID();
96  LE.write<uint64_t>(UID.getFile());
97  LE.write<uint64_t>(UID.getDevice());
98  LE.write<uint64_t>(FE->getModificationTime());
99  LE.write<uint64_t>(FE->getSize());
100  } break;
101  case IsDE:
102  // Emit stat information.
103  LE.write<uint64_t>(Data->UniqueID.getFile());
104  LE.write<uint64_t>(Data->UniqueID.getDevice());
105  LE.write<uint64_t>(Data->ModTime);
106  LE.write<uint64_t>(Data->Size);
107  delete Data;
108  break;
109  default:
110  break;
111  }
112  }
113 
114  unsigned getRepresentationLength() const {
115  return Kind == IsNoExist ? 0 : 4 * 8;
116  }
117 };
118 
119 class FileEntryPTHEntryInfo {
120 public:
121  typedef PTHEntryKeyVariant key_type;
122  typedef key_type key_type_ref;
123 
124  typedef PTHEntry data_type;
125  typedef const PTHEntry& data_type_ref;
126 
127  typedef unsigned hash_value_type;
128  typedef unsigned offset_type;
129 
130  static hash_value_type ComputeHash(PTHEntryKeyVariant V) {
131  return llvm::djbHash(V.getString());
132  }
133 
134  static std::pair<unsigned,unsigned>
135  EmitKeyDataLength(raw_ostream& Out, PTHEntryKeyVariant V,
136  const PTHEntry& E) {
137  using namespace llvm::support;
138  endian::Writer LE(Out, little);
139 
140  unsigned n = V.getString().size() + 1 + 1;
141  LE.write<uint16_t>(n);
142 
143  unsigned m = V.getRepresentationLength() + (V.isFile() ? 4 + 4 : 0);
144  LE.write<uint8_t>(m);
145 
146  return std::make_pair(n, m);
147  }
148 
149  static void EmitKey(raw_ostream& Out, PTHEntryKeyVariant V, unsigned n){
150  using namespace llvm::support;
151  // Emit the entry kind.
152  Out << char(V.getKind());
153  // Emit the string.
154  Out.write(V.getString().data(), n - 1);
155  }
156 
157  static void EmitData(raw_ostream& Out, PTHEntryKeyVariant V,
158  const PTHEntry& E, unsigned) {
159  using namespace llvm::support;
160  endian::Writer LE(Out, little);
161 
162  // For file entries emit the offsets into the PTH file for token data
163  // and the preprocessor blocks table.
164  if (V.isFile()) {
165  LE.write<uint32_t>(E.getTokenOffset());
166  LE.write<uint32_t>(E.getPPCondTableOffset());
167  }
168 
169  // Emit any other data associated with the key (i.e., stat information).
170  V.EmitData(Out);
171  }
172 };
173 
174 class OffsetOpt {
175  bool valid;
176  Offset off;
177 public:
178  OffsetOpt() : valid(false) {}
179  bool hasOffset() const { return valid; }
180  Offset getOffset() const { assert(valid); return off; }
181  void setOffset(Offset o) { off = o; valid = true; }
182 };
183 } // end anonymous namespace
184 
185 typedef llvm::OnDiskChainedHashTableGenerator<FileEntryPTHEntryInfo> PTHMap;
186 
187 namespace {
188 class PTHWriter {
189  typedef llvm::DenseMap<const IdentifierInfo*,uint32_t> IDMap;
190  typedef llvm::StringMap<OffsetOpt, llvm::BumpPtrAllocator> CachedStrsTy;
191 
192  raw_pwrite_stream &Out;
193  Preprocessor& PP;
194  IDMap IM;
195  std::vector<llvm::StringMapEntry<OffsetOpt>*> StrEntries;
196  PTHMap PM;
197  CachedStrsTy CachedStrs;
198  uint32_t idcount;
199  Offset CurStrOffset;
200 
201  //// Get the persistent id for the given IdentifierInfo*.
202  uint32_t ResolveID(const IdentifierInfo* II);
203 
204  /// Emit a token to the PTH file.
205  void EmitToken(const Token& T);
206 
207  void Emit8(uint32_t V) {
208  Out << char(V);
209  }
210 
211  void Emit16(uint32_t V) {
212  using namespace llvm::support;
213  endian::write<uint16_t>(Out, V, little);
214  }
215 
216  void Emit32(uint32_t V) {
217  using namespace llvm::support;
218  endian::write<uint32_t>(Out, V, little);
219  }
220 
221  void EmitBuf(const char *Ptr, unsigned NumBytes) {
222  Out.write(Ptr, NumBytes);
223  }
224 
225  void EmitString(StringRef V) {
226  using namespace llvm::support;
227  endian::write<uint16_t>(Out, V.size(), little);
228  EmitBuf(V.data(), V.size());
229  }
230 
231  /// EmitIdentifierTable - Emits two tables to the PTH file. The first is
232  /// a hashtable mapping from identifier strings to persistent IDs.
233  /// The second is a straight table mapping from persistent IDs to string data
234  /// (the keys of the first table).
235  std::pair<Offset, Offset> EmitIdentifierTable();
236 
237  /// EmitFileTable - Emit a table mapping from file name strings to PTH
238  /// token data.
239  Offset EmitFileTable() { return PM.Emit(Out); }
240 
241  PTHEntry LexTokens(Lexer& L);
242  Offset EmitCachedSpellings();
243 
244 public:
245  PTHWriter(raw_pwrite_stream &out, Preprocessor &pp)
246  : Out(out), PP(pp), idcount(0), CurStrOffset(0) {}
247 
248  PTHMap &getPM() { return PM; }
249  void GeneratePTH(StringRef MainFile);
250 };
251 } // end anonymous namespace
252 
253 uint32_t PTHWriter::ResolveID(const IdentifierInfo* II) {
254  // Null IdentifierInfo's map to the persistent ID 0.
255  if (!II)
256  return 0;
257 
258  IDMap::iterator I = IM.find(II);
259  if (I != IM.end())
260  return I->second; // We've already added 1.
261 
262  IM[II] = ++idcount; // Pre-increment since '0' is reserved for NULL.
263  return idcount;
264 }
265 
266 void PTHWriter::EmitToken(const Token& T) {
267  // Emit the token kind, flags, and length.
268  Emit32(((uint32_t) T.getKind()) | ((((uint32_t) T.getFlags())) << 8)|
269  (((uint32_t) T.getLength()) << 16));
270 
271  if (!T.isLiteral()) {
272  Emit32(ResolveID(T.getIdentifierInfo()));
273  } else {
274  // We cache *un-cleaned* spellings. This gives us 100% fidelity with the
275  // source code.
276  StringRef s(T.getLiteralData(), T.getLength());
277 
278  // Get the string entry.
279  auto &E = *CachedStrs.insert(std::make_pair(s, OffsetOpt())).first;
280 
281  // If this is a new string entry, bump the PTH offset.
282  if (!E.second.hasOffset()) {
283  E.second.setOffset(CurStrOffset);
284  StrEntries.push_back(&E);
285  CurStrOffset += s.size() + 1;
286  }
287 
288  // Emit the relative offset into the PTH file for the spelling string.
289  Emit32(E.second.getOffset());
290  }
291 
292  // Emit the offset into the original source file of this token so that we
293  // can reconstruct its SourceLocation.
294  Emit32(PP.getSourceManager().getFileOffset(T.getLocation()));
295 }
296 
297 PTHEntry PTHWriter::LexTokens(Lexer& L) {
298  // Pad 0's so that we emit tokens to a 4-byte alignment.
299  // This speed up reading them back in.
300  using namespace llvm::support;
301  endian::Writer LE(Out, little);
302  uint32_t TokenOff = Out.tell();
303  for (uint64_t N = llvm::OffsetToAlignment(TokenOff, 4); N; --N, ++TokenOff)
304  LE.write<uint8_t>(0);
305 
306  // Keep track of matching '#if' ... '#endif'.
307  typedef std::vector<std::pair<Offset, unsigned> > PPCondTable;
308  PPCondTable PPCond;
309  std::vector<unsigned> PPStartCond;
310  bool ParsingPreprocessorDirective = false;
311  Token Tok;
312 
313  do {
314  L.LexFromRawLexer(Tok);
315  NextToken:
316 
317  if ((Tok.isAtStartOfLine() || Tok.is(tok::eof)) &&
318  ParsingPreprocessorDirective) {
319  // Insert an eod token into the token cache. It has the same
320  // position as the next token that is not on the same line as the
321  // preprocessor directive. Observe that we continue processing
322  // 'Tok' when we exit this branch.
323  Token Tmp = Tok;
324  Tmp.setKind(tok::eod);
326  Tmp.setIdentifierInfo(nullptr);
327  EmitToken(Tmp);
328  ParsingPreprocessorDirective = false;
329  }
330 
331  if (Tok.is(tok::raw_identifier)) {
332  PP.LookUpIdentifierInfo(Tok);
333  EmitToken(Tok);
334  continue;
335  }
336 
337  if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) {
338  // Special processing for #include. Store the '#' token and lex
339  // the next token.
340  assert(!ParsingPreprocessorDirective);
341  Offset HashOff = (Offset) Out.tell();
342 
343  // Get the next token.
344  Token NextTok;
345  L.LexFromRawLexer(NextTok);
346 
347  // If we see the start of line, then we had a null directive "#". In
348  // this case, discard both tokens.
349  if (NextTok.isAtStartOfLine())
350  goto NextToken;
351 
352  // The token is the start of a directive. Emit it.
353  EmitToken(Tok);
354  Tok = NextTok;
355 
356  // Did we see 'include'/'import'/'include_next'?
357  if (Tok.isNot(tok::raw_identifier)) {
358  EmitToken(Tok);
359  continue;
360  }
361 
362  IdentifierInfo* II = PP.LookUpIdentifierInfo(Tok);
364 
365  ParsingPreprocessorDirective = true;
366 
367  switch (K) {
368  case tok::pp_not_keyword:
369  // Invalid directives "#foo" can occur in #if 0 blocks etc, just pass
370  // them through.
371  default:
372  break;
373 
374  case tok::pp_include:
375  case tok::pp_import:
376  case tok::pp_include_next: {
377  // Save the 'include' token.
378  EmitToken(Tok);
379  // Lex the next token as an include string.
381  L.LexIncludeFilename(Tok);
383  assert(!Tok.isAtStartOfLine());
384  if (Tok.is(tok::raw_identifier))
385  PP.LookUpIdentifierInfo(Tok);
386 
387  break;
388  }
389  case tok::pp_if:
390  case tok::pp_ifdef:
391  case tok::pp_ifndef: {
392  // Add an entry for '#if' and friends. We initially set the target
393  // index to 0. This will get backpatched when we hit #endif.
394  PPStartCond.push_back(PPCond.size());
395  PPCond.push_back(std::make_pair(HashOff, 0U));
396  break;
397  }
398  case tok::pp_endif: {
399  // Add an entry for '#endif'. We set the target table index to itself.
400  // This will later be set to zero when emitting to the PTH file. We
401  // use 0 for uninitialized indices because that is easier to debug.
402  unsigned index = PPCond.size();
403  // Backpatch the opening '#if' entry.
404  assert(!PPStartCond.empty());
405  assert(PPCond.size() > PPStartCond.back());
406  assert(PPCond[PPStartCond.back()].second == 0);
407  PPCond[PPStartCond.back()].second = index;
408  PPStartCond.pop_back();
409  // Add the new entry to PPCond.
410  PPCond.push_back(std::make_pair(HashOff, index));
411  EmitToken(Tok);
412 
413  // Some files have gibberish on the same line as '#endif'.
414  // Discard these tokens.
415  do
416  L.LexFromRawLexer(Tok);
417  while (Tok.isNot(tok::eof) && !Tok.isAtStartOfLine());
418  // We have the next token in hand.
419  // Don't immediately lex the next one.
420  goto NextToken;
421  }
422  case tok::pp_elif:
423  case tok::pp_else: {
424  // Add an entry for #elif or #else.
425  // This serves as both a closing and opening of a conditional block.
426  // This means that its entry will get backpatched later.
427  unsigned index = PPCond.size();
428  // Backpatch the previous '#if' entry.
429  assert(!PPStartCond.empty());
430  assert(PPCond.size() > PPStartCond.back());
431  assert(PPCond[PPStartCond.back()].second == 0);
432  PPCond[PPStartCond.back()].second = index;
433  PPStartCond.pop_back();
434  // Now add '#elif' as a new block opening.
435  PPCond.push_back(std::make_pair(HashOff, 0U));
436  PPStartCond.push_back(index);
437  break;
438  }
439  }
440  }
441 
442  EmitToken(Tok);
443  }
444  while (Tok.isNot(tok::eof));
445 
446  assert(PPStartCond.empty() && "Error: imblanced preprocessor conditionals.");
447 
448  // Next write out PPCond.
449  Offset PPCondOff = (Offset) Out.tell();
450 
451  // Write out the size of PPCond so that clients can identifer empty tables.
452  Emit32(PPCond.size());
453 
454  for (unsigned i = 0, e = PPCond.size(); i!=e; ++i) {
455  Emit32(PPCond[i].first - TokenOff);
456  uint32_t x = PPCond[i].second;
457  assert(x != 0 && "PPCond entry not backpatched.");
458  // Emit zero for #endifs. This allows us to do checking when
459  // we read the PTH file back in.
460  Emit32(x == i ? 0 : x);
461  }
462 
463  return PTHEntry(TokenOff, PPCondOff);
464 }
465 
466 Offset PTHWriter::EmitCachedSpellings() {
467  // Write each cached strings to the PTH file.
468  Offset SpellingsOff = Out.tell();
469 
470  for (std::vector<llvm::StringMapEntry<OffsetOpt>*>::iterator
471  I = StrEntries.begin(), E = StrEntries.end(); I!=E; ++I)
472  EmitBuf((*I)->getKeyData(), (*I)->getKeyLength()+1 /*nul included*/);
473 
474  return SpellingsOff;
475 }
476 
477 static uint32_t swap32le(uint32_t X) {
478  return llvm::support::endian::byte_swap<uint32_t, llvm::support::little>(X);
479 }
480 
481 static void pwrite32le(raw_pwrite_stream &OS, uint32_t Val, uint64_t &Off) {
482  uint32_t LEVal = swap32le(Val);
483  OS.pwrite(reinterpret_cast<const char *>(&LEVal), 4, Off);
484  Off += 4;
485 }
486 
487 void PTHWriter::GeneratePTH(StringRef MainFile) {
488  // Generate the prologue.
489  Out << "cfe-pth" << '\0';
490  Emit32(PTHManager::Version);
491 
492  // Leave 4 words for the prologue.
493  Offset PrologueOffset = Out.tell();
494  for (unsigned i = 0; i < 4; ++i)
495  Emit32(0);
496 
497  // Write the name of the MainFile.
498  if (!MainFile.empty()) {
499  EmitString(MainFile);
500  } else {
501  // String with 0 bytes.
502  Emit16(0);
503  }
504  Emit8(0);
505 
506  // Iterate over all the files in SourceManager. Create a lexer
507  // for each file and cache the tokens.
508  SourceManager &SM = PP.getSourceManager();
509  const LangOptions &LOpts = PP.getLangOpts();
510 
512  E = SM.fileinfo_end(); I != E; ++I) {
513  const SrcMgr::ContentCache &C = *I->second;
514  const FileEntry *FE = C.OrigEntry;
515 
516  // FIXME: Handle files with non-absolute paths.
517  if (llvm::sys::path::is_relative(FE->getName()))
518  continue;
519 
520  const llvm::MemoryBuffer *B = C.getBuffer(PP.getDiagnostics(), SM);
521  if (!B) continue;
522 
524  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
525  Lexer L(FID, FromFile, SM, LOpts);
526  PM.insert(FE, LexTokens(L));
527  }
528 
529  // Write out the identifier table.
530  const std::pair<Offset,Offset> &IdTableOff = EmitIdentifierTable();
531 
532  // Write out the cached strings table.
533  Offset SpellingOff = EmitCachedSpellings();
534 
535  // Write out the file table.
536  Offset FileTableOff = EmitFileTable();
537 
538  // Finally, write the prologue.
539  uint64_t Off = PrologueOffset;
540  pwrite32le(Out, IdTableOff.first, Off);
541  pwrite32le(Out, IdTableOff.second, Off);
542  pwrite32le(Out, FileTableOff, Off);
543  pwrite32le(Out, SpellingOff, Off);
544 }
545 
546 namespace {
547 /// StatListener - A simple "interpose" object used to monitor stat calls
548 /// invoked by FileManager while processing the original sources used
549 /// as input to PTH generation. StatListener populates the PTHWriter's
550 /// file map with stat information for directories as well as negative stats.
551 /// Stat information for files are populated elsewhere.
552 class StatListener : public FileSystemStatCache {
553  PTHMap &PM;
554 public:
555  StatListener(PTHMap &pm) : PM(pm) {}
556  ~StatListener() override {}
557 
558  LookupResult getStat(StringRef Path, FileData &Data, bool isFile,
559  std::unique_ptr<vfs::File> *F,
560  vfs::FileSystem &FS) override {
561  LookupResult Result = statChained(Path, Data, isFile, F, FS);
562 
563  if (Result == CacheMissing) // Failed 'stat'.
564  PM.insert(PTHEntryKeyVariant(Path), PTHEntry());
565  else if (Data.IsDirectory) {
566  // Only cache directories with absolute paths.
567  if (llvm::sys::path::is_relative(Path))
568  return Result;
569 
570  PM.insert(PTHEntryKeyVariant(&Data, Path), PTHEntry());
571  }
572 
573  return Result;
574  }
575 };
576 } // end anonymous namespace
577 
578 void clang::CacheTokens(Preprocessor &PP, raw_pwrite_stream *OS) {
579  // Get the name of the main file.
580  const SourceManager &SrcMgr = PP.getSourceManager();
581  const FileEntry *MainFile = SrcMgr.getFileEntryForID(SrcMgr.getMainFileID());
582  SmallString<128> MainFilePath(MainFile->getName());
583 
584  llvm::sys::fs::make_absolute(MainFilePath);
585 
586  // Create the PTHWriter.
587  PTHWriter PW(*OS, PP);
588 
589  // Install the 'stat' system call listener in the FileManager.
590  auto StatCacheOwner = llvm::make_unique<StatListener>(PW.getPM());
591  StatListener *StatCache = StatCacheOwner.get();
592  PP.getFileManager().addStatCache(std::move(StatCacheOwner),
593  /*AtBeginning=*/true);
594 
595  // Lex through the entire file. This will populate SourceManager with
596  // all of the header information.
597  Token Tok;
598  PP.EnterMainSourceFile();
599  do { PP.Lex(Tok); } while (Tok.isNot(tok::eof));
600 
601  // Generate the PTH file.
602  PP.getFileManager().removeStatCache(StatCache);
603  PW.GeneratePTH(MainFilePath.str());
604 }
605 
606 //===----------------------------------------------------------------------===//
607 
608 namespace {
609 class PTHIdKey {
610 public:
611  const IdentifierInfo* II;
612  uint32_t FileOffset;
613 };
614 
615 class PTHIdentifierTableTrait {
616 public:
617  typedef PTHIdKey* key_type;
618  typedef key_type key_type_ref;
619 
620  typedef uint32_t data_type;
621  typedef data_type data_type_ref;
622 
623  typedef unsigned hash_value_type;
624  typedef unsigned offset_type;
625 
626  static hash_value_type ComputeHash(PTHIdKey* key) {
627  return llvm::djbHash(key->II->getName());
628  }
629 
630  static std::pair<unsigned,unsigned>
631  EmitKeyDataLength(raw_ostream& Out, const PTHIdKey* key, uint32_t) {
632  using namespace llvm::support;
633  unsigned n = key->II->getLength() + 1;
634  endian::write<uint16_t>(Out, n, little);
635  return std::make_pair(n, sizeof(uint32_t));
636  }
637 
638  static void EmitKey(raw_ostream& Out, PTHIdKey* key, unsigned n) {
639  // Record the location of the key data. This is used when generating
640  // the mapping from persistent IDs to strings.
641  key->FileOffset = Out.tell();
642  Out.write(key->II->getNameStart(), n);
643  }
644 
645  static void EmitData(raw_ostream& Out, PTHIdKey*, uint32_t pID,
646  unsigned) {
647  using namespace llvm::support;
648  endian::write<uint32_t>(Out, pID, little);
649  }
650 };
651 } // end anonymous namespace
652 
653 /// EmitIdentifierTable - Emits two tables to the PTH file. The first is
654 /// a hashtable mapping from identifier strings to persistent IDs. The second
655 /// is a straight table mapping from persistent IDs to string data (the
656 /// keys of the first table).
657 ///
658 std::pair<Offset,Offset> PTHWriter::EmitIdentifierTable() {
659  // Build two maps:
660  // (1) an inverse map from persistent IDs -> (IdentifierInfo*,Offset)
661  // (2) a map from (IdentifierInfo*, Offset)* -> persistent IDs
662 
663  // Note that we use 'calloc', so all the bytes are 0.
664  PTHIdKey *IIDMap = static_cast<PTHIdKey*>(
665  llvm::safe_calloc(idcount, sizeof(PTHIdKey)));
666 
667  // Create the hashtable.
668  llvm::OnDiskChainedHashTableGenerator<PTHIdentifierTableTrait> IIOffMap;
669 
670  // Generate mapping from persistent IDs -> IdentifierInfo*.
671  for (IDMap::iterator I = IM.begin(), E = IM.end(); I != E; ++I) {
672  // Decrement by 1 because we are using a vector for the lookup and
673  // 0 is reserved for NULL.
674  assert(I->second > 0);
675  assert(I->second-1 < idcount);
676  unsigned idx = I->second-1;
677 
678  // Store the mapping from persistent ID to IdentifierInfo*
679  IIDMap[idx].II = I->first;
680 
681  // Store the reverse mapping in a hashtable.
682  IIOffMap.insert(&IIDMap[idx], I->second);
683  }
684 
685  // Write out the inverse map first. This causes the PCIDKey entries to
686  // record PTH file offsets for the string data. This is used to write
687  // the second table.
688  Offset StringTableOffset = IIOffMap.Emit(Out);
689 
690  // Now emit the table mapping from persistent IDs to PTH file offsets.
691  Offset IDOff = Out.tell();
692  Emit32(idcount); // Emit the number of identifiers.
693  for (unsigned i = 0 ; i < idcount; ++i)
694  Emit32(IIDMap[i].FileOffset);
695 
696  // Finally, release the inverse map.
697  free(IIDMap);
698 
699  return std::make_pair(IDOff, StringTableOffset);
700 }
const FileEntry * OrigEntry
Reference to the file entry representing this ContentCache.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:77
Defines the clang::FileManager interface and associated types.
bool LexFromRawLexer(Token &Result)
LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...
Definition: Lexer.h:195
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
Defines the SourceManager interface.
FileManager & getFileManager() const
Definition: Preprocessor.h:824
fileinfo_iterator fileinfo_end() const
Defines the FileSystemStatCache interface.
tok::TokenKind getKind() const
Definition: Token.h:90
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:113
The virtual file system interface.
One of these records is kept for each identifier that is lexed.
One instance of this struct is kept for every file loaded or used.
Definition: SourceManager.h:95
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
static void pwrite32le(raw_pwrite_stream &OS, uint32_t Val, uint64_t &Off)
llvm::OnDiskChainedHashTableGenerator< FileEntryPTHEntryInfo > PTHMap
void setKind(tok::TokenKind K)
Definition: Token.h:91
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:50
Abstract interface for introducing a FileManager cache for &#39;stat&#39; system calls, which is used by prec...
Represents the results of name lookup.
Definition: Lookup.h:47
uint32_t Offset
Definition: CacheTokens.cpp:43
const FormatToken & Tok
Defines the Diagnostic-related interfaces.
void setParsingPreprocessorDirective(bool f)
Inform the lexer whether or not we are currently lexing a preprocessor directive. ...
void CacheTokens(Preprocessor &PP, raw_pwrite_stream *OS)
Cache tokens for use with PCH. Note that this requires a seekable stream.
llvm::MemoryBuffer * getBuffer(DiagnosticsEngine &Diag, const SourceManager &SM, SourceLocation Loc=SourceLocation(), bool *Invalid=nullptr) const
Returns the memory buffer for the associated content.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, unsigned LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
Defines the clang::Preprocessor interface.
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a &#39;#&#39; at the beginning of the line...
Definition: TokenKinds.h:33
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
fileinfo_iterator fileinfo_begin() const
The result type of a method or function.
const SourceManager & SM
Definition: Format.cpp:1474
SourceManager & getSourceManager() const
Definition: Preprocessor.h:825
static uint32_t swap32le(uint32_t X)
Kind
llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
void addStatCache(std::unique_ptr< FileSystemStatCache > statCache, bool AtBeginning=false)
Installs the provided FileSystemStatCache object within the FileManager.
Definition: FileManager.cpp:66
Encodes a location in the source.
llvm::sys::fs::UniqueID UniqueID
StringRef getName() const
Definition: FileManager.h:84
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:177
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
void setIdentifierInfo(IdentifierInfo *II)
Definition: Token.h:186
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:266
llvm::DenseMap< const FileEntry *, SrcMgr::ContentCache * >::const_iterator fileinfo_iterator
unsigned ComputeHash(Selector Sel)
Definition: ASTCommon.cpp:242
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:96
Dataflow Directional Tag Classes.
void removeStatCache(FileSystemStatCache *statCache)
Removes the specified FileSystemStatCache object from the manager.
Definition: FileManager.cpp:82
FileID getMainFileID() const
Returns the FileID of the main source file.
tok::PPKeywordKind getPPKeywordID() const
Return the preprocessor keyword ID for this identifier.
unsigned getLength() const
Definition: Token.h:127
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:215
void LexIncludeFilename(Token &Result)
After the preprocessor has parsed a #include, lex and (potentially) macro expand the filename...
raw_ostream & EmitString(raw_ostream &o, StringRef s)
Definition: PlistSupport.h:68
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
Definition: SemaDecl.cpp:13719
static Decl::Kind getKind(const Decl *D)
Definition: DeclBase.cpp:930
Generate pre-tokenized header.
unsigned getFlags() const
Return the internal represtation of the flags.
Definition: Token.h:252
void clearFlag(TokenFlags Flag)
Unset the specified flag.
Definition: Token.h:244
This class handles loading and caching of source files into memory.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:127