clang  6.0.0svn
PTHLexer.cpp
Go to the documentation of this file.
1 //===--- PTHLexer.cpp - Lex from a token stream ---------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the PTHLexer interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "clang/Lex/PTHLexer.h"
18 #include "clang/Basic/TokenKinds.h"
20 #include "clang/Lex/PTHManager.h"
21 #include "clang/Lex/Preprocessor.h"
22 #include "clang/Lex/Token.h"
23 #include "llvm/ADT/StringExtras.h"
24 #include "llvm/Support/EndianStream.h"
25 #include "llvm/Support/MemoryBuffer.h"
26 #include <memory>
27 #include <system_error>
28 using namespace clang;
29 
30 static const unsigned StoredTokenSize = 1 + 1 + 2 + 4 + 4;
31 
32 //===----------------------------------------------------------------------===//
33 // PTHLexer methods.
34 //===----------------------------------------------------------------------===//
35 
36 PTHLexer::PTHLexer(Preprocessor &PP, FileID FID, const unsigned char *D,
37  const unsigned char *ppcond, PTHManager &PM)
38  : PreprocessorLexer(&PP, FID), TokBuf(D), CurPtr(D), LastHashTokPtr(nullptr),
39  PPCond(ppcond), CurPPCondPtr(ppcond), PTHMgr(PM) {
40 
41  FileStartLoc = PP.getSourceManager().getLocForStartOfFile(FID);
42 }
43 
45  //===--------------------------------------==//
46  // Read the raw token data.
47  //===--------------------------------------==//
48  using namespace llvm::support;
49 
50  // Shadow CurPtr into an automatic variable.
51  const unsigned char *CurPtrShadow = CurPtr;
52 
53  // Read in the data for the token.
54  unsigned Word0 = endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
55  uint32_t IdentifierID =
56  endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
57  uint32_t FileOffset =
58  endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
59 
60  tok::TokenKind TKind = (tok::TokenKind) (Word0 & 0xFF);
61  Token::TokenFlags TFlags = (Token::TokenFlags) ((Word0 >> 8) & 0xFF);
62  uint32_t Len = Word0 >> 16;
63 
64  CurPtr = CurPtrShadow;
65 
66  //===--------------------------------------==//
67  // Construct the token itself.
68  //===--------------------------------------==//
69 
70  Tok.startToken();
71  Tok.setKind(TKind);
72  Tok.setFlag(TFlags);
73  assert(!LexingRawMode);
74  Tok.setLocation(FileStartLoc.getLocWithOffset(FileOffset));
75  Tok.setLength(Len);
76 
77  // Handle identifiers.
78  if (Tok.isLiteral()) {
79  Tok.setLiteralData((const char*) (PTHMgr.SpellingBase + IdentifierID));
80  }
81  else if (IdentifierID) {
82  MIOpt.ReadToken();
83  IdentifierInfo *II = PTHMgr.GetIdentifierInfo(IdentifierID-1);
84 
85  Tok.setIdentifierInfo(II);
86 
87  // Change the kind of this identifier to the appropriate token kind, e.g.
88  // turning "for" into a keyword.
89  Tok.setKind(II->getTokenID());
90 
91  if (II->isHandleIdentifierCase())
92  return PP->HandleIdentifier(Tok);
93 
94  return true;
95  }
96 
97  //===--------------------------------------==//
98  // Process the token.
99  //===--------------------------------------==//
100  if (TKind == tok::eof) {
101  // Save the end-of-file token.
102  EofToken = Tok;
103 
105  assert(!LexingRawMode);
106 
107  return LexEndOfFile(Tok);
108  }
109 
110  if (TKind == tok::hash && Tok.isAtStartOfLine()) {
111  LastHashTokPtr = CurPtr - StoredTokenSize;
112  assert(!LexingRawMode);
113  PP->HandleDirective(Tok);
114 
115  return false;
116  }
117 
118  if (TKind == tok::eod) {
121  return true;
122  }
123 
124  MIOpt.ReadToken();
125  return true;
126 }
127 
128 bool PTHLexer::LexEndOfFile(Token &Result) {
129  // If we hit the end of the file while parsing a preprocessor directive,
130  // end the preprocessor directive first. The next token returned will
131  // then be the end of file.
133  ParsingPreprocessorDirective = false; // Done parsing the "line".
134  return true; // Have a token.
135  }
136 
137  assert(!LexingRawMode);
138 
139  // If we are in a #if directive, emit an error.
140  while (!ConditionalStack.empty()) {
141  if (PP->getCodeCompletionFileLoc() != FileStartLoc)
142  PP->Diag(ConditionalStack.back().IfLoc,
143  diag::err_pp_unterminated_conditional);
144  ConditionalStack.pop_back();
145  }
146 
147  // Finally, let the preprocessor handle this.
148  return PP->HandleEndOfFile(Result);
149 }
150 
151 // FIXME: We can just grab the last token instead of storing a copy
152 // into EofToken.
154  assert(EofToken.is(tok::eof));
155  Tok = EofToken;
156 }
157 
159  assert(ParsingPreprocessorDirective && ParsingFilename == false &&
160  "Must be in a preprocessing directive!");
161 
162  // We assume that if the preprocessor wishes to discard to the end of
163  // the line that it also means to end the current preprocessor directive.
165 
166  // Skip tokens by only peeking at their token kind and the flags.
167  // We don't need to actually reconstruct full tokens from the token buffer.
168  // This saves some copies and it also reduces IdentifierInfo* lookup.
169  const unsigned char* p = CurPtr;
170  while (1) {
171  // Read the token kind. Are we at the end of the file?
172  tok::TokenKind x = (tok::TokenKind) (uint8_t) *p;
173  if (x == tok::eof) break;
174 
175  // Read the token flags. Are we at the start of the next line?
176  Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1];
177  if (y & Token::StartOfLine) break;
178 
179  // Skip to the next token.
180  p += StoredTokenSize;
181  }
182 
183  CurPtr = p;
184 }
185 
186 /// SkipBlock - Used by Preprocessor to skip the current conditional block.
188  using namespace llvm::support;
189  assert(CurPPCondPtr && "No cached PP conditional information.");
190  assert(LastHashTokPtr && "No known '#' token.");
191 
192  const unsigned char *HashEntryI = nullptr;
193  uint32_t TableIdx;
194 
195  do {
196  // Read the token offset from the side-table.
197  uint32_t Offset = endian::readNext<uint32_t, little, aligned>(CurPPCondPtr);
198 
199  // Read the target table index from the side-table.
200  TableIdx = endian::readNext<uint32_t, little, aligned>(CurPPCondPtr);
201 
202  // Compute the actual memory address of the '#' token data for this entry.
203  HashEntryI = TokBuf + Offset;
204 
205  // Optmization: "Sibling jumping". #if...#else...#endif blocks can
206  // contain nested blocks. In the side-table we can jump over these
207  // nested blocks instead of doing a linear search if the next "sibling"
208  // entry is not at a location greater than LastHashTokPtr.
209  if (HashEntryI < LastHashTokPtr && TableIdx) {
210  // In the side-table we are still at an entry for a '#' token that
211  // is earlier than the last one we saw. Check if the location we would
212  // stride gets us closer.
213  const unsigned char* NextPPCondPtr =
214  PPCond + TableIdx*(sizeof(uint32_t)*2);
215  assert(NextPPCondPtr >= CurPPCondPtr);
216  // Read where we should jump to.
217  const unsigned char *HashEntryJ =
218  TokBuf + endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
219 
220  if (HashEntryJ <= LastHashTokPtr) {
221  // Jump directly to the next entry in the side table.
222  HashEntryI = HashEntryJ;
223  TableIdx = endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
224  CurPPCondPtr = NextPPCondPtr;
225  }
226  }
227  }
228  while (HashEntryI < LastHashTokPtr);
229  assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'");
230  assert(TableIdx && "No jumping from #endifs.");
231 
232  // Update our side-table iterator.
233  const unsigned char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2);
234  assert(NextPPCondPtr >= CurPPCondPtr);
235  CurPPCondPtr = NextPPCondPtr;
236 
237  // Read where we should jump to.
238  HashEntryI =
239  TokBuf + endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
240  uint32_t NextIdx = endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
241 
242  // By construction NextIdx will be zero if this is a #endif. This is useful
243  // to know to obviate lexing another token.
244  bool isEndif = NextIdx == 0;
245 
246  // This case can occur when we see something like this:
247  //
248  // #if ...
249  // /* a comment or nothing */
250  // #elif
251  //
252  // If we are skipping the first #if block it will be the case that CurPtr
253  // already points 'elif'. Just return.
254 
255  if (CurPtr > HashEntryI) {
256  assert(CurPtr == HashEntryI + StoredTokenSize);
257  // Did we reach a #endif? If so, go ahead and consume that token as well.
258  if (isEndif)
259  CurPtr += StoredTokenSize * 2;
260  else
261  LastHashTokPtr = HashEntryI;
262 
263  return isEndif;
264  }
265 
266  // Otherwise, we need to advance. Update CurPtr to point to the '#' token.
267  CurPtr = HashEntryI;
268 
269  // Update the location of the last observed '#'. This is useful if we
270  // are skipping multiple blocks.
271  LastHashTokPtr = CurPtr;
272 
273  // Skip the '#' token.
274  assert(((tok::TokenKind)*CurPtr) == tok::hash);
275  CurPtr += StoredTokenSize;
276 
277  // Did we reach a #endif? If so, go ahead and consume that token as well.
278  if (isEndif) {
279  CurPtr += StoredTokenSize * 2;
280  }
281 
282  return isEndif;
283 }
284 
286  // getSourceLocation is not on the hot path. It is used to get the location
287  // of the next token when transitioning back to this lexer when done
288  // handling a #included file. Just read the necessary data from the token
289  // data buffer to construct the SourceLocation object.
290  // NOTE: This is a virtual function; hence it is defined out-of-line.
291  using namespace llvm::support;
292 
293  const unsigned char *OffsetPtr = CurPtr + (StoredTokenSize - 4);
294  uint32_t Offset = endian::readNext<uint32_t, little, aligned>(OffsetPtr);
295  return FileStartLoc.getLocWithOffset(Offset);
296 }
297 
298 //===----------------------------------------------------------------------===//
299 // PTH file lookup: map from strings to file data.
300 //===----------------------------------------------------------------------===//
301 
302 /// PTHFileLookup - This internal data structure is used by the PTHManager
303 /// to map from FileEntry objects managed by FileManager to offsets within
304 /// the PTH file.
305 namespace {
306 class PTHFileData {
307  const uint32_t TokenOff;
308  const uint32_t PPCondOff;
309 public:
310  PTHFileData(uint32_t tokenOff, uint32_t ppCondOff)
311  : TokenOff(tokenOff), PPCondOff(ppCondOff) {}
312 
313  uint32_t getTokenOffset() const { return TokenOff; }
314  uint32_t getPPCondOffset() const { return PPCondOff; }
315 };
316 
317 
318 class PTHFileLookupCommonTrait {
319 public:
320  typedef std::pair<unsigned char, StringRef> internal_key_type;
321  typedef unsigned hash_value_type;
322  typedef unsigned offset_type;
323 
324  static hash_value_type ComputeHash(internal_key_type x) {
325  return llvm::HashString(x.second);
326  }
327 
328  static std::pair<unsigned, unsigned>
329  ReadKeyDataLength(const unsigned char*& d) {
330  using namespace llvm::support;
331  unsigned keyLen =
332  (unsigned)endian::readNext<uint16_t, little, unaligned>(d);
333  unsigned dataLen = (unsigned) *(d++);
334  return std::make_pair(keyLen, dataLen);
335  }
336 
337  static internal_key_type ReadKey(const unsigned char* d, unsigned) {
338  unsigned char k = *(d++); // Read the entry kind.
339  return std::make_pair(k, (const char*) d);
340  }
341 };
342 
343 } // end anonymous namespace
344 
345 class PTHManager::PTHFileLookupTrait : public PTHFileLookupCommonTrait {
346 public:
347  typedef const FileEntry* external_key_type;
348  typedef PTHFileData data_type;
349 
350  static internal_key_type GetInternalKey(const FileEntry* FE) {
351  return std::make_pair((unsigned char) 0x1, FE->getName());
352  }
353 
354  static bool EqualKey(internal_key_type a, internal_key_type b) {
355  return a.first == b.first && a.second == b.second;
356  }
357 
358  static PTHFileData ReadData(const internal_key_type& k,
359  const unsigned char* d, unsigned) {
360  assert(k.first == 0x1 && "Only file lookups can match!");
361  using namespace llvm::support;
362  uint32_t x = endian::readNext<uint32_t, little, unaligned>(d);
363  uint32_t y = endian::readNext<uint32_t, little, unaligned>(d);
364  return PTHFileData(x, y);
365  }
366 };
367 
369 public:
370  typedef uint32_t data_type;
371  typedef const std::pair<const char*, unsigned> external_key_type;
372  typedef external_key_type internal_key_type;
373  typedef uint32_t hash_value_type;
374  typedef unsigned offset_type;
375 
376  static bool EqualKey(const internal_key_type& a,
377  const internal_key_type& b) {
378  return (a.second == b.second) ? memcmp(a.first, b.first, a.second) == 0
379  : false;
380  }
381 
382  static hash_value_type ComputeHash(const internal_key_type& a) {
383  return llvm::HashString(StringRef(a.first, a.second));
384  }
385 
386  // This hopefully will just get inlined and removed by the optimizer.
387  static const internal_key_type&
388  GetInternalKey(const external_key_type& x) { return x; }
389 
390  static std::pair<unsigned, unsigned>
391  ReadKeyDataLength(const unsigned char*& d) {
392  using namespace llvm::support;
393  return std::make_pair(
394  (unsigned)endian::readNext<uint16_t, little, unaligned>(d),
395  sizeof(uint32_t));
396  }
397 
398  static std::pair<const char*, unsigned>
399  ReadKey(const unsigned char* d, unsigned n) {
400  assert(n >= 2 && d[n-1] == '\0');
401  return std::make_pair((const char*) d, n-1);
402  }
403 
404  static uint32_t ReadData(const internal_key_type& k, const unsigned char* d,
405  unsigned) {
406  using namespace llvm::support;
407  return endian::readNext<uint32_t, little, unaligned>(d);
408  }
409 };
410 
411 //===----------------------------------------------------------------------===//
412 // PTHManager methods.
413 //===----------------------------------------------------------------------===//
414 
415 PTHManager::PTHManager(
416  std::unique_ptr<const llvm::MemoryBuffer> buf,
417  std::unique_ptr<PTHFileLookup> fileLookup, const unsigned char *idDataTable,
418  std::unique_ptr<IdentifierInfo *[], llvm::FreeDeleter> perIDCache,
419  std::unique_ptr<PTHStringIdLookup> stringIdLookup, unsigned numIds,
420  const unsigned char *spellingBase, const char *originalSourceFile)
421  : Buf(std::move(buf)), PerIDCache(std::move(perIDCache)),
422  FileLookup(std::move(fileLookup)), IdDataTable(idDataTable),
423  StringIdLookup(std::move(stringIdLookup)), NumIds(numIds), PP(nullptr),
424  SpellingBase(spellingBase), OriginalSourceFile(originalSourceFile) {}
425 
427 }
428 
429 static void InvalidPTH(DiagnosticsEngine &Diags, const char *Msg) {
430  Diags.Report(Diags.getCustomDiagID(DiagnosticsEngine::Error, "%0")) << Msg;
431 }
432 
434  // Memory map the PTH file.
435  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileOrErr =
436  llvm::MemoryBuffer::getFile(file);
437 
438  if (!FileOrErr) {
439  // FIXME: Add ec.message() to this diag.
440  Diags.Report(diag::err_invalid_pth_file) << file;
441  return nullptr;
442  }
443  std::unique_ptr<llvm::MemoryBuffer> File = std::move(FileOrErr.get());
444 
445  using namespace llvm::support;
446 
447  // Get the buffer ranges and check if there are at least three 32-bit
448  // words at the end of the file.
449  const unsigned char *BufBeg = (const unsigned char*)File->getBufferStart();
450  const unsigned char *BufEnd = (const unsigned char*)File->getBufferEnd();
451 
452  // Check the prologue of the file.
453  if ((BufEnd - BufBeg) < (signed)(sizeof("cfe-pth") + 4 + 4) ||
454  memcmp(BufBeg, "cfe-pth", sizeof("cfe-pth")) != 0) {
455  Diags.Report(diag::err_invalid_pth_file) << file;
456  return nullptr;
457  }
458 
459  // Read the PTH version.
460  const unsigned char *p = BufBeg + (sizeof("cfe-pth"));
461  unsigned Version = endian::readNext<uint32_t, little, aligned>(p);
462 
463  if (Version < PTHManager::Version) {
464  InvalidPTH(Diags,
465  Version < PTHManager::Version
466  ? "PTH file uses an older PTH format that is no longer supported"
467  : "PTH file uses a newer PTH format that cannot be read");
468  return nullptr;
469  }
470 
471  // Compute the address of the index table at the end of the PTH file.
472  const unsigned char *PrologueOffset = p;
473 
474  if (PrologueOffset >= BufEnd) {
475  Diags.Report(diag::err_invalid_pth_file) << file;
476  return nullptr;
477  }
478 
479  // Construct the file lookup table. This will be used for mapping from
480  // FileEntry*'s to cached tokens.
481  const unsigned char* FileTableOffset = PrologueOffset + sizeof(uint32_t)*2;
482  const unsigned char *FileTable =
483  BufBeg + endian::readNext<uint32_t, little, aligned>(FileTableOffset);
484 
485  if (!(FileTable > BufBeg && FileTable < BufEnd)) {
486  Diags.Report(diag::err_invalid_pth_file) << file;
487  return nullptr; // FIXME: Proper error diagnostic?
488  }
489 
490  std::unique_ptr<PTHFileLookup> FL(PTHFileLookup::Create(FileTable, BufBeg));
491 
492  // Warn if the PTH file is empty. We still want to create a PTHManager
493  // as the PTH could be used with -include-pth.
494  if (FL->isEmpty())
495  InvalidPTH(Diags, "PTH file contains no cached source data");
496 
497  // Get the location of the table mapping from persistent ids to the
498  // data needed to reconstruct identifiers.
499  const unsigned char* IDTableOffset = PrologueOffset + sizeof(uint32_t)*0;
500  const unsigned char *IData =
501  BufBeg + endian::readNext<uint32_t, little, aligned>(IDTableOffset);
502 
503  if (!(IData >= BufBeg && IData < BufEnd)) {
504  Diags.Report(diag::err_invalid_pth_file) << file;
505  return nullptr;
506  }
507 
508  // Get the location of the hashtable mapping between strings and
509  // persistent IDs.
510  const unsigned char* StringIdTableOffset = PrologueOffset + sizeof(uint32_t)*1;
511  const unsigned char *StringIdTable =
512  BufBeg + endian::readNext<uint32_t, little, aligned>(StringIdTableOffset);
513  if (!(StringIdTable >= BufBeg && StringIdTable < BufEnd)) {
514  Diags.Report(diag::err_invalid_pth_file) << file;
515  return nullptr;
516  }
517 
518  std::unique_ptr<PTHStringIdLookup> SL(
519  PTHStringIdLookup::Create(StringIdTable, BufBeg));
520 
521  // Get the location of the spelling cache.
522  const unsigned char* spellingBaseOffset = PrologueOffset + sizeof(uint32_t)*3;
523  const unsigned char *spellingBase =
524  BufBeg + endian::readNext<uint32_t, little, aligned>(spellingBaseOffset);
525  if (!(spellingBase >= BufBeg && spellingBase < BufEnd)) {
526  Diags.Report(diag::err_invalid_pth_file) << file;
527  return nullptr;
528  }
529 
530  // Get the number of IdentifierInfos and pre-allocate the identifier cache.
531  uint32_t NumIds = endian::readNext<uint32_t, little, aligned>(IData);
532 
533  // Pre-allocate the persistent ID -> IdentifierInfo* cache. We use calloc()
534  // so that we in the best case only zero out memory once when the OS returns
535  // us new pages.
536  std::unique_ptr<IdentifierInfo *[], llvm::FreeDeleter> PerIDCache;
537 
538  if (NumIds) {
539  PerIDCache.reset((IdentifierInfo **)calloc(NumIds, sizeof(PerIDCache[0])));
540  if (!PerIDCache) {
541  InvalidPTH(Diags, "Could not allocate memory for processing PTH file");
542  return nullptr;
543  }
544  }
545 
546  // Compute the address of the original source file.
547  const unsigned char* originalSourceBase = PrologueOffset + sizeof(uint32_t)*4;
548  unsigned len =
549  endian::readNext<uint16_t, little, unaligned>(originalSourceBase);
550  if (!len) originalSourceBase = nullptr;
551 
552  // Create the new PTHManager.
553  return new PTHManager(std::move(File), std::move(FL), IData,
554  std::move(PerIDCache), std::move(SL), NumIds,
555  spellingBase, (const char *)originalSourceBase);
556 }
557 
558 IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(unsigned PersistentID) {
559  using namespace llvm::support;
560  // Look in the PTH file for the string data for the IdentifierInfo object.
561  const unsigned char* TableEntry = IdDataTable + sizeof(uint32_t)*PersistentID;
562  const unsigned char *IDData =
563  (const unsigned char *)Buf->getBufferStart() +
564  endian::readNext<uint32_t, little, aligned>(TableEntry);
565  assert(IDData < (const unsigned char*)Buf->getBufferEnd());
566 
567  // Allocate the object.
568  std::pair<IdentifierInfo,const unsigned char*> *Mem =
569  Alloc.Allocate<std::pair<IdentifierInfo,const unsigned char*> >();
570 
571  Mem->second = IDData;
572  assert(IDData[0] != '\0');
573  IdentifierInfo *II = new ((void*) Mem) IdentifierInfo();
574 
575  // Store the new IdentifierInfo in the cache.
576  PerIDCache[PersistentID] = II;
577  assert(II->getNameStart() && II->getNameStart()[0] != '\0');
578  return II;
579 }
580 
581 IdentifierInfo* PTHManager::get(StringRef Name) {
582  // Double check our assumption that the last character isn't '\0'.
583  assert(Name.empty() || Name.back() != '\0');
584  PTHStringIdLookup::iterator I =
585  StringIdLookup->find(std::make_pair(Name.data(), Name.size()));
586  if (I == StringIdLookup->end()) // No identifier found?
587  return nullptr;
588 
589  // Match found. Return the identifier!
590  assert(*I > 0);
591  return GetIdentifierInfo(*I-1);
592 }
593 
595  const FileEntry *FE = PP->getSourceManager().getFileEntryForID(FID);
596  if (!FE)
597  return nullptr;
598 
599  using namespace llvm::support;
600 
601  // Lookup the FileEntry object in our file lookup data structure. It will
602  // return a variant that indicates whether or not there is an offset within
603  // the PTH file that contains cached tokens.
604  PTHFileLookup::iterator I = FileLookup->find(FE);
605 
606  if (I == FileLookup->end()) // No tokens available?
607  return nullptr;
608 
609  const PTHFileData& FileData = *I;
610 
611  const unsigned char *BufStart = (const unsigned char *)Buf->getBufferStart();
612  // Compute the offset of the token data within the buffer.
613  const unsigned char* data = BufStart + FileData.getTokenOffset();
614 
615  // Get the location of pp-conditional table.
616  const unsigned char* ppcond = BufStart + FileData.getPPCondOffset();
617  uint32_t Len = endian::readNext<uint32_t, little, aligned>(ppcond);
618  if (Len == 0) ppcond = nullptr;
619 
620  assert(PP && "No preprocessor set yet!");
621  return new PTHLexer(*PP, FID, data, ppcond, *this);
622 }
623 
624 //===----------------------------------------------------------------------===//
625 // 'stat' caching.
626 //===----------------------------------------------------------------------===//
627 
628 namespace {
629 class PTHStatData {
630 public:
631  uint64_t Size;
632  time_t ModTime;
633  llvm::sys::fs::UniqueID UniqueID;
634  const bool HasData;
635  bool IsDirectory;
636 
637  PTHStatData(uint64_t Size, time_t ModTime, llvm::sys::fs::UniqueID UniqueID,
638  bool IsDirectory)
639  : Size(Size), ModTime(ModTime), UniqueID(UniqueID), HasData(true),
640  IsDirectory(IsDirectory) {}
641 
642  PTHStatData() : HasData(false) {}
643 };
644 
645 class PTHStatLookupTrait : public PTHFileLookupCommonTrait {
646 public:
647  typedef StringRef external_key_type; // const char*
648  typedef PTHStatData data_type;
649 
650  static internal_key_type GetInternalKey(StringRef path) {
651  // The key 'kind' doesn't matter here because it is ignored in EqualKey.
652  return std::make_pair((unsigned char) 0x0, path);
653  }
654 
655  static bool EqualKey(internal_key_type a, internal_key_type b) {
656  // When doing 'stat' lookups we don't care about the kind of 'a' and 'b',
657  // just the paths.
658  return a.second == b.second;
659  }
660 
661  static data_type ReadData(const internal_key_type& k, const unsigned char* d,
662  unsigned) {
663 
664  if (k.first /* File or Directory */) {
665  bool IsDirectory = true;
666  if (k.first == 0x1 /* File */) {
667  IsDirectory = false;
668  d += 4 * 2; // Skip the first 2 words.
669  }
670 
671  using namespace llvm::support;
672 
673  uint64_t File = endian::readNext<uint64_t, little, unaligned>(d);
674  uint64_t Device = endian::readNext<uint64_t, little, unaligned>(d);
675  llvm::sys::fs::UniqueID UniqueID(Device, File);
676  time_t ModTime = endian::readNext<uint64_t, little, unaligned>(d);
677  uint64_t Size = endian::readNext<uint64_t, little, unaligned>(d);
678  return data_type(Size, ModTime, UniqueID, IsDirectory);
679  }
680 
681  // Negative stat. Don't read anything.
682  return data_type();
683  }
684 };
685 } // end anonymous namespace
686 
687 namespace clang {
690  CacheTy Cache;
691 
692 public:
694  : Cache(FL.getNumBuckets(), FL.getNumEntries(), FL.getBuckets(),
695  FL.getBase()) {}
696 
697  LookupResult getStat(StringRef Path, FileData &Data, bool isFile,
698  std::unique_ptr<vfs::File> *F,
699  vfs::FileSystem &FS) override {
700  // Do the lookup for the file's data in the PTH file.
701  CacheTy::iterator I = Cache.find(Path);
702 
703  // If we don't get a hit in the PTH file just forward to 'stat'.
704  if (I == Cache.end())
705  return statChained(Path, Data, isFile, F, FS);
706 
707  const PTHStatData &D = *I;
708 
709  if (!D.HasData)
710  return CacheMissing;
711 
712  Data.Name = Path;
713  Data.Size = D.Size;
714  Data.ModTime = D.ModTime;
715  Data.UniqueID = D.UniqueID;
716  Data.IsDirectory = D.IsDirectory;
717  Data.IsNamedPipe = false;
718  Data.InPCH = true;
719 
720  return CacheExists;
721  }
722 };
723 }
724 
725 std::unique_ptr<FileSystemStatCache> PTHManager::createStatCache() {
726  return llvm::make_unique<PTHStatCache>(*FileLookup);
727 }
IdentifierInfo * get(StringRef Name) override
get - Return the identifier token info for the specified named identifier.
Definition: PTHLexer.cpp:581
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file. ...
LookupResult getStat(StringRef Path, FileData &Data, bool isFile, std::unique_ptr< vfs::File > *F, vfs::FileSystem &FS) override
Definition: PTHLexer.cpp:697
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
static const unsigned StoredTokenSize
Definition: PTHLexer.cpp:30
Defines the clang::FileManager interface and associated types.
bool Lex(Token &Tok)
Lex - Return the next token.
Definition: PTHLexer.cpp:44
static PTHFileData ReadData(const internal_key_type &k, const unsigned char *d, unsigned)
Definition: PTHLexer.cpp:358
static std::pair< unsigned, unsigned > ReadKeyDataLength(const unsigned char *&d)
Definition: PTHLexer.cpp:391
Defines the FileSystemStatCache interface.
TypePropertyCache< Private > Cache
Definition: Type.cpp:3362
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1205
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:234
PTHLexer * CreateLexer(FileID FID)
CreateLexer - Return a PTHLexer that "lexes" the cached tokens for the specified file.
Definition: PTHLexer.cpp:594
uint32_t IdentifierID
An ID number that refers to an identifier in an AST file.
Definition: ASTBitCodes.h:55
static PTHManager * Create(StringRef file, DiagnosticsEngine &Diags)
Create - This method creates PTHManager objects.
Definition: PTHLexer.cpp:433
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:113
static hash_value_type ComputeHash(const internal_key_type &a)
Definition: PTHLexer.cpp:382
The virtual file system interface.
One of these records is kept for each identifier that is lexed.
bool ParsingPreprocessorDirective
True when parsing #XXX; turns &#39;\n&#39; into a tok::eod token.
SmallVector< PPConditionalInfo, 4 > ConditionalStack
Information about the set of #if/#ifdef/#ifndef blocks we are currently in.
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
static uint32_t ReadData(const internal_key_type &k, const unsigned char *d, unsigned)
Definition: PTHLexer.cpp:404
void setKind(tok::TokenKind K)
Definition: Token.h:91
Abstract interface for introducing a FileManager cache for &#39;stat&#39; system calls, which is used by prec...
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
uint32_t Offset
Definition: CacheTokens.cpp:43
const FormatToken & Tok
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:147
std::unique_ptr< FileSystemStatCache > createStatCache()
createStatCache - Returns a FileSystemStatCache object for use with FileManager objects.
Definition: PTHLexer.cpp:725
static internal_key_type GetInternalKey(const FileEntry *FE)
Definition: PTHLexer.cpp:350
const FileID FID
The SourceManager FileID corresponding to the file being lexed.
bool ParsingFilename
True after #include; turns <xx> into a tok::angle_string_literal token.
bool LexingRawMode
True if in raw mode.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
Defines the clang::Preprocessor interface.
MultipleIncludeOpt MIOpt
A state machine that detects the #ifndef-wrapping a file idiom for the multiple-include optimization...
void DiscardToEndOfLine()
DiscardToEndOfLine - Read the rest of the current preprocessor line as an uninterpreted string...
Definition: PTHLexer.cpp:158
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
The result type of a method or function.
friend class PTHManager
Definition: PTHLexer.h:61
SourceManager & getSourceManager() const
Definition: Preprocessor.h:732
SourceLocation getSourceLocation() override
getSourceLocation - Return a source location for the token in the current file.
Definition: PTHLexer.cpp:285
Encodes a location in the source.
llvm::sys::fs::UniqueID UniqueID
StringRef getName() const
Definition: FileManager.h:84
void setLength(unsigned Len)
Definition: Token.h:133
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
void setIdentifierInfo(IdentifierInfo *II)
Definition: Token.h:186
const char * getNameStart() const
Return the beginning of the actual null-terminated string for this identifier.
static bool EqualKey(const internal_key_type &a, const internal_key_type &b)
Definition: PTHLexer.cpp:376
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:266
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
Definition: Diagnostic.h:689
unsigned ComputeHash(Selector Sel)
Definition: ASTCommon.cpp:167
void getEOF(Token &Tok)
Definition: PTHLexer.cpp:153
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool SkipBlock()
SkipBlock - Used by Preprocessor to skip the current conditional block.
Definition: PTHLexer.cpp:187
Dataflow Directional Tag Classes.
~PTHManager() override
Definition: PTHLexer.cpp:426
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
void setLiteralData(const char *Ptr)
Definition: Token.h:219
PTHStatCache(PTHManager::PTHFileLookup &FL)
Definition: PTHLexer.cpp:693
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
Defines the clang::TokenKind enum and support functions.
const std::pair< const char *, unsigned > external_key_type
Definition: PTHLexer.cpp:371
void setLocation(SourceLocation L)
Definition: Token.h:132
static void InvalidPTH(DiagnosticsEngine &Diags, const char *Msg)
Definition: PTHLexer.cpp:429
static const internal_key_type & GetInternalKey(const external_key_type &x)
Definition: PTHLexer.cpp:388
static std::pair< const char *, unsigned > ReadKey(const unsigned char *d, unsigned n)
Definition: PTHLexer.cpp:399
static bool EqualKey(internal_key_type a, internal_key_type b)
Definition: PTHLexer.cpp:354
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void startToken()
Reset all flags to cleared.
Definition: Token.h:169
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:98
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.