clang  6.0.0svn
PTHLexer.cpp
Go to the documentation of this file.
1 //===- PTHLexer.cpp - Lex from a token stream -----------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the PTHLexer interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "clang/Lex/PTHLexer.h"
15 #include "clang/Basic/Diagnostic.h"
20 #include "clang/Basic/TokenKinds.h"
22 #include "clang/Lex/PTHManager.h"
23 #include "clang/Lex/Preprocessor.h"
24 #include "clang/Lex/Token.h"
25 #include "llvm/ADT/STLExtras.h"
26 #include "llvm/ADT/StringExtras.h"
27 #include "llvm/ADT/StringRef.h"
28 #include "llvm/Support/Endian.h"
29 #include "llvm/Support/ErrorOr.h"
30 #include "llvm/Support/FileSystem.h"
31 #include "llvm/Support/MemoryBuffer.h"
32 #include "llvm/Support/OnDiskHashTable.h"
33 #include <cassert>
34 #include <cstdint>
35 #include <cstdlib>
36 #include <cstring>
37 #include <ctime>
38 #include <memory>
39 #include <utility>
40 
41 using namespace clang;
42 
43 static const unsigned StoredTokenSize = 1 + 1 + 2 + 4 + 4;
44 
45 //===----------------------------------------------------------------------===//
46 // PTHLexer methods.
47 //===----------------------------------------------------------------------===//
48 
49 PTHLexer::PTHLexer(Preprocessor &PP, FileID FID, const unsigned char *D,
50  const unsigned char *ppcond, PTHManager &PM)
51  : PreprocessorLexer(&PP, FID), TokBuf(D), CurPtr(D), PPCond(ppcond),
52  CurPPCondPtr(ppcond), PTHMgr(PM) {
53  FileStartLoc = PP.getSourceManager().getLocForStartOfFile(FID);
54 }
55 
57  //===--------------------------------------==//
58  // Read the raw token data.
59  //===--------------------------------------==//
60  using namespace llvm::support;
61 
62  // Shadow CurPtr into an automatic variable.
63  const unsigned char *CurPtrShadow = CurPtr;
64 
65  // Read in the data for the token.
66  unsigned Word0 = endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
67  uint32_t IdentifierID =
68  endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
69  uint32_t FileOffset =
70  endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
71 
72  tok::TokenKind TKind = (tok::TokenKind) (Word0 & 0xFF);
73  Token::TokenFlags TFlags = (Token::TokenFlags) ((Word0 >> 8) & 0xFF);
74  uint32_t Len = Word0 >> 16;
75 
76  CurPtr = CurPtrShadow;
77 
78  //===--------------------------------------==//
79  // Construct the token itself.
80  //===--------------------------------------==//
81 
82  Tok.startToken();
83  Tok.setKind(TKind);
84  Tok.setFlag(TFlags);
85  assert(!LexingRawMode);
86  Tok.setLocation(FileStartLoc.getLocWithOffset(FileOffset));
87  Tok.setLength(Len);
88 
89  // Handle identifiers.
90  if (Tok.isLiteral()) {
91  Tok.setLiteralData((const char*) (PTHMgr.SpellingBase + IdentifierID));
92  }
93  else if (IdentifierID) {
94  MIOpt.ReadToken();
95  IdentifierInfo *II = PTHMgr.GetIdentifierInfo(IdentifierID-1);
96 
97  Tok.setIdentifierInfo(II);
98 
99  // Change the kind of this identifier to the appropriate token kind, e.g.
100  // turning "for" into a keyword.
101  Tok.setKind(II->getTokenID());
102 
103  if (II->isHandleIdentifierCase())
104  return PP->HandleIdentifier(Tok);
105 
106  return true;
107  }
108 
109  //===--------------------------------------==//
110  // Process the token.
111  //===--------------------------------------==//
112  if (TKind == tok::eof) {
113  // Save the end-of-file token.
114  EofToken = Tok;
115 
117  assert(!LexingRawMode);
118 
119  return LexEndOfFile(Tok);
120  }
121 
122  if (TKind == tok::hash && Tok.isAtStartOfLine()) {
123  LastHashTokPtr = CurPtr - StoredTokenSize;
124  assert(!LexingRawMode);
125  PP->HandleDirective(Tok);
126 
127  return false;
128  }
129 
130  if (TKind == tok::eod) {
133  return true;
134  }
135 
136  MIOpt.ReadToken();
137  return true;
138 }
139 
140 bool PTHLexer::LexEndOfFile(Token &Result) {
141  // If we hit the end of the file while parsing a preprocessor directive,
142  // end the preprocessor directive first. The next token returned will
143  // then be the end of file.
145  ParsingPreprocessorDirective = false; // Done parsing the "line".
146  return true; // Have a token.
147  }
148 
149  assert(!LexingRawMode);
150 
151  // If we are in a #if directive, emit an error.
152  while (!ConditionalStack.empty()) {
153  if (PP->getCodeCompletionFileLoc() != FileStartLoc)
154  PP->Diag(ConditionalStack.back().IfLoc,
155  diag::err_pp_unterminated_conditional);
156  ConditionalStack.pop_back();
157  }
158 
159  // Finally, let the preprocessor handle this.
160  return PP->HandleEndOfFile(Result);
161 }
162 
163 // FIXME: We can just grab the last token instead of storing a copy
164 // into EofToken.
166  assert(EofToken.is(tok::eof));
167  Tok = EofToken;
168 }
169 
171  assert(ParsingPreprocessorDirective && ParsingFilename == false &&
172  "Must be in a preprocessing directive!");
173 
174  // We assume that if the preprocessor wishes to discard to the end of
175  // the line that it also means to end the current preprocessor directive.
177 
178  // Skip tokens by only peeking at their token kind and the flags.
179  // We don't need to actually reconstruct full tokens from the token buffer.
180  // This saves some copies and it also reduces IdentifierInfo* lookup.
181  const unsigned char* p = CurPtr;
182  while (true) {
183  // Read the token kind. Are we at the end of the file?
184  tok::TokenKind x = (tok::TokenKind) (uint8_t) *p;
185  if (x == tok::eof) break;
186 
187  // Read the token flags. Are we at the start of the next line?
188  Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1];
189  if (y & Token::StartOfLine) break;
190 
191  // Skip to the next token.
192  p += StoredTokenSize;
193  }
194 
195  CurPtr = p;
196 }
197 
198 /// SkipBlock - Used by Preprocessor to skip the current conditional block.
200  using namespace llvm::support;
201 
202  assert(CurPPCondPtr && "No cached PP conditional information.");
203  assert(LastHashTokPtr && "No known '#' token.");
204 
205  const unsigned char *HashEntryI = nullptr;
206  uint32_t TableIdx;
207 
208  do {
209  // Read the token offset from the side-table.
210  uint32_t Offset = endian::readNext<uint32_t, little, aligned>(CurPPCondPtr);
211 
212  // Read the target table index from the side-table.
213  TableIdx = endian::readNext<uint32_t, little, aligned>(CurPPCondPtr);
214 
215  // Compute the actual memory address of the '#' token data for this entry.
216  HashEntryI = TokBuf + Offset;
217 
218  // Optmization: "Sibling jumping". #if...#else...#endif blocks can
219  // contain nested blocks. In the side-table we can jump over these
220  // nested blocks instead of doing a linear search if the next "sibling"
221  // entry is not at a location greater than LastHashTokPtr.
222  if (HashEntryI < LastHashTokPtr && TableIdx) {
223  // In the side-table we are still at an entry for a '#' token that
224  // is earlier than the last one we saw. Check if the location we would
225  // stride gets us closer.
226  const unsigned char* NextPPCondPtr =
227  PPCond + TableIdx*(sizeof(uint32_t)*2);
228  assert(NextPPCondPtr >= CurPPCondPtr);
229  // Read where we should jump to.
230  const unsigned char *HashEntryJ =
231  TokBuf + endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
232 
233  if (HashEntryJ <= LastHashTokPtr) {
234  // Jump directly to the next entry in the side table.
235  HashEntryI = HashEntryJ;
236  TableIdx = endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
237  CurPPCondPtr = NextPPCondPtr;
238  }
239  }
240  }
241  while (HashEntryI < LastHashTokPtr);
242  assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'");
243  assert(TableIdx && "No jumping from #endifs.");
244 
245  // Update our side-table iterator.
246  const unsigned char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2);
247  assert(NextPPCondPtr >= CurPPCondPtr);
248  CurPPCondPtr = NextPPCondPtr;
249 
250  // Read where we should jump to.
251  HashEntryI =
252  TokBuf + endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
253  uint32_t NextIdx = endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
254 
255  // By construction NextIdx will be zero if this is a #endif. This is useful
256  // to know to obviate lexing another token.
257  bool isEndif = NextIdx == 0;
258 
259  // This case can occur when we see something like this:
260  //
261  // #if ...
262  // /* a comment or nothing */
263  // #elif
264  //
265  // If we are skipping the first #if block it will be the case that CurPtr
266  // already points 'elif'. Just return.
267 
268  if (CurPtr > HashEntryI) {
269  assert(CurPtr == HashEntryI + StoredTokenSize);
270  // Did we reach a #endif? If so, go ahead and consume that token as well.
271  if (isEndif)
272  CurPtr += StoredTokenSize * 2;
273  else
274  LastHashTokPtr = HashEntryI;
275 
276  return isEndif;
277  }
278 
279  // Otherwise, we need to advance. Update CurPtr to point to the '#' token.
280  CurPtr = HashEntryI;
281 
282  // Update the location of the last observed '#'. This is useful if we
283  // are skipping multiple blocks.
284  LastHashTokPtr = CurPtr;
285 
286  // Skip the '#' token.
287  assert(((tok::TokenKind)*CurPtr) == tok::hash);
288  CurPtr += StoredTokenSize;
289 
290  // Did we reach a #endif? If so, go ahead and consume that token as well.
291  if (isEndif) {
292  CurPtr += StoredTokenSize * 2;
293  }
294 
295  return isEndif;
296 }
297 
299  // getSourceLocation is not on the hot path. It is used to get the location
300  // of the next token when transitioning back to this lexer when done
301  // handling a #included file. Just read the necessary data from the token
302  // data buffer to construct the SourceLocation object.
303  // NOTE: This is a virtual function; hence it is defined out-of-line.
304  using namespace llvm::support;
305 
306  const unsigned char *OffsetPtr = CurPtr + (StoredTokenSize - 4);
307  uint32_t Offset = endian::readNext<uint32_t, little, aligned>(OffsetPtr);
308  return FileStartLoc.getLocWithOffset(Offset);
309 }
310 
311 //===----------------------------------------------------------------------===//
312 // PTH file lookup: map from strings to file data.
313 //===----------------------------------------------------------------------===//
314 
315 /// PTHFileLookup - This internal data structure is used by the PTHManager
316 /// to map from FileEntry objects managed by FileManager to offsets within
317 /// the PTH file.
318 namespace {
319 
320 class PTHFileData {
321  const uint32_t TokenOff;
322  const uint32_t PPCondOff;
323 
324 public:
325  PTHFileData(uint32_t tokenOff, uint32_t ppCondOff)
326  : TokenOff(tokenOff), PPCondOff(ppCondOff) {}
327 
328  uint32_t getTokenOffset() const { return TokenOff; }
329  uint32_t getPPCondOffset() const { return PPCondOff; }
330 };
331 
332 class PTHFileLookupCommonTrait {
333 public:
334  using internal_key_type = std::pair<unsigned char, StringRef>;
335  using hash_value_type = unsigned;
336  using offset_type = unsigned;
337 
338  static hash_value_type ComputeHash(internal_key_type x) {
339  return llvm::HashString(x.second);
340  }
341 
342  static std::pair<unsigned, unsigned>
343  ReadKeyDataLength(const unsigned char*& d) {
344  using namespace llvm::support;
345 
346  unsigned keyLen =
347  (unsigned)endian::readNext<uint16_t, little, unaligned>(d);
348  unsigned dataLen = (unsigned) *(d++);
349  return std::make_pair(keyLen, dataLen);
350  }
351 
352  static internal_key_type ReadKey(const unsigned char* d, unsigned) {
353  unsigned char k = *(d++); // Read the entry kind.
354  return std::make_pair(k, (const char*) d);
355  }
356 };
357 
358 } // namespace
359 
360 class PTHManager::PTHFileLookupTrait : public PTHFileLookupCommonTrait {
361 public:
362  using external_key_type = const FileEntry *;
363  using data_type = PTHFileData;
364 
365  static internal_key_type GetInternalKey(const FileEntry* FE) {
366  return std::make_pair((unsigned char) 0x1, FE->getName());
367  }
368 
369  static bool EqualKey(internal_key_type a, internal_key_type b) {
370  return a.first == b.first && a.second == b.second;
371  }
372 
373  static PTHFileData ReadData(const internal_key_type& k,
374  const unsigned char* d, unsigned) {
375  using namespace llvm::support;
376 
377  assert(k.first == 0x1 && "Only file lookups can match!");
378  uint32_t x = endian::readNext<uint32_t, little, unaligned>(d);
379  uint32_t y = endian::readNext<uint32_t, little, unaligned>(d);
380  return PTHFileData(x, y);
381  }
382 };
383 
385 public:
386  using data_type = uint32_t;
387  using external_key_type = const std::pair<const char *, unsigned>;
389  using hash_value_type = uint32_t;
390  using offset_type = unsigned;
391 
392  static bool EqualKey(const internal_key_type& a,
393  const internal_key_type& b) {
394  return (a.second == b.second) ? memcmp(a.first, b.first, a.second) == 0
395  : false;
396  }
397 
399  return llvm::HashString(StringRef(a.first, a.second));
400  }
401 
402  // This hopefully will just get inlined and removed by the optimizer.
403  static const internal_key_type&
404  GetInternalKey(const external_key_type& x) { return x; }
405 
406  static std::pair<unsigned, unsigned>
407  ReadKeyDataLength(const unsigned char*& d) {
408  using namespace llvm::support;
409 
410  return std::make_pair(
411  (unsigned)endian::readNext<uint16_t, little, unaligned>(d),
412  sizeof(uint32_t));
413  }
414 
415  static std::pair<const char*, unsigned>
416  ReadKey(const unsigned char* d, unsigned n) {
417  assert(n >= 2 && d[n-1] == '\0');
418  return std::make_pair((const char*) d, n-1);
419  }
420 
421  static uint32_t ReadData(const internal_key_type& k, const unsigned char* d,
422  unsigned) {
423  using namespace llvm::support;
424 
425  return endian::readNext<uint32_t, little, unaligned>(d);
426  }
427 };
428 
429 //===----------------------------------------------------------------------===//
430 // PTHManager methods.
431 //===----------------------------------------------------------------------===//
432 
433 PTHManager::PTHManager(
434  std::unique_ptr<const llvm::MemoryBuffer> buf,
435  std::unique_ptr<PTHFileLookup> fileLookup, const unsigned char *idDataTable,
436  std::unique_ptr<IdentifierInfo *[], llvm::FreeDeleter> perIDCache,
437  std::unique_ptr<PTHStringIdLookup> stringIdLookup, unsigned numIds,
438  const unsigned char *spellingBase, const char *originalSourceFile)
439  : Buf(std::move(buf)), PerIDCache(std::move(perIDCache)),
440  FileLookup(std::move(fileLookup)), IdDataTable(idDataTable),
441  StringIdLookup(std::move(stringIdLookup)), NumIds(numIds),
442  SpellingBase(spellingBase), OriginalSourceFile(originalSourceFile) {}
443 
444 PTHManager::~PTHManager() = default;
445 
446 static void InvalidPTH(DiagnosticsEngine &Diags, const char *Msg) {
447  Diags.Report(Diags.getCustomDiagID(DiagnosticsEngine::Error, "%0")) << Msg;
448 }
449 
451  // Memory map the PTH file.
452  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileOrErr =
453  llvm::MemoryBuffer::getFile(file);
454 
455  if (!FileOrErr) {
456  // FIXME: Add ec.message() to this diag.
457  Diags.Report(diag::err_invalid_pth_file) << file;
458  return nullptr;
459  }
460  std::unique_ptr<llvm::MemoryBuffer> File = std::move(FileOrErr.get());
461 
462  using namespace llvm::support;
463 
464  // Get the buffer ranges and check if there are at least three 32-bit
465  // words at the end of the file.
466  const unsigned char *BufBeg = (const unsigned char*)File->getBufferStart();
467  const unsigned char *BufEnd = (const unsigned char*)File->getBufferEnd();
468 
469  // Check the prologue of the file.
470  if ((BufEnd - BufBeg) < (signed)(sizeof("cfe-pth") + 4 + 4) ||
471  memcmp(BufBeg, "cfe-pth", sizeof("cfe-pth")) != 0) {
472  Diags.Report(diag::err_invalid_pth_file) << file;
473  return nullptr;
474  }
475 
476  // Read the PTH version.
477  const unsigned char *p = BufBeg + (sizeof("cfe-pth"));
478  unsigned Version = endian::readNext<uint32_t, little, aligned>(p);
479 
480  if (Version < PTHManager::Version) {
481  InvalidPTH(Diags,
482  Version < PTHManager::Version
483  ? "PTH file uses an older PTH format that is no longer supported"
484  : "PTH file uses a newer PTH format that cannot be read");
485  return nullptr;
486  }
487 
488  // Compute the address of the index table at the end of the PTH file.
489  const unsigned char *PrologueOffset = p;
490 
491  if (PrologueOffset >= BufEnd) {
492  Diags.Report(diag::err_invalid_pth_file) << file;
493  return nullptr;
494  }
495 
496  // Construct the file lookup table. This will be used for mapping from
497  // FileEntry*'s to cached tokens.
498  const unsigned char* FileTableOffset = PrologueOffset + sizeof(uint32_t)*2;
499  const unsigned char *FileTable =
500  BufBeg + endian::readNext<uint32_t, little, aligned>(FileTableOffset);
501 
502  if (!(FileTable > BufBeg && FileTable < BufEnd)) {
503  Diags.Report(diag::err_invalid_pth_file) << file;
504  return nullptr; // FIXME: Proper error diagnostic?
505  }
506 
507  std::unique_ptr<PTHFileLookup> FL(PTHFileLookup::Create(FileTable, BufBeg));
508 
509  // Warn if the PTH file is empty. We still want to create a PTHManager
510  // as the PTH could be used with -include-pth.
511  if (FL->isEmpty())
512  InvalidPTH(Diags, "PTH file contains no cached source data");
513 
514  // Get the location of the table mapping from persistent ids to the
515  // data needed to reconstruct identifiers.
516  const unsigned char* IDTableOffset = PrologueOffset + sizeof(uint32_t)*0;
517  const unsigned char *IData =
518  BufBeg + endian::readNext<uint32_t, little, aligned>(IDTableOffset);
519 
520  if (!(IData >= BufBeg && IData < BufEnd)) {
521  Diags.Report(diag::err_invalid_pth_file) << file;
522  return nullptr;
523  }
524 
525  // Get the location of the hashtable mapping between strings and
526  // persistent IDs.
527  const unsigned char* StringIdTableOffset = PrologueOffset + sizeof(uint32_t)*1;
528  const unsigned char *StringIdTable =
529  BufBeg + endian::readNext<uint32_t, little, aligned>(StringIdTableOffset);
530  if (!(StringIdTable >= BufBeg && StringIdTable < BufEnd)) {
531  Diags.Report(diag::err_invalid_pth_file) << file;
532  return nullptr;
533  }
534 
535  std::unique_ptr<PTHStringIdLookup> SL(
536  PTHStringIdLookup::Create(StringIdTable, BufBeg));
537 
538  // Get the location of the spelling cache.
539  const unsigned char* spellingBaseOffset = PrologueOffset + sizeof(uint32_t)*3;
540  const unsigned char *spellingBase =
541  BufBeg + endian::readNext<uint32_t, little, aligned>(spellingBaseOffset);
542  if (!(spellingBase >= BufBeg && spellingBase < BufEnd)) {
543  Diags.Report(diag::err_invalid_pth_file) << file;
544  return nullptr;
545  }
546 
547  // Get the number of IdentifierInfos and pre-allocate the identifier cache.
548  uint32_t NumIds = endian::readNext<uint32_t, little, aligned>(IData);
549 
550  // Pre-allocate the persistent ID -> IdentifierInfo* cache. We use calloc()
551  // so that we in the best case only zero out memory once when the OS returns
552  // us new pages.
553  std::unique_ptr<IdentifierInfo *[], llvm::FreeDeleter> PerIDCache;
554 
555  if (NumIds) {
556  PerIDCache.reset((IdentifierInfo **)calloc(NumIds, sizeof(PerIDCache[0])));
557  if (!PerIDCache) {
558  InvalidPTH(Diags, "Could not allocate memory for processing PTH file");
559  return nullptr;
560  }
561  }
562 
563  // Compute the address of the original source file.
564  const unsigned char* originalSourceBase = PrologueOffset + sizeof(uint32_t)*4;
565  unsigned len =
566  endian::readNext<uint16_t, little, unaligned>(originalSourceBase);
567  if (!len) originalSourceBase = nullptr;
568 
569  // Create the new PTHManager.
570  return new PTHManager(std::move(File), std::move(FL), IData,
571  std::move(PerIDCache), std::move(SL), NumIds,
572  spellingBase, (const char *)originalSourceBase);
573 }
574 
575 IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(unsigned PersistentID) {
576  using namespace llvm::support;
577 
578  // Look in the PTH file for the string data for the IdentifierInfo object.
579  const unsigned char* TableEntry = IdDataTable + sizeof(uint32_t)*PersistentID;
580  const unsigned char *IDData =
581  (const unsigned char *)Buf->getBufferStart() +
582  endian::readNext<uint32_t, little, aligned>(TableEntry);
583  assert(IDData < (const unsigned char*)Buf->getBufferEnd());
584 
585  // Allocate the object.
586  std::pair<IdentifierInfo,const unsigned char*> *Mem =
587  Alloc.Allocate<std::pair<IdentifierInfo, const unsigned char *>>();
588 
589  Mem->second = IDData;
590  assert(IDData[0] != '\0');
591  IdentifierInfo *II = new ((void*) Mem) IdentifierInfo();
592 
593  // Store the new IdentifierInfo in the cache.
594  PerIDCache[PersistentID] = II;
595  assert(II->getNameStart() && II->getNameStart()[0] != '\0');
596  return II;
597 }
598 
599 IdentifierInfo* PTHManager::get(StringRef Name) {
600  // Double check our assumption that the last character isn't '\0'.
601  assert(Name.empty() || Name.back() != '\0');
602  PTHStringIdLookup::iterator I =
603  StringIdLookup->find(std::make_pair(Name.data(), Name.size()));
604  if (I == StringIdLookup->end()) // No identifier found?
605  return nullptr;
606 
607  // Match found. Return the identifier!
608  assert(*I > 0);
609  return GetIdentifierInfo(*I-1);
610 }
611 
613  const FileEntry *FE = PP->getSourceManager().getFileEntryForID(FID);
614  if (!FE)
615  return nullptr;
616 
617  using namespace llvm::support;
618 
619  // Lookup the FileEntry object in our file lookup data structure. It will
620  // return a variant that indicates whether or not there is an offset within
621  // the PTH file that contains cached tokens.
622  PTHFileLookup::iterator I = FileLookup->find(FE);
623 
624  if (I == FileLookup->end()) // No tokens available?
625  return nullptr;
626 
627  const PTHFileData& FileData = *I;
628 
629  const unsigned char *BufStart = (const unsigned char *)Buf->getBufferStart();
630  // Compute the offset of the token data within the buffer.
631  const unsigned char* data = BufStart + FileData.getTokenOffset();
632 
633  // Get the location of pp-conditional table.
634  const unsigned char* ppcond = BufStart + FileData.getPPCondOffset();
635  uint32_t Len = endian::readNext<uint32_t, little, aligned>(ppcond);
636  if (Len == 0) ppcond = nullptr;
637 
638  assert(PP && "No preprocessor set yet!");
639  return new PTHLexer(*PP, FID, data, ppcond, *this);
640 }
641 
642 //===----------------------------------------------------------------------===//
643 // 'stat' caching.
644 //===----------------------------------------------------------------------===//
645 
646 namespace {
647 
648 class PTHStatData {
649 public:
650  uint64_t Size;
651  time_t ModTime;
652  llvm::sys::fs::UniqueID UniqueID;
653  const bool HasData = false;
654  bool IsDirectory;
655 
656  PTHStatData() = default;
657  PTHStatData(uint64_t Size, time_t ModTime, llvm::sys::fs::UniqueID UniqueID,
658  bool IsDirectory)
659  : Size(Size), ModTime(ModTime), UniqueID(UniqueID), HasData(true),
660  IsDirectory(IsDirectory) {}
661 };
662 
663 class PTHStatLookupTrait : public PTHFileLookupCommonTrait {
664 public:
665  using external_key_type = StringRef; // const char*
666  using data_type = PTHStatData;
667 
668  static internal_key_type GetInternalKey(StringRef path) {
669  // The key 'kind' doesn't matter here because it is ignored in EqualKey.
670  return std::make_pair((unsigned char) 0x0, path);
671  }
672 
673  static bool EqualKey(internal_key_type a, internal_key_type b) {
674  // When doing 'stat' lookups we don't care about the kind of 'a' and 'b',
675  // just the paths.
676  return a.second == b.second;
677  }
678 
679  static data_type ReadData(const internal_key_type& k, const unsigned char* d,
680  unsigned) {
681  if (k.first /* File or Directory */) {
682  bool IsDirectory = true;
683  if (k.first == 0x1 /* File */) {
684  IsDirectory = false;
685  d += 4 * 2; // Skip the first 2 words.
686  }
687 
688  using namespace llvm::support;
689 
690  uint64_t File = endian::readNext<uint64_t, little, unaligned>(d);
691  uint64_t Device = endian::readNext<uint64_t, little, unaligned>(d);
692  llvm::sys::fs::UniqueID UniqueID(Device, File);
693  time_t ModTime = endian::readNext<uint64_t, little, unaligned>(d);
694  uint64_t Size = endian::readNext<uint64_t, little, unaligned>(d);
695  return data_type(Size, ModTime, UniqueID, IsDirectory);
696  }
697 
698  // Negative stat. Don't read anything.
699  return data_type();
700  }
701 };
702 
703 } // namespace
704 
705 namespace clang {
706 
709 
710  CacheTy Cache;
711 
712 public:
714  : Cache(FL.getNumBuckets(), FL.getNumEntries(), FL.getBuckets(),
715  FL.getBase()) {}
716 
717  LookupResult getStat(StringRef Path, FileData &Data, bool isFile,
718  std::unique_ptr<vfs::File> *F,
719  vfs::FileSystem &FS) override {
720  // Do the lookup for the file's data in the PTH file.
721  CacheTy::iterator I = Cache.find(Path);
722 
723  // If we don't get a hit in the PTH file just forward to 'stat'.
724  if (I == Cache.end())
725  return statChained(Path, Data, isFile, F, FS);
726 
727  const PTHStatData &D = *I;
728 
729  if (!D.HasData)
730  return CacheMissing;
731 
732  Data.Name = Path;
733  Data.Size = D.Size;
734  Data.ModTime = D.ModTime;
735  Data.UniqueID = D.UniqueID;
736  Data.IsDirectory = D.IsDirectory;
737  Data.IsNamedPipe = false;
738  Data.InPCH = true;
739 
740  return CacheExists;
741  }
742 };
743 
744 } // namespace clang
745 
746 std::unique_ptr<FileSystemStatCache> PTHManager::createStatCache() {
747  return llvm::make_unique<PTHStatCache>(*FileLookup);
748 }
IdentifierInfo * get(StringRef Name) override
get - Return the identifier token info for the specified named identifier.
Definition: PTHLexer.cpp:599
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file. ...
LookupResult getStat(StringRef Path, FileData &Data, bool isFile, std::unique_ptr< vfs::File > *F, vfs::FileSystem &FS) override
Definition: PTHLexer.cpp:717
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
static const unsigned StoredTokenSize
Definition: PTHLexer.cpp:43
Defines the clang::FileManager interface and associated types.
TypePropertyCache< Private > Cache
Definition: Type.cpp:3396
bool Lex(Token &Tok)
Lex - Return the next token.
Definition: PTHLexer.cpp:56
Defines the SourceManager interface.
static PTHFileData ReadData(const internal_key_type &k, const unsigned char *d, unsigned)
Definition: PTHLexer.cpp:373
static std::pair< unsigned, unsigned > ReadKeyDataLength(const unsigned char *&d)
Definition: PTHLexer.cpp:407
Defines the FileSystemStatCache interface.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1207
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:234
PTHLexer * CreateLexer(FileID FID)
CreateLexer - Return a PTHLexer that "lexes" the cached tokens for the specified file.
Definition: PTHLexer.cpp:612
PTHLexer(Preprocessor &pp, FileID FID, const unsigned char *D, const unsigned char *ppcond, PTHManager &PM)
Create a PTHLexer for the specified token stream.
Definition: PTHLexer.cpp:49
static PTHManager * Create(StringRef file, DiagnosticsEngine &Diags)
Create - This method creates PTHManager objects.
Definition: PTHLexer.cpp:450
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:113
static hash_value_type ComputeHash(const internal_key_type &a)
Definition: PTHLexer.cpp:398
The virtual file system interface.
One of these records is kept for each identifier that is lexed.
bool ParsingPreprocessorDirective
True when parsing #XXX; turns &#39;\n&#39; into a tok::eod token.
SmallVector< PPConditionalInfo, 4 > ConditionalStack
Information about the set of #if/#ifdef/#ifndef blocks we are currently in.
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
static uint32_t ReadData(const internal_key_type &k, const unsigned char *d, unsigned)
Definition: PTHLexer.cpp:421
void setKind(tok::TokenKind K)
Definition: Token.h:91
Abstract interface for introducing a FileManager cache for &#39;stat&#39; system calls, which is used by prec...
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
uint32_t Offset
Definition: CacheTokens.cpp:43
const FormatToken & Tok
const std::pair< const char *, unsigned > external_key_type
Definition: PTHLexer.cpp:387
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:147
std::unique_ptr< FileSystemStatCache > createStatCache()
createStatCache - Returns a FileSystemStatCache object for use with FileManager objects.
Definition: PTHLexer.cpp:746
Defines the Diagnostic-related interfaces.
static internal_key_type GetInternalKey(const FileEntry *FE)
Definition: PTHLexer.cpp:365
const FileID FID
The SourceManager FileID corresponding to the file being lexed.
bool ParsingFilename
True after #include; turns <xx> into a tok::angle_string_literal token.
bool LexingRawMode
True if in raw mode.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
Defines the clang::Preprocessor interface.
MultipleIncludeOpt MIOpt
A state machine that detects the #ifndef-wrapping a file idiom for the multiple-include optimization...
void DiscardToEndOfLine()
DiscardToEndOfLine - Read the rest of the current preprocessor line as an uninterpreted string...
Definition: PTHLexer.cpp:170
uint32_t IdentifierID
An ID number that refers to an identifier in an AST file.
Definition: ASTBitCodes.h:61
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
The result type of a method or function.
friend class PTHManager
Definition: PTHLexer.h:61
SourceManager & getSourceManager() const
Definition: Preprocessor.h:819
SourceLocation getSourceLocation() override
getSourceLocation - Return a source location for the token in the current file.
Definition: PTHLexer.cpp:298
Encodes a location in the source.
llvm::sys::fs::UniqueID UniqueID
StringRef getName() const
Definition: FileManager.h:84
void setLength(unsigned Len)
Definition: Token.h:133
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
void setIdentifierInfo(IdentifierInfo *II)
Definition: Token.h:186
const char * getNameStart() const
Return the beginning of the actual null-terminated string for this identifier.
static bool EqualKey(const internal_key_type &a, const internal_key_type &b)
Definition: PTHLexer.cpp:392
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:266
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
Definition: Diagnostic.h:691
unsigned ComputeHash(Selector Sel)
Definition: ASTCommon.cpp:167
void getEOF(Token &Tok)
Definition: PTHLexer.cpp:165
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool SkipBlock()
SkipBlock - Used by Preprocessor to skip the current conditional block.
Definition: PTHLexer.cpp:199
Dataflow Directional Tag Classes.
~PTHManager() override
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
void setLiteralData(const char *Ptr)
Definition: Token.h:219
PTHStatCache(PTHManager::PTHFileLookup &FL)
Definition: PTHLexer.cpp:713
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
Defines the clang::TokenKind enum and support functions.
void setLocation(SourceLocation L)
Definition: Token.h:132
static void InvalidPTH(DiagnosticsEngine &Diags, const char *Msg)
Definition: PTHLexer.cpp:446
static const internal_key_type & GetInternalKey(const external_key_type &x)
Definition: PTHLexer.cpp:404
static std::pair< const char *, unsigned > ReadKey(const unsigned char *d, unsigned n)
Definition: PTHLexer.cpp:416
static bool EqualKey(internal_key_type a, internal_key_type b)
Definition: PTHLexer.cpp:369
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void startToken()
Reset all flags to cleared.
Definition: Token.h:169
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:127
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.