clang  9.0.0svn
Preprocessor.cpp
Go to the documentation of this file.
1 //===- Preprocess.cpp - C Language Family Preprocessor Implementation -----===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the Preprocessor interface.
11 //
12 //===----------------------------------------------------------------------===//
13 //
14 // Options to support:
15 // -H - Print the name of each header file used.
16 // -d[DNI] - Dump various things.
17 // -fworking-directory - #line's with preprocessor's working dir.
18 // -fpreprocessed
19 // -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
20 // -W*
21 // -w
22 //
23 // Messages to emit:
24 // "Multiple include guards may be useful for:\n"
25 //
26 //===----------------------------------------------------------------------===//
27 
28 #include "clang/Lex/Preprocessor.h"
32 #include "clang/Basic/LLVM.h"
34 #include "clang/Basic/Module.h"
37 #include "clang/Basic/TargetInfo.h"
40 #include "clang/Lex/HeaderSearch.h"
42 #include "clang/Lex/Lexer.h"
44 #include "clang/Lex/MacroArgs.h"
45 #include "clang/Lex/MacroInfo.h"
46 #include "clang/Lex/ModuleLoader.h"
47 #include "clang/Lex/Pragma.h"
52 #include "clang/Lex/Token.h"
53 #include "clang/Lex/TokenLexer.h"
54 #include "llvm/ADT/APInt.h"
55 #include "llvm/ADT/ArrayRef.h"
56 #include "llvm/ADT/DenseMap.h"
57 #include "llvm/ADT/SmallString.h"
58 #include "llvm/ADT/SmallVector.h"
59 #include "llvm/ADT/STLExtras.h"
60 #include "llvm/ADT/StringRef.h"
61 #include "llvm/ADT/StringSwitch.h"
62 #include "llvm/Support/Capacity.h"
63 #include "llvm/Support/ErrorHandling.h"
64 #include "llvm/Support/MemoryBuffer.h"
65 #include "llvm/Support/raw_ostream.h"
66 #include <algorithm>
67 #include <cassert>
68 #include <memory>
69 #include <string>
70 #include <utility>
71 #include <vector>
72 
73 using namespace clang;
74 
75 LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
76 
78 
79 Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
80  DiagnosticsEngine &diags, LangOptions &opts,
82  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
83  IdentifierInfoLookup *IILookup, bool OwnsHeaders,
84  TranslationUnitKind TUKind)
85  : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts),
86  FileMgr(Headers.getFileMgr()), SourceMgr(SM), PCMCache(PCMCache),
87  ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
88  TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
89  // As the language options may have not been loaded yet (when
90  // deserializing an ASTUnit), adding keywords to the identifier table is
91  // deferred to Preprocessor::Initialize().
92  Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
93  TUKind(TUKind), SkipMainFilePreamble(0, true),
94  CurSubmoduleState(&NullSubmoduleState) {
95  OwnsHeaderSearch = OwnsHeaders;
96 
97  // Default to discarding comments.
98  KeepComments = false;
99  KeepMacroComments = false;
100  SuppressIncludeNotFoundError = false;
101 
102  // Macro expansion is enabled.
103  DisableMacroExpansion = false;
104  MacroExpansionInDirectivesOverride = false;
105  InMacroArgs = false;
106  InMacroArgPreExpansion = false;
107  NumCachedTokenLexers = 0;
108  PragmasEnabled = true;
109  ParsingIfOrElifDirective = false;
110  PreprocessedOutput = false;
111 
112  // We haven't read anything from the external source.
113  ReadMacrosFromExternalSource = false;
114 
115  // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
116  // a macro. They get unpoisoned where it is allowed.
117  (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
118  SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
119  if (getLangOpts().CPlusPlus2a) {
120  (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned();
121  SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use);
122  } else {
123  Ident__VA_OPT__ = nullptr;
124  }
125 
126  // Initialize the pragma handlers.
127  RegisterBuiltinPragmas();
128 
129  // Initialize builtin macros like __LINE__ and friends.
130  RegisterBuiltinMacros();
131 
132  if(LangOpts.Borland) {
133  Ident__exception_info = getIdentifierInfo("_exception_info");
134  Ident___exception_info = getIdentifierInfo("__exception_info");
135  Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation");
136  Ident__exception_code = getIdentifierInfo("_exception_code");
137  Ident___exception_code = getIdentifierInfo("__exception_code");
138  Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode");
139  Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination");
140  Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
141  Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination");
142  } else {
143  Ident__exception_info = Ident__exception_code = nullptr;
144  Ident__abnormal_termination = Ident___exception_info = nullptr;
145  Ident___exception_code = Ident___abnormal_termination = nullptr;
146  Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
147  Ident_AbnormalTermination = nullptr;
148  }
149 
150  // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
152  SkippingUntilPragmaHdrStop = true;
153 
154  // If using a PCH with a through header, start skipping tokens.
155  if (!this->PPOpts->PCHThroughHeader.empty() &&
156  !this->PPOpts->ImplicitPCHInclude.empty())
157  SkippingUntilPCHThroughHeader = true;
158 
159  if (this->PPOpts->GeneratePreamble)
160  PreambleConditionalStack.startRecording();
161 }
162 
164  assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
165 
166  IncludeMacroStack.clear();
167 
168  // Destroy any macro definitions.
169  while (MacroInfoChain *I = MIChainHead) {
170  MIChainHead = I->Next;
171  I->~MacroInfoChain();
172  }
173 
174  // Free any cached macro expanders.
175  // This populates MacroArgCache, so all TokenLexers need to be destroyed
176  // before the code below that frees up the MacroArgCache list.
177  std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
178  CurTokenLexer.reset();
179 
180  // Free any cached MacroArgs.
181  for (MacroArgs *ArgList = MacroArgCache; ArgList;)
182  ArgList = ArgList->deallocate();
183 
184  // Delete the header search info, if we own it.
185  if (OwnsHeaderSearch)
186  delete &HeaderInfo;
187 }
188 
190  const TargetInfo *AuxTarget) {
191  assert((!this->Target || this->Target == &Target) &&
192  "Invalid override of target information");
193  this->Target = &Target;
194 
195  assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
196  "Invalid override of aux target information.");
197  this->AuxTarget = AuxTarget;
198 
199  // Initialize information about built-ins.
200  BuiltinInfo.InitializeTarget(Target, AuxTarget);
201  HeaderInfo.setTarget(Target);
202 
203  // Populate the identifier table with info about keywords for the current language.
204  Identifiers.AddKeywords(LangOpts);
205 }
206 
208  NumEnteredSourceFiles = 0;
209 
210  // Reset pragmas
211  PragmaHandlersBackup = std::move(PragmaHandlers);
212  PragmaHandlers = llvm::make_unique<PragmaNamespace>(StringRef());
213  RegisterBuiltinPragmas();
214 
215  // Reset PredefinesFileID
216  PredefinesFileID = FileID();
217 }
218 
220  NumEnteredSourceFiles = 1;
221 
222  PragmaHandlers = std::move(PragmaHandlersBackup);
223 }
224 
225 void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
226  llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
227  << getSpelling(Tok) << "'";
228 
229  if (!DumpFlags) return;
230 
231  llvm::errs() << "\t";
232  if (Tok.isAtStartOfLine())
233  llvm::errs() << " [StartOfLine]";
234  if (Tok.hasLeadingSpace())
235  llvm::errs() << " [LeadingSpace]";
236  if (Tok.isExpandDisabled())
237  llvm::errs() << " [ExpandDisabled]";
238  if (Tok.needsCleaning()) {
239  const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
240  llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
241  << "']";
242  }
243 
244  llvm::errs() << "\tLoc=<";
245  DumpLocation(Tok.getLocation());
246  llvm::errs() << ">";
247 }
248 
250  Loc.print(llvm::errs(), SourceMgr);
251 }
252 
253 void Preprocessor::DumpMacro(const MacroInfo &MI) const {
254  llvm::errs() << "MACRO: ";
255  for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
257  llvm::errs() << " ";
258  }
259  llvm::errs() << "\n";
260 }
261 
263  llvm::errs() << "\n*** Preprocessor Stats:\n";
264  llvm::errs() << NumDirectives << " directives found:\n";
265  llvm::errs() << " " << NumDefined << " #define.\n";
266  llvm::errs() << " " << NumUndefined << " #undef.\n";
267  llvm::errs() << " #include/#include_next/#import:\n";
268  llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
269  llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
270  llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
271  llvm::errs() << " " << NumElse << " #else/#elif.\n";
272  llvm::errs() << " " << NumEndif << " #endif.\n";
273  llvm::errs() << " " << NumPragma << " #pragma.\n";
274  llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
275 
276  llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
277  << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
278  << NumFastMacroExpanded << " on the fast path.\n";
279  llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
280  << " token paste (##) operations performed, "
281  << NumFastTokenPaste << " on the fast path.\n";
282 
283  llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
284 
285  llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
286  llvm::errs() << "\n Macro Expanded Tokens: "
287  << llvm::capacity_in_bytes(MacroExpandedTokens);
288  llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
289  // FIXME: List information for all submodules.
290  llvm::errs() << "\n Macros: "
291  << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
292  llvm::errs() << "\n #pragma push_macro Info: "
293  << llvm::capacity_in_bytes(PragmaPushMacroInfo);
294  llvm::errs() << "\n Poison Reasons: "
295  << llvm::capacity_in_bytes(PoisonReasons);
296  llvm::errs() << "\n Comment Handlers: "
297  << llvm::capacity_in_bytes(CommentHandlers) << "\n";
298 }
299 
301 Preprocessor::macro_begin(bool IncludeExternalMacros) const {
302  if (IncludeExternalMacros && ExternalSource &&
303  !ReadMacrosFromExternalSource) {
304  ReadMacrosFromExternalSource = true;
305  ExternalSource->ReadDefinedMacros();
306  }
307 
308  // Make sure we cover all macros in visible modules.
309  for (const ModuleMacro &Macro : ModuleMacros)
310  CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
311 
312  return CurSubmoduleState->Macros.begin();
313 }
314 
316  return BP.getTotalMemory()
317  + llvm::capacity_in_bytes(MacroExpandedTokens)
318  + Predefines.capacity() /* Predefines buffer. */
319  // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
320  // and ModuleMacros.
321  + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
322  + llvm::capacity_in_bytes(PragmaPushMacroInfo)
323  + llvm::capacity_in_bytes(PoisonReasons)
324  + llvm::capacity_in_bytes(CommentHandlers);
325 }
326 
328 Preprocessor::macro_end(bool IncludeExternalMacros) const {
329  if (IncludeExternalMacros && ExternalSource &&
330  !ReadMacrosFromExternalSource) {
331  ReadMacrosFromExternalSource = true;
332  ExternalSource->ReadDefinedMacros();
333  }
334 
335  return CurSubmoduleState->Macros.end();
336 }
337 
338 /// Compares macro tokens with a specified token value sequence.
339 static bool MacroDefinitionEquals(const MacroInfo *MI,
340  ArrayRef<TokenValue> Tokens) {
341  return Tokens.size() == MI->getNumTokens() &&
342  std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
343 }
344 
346  SourceLocation Loc,
347  ArrayRef<TokenValue> Tokens) const {
348  SourceLocation BestLocation;
349  StringRef BestSpelling;
351  I != E; ++I) {
353  Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
354  if (!Def || !Def.getMacroInfo())
355  continue;
356  if (!Def.getMacroInfo()->isObjectLike())
357  continue;
358  if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
359  continue;
360  SourceLocation Location = Def.getLocation();
361  // Choose the macro defined latest.
362  if (BestLocation.isInvalid() ||
363  (Location.isValid() &&
364  SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
365  BestLocation = Location;
366  BestSpelling = I->first->getName();
367  }
368  }
369  return BestSpelling;
370 }
371 
373  if (CurLexer)
374  CurLexerKind = CLK_Lexer;
375  else if (CurTokenLexer)
376  CurLexerKind = CLK_TokenLexer;
377  else
378  CurLexerKind = CLK_CachingLexer;
379 }
380 
382  unsigned CompleteLine,
383  unsigned CompleteColumn) {
384  assert(File);
385  assert(CompleteLine && CompleteColumn && "Starts from 1:1");
386  assert(!CodeCompletionFile && "Already set");
387 
388  using llvm::MemoryBuffer;
389 
390  // Load the actual file's contents.
391  bool Invalid = false;
392  const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
393  if (Invalid)
394  return true;
395 
396  // Find the byte position of the truncation point.
397  const char *Position = Buffer->getBufferStart();
398  for (unsigned Line = 1; Line < CompleteLine; ++Line) {
399  for (; *Position; ++Position) {
400  if (*Position != '\r' && *Position != '\n')
401  continue;
402 
403  // Eat \r\n or \n\r as a single line.
404  if ((Position[1] == '\r' || Position[1] == '\n') &&
405  Position[0] != Position[1])
406  ++Position;
407  ++Position;
408  break;
409  }
410  }
411 
412  Position += CompleteColumn - 1;
413 
414  // If pointing inside the preamble, adjust the position at the beginning of
415  // the file after the preamble.
416  if (SkipMainFilePreamble.first &&
417  SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) {
418  if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
419  Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
420  }
421 
422  if (Position > Buffer->getBufferEnd())
423  Position = Buffer->getBufferEnd();
424 
425  CodeCompletionFile = File;
426  CodeCompletionOffset = Position - Buffer->getBufferStart();
427 
428  auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
429  Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
430  char *NewBuf = NewBuffer->getBufferStart();
431  char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
432  *NewPos = '\0';
433  std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
434  SourceMgr.overrideFileContents(File, std::move(NewBuffer));
435 
436  return false;
437 }
438 
439 void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
440  bool IsAngled) {
441  if (CodeComplete)
442  CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
444 }
445 
447  if (CodeComplete)
448  CodeComplete->CodeCompleteNaturalLanguage();
450 }
451 
452 /// getSpelling - This method is used to get the spelling of a token into a
453 /// SmallVector. Note that the returned StringRef may not point to the
454 /// supplied buffer if a copy can be avoided.
456  SmallVectorImpl<char> &Buffer,
457  bool *Invalid) const {
458  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
459  if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
460  // Try the fast path.
461  if (const IdentifierInfo *II = Tok.getIdentifierInfo())
462  return II->getName();
463  }
464 
465  // Resize the buffer if we need to copy into it.
466  if (Tok.needsCleaning())
467  Buffer.resize(Tok.getLength());
468 
469  const char *Ptr = Buffer.data();
470  unsigned Len = getSpelling(Tok, Ptr, Invalid);
471  return StringRef(Ptr, Len);
472 }
473 
474 /// CreateString - Plop the specified string into a scratch buffer and return a
475 /// location for it. If specified, the source location provides a source
476 /// location for the token.
477 void Preprocessor::CreateString(StringRef Str, Token &Tok,
478  SourceLocation ExpansionLocStart,
479  SourceLocation ExpansionLocEnd) {
480  Tok.setLength(Str.size());
481 
482  const char *DestPtr;
483  SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
484 
485  if (ExpansionLocStart.isValid())
486  Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
487  ExpansionLocEnd, Str.size());
488  Tok.setLocation(Loc);
489 
490  // If this is a raw identifier or a literal token, set the pointer data.
491  if (Tok.is(tok::raw_identifier))
492  Tok.setRawIdentifierData(DestPtr);
493  else if (Tok.isLiteral())
494  Tok.setLiteralData(DestPtr);
495 }
496 
498  auto &SM = getSourceManager();
499  SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
500  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc);
501  bool Invalid = false;
502  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
503  if (Invalid)
504  return SourceLocation();
505 
506  // FIXME: We could consider re-using spelling for tokens we see repeatedly.
507  const char *DestPtr;
508  SourceLocation Spelling =
509  ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
510  return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length));
511 }
512 
514  if (!getLangOpts().isCompilingModule())
515  return nullptr;
516 
517  return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
518 }
519 
520 //===----------------------------------------------------------------------===//
521 // Preprocessor Initialization Methods
522 //===----------------------------------------------------------------------===//
523 
524 /// EnterMainSourceFile - Enter the specified FileID as the main source file,
525 /// which implicitly adds the builtin defines etc.
527  // We do not allow the preprocessor to reenter the main file. Doing so will
528  // cause FileID's to accumulate information from both runs (e.g. #line
529  // information) and predefined macros aren't guaranteed to be set properly.
530  assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
531  FileID MainFileID = SourceMgr.getMainFileID();
532 
533  // If MainFileID is loaded it means we loaded an AST file, no need to enter
534  // a main file.
535  if (!SourceMgr.isLoadedFileID(MainFileID)) {
536  // Enter the main file source buffer.
537  EnterSourceFile(MainFileID, nullptr, SourceLocation());
538 
539  // If we've been asked to skip bytes in the main file (e.g., as part of a
540  // precompiled preamble), do so now.
541  if (SkipMainFilePreamble.first > 0)
542  CurLexer->SetByteOffset(SkipMainFilePreamble.first,
543  SkipMainFilePreamble.second);
544 
545  // Tell the header info that the main file was entered. If the file is later
546  // #imported, it won't be re-entered.
547  if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
548  HeaderInfo.IncrementIncludeCount(FE);
549  }
550 
551  // Preprocess Predefines to populate the initial preprocessor state.
552  std::unique_ptr<llvm::MemoryBuffer> SB =
553  llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
554  assert(SB && "Cannot create predefined source buffer");
555  FileID FID = SourceMgr.createFileID(std::move(SB));
556  assert(FID.isValid() && "Could not create FileID for predefines?");
557  setPredefinesFileID(FID);
558 
559  // Start parsing the predefines.
560  EnterSourceFile(FID, nullptr, SourceLocation());
561 
562  if (!PPOpts->PCHThroughHeader.empty()) {
563  // Lookup and save the FileID for the through header. If it isn't found
564  // in the search path, it's a fatal error.
565  const DirectoryLookup *CurDir;
566  const FileEntry *File = LookupFile(
567  SourceLocation(), PPOpts->PCHThroughHeader,
568  /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr, CurDir,
569  /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
570  /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr);
571  if (!File) {
572  Diag(SourceLocation(), diag::err_pp_through_header_not_found)
573  << PPOpts->PCHThroughHeader;
574  return;
575  }
576  setPCHThroughHeaderFileID(
577  SourceMgr.createFileID(File, SourceLocation(), SrcMgr::C_User));
578  }
579 
580  // Skip tokens from the Predefines and if needed the main file.
581  if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
582  (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
584 }
585 
586 void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
587  assert(PCHThroughHeaderFileID.isInvalid() &&
588  "PCHThroughHeaderFileID already set!");
589  PCHThroughHeaderFileID = FID;
590 }
591 
593  assert(PCHThroughHeaderFileID.isValid() &&
594  "Invalid PCH through header FileID");
595  return FE == SourceMgr.getFileEntryForID(PCHThroughHeaderFileID);
596 }
597 
599  return TUKind == TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
600  PCHThroughHeaderFileID.isValid();
601 }
602 
604  return TUKind != TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
605  PCHThroughHeaderFileID.isValid();
606 }
607 
609  return TUKind == TU_Prefix && PPOpts->PCHWithHdrStop;
610 }
611 
613  return TUKind != TU_Prefix && PPOpts->PCHWithHdrStop;
614 }
615 
616 /// Skip tokens until after the #include of the through header or
617 /// until after a #pragma hdrstop is seen. Tokens in the predefines file
618 /// and the main file may be skipped. If the end of the predefines file
619 /// is reached, skipping continues into the main file. If the end of the
620 /// main file is reached, it's a fatal error.
622  bool ReachedMainFileEOF = false;
623  bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
624  bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
625  Token Tok;
626  while (true) {
627  bool InPredefines = (CurLexer->getFileID() == getPredefinesFileID());
628  CurLexer->Lex(Tok);
629  if (Tok.is(tok::eof) && !InPredefines) {
630  ReachedMainFileEOF = true;
631  break;
632  }
633  if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
634  break;
635  if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
636  break;
637  }
638  if (ReachedMainFileEOF) {
639  if (UsingPCHThroughHeader)
640  Diag(SourceLocation(), diag::err_pp_through_header_not_seen)
641  << PPOpts->PCHThroughHeader << 1;
642  else if (!PPOpts->PCHWithHdrStopCreate)
643  Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen);
644  }
645 }
646 
647 void Preprocessor::replayPreambleConditionalStack() {
648  // Restore the conditional stack from the preamble, if there is one.
649  if (PreambleConditionalStack.isReplaying()) {
650  assert(CurPPLexer &&
651  "CurPPLexer is null when calling replayPreambleConditionalStack.");
652  CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
653  PreambleConditionalStack.doneReplaying();
654  if (PreambleConditionalStack.reachedEOFWhileSkipping())
655  SkipExcludedConditionalBlock(
656  PreambleConditionalStack.SkipInfo->HashTokenLoc,
657  PreambleConditionalStack.SkipInfo->IfTokenLoc,
658  PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
659  PreambleConditionalStack.SkipInfo->FoundElse,
660  PreambleConditionalStack.SkipInfo->ElseLoc);
661  }
662 }
663 
665  // Notify the client that we reached the end of the source file.
666  if (Callbacks)
667  Callbacks->EndOfMainFile();
668 }
669 
670 //===----------------------------------------------------------------------===//
671 // Lexer Event Handling.
672 //===----------------------------------------------------------------------===//
673 
674 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
675 /// identifier information for the token and install it into the token,
676 /// updating the token kind accordingly.
678  assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
679 
680  // Look up this token, see if it is a macro, or if it is a language keyword.
681  IdentifierInfo *II;
682  if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
683  // No cleaning needed, just use the characters from the lexed buffer.
684  II = getIdentifierInfo(Identifier.getRawIdentifier());
685  } else {
686  // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
687  SmallString<64> IdentifierBuffer;
688  StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
689 
690  if (Identifier.hasUCN()) {
691  SmallString<64> UCNIdentifierBuffer;
692  expandUCNs(UCNIdentifierBuffer, CleanedStr);
693  II = getIdentifierInfo(UCNIdentifierBuffer);
694  } else {
695  II = getIdentifierInfo(CleanedStr);
696  }
697  }
698 
699  // Update the token info (identifier info and appropriate token kind).
700  Identifier.setIdentifierInfo(II);
701  if (getLangOpts().MSVCCompat && II->isCPlusPlusOperatorKeyword() &&
703  Identifier.setKind(tok::identifier);
704  else
705  Identifier.setKind(II->getTokenID());
706 
707  return II;
708 }
709 
710 void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
711  PoisonReasons[II] = DiagID;
712 }
713 
715  assert(Ident__exception_code && Ident__exception_info);
716  assert(Ident___exception_code && Ident___exception_info);
717  Ident__exception_code->setIsPoisoned(Poison);
718  Ident___exception_code->setIsPoisoned(Poison);
719  Ident_GetExceptionCode->setIsPoisoned(Poison);
720  Ident__exception_info->setIsPoisoned(Poison);
721  Ident___exception_info->setIsPoisoned(Poison);
722  Ident_GetExceptionInfo->setIsPoisoned(Poison);
723  Ident__abnormal_termination->setIsPoisoned(Poison);
724  Ident___abnormal_termination->setIsPoisoned(Poison);
725  Ident_AbnormalTermination->setIsPoisoned(Poison);
726 }
727 
729  assert(Identifier.getIdentifierInfo() &&
730  "Can't handle identifiers without identifier info!");
731  llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
732  PoisonReasons.find(Identifier.getIdentifierInfo());
733  if(it == PoisonReasons.end())
734  Diag(Identifier, diag::err_pp_used_poisoned_id);
735  else
736  Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
737 }
738 
739 /// Returns a diagnostic message kind for reporting a future keyword as
740 /// appropriate for the identifier and specified language.
742  const LangOptions &LangOpts) {
743  assert(II.isFutureCompatKeyword() && "diagnostic should not be needed");
744 
745  if (LangOpts.CPlusPlus)
746  return llvm::StringSwitch<diag::kind>(II.getName())
747 #define CXX11_KEYWORD(NAME, FLAGS) \
748  .Case(#NAME, diag::warn_cxx11_keyword)
749 #define CXX2A_KEYWORD(NAME, FLAGS) \
750  .Case(#NAME, diag::warn_cxx2a_keyword)
751 #include "clang/Basic/TokenKinds.def"
752  ;
753 
754  llvm_unreachable(
755  "Keyword not known to come from a newer Standard or proposed Standard");
756 }
757 
758 void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const {
759  assert(II.isOutOfDate() && "not out of date");
761 }
762 
763 /// HandleIdentifier - This callback is invoked when the lexer reads an
764 /// identifier. This callback looks up the identifier in the map and/or
765 /// potentially macro expands it or turns it into a named token (like 'for').
766 ///
767 /// Note that callers of this method are guarded by checking the
768 /// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
769 /// IdentifierInfo methods that compute these properties will need to change to
770 /// match.
772  assert(Identifier.getIdentifierInfo() &&
773  "Can't handle identifiers without identifier info!");
774 
775  IdentifierInfo &II = *Identifier.getIdentifierInfo();
776 
777  // If the information about this identifier is out of date, update it from
778  // the external source.
779  // We have to treat __VA_ARGS__ in a special way, since it gets
780  // serialized with isPoisoned = true, but our preprocessor may have
781  // unpoisoned it if we're defining a C99 macro.
782  if (II.isOutOfDate()) {
783  bool CurrentIsPoisoned = false;
784  const bool IsSpecialVariadicMacro =
785  &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
786  if (IsSpecialVariadicMacro)
787  CurrentIsPoisoned = II.isPoisoned();
788 
789  updateOutOfDateIdentifier(II);
790  Identifier.setKind(II.getTokenID());
791 
792  if (IsSpecialVariadicMacro)
793  II.setIsPoisoned(CurrentIsPoisoned);
794  }
795 
796  // If this identifier was poisoned, and if it was not produced from a macro
797  // expansion, emit an error.
798  if (II.isPoisoned() && CurPPLexer) {
799  HandlePoisonedIdentifier(Identifier);
800  }
801 
802  // If this is a macro to be expanded, do it.
803  if (MacroDefinition MD = getMacroDefinition(&II)) {
804  auto *MI = MD.getMacroInfo();
805  assert(MI && "macro definition with no macro info?");
806  if (!DisableMacroExpansion) {
807  if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
808  // C99 6.10.3p10: If the preprocessing token immediately after the
809  // macro name isn't a '(', this macro should not be expanded.
810  if (!MI->isFunctionLike() || isNextPPTokenLParen())
811  return HandleMacroExpandedIdentifier(Identifier, MD);
812  } else {
813  // C99 6.10.3.4p2 says that a disabled macro may never again be
814  // expanded, even if it's in a context where it could be expanded in the
815  // future.
816  Identifier.setFlag(Token::DisableExpand);
817  if (MI->isObjectLike() || isNextPPTokenLParen())
818  Diag(Identifier, diag::pp_disabled_macro_expansion);
819  }
820  }
821  }
822 
823  // If this identifier is a keyword in a newer Standard or proposed Standard,
824  // produce a warning. Don't warn if we're not considering macro expansion,
825  // since this identifier might be the name of a macro.
826  // FIXME: This warning is disabled in cases where it shouldn't be, like
827  // "#define constexpr constexpr", "int constexpr;"
828  if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
829  Diag(Identifier, getFutureCompatDiagKind(II, getLangOpts()))
830  << II.getName();
831  // Don't diagnose this keyword again in this translation unit.
832  II.setIsFutureCompatKeyword(false);
833  }
834 
835  // If this is an extension token, diagnose its use.
836  // We avoid diagnosing tokens that originate from macro definitions.
837  // FIXME: This warning is disabled in cases where it shouldn't be,
838  // like "#define TY typeof", "TY(1) x".
839  if (II.isExtensionToken() && !DisableMacroExpansion)
840  Diag(Identifier, diag::ext_token_used);
841 
842  // If this is the 'import' contextual keyword following an '@', note
843  // that the next token indicates a module name.
844  //
845  // Note that we do not treat 'import' as a contextual
846  // keyword when we're in a caching lexer, because caching lexers only get
847  // used in contexts where import declarations are disallowed.
848  //
849  // Likewise if this is the C++ Modules TS import keyword.
850  if (((LastTokenWasAt && II.isModulesImport()) ||
851  Identifier.is(tok::kw_import)) &&
852  !InMacroArgs && !DisableMacroExpansion &&
853  (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
854  CurLexerKind != CLK_CachingLexer) {
855  ModuleImportLoc = Identifier.getLocation();
856  ModuleImportPath.clear();
857  ModuleImportExpectsIdentifier = true;
858  CurLexerKind = CLK_LexAfterModuleImport;
859  }
860  return true;
861 }
862 
864  // We loop here until a lex function returns a token; this avoids recursion.
865  bool ReturnedToken;
866  do {
867  switch (CurLexerKind) {
868  case CLK_Lexer:
869  ReturnedToken = CurLexer->Lex(Result);
870  break;
871  case CLK_TokenLexer:
872  ReturnedToken = CurTokenLexer->Lex(Result);
873  break;
874  case CLK_CachingLexer:
875  CachingLex(Result);
876  ReturnedToken = true;
877  break;
878  case CLK_LexAfterModuleImport:
879  LexAfterModuleImport(Result);
880  ReturnedToken = true;
881  break;
882  }
883  } while (!ReturnedToken);
884 
885  if (Result.is(tok::code_completion) && Result.getIdentifierInfo()) {
886  // Remember the identifier before code completion token.
889  // Set IdenfitierInfo to null to avoid confusing code that handles both
890  // identifiers and completion tokens.
891  Result.setIdentifierInfo(nullptr);
892  }
893 
894  LastTokenWasAt = Result.is(tok::at);
895 }
896 
897 /// Lex a token following the 'import' contextual keyword.
898 ///
900  // Figure out what kind of lexer we actually have.
902 
903  // Lex the next token.
904  Lex(Result);
905 
906  // The token sequence
907  //
908  // import identifier (. identifier)*
909  //
910  // indicates a module import directive. We already saw the 'import'
911  // contextual keyword, so now we're looking for the identifiers.
912  if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
913  // We expected to see an identifier here, and we did; continue handling
914  // identifiers.
915  ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
916  Result.getLocation()));
917  ModuleImportExpectsIdentifier = false;
918  CurLexerKind = CLK_LexAfterModuleImport;
919  return;
920  }
921 
922  // If we're expecting a '.' or a ';', and we got a '.', then wait until we
923  // see the next identifier. (We can also see a '[[' that begins an
924  // attribute-specifier-seq here under the C++ Modules TS.)
925  if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
926  ModuleImportExpectsIdentifier = true;
927  CurLexerKind = CLK_LexAfterModuleImport;
928  return;
929  }
930 
931  // If we have a non-empty module path, load the named module.
932  if (!ModuleImportPath.empty()) {
933  // Under the Modules TS, the dot is just part of the module name, and not
934  // a real hierarchy separator. Flatten such module names now.
935  //
936  // FIXME: Is this the right level to be performing this transformation?
937  std::string FlatModuleName;
938  if (getLangOpts().ModulesTS) {
939  for (auto &Piece : ModuleImportPath) {
940  if (!FlatModuleName.empty())
941  FlatModuleName += ".";
942  FlatModuleName += Piece.first->getName();
943  }
944  SourceLocation FirstPathLoc = ModuleImportPath[0].second;
945  ModuleImportPath.clear();
946  ModuleImportPath.push_back(
947  std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
948  }
949 
950  Module *Imported = nullptr;
951  if (getLangOpts().Modules) {
952  Imported = TheModuleLoader.loadModule(ModuleImportLoc,
953  ModuleImportPath,
955  /*IsIncludeDirective=*/false);
956  if (Imported)
957  makeModuleVisible(Imported, ModuleImportLoc);
958  }
959  if (Callbacks && (getLangOpts().Modules || getLangOpts().DebuggerSupport))
960  Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
961  }
962 }
963 
965  CurSubmoduleState->VisibleModules.setVisible(
966  M, Loc, [](Module *) {},
967  [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
968  // FIXME: Include the path in the diagnostic.
969  // FIXME: Include the import location for the conflicting module.
970  Diag(ModuleImportLoc, diag::warn_module_conflict)
971  << Path[0]->getFullModuleName()
972  << Conflict->getFullModuleName()
973  << Message;
974  });
975 
976  // Add this module to the imports list of the currently-built submodule.
977  if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
978  BuildingSubmoduleStack.back().M->Imports.insert(M);
979 }
980 
982  const char *DiagnosticTag,
983  bool AllowMacroExpansion) {
984  // We need at least one string literal.
985  if (Result.isNot(tok::string_literal)) {
986  Diag(Result, diag::err_expected_string_literal)
987  << /*Source='in...'*/0 << DiagnosticTag;
988  return false;
989  }
990 
991  // Lex string literal tokens, optionally with macro expansion.
992  SmallVector<Token, 4> StrToks;
993  do {
994  StrToks.push_back(Result);
995 
996  if (Result.hasUDSuffix())
997  Diag(Result, diag::err_invalid_string_udl);
998 
999  if (AllowMacroExpansion)
1000  Lex(Result);
1001  else
1002  LexUnexpandedToken(Result);
1003  } while (Result.is(tok::string_literal));
1004 
1005  // Concatenate and parse the strings.
1006  StringLiteralParser Literal(StrToks, *this);
1007  assert(Literal.isAscii() && "Didn't allow wide strings in");
1008 
1009  if (Literal.hadError)
1010  return false;
1011 
1012  if (Literal.Pascal) {
1013  Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1014  << /*Source='in...'*/0 << DiagnosticTag;
1015  return false;
1016  }
1017 
1018  String = Literal.GetString();
1019  return true;
1020 }
1021 
1023  assert(Tok.is(tok::numeric_constant));
1024  SmallString<8> IntegerBuffer;
1025  bool NumberInvalid = false;
1026  StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1027  if (NumberInvalid)
1028  return false;
1029  NumericLiteralParser Literal(Spelling, Tok.getLocation(), *this);
1030  if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1031  return false;
1032  llvm::APInt APVal(64, 0);
1033  if (Literal.GetIntegerValue(APVal))
1034  return false;
1035  Lex(Tok);
1036  Value = APVal.getLimitedValue();
1037  return true;
1038 }
1039 
1041  assert(Handler && "NULL comment handler");
1042  assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
1043  CommentHandlers.end() && "Comment handler already registered");
1044  CommentHandlers.push_back(Handler);
1045 }
1046 
1048  std::vector<CommentHandler *>::iterator Pos =
1049  std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
1050  assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1051  CommentHandlers.erase(Pos);
1052 }
1053 
1055  bool AnyPendingTokens = false;
1056  for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1057  HEnd = CommentHandlers.end();
1058  H != HEnd; ++H) {
1059  if ((*H)->HandleComment(*this, Comment))
1060  AnyPendingTokens = true;
1061  }
1062  if (!AnyPendingTokens || getCommentRetentionState())
1063  return false;
1064  Lex(result);
1065  return true;
1066 }
1067 
1068 ModuleLoader::~ModuleLoader() = default;
1069 
1071 
1073 
1075  if (Record)
1076  return;
1077 
1078  Record = new PreprocessingRecord(getSourceManager());
1079  addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));
1080 }
StringRef Identifier
Definition: Format.cpp:1636
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void FinalizeForModelFile()
Cleanup after model file parsing.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop...
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
Defines the clang::FileManager interface and associated types.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
Defines the SourceManager interface.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
Defines the clang::Module class, which describes a module in the source code.
Module * getCurrentModule()
Retrieves the module that we&#39;re currently building, if any.
Defines the FileSystemStatCache interface.
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
The translation unit is a prefix to a translation unit, and is not complete.
Definition: LangOptions.h:368
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:565
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
void LexAfterModuleImport(Token &Result)
Lex a token following the &#39;import&#39; contextual keyword.
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:234
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions...
StringRef getBufferData(FileID FID, bool *Invalid=nullptr) const
Return a StringRef to the source buffer data for the specified FileID.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
Manage memory buffers across multiple users.
const FileEntry * LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, const DirectoryLookup *FromDir, const FileEntry *FromFile, const DirectoryLookup *&CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
tokens_iterator tokens_begin() const
Definition: MacroInfo.h:243
tok::TokenKind getKind() const
Definition: Token.h:90
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:113
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
One of these records is kept for each identifier that is lexed.
Represents a macro directive exported by a module.
Definition: MacroInfo.h:489
void print(raw_ostream &OS, const SourceManager &SM) const
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:207
Definition: Format.h:2072
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, MemoryBufferCache &PCMCache, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
void setTarget(const TargetInfo &Target)
Set the target information for the header search, if not already known.
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
void setKind(tok::TokenKind K)
Definition: Token.h:91
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an #include directive...
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:50
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:815
Describes a module or submodule.
Definition: Module.h:65
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
virtual ~ModuleLoader()
bool isInvalid() const
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:821
const Token & getReplacementToken(unsigned Tok) const
Definition: MacroInfo.h:236
bool hasUDSuffix() const
Return true if this token is a string or character literal which has a ud-suffix. ...
Definition: Token.h:291
const FormatToken & Tok
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:149
SourceLocation getSpellingLoc(SourceLocation Loc) const
Given a SourceLocation object, return the spelling location referenced by the ID. ...
macro_iterator macro_end(bool IncludeExternalMacros=true) const
bool HandleComment(Token &result, SourceRange Comment)
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:148
Provides lookups to, and iteration over, IdentiferInfo objects.
bool isValid() const
Exposes information about the current target.
Definition: TargetInfo.h:54
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Defines the clang::LangOptions interface.
void makeModuleVisible(Module *M, SourceLocation Loc)
SourceLocation getLocation() const
Definition: MacroInfo.h:463
bool isInSystemHeader(SourceLocation Loc) const
Returns if a SourceLocation is in a system header.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
const AnnotatedLine * Line
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, unsigned LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
virtual void ReadDefinedMacros()=0
Read the set of macros defined by this external macro source.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
Defines the clang::Preprocessor interface.
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
size_t getTotalMemory() const
bool getCommentRetentionState() const
Definition: Preprocessor.h:855
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool isObjectLike() const
Definition: MacroInfo.h:201
void IncrementIncludeCount(const FileEntry *File)
Increment the count for the number of times the specified FileEntry has been entered.
Definition: HeaderSearch.h:455
The result type of a method or function.
StringRef GetString() const
const SourceManager & SM
Definition: Format.cpp:1490
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
SourceManager & getSourceManager() const
Definition: Preprocessor.h:819
void DumpMacro(const MacroInfo &MI) const
void overrideFileContents(const FileEntry *SourceFile, llvm::MemoryBuffer *Buffer, bool DoNotFree)
Override the contents of the given source file by providing an already-allocated buffer.
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:833
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6...
Definition: Token.h:274
Encodes a location in the source.
void setLength(unsigned Len)
Definition: Token.h:133
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, bool ExpansionIsTokenRange=true, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:940
All of the names in this module are hidden.
Definition: Module.h:269
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:177
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:60
void setIdentifierInfo(IdentifierInfo *II)
Definition: Token.h:186
void Lex(Token &Result)
Lex the next token for this preprocessor.
llvm::MemoryBuffer * getMemoryBufferForFile(const FileEntry *File, bool *Invalid=nullptr)
Retrieve the memory buffer associated with the given file.
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:266
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
StringRef getName() const
Return the actual identifier string.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:96
bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
Definition: MacroInfo.h:234
static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II, const LangOptions &LangOpts)
Returns a diagnostic message kind for reporting a future keyword as appropriate for the identifier an...
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
FileID getMainFileID() const
Returns the FileID of the main source file.
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
Abstract interface for a module loader.
Definition: ModuleLoader.h:74
unsigned getLength() const
Definition: Token.h:127
void PoisonSEHIdentifiers(bool Poison=true)
void setLiteralData(const char *Ptr)
Definition: Token.h:219
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:40
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
Definition: Pragma.h:90
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token&#39;s location and length to ...
Module * lookupModule(StringRef ModuleName, bool AllowSearch=true, bool AllowExtraModuleMapSearch=false)
Lookup a module Search for a module with the given name.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
#define CXX11_KEYWORD(NAME, FLAGS)
Defines the clang::SourceLocation class and associated facilities.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:900
StringRef getRawIdentifier() const
getRawIdentifier - For a raw identifier token (i.e., an identifier lexed in raw mode), returns a reference to the text substring in the buffer if known.
Definition: Token.h:203
SourceLocation createTokenSplitLoc(SourceLocation SpellingLoc, SourceLocation TokenStart, SourceLocation TokenEnd)
Return a new SourceLocation that encodes that the token starting at TokenStart ends prematurely at To...
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:362
void DumpLocation(SourceLocation Loc) const
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
unsigned kind
All of the diagnostics that can be emitted by the frontend.
Definition: DiagnosticIDs.h:61
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
Definition: TokenKinds.cpp:25
Defines the clang::TargetInfo interface.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:283
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber...
MacroMap::const_iterator macro_iterator
Abstract base class that describes a handler that will receive source ranges for each of the comments...
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void setLocation(SourceLocation L)
Definition: Token.h:132
#define true
Definition: stdbool.h:32
A trivial tuple used to represent a source range.
bool hasUCN() const
Returns true if this token contains a universal character name.
Definition: Token.h:294
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
Defines the PreprocessorLexer interface.
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens...
Definition: ScratchBuffer.h:25
void InitializeTarget(const TargetInfo &Target, const TargetInfo *AuxTarget)
Perform target-specific initialization.
Definition: Builtins.cpp:43
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:270
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:908
This class handles loading and caching of source files into memory.
std::pair< FileID, unsigned > getDecomposedLoc(SourceLocation Loc) const
Decompose the specified location into a raw FileID + Offset pair.
SourceLocation getEndLoc() const
Definition: Token.h:151