clang  6.0.0svn
Preprocessor.h
Go to the documentation of this file.
1 //===--- Preprocessor.h - C Language Family Preprocessor --------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief Defines the clang::Preprocessor interface.
12 ///
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
16 #define LLVM_CLANG_LEX_PREPROCESSOR_H
17 
18 #include "clang/Basic/Builtins.h"
19 #include "clang/Basic/Diagnostic.h"
22 #include "clang/Lex/Lexer.h"
23 #include "clang/Lex/MacroInfo.h"
24 #include "clang/Lex/ModuleMap.h"
25 #include "clang/Lex/PPCallbacks.h"
26 #include "clang/Lex/PTHLexer.h"
27 #include "clang/Lex/TokenLexer.h"
28 #include "llvm/ADT/ArrayRef.h"
29 #include "llvm/ADT/DenseMap.h"
30 #include "llvm/ADT/IntrusiveRefCntPtr.h"
31 #include "llvm/ADT/SmallPtrSet.h"
32 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/ADT/TinyPtrVector.h"
34 #include "llvm/Support/Allocator.h"
35 #include "llvm/Support/Registry.h"
36 #include <memory>
37 #include <vector>
38 
39 namespace llvm {
40  template<unsigned InternalLen> class SmallString;
41 }
42 
43 namespace clang {
44 
45 class SourceManager;
46 class ExternalPreprocessorSource;
47 class FileManager;
48 class FileEntry;
49 class HeaderSearch;
50 class MemoryBufferCache;
51 class PragmaNamespace;
52 class PragmaHandler;
53 class CommentHandler;
54 class ScratchBuffer;
55 class TargetInfo;
56 class PPCallbacks;
57 class CodeCompletionHandler;
58 class DirectoryLookup;
59 class PreprocessingRecord;
60 class ModuleLoader;
61 class PTHManager;
62 class PreprocessorOptions;
63 
64 /// \brief Stores token information for comparing actual tokens with
65 /// predefined values. Only handles simple tokens and identifiers.
66 class TokenValue {
68  IdentifierInfo *II;
69 
70 public:
71  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
72  assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
73  assert(Kind != tok::identifier &&
74  "Identifiers should be created by TokenValue(IdentifierInfo *)");
75  assert(!tok::isLiteral(Kind) && "Literals are not supported.");
76  assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
77  }
78  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
79  bool operator==(const Token &Tok) const {
80  return Tok.getKind() == Kind &&
81  (!II || II == Tok.getIdentifierInfo());
82  }
83 };
84 
85 /// \brief Context in which macro name is used.
86 enum MacroUse {
87  MU_Other = 0, // other than #define or #undef
88  MU_Define = 1, // macro name specified in #define
89  MU_Undef = 2 // macro name specified in #undef
90 };
91 
92 /// \brief Engages in a tight little dance with the lexer to efficiently
93 /// preprocess tokens.
94 ///
95 /// Lexers know only about tokens within a single source file, and don't
96 /// know anything about preprocessor-level issues like the \#include stack,
97 /// token expansion, etc.
98 class Preprocessor {
101  std::shared_ptr<PreprocessorOptions> PPOpts;
102  DiagnosticsEngine *Diags;
103  LangOptions &LangOpts;
104  const TargetInfo *Target;
105  const TargetInfo *AuxTarget;
106  FileManager &FileMgr;
107  SourceManager &SourceMgr;
108  MemoryBufferCache &PCMCache;
109  std::unique_ptr<ScratchBuffer> ScratchBuf;
110  HeaderSearch &HeaderInfo;
111  ModuleLoader &TheModuleLoader;
112 
113  /// \brief External source of macros.
114  ExternalPreprocessorSource *ExternalSource;
115 
116 
117  /// An optional PTHManager object used for getting tokens from
118  /// a token cache rather than lexing the original source file.
119  std::unique_ptr<PTHManager> PTH;
120 
121  /// A BumpPtrAllocator object used to quickly allocate and release
122  /// objects internal to the Preprocessor.
123  llvm::BumpPtrAllocator BP;
124 
125  /// Identifiers for builtin macros and other builtins.
126  IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
127  IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
128  IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
129  IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
130  IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
131  IdentifierInfo *Ident__COUNTER__; // __COUNTER__
132  IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
133  IdentifierInfo *Ident__identifier; // __identifier
134  IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
135  IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
136  IdentifierInfo *Ident__has_feature; // __has_feature
137  IdentifierInfo *Ident__has_extension; // __has_extension
138  IdentifierInfo *Ident__has_builtin; // __has_builtin
139  IdentifierInfo *Ident__has_attribute; // __has_attribute
140  IdentifierInfo *Ident__has_include; // __has_include
141  IdentifierInfo *Ident__has_include_next; // __has_include_next
142  IdentifierInfo *Ident__has_warning; // __has_warning
143  IdentifierInfo *Ident__is_identifier; // __is_identifier
144  IdentifierInfo *Ident__building_module; // __building_module
145  IdentifierInfo *Ident__MODULE__; // __MODULE__
146  IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
147  IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
148 
149  SourceLocation DATELoc, TIMELoc;
150  unsigned CounterValue; // Next __COUNTER__ value.
151 
152  enum {
153  /// \brief Maximum depth of \#includes.
154  MaxAllowedIncludeStackDepth = 200
155  };
156 
157  // State that is set before the preprocessor begins.
158  bool KeepComments : 1;
159  bool KeepMacroComments : 1;
160  bool SuppressIncludeNotFoundError : 1;
161 
162  // State that changes while the preprocessor runs:
163  bool InMacroArgs : 1; // True if parsing fn macro invocation args.
164 
165  /// Whether the preprocessor owns the header search object.
166  bool OwnsHeaderSearch : 1;
167 
168  /// True if macro expansion is disabled.
169  bool DisableMacroExpansion : 1;
170 
171  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
172  /// when parsing preprocessor directives.
173  bool MacroExpansionInDirectivesOverride : 1;
174 
176 
177  /// \brief Whether we have already loaded macros from the external source.
178  mutable bool ReadMacrosFromExternalSource : 1;
179 
180  /// \brief True if pragmas are enabled.
181  bool PragmasEnabled : 1;
182 
183  /// \brief True if the current build action is a preprocessing action.
184  bool PreprocessedOutput : 1;
185 
186  /// \brief True if we are currently preprocessing a #if or #elif directive
187  bool ParsingIfOrElifDirective;
188 
189  /// \brief True if we are pre-expanding macro arguments.
190  bool InMacroArgPreExpansion;
191 
192  /// \brief Mapping/lookup information for all identifiers in
193  /// the program, including program keywords.
194  mutable IdentifierTable Identifiers;
195 
196  /// \brief This table contains all the selectors in the program.
197  ///
198  /// Unlike IdentifierTable above, this table *isn't* populated by the
199  /// preprocessor. It is declared/expanded here because its role/lifetime is
200  /// conceptually similar to the IdentifierTable. In addition, the current
201  /// control flow (in clang::ParseAST()), make it convenient to put here.
202  ///
203  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
204  /// the lifetime of the preprocessor.
205  SelectorTable Selectors;
206 
207  /// \brief Information about builtins.
209 
210  /// \brief Tracks all of the pragmas that the client registered
211  /// with this preprocessor.
212  std::unique_ptr<PragmaNamespace> PragmaHandlers;
213 
214  /// \brief Pragma handlers of the original source is stored here during the
215  /// parsing of a model file.
216  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
217 
218  /// \brief Tracks all of the comment handlers that the client registered
219  /// with this preprocessor.
220  std::vector<CommentHandler *> CommentHandlers;
221 
222  /// \brief True if we want to ignore EOF token and continue later on (thus
223  /// avoid tearing the Lexer and etc. down).
224  bool IncrementalProcessing;
225 
226  /// The kind of translation unit we are processing.
227  TranslationUnitKind TUKind;
228 
229  /// \brief The code-completion handler.
230  CodeCompletionHandler *CodeComplete;
231 
232  /// \brief The file that we're performing code-completion for, if any.
233  const FileEntry *CodeCompletionFile;
234 
235  /// \brief The offset in file for the code-completion point.
236  unsigned CodeCompletionOffset;
237 
238  /// \brief The location for the code-completion point. This gets instantiated
239  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
240  SourceLocation CodeCompletionLoc;
241 
242  /// \brief The start location for the file of the code-completion point.
243  ///
244  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
245  /// for preprocessing.
246  SourceLocation CodeCompletionFileLoc;
247 
248  /// \brief The source location of the \c import contextual keyword we just
249  /// lexed, if any.
250  SourceLocation ModuleImportLoc;
251 
252  /// \brief The module import path that we're currently processing.
254 
255  /// \brief Whether the last token we lexed was an '@'.
256  bool LastTokenWasAt;
257 
258  /// \brief Whether the module import expects an identifier next. Otherwise,
259  /// it expects a '.' or ';'.
260  bool ModuleImportExpectsIdentifier;
261 
262  /// \brief The source location of the currently-active
263  /// \#pragma clang arc_cf_code_audited begin.
264  SourceLocation PragmaARCCFCodeAuditedLoc;
265 
266  /// \brief The source location of the currently-active
267  /// \#pragma clang assume_nonnull begin.
268  SourceLocation PragmaAssumeNonNullLoc;
269 
270  /// \brief True if we hit the code-completion point.
271  bool CodeCompletionReached;
272 
273  /// \brief The code completion token containing the information
274  /// on the stem that is to be code completed.
275  IdentifierInfo *CodeCompletionII;
276 
277  /// \brief The directory that the main file should be considered to occupy,
278  /// if it does not correspond to a real file (as happens when building a
279  /// module).
280  const DirectoryEntry *MainFileDir;
281 
282  /// \brief The number of bytes that we will initially skip when entering the
283  /// main file, along with a flag that indicates whether skipping this number
284  /// of bytes will place the lexer at the start of a line.
285  ///
286  /// This is used when loading a precompiled preamble.
287  std::pair<int, bool> SkipMainFilePreamble;
288 
289 public:
292  bool FoundNonSkipPortion, bool FoundElse,
293  SourceLocation ElseLoc)
294  : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
295  FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
296  ElseLoc(ElseLoc) {}
297 
301  bool FoundElse;
303  };
304 
305 private:
306  class PreambleConditionalStackStore {
307  enum State {
308  Off = 0,
309  Recording = 1,
310  Replaying = 2,
311  };
312 
313  public:
314  PreambleConditionalStackStore() : ConditionalStackState(Off) {}
315 
316  void startRecording() { ConditionalStackState = Recording; }
317  void startReplaying() { ConditionalStackState = Replaying; }
318  bool isRecording() const { return ConditionalStackState == Recording; }
319  bool isReplaying() const { return ConditionalStackState == Replaying; }
320 
321  ArrayRef<PPConditionalInfo> getStack() const {
322  return ConditionalStack;
323  }
324 
325  void doneReplaying() {
326  ConditionalStack.clear();
327  ConditionalStackState = Off;
328  }
329 
330  void setStack(ArrayRef<PPConditionalInfo> s) {
331  if (!isRecording() && !isReplaying())
332  return;
333  ConditionalStack.clear();
334  ConditionalStack.append(s.begin(), s.end());
335  }
336 
337  bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
338 
339  bool reachedEOFWhileSkipping() const { return SkipInfo.hasValue(); }
340 
341  void clearSkipInfo() { SkipInfo.reset(); }
342 
344 
345  private:
346  SmallVector<PPConditionalInfo, 4> ConditionalStack;
347  State ConditionalStackState;
348  } PreambleConditionalStack;
349 
350  /// \brief The current top of the stack that we're lexing from if
351  /// not expanding a macro and we are lexing directly from source code.
352  ///
353  /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
354  std::unique_ptr<Lexer> CurLexer;
355 
356  /// \brief The current top of stack that we're lexing from if
357  /// not expanding from a macro and we are lexing from a PTH cache.
358  ///
359  /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
360  std::unique_ptr<PTHLexer> CurPTHLexer;
361 
362  /// \brief The current top of the stack what we're lexing from
363  /// if not expanding a macro.
364  ///
365  /// This is an alias for either CurLexer or CurPTHLexer.
366  PreprocessorLexer *CurPPLexer;
367 
368  /// \brief Used to find the current FileEntry, if CurLexer is non-null
369  /// and if applicable.
370  ///
371  /// This allows us to implement \#include_next and find directory-specific
372  /// properties.
373  const DirectoryLookup *CurDirLookup;
374 
375  /// \brief The current macro we are expanding, if we are expanding a macro.
376  ///
377  /// One of CurLexer and CurTokenLexer must be null.
378  std::unique_ptr<TokenLexer> CurTokenLexer;
379 
380  /// \brief The kind of lexer we're currently working with.
381  enum CurLexerKind {
382  CLK_Lexer,
383  CLK_PTHLexer,
384  CLK_TokenLexer,
385  CLK_CachingLexer,
386  CLK_LexAfterModuleImport
387  } CurLexerKind;
388 
389  /// \brief If the current lexer is for a submodule that is being built, this
390  /// is that submodule.
391  Module *CurLexerSubmodule;
392 
393  /// \brief Keeps track of the stack of files currently
394  /// \#included, and macros currently being expanded from, not counting
395  /// CurLexer/CurTokenLexer.
396  struct IncludeStackInfo {
397  enum CurLexerKind CurLexerKind;
398  Module *TheSubmodule;
399  std::unique_ptr<Lexer> TheLexer;
400  std::unique_ptr<PTHLexer> ThePTHLexer;
401  PreprocessorLexer *ThePPLexer;
402  std::unique_ptr<TokenLexer> TheTokenLexer;
403  const DirectoryLookup *TheDirLookup;
404 
405  // The following constructors are completely useless copies of the default
406  // versions, only needed to pacify MSVC.
407  IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
408  std::unique_ptr<Lexer> &&TheLexer,
409  std::unique_ptr<PTHLexer> &&ThePTHLexer,
410  PreprocessorLexer *ThePPLexer,
411  std::unique_ptr<TokenLexer> &&TheTokenLexer,
412  const DirectoryLookup *TheDirLookup)
413  : CurLexerKind(std::move(CurLexerKind)),
414  TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
415  ThePTHLexer(std::move(ThePTHLexer)),
416  ThePPLexer(std::move(ThePPLexer)),
417  TheTokenLexer(std::move(TheTokenLexer)),
418  TheDirLookup(std::move(TheDirLookup)) {}
419  };
420  std::vector<IncludeStackInfo> IncludeMacroStack;
421 
422  /// \brief Actions invoked when some preprocessor activity is
423  /// encountered (e.g. a file is \#included, etc).
424  std::unique_ptr<PPCallbacks> Callbacks;
425 
426  struct MacroExpandsInfo {
427  Token Tok;
428  MacroDefinition MD;
429  SourceRange Range;
430  MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
431  : Tok(Tok), MD(MD), Range(Range) { }
432  };
433  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
434 
435  /// Information about a name that has been used to define a module macro.
436  struct ModuleMacroInfo {
437  ModuleMacroInfo(MacroDirective *MD)
438  : MD(MD), ActiveModuleMacrosGeneration(0), IsAmbiguous(false) {}
439 
440  /// The most recent macro directive for this identifier.
441  MacroDirective *MD;
442  /// The active module macros for this identifier.
443  llvm::TinyPtrVector<ModuleMacro*> ActiveModuleMacros;
444  /// The generation number at which we last updated ActiveModuleMacros.
445  /// \see Preprocessor::VisibleModules.
446  unsigned ActiveModuleMacrosGeneration;
447  /// Whether this macro name is ambiguous.
448  bool IsAmbiguous;
449  /// The module macros that are overridden by this macro.
450  llvm::TinyPtrVector<ModuleMacro*> OverriddenMacros;
451  };
452 
453  /// The state of a macro for an identifier.
454  class MacroState {
455  mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
456 
457  ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
458  const IdentifierInfo *II) const {
459  if (II->isOutOfDate())
460  PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
461  // FIXME: Find a spare bit on IdentifierInfo and store a
462  // HasModuleMacros flag.
463  if (!II->hasMacroDefinition() ||
464  (!PP.getLangOpts().Modules &&
465  !PP.getLangOpts().ModulesLocalVisibility) ||
466  !PP.CurSubmoduleState->VisibleModules.getGeneration())
467  return nullptr;
468 
469  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
470  if (!Info) {
471  Info = new (PP.getPreprocessorAllocator())
472  ModuleMacroInfo(State.get<MacroDirective *>());
473  State = Info;
474  }
475 
476  if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
477  Info->ActiveModuleMacrosGeneration)
478  PP.updateModuleMacroInfo(II, *Info);
479  return Info;
480  }
481 
482  public:
483  MacroState() : MacroState(nullptr) {}
484  MacroState(MacroDirective *MD) : State(MD) {}
485  MacroState(MacroState &&O) noexcept : State(O.State) {
486  O.State = (MacroDirective *)nullptr;
487  }
488  MacroState &operator=(MacroState &&O) noexcept {
489  auto S = O.State;
490  O.State = (MacroDirective *)nullptr;
491  State = S;
492  return *this;
493  }
494  ~MacroState() {
495  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
496  Info->~ModuleMacroInfo();
497  }
498 
499  MacroDirective *getLatest() const {
500  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
501  return Info->MD;
502  return State.get<MacroDirective*>();
503  }
504  void setLatest(MacroDirective *MD) {
505  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
506  Info->MD = MD;
507  else
508  State = MD;
509  }
510 
511  bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
512  auto *Info = getModuleInfo(PP, II);
513  return Info ? Info->IsAmbiguous : false;
514  }
516  getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
517  if (auto *Info = getModuleInfo(PP, II))
518  return Info->ActiveModuleMacros;
519  return None;
520  }
521 
522  MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
523  SourceManager &SourceMgr) const {
524  // FIXME: Incorporate module macros into the result of this.
525  if (auto *Latest = getLatest())
526  return Latest->findDirectiveAtLoc(Loc, SourceMgr);
527  return MacroDirective::DefInfo();
528  }
529 
530  void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
531  if (auto *Info = getModuleInfo(PP, II)) {
532  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
533  Info->ActiveModuleMacros.begin(),
534  Info->ActiveModuleMacros.end());
535  Info->ActiveModuleMacros.clear();
536  Info->IsAmbiguous = false;
537  }
538  }
539  ArrayRef<ModuleMacro*> getOverriddenMacros() const {
540  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
541  return Info->OverriddenMacros;
542  return None;
543  }
544  void setOverriddenMacros(Preprocessor &PP,
545  ArrayRef<ModuleMacro *> Overrides) {
546  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
547  if (!Info) {
548  if (Overrides.empty())
549  return;
550  Info = new (PP.getPreprocessorAllocator())
551  ModuleMacroInfo(State.get<MacroDirective *>());
552  State = Info;
553  }
554  Info->OverriddenMacros.clear();
555  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
556  Overrides.begin(), Overrides.end());
557  Info->ActiveModuleMacrosGeneration = 0;
558  }
559  };
560 
561  /// For each IdentifierInfo that was associated with a macro, we
562  /// keep a mapping to the history of all macro definitions and #undefs in
563  /// the reverse order (the latest one is in the head of the list).
564  ///
565  /// This mapping lives within the \p CurSubmoduleState.
566  typedef llvm::DenseMap<const IdentifierInfo *, MacroState> MacroMap;
567 
568  friend class ASTReader;
569 
570  struct SubmoduleState;
571 
572  /// \brief Information about a submodule that we're currently building.
573  struct BuildingSubmoduleInfo {
574  BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
575  SubmoduleState *OuterSubmoduleState,
576  unsigned OuterPendingModuleMacroNames)
577  : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
578  OuterSubmoduleState(OuterSubmoduleState),
579  OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
580 
581  /// The module that we are building.
582  Module *M;
583  /// The location at which the module was included.
584  SourceLocation ImportLoc;
585  /// Whether we entered this submodule via a pragma.
586  bool IsPragma;
587  /// The previous SubmoduleState.
588  SubmoduleState *OuterSubmoduleState;
589  /// The number of pending module macro names when we started building this.
590  unsigned OuterPendingModuleMacroNames;
591  };
592  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
593 
594  /// \brief Information about a submodule's preprocessor state.
595  struct SubmoduleState {
596  /// The macros for the submodule.
597  MacroMap Macros;
598  /// The set of modules that are visible within the submodule.
599  VisibleModuleSet VisibleModules;
600  // FIXME: CounterValue?
601  // FIXME: PragmaPushMacroInfo?
602  };
603  std::map<Module*, SubmoduleState> Submodules;
604 
605  /// The preprocessor state for preprocessing outside of any submodule.
606  SubmoduleState NullSubmoduleState;
607 
608  /// The current submodule state. Will be \p NullSubmoduleState if we're not
609  /// in a submodule.
610  SubmoduleState *CurSubmoduleState;
611 
612  /// The set of known macros exported from modules.
613  llvm::FoldingSet<ModuleMacro> ModuleMacros;
614 
615  /// The names of potential module macros that we've not yet processed.
616  llvm::SmallVector<const IdentifierInfo*, 32> PendingModuleMacroNames;
617 
618  /// The list of module macros, for each identifier, that are not overridden by
619  /// any other module macro.
620  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro*>>
621  LeafModuleMacros;
622 
623  /// \brief Macros that we want to warn because they are not used at the end
624  /// of the translation unit.
625  ///
626  /// We store just their SourceLocations instead of
627  /// something like MacroInfo*. The benefit of this is that when we are
628  /// deserializing from PCH, we don't need to deserialize identifier & macros
629  /// just so that we can report that they are unused, we just warn using
630  /// the SourceLocations of this set (that will be filled by the ASTReader).
631  /// We are using SmallPtrSet instead of a vector for faster removal.
632  typedef llvm::SmallPtrSet<SourceLocation, 32> WarnUnusedMacroLocsTy;
633  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
634 
635  /// \brief A "freelist" of MacroArg objects that can be
636  /// reused for quick allocation.
637  MacroArgs *MacroArgCache;
638  friend class MacroArgs;
639 
640  /// For each IdentifierInfo used in a \#pragma push_macro directive,
641  /// we keep a MacroInfo stack used to restore the previous macro value.
642  llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> > PragmaPushMacroInfo;
643 
644  // Various statistics we track for performance analysis.
645  unsigned NumDirectives, NumDefined, NumUndefined, NumPragma;
646  unsigned NumIf, NumElse, NumEndif;
647  unsigned NumEnteredSourceFiles, MaxIncludeStackDepth;
648  unsigned NumMacroExpanded, NumFnMacroExpanded, NumBuiltinMacroExpanded;
649  unsigned NumFastMacroExpanded, NumTokenPaste, NumFastTokenPaste;
650  unsigned NumSkipped;
651 
652  /// \brief The predefined macros that preprocessor should use from the
653  /// command line etc.
654  std::string Predefines;
655 
656  /// \brief The file ID for the preprocessor predefines.
657  FileID PredefinesFileID;
658 
659  /// \{
660  /// \brief Cache of macro expanders to reduce malloc traffic.
661  enum { TokenLexerCacheSize = 8 };
662  unsigned NumCachedTokenLexers;
663  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
664  /// \}
665 
666  /// \brief Keeps macro expanded tokens for TokenLexers.
667  //
668  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
669  /// going to lex in the cache and when it finishes the tokens are removed
670  /// from the end of the cache.
671  SmallVector<Token, 16> MacroExpandedTokens;
672  std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack;
673 
674  /// \brief A record of the macro definitions and expansions that
675  /// occurred during preprocessing.
676  ///
677  /// This is an optional side structure that can be enabled with
678  /// \c createPreprocessingRecord() prior to preprocessing.
679  PreprocessingRecord *Record;
680 
681  /// Cached tokens state.
683 
684  /// \brief Cached tokens are stored here when we do backtracking or
685  /// lookahead. They are "lexed" by the CachingLex() method.
686  CachedTokensTy CachedTokens;
687 
688  /// \brief The position of the cached token that CachingLex() should
689  /// "lex" next.
690  ///
691  /// If it points beyond the CachedTokens vector, it means that a normal
692  /// Lex() should be invoked.
693  CachedTokensTy::size_type CachedLexPos;
694 
695  /// \brief Stack of backtrack positions, allowing nested backtracks.
696  ///
697  /// The EnableBacktrackAtThisPos() method pushes a position to
698  /// indicate where CachedLexPos should be set when the BackTrack() method is
699  /// invoked (at which point the last position is popped).
700  std::vector<CachedTokensTy::size_type> BacktrackPositions;
701 
702  struct MacroInfoChain {
703  MacroInfo MI;
704  MacroInfoChain *Next;
705  };
706 
707  /// MacroInfos are managed as a chain for easy disposal. This is the head
708  /// of that list.
709  MacroInfoChain *MIChainHead;
710 
711  void updateOutOfDateIdentifier(IdentifierInfo &II) const;
712 
713 public:
714  Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
716  MemoryBufferCache &PCMCache,
717  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
718  IdentifierInfoLookup *IILookup = nullptr,
719  bool OwnsHeaderSearch = false,
721 
722  ~Preprocessor();
723 
724  /// \brief Initialize the preprocessor using information about the target.
725  ///
726  /// \param Target is owned by the caller and must remain valid for the
727  /// lifetime of the preprocessor.
728  /// \param AuxTarget is owned by the caller and must remain valid for
729  /// the lifetime of the preprocessor.
730  void Initialize(const TargetInfo &Target,
731  const TargetInfo *AuxTarget = nullptr);
732 
733  /// \brief Initialize the preprocessor to parse a model file
734  ///
735  /// To parse model files the preprocessor of the original source is reused to
736  /// preserver the identifier table. However to avoid some duplicate
737  /// information in the preprocessor some cleanup is needed before it is used
738  /// to parse model files. This method does that cleanup.
739  void InitializeForModelFile();
740 
741  /// \brief Cleanup after model file parsing
742  void FinalizeForModelFile();
743 
744  /// \brief Retrieve the preprocessor options used to initialize this
745  /// preprocessor.
746  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
747 
748  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
749  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
750 
751  const LangOptions &getLangOpts() const { return LangOpts; }
752  const TargetInfo &getTargetInfo() const { return *Target; }
753  const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
754  FileManager &getFileManager() const { return FileMgr; }
755  SourceManager &getSourceManager() const { return SourceMgr; }
756  MemoryBufferCache &getPCMCache() const { return PCMCache; }
757  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
758 
759  IdentifierTable &getIdentifierTable() { return Identifiers; }
760  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
761  SelectorTable &getSelectorTable() { return Selectors; }
763  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
764 
765  void setPTHManager(PTHManager* pm);
766 
767  PTHManager *getPTHManager() { return PTH.get(); }
768 
770  ExternalSource = Source;
771  }
772 
774  return ExternalSource;
775  }
776 
777  /// \brief Retrieve the module loader associated with this preprocessor.
778  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
779 
781  return TheModuleLoader.HadFatalFailure;
782  }
783 
784  /// \brief True if we are currently preprocessing a #if or #elif directive
786  return ParsingIfOrElifDirective;
787  }
788 
789  /// \brief Control whether the preprocessor retains comments in output.
790  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
791  this->KeepComments = KeepComments | KeepMacroComments;
792  this->KeepMacroComments = KeepMacroComments;
793  }
794 
795  bool getCommentRetentionState() const { return KeepComments; }
796 
797  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
798  bool getPragmasEnabled() const { return PragmasEnabled; }
799 
800  void SetSuppressIncludeNotFoundError(bool Suppress) {
801  SuppressIncludeNotFoundError = Suppress;
802  }
803 
805  return SuppressIncludeNotFoundError;
806  }
807 
808  /// Sets whether the preprocessor is responsible for producing output or if
809  /// it is producing tokens to be consumed by Parse and Sema.
810  void setPreprocessedOutput(bool IsPreprocessedOutput) {
811  PreprocessedOutput = IsPreprocessedOutput;
812  }
813 
814  /// Returns true if the preprocessor is responsible for generating output,
815  /// false if it is producing tokens to be consumed by Parse and Sema.
816  bool isPreprocessedOutput() const { return PreprocessedOutput; }
817 
818  /// \brief Return true if we are lexing directly from the specified lexer.
819  bool isCurrentLexer(const PreprocessorLexer *L) const {
820  return CurPPLexer == L;
821  }
822 
823  /// \brief Return the current lexer being lexed from.
824  ///
825  /// Note that this ignores any potentially active macro expansions and _Pragma
826  /// expansions going on at the time.
827  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
828 
829  /// \brief Return the current file lexer being lexed from.
830  ///
831  /// Note that this ignores any potentially active macro expansions and _Pragma
832  /// expansions going on at the time.
833  PreprocessorLexer *getCurrentFileLexer() const;
834 
835  /// \brief Return the submodule owning the file being lexed. This may not be
836  /// the current module if we have changed modules since entering the file.
837  Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
838 
839  /// \brief Returns the FileID for the preprocessor predefines.
840  FileID getPredefinesFileID() const { return PredefinesFileID; }
841 
842  /// \{
843  /// \brief Accessors for preprocessor callbacks.
844  ///
845  /// Note that this class takes ownership of any PPCallbacks object given to
846  /// it.
847  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
848  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
849  if (Callbacks)
850  C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
851  std::move(Callbacks));
852  Callbacks = std::move(C);
853  }
854  /// \}
855 
856  bool isMacroDefined(StringRef Id) {
857  return isMacroDefined(&Identifiers.get(Id));
858  }
859  bool isMacroDefined(const IdentifierInfo *II) {
860  return II->hasMacroDefinition() &&
861  (!getLangOpts().Modules || (bool)getMacroDefinition(II));
862  }
863 
864  /// \brief Determine whether II is defined as a macro within the module M,
865  /// if that is a module that we've already preprocessed. Does not check for
866  /// macros imported into M.
868  if (!II->hasMacroDefinition())
869  return false;
870  auto I = Submodules.find(M);
871  if (I == Submodules.end())
872  return false;
873  auto J = I->second.Macros.find(II);
874  if (J == I->second.Macros.end())
875  return false;
876  auto *MD = J->second.getLatest();
877  return MD && MD->isDefined();
878  }
879 
881  if (!II->hasMacroDefinition())
882  return MacroDefinition();
883 
884  MacroState &S = CurSubmoduleState->Macros[II];
885  auto *MD = S.getLatest();
886  while (MD && isa<VisibilityMacroDirective>(MD))
887  MD = MD->getPrevious();
888  return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
889  S.getActiveModuleMacros(*this, II),
890  S.isAmbiguous(*this, II));
891  }
892 
894  SourceLocation Loc) {
895  if (!II->hadMacroDefinition())
896  return MacroDefinition();
897 
898  MacroState &S = CurSubmoduleState->Macros[II];
900  if (auto *MD = S.getLatest())
901  DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
902  // FIXME: Compute the set of active module macros at the specified location.
903  return MacroDefinition(DI.getDirective(),
904  S.getActiveModuleMacros(*this, II),
905  S.isAmbiguous(*this, II));
906  }
907 
908  /// \brief Given an identifier, return its latest non-imported MacroDirective
909  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
911  if (!II->hasMacroDefinition())
912  return nullptr;
913 
914  auto *MD = getLocalMacroDirectiveHistory(II);
915  if (!MD || MD->getDefinition().isUndefined())
916  return nullptr;
917 
918  return MD;
919  }
920 
921  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
922  return const_cast<Preprocessor*>(this)->getMacroInfo(II);
923  }
924 
926  if (!II->hasMacroDefinition())
927  return nullptr;
928  if (auto MD = getMacroDefinition(II))
929  return MD.getMacroInfo();
930  return nullptr;
931  }
932 
933  /// \brief Given an identifier, return the latest non-imported macro
934  /// directive for that identifier.
935  ///
936  /// One can iterate over all previous macro directives from the most recent
937  /// one.
938  MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const;
939 
940  /// \brief Add a directive to the macro directive history for this identifier.
941  void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
943  SourceLocation Loc) {
944  DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
945  appendMacroDirective(II, MD);
946  return MD;
947  }
949  MacroInfo *MI) {
950  return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
951  }
952  /// \brief Set a MacroDirective that was loaded from a PCH file.
953  void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED,
954  MacroDirective *MD);
955 
956  /// \brief Register an exported macro for a module and identifier.
957  ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro,
958  ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
959  ModuleMacro *getModuleMacro(Module *Mod, IdentifierInfo *II);
960 
961  /// \brief Get the list of leaf (non-overridden) module macros for a name.
963  if (II->isOutOfDate())
964  updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
965  auto I = LeafModuleMacros.find(II);
966  if (I != LeafModuleMacros.end())
967  return I->second;
968  return None;
969  }
970 
971  /// \{
972  /// Iterators for the macro history table. Currently defined macros have
973  /// IdentifierInfo::hasMacroDefinition() set and an empty
974  /// MacroInfo::getUndefLoc() at the head of the list.
975  typedef MacroMap::const_iterator macro_iterator;
976  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
977  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
978  llvm::iterator_range<macro_iterator>
979  macros(bool IncludeExternalMacros = true) const {
980  return llvm::make_range(macro_begin(IncludeExternalMacros),
981  macro_end(IncludeExternalMacros));
982  }
983  /// \}
984 
985  /// \brief Return the name of the macro defined before \p Loc that has
986  /// spelling \p Tokens. If there are multiple macros with same spelling,
987  /// return the last one defined.
988  StringRef getLastMacroWithSpelling(SourceLocation Loc,
989  ArrayRef<TokenValue> Tokens) const;
990 
991  const std::string &getPredefines() const { return Predefines; }
992  /// \brief Set the predefines for this Preprocessor.
993  ///
994  /// These predefines are automatically injected when parsing the main file.
995  void setPredefines(const char *P) { Predefines = P; }
996  void setPredefines(StringRef P) { Predefines = P; }
997 
998  /// Return information about the specified preprocessor
999  /// identifier token.
1000  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1001  return &Identifiers.get(Name);
1002  }
1003 
1004  /// \brief Add the specified pragma handler to this preprocessor.
1005  ///
1006  /// If \p Namespace is non-null, then it is a token required to exist on the
1007  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1008  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1010  AddPragmaHandler(StringRef(), Handler);
1011  }
1012 
1013  /// \brief Remove the specific pragma handler from this preprocessor.
1014  ///
1015  /// If \p Namespace is non-null, then it should be the namespace that
1016  /// \p Handler was added to. It is an error to remove a handler that
1017  /// has not been registered.
1018  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1020  RemovePragmaHandler(StringRef(), Handler);
1021  }
1022 
1023  /// Install empty handlers for all pragmas (making them ignored).
1024  void IgnorePragmas();
1025 
1026  /// \brief Add the specified comment handler to the preprocessor.
1027  void addCommentHandler(CommentHandler *Handler);
1028 
1029  /// \brief Remove the specified comment handler.
1030  ///
1031  /// It is an error to remove a handler that has not been registered.
1032  void removeCommentHandler(CommentHandler *Handler);
1033 
1034  /// \brief Set the code completion handler to the given object.
1036  CodeComplete = &Handler;
1037  }
1038 
1039  /// \brief Retrieve the current code-completion handler.
1041  return CodeComplete;
1042  }
1043 
1044  /// \brief Clear out the code completion handler.
1046  CodeComplete = nullptr;
1047  }
1048 
1049  /// \brief Hook used by the lexer to invoke the "natural language" code
1050  /// completion point.
1051  void CodeCompleteNaturalLanguage();
1052 
1053  /// \brief Set the code completion token for filtering purposes.
1055  CodeCompletionII = Filter;
1056  }
1057 
1058  /// \brief Get the code completion token for filtering purposes.
1060  if (CodeCompletionII)
1061  return CodeCompletionII->getName();
1062  return {};
1063  }
1064 
1065  /// \brief Retrieve the preprocessing record, or NULL if there is no
1066  /// preprocessing record.
1067  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1068 
1069  /// \brief Create a new preprocessing record, which will keep track of
1070  /// all macro expansions, macro definitions, etc.
1071  void createPreprocessingRecord();
1072 
1073  /// \brief Enter the specified FileID as the main source file,
1074  /// which implicitly adds the builtin defines etc.
1075  void EnterMainSourceFile();
1076 
1077  /// \brief Inform the preprocessor callbacks that processing is complete.
1078  void EndSourceFile();
1079 
1080  /// \brief Add a source file to the top of the include stack and
1081  /// start lexing tokens from it instead of the current buffer.
1082  ///
1083  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1084  bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir,
1085  SourceLocation Loc);
1086 
1087  /// \brief Add a Macro to the top of the include stack and start lexing
1088  /// tokens from it instead of the current buffer.
1089  ///
1090  /// \param Args specifies the tokens input to a function-like macro.
1091  /// \param ILEnd specifies the location of the ')' for a function-like macro
1092  /// or the identifier for an object-like macro.
1093  void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro,
1094  MacroArgs *Args);
1095 
1096  /// \brief Add a "macro" context to the top of the include stack,
1097  /// which will cause the lexer to start returning the specified tokens.
1098  ///
1099  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1100  /// will not be subject to further macro expansion. Otherwise, these tokens
1101  /// will be re-macro-expanded when/if expansion is enabled.
1102  ///
1103  /// If \p OwnsTokens is false, this method assumes that the specified stream
1104  /// of tokens has a permanent owner somewhere, so they do not need to be
1105  /// copied. If it is true, it assumes the array of tokens is allocated with
1106  /// \c new[] and the Preprocessor will delete[] it.
1107 private:
1108  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1109  bool DisableMacroExpansion, bool OwnsTokens);
1110 
1111 public:
1112  void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1113  bool DisableMacroExpansion) {
1114  EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true);
1115  }
1116  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion) {
1117  EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false);
1118  }
1119 
1120  /// \brief Pop the current lexer/macro exp off the top of the lexer stack.
1121  ///
1122  /// This should only be used in situations where the current state of the
1123  /// top-of-stack lexer is known.
1124  void RemoveTopOfLexerStack();
1125 
1126  /// From the point that this method is called, and until
1127  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1128  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1129  /// make the Preprocessor re-lex the same tokens.
1130  ///
1131  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1132  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1133  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1134  ///
1135  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1136  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1137  /// tokens will continue indefinitely.
1138  ///
1139  void EnableBacktrackAtThisPos();
1140 
1141  /// \brief Disable the last EnableBacktrackAtThisPos call.
1142  void CommitBacktrackedTokens();
1143 
1145  CachedTokensTy::size_type Begin, End;
1146  };
1147 
1148 private:
1149  /// \brief A range of cached tokens that should be erased after lexing
1150  /// when backtracking requires the erasure of such cached tokens.
1151  Optional<CachedTokensRange> CachedTokenRangeToErase;
1152 
1153 public:
1154  /// \brief Returns the range of cached tokens that were lexed since
1155  /// EnableBacktrackAtThisPos() was previously called.
1156  CachedTokensRange LastCachedTokenRange();
1157 
1158  /// \brief Erase the range of cached tokens that were lexed since
1159  /// EnableBacktrackAtThisPos() was previously called.
1160  void EraseCachedTokens(CachedTokensRange TokenRange);
1161 
1162  /// \brief Make Preprocessor re-lex the tokens that were lexed since
1163  /// EnableBacktrackAtThisPos() was previously called.
1164  void Backtrack();
1165 
1166  /// \brief True if EnableBacktrackAtThisPos() was called and
1167  /// caching of tokens is on.
1168  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1169 
1170  /// \brief Lex the next token for this preprocessor.
1171  void Lex(Token &Result);
1172 
1173  void LexAfterModuleImport(Token &Result);
1174 
1175  void makeModuleVisible(Module *M, SourceLocation Loc);
1176 
1178  return CurSubmoduleState->VisibleModules.getImportLoc(M);
1179  }
1180 
1181  /// \brief Lex a string literal, which may be the concatenation of multiple
1182  /// string literals and may even come from macro expansion.
1183  /// \returns true on success, false if a error diagnostic has been generated.
1184  bool LexStringLiteral(Token &Result, std::string &String,
1185  const char *DiagnosticTag, bool AllowMacroExpansion) {
1186  if (AllowMacroExpansion)
1187  Lex(Result);
1188  else
1189  LexUnexpandedToken(Result);
1190  return FinishLexStringLiteral(Result, String, DiagnosticTag,
1191  AllowMacroExpansion);
1192  }
1193 
1194  /// \brief Complete the lexing of a string literal where the first token has
1195  /// already been lexed (see LexStringLiteral).
1196  bool FinishLexStringLiteral(Token &Result, std::string &String,
1197  const char *DiagnosticTag,
1198  bool AllowMacroExpansion);
1199 
1200  /// \brief Lex a token. If it's a comment, keep lexing until we get
1201  /// something not a comment.
1202  ///
1203  /// This is useful in -E -C mode where comments would foul up preprocessor
1204  /// directive handling.
1205  void LexNonComment(Token &Result) {
1206  do
1207  Lex(Result);
1208  while (Result.getKind() == tok::comment);
1209  }
1210 
1211  /// \brief Just like Lex, but disables macro expansion of identifier tokens.
1212  void LexUnexpandedToken(Token &Result) {
1213  // Disable macro expansion.
1214  bool OldVal = DisableMacroExpansion;
1215  DisableMacroExpansion = true;
1216  // Lex the token.
1217  Lex(Result);
1218 
1219  // Reenable it.
1220  DisableMacroExpansion = OldVal;
1221  }
1222 
1223  /// \brief Like LexNonComment, but this disables macro expansion of
1224  /// identifier tokens.
1226  do
1227  LexUnexpandedToken(Result);
1228  while (Result.getKind() == tok::comment);
1229  }
1230 
1231  /// \brief Parses a simple integer literal to get its numeric value. Floating
1232  /// point literals and user defined literals are rejected. Used primarily to
1233  /// handle pragmas that accept integer arguments.
1234  bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1235 
1236  /// Disables macro expansion everywhere except for preprocessor directives.
1238  DisableMacroExpansion = true;
1239  MacroExpansionInDirectivesOverride = true;
1240  }
1241 
1242  /// \brief Peeks ahead N tokens and returns that token without consuming any
1243  /// tokens.
1244  ///
1245  /// LookAhead(0) returns the next token that would be returned by Lex(),
1246  /// LookAhead(1) returns the token after it, etc. This returns normal
1247  /// tokens after phase 5. As such, it is equivalent to using
1248  /// 'Lex', not 'LexUnexpandedToken'.
1249  const Token &LookAhead(unsigned N) {
1250  if (CachedLexPos + N < CachedTokens.size())
1251  return CachedTokens[CachedLexPos+N];
1252  else
1253  return PeekAhead(N+1);
1254  }
1255 
1256  /// \brief When backtracking is enabled and tokens are cached,
1257  /// this allows to revert a specific number of tokens.
1258  ///
1259  /// Note that the number of tokens being reverted should be up to the last
1260  /// backtrack position, not more.
1261  void RevertCachedTokens(unsigned N) {
1262  assert(isBacktrackEnabled() &&
1263  "Should only be called when tokens are cached for backtracking");
1264  assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1265  && "Should revert tokens up to the last backtrack position, not more");
1266  assert(signed(CachedLexPos) - signed(N) >= 0 &&
1267  "Corrupted backtrack positions ?");
1268  CachedLexPos -= N;
1269  }
1270 
1271  /// \brief Enters a token in the token stream to be lexed next.
1272  ///
1273  /// If BackTrack() is called afterwards, the token will remain at the
1274  /// insertion point.
1275  void EnterToken(const Token &Tok) {
1276  EnterCachingLexMode();
1277  CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1278  }
1279 
1280  /// We notify the Preprocessor that if it is caching tokens (because
1281  /// backtrack is enabled) it should replace the most recent cached tokens
1282  /// with the given annotation token. This function has no effect if
1283  /// backtracking is not enabled.
1284  ///
1285  /// Note that the use of this function is just for optimization, so that the
1286  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1287  /// invoked.
1288  void AnnotateCachedTokens(const Token &Tok) {
1289  assert(Tok.isAnnotation() && "Expected annotation token");
1290  if (CachedLexPos != 0 && isBacktrackEnabled())
1291  AnnotatePreviousCachedTokens(Tok);
1292  }
1293 
1294  /// Get the location of the last cached token, suitable for setting the end
1295  /// location of an annotation token.
1297  assert(CachedLexPos != 0);
1298  return CachedTokens[CachedLexPos-1].getLastLoc();
1299  }
1300 
1301  /// \brief Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1302  /// CachedTokens.
1303  bool IsPreviousCachedToken(const Token &Tok) const;
1304 
1305  /// \brief Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1306  /// in \p NewToks.
1307  ///
1308  /// Useful when a token needs to be split in smaller ones and CachedTokens
1309  /// most recent token must to be updated to reflect that.
1310  void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1311 
1312  /// \brief Replace the last token with an annotation token.
1313  ///
1314  /// Like AnnotateCachedTokens(), this routine replaces an
1315  /// already-parsed (and resolved) token with an annotation
1316  /// token. However, this routine only replaces the last token with
1317  /// the annotation token; it does not affect any other cached
1318  /// tokens. This function has no effect if backtracking is not
1319  /// enabled.
1321  assert(Tok.isAnnotation() && "Expected annotation token");
1322  if (CachedLexPos != 0 && isBacktrackEnabled())
1323  CachedTokens[CachedLexPos-1] = Tok;
1324  }
1325 
1326  /// Enter an annotation token into the token stream.
1327  void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1328  void *AnnotationVal);
1329 
1330  /// Update the current token to represent the provided
1331  /// identifier, in order to cache an action performed by typo correction.
1332  void TypoCorrectToken(const Token &Tok) {
1333  assert(Tok.getIdentifierInfo() && "Expected identifier token");
1334  if (CachedLexPos != 0 && isBacktrackEnabled())
1335  CachedTokens[CachedLexPos-1] = Tok;
1336  }
1337 
1338  /// \brief Recompute the current lexer kind based on the CurLexer/CurPTHLexer/
1339  /// CurTokenLexer pointers.
1340  void recomputeCurLexerKind();
1341 
1342  /// \brief Returns true if incremental processing is enabled
1343  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1344 
1345  /// \brief Enables the incremental processing
1346  void enableIncrementalProcessing(bool value = true) {
1347  IncrementalProcessing = value;
1348  }
1349 
1350  /// \brief Specify the point at which code-completion will be performed.
1351  ///
1352  /// \param File the file in which code completion should occur. If
1353  /// this file is included multiple times, code-completion will
1354  /// perform completion the first time it is included. If NULL, this
1355  /// function clears out the code-completion point.
1356  ///
1357  /// \param Line the line at which code completion should occur
1358  /// (1-based).
1359  ///
1360  /// \param Column the column at which code completion should occur
1361  /// (1-based).
1362  ///
1363  /// \returns true if an error occurred, false otherwise.
1364  bool SetCodeCompletionPoint(const FileEntry *File,
1365  unsigned Line, unsigned Column);
1366 
1367  /// \brief Determine if we are performing code completion.
1368  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1369 
1370  /// \brief Returns the location of the code-completion point.
1371  ///
1372  /// Returns an invalid location if code-completion is not enabled or the file
1373  /// containing the code-completion point has not been lexed yet.
1374  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1375 
1376  /// \brief Returns the start location of the file of code-completion point.
1377  ///
1378  /// Returns an invalid location if code-completion is not enabled or the file
1379  /// containing the code-completion point has not been lexed yet.
1381  return CodeCompletionFileLoc;
1382  }
1383 
1384  /// \brief Returns true if code-completion is enabled and we have hit the
1385  /// code-completion point.
1386  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1387 
1388  /// \brief Note that we hit the code-completion point.
1390  assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1391  CodeCompletionReached = true;
1392  // Silence any diagnostics that occur after we hit the code-completion.
1393  getDiagnostics().setSuppressAllDiagnostics(true);
1394  }
1395 
1396  /// \brief The location of the currently-active \#pragma clang
1397  /// arc_cf_code_audited begin.
1398  ///
1399  /// Returns an invalid location if there is no such pragma active.
1401  return PragmaARCCFCodeAuditedLoc;
1402  }
1403 
1404  /// \brief Set the location of the currently-active \#pragma clang
1405  /// arc_cf_code_audited begin. An invalid location ends the pragma.
1407  PragmaARCCFCodeAuditedLoc = Loc;
1408  }
1409 
1410  /// \brief The location of the currently-active \#pragma clang
1411  /// assume_nonnull begin.
1412  ///
1413  /// Returns an invalid location if there is no such pragma active.
1415  return PragmaAssumeNonNullLoc;
1416  }
1417 
1418  /// \brief Set the location of the currently-active \#pragma clang
1419  /// assume_nonnull begin. An invalid location ends the pragma.
1421  PragmaAssumeNonNullLoc = Loc;
1422  }
1423 
1424  /// \brief Set the directory in which the main file should be considered
1425  /// to have been found, if it is not a real file.
1426  void setMainFileDir(const DirectoryEntry *Dir) {
1427  MainFileDir = Dir;
1428  }
1429 
1430  /// \brief Instruct the preprocessor to skip part of the main source file.
1431  ///
1432  /// \param Bytes The number of bytes in the preamble to skip.
1433  ///
1434  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1435  /// start of a line.
1436  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1437  SkipMainFilePreamble.first = Bytes;
1438  SkipMainFilePreamble.second = StartOfLine;
1439  }
1440 
1441  /// Forwarding function for diagnostics. This emits a diagnostic at
1442  /// the specified Token's location, translating the token's start
1443  /// position in the current buffer into a SourcePosition object for rendering.
1444  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1445  return Diags->Report(Loc, DiagID);
1446  }
1447 
1448  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1449  return Diags->Report(Tok.getLocation(), DiagID);
1450  }
1451 
1452  /// Return the 'spelling' of the token at the given
1453  /// location; does not go up to the spelling location or down to the
1454  /// expansion location.
1455  ///
1456  /// \param buffer A buffer which will be used only if the token requires
1457  /// "cleaning", e.g. if it contains trigraphs or escaped newlines
1458  /// \param invalid If non-null, will be set \c true if an error occurs.
1460  SmallVectorImpl<char> &buffer,
1461  bool *invalid = nullptr) const {
1462  return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1463  }
1464 
1465  /// \brief Return the 'spelling' of the Tok token.
1466  ///
1467  /// The spelling of a token is the characters used to represent the token in
1468  /// the source file after trigraph expansion and escaped-newline folding. In
1469  /// particular, this wants to get the true, uncanonicalized, spelling of
1470  /// things like digraphs, UCNs, etc.
1471  ///
1472  /// \param Invalid If non-null, will be set \c true if an error occurs.
1473  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1474  return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1475  }
1476 
1477  /// \brief Get the spelling of a token into a preallocated buffer, instead
1478  /// of as an std::string.
1479  ///
1480  /// The caller is required to allocate enough space for the token, which is
1481  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1482  /// actual result is returned.
1483  ///
1484  /// Note that this method may do two possible things: it may either fill in
1485  /// the buffer specified with characters, or it may *change the input pointer*
1486  /// to point to a constant buffer with the data already in it (avoiding a
1487  /// copy). The caller is not allowed to modify the returned buffer pointer
1488  /// if an internal buffer is returned.
1489  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1490  bool *Invalid = nullptr) const {
1491  return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1492  }
1493 
1494  /// \brief Get the spelling of a token into a SmallVector.
1495  ///
1496  /// Note that the returned StringRef may not point to the
1497  /// supplied buffer if a copy can be avoided.
1498  StringRef getSpelling(const Token &Tok,
1499  SmallVectorImpl<char> &Buffer,
1500  bool *Invalid = nullptr) const;
1501 
1502  /// \brief Relex the token at the specified location.
1503  /// \returns true if there was a failure, false on success.
1504  bool getRawToken(SourceLocation Loc, Token &Result,
1505  bool IgnoreWhiteSpace = false) {
1506  return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1507  }
1508 
1509  /// \brief Given a Token \p Tok that is a numeric constant with length 1,
1510  /// return the character.
1511  char
1513  bool *Invalid = nullptr) const {
1514  assert(Tok.is(tok::numeric_constant) &&
1515  Tok.getLength() == 1 && "Called on unsupported token");
1516  assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1517 
1518  // If the token is carrying a literal data pointer, just use it.
1519  if (const char *D = Tok.getLiteralData())
1520  return *D;
1521 
1522  // Otherwise, fall back on getCharacterData, which is slower, but always
1523  // works.
1524  return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1525  }
1526 
1527  /// \brief Retrieve the name of the immediate macro expansion.
1528  ///
1529  /// This routine starts from a source location, and finds the name of the
1530  /// macro responsible for its immediate expansion. It looks through any
1531  /// intervening macro argument expansions to compute this. It returns a
1532  /// StringRef that refers to the SourceManager-owned buffer of the source
1533  /// where that macro name is spelled. Thus, the result shouldn't out-live
1534  /// the SourceManager.
1536  return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1537  }
1538 
1539  /// \brief Plop the specified string into a scratch buffer and set the
1540  /// specified token's location and length to it.
1541  ///
1542  /// If specified, the source location provides a location of the expansion
1543  /// point of the token.
1544  void CreateString(StringRef Str, Token &Tok,
1545  SourceLocation ExpansionLocStart = SourceLocation(),
1546  SourceLocation ExpansionLocEnd = SourceLocation());
1547 
1548  /// \brief Computes the source location just past the end of the
1549  /// token at this source location.
1550  ///
1551  /// This routine can be used to produce a source location that
1552  /// points just past the end of the token referenced by \p Loc, and
1553  /// is generally used when a diagnostic needs to point just after a
1554  /// token where it expected something different that it received. If
1555  /// the returned source location would not be meaningful (e.g., if
1556  /// it points into a macro), this routine returns an invalid
1557  /// source location.
1558  ///
1559  /// \param Offset an offset from the end of the token, where the source
1560  /// location should refer to. The default offset (0) produces a source
1561  /// location pointing just past the end of the token; an offset of 1 produces
1562  /// a source location pointing to the last character in the token, etc.
1564  return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1565  }
1566 
1567  /// \brief Returns true if the given MacroID location points at the first
1568  /// token of the macro expansion.
1569  ///
1570  /// \param MacroBegin If non-null and function returns true, it is set to
1571  /// begin location of the macro.
1573  SourceLocation *MacroBegin = nullptr) const {
1574  return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1575  MacroBegin);
1576  }
1577 
1578  /// \brief Returns true if the given MacroID location points at the last
1579  /// token of the macro expansion.
1580  ///
1581  /// \param MacroEnd If non-null and function returns true, it is set to
1582  /// end location of the macro.
1584  SourceLocation *MacroEnd = nullptr) const {
1585  return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1586  }
1587 
1588  /// \brief Print the token to stderr, used for debugging.
1589  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1590  void DumpLocation(SourceLocation Loc) const;
1591  void DumpMacro(const MacroInfo &MI) const;
1592  void dumpMacroInfo(const IdentifierInfo *II);
1593 
1594  /// \brief Given a location that specifies the start of a
1595  /// token, return a new location that specifies a character within the token.
1597  unsigned Char) const {
1598  return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1599  }
1600 
1601  /// \brief Increment the counters for the number of token paste operations
1602  /// performed.
1603  ///
1604  /// If fast was specified, this is a 'fast paste' case we handled.
1605  void IncrementPasteCounter(bool isFast) {
1606  if (isFast)
1607  ++NumFastTokenPaste;
1608  else
1609  ++NumTokenPaste;
1610  }
1611 
1612  void PrintStats();
1613 
1614  size_t getTotalMemory() const;
1615 
1616  /// When the macro expander pastes together a comment (/##/) in Microsoft
1617  /// mode, this method handles updating the current state, returning the
1618  /// token on the next source line.
1619  void HandleMicrosoftCommentPaste(Token &Tok);
1620 
1621  //===--------------------------------------------------------------------===//
1622  // Preprocessor callback methods. These are invoked by a lexer as various
1623  // directives and events are found.
1624 
1625  /// Given a tok::raw_identifier token, look up the
1626  /// identifier information for the token and install it into the token,
1627  /// updating the token kind accordingly.
1628  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1629 
1630 private:
1631  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1632 
1633 public:
1634 
1635  /// \brief Specifies the reason for poisoning an identifier.
1636  ///
1637  /// If that identifier is accessed while poisoned, then this reason will be
1638  /// used instead of the default "poisoned" diagnostic.
1639  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1640 
1641  /// \brief Display reason for poisoned identifier.
1642  void HandlePoisonedIdentifier(Token & Tok);
1643 
1645  if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1646  if(II->isPoisoned()) {
1647  HandlePoisonedIdentifier(Identifier);
1648  }
1649  }
1650  }
1651 
1652 private:
1653  /// Identifiers used for SEH handling in Borland. These are only
1654  /// allowed in particular circumstances
1655  // __except block
1656  IdentifierInfo *Ident__exception_code,
1657  *Ident___exception_code,
1658  *Ident_GetExceptionCode;
1659  // __except filter expression
1660  IdentifierInfo *Ident__exception_info,
1661  *Ident___exception_info,
1662  *Ident_GetExceptionInfo;
1663  // __finally
1664  IdentifierInfo *Ident__abnormal_termination,
1665  *Ident___abnormal_termination,
1666  *Ident_AbnormalTermination;
1667 
1668  const char *getCurLexerEndPos();
1669  void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
1670 
1671 public:
1672  void PoisonSEHIdentifiers(bool Poison = true); // Borland
1673 
1674  /// \brief Callback invoked when the lexer reads an identifier and has
1675  /// filled in the tokens IdentifierInfo member.
1676  ///
1677  /// This callback potentially macro expands it or turns it into a named
1678  /// token (like 'for').
1679  ///
1680  /// \returns true if we actually computed a token, false if we need to
1681  /// lex again.
1682  bool HandleIdentifier(Token &Identifier);
1683 
1684 
1685  /// \brief Callback invoked when the lexer hits the end of the current file.
1686  ///
1687  /// This either returns the EOF token and returns true, or
1688  /// pops a level off the include stack and returns false, at which point the
1689  /// client should call lex again.
1690  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1691 
1692  /// \brief Callback invoked when the current TokenLexer hits the end of its
1693  /// token stream.
1694  bool HandleEndOfTokenLexer(Token &Result);
1695 
1696  /// \brief Callback invoked when the lexer sees a # token at the start of a
1697  /// line.
1698  ///
1699  /// This consumes the directive, modifies the lexer/preprocessor state, and
1700  /// advances the lexer(s) so that the next token read is the correct one.
1701  void HandleDirective(Token &Result);
1702 
1703  /// \brief Ensure that the next token is a tok::eod token.
1704  ///
1705  /// If not, emit a diagnostic and consume up until the eod.
1706  /// If \p EnableMacros is true, then we consider macros that expand to zero
1707  /// tokens as being ok.
1708  void CheckEndOfDirective(const char *Directive, bool EnableMacros = false);
1709 
1710  /// \brief Read and discard all tokens remaining on the current line until
1711  /// the tok::eod token is found.
1712  void DiscardUntilEndOfDirective();
1713 
1714  /// \brief Returns true if the preprocessor has seen a use of
1715  /// __DATE__ or __TIME__ in the file so far.
1716  bool SawDateOrTime() const {
1717  return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
1718  }
1719  unsigned getCounterValue() const { return CounterValue; }
1720  void setCounterValue(unsigned V) { CounterValue = V; }
1721 
1722  /// \brief Retrieves the module that we're currently building, if any.
1723  Module *getCurrentModule();
1724 
1725  /// \brief Allocate a new MacroInfo object with the provided SourceLocation.
1726  MacroInfo *AllocateMacroInfo(SourceLocation L);
1727 
1728  /// \brief Turn the specified lexer token into a fully checked and spelled
1729  /// filename, e.g. as an operand of \#include.
1730  ///
1731  /// The caller is expected to provide a buffer that is large enough to hold
1732  /// the spelling of the filename, but is also expected to handle the case
1733  /// when this method decides to use a different buffer.
1734  ///
1735  /// \returns true if the input filename was in <>'s or false if it was
1736  /// in ""'s.
1737  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Filename);
1738 
1739  /// \brief Given a "foo" or <foo> reference, look up the indicated file.
1740  ///
1741  /// Returns null on failure. \p isAngled indicates whether the file
1742  /// reference is for system \#include's or not (i.e. using <> instead of "").
1743  const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename,
1744  bool isAngled, const DirectoryLookup *FromDir,
1745  const FileEntry *FromFile,
1746  const DirectoryLookup *&CurDir,
1747  SmallVectorImpl<char> *SearchPath,
1748  SmallVectorImpl<char> *RelativePath,
1749  ModuleMap::KnownHeader *SuggestedModule,
1750  bool *IsMapped, bool SkipCache = false);
1751 
1752  /// \brief Get the DirectoryLookup structure used to find the current
1753  /// FileEntry, if CurLexer is non-null and if applicable.
1754  ///
1755  /// This allows us to implement \#include_next and find directory-specific
1756  /// properties.
1757  const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1758 
1759  /// \brief Return true if we're in the top-level file, not in a \#include.
1760  bool isInPrimaryFile() const;
1761 
1762  /// \brief Handle cases where the \#include name is expanded
1763  /// from a macro as multiple tokens, which need to be glued together.
1764  ///
1765  /// This occurs for code like:
1766  /// \code
1767  /// \#define FOO <x/y.h>
1768  /// \#include FOO
1769  /// \endcode
1770  /// because in this case, "<x/y.h>" is returned as 7 tokens, not one.
1771  ///
1772  /// This code concatenates and consumes tokens up to the '>' token. It
1773  /// returns false if the > was found, otherwise it returns true if it finds
1774  /// and consumes the EOD marker.
1775  bool ConcatenateIncludeName(SmallString<128> &FilenameBuffer,
1776  SourceLocation &End);
1777 
1778  /// \brief Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1779  /// followed by EOD. Return true if the token is not a valid on-off-switch.
1780  bool LexOnOffSwitch(tok::OnOffSwitch &OOS);
1781 
1782  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
1783  bool *ShadowFlag = nullptr);
1784 
1785  void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
1786  Module *LeaveSubmodule(bool ForPragma);
1787 
1788 private:
1789  void PushIncludeMacroStack() {
1790  assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
1791  IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
1792  std::move(CurLexer), std::move(CurPTHLexer),
1793  CurPPLexer, std::move(CurTokenLexer),
1794  CurDirLookup);
1795  CurPPLexer = nullptr;
1796  }
1797 
1798  void PopIncludeMacroStack() {
1799  CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1800  CurPTHLexer = std::move(IncludeMacroStack.back().ThePTHLexer);
1801  CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1802  CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1803  CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1804  CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
1805  CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1806  IncludeMacroStack.pop_back();
1807  }
1808 
1809  void PropagateLineStartLeadingSpaceInfo(Token &Result);
1810 
1811  /// Determine whether we need to create module macros for #defines in the
1812  /// current context.
1813  bool needModuleMacros() const;
1814 
1815  /// Update the set of active module macros and ambiguity flag for a module
1816  /// macro name.
1817  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
1818 
1819  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
1820  SourceLocation Loc);
1821  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
1822  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
1823  bool isPublic);
1824 
1825  /// \brief Lex and validate a macro name, which occurs after a
1826  /// \#define or \#undef.
1827  ///
1828  /// \param MacroNameTok Token that represents the name defined or undefined.
1829  /// \param IsDefineUndef Kind if preprocessor directive.
1830  /// \param ShadowFlag Points to flag that is set if macro name shadows
1831  /// a keyword.
1832  ///
1833  /// This emits a diagnostic, sets the token kind to eod,
1834  /// and discards the rest of the macro line if the macro name is invalid.
1835  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
1836  bool *ShadowFlag = nullptr);
1837 
1838  /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
1839  /// entire line) of the macro's tokens and adds them to MacroInfo, and while
1840  /// doing so performs certain validity checks including (but not limited to):
1841  /// - # (stringization) is followed by a macro parameter
1842  /// \param MacroNameTok - Token that represents the macro name
1843  /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
1844  ///
1845  /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
1846  /// returns a nullptr if an invalid sequence of tokens is encountered.
1847 
1848  MacroInfo *ReadOptionalMacroParameterListAndBody(
1849  const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
1850 
1851  /// The ( starting an argument list of a macro definition has just been read.
1852  /// Lex the rest of the parameters and the closing ), updating \p MI with
1853  /// what we learn and saving in \p LastTok the last token read.
1854  /// Return true if an error occurs parsing the arg list.
1855  bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
1856 
1857  /// We just read a \#if or related directive and decided that the
1858  /// subsequent tokens are in the \#if'd out portion of the
1859  /// file. Lex the rest of the file, until we see an \#endif. If \p
1860  /// FoundNonSkipPortion is true, then we have already emitted code for part of
1861  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
1862  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
1863  /// already seen one so a \#else directive is a duplicate. When this returns,
1864  /// the caller can lex the first valid token.
1865  void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
1866  SourceLocation IfTokenLoc,
1867  bool FoundNonSkipPortion, bool FoundElse,
1868  SourceLocation ElseLoc = SourceLocation());
1869 
1870  /// \brief A fast PTH version of SkipExcludedConditionalBlock.
1871  void PTHSkipExcludedConditionalBlock();
1872 
1873  /// Information about the result for evaluating an expression for a
1874  /// preprocessor directive.
1875  struct DirectiveEvalResult {
1876  /// Whether the expression was evaluated as true or not.
1877  bool Conditional;
1878  /// True if the expression contained identifiers that were undefined.
1879  bool IncludedUndefinedIds;
1880  };
1881 
1882  /// \brief Evaluate an integer constant expression that may occur after a
1883  /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
1884  ///
1885  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
1886  DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1887 
1888  /// \brief Install the standard preprocessor pragmas:
1889  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
1890  void RegisterBuiltinPragmas();
1891 
1892  /// \brief Register builtin macros such as __LINE__ with the identifier table.
1893  void RegisterBuiltinMacros();
1894 
1895  /// If an identifier token is read that is to be expanded as a macro, handle
1896  /// it and return the next token as 'Tok'. If we lexed a token, return true;
1897  /// otherwise the caller should lex again.
1898  bool HandleMacroExpandedIdentifier(Token &Tok, const MacroDefinition &MD);
1899 
1900  /// \brief Cache macro expanded tokens for TokenLexers.
1901  //
1902  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
1903  /// going to lex in the cache and when it finishes the tokens are removed
1904  /// from the end of the cache.
1905  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
1906  ArrayRef<Token> tokens);
1907  void removeCachedMacroExpandedTokensOfLastLexer();
1908  friend void TokenLexer::ExpandFunctionArguments();
1909 
1910  /// Determine whether the next preprocessor token to be
1911  /// lexed is a '('. If so, consume the token and return true, if not, this
1912  /// method should have no observable side-effect on the lexed tokens.
1913  bool isNextPPTokenLParen();
1914 
1915  /// After reading "MACRO(", this method is invoked to read all of the formal
1916  /// arguments specified for the macro invocation. Returns null on error.
1917  MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
1918  SourceLocation &ExpansionEnd);
1919 
1920  /// \brief If an identifier token is read that is to be expanded
1921  /// as a builtin macro, handle it and return the next token as 'Tok'.
1922  void ExpandBuiltinMacro(Token &Tok);
1923 
1924  /// \brief Read a \c _Pragma directive, slice it up, process it, then
1925  /// return the first token after the directive.
1926  /// This assumes that the \c _Pragma token has just been read into \p Tok.
1927  void Handle_Pragma(Token &Tok);
1928 
1929  /// \brief Like Handle_Pragma except the pragma text is not enclosed within
1930  /// a string literal.
1931  void HandleMicrosoft__pragma(Token &Tok);
1932 
1933  /// \brief Add a lexer to the top of the include stack and
1934  /// start lexing tokens from it instead of the current buffer.
1935  void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
1936 
1937  /// \brief Add a lexer to the top of the include stack and
1938  /// start getting tokens from it using the PTH cache.
1939  void EnterSourceFileWithPTH(PTHLexer *PL, const DirectoryLookup *Dir);
1940 
1941  /// \brief Set the FileID for the preprocessor predefines.
1942  void setPredefinesFileID(FileID FID) {
1943  assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
1944  PredefinesFileID = FID;
1945  }
1946 
1947  /// \brief Returns true if we are lexing from a file and not a
1948  /// pragma or a macro.
1949  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
1950  return L ? !L->isPragmaLexer() : P != nullptr;
1951  }
1952 
1953  static bool IsFileLexer(const IncludeStackInfo& I) {
1954  return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
1955  }
1956 
1957  bool IsFileLexer() const {
1958  return IsFileLexer(CurLexer.get(), CurPPLexer);
1959  }
1960 
1961  //===--------------------------------------------------------------------===//
1962  // Caching stuff.
1963  void CachingLex(Token &Result);
1964  bool InCachingLexMode() const {
1965  // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
1966  // that we are past EOF, not that we are in CachingLex mode.
1967  return !CurPPLexer && !CurTokenLexer && !CurPTHLexer &&
1968  !IncludeMacroStack.empty();
1969  }
1970  void EnterCachingLexMode();
1971  void ExitCachingLexMode() {
1972  if (InCachingLexMode())
1973  RemoveTopOfLexerStack();
1974  }
1975  const Token &PeekAhead(unsigned N);
1976  void AnnotatePreviousCachedTokens(const Token &Tok);
1977 
1978  //===--------------------------------------------------------------------===//
1979  /// Handle*Directive - implement the various preprocessor directives. These
1980  /// should side-effect the current preprocessor object so that the next call
1981  /// to Lex() will return the appropriate token next.
1982  void HandleLineDirective();
1983  void HandleDigitDirective(Token &Tok);
1984  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
1985  void HandleIdentSCCSDirective(Token &Tok);
1986  void HandleMacroPublicDirective(Token &Tok);
1987  void HandleMacroPrivateDirective();
1988 
1989  // File inclusion.
1990  void HandleIncludeDirective(SourceLocation HashLoc,
1991  Token &Tok,
1992  const DirectoryLookup *LookupFrom = nullptr,
1993  const FileEntry *LookupFromFile = nullptr,
1994  bool isImport = false);
1995  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
1996  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
1997  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
1998  void HandleMicrosoftImportDirective(Token &Tok);
1999 
2000 public:
2001  /// Check that the given module is available, producing a diagnostic if not.
2002  /// \return \c true if the check failed (because the module is not available).
2003  /// \c false if the module appears to be usable.
2004  static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2005  const TargetInfo &TargetInfo,
2006  DiagnosticsEngine &Diags, Module *M);
2007 
2008  // Module inclusion testing.
2009  /// \brief Find the module that owns the source or header file that
2010  /// \p Loc points to. If the location is in a file that was included
2011  /// into a module, or is outside any module, returns nullptr.
2012  Module *getModuleForLocation(SourceLocation Loc);
2013 
2014  /// \brief We want to produce a diagnostic at location IncLoc concerning a
2015  /// missing module import.
2016  ///
2017  /// \param IncLoc The location at which the missing import was detected.
2018  /// \param M The desired module.
2019  /// \param MLoc A location within the desired module at which some desired
2020  /// effect occurred (eg, where a desired entity was declared).
2021  ///
2022  /// \return A file that can be #included to import a module containing MLoc.
2023  /// Null if no such file could be determined or if a #include is not
2024  /// appropriate.
2025  const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2026  Module *M,
2027  SourceLocation MLoc);
2028 
2029  bool isRecordingPreamble() const {
2030  return PreambleConditionalStack.isRecording();
2031  }
2032 
2033  bool hasRecordedPreamble() const {
2034  return PreambleConditionalStack.hasRecordedPreamble();
2035  }
2036 
2038  return PreambleConditionalStack.getStack();
2039  }
2040 
2042  PreambleConditionalStack.setStack(s);
2043  }
2044 
2047  PreambleConditionalStack.startReplaying();
2048  PreambleConditionalStack.setStack(s);
2049  PreambleConditionalStack.SkipInfo = SkipInfo;
2050  }
2051 
2053  return PreambleConditionalStack.SkipInfo;
2054  }
2055 
2056 private:
2057  /// \brief After processing predefined file, initialize the conditional stack from
2058  /// the preamble.
2059  void replayPreambleConditionalStack();
2060 
2061  // Macro handling.
2062  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterTopLevelIfndef);
2063  void HandleUndefDirective();
2064 
2065  // Conditional Inclusion.
2066  void HandleIfdefDirective(Token &Tok, const Token &HashToken,
2067  bool isIfndef, bool ReadAnyTokensBeforeDirective);
2068  void HandleIfDirective(Token &Tok, const Token &HashToken,
2069  bool ReadAnyTokensBeforeDirective);
2070  void HandleEndifDirective(Token &Tok);
2071  void HandleElseDirective(Token &Tok, const Token &HashToken);
2072  void HandleElifDirective(Token &Tok, const Token &HashToken);
2073 
2074  // Pragmas.
2075  void HandlePragmaDirective(SourceLocation IntroducerLoc,
2076  PragmaIntroducerKind Introducer);
2077 public:
2078  void HandlePragmaOnce(Token &OnceTok);
2079  void HandlePragmaMark();
2080  void HandlePragmaPoison();
2081  void HandlePragmaSystemHeader(Token &SysHeaderTok);
2082  void HandlePragmaDependency(Token &DependencyTok);
2083  void HandlePragmaPushMacro(Token &Tok);
2084  void HandlePragmaPopMacro(Token &Tok);
2085  void HandlePragmaIncludeAlias(Token &Tok);
2086  void HandlePragmaModuleBuild(Token &Tok);
2087  IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2088 
2089  // Return true and store the first token only if any CommentHandler
2090  // has inserted some tokens and getCommentRetentionState() is false.
2091  bool HandleComment(Token &Token, SourceRange Comment);
2092 
2093  /// \brief A macro is used, update information about macros that need unused
2094  /// warnings.
2095  void markMacroAsUsed(MacroInfo *MI);
2096 };
2097 
2098 /// \brief Abstract base class that describes a handler that will receive
2099 /// source ranges for each of the comments encountered in the source file.
2101 public:
2102  virtual ~CommentHandler();
2103 
2104  // The handler shall return true if it has pushed any tokens
2105  // to be read using e.g. EnterToken or EnterTokenStream.
2106  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2107 };
2108 
2109 /// \brief Registry of pragma handlers added by plugins
2110 typedef llvm::Registry<PragmaHandler> PragmaHandlerRegistry;
2111 
2112 } // end namespace clang
2113 
2114 #endif
IdentifierInfo * get(StringRef Name) override
get - Return the identifier token info for the specified named identifier.
Definition: PTHLexer.cpp:581
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
Definition: Preprocessor.h:291
A set of visible modules.
Definition: Module.h:571
llvm::BumpPtrAllocator & getPreprocessorAllocator()
Definition: Preprocessor.h:763
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:63
SelectorTable & getSelectorTable()
Definition: Preprocessor.h:761
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:68
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:116
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the &#39;spelling&#39; of the Tok token.
MemoryBufferCache & getPCMCache() const
Definition: Preprocessor.h:756
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:26
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
static const Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:21
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
FileManager & getFileManager() const
Definition: Preprocessor.h:754
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
SourceLocation getPragmaARCCFCodeAuditedLoc() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
StringRef P
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:545
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1205
A directive for an undefined macro.
Definition: MacroInfo.h:419
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
MacroMap::const_iterator macro_iterator
Definition: Preprocessor.h:975
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool hadModuleLoaderFatalFailure() const
Definition: Preprocessor.h:780
void EnterToken(const Token &Tok)
Enters a token in the token stream to be lexed next.
Manage memory buffers across multiple users.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:79
Builtin::Context & getBuiltinInfo()
Definition: Preprocessor.h:762
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
Definition: Preprocessor.h:995
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:36
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:118
tok::TokenKind getKind() const
Definition: Token.h:90
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:71
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
Definition: Preprocessor.h:942
One of these records is kept for each identifier that is lexed.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
Represents a macro directive exported by a module.
Definition: MacroInfo.h:476
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:397
This table allows us to fully hide how we implement multi-keyword caching.
LineState State
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:752
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Definition: Preprocessor.h:790
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:48
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:751
Describes a module or submodule.
Definition: Module.h:65
bool isPragmaLexer() const
isPragmaLexer - Returns true if this Lexer is being used to lex a pragma.
Definition: Lexer.h:160
A directive for setting the module visibility of a macro.
Definition: MacroInfo.h:433
bool isInvalid() const
void SetSuppressIncludeNotFoundError(bool Suppress)
Definition: Preprocessor.h:800
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:86
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
Definition: Preprocessor.h:837
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:757
uint32_t Offset
Definition: CacheTokens.cpp:43
const FormatToken & Tok
void LexNonComment(Token &Result)
Lex a token.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:147
bool isMacroDefined(const IdentifierInfo *II)
Definition: Preprocessor.h:859
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we&#39;ve alread...
Definition: Preprocessor.h:867
Defines the Diagnostic-related interfaces.
const TargetInfo * getAuxTargetInfo() const
Definition: Preprocessor.h:753
TokenLexer - This implements a lexer that returns tokens from a macro body or token stream instead of...
Definition: TokenLexer.h:31
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
Definition: Preprocessor.h:810
PragmaIntroducerKind
Describes how the pragma was introduced, e.g., with #pragma, _Pragma, or __pragma.
Definition: Pragma.h:32
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:137
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:953
void clearCodeCompletionHandler()
Clear out the code completion handler.
StringRef Filename
Definition: Format.cpp:1335
Provides lookups to, and iteration over, IdentiferInfo objects.
Exposes information about the current target.
Definition: TargetInfo.h:55
const IdentifierTable & getIdentifierTable() const
Definition: Preprocessor.h:760
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Abstract interface for external sources of preprocessor information.
unsigned getCounterValue() const
SourceLocation End
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:343
int Id
Definition: ASTDiff.cpp:191
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
const AnnotatedLine * Line
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Definition: Preprocessor.h:819
Implements an efficient mapping from strings to IdentifierInfo nodes.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
A class for tracking whether we&#39;re inside a VA_OPT during a traversal of the tokens of a variadic mac...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
#define bool
Definition: stdbool.h:31
SourceLocation Begin
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:66
llvm::Optional< PreambleSkipInfo > getPreambleSkipInfo() const
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
Definition: Preprocessor.h:921
bool getCommentRetentionState() const
Definition: Preprocessor.h:795
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:78
PPCallbacks * getPPCallbacks() const
Accessors for preprocessor callbacks.
Definition: Preprocessor.h:847
const SourceManager & SM
Definition: Format.cpp:1327
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
bool isRecordingPreamble() const
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
SourceManager & getSourceManager() const
Definition: Preprocessor.h:755
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define&#39;d and not #undef&#39;...
Definition: Preprocessor.h:910
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:773
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:286
Kind
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
void setExternalSource(ExternalPreprocessorSource *Source)
Definition: Preprocessor.h:769
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.h:95
void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:880
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:177
IdentifierTable & getIdentifierTable()
Definition: Preprocessor.h:759
void setPragmasEnabled(bool Enabled)
Definition: Preprocessor.h:797
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:87
bool isPoisoned() const
Return true if this token has been poisoned.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
Definition: Preprocessor.h:893
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
Definition: Preprocessor.h:827
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
StringRef getName() const
Return the actual identifier string.
bool isMacroDefined(StringRef Id)
Definition: Preprocessor.h:856
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
Dataflow Directional Tag Classes.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion)
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
Definition: DeclSpec.h:1128
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion)
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:358
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:59
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
Definition: Preprocessor.h:979
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Abstract interface for a module loader.
Definition: ModuleLoader.h:69
unsigned getLength() const
Definition: Token.h:127
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:34
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:49
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:215
bool getPragmasEnabled() const
Definition: Preprocessor.h:798
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:117
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
Definition: Preprocessor.h:778
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Definition: Preprocessor.h:746
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:45
Defines the PPCallbacks interface.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
Definition: Preprocessor.h:948
SourceLocation getModuleImportLoc(Module *M) const
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:840
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:748
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:242
bool hasRecordedPreamble() const
const std::string & getPredefines() const
Definition: Preprocessor.h:991
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:283
void setPredefines(StringRef P)
Definition: Preprocessor.h:996
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
Definition: Preprocessor.h:962
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
The translation unit is a complete translation unit.
Definition: LangOptions.h:244
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
Abstract base class that describes a handler that will receive source ranges for each of the comments...
void setCounterValue(unsigned V)
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
Definition: Preprocessor.h:785
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
Definition: Preprocessor.h:804
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, llvm::Optional< PreambleSkipInfo > SkipInfo)
Directive - Abstract class representing a parsed verify directive.
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Definition: Preprocessor.h:925
Callback handler that receives notifications when performing code completion within the preprocessor...
A header that is known to reside within a given module, whether it was included or excluded...
Definition: ModuleMap.h:131
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
Definition: Preprocessor.h:816
PTHManager * getPTHManager()
Definition: Preprocessor.h:767
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:848
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:98
void setDiagnostics(DiagnosticsEngine &D)
Definition: Preprocessor.h:749