clang  6.0.0svn
Preprocessor.h
Go to the documentation of this file.
1 //===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// \brief Defines the clang::Preprocessor interface.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
16 #define LLVM_CLANG_LEX_PREPROCESSOR_H
17 
18 #include "clang/Basic/Builtins.h"
19 #include "clang/Basic/Diagnostic.h"
21 #include "clang/Basic/LLVM.h"
23 #include "clang/Basic/Module.h"
26 #include "clang/Basic/TokenKinds.h"
27 #include "clang/Lex/Lexer.h"
28 #include "clang/Lex/MacroInfo.h"
29 #include "clang/Lex/ModuleLoader.h"
30 #include "clang/Lex/ModuleMap.h"
31 #include "clang/Lex/PPCallbacks.h"
32 #include "clang/Lex/PTHLexer.h"
33 #include "clang/Lex/Token.h"
34 #include "clang/Lex/TokenLexer.h"
35 #include "llvm/ADT/ArrayRef.h"
36 #include "llvm/ADT/DenseMap.h"
37 #include "llvm/ADT/FoldingSet.h"
38 #include "llvm/ADT/None.h"
39 #include "llvm/ADT/Optional.h"
40 #include "llvm/ADT/PointerUnion.h"
41 #include "llvm/ADT/STLExtras.h"
42 #include "llvm/ADT/SmallPtrSet.h"
43 #include "llvm/ADT/SmallVector.h"
44 #include "llvm/ADT/StringRef.h"
45 #include "llvm/ADT/TinyPtrVector.h"
46 #include "llvm/ADT/iterator_range.h"
47 #include "llvm/Support/Allocator.h"
48 #include "llvm/Support/Casting.h"
49 #include "llvm/Support/Registry.h"
50 #include <cassert>
51 #include <cstddef>
52 #include <cstdint>
53 #include <memory>
54 #include <map>
55 #include <string>
56 #include <utility>
57 #include <vector>
58 
59 namespace llvm {
60 
61 template<unsigned InternalLen> class SmallString;
62 
63 } // namespace llvm
64 
65 namespace clang {
66 
67 class CodeCompletionHandler;
68 class CommentHandler;
69 class DirectoryEntry;
70 class DirectoryLookup;
71 class ExternalPreprocessorSource;
72 class FileEntry;
73 class FileManager;
74 class HeaderSearch;
75 class MacroArgs;
76 class MemoryBufferCache;
77 class PragmaHandler;
78 class PragmaNamespace;
79 class PreprocessingRecord;
80 class PreprocessorLexer;
81 class PreprocessorOptions;
82 class PTHManager;
83 class ScratchBuffer;
84 class TargetInfo;
85 
86 /// \brief Stores token information for comparing actual tokens with
87 /// predefined values. Only handles simple tokens and identifiers.
88 class TokenValue {
90  IdentifierInfo *II;
91 
92 public:
93  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
94  assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
95  assert(Kind != tok::identifier &&
96  "Identifiers should be created by TokenValue(IdentifierInfo *)");
97  assert(!tok::isLiteral(Kind) && "Literals are not supported.");
98  assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
99  }
100 
101  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
102 
103  bool operator==(const Token &Tok) const {
104  return Tok.getKind() == Kind &&
105  (!II || II == Tok.getIdentifierInfo());
106  }
107 };
108 
109 /// \brief Context in which macro name is used.
110 enum MacroUse {
111  // other than #define or #undef
112  MU_Other = 0,
113 
114  // macro name specified in #define
116 
117  // macro name specified in #undef
119 };
120 
121 /// \brief Engages in a tight little dance with the lexer to efficiently
122 /// preprocess tokens.
123 ///
124 /// Lexers know only about tokens within a single source file, and don't
125 /// know anything about preprocessor-level issues like the \#include stack,
126 /// token expansion, etc.
130 
131  std::shared_ptr<PreprocessorOptions> PPOpts;
132  DiagnosticsEngine *Diags;
133  LangOptions &LangOpts;
134  const TargetInfo *Target = nullptr;
135  const TargetInfo *AuxTarget = nullptr;
136  FileManager &FileMgr;
137  SourceManager &SourceMgr;
138  MemoryBufferCache &PCMCache;
139  std::unique_ptr<ScratchBuffer> ScratchBuf;
140  HeaderSearch &HeaderInfo;
141  ModuleLoader &TheModuleLoader;
142 
143  /// \brief External source of macros.
144  ExternalPreprocessorSource *ExternalSource;
145 
146  /// An optional PTHManager object used for getting tokens from
147  /// a token cache rather than lexing the original source file.
148  std::unique_ptr<PTHManager> PTH;
149 
150  /// A BumpPtrAllocator object used to quickly allocate and release
151  /// objects internal to the Preprocessor.
152  llvm::BumpPtrAllocator BP;
153 
154  /// Identifiers for builtin macros and other builtins.
155  IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
156  IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
157  IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
158  IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
159  IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
160  IdentifierInfo *Ident__COUNTER__; // __COUNTER__
161  IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
162  IdentifierInfo *Ident__identifier; // __identifier
163  IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
164  IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
165  IdentifierInfo *Ident__has_feature; // __has_feature
166  IdentifierInfo *Ident__has_extension; // __has_extension
167  IdentifierInfo *Ident__has_builtin; // __has_builtin
168  IdentifierInfo *Ident__has_attribute; // __has_attribute
169  IdentifierInfo *Ident__has_include; // __has_include
170  IdentifierInfo *Ident__has_include_next; // __has_include_next
171  IdentifierInfo *Ident__has_warning; // __has_warning
172  IdentifierInfo *Ident__is_identifier; // __is_identifier
173  IdentifierInfo *Ident__building_module; // __building_module
174  IdentifierInfo *Ident__MODULE__; // __MODULE__
175  IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
176  IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
177  IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
178 
179  SourceLocation DATELoc, TIMELoc;
180 
181  // Next __COUNTER__ value, starts at 0.
182  unsigned CounterValue = 0;
183 
184  enum {
185  /// \brief Maximum depth of \#includes.
186  MaxAllowedIncludeStackDepth = 200
187  };
188 
189  // State that is set before the preprocessor begins.
190  bool KeepComments : 1;
191  bool KeepMacroComments : 1;
192  bool SuppressIncludeNotFoundError : 1;
193 
194  // State that changes while the preprocessor runs:
195  bool InMacroArgs : 1; // True if parsing fn macro invocation args.
196 
197  /// Whether the preprocessor owns the header search object.
198  bool OwnsHeaderSearch : 1;
199 
200  /// True if macro expansion is disabled.
201  bool DisableMacroExpansion : 1;
202 
203  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
204  /// when parsing preprocessor directives.
205  bool MacroExpansionInDirectivesOverride : 1;
206 
208 
209  /// \brief Whether we have already loaded macros from the external source.
210  mutable bool ReadMacrosFromExternalSource : 1;
211 
212  /// \brief True if pragmas are enabled.
213  bool PragmasEnabled : 1;
214 
215  /// \brief True if the current build action is a preprocessing action.
216  bool PreprocessedOutput : 1;
217 
218  /// \brief True if we are currently preprocessing a #if or #elif directive
219  bool ParsingIfOrElifDirective;
220 
221  /// \brief True if we are pre-expanding macro arguments.
222  bool InMacroArgPreExpansion;
223 
224  /// \brief Mapping/lookup information for all identifiers in
225  /// the program, including program keywords.
226  mutable IdentifierTable Identifiers;
227 
228  /// \brief This table contains all the selectors in the program.
229  ///
230  /// Unlike IdentifierTable above, this table *isn't* populated by the
231  /// preprocessor. It is declared/expanded here because its role/lifetime is
232  /// conceptually similar to the IdentifierTable. In addition, the current
233  /// control flow (in clang::ParseAST()), make it convenient to put here.
234  ///
235  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
236  /// the lifetime of the preprocessor.
237  SelectorTable Selectors;
238 
239  /// \brief Information about builtins.
241 
242  /// \brief Tracks all of the pragmas that the client registered
243  /// with this preprocessor.
244  std::unique_ptr<PragmaNamespace> PragmaHandlers;
245 
246  /// \brief Pragma handlers of the original source is stored here during the
247  /// parsing of a model file.
248  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
249 
250  /// \brief Tracks all of the comment handlers that the client registered
251  /// with this preprocessor.
252  std::vector<CommentHandler *> CommentHandlers;
253 
254  /// \brief True if we want to ignore EOF token and continue later on (thus
255  /// avoid tearing the Lexer and etc. down).
256  bool IncrementalProcessing = false;
257 
258  /// The kind of translation unit we are processing.
259  TranslationUnitKind TUKind;
260 
261  /// \brief The code-completion handler.
262  CodeCompletionHandler *CodeComplete = nullptr;
263 
264  /// \brief The file that we're performing code-completion for, if any.
265  const FileEntry *CodeCompletionFile = nullptr;
266 
267  /// \brief The offset in file for the code-completion point.
268  unsigned CodeCompletionOffset = 0;
269 
270  /// \brief The location for the code-completion point. This gets instantiated
271  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
272  SourceLocation CodeCompletionLoc;
273 
274  /// \brief The start location for the file of the code-completion point.
275  ///
276  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
277  /// for preprocessing.
278  SourceLocation CodeCompletionFileLoc;
279 
280  /// \brief The source location of the \c import contextual keyword we just
281  /// lexed, if any.
282  SourceLocation ModuleImportLoc;
283 
284  /// \brief The module import path that we're currently processing.
286 
287  /// \brief Whether the last token we lexed was an '@'.
288  bool LastTokenWasAt = false;
289 
290  /// \brief Whether the module import expects an identifier next. Otherwise,
291  /// it expects a '.' or ';'.
292  bool ModuleImportExpectsIdentifier = false;
293 
294  /// \brief The source location of the currently-active
295  /// \#pragma clang arc_cf_code_audited begin.
296  SourceLocation PragmaARCCFCodeAuditedLoc;
297 
298  /// \brief The source location of the currently-active
299  /// \#pragma clang assume_nonnull begin.
300  SourceLocation PragmaAssumeNonNullLoc;
301 
302  /// \brief True if we hit the code-completion point.
303  bool CodeCompletionReached = false;
304 
305  /// \brief The code completion token containing the information
306  /// on the stem that is to be code completed.
307  IdentifierInfo *CodeCompletionII = nullptr;
308 
309  /// \brief The directory that the main file should be considered to occupy,
310  /// if it does not correspond to a real file (as happens when building a
311  /// module).
312  const DirectoryEntry *MainFileDir = nullptr;
313 
314  /// \brief The number of bytes that we will initially skip when entering the
315  /// main file, along with a flag that indicates whether skipping this number
316  /// of bytes will place the lexer at the start of a line.
317  ///
318  /// This is used when loading a precompiled preamble.
319  std::pair<int, bool> SkipMainFilePreamble;
320 
321 public:
326  bool FoundElse;
328 
330  bool FoundNonSkipPortion, bool FoundElse,
331  SourceLocation ElseLoc)
332  : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
333  FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
334  ElseLoc(ElseLoc) {}
335  };
336 
337 private:
338  friend class ASTReader;
339  friend class MacroArgs;
340 
341  class PreambleConditionalStackStore {
342  enum State {
343  Off = 0,
344  Recording = 1,
345  Replaying = 2,
346  };
347 
348  public:
349  PreambleConditionalStackStore() = default;
350 
351  void startRecording() { ConditionalStackState = Recording; }
352  void startReplaying() { ConditionalStackState = Replaying; }
353  bool isRecording() const { return ConditionalStackState == Recording; }
354  bool isReplaying() const { return ConditionalStackState == Replaying; }
355 
356  ArrayRef<PPConditionalInfo> getStack() const {
357  return ConditionalStack;
358  }
359 
360  void doneReplaying() {
361  ConditionalStack.clear();
362  ConditionalStackState = Off;
363  }
364 
365  void setStack(ArrayRef<PPConditionalInfo> s) {
366  if (!isRecording() && !isReplaying())
367  return;
368  ConditionalStack.clear();
369  ConditionalStack.append(s.begin(), s.end());
370  }
371 
372  bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
373 
374  bool reachedEOFWhileSkipping() const { return SkipInfo.hasValue(); }
375 
376  void clearSkipInfo() { SkipInfo.reset(); }
377 
379 
380  private:
381  SmallVector<PPConditionalInfo, 4> ConditionalStack;
382  State ConditionalStackState = Off;
383  } PreambleConditionalStack;
384 
385  /// \brief The current top of the stack that we're lexing from if
386  /// not expanding a macro and we are lexing directly from source code.
387  ///
388  /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
389  std::unique_ptr<Lexer> CurLexer;
390 
391  /// \brief The current top of stack that we're lexing from if
392  /// not expanding from a macro and we are lexing from a PTH cache.
393  ///
394  /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
395  std::unique_ptr<PTHLexer> CurPTHLexer;
396 
397  /// \brief The current top of the stack what we're lexing from
398  /// if not expanding a macro.
399  ///
400  /// This is an alias for either CurLexer or CurPTHLexer.
401  PreprocessorLexer *CurPPLexer = nullptr;
402 
403  /// \brief Used to find the current FileEntry, if CurLexer is non-null
404  /// and if applicable.
405  ///
406  /// This allows us to implement \#include_next and find directory-specific
407  /// properties.
408  const DirectoryLookup *CurDirLookup = nullptr;
409 
410  /// \brief The current macro we are expanding, if we are expanding a macro.
411  ///
412  /// One of CurLexer and CurTokenLexer must be null.
413  std::unique_ptr<TokenLexer> CurTokenLexer;
414 
415  /// \brief The kind of lexer we're currently working with.
416  enum CurLexerKind {
417  CLK_Lexer,
418  CLK_PTHLexer,
419  CLK_TokenLexer,
420  CLK_CachingLexer,
421  CLK_LexAfterModuleImport
422  } CurLexerKind = CLK_Lexer;
423 
424  /// \brief If the current lexer is for a submodule that is being built, this
425  /// is that submodule.
426  Module *CurLexerSubmodule = nullptr;
427 
428  /// \brief Keeps track of the stack of files currently
429  /// \#included, and macros currently being expanded from, not counting
430  /// CurLexer/CurTokenLexer.
431  struct IncludeStackInfo {
432  enum CurLexerKind CurLexerKind;
433  Module *TheSubmodule;
434  std::unique_ptr<Lexer> TheLexer;
435  std::unique_ptr<PTHLexer> ThePTHLexer;
436  PreprocessorLexer *ThePPLexer;
437  std::unique_ptr<TokenLexer> TheTokenLexer;
438  const DirectoryLookup *TheDirLookup;
439 
440  // The following constructors are completely useless copies of the default
441  // versions, only needed to pacify MSVC.
442  IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
443  std::unique_ptr<Lexer> &&TheLexer,
444  std::unique_ptr<PTHLexer> &&ThePTHLexer,
445  PreprocessorLexer *ThePPLexer,
446  std::unique_ptr<TokenLexer> &&TheTokenLexer,
447  const DirectoryLookup *TheDirLookup)
448  : CurLexerKind(std::move(CurLexerKind)),
449  TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
450  ThePTHLexer(std::move(ThePTHLexer)),
451  ThePPLexer(std::move(ThePPLexer)),
452  TheTokenLexer(std::move(TheTokenLexer)),
453  TheDirLookup(std::move(TheDirLookup)) {}
454  };
455  std::vector<IncludeStackInfo> IncludeMacroStack;
456 
457  /// \brief Actions invoked when some preprocessor activity is
458  /// encountered (e.g. a file is \#included, etc).
459  std::unique_ptr<PPCallbacks> Callbacks;
460 
461  struct MacroExpandsInfo {
462  Token Tok;
463  MacroDefinition MD;
464  SourceRange Range;
465 
466  MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
467  : Tok(Tok), MD(MD), Range(Range) {}
468  };
469  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
470 
471  /// Information about a name that has been used to define a module macro.
472  struct ModuleMacroInfo {
473  /// The most recent macro directive for this identifier.
474  MacroDirective *MD;
475 
476  /// The active module macros for this identifier.
477  llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
478 
479  /// The generation number at which we last updated ActiveModuleMacros.
480  /// \see Preprocessor::VisibleModules.
481  unsigned ActiveModuleMacrosGeneration = 0;
482 
483  /// Whether this macro name is ambiguous.
484  bool IsAmbiguous = false;
485 
486  /// The module macros that are overridden by this macro.
487  llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
488 
489  ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
490  };
491 
492  /// The state of a macro for an identifier.
493  class MacroState {
494  mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
495 
496  ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
497  const IdentifierInfo *II) const {
498  if (II->isOutOfDate())
499  PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
500  // FIXME: Find a spare bit on IdentifierInfo and store a
501  // HasModuleMacros flag.
502  if (!II->hasMacroDefinition() ||
503  (!PP.getLangOpts().Modules &&
504  !PP.getLangOpts().ModulesLocalVisibility) ||
505  !PP.CurSubmoduleState->VisibleModules.getGeneration())
506  return nullptr;
507 
508  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
509  if (!Info) {
510  Info = new (PP.getPreprocessorAllocator())
511  ModuleMacroInfo(State.get<MacroDirective *>());
512  State = Info;
513  }
514 
515  if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
516  Info->ActiveModuleMacrosGeneration)
517  PP.updateModuleMacroInfo(II, *Info);
518  return Info;
519  }
520 
521  public:
522  MacroState() : MacroState(nullptr) {}
523  MacroState(MacroDirective *MD) : State(MD) {}
524 
525  MacroState(MacroState &&O) noexcept : State(O.State) {
526  O.State = (MacroDirective *)nullptr;
527  }
528 
529  MacroState &operator=(MacroState &&O) noexcept {
530  auto S = O.State;
531  O.State = (MacroDirective *)nullptr;
532  State = S;
533  return *this;
534  }
535 
536  ~MacroState() {
537  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
538  Info->~ModuleMacroInfo();
539  }
540 
541  MacroDirective *getLatest() const {
542  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
543  return Info->MD;
544  return State.get<MacroDirective*>();
545  }
546 
547  void setLatest(MacroDirective *MD) {
548  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
549  Info->MD = MD;
550  else
551  State = MD;
552  }
553 
554  bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
555  auto *Info = getModuleInfo(PP, II);
556  return Info ? Info->IsAmbiguous : false;
557  }
558 
560  getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
561  if (auto *Info = getModuleInfo(PP, II))
562  return Info->ActiveModuleMacros;
563  return None;
564  }
565 
566  MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
567  SourceManager &SourceMgr) const {
568  // FIXME: Incorporate module macros into the result of this.
569  if (auto *Latest = getLatest())
570  return Latest->findDirectiveAtLoc(Loc, SourceMgr);
571  return {};
572  }
573 
574  void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
575  if (auto *Info = getModuleInfo(PP, II)) {
576  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
577  Info->ActiveModuleMacros.begin(),
578  Info->ActiveModuleMacros.end());
579  Info->ActiveModuleMacros.clear();
580  Info->IsAmbiguous = false;
581  }
582  }
583 
584  ArrayRef<ModuleMacro*> getOverriddenMacros() const {
585  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
586  return Info->OverriddenMacros;
587  return None;
588  }
589 
590  void setOverriddenMacros(Preprocessor &PP,
591  ArrayRef<ModuleMacro *> Overrides) {
592  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
593  if (!Info) {
594  if (Overrides.empty())
595  return;
596  Info = new (PP.getPreprocessorAllocator())
597  ModuleMacroInfo(State.get<MacroDirective *>());
598  State = Info;
599  }
600  Info->OverriddenMacros.clear();
601  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
602  Overrides.begin(), Overrides.end());
603  Info->ActiveModuleMacrosGeneration = 0;
604  }
605  };
606 
607  /// For each IdentifierInfo that was associated with a macro, we
608  /// keep a mapping to the history of all macro definitions and #undefs in
609  /// the reverse order (the latest one is in the head of the list).
610  ///
611  /// This mapping lives within the \p CurSubmoduleState.
612  using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
613 
614  struct SubmoduleState;
615 
616  /// \brief Information about a submodule that we're currently building.
617  struct BuildingSubmoduleInfo {
618  /// The module that we are building.
619  Module *M;
620 
621  /// The location at which the module was included.
622  SourceLocation ImportLoc;
623 
624  /// Whether we entered this submodule via a pragma.
625  bool IsPragma;
626 
627  /// The previous SubmoduleState.
628  SubmoduleState *OuterSubmoduleState;
629 
630  /// The number of pending module macro names when we started building this.
631  unsigned OuterPendingModuleMacroNames;
632 
633  BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
634  SubmoduleState *OuterSubmoduleState,
635  unsigned OuterPendingModuleMacroNames)
636  : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
637  OuterSubmoduleState(OuterSubmoduleState),
638  OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
639  };
640  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
641 
642  /// \brief Information about a submodule's preprocessor state.
643  struct SubmoduleState {
644  /// The macros for the submodule.
645  MacroMap Macros;
646 
647  /// The set of modules that are visible within the submodule.
648  VisibleModuleSet VisibleModules;
649 
650  // FIXME: CounterValue?
651  // FIXME: PragmaPushMacroInfo?
652  };
653  std::map<Module *, SubmoduleState> Submodules;
654 
655  /// The preprocessor state for preprocessing outside of any submodule.
656  SubmoduleState NullSubmoduleState;
657 
658  /// The current submodule state. Will be \p NullSubmoduleState if we're not
659  /// in a submodule.
660  SubmoduleState *CurSubmoduleState;
661 
662  /// The set of known macros exported from modules.
663  llvm::FoldingSet<ModuleMacro> ModuleMacros;
664 
665  /// The names of potential module macros that we've not yet processed.
666  llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
667 
668  /// The list of module macros, for each identifier, that are not overridden by
669  /// any other module macro.
670  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
671  LeafModuleMacros;
672 
673  /// \brief Macros that we want to warn because they are not used at the end
674  /// of the translation unit.
675  ///
676  /// We store just their SourceLocations instead of
677  /// something like MacroInfo*. The benefit of this is that when we are
678  /// deserializing from PCH, we don't need to deserialize identifier & macros
679  /// just so that we can report that they are unused, we just warn using
680  /// the SourceLocations of this set (that will be filled by the ASTReader).
681  /// We are using SmallPtrSet instead of a vector for faster removal.
682  using WarnUnusedMacroLocsTy = llvm::SmallPtrSet<SourceLocation, 32>;
683  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
684 
685  /// \brief A "freelist" of MacroArg objects that can be
686  /// reused for quick allocation.
687  MacroArgs *MacroArgCache = nullptr;
688 
689  /// For each IdentifierInfo used in a \#pragma push_macro directive,
690  /// we keep a MacroInfo stack used to restore the previous macro value.
691  llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
692  PragmaPushMacroInfo;
693 
694  // Various statistics we track for performance analysis.
695  unsigned NumDirectives = 0;
696  unsigned NumDefined = 0;
697  unsigned NumUndefined = 0;
698  unsigned NumPragma = 0;
699  unsigned NumIf = 0;
700  unsigned NumElse = 0;
701  unsigned NumEndif = 0;
702  unsigned NumEnteredSourceFiles = 0;
703  unsigned MaxIncludeStackDepth = 0;
704  unsigned NumMacroExpanded = 0;
705  unsigned NumFnMacroExpanded = 0;
706  unsigned NumBuiltinMacroExpanded = 0;
707  unsigned NumFastMacroExpanded = 0;
708  unsigned NumTokenPaste = 0;
709  unsigned NumFastTokenPaste = 0;
710  unsigned NumSkipped = 0;
711 
712  /// \brief The predefined macros that preprocessor should use from the
713  /// command line etc.
714  std::string Predefines;
715 
716  /// \brief The file ID for the preprocessor predefines.
717  FileID PredefinesFileID;
718 
719  /// \{
720  /// \brief Cache of macro expanders to reduce malloc traffic.
721  enum { TokenLexerCacheSize = 8 };
722  unsigned NumCachedTokenLexers;
723  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
724  /// \}
725 
726  /// \brief Keeps macro expanded tokens for TokenLexers.
727  //
728  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
729  /// going to lex in the cache and when it finishes the tokens are removed
730  /// from the end of the cache.
731  SmallVector<Token, 16> MacroExpandedTokens;
732  std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
733 
734  /// \brief A record of the macro definitions and expansions that
735  /// occurred during preprocessing.
736  ///
737  /// This is an optional side structure that can be enabled with
738  /// \c createPreprocessingRecord() prior to preprocessing.
739  PreprocessingRecord *Record = nullptr;
740 
741  /// Cached tokens state.
743 
744  /// \brief Cached tokens are stored here when we do backtracking or
745  /// lookahead. They are "lexed" by the CachingLex() method.
747 
748  /// \brief The position of the cached token that CachingLex() should
749  /// "lex" next.
750  ///
751  /// If it points beyond the CachedTokens vector, it means that a normal
752  /// Lex() should be invoked.
753  CachedTokensTy::size_type CachedLexPos = 0;
754 
755  /// \brief Stack of backtrack positions, allowing nested backtracks.
756  ///
757  /// The EnableBacktrackAtThisPos() method pushes a position to
758  /// indicate where CachedLexPos should be set when the BackTrack() method is
759  /// invoked (at which point the last position is popped).
760  std::vector<CachedTokensTy::size_type> BacktrackPositions;
761 
762  struct MacroInfoChain {
763  MacroInfo MI;
764  MacroInfoChain *Next;
765  };
766 
767  /// MacroInfos are managed as a chain for easy disposal. This is the head
768  /// of that list.
769  MacroInfoChain *MIChainHead = nullptr;
770 
771  void updateOutOfDateIdentifier(IdentifierInfo &II) const;
772 
773 public:
774  Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
776  MemoryBufferCache &PCMCache,
777  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
778  IdentifierInfoLookup *IILookup = nullptr,
779  bool OwnsHeaderSearch = false,
781 
782  ~Preprocessor();
783 
784  /// \brief Initialize the preprocessor using information about the target.
785  ///
786  /// \param Target is owned by the caller and must remain valid for the
787  /// lifetime of the preprocessor.
788  /// \param AuxTarget is owned by the caller and must remain valid for
789  /// the lifetime of the preprocessor.
790  void Initialize(const TargetInfo &Target,
791  const TargetInfo *AuxTarget = nullptr);
792 
793  /// \brief Initialize the preprocessor to parse a model file
794  ///
795  /// To parse model files the preprocessor of the original source is reused to
796  /// preserver the identifier table. However to avoid some duplicate
797  /// information in the preprocessor some cleanup is needed before it is used
798  /// to parse model files. This method does that cleanup.
799  void InitializeForModelFile();
800 
801  /// \brief Cleanup after model file parsing
802  void FinalizeForModelFile();
803 
804  /// \brief Retrieve the preprocessor options used to initialize this
805  /// preprocessor.
806  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
807 
808  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
809  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
810 
811  const LangOptions &getLangOpts() const { return LangOpts; }
812  const TargetInfo &getTargetInfo() const { return *Target; }
813  const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
814  FileManager &getFileManager() const { return FileMgr; }
815  SourceManager &getSourceManager() const { return SourceMgr; }
816  MemoryBufferCache &getPCMCache() const { return PCMCache; }
817  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
818 
819  IdentifierTable &getIdentifierTable() { return Identifiers; }
820  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
821  SelectorTable &getSelectorTable() { return Selectors; }
823  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
824 
825  void setPTHManager(PTHManager* pm);
826 
827  PTHManager *getPTHManager() { return PTH.get(); }
828 
830  ExternalSource = Source;
831  }
832 
834  return ExternalSource;
835  }
836 
837  /// \brief Retrieve the module loader associated with this preprocessor.
838  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
839 
841  return TheModuleLoader.HadFatalFailure;
842  }
843 
844  /// \brief True if we are currently preprocessing a #if or #elif directive
846  return ParsingIfOrElifDirective;
847  }
848 
849  /// \brief Control whether the preprocessor retains comments in output.
850  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
851  this->KeepComments = KeepComments | KeepMacroComments;
852  this->KeepMacroComments = KeepMacroComments;
853  }
854 
855  bool getCommentRetentionState() const { return KeepComments; }
856 
857  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
858  bool getPragmasEnabled() const { return PragmasEnabled; }
859 
860  void SetSuppressIncludeNotFoundError(bool Suppress) {
861  SuppressIncludeNotFoundError = Suppress;
862  }
863 
865  return SuppressIncludeNotFoundError;
866  }
867 
868  /// Sets whether the preprocessor is responsible for producing output or if
869  /// it is producing tokens to be consumed by Parse and Sema.
870  void setPreprocessedOutput(bool IsPreprocessedOutput) {
871  PreprocessedOutput = IsPreprocessedOutput;
872  }
873 
874  /// Returns true if the preprocessor is responsible for generating output,
875  /// false if it is producing tokens to be consumed by Parse and Sema.
876  bool isPreprocessedOutput() const { return PreprocessedOutput; }
877 
878  /// \brief Return true if we are lexing directly from the specified lexer.
879  bool isCurrentLexer(const PreprocessorLexer *L) const {
880  return CurPPLexer == L;
881  }
882 
883  /// \brief Return the current lexer being lexed from.
884  ///
885  /// Note that this ignores any potentially active macro expansions and _Pragma
886  /// expansions going on at the time.
887  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
888 
889  /// \brief Return the current file lexer being lexed from.
890  ///
891  /// Note that this ignores any potentially active macro expansions and _Pragma
892  /// expansions going on at the time.
893  PreprocessorLexer *getCurrentFileLexer() const;
894 
895  /// \brief Return the submodule owning the file being lexed. This may not be
896  /// the current module if we have changed modules since entering the file.
897  Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
898 
899  /// \brief Returns the FileID for the preprocessor predefines.
900  FileID getPredefinesFileID() const { return PredefinesFileID; }
901 
902  /// \{
903  /// \brief Accessors for preprocessor callbacks.
904  ///
905  /// Note that this class takes ownership of any PPCallbacks object given to
906  /// it.
907  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
908  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
909  if (Callbacks)
910  C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
911  std::move(Callbacks));
912  Callbacks = std::move(C);
913  }
914  /// \}
915 
916  bool isMacroDefined(StringRef Id) {
917  return isMacroDefined(&Identifiers.get(Id));
918  }
919  bool isMacroDefined(const IdentifierInfo *II) {
920  return II->hasMacroDefinition() &&
921  (!getLangOpts().Modules || (bool)getMacroDefinition(II));
922  }
923 
924  /// \brief Determine whether II is defined as a macro within the module M,
925  /// if that is a module that we've already preprocessed. Does not check for
926  /// macros imported into M.
928  if (!II->hasMacroDefinition())
929  return false;
930  auto I = Submodules.find(M);
931  if (I == Submodules.end())
932  return false;
933  auto J = I->second.Macros.find(II);
934  if (J == I->second.Macros.end())
935  return false;
936  auto *MD = J->second.getLatest();
937  return MD && MD->isDefined();
938  }
939 
941  if (!II->hasMacroDefinition())
942  return {};
943 
944  MacroState &S = CurSubmoduleState->Macros[II];
945  auto *MD = S.getLatest();
946  while (MD && isa<VisibilityMacroDirective>(MD))
947  MD = MD->getPrevious();
948  return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
949  S.getActiveModuleMacros(*this, II),
950  S.isAmbiguous(*this, II));
951  }
952 
954  SourceLocation Loc) {
955  if (!II->hadMacroDefinition())
956  return {};
957 
958  MacroState &S = CurSubmoduleState->Macros[II];
960  if (auto *MD = S.getLatest())
961  DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
962  // FIXME: Compute the set of active module macros at the specified location.
963  return MacroDefinition(DI.getDirective(),
964  S.getActiveModuleMacros(*this, II),
965  S.isAmbiguous(*this, II));
966  }
967 
968  /// \brief Given an identifier, return its latest non-imported MacroDirective
969  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
971  if (!II->hasMacroDefinition())
972  return nullptr;
973 
974  auto *MD = getLocalMacroDirectiveHistory(II);
975  if (!MD || MD->getDefinition().isUndefined())
976  return nullptr;
977 
978  return MD;
979  }
980 
981  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
982  return const_cast<Preprocessor*>(this)->getMacroInfo(II);
983  }
984 
986  if (!II->hasMacroDefinition())
987  return nullptr;
988  if (auto MD = getMacroDefinition(II))
989  return MD.getMacroInfo();
990  return nullptr;
991  }
992 
993  /// \brief Given an identifier, return the latest non-imported macro
994  /// directive for that identifier.
995  ///
996  /// One can iterate over all previous macro directives from the most recent
997  /// one.
998  MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const;
999 
1000  /// \brief Add a directive to the macro directive history for this identifier.
1001  void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
1003  SourceLocation Loc) {
1004  DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1005  appendMacroDirective(II, MD);
1006  return MD;
1007  }
1009  MacroInfo *MI) {
1010  return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1011  }
1012 
1013  /// \brief Set a MacroDirective that was loaded from a PCH file.
1014  void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED,
1015  MacroDirective *MD);
1016 
1017  /// \brief Register an exported macro for a module and identifier.
1018  ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro,
1019  ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1020  ModuleMacro *getModuleMacro(Module *Mod, IdentifierInfo *II);
1021 
1022  /// \brief Get the list of leaf (non-overridden) module macros for a name.
1024  if (II->isOutOfDate())
1025  updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1026  auto I = LeafModuleMacros.find(II);
1027  if (I != LeafModuleMacros.end())
1028  return I->second;
1029  return None;
1030  }
1031 
1032  /// \{
1033  /// Iterators for the macro history table. Currently defined macros have
1034  /// IdentifierInfo::hasMacroDefinition() set and an empty
1035  /// MacroInfo::getUndefLoc() at the head of the list.
1036  using macro_iterator = MacroMap::const_iterator;
1037 
1038  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1039  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1040  llvm::iterator_range<macro_iterator>
1041 
1042  macros(bool IncludeExternalMacros = true) const {
1043  return llvm::make_range(macro_begin(IncludeExternalMacros),
1044  macro_end(IncludeExternalMacros));
1045  }
1046  /// \}
1047 
1048  /// \brief Return the name of the macro defined before \p Loc that has
1049  /// spelling \p Tokens. If there are multiple macros with same spelling,
1050  /// return the last one defined.
1051  StringRef getLastMacroWithSpelling(SourceLocation Loc,
1052  ArrayRef<TokenValue> Tokens) const;
1053 
1054  const std::string &getPredefines() const { return Predefines; }
1055 
1056  /// \brief Set the predefines for this Preprocessor.
1057  ///
1058  /// These predefines are automatically injected when parsing the main file.
1059  void setPredefines(const char *P) { Predefines = P; }
1060  void setPredefines(StringRef P) { Predefines = P; }
1061 
1062  /// Return information about the specified preprocessor
1063  /// identifier token.
1064  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1065  return &Identifiers.get(Name);
1066  }
1067 
1068  /// \brief Add the specified pragma handler to this preprocessor.
1069  ///
1070  /// If \p Namespace is non-null, then it is a token required to exist on the
1071  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1072  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1074  AddPragmaHandler(StringRef(), Handler);
1075  }
1076 
1077  /// \brief Remove the specific pragma handler from this preprocessor.
1078  ///
1079  /// If \p Namespace is non-null, then it should be the namespace that
1080  /// \p Handler was added to. It is an error to remove a handler that
1081  /// has not been registered.
1082  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1084  RemovePragmaHandler(StringRef(), Handler);
1085  }
1086 
1087  /// Install empty handlers for all pragmas (making them ignored).
1088  void IgnorePragmas();
1089 
1090  /// \brief Add the specified comment handler to the preprocessor.
1091  void addCommentHandler(CommentHandler *Handler);
1092 
1093  /// \brief Remove the specified comment handler.
1094  ///
1095  /// It is an error to remove a handler that has not been registered.
1096  void removeCommentHandler(CommentHandler *Handler);
1097 
1098  /// \brief Set the code completion handler to the given object.
1100  CodeComplete = &Handler;
1101  }
1102 
1103  /// \brief Retrieve the current code-completion handler.
1105  return CodeComplete;
1106  }
1107 
1108  /// \brief Clear out the code completion handler.
1110  CodeComplete = nullptr;
1111  }
1112 
1113  /// \brief Hook used by the lexer to invoke the "natural language" code
1114  /// completion point.
1115  void CodeCompleteNaturalLanguage();
1116 
1117  /// \brief Set the code completion token for filtering purposes.
1119  CodeCompletionII = Filter;
1120  }
1121 
1122  /// \brief Get the code completion token for filtering purposes.
1124  if (CodeCompletionII)
1125  return CodeCompletionII->getName();
1126  return {};
1127  }
1128 
1129  /// \brief Retrieve the preprocessing record, or NULL if there is no
1130  /// preprocessing record.
1131  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1132 
1133  /// \brief Create a new preprocessing record, which will keep track of
1134  /// all macro expansions, macro definitions, etc.
1135  void createPreprocessingRecord();
1136 
1137  /// \brief Enter the specified FileID as the main source file,
1138  /// which implicitly adds the builtin defines etc.
1139  void EnterMainSourceFile();
1140 
1141  /// \brief Inform the preprocessor callbacks that processing is complete.
1142  void EndSourceFile();
1143 
1144  /// \brief Add a source file to the top of the include stack and
1145  /// start lexing tokens from it instead of the current buffer.
1146  ///
1147  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1148  bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir,
1149  SourceLocation Loc);
1150 
1151  /// \brief Add a Macro to the top of the include stack and start lexing
1152  /// tokens from it instead of the current buffer.
1153  ///
1154  /// \param Args specifies the tokens input to a function-like macro.
1155  /// \param ILEnd specifies the location of the ')' for a function-like macro
1156  /// or the identifier for an object-like macro.
1157  void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro,
1158  MacroArgs *Args);
1159 
1160  /// \brief Add a "macro" context to the top of the include stack,
1161  /// which will cause the lexer to start returning the specified tokens.
1162  ///
1163  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1164  /// will not be subject to further macro expansion. Otherwise, these tokens
1165  /// will be re-macro-expanded when/if expansion is enabled.
1166  ///
1167  /// If \p OwnsTokens is false, this method assumes that the specified stream
1168  /// of tokens has a permanent owner somewhere, so they do not need to be
1169  /// copied. If it is true, it assumes the array of tokens is allocated with
1170  /// \c new[] and the Preprocessor will delete[] it.
1171 private:
1172  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1173  bool DisableMacroExpansion, bool OwnsTokens);
1174 
1175 public:
1176  void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1177  bool DisableMacroExpansion) {
1178  EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true);
1179  }
1180 
1181  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion) {
1182  EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false);
1183  }
1184 
1185  /// \brief Pop the current lexer/macro exp off the top of the lexer stack.
1186  ///
1187  /// This should only be used in situations where the current state of the
1188  /// top-of-stack lexer is known.
1189  void RemoveTopOfLexerStack();
1190 
1191  /// From the point that this method is called, and until
1192  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1193  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1194  /// make the Preprocessor re-lex the same tokens.
1195  ///
1196  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1197  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1198  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1199  ///
1200  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1201  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1202  /// tokens will continue indefinitely.
1203  ///
1204  void EnableBacktrackAtThisPos();
1205 
1206  /// \brief Disable the last EnableBacktrackAtThisPos call.
1207  void CommitBacktrackedTokens();
1208 
1210  CachedTokensTy::size_type Begin, End;
1211  };
1212 
1213 private:
1214  /// \brief A range of cached tokens that should be erased after lexing
1215  /// when backtracking requires the erasure of such cached tokens.
1216  Optional<CachedTokensRange> CachedTokenRangeToErase;
1217 
1218 public:
1219  /// \brief Returns the range of cached tokens that were lexed since
1220  /// EnableBacktrackAtThisPos() was previously called.
1221  CachedTokensRange LastCachedTokenRange();
1222 
1223  /// \brief Erase the range of cached tokens that were lexed since
1224  /// EnableBacktrackAtThisPos() was previously called.
1225  void EraseCachedTokens(CachedTokensRange TokenRange);
1226 
1227  /// \brief Make Preprocessor re-lex the tokens that were lexed since
1228  /// EnableBacktrackAtThisPos() was previously called.
1229  void Backtrack();
1230 
1231  /// \brief True if EnableBacktrackAtThisPos() was called and
1232  /// caching of tokens is on.
1233  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1234 
1235  /// \brief Lex the next token for this preprocessor.
1236  void Lex(Token &Result);
1237 
1238  void LexAfterModuleImport(Token &Result);
1239 
1240  void makeModuleVisible(Module *M, SourceLocation Loc);
1241 
1243  return CurSubmoduleState->VisibleModules.getImportLoc(M);
1244  }
1245 
1246  /// \brief Lex a string literal, which may be the concatenation of multiple
1247  /// string literals and may even come from macro expansion.
1248  /// \returns true on success, false if a error diagnostic has been generated.
1249  bool LexStringLiteral(Token &Result, std::string &String,
1250  const char *DiagnosticTag, bool AllowMacroExpansion) {
1251  if (AllowMacroExpansion)
1252  Lex(Result);
1253  else
1254  LexUnexpandedToken(Result);
1255  return FinishLexStringLiteral(Result, String, DiagnosticTag,
1256  AllowMacroExpansion);
1257  }
1258 
1259  /// \brief Complete the lexing of a string literal where the first token has
1260  /// already been lexed (see LexStringLiteral).
1261  bool FinishLexStringLiteral(Token &Result, std::string &String,
1262  const char *DiagnosticTag,
1263  bool AllowMacroExpansion);
1264 
1265  /// \brief Lex a token. If it's a comment, keep lexing until we get
1266  /// something not a comment.
1267  ///
1268  /// This is useful in -E -C mode where comments would foul up preprocessor
1269  /// directive handling.
1270  void LexNonComment(Token &Result) {
1271  do
1272  Lex(Result);
1273  while (Result.getKind() == tok::comment);
1274  }
1275 
1276  /// \brief Just like Lex, but disables macro expansion of identifier tokens.
1277  void LexUnexpandedToken(Token &Result) {
1278  // Disable macro expansion.
1279  bool OldVal = DisableMacroExpansion;
1280  DisableMacroExpansion = true;
1281  // Lex the token.
1282  Lex(Result);
1283 
1284  // Reenable it.
1285  DisableMacroExpansion = OldVal;
1286  }
1287 
1288  /// \brief Like LexNonComment, but this disables macro expansion of
1289  /// identifier tokens.
1291  do
1292  LexUnexpandedToken(Result);
1293  while (Result.getKind() == tok::comment);
1294  }
1295 
1296  /// \brief Parses a simple integer literal to get its numeric value. Floating
1297  /// point literals and user defined literals are rejected. Used primarily to
1298  /// handle pragmas that accept integer arguments.
1299  bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1300 
1301  /// Disables macro expansion everywhere except for preprocessor directives.
1303  DisableMacroExpansion = true;
1304  MacroExpansionInDirectivesOverride = true;
1305  }
1306 
1307  /// \brief Peeks ahead N tokens and returns that token without consuming any
1308  /// tokens.
1309  ///
1310  /// LookAhead(0) returns the next token that would be returned by Lex(),
1311  /// LookAhead(1) returns the token after it, etc. This returns normal
1312  /// tokens after phase 5. As such, it is equivalent to using
1313  /// 'Lex', not 'LexUnexpandedToken'.
1314  const Token &LookAhead(unsigned N) {
1315  if (CachedLexPos + N < CachedTokens.size())
1316  return CachedTokens[CachedLexPos+N];
1317  else
1318  return PeekAhead(N+1);
1319  }
1320 
1321  /// \brief When backtracking is enabled and tokens are cached,
1322  /// this allows to revert a specific number of tokens.
1323  ///
1324  /// Note that the number of tokens being reverted should be up to the last
1325  /// backtrack position, not more.
1326  void RevertCachedTokens(unsigned N) {
1327  assert(isBacktrackEnabled() &&
1328  "Should only be called when tokens are cached for backtracking");
1329  assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1330  && "Should revert tokens up to the last backtrack position, not more");
1331  assert(signed(CachedLexPos) - signed(N) >= 0 &&
1332  "Corrupted backtrack positions ?");
1333  CachedLexPos -= N;
1334  }
1335 
1336  /// \brief Enters a token in the token stream to be lexed next.
1337  ///
1338  /// If BackTrack() is called afterwards, the token will remain at the
1339  /// insertion point.
1340  void EnterToken(const Token &Tok) {
1341  EnterCachingLexMode();
1342  CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1343  }
1344 
1345  /// We notify the Preprocessor that if it is caching tokens (because
1346  /// backtrack is enabled) it should replace the most recent cached tokens
1347  /// with the given annotation token. This function has no effect if
1348  /// backtracking is not enabled.
1349  ///
1350  /// Note that the use of this function is just for optimization, so that the
1351  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1352  /// invoked.
1353  void AnnotateCachedTokens(const Token &Tok) {
1354  assert(Tok.isAnnotation() && "Expected annotation token");
1355  if (CachedLexPos != 0 && isBacktrackEnabled())
1356  AnnotatePreviousCachedTokens(Tok);
1357  }
1358 
1359  /// Get the location of the last cached token, suitable for setting the end
1360  /// location of an annotation token.
1362  assert(CachedLexPos != 0);
1363  return CachedTokens[CachedLexPos-1].getLastLoc();
1364  }
1365 
1366  /// \brief Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1367  /// CachedTokens.
1368  bool IsPreviousCachedToken(const Token &Tok) const;
1369 
1370  /// \brief Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1371  /// in \p NewToks.
1372  ///
1373  /// Useful when a token needs to be split in smaller ones and CachedTokens
1374  /// most recent token must to be updated to reflect that.
1375  void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1376 
1377  /// \brief Replace the last token with an annotation token.
1378  ///
1379  /// Like AnnotateCachedTokens(), this routine replaces an
1380  /// already-parsed (and resolved) token with an annotation
1381  /// token. However, this routine only replaces the last token with
1382  /// the annotation token; it does not affect any other cached
1383  /// tokens. This function has no effect if backtracking is not
1384  /// enabled.
1386  assert(Tok.isAnnotation() && "Expected annotation token");
1387  if (CachedLexPos != 0 && isBacktrackEnabled())
1388  CachedTokens[CachedLexPos-1] = Tok;
1389  }
1390 
1391  /// Enter an annotation token into the token stream.
1392  void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1393  void *AnnotationVal);
1394 
1395  /// Update the current token to represent the provided
1396  /// identifier, in order to cache an action performed by typo correction.
1397  void TypoCorrectToken(const Token &Tok) {
1398  assert(Tok.getIdentifierInfo() && "Expected identifier token");
1399  if (CachedLexPos != 0 && isBacktrackEnabled())
1400  CachedTokens[CachedLexPos-1] = Tok;
1401  }
1402 
1403  /// \brief Recompute the current lexer kind based on the CurLexer/CurPTHLexer/
1404  /// CurTokenLexer pointers.
1405  void recomputeCurLexerKind();
1406 
1407  /// \brief Returns true if incremental processing is enabled
1408  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1409 
1410  /// \brief Enables the incremental processing
1411  void enableIncrementalProcessing(bool value = true) {
1412  IncrementalProcessing = value;
1413  }
1414 
1415  /// \brief Specify the point at which code-completion will be performed.
1416  ///
1417  /// \param File the file in which code completion should occur. If
1418  /// this file is included multiple times, code-completion will
1419  /// perform completion the first time it is included. If NULL, this
1420  /// function clears out the code-completion point.
1421  ///
1422  /// \param Line the line at which code completion should occur
1423  /// (1-based).
1424  ///
1425  /// \param Column the column at which code completion should occur
1426  /// (1-based).
1427  ///
1428  /// \returns true if an error occurred, false otherwise.
1429  bool SetCodeCompletionPoint(const FileEntry *File,
1430  unsigned Line, unsigned Column);
1431 
1432  /// \brief Determine if we are performing code completion.
1433  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1434 
1435  /// \brief Returns the location of the code-completion point.
1436  ///
1437  /// Returns an invalid location if code-completion is not enabled or the file
1438  /// containing the code-completion point has not been lexed yet.
1439  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1440 
1441  /// \brief Returns the start location of the file of code-completion point.
1442  ///
1443  /// Returns an invalid location if code-completion is not enabled or the file
1444  /// containing the code-completion point has not been lexed yet.
1446  return CodeCompletionFileLoc;
1447  }
1448 
1449  /// \brief Returns true if code-completion is enabled and we have hit the
1450  /// code-completion point.
1451  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1452 
1453  /// \brief Note that we hit the code-completion point.
1455  assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1456  CodeCompletionReached = true;
1457  // Silence any diagnostics that occur after we hit the code-completion.
1458  getDiagnostics().setSuppressAllDiagnostics(true);
1459  }
1460 
1461  /// \brief The location of the currently-active \#pragma clang
1462  /// arc_cf_code_audited begin.
1463  ///
1464  /// Returns an invalid location if there is no such pragma active.
1466  return PragmaARCCFCodeAuditedLoc;
1467  }
1468 
1469  /// \brief Set the location of the currently-active \#pragma clang
1470  /// arc_cf_code_audited begin. An invalid location ends the pragma.
1472  PragmaARCCFCodeAuditedLoc = Loc;
1473  }
1474 
1475  /// \brief The location of the currently-active \#pragma clang
1476  /// assume_nonnull begin.
1477  ///
1478  /// Returns an invalid location if there is no such pragma active.
1480  return PragmaAssumeNonNullLoc;
1481  }
1482 
1483  /// \brief Set the location of the currently-active \#pragma clang
1484  /// assume_nonnull begin. An invalid location ends the pragma.
1486  PragmaAssumeNonNullLoc = Loc;
1487  }
1488 
1489  /// \brief Set the directory in which the main file should be considered
1490  /// to have been found, if it is not a real file.
1491  void setMainFileDir(const DirectoryEntry *Dir) {
1492  MainFileDir = Dir;
1493  }
1494 
1495  /// \brief Instruct the preprocessor to skip part of the main source file.
1496  ///
1497  /// \param Bytes The number of bytes in the preamble to skip.
1498  ///
1499  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1500  /// start of a line.
1501  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1502  SkipMainFilePreamble.first = Bytes;
1503  SkipMainFilePreamble.second = StartOfLine;
1504  }
1505 
1506  /// Forwarding function for diagnostics. This emits a diagnostic at
1507  /// the specified Token's location, translating the token's start
1508  /// position in the current buffer into a SourcePosition object for rendering.
1509  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1510  return Diags->Report(Loc, DiagID);
1511  }
1512 
1513  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1514  return Diags->Report(Tok.getLocation(), DiagID);
1515  }
1516 
1517  /// Return the 'spelling' of the token at the given
1518  /// location; does not go up to the spelling location or down to the
1519  /// expansion location.
1520  ///
1521  /// \param buffer A buffer which will be used only if the token requires
1522  /// "cleaning", e.g. if it contains trigraphs or escaped newlines
1523  /// \param invalid If non-null, will be set \c true if an error occurs.
1525  SmallVectorImpl<char> &buffer,
1526  bool *invalid = nullptr) const {
1527  return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1528  }
1529 
1530  /// \brief Return the 'spelling' of the Tok token.
1531  ///
1532  /// The spelling of a token is the characters used to represent the token in
1533  /// the source file after trigraph expansion and escaped-newline folding. In
1534  /// particular, this wants to get the true, uncanonicalized, spelling of
1535  /// things like digraphs, UCNs, etc.
1536  ///
1537  /// \param Invalid If non-null, will be set \c true if an error occurs.
1538  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1539  return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1540  }
1541 
1542  /// \brief Get the spelling of a token into a preallocated buffer, instead
1543  /// of as an std::string.
1544  ///
1545  /// The caller is required to allocate enough space for the token, which is
1546  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1547  /// actual result is returned.
1548  ///
1549  /// Note that this method may do two possible things: it may either fill in
1550  /// the buffer specified with characters, or it may *change the input pointer*
1551  /// to point to a constant buffer with the data already in it (avoiding a
1552  /// copy). The caller is not allowed to modify the returned buffer pointer
1553  /// if an internal buffer is returned.
1554  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1555  bool *Invalid = nullptr) const {
1556  return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1557  }
1558 
1559  /// \brief Get the spelling of a token into a SmallVector.
1560  ///
1561  /// Note that the returned StringRef may not point to the
1562  /// supplied buffer if a copy can be avoided.
1563  StringRef getSpelling(const Token &Tok,
1564  SmallVectorImpl<char> &Buffer,
1565  bool *Invalid = nullptr) const;
1566 
1567  /// \brief Relex the token at the specified location.
1568  /// \returns true if there was a failure, false on success.
1569  bool getRawToken(SourceLocation Loc, Token &Result,
1570  bool IgnoreWhiteSpace = false) {
1571  return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1572  }
1573 
1574  /// \brief Given a Token \p Tok that is a numeric constant with length 1,
1575  /// return the character.
1576  char
1578  bool *Invalid = nullptr) const {
1579  assert(Tok.is(tok::numeric_constant) &&
1580  Tok.getLength() == 1 && "Called on unsupported token");
1581  assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1582 
1583  // If the token is carrying a literal data pointer, just use it.
1584  if (const char *D = Tok.getLiteralData())
1585  return *D;
1586 
1587  // Otherwise, fall back on getCharacterData, which is slower, but always
1588  // works.
1589  return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1590  }
1591 
1592  /// \brief Retrieve the name of the immediate macro expansion.
1593  ///
1594  /// This routine starts from a source location, and finds the name of the
1595  /// macro responsible for its immediate expansion. It looks through any
1596  /// intervening macro argument expansions to compute this. It returns a
1597  /// StringRef that refers to the SourceManager-owned buffer of the source
1598  /// where that macro name is spelled. Thus, the result shouldn't out-live
1599  /// the SourceManager.
1601  return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1602  }
1603 
1604  /// \brief Plop the specified string into a scratch buffer and set the
1605  /// specified token's location and length to it.
1606  ///
1607  /// If specified, the source location provides a location of the expansion
1608  /// point of the token.
1609  void CreateString(StringRef Str, Token &Tok,
1610  SourceLocation ExpansionLocStart = SourceLocation(),
1611  SourceLocation ExpansionLocEnd = SourceLocation());
1612 
1613  /// \brief Computes the source location just past the end of the
1614  /// token at this source location.
1615  ///
1616  /// This routine can be used to produce a source location that
1617  /// points just past the end of the token referenced by \p Loc, and
1618  /// is generally used when a diagnostic needs to point just after a
1619  /// token where it expected something different that it received. If
1620  /// the returned source location would not be meaningful (e.g., if
1621  /// it points into a macro), this routine returns an invalid
1622  /// source location.
1623  ///
1624  /// \param Offset an offset from the end of the token, where the source
1625  /// location should refer to. The default offset (0) produces a source
1626  /// location pointing just past the end of the token; an offset of 1 produces
1627  /// a source location pointing to the last character in the token, etc.
1629  return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1630  }
1631 
1632  /// \brief Returns true if the given MacroID location points at the first
1633  /// token of the macro expansion.
1634  ///
1635  /// \param MacroBegin If non-null and function returns true, it is set to
1636  /// begin location of the macro.
1638  SourceLocation *MacroBegin = nullptr) const {
1639  return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1640  MacroBegin);
1641  }
1642 
1643  /// \brief Returns true if the given MacroID location points at the last
1644  /// token of the macro expansion.
1645  ///
1646  /// \param MacroEnd If non-null and function returns true, it is set to
1647  /// end location of the macro.
1649  SourceLocation *MacroEnd = nullptr) const {
1650  return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1651  }
1652 
1653  /// \brief Print the token to stderr, used for debugging.
1654  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1655  void DumpLocation(SourceLocation Loc) const;
1656  void DumpMacro(const MacroInfo &MI) const;
1657  void dumpMacroInfo(const IdentifierInfo *II);
1658 
1659  /// \brief Given a location that specifies the start of a
1660  /// token, return a new location that specifies a character within the token.
1662  unsigned Char) const {
1663  return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1664  }
1665 
1666  /// \brief Increment the counters for the number of token paste operations
1667  /// performed.
1668  ///
1669  /// If fast was specified, this is a 'fast paste' case we handled.
1670  void IncrementPasteCounter(bool isFast) {
1671  if (isFast)
1672  ++NumFastTokenPaste;
1673  else
1674  ++NumTokenPaste;
1675  }
1676 
1677  void PrintStats();
1678 
1679  size_t getTotalMemory() const;
1680 
1681  /// When the macro expander pastes together a comment (/##/) in Microsoft
1682  /// mode, this method handles updating the current state, returning the
1683  /// token on the next source line.
1684  void HandleMicrosoftCommentPaste(Token &Tok);
1685 
1686  //===--------------------------------------------------------------------===//
1687  // Preprocessor callback methods. These are invoked by a lexer as various
1688  // directives and events are found.
1689 
1690  /// Given a tok::raw_identifier token, look up the
1691  /// identifier information for the token and install it into the token,
1692  /// updating the token kind accordingly.
1693  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1694 
1695 private:
1696  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1697 
1698 public:
1699  /// \brief Specifies the reason for poisoning an identifier.
1700  ///
1701  /// If that identifier is accessed while poisoned, then this reason will be
1702  /// used instead of the default "poisoned" diagnostic.
1703  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1704 
1705  /// \brief Display reason for poisoned identifier.
1706  void HandlePoisonedIdentifier(Token & Tok);
1707 
1709  if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1710  if(II->isPoisoned()) {
1711  HandlePoisonedIdentifier(Identifier);
1712  }
1713  }
1714  }
1715 
1716 private:
1717  /// Identifiers used for SEH handling in Borland. These are only
1718  /// allowed in particular circumstances
1719  // __except block
1720  IdentifierInfo *Ident__exception_code,
1721  *Ident___exception_code,
1722  *Ident_GetExceptionCode;
1723  // __except filter expression
1724  IdentifierInfo *Ident__exception_info,
1725  *Ident___exception_info,
1726  *Ident_GetExceptionInfo;
1727  // __finally
1728  IdentifierInfo *Ident__abnormal_termination,
1729  *Ident___abnormal_termination,
1730  *Ident_AbnormalTermination;
1731 
1732  const char *getCurLexerEndPos();
1733  void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
1734 
1735 public:
1736  void PoisonSEHIdentifiers(bool Poison = true); // Borland
1737 
1738  /// \brief Callback invoked when the lexer reads an identifier and has
1739  /// filled in the tokens IdentifierInfo member.
1740  ///
1741  /// This callback potentially macro expands it or turns it into a named
1742  /// token (like 'for').
1743  ///
1744  /// \returns true if we actually computed a token, false if we need to
1745  /// lex again.
1746  bool HandleIdentifier(Token &Identifier);
1747 
1748  /// \brief Callback invoked when the lexer hits the end of the current file.
1749  ///
1750  /// This either returns the EOF token and returns true, or
1751  /// pops a level off the include stack and returns false, at which point the
1752  /// client should call lex again.
1753  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1754 
1755  /// \brief Callback invoked when the current TokenLexer hits the end of its
1756  /// token stream.
1757  bool HandleEndOfTokenLexer(Token &Result);
1758 
1759  /// \brief Callback invoked when the lexer sees a # token at the start of a
1760  /// line.
1761  ///
1762  /// This consumes the directive, modifies the lexer/preprocessor state, and
1763  /// advances the lexer(s) so that the next token read is the correct one.
1764  void HandleDirective(Token &Result);
1765 
1766  /// \brief Ensure that the next token is a tok::eod token.
1767  ///
1768  /// If not, emit a diagnostic and consume up until the eod.
1769  /// If \p EnableMacros is true, then we consider macros that expand to zero
1770  /// tokens as being ok.
1771  void CheckEndOfDirective(const char *Directive, bool EnableMacros = false);
1772 
1773  /// \brief Read and discard all tokens remaining on the current line until
1774  /// the tok::eod token is found.
1775  void DiscardUntilEndOfDirective();
1776 
1777  /// \brief Returns true if the preprocessor has seen a use of
1778  /// __DATE__ or __TIME__ in the file so far.
1779  bool SawDateOrTime() const {
1780  return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
1781  }
1782  unsigned getCounterValue() const { return CounterValue; }
1783  void setCounterValue(unsigned V) { CounterValue = V; }
1784 
1785  /// \brief Retrieves the module that we're currently building, if any.
1786  Module *getCurrentModule();
1787 
1788  /// \brief Allocate a new MacroInfo object with the provided SourceLocation.
1789  MacroInfo *AllocateMacroInfo(SourceLocation L);
1790 
1791  /// \brief Turn the specified lexer token into a fully checked and spelled
1792  /// filename, e.g. as an operand of \#include.
1793  ///
1794  /// The caller is expected to provide a buffer that is large enough to hold
1795  /// the spelling of the filename, but is also expected to handle the case
1796  /// when this method decides to use a different buffer.
1797  ///
1798  /// \returns true if the input filename was in <>'s or false if it was
1799  /// in ""'s.
1800  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Filename);
1801 
1802  /// \brief Given a "foo" or <foo> reference, look up the indicated file.
1803  ///
1804  /// Returns null on failure. \p isAngled indicates whether the file
1805  /// reference is for system \#include's or not (i.e. using <> instead of "").
1806  const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename,
1807  bool isAngled, const DirectoryLookup *FromDir,
1808  const FileEntry *FromFile,
1809  const DirectoryLookup *&CurDir,
1810  SmallVectorImpl<char> *SearchPath,
1811  SmallVectorImpl<char> *RelativePath,
1812  ModuleMap::KnownHeader *SuggestedModule,
1813  bool *IsMapped, bool SkipCache = false);
1814 
1815  /// \brief Get the DirectoryLookup structure used to find the current
1816  /// FileEntry, if CurLexer is non-null and if applicable.
1817  ///
1818  /// This allows us to implement \#include_next and find directory-specific
1819  /// properties.
1820  const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1821 
1822  /// \brief Return true if we're in the top-level file, not in a \#include.
1823  bool isInPrimaryFile() const;
1824 
1825  /// \brief Handle cases where the \#include name is expanded
1826  /// from a macro as multiple tokens, which need to be glued together.
1827  ///
1828  /// This occurs for code like:
1829  /// \code
1830  /// \#define FOO <x/y.h>
1831  /// \#include FOO
1832  /// \endcode
1833  /// because in this case, "<x/y.h>" is returned as 7 tokens, not one.
1834  ///
1835  /// This code concatenates and consumes tokens up to the '>' token. It
1836  /// returns false if the > was found, otherwise it returns true if it finds
1837  /// and consumes the EOD marker.
1838  bool ConcatenateIncludeName(SmallString<128> &FilenameBuffer,
1839  SourceLocation &End);
1840 
1841  /// \brief Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1842  /// followed by EOD. Return true if the token is not a valid on-off-switch.
1843  bool LexOnOffSwitch(tok::OnOffSwitch &OOS);
1844 
1845  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
1846  bool *ShadowFlag = nullptr);
1847 
1848  void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
1849  Module *LeaveSubmodule(bool ForPragma);
1850 
1851 private:
1852  friend void TokenLexer::ExpandFunctionArguments();
1853 
1854  void PushIncludeMacroStack() {
1855  assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
1856  IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
1857  std::move(CurLexer), std::move(CurPTHLexer),
1858  CurPPLexer, std::move(CurTokenLexer),
1859  CurDirLookup);
1860  CurPPLexer = nullptr;
1861  }
1862 
1863  void PopIncludeMacroStack() {
1864  CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1865  CurPTHLexer = std::move(IncludeMacroStack.back().ThePTHLexer);
1866  CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1867  CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1868  CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1869  CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
1870  CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1871  IncludeMacroStack.pop_back();
1872  }
1873 
1874  void PropagateLineStartLeadingSpaceInfo(Token &Result);
1875 
1876  /// Determine whether we need to create module macros for #defines in the
1877  /// current context.
1878  bool needModuleMacros() const;
1879 
1880  /// Update the set of active module macros and ambiguity flag for a module
1881  /// macro name.
1882  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
1883 
1884  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
1885  SourceLocation Loc);
1886  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
1887  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
1888  bool isPublic);
1889 
1890  /// \brief Lex and validate a macro name, which occurs after a
1891  /// \#define or \#undef.
1892  ///
1893  /// \param MacroNameTok Token that represents the name defined or undefined.
1894  /// \param IsDefineUndef Kind if preprocessor directive.
1895  /// \param ShadowFlag Points to flag that is set if macro name shadows
1896  /// a keyword.
1897  ///
1898  /// This emits a diagnostic, sets the token kind to eod,
1899  /// and discards the rest of the macro line if the macro name is invalid.
1900  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
1901  bool *ShadowFlag = nullptr);
1902 
1903  /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
1904  /// entire line) of the macro's tokens and adds them to MacroInfo, and while
1905  /// doing so performs certain validity checks including (but not limited to):
1906  /// - # (stringization) is followed by a macro parameter
1907  /// \param MacroNameTok - Token that represents the macro name
1908  /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
1909  ///
1910  /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
1911  /// returns a nullptr if an invalid sequence of tokens is encountered.
1912  MacroInfo *ReadOptionalMacroParameterListAndBody(
1913  const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
1914 
1915  /// The ( starting an argument list of a macro definition has just been read.
1916  /// Lex the rest of the parameters and the closing ), updating \p MI with
1917  /// what we learn and saving in \p LastTok the last token read.
1918  /// Return true if an error occurs parsing the arg list.
1919  bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
1920 
1921  /// We just read a \#if or related directive and decided that the
1922  /// subsequent tokens are in the \#if'd out portion of the
1923  /// file. Lex the rest of the file, until we see an \#endif. If \p
1924  /// FoundNonSkipPortion is true, then we have already emitted code for part of
1925  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
1926  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
1927  /// already seen one so a \#else directive is a duplicate. When this returns,
1928  /// the caller can lex the first valid token.
1929  void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
1930  SourceLocation IfTokenLoc,
1931  bool FoundNonSkipPortion, bool FoundElse,
1932  SourceLocation ElseLoc = SourceLocation());
1933 
1934  /// \brief A fast PTH version of SkipExcludedConditionalBlock.
1935  void PTHSkipExcludedConditionalBlock();
1936 
1937  /// Information about the result for evaluating an expression for a
1938  /// preprocessor directive.
1939  struct DirectiveEvalResult {
1940  /// Whether the expression was evaluated as true or not.
1941  bool Conditional;
1942 
1943  /// True if the expression contained identifiers that were undefined.
1944  bool IncludedUndefinedIds;
1945  };
1946 
1947  /// \brief Evaluate an integer constant expression that may occur after a
1948  /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
1949  ///
1950  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
1951  DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1952 
1953  /// \brief Install the standard preprocessor pragmas:
1954  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
1955  void RegisterBuiltinPragmas();
1956 
1957  /// \brief Register builtin macros such as __LINE__ with the identifier table.
1958  void RegisterBuiltinMacros();
1959 
1960  /// If an identifier token is read that is to be expanded as a macro, handle
1961  /// it and return the next token as 'Tok'. If we lexed a token, return true;
1962  /// otherwise the caller should lex again.
1963  bool HandleMacroExpandedIdentifier(Token &Tok, const MacroDefinition &MD);
1964 
1965  /// \brief Cache macro expanded tokens for TokenLexers.
1966  //
1967  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
1968  /// going to lex in the cache and when it finishes the tokens are removed
1969  /// from the end of the cache.
1970  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
1971  ArrayRef<Token> tokens);
1972 
1973  void removeCachedMacroExpandedTokensOfLastLexer();
1974 
1975  /// Determine whether the next preprocessor token to be
1976  /// lexed is a '('. If so, consume the token and return true, if not, this
1977  /// method should have no observable side-effect on the lexed tokens.
1978  bool isNextPPTokenLParen();
1979 
1980  /// After reading "MACRO(", this method is invoked to read all of the formal
1981  /// arguments specified for the macro invocation. Returns null on error.
1982  MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
1983  SourceLocation &ExpansionEnd);
1984 
1985  /// \brief If an identifier token is read that is to be expanded
1986  /// as a builtin macro, handle it and return the next token as 'Tok'.
1987  void ExpandBuiltinMacro(Token &Tok);
1988 
1989  /// \brief Read a \c _Pragma directive, slice it up, process it, then
1990  /// return the first token after the directive.
1991  /// This assumes that the \c _Pragma token has just been read into \p Tok.
1992  void Handle_Pragma(Token &Tok);
1993 
1994  /// \brief Like Handle_Pragma except the pragma text is not enclosed within
1995  /// a string literal.
1996  void HandleMicrosoft__pragma(Token &Tok);
1997 
1998  /// \brief Add a lexer to the top of the include stack and
1999  /// start lexing tokens from it instead of the current buffer.
2000  void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
2001 
2002  /// \brief Add a lexer to the top of the include stack and
2003  /// start getting tokens from it using the PTH cache.
2004  void EnterSourceFileWithPTH(PTHLexer *PL, const DirectoryLookup *Dir);
2005 
2006  /// \brief Set the FileID for the preprocessor predefines.
2007  void setPredefinesFileID(FileID FID) {
2008  assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2009  PredefinesFileID = FID;
2010  }
2011 
2012  /// \brief Returns true if we are lexing from a file and not a
2013  /// pragma or a macro.
2014  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2015  return L ? !L->isPragmaLexer() : P != nullptr;
2016  }
2017 
2018  static bool IsFileLexer(const IncludeStackInfo& I) {
2019  return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2020  }
2021 
2022  bool IsFileLexer() const {
2023  return IsFileLexer(CurLexer.get(), CurPPLexer);
2024  }
2025 
2026  //===--------------------------------------------------------------------===//
2027  // Caching stuff.
2028  void CachingLex(Token &Result);
2029 
2030  bool InCachingLexMode() const {
2031  // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2032  // that we are past EOF, not that we are in CachingLex mode.
2033  return !CurPPLexer && !CurTokenLexer && !CurPTHLexer &&
2034  !IncludeMacroStack.empty();
2035  }
2036 
2037  void EnterCachingLexMode();
2038 
2039  void ExitCachingLexMode() {
2040  if (InCachingLexMode())
2041  RemoveTopOfLexerStack();
2042  }
2043 
2044  const Token &PeekAhead(unsigned N);
2045  void AnnotatePreviousCachedTokens(const Token &Tok);
2046 
2047  //===--------------------------------------------------------------------===//
2048  /// Handle*Directive - implement the various preprocessor directives. These
2049  /// should side-effect the current preprocessor object so that the next call
2050  /// to Lex() will return the appropriate token next.
2051  void HandleLineDirective();
2052  void HandleDigitDirective(Token &Tok);
2053  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2054  void HandleIdentSCCSDirective(Token &Tok);
2055  void HandleMacroPublicDirective(Token &Tok);
2056  void HandleMacroPrivateDirective();
2057 
2058  // File inclusion.
2059  void HandleIncludeDirective(SourceLocation HashLoc,
2060  Token &Tok,
2061  const DirectoryLookup *LookupFrom = nullptr,
2062  const FileEntry *LookupFromFile = nullptr,
2063  bool isImport = false);
2064  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2065  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2066  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2067  void HandleMicrosoftImportDirective(Token &Tok);
2068 
2069 public:
2070  /// Check that the given module is available, producing a diagnostic if not.
2071  /// \return \c true if the check failed (because the module is not available).
2072  /// \c false if the module appears to be usable.
2073  static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2074  const TargetInfo &TargetInfo,
2075  DiagnosticsEngine &Diags, Module *M);
2076 
2077  // Module inclusion testing.
2078  /// \brief Find the module that owns the source or header file that
2079  /// \p Loc points to. If the location is in a file that was included
2080  /// into a module, or is outside any module, returns nullptr.
2081  Module *getModuleForLocation(SourceLocation Loc);
2082 
2083  /// \brief We want to produce a diagnostic at location IncLoc concerning a
2084  /// missing module import.
2085  ///
2086  /// \param IncLoc The location at which the missing import was detected.
2087  /// \param M The desired module.
2088  /// \param MLoc A location within the desired module at which some desired
2089  /// effect occurred (eg, where a desired entity was declared).
2090  ///
2091  /// \return A file that can be #included to import a module containing MLoc.
2092  /// Null if no such file could be determined or if a #include is not
2093  /// appropriate.
2094  const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2095  Module *M,
2096  SourceLocation MLoc);
2097 
2098  bool isRecordingPreamble() const {
2099  return PreambleConditionalStack.isRecording();
2100  }
2101 
2102  bool hasRecordedPreamble() const {
2103  return PreambleConditionalStack.hasRecordedPreamble();
2104  }
2105 
2107  return PreambleConditionalStack.getStack();
2108  }
2109 
2111  PreambleConditionalStack.setStack(s);
2112  }
2113 
2116  PreambleConditionalStack.startReplaying();
2117  PreambleConditionalStack.setStack(s);
2118  PreambleConditionalStack.SkipInfo = SkipInfo;
2119  }
2120 
2122  return PreambleConditionalStack.SkipInfo;
2123  }
2124 
2125 private:
2126  /// \brief After processing predefined file, initialize the conditional stack from
2127  /// the preamble.
2128  void replayPreambleConditionalStack();
2129 
2130  // Macro handling.
2131  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterTopLevelIfndef);
2132  void HandleUndefDirective();
2133 
2134  // Conditional Inclusion.
2135  void HandleIfdefDirective(Token &Tok, const Token &HashToken,
2136  bool isIfndef, bool ReadAnyTokensBeforeDirective);
2137  void HandleIfDirective(Token &Tok, const Token &HashToken,
2138  bool ReadAnyTokensBeforeDirective);
2139  void HandleEndifDirective(Token &Tok);
2140  void HandleElseDirective(Token &Tok, const Token &HashToken);
2141  void HandleElifDirective(Token &Tok, const Token &HashToken);
2142 
2143  // Pragmas.
2144  void HandlePragmaDirective(SourceLocation IntroducerLoc,
2145  PragmaIntroducerKind Introducer);
2146 
2147 public:
2148  void HandlePragmaOnce(Token &OnceTok);
2149  void HandlePragmaMark();
2150  void HandlePragmaPoison();
2151  void HandlePragmaSystemHeader(Token &SysHeaderTok);
2152  void HandlePragmaDependency(Token &DependencyTok);
2153  void HandlePragmaPushMacro(Token &Tok);
2154  void HandlePragmaPopMacro(Token &Tok);
2155  void HandlePragmaIncludeAlias(Token &Tok);
2156  void HandlePragmaModuleBuild(Token &Tok);
2157  IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2158 
2159  // Return true and store the first token only if any CommentHandler
2160  // has inserted some tokens and getCommentRetentionState() is false.
2161  bool HandleComment(Token &Token, SourceRange Comment);
2162 
2163  /// \brief A macro is used, update information about macros that need unused
2164  /// warnings.
2165  void markMacroAsUsed(MacroInfo *MI);
2166 };
2167 
2168 /// \brief Abstract base class that describes a handler that will receive
2169 /// source ranges for each of the comments encountered in the source file.
2171 public:
2172  virtual ~CommentHandler();
2173 
2174  // The handler shall return true if it has pushed any tokens
2175  // to be read using e.g. EnterToken or EnterTokenStream.
2176  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2177 };
2178 
2179 /// \brief Registry of pragma handlers added by plugins
2180 using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2181 
2182 } // namespace clang
2183 
2184 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H
IdentifierInfo * get(StringRef Name) override
get - Return the identifier token info for the specified named identifier.
Definition: PTHLexer.cpp:599
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
Definition: Preprocessor.h:329
A set of visible modules.
Definition: Module.h:571
llvm::BumpPtrAllocator & getPreprocessorAllocator()
Definition: Preprocessor.h:823
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:77
SelectorTable & getSelectorTable()
Definition: Preprocessor.h:821
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:68
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:116
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the &#39;spelling&#39; of the Tok token.
MemoryBufferCache & getPCMCache() const
Definition: Preprocessor.h:816
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:26
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
Defines the SourceManager interface.
static const Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:21
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
FileManager & getFileManager() const
Definition: Preprocessor.h:814
Defines the clang::Module class, which describes a module in the source code.
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
SourceLocation getPragmaARCCFCodeAuditedLoc() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
StringRef P
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:564
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1207
A directive for an undefined macro.
Definition: MacroInfo.h:429
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool hadModuleLoaderFatalFailure() const
Definition: Preprocessor.h:840
const DefInfo findDirectiveAtLoc(SourceLocation L, SourceManager &SM) const
Find macro definition active in the specified source location.
Definition: MacroInfo.cpp:203
void EnterToken(const Token &Tok)
Enters a token in the token stream to be lexed next.
Manage memory buffers across multiple users.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:103
Builtin::Context & getBuiltinInfo()
Definition: Preprocessor.h:822
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:36
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:118
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Definition: MacroInfo.h:329
tok::TokenKind getKind() const
Definition: Token.h:90
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:93
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
One of these records is kept for each identifier that is lexed.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
Represents a macro directive exported by a module.
Definition: MacroInfo.h:488
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:406
This table allows us to fully hide how we implement multi-keyword caching.
LineState State
Definition: Format.h:1900
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:812
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Definition: Preprocessor.h:850
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:48
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:811
Describes a module or submodule.
Definition: Module.h:65
bool isPragmaLexer() const
isPragmaLexer - Returns true if this Lexer is being used to lex a pragma.
Definition: Lexer.h:184
A directive for setting the module visibility of a macro.
Definition: MacroInfo.h:444
bool isInvalid() const
void SetSuppressIncludeNotFoundError(bool Suppress)
Definition: Preprocessor.h:860
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:110
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
Definition: Preprocessor.h:897
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:817
uint32_t Offset
Definition: CacheTokens.cpp:43
const FormatToken & Tok
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void LexNonComment(Token &Result)
Lex a token.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:147
bool isMacroDefined(const IdentifierInfo *II)
Definition: Preprocessor.h:919
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we&#39;ve alread...
Definition: Preprocessor.h:927
Defines the Diagnostic-related interfaces.
const TargetInfo * getAuxTargetInfo() const
Definition: Preprocessor.h:813
TokenLexer - This implements a lexer that returns tokens from a macro body or token stream instead of...
Definition: TokenLexer.h:31
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
Definition: Preprocessor.h:870
PragmaIntroducerKind
Describes how the pragma was introduced, e.g., with #pragma, _Pragma, or __pragma.
Definition: Pragma.h:32
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:148
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:955
void clearCodeCompletionHandler()
Clear out the code completion handler.
StringRef Filename
Definition: Format.cpp:1345
Provides lookups to, and iteration over, IdentiferInfo objects.
Exposes information about the current target.
Definition: TargetInfo.h:54
const IdentifierTable & getIdentifierTable() const
Definition: Preprocessor.h:820
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Abstract interface for external sources of preprocessor information.
Defines the clang::LangOptions interface.
unsigned getCounterValue() const
SourceLocation End
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:350
int Id
Definition: ASTDiff.cpp:191
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
const AnnotatedLine * Line
bool isDefined() const
Definition: MacroInfo.h:385
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Definition: Preprocessor.h:879
Implements an efficient mapping from strings to IdentifierInfo nodes.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
A class for tracking whether we&#39;re inside a VA_OPT during a traversal of the tokens of a variadic mac...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
#define bool
Definition: stdbool.h:31
DefInfo getDefinition()
Traverses the macro directives history and returns the next macro definition directive along with inf...
Definition: MacroInfo.cpp:179
SourceLocation Begin
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:88
llvm::Optional< PreambleSkipInfo > getPreambleSkipInfo() const
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
Definition: Preprocessor.h:981
bool getCommentRetentionState() const
Definition: Preprocessor.h:855
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:101
PPCallbacks * getPPCallbacks() const
Accessors for preprocessor callbacks.
Definition: Preprocessor.h:907
const SourceManager & SM
Definition: Format.cpp:1337
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
bool isRecordingPreamble() const
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
SourceManager & getSourceManager() const
Definition: Preprocessor.h:815
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define&#39;d and not #undef&#39;...
Definition: Preprocessor.h:970
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:833
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:291
Kind
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
void setExternalSource(ExternalPreprocessorSource *Source)
Definition: Preprocessor.h:829
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.h:95
void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:940
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:177
IdentifierTable & getIdentifierTable()
Definition: Preprocessor.h:819
void setPragmasEnabled(bool Enabled)
Definition: Preprocessor.h:857
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:87
bool isPoisoned() const
Return true if this token has been poisoned.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
Definition: Preprocessor.h:953
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
Definition: Preprocessor.h:887
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
const MacroInfo * getMacroInfo() const
Definition: MacroInfo.h:391
StringRef getName() const
Return the actual identifier string.
bool isMacroDefined(StringRef Id)
Definition: Preprocessor.h:916
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
Dataflow Directional Tag Classes.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion)
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
Definition: DeclSpec.h:1114
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion)
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:358
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:59
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Abstract interface for a module loader.
Definition: ModuleLoader.h:74
unsigned getLength() const
Definition: Token.h:127
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:40
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:49
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:215
bool getPragmasEnabled() const
Definition: Preprocessor.h:858
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:124
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
Definition: Preprocessor.h:838
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Definition: Preprocessor.h:806
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:45
Defines the PPCallbacks interface.
Defines the clang::TokenKind enum and support functions.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
SourceLocation getModuleImportLoc(Module *M) const
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:900
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:808
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:242
bool hasRecordedPreamble() const
const std::string & getPredefines() const
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:283
void setPredefines(StringRef P)
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
The translation unit is a complete translation unit.
Definition: LangOptions.h:244
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
MacroMap::const_iterator macro_iterator
Abstract base class that describes a handler that will receive source ranges for each of the comments...
void setCounterValue(unsigned V)
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
Definition: Preprocessor.h:845
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
Definition: Preprocessor.h:864
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, llvm::Optional< PreambleSkipInfo > SkipInfo)
Directive - Abstract class representing a parsed verify directive.
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Definition: Preprocessor.h:985
Callback handler that receives notifications when performing code completion within the preprocessor...
A header that is known to reside within a given module, whether it was included or excluded...
Definition: ModuleMap.h:133
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
Definition: Preprocessor.h:876
PTHManager * getPTHManager()
Definition: Preprocessor.h:827
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:908
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:127
void setDiagnostics(DiagnosticsEngine &D)
Definition: Preprocessor.h:809