clang  9.0.0svn
Preprocessor.h
Go to the documentation of this file.
1 //===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines the clang::Preprocessor interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15 #define LLVM_CLANG_LEX_PREPROCESSOR_H
16 
17 #include "clang/Basic/Builtins.h"
18 #include "clang/Basic/Diagnostic.h"
20 #include "clang/Basic/LLVM.h"
22 #include "clang/Basic/Module.h"
25 #include "clang/Basic/TokenKinds.h"
26 #include "clang/Lex/Lexer.h"
27 #include "clang/Lex/MacroInfo.h"
28 #include "clang/Lex/ModuleLoader.h"
29 #include "clang/Lex/ModuleMap.h"
30 #include "clang/Lex/PPCallbacks.h"
31 #include "clang/Lex/Token.h"
32 #include "clang/Lex/TokenLexer.h"
33 #include "llvm/ADT/ArrayRef.h"
34 #include "llvm/ADT/DenseMap.h"
35 #include "llvm/ADT/FoldingSet.h"
36 #include "llvm/ADT/None.h"
37 #include "llvm/ADT/Optional.h"
38 #include "llvm/ADT/PointerUnion.h"
39 #include "llvm/ADT/STLExtras.h"
40 #include "llvm/ADT/SmallPtrSet.h"
41 #include "llvm/ADT/SmallVector.h"
42 #include "llvm/ADT/StringRef.h"
43 #include "llvm/ADT/TinyPtrVector.h"
44 #include "llvm/ADT/iterator_range.h"
45 #include "llvm/Support/Allocator.h"
46 #include "llvm/Support/Casting.h"
47 #include "llvm/Support/Registry.h"
48 #include <cassert>
49 #include <cstddef>
50 #include <cstdint>
51 #include <memory>
52 #include <map>
53 #include <string>
54 #include <utility>
55 #include <vector>
56 
57 namespace llvm {
58 
59 template<unsigned InternalLen> class SmallString;
60 
61 } // namespace llvm
62 
63 namespace clang {
64 
65 class CodeCompletionHandler;
66 class CommentHandler;
67 class DirectoryEntry;
68 class DirectoryLookup;
69 class ExternalPreprocessorSource;
70 class FileEntry;
71 class FileManager;
72 class HeaderSearch;
73 class MacroArgs;
74 class MemoryBufferCache;
75 class PragmaHandler;
76 class PragmaNamespace;
77 class PreprocessingRecord;
78 class PreprocessorLexer;
79 class PreprocessorOptions;
80 class ScratchBuffer;
81 class TargetInfo;
82 
83 /// Stores token information for comparing actual tokens with
84 /// predefined values. Only handles simple tokens and identifiers.
85 class TokenValue {
87  IdentifierInfo *II;
88 
89 public:
90  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
91  assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
92  assert(Kind != tok::identifier &&
93  "Identifiers should be created by TokenValue(IdentifierInfo *)");
94  assert(!tok::isLiteral(Kind) && "Literals are not supported.");
95  assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
96  }
97 
98  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
99 
100  bool operator==(const Token &Tok) const {
101  return Tok.getKind() == Kind &&
102  (!II || II == Tok.getIdentifierInfo());
103  }
104 };
105 
106 /// Context in which macro name is used.
107 enum MacroUse {
108  // other than #define or #undef
109  MU_Other = 0,
110 
111  // macro name specified in #define
113 
114  // macro name specified in #undef
116 };
117 
118 /// Engages in a tight little dance with the lexer to efficiently
119 /// preprocess tokens.
120 ///
121 /// Lexers know only about tokens within a single source file, and don't
122 /// know anything about preprocessor-level issues like the \#include stack,
123 /// token expansion, etc.
127 
128  std::shared_ptr<PreprocessorOptions> PPOpts;
129  DiagnosticsEngine *Diags;
130  LangOptions &LangOpts;
131  const TargetInfo *Target = nullptr;
132  const TargetInfo *AuxTarget = nullptr;
133  FileManager &FileMgr;
134  SourceManager &SourceMgr;
135  MemoryBufferCache &PCMCache;
136  std::unique_ptr<ScratchBuffer> ScratchBuf;
137  HeaderSearch &HeaderInfo;
138  ModuleLoader &TheModuleLoader;
139 
140  /// External source of macros.
141  ExternalPreprocessorSource *ExternalSource;
142 
143  /// A BumpPtrAllocator object used to quickly allocate and release
144  /// objects internal to the Preprocessor.
145  llvm::BumpPtrAllocator BP;
146 
147  /// Identifiers for builtin macros and other builtins.
148  IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
149  IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
150  IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
151  IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
152  IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
153  IdentifierInfo *Ident__COUNTER__; // __COUNTER__
154  IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
155  IdentifierInfo *Ident__identifier; // __identifier
156  IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
157  IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
158  IdentifierInfo *Ident__has_feature; // __has_feature
159  IdentifierInfo *Ident__has_extension; // __has_extension
160  IdentifierInfo *Ident__has_builtin; // __has_builtin
161  IdentifierInfo *Ident__has_attribute; // __has_attribute
162  IdentifierInfo *Ident__has_include; // __has_include
163  IdentifierInfo *Ident__has_include_next; // __has_include_next
164  IdentifierInfo *Ident__has_warning; // __has_warning
165  IdentifierInfo *Ident__is_identifier; // __is_identifier
166  IdentifierInfo *Ident__building_module; // __building_module
167  IdentifierInfo *Ident__MODULE__; // __MODULE__
168  IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
169  IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
170  IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
171  IdentifierInfo *Ident__is_target_arch; // __is_target_arch
172  IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
173  IdentifierInfo *Ident__is_target_os; // __is_target_os
174  IdentifierInfo *Ident__is_target_environment; // __is_target_environment
175 
176  SourceLocation DATELoc, TIMELoc;
177 
178  // Next __COUNTER__ value, starts at 0.
179  unsigned CounterValue = 0;
180 
181  enum {
182  /// Maximum depth of \#includes.
183  MaxAllowedIncludeStackDepth = 200
184  };
185 
186  // State that is set before the preprocessor begins.
187  bool KeepComments : 1;
188  bool KeepMacroComments : 1;
189  bool SuppressIncludeNotFoundError : 1;
190 
191  // State that changes while the preprocessor runs:
192  bool InMacroArgs : 1; // True if parsing fn macro invocation args.
193 
194  /// Whether the preprocessor owns the header search object.
195  bool OwnsHeaderSearch : 1;
196 
197  /// True if macro expansion is disabled.
198  bool DisableMacroExpansion : 1;
199 
200  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
201  /// when parsing preprocessor directives.
202  bool MacroExpansionInDirectivesOverride : 1;
203 
205 
206  /// Whether we have already loaded macros from the external source.
207  mutable bool ReadMacrosFromExternalSource : 1;
208 
209  /// True if pragmas are enabled.
210  bool PragmasEnabled : 1;
211 
212  /// True if the current build action is a preprocessing action.
213  bool PreprocessedOutput : 1;
214 
215  /// True if we are currently preprocessing a #if or #elif directive
216  bool ParsingIfOrElifDirective;
217 
218  /// True if we are pre-expanding macro arguments.
219  bool InMacroArgPreExpansion;
220 
221  /// Mapping/lookup information for all identifiers in
222  /// the program, including program keywords.
223  mutable IdentifierTable Identifiers;
224 
225  /// This table contains all the selectors in the program.
226  ///
227  /// Unlike IdentifierTable above, this table *isn't* populated by the
228  /// preprocessor. It is declared/expanded here because its role/lifetime is
229  /// conceptually similar to the IdentifierTable. In addition, the current
230  /// control flow (in clang::ParseAST()), make it convenient to put here.
231  ///
232  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
233  /// the lifetime of the preprocessor.
234  SelectorTable Selectors;
235 
236  /// Information about builtins.
238 
239  /// Tracks all of the pragmas that the client registered
240  /// with this preprocessor.
241  std::unique_ptr<PragmaNamespace> PragmaHandlers;
242 
243  /// Pragma handlers of the original source is stored here during the
244  /// parsing of a model file.
245  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
246 
247  /// Tracks all of the comment handlers that the client registered
248  /// with this preprocessor.
249  std::vector<CommentHandler *> CommentHandlers;
250 
251  /// True if we want to ignore EOF token and continue later on (thus
252  /// avoid tearing the Lexer and etc. down).
253  bool IncrementalProcessing = false;
254 
255  /// The kind of translation unit we are processing.
256  TranslationUnitKind TUKind;
257 
258  /// The code-completion handler.
259  CodeCompletionHandler *CodeComplete = nullptr;
260 
261  /// The file that we're performing code-completion for, if any.
262  const FileEntry *CodeCompletionFile = nullptr;
263 
264  /// The offset in file for the code-completion point.
265  unsigned CodeCompletionOffset = 0;
266 
267  /// The location for the code-completion point. This gets instantiated
268  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
269  SourceLocation CodeCompletionLoc;
270 
271  /// The start location for the file of the code-completion point.
272  ///
273  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
274  /// for preprocessing.
275  SourceLocation CodeCompletionFileLoc;
276 
277  /// The source location of the \c import contextual keyword we just
278  /// lexed, if any.
279  SourceLocation ModuleImportLoc;
280 
281  /// The module import path that we're currently processing.
283 
284  /// Whether the last token we lexed was an '@'.
285  bool LastTokenWasAt = false;
286 
287  /// Whether the module import expects an identifier next. Otherwise,
288  /// it expects a '.' or ';'.
289  bool ModuleImportExpectsIdentifier = false;
290 
291  /// The source location of the currently-active
292  /// \#pragma clang arc_cf_code_audited begin.
293  SourceLocation PragmaARCCFCodeAuditedLoc;
294 
295  /// The source location of the currently-active
296  /// \#pragma clang assume_nonnull begin.
297  SourceLocation PragmaAssumeNonNullLoc;
298 
299  /// True if we hit the code-completion point.
300  bool CodeCompletionReached = false;
301 
302  /// The code completion token containing the information
303  /// on the stem that is to be code completed.
304  IdentifierInfo *CodeCompletionII = nullptr;
305 
306  /// Range for the code completion token.
307  SourceRange CodeCompletionTokenRange;
308 
309  /// The directory that the main file should be considered to occupy,
310  /// if it does not correspond to a real file (as happens when building a
311  /// module).
312  const DirectoryEntry *MainFileDir = nullptr;
313 
314  /// The number of bytes that we will initially skip when entering the
315  /// main file, along with a flag that indicates whether skipping this number
316  /// of bytes will place the lexer at the start of a line.
317  ///
318  /// This is used when loading a precompiled preamble.
319  std::pair<int, bool> SkipMainFilePreamble;
320 
321  /// Whether we hit an error due to reaching max allowed include depth. Allows
322  /// to avoid hitting the same error over and over again.
323  bool HasReachedMaxIncludeDepth = false;
324 
325 public:
330  bool FoundElse;
332 
334  bool FoundNonSkipPortion, bool FoundElse,
335  SourceLocation ElseLoc)
336  : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
337  FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
338  ElseLoc(ElseLoc) {}
339  };
340 
341 private:
342  friend class ASTReader;
343  friend class MacroArgs;
344 
345  class PreambleConditionalStackStore {
346  enum State {
347  Off = 0,
348  Recording = 1,
349  Replaying = 2,
350  };
351 
352  public:
353  PreambleConditionalStackStore() = default;
354 
355  void startRecording() { ConditionalStackState = Recording; }
356  void startReplaying() { ConditionalStackState = Replaying; }
357  bool isRecording() const { return ConditionalStackState == Recording; }
358  bool isReplaying() const { return ConditionalStackState == Replaying; }
359 
360  ArrayRef<PPConditionalInfo> getStack() const {
361  return ConditionalStack;
362  }
363 
364  void doneReplaying() {
365  ConditionalStack.clear();
366  ConditionalStackState = Off;
367  }
368 
369  void setStack(ArrayRef<PPConditionalInfo> s) {
370  if (!isRecording() && !isReplaying())
371  return;
372  ConditionalStack.clear();
373  ConditionalStack.append(s.begin(), s.end());
374  }
375 
376  bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
377 
378  bool reachedEOFWhileSkipping() const { return SkipInfo.hasValue(); }
379 
380  void clearSkipInfo() { SkipInfo.reset(); }
381 
383 
384  private:
385  SmallVector<PPConditionalInfo, 4> ConditionalStack;
386  State ConditionalStackState = Off;
387  } PreambleConditionalStack;
388 
389  /// The current top of the stack that we're lexing from if
390  /// not expanding a macro and we are lexing directly from source code.
391  ///
392  /// Only one of CurLexer, or CurTokenLexer will be non-null.
393  std::unique_ptr<Lexer> CurLexer;
394 
395  /// The current top of the stack what we're lexing from
396  /// if not expanding a macro.
397  ///
398  /// This is an alias for CurLexer.
399  PreprocessorLexer *CurPPLexer = nullptr;
400 
401  /// Used to find the current FileEntry, if CurLexer is non-null
402  /// and if applicable.
403  ///
404  /// This allows us to implement \#include_next and find directory-specific
405  /// properties.
406  const DirectoryLookup *CurDirLookup = nullptr;
407 
408  /// The current macro we are expanding, if we are expanding a macro.
409  ///
410  /// One of CurLexer and CurTokenLexer must be null.
411  std::unique_ptr<TokenLexer> CurTokenLexer;
412 
413  /// The kind of lexer we're currently working with.
414  enum CurLexerKind {
415  CLK_Lexer,
416  CLK_TokenLexer,
417  CLK_CachingLexer,
418  CLK_LexAfterModuleImport
419  } CurLexerKind = CLK_Lexer;
420 
421  /// If the current lexer is for a submodule that is being built, this
422  /// is that submodule.
423  Module *CurLexerSubmodule = nullptr;
424 
425  /// Keeps track of the stack of files currently
426  /// \#included, and macros currently being expanded from, not counting
427  /// CurLexer/CurTokenLexer.
428  struct IncludeStackInfo {
429  enum CurLexerKind CurLexerKind;
430  Module *TheSubmodule;
431  std::unique_ptr<Lexer> TheLexer;
432  PreprocessorLexer *ThePPLexer;
433  std::unique_ptr<TokenLexer> TheTokenLexer;
434  const DirectoryLookup *TheDirLookup;
435 
436  // The following constructors are completely useless copies of the default
437  // versions, only needed to pacify MSVC.
438  IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
439  std::unique_ptr<Lexer> &&TheLexer,
440  PreprocessorLexer *ThePPLexer,
441  std::unique_ptr<TokenLexer> &&TheTokenLexer,
442  const DirectoryLookup *TheDirLookup)
443  : CurLexerKind(std::move(CurLexerKind)),
444  TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
445  ThePPLexer(std::move(ThePPLexer)),
446  TheTokenLexer(std::move(TheTokenLexer)),
447  TheDirLookup(std::move(TheDirLookup)) {}
448  };
449  std::vector<IncludeStackInfo> IncludeMacroStack;
450 
451  /// Actions invoked when some preprocessor activity is
452  /// encountered (e.g. a file is \#included, etc).
453  std::unique_ptr<PPCallbacks> Callbacks;
454 
455  struct MacroExpandsInfo {
456  Token Tok;
457  MacroDefinition MD;
458  SourceRange Range;
459 
460  MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
461  : Tok(Tok), MD(MD), Range(Range) {}
462  };
463  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
464 
465  /// Information about a name that has been used to define a module macro.
466  struct ModuleMacroInfo {
467  /// The most recent macro directive for this identifier.
468  MacroDirective *MD;
469 
470  /// The active module macros for this identifier.
471  llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
472 
473  /// The generation number at which we last updated ActiveModuleMacros.
474  /// \see Preprocessor::VisibleModules.
475  unsigned ActiveModuleMacrosGeneration = 0;
476 
477  /// Whether this macro name is ambiguous.
478  bool IsAmbiguous = false;
479 
480  /// The module macros that are overridden by this macro.
481  llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
482 
483  ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
484  };
485 
486  /// The state of a macro for an identifier.
487  class MacroState {
488  mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
489 
490  ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
491  const IdentifierInfo *II) const {
492  if (II->isOutOfDate())
493  PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
494  // FIXME: Find a spare bit on IdentifierInfo and store a
495  // HasModuleMacros flag.
496  if (!II->hasMacroDefinition() ||
497  (!PP.getLangOpts().Modules &&
498  !PP.getLangOpts().ModulesLocalVisibility) ||
499  !PP.CurSubmoduleState->VisibleModules.getGeneration())
500  return nullptr;
501 
502  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
503  if (!Info) {
504  Info = new (PP.getPreprocessorAllocator())
505  ModuleMacroInfo(State.get<MacroDirective *>());
506  State = Info;
507  }
508 
509  if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
510  Info->ActiveModuleMacrosGeneration)
511  PP.updateModuleMacroInfo(II, *Info);
512  return Info;
513  }
514 
515  public:
516  MacroState() : MacroState(nullptr) {}
517  MacroState(MacroDirective *MD) : State(MD) {}
518 
519  MacroState(MacroState &&O) noexcept : State(O.State) {
520  O.State = (MacroDirective *)nullptr;
521  }
522 
523  MacroState &operator=(MacroState &&O) noexcept {
524  auto S = O.State;
525  O.State = (MacroDirective *)nullptr;
526  State = S;
527  return *this;
528  }
529 
530  ~MacroState() {
531  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
532  Info->~ModuleMacroInfo();
533  }
534 
535  MacroDirective *getLatest() const {
536  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
537  return Info->MD;
538  return State.get<MacroDirective*>();
539  }
540 
541  void setLatest(MacroDirective *MD) {
542  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
543  Info->MD = MD;
544  else
545  State = MD;
546  }
547 
548  bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
549  auto *Info = getModuleInfo(PP, II);
550  return Info ? Info->IsAmbiguous : false;
551  }
552 
554  getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
555  if (auto *Info = getModuleInfo(PP, II))
556  return Info->ActiveModuleMacros;
557  return None;
558  }
559 
560  MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
561  SourceManager &SourceMgr) const {
562  // FIXME: Incorporate module macros into the result of this.
563  if (auto *Latest = getLatest())
564  return Latest->findDirectiveAtLoc(Loc, SourceMgr);
565  return {};
566  }
567 
568  void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
569  if (auto *Info = getModuleInfo(PP, II)) {
570  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
571  Info->ActiveModuleMacros.begin(),
572  Info->ActiveModuleMacros.end());
573  Info->ActiveModuleMacros.clear();
574  Info->IsAmbiguous = false;
575  }
576  }
577 
578  ArrayRef<ModuleMacro*> getOverriddenMacros() const {
579  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
580  return Info->OverriddenMacros;
581  return None;
582  }
583 
584  void setOverriddenMacros(Preprocessor &PP,
585  ArrayRef<ModuleMacro *> Overrides) {
586  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
587  if (!Info) {
588  if (Overrides.empty())
589  return;
590  Info = new (PP.getPreprocessorAllocator())
591  ModuleMacroInfo(State.get<MacroDirective *>());
592  State = Info;
593  }
594  Info->OverriddenMacros.clear();
595  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
596  Overrides.begin(), Overrides.end());
597  Info->ActiveModuleMacrosGeneration = 0;
598  }
599  };
600 
601  /// For each IdentifierInfo that was associated with a macro, we
602  /// keep a mapping to the history of all macro definitions and #undefs in
603  /// the reverse order (the latest one is in the head of the list).
604  ///
605  /// This mapping lives within the \p CurSubmoduleState.
606  using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
607 
608  struct SubmoduleState;
609 
610  /// Information about a submodule that we're currently building.
611  struct BuildingSubmoduleInfo {
612  /// The module that we are building.
613  Module *M;
614 
615  /// The location at which the module was included.
616  SourceLocation ImportLoc;
617 
618  /// Whether we entered this submodule via a pragma.
619  bool IsPragma;
620 
621  /// The previous SubmoduleState.
622  SubmoduleState *OuterSubmoduleState;
623 
624  /// The number of pending module macro names when we started building this.
625  unsigned OuterPendingModuleMacroNames;
626 
627  BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
628  SubmoduleState *OuterSubmoduleState,
629  unsigned OuterPendingModuleMacroNames)
630  : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
631  OuterSubmoduleState(OuterSubmoduleState),
632  OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
633  };
634  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
635 
636  /// Information about a submodule's preprocessor state.
637  struct SubmoduleState {
638  /// The macros for the submodule.
639  MacroMap Macros;
640 
641  /// The set of modules that are visible within the submodule.
642  VisibleModuleSet VisibleModules;
643 
644  // FIXME: CounterValue?
645  // FIXME: PragmaPushMacroInfo?
646  };
647  std::map<Module *, SubmoduleState> Submodules;
648 
649  /// The preprocessor state for preprocessing outside of any submodule.
650  SubmoduleState NullSubmoduleState;
651 
652  /// The current submodule state. Will be \p NullSubmoduleState if we're not
653  /// in a submodule.
654  SubmoduleState *CurSubmoduleState;
655 
656  /// The set of known macros exported from modules.
657  llvm::FoldingSet<ModuleMacro> ModuleMacros;
658 
659  /// The names of potential module macros that we've not yet processed.
660  llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
661 
662  /// The list of module macros, for each identifier, that are not overridden by
663  /// any other module macro.
664  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
665  LeafModuleMacros;
666 
667  /// Macros that we want to warn because they are not used at the end
668  /// of the translation unit.
669  ///
670  /// We store just their SourceLocations instead of
671  /// something like MacroInfo*. The benefit of this is that when we are
672  /// deserializing from PCH, we don't need to deserialize identifier & macros
673  /// just so that we can report that they are unused, we just warn using
674  /// the SourceLocations of this set (that will be filled by the ASTReader).
675  /// We are using SmallPtrSet instead of a vector for faster removal.
676  using WarnUnusedMacroLocsTy = llvm::SmallPtrSet<SourceLocation, 32>;
677  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
678 
679  /// A "freelist" of MacroArg objects that can be
680  /// reused for quick allocation.
681  MacroArgs *MacroArgCache = nullptr;
682 
683  /// For each IdentifierInfo used in a \#pragma push_macro directive,
684  /// we keep a MacroInfo stack used to restore the previous macro value.
685  llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
686  PragmaPushMacroInfo;
687 
688  // Various statistics we track for performance analysis.
689  unsigned NumDirectives = 0;
690  unsigned NumDefined = 0;
691  unsigned NumUndefined = 0;
692  unsigned NumPragma = 0;
693  unsigned NumIf = 0;
694  unsigned NumElse = 0;
695  unsigned NumEndif = 0;
696  unsigned NumEnteredSourceFiles = 0;
697  unsigned MaxIncludeStackDepth = 0;
698  unsigned NumMacroExpanded = 0;
699  unsigned NumFnMacroExpanded = 0;
700  unsigned NumBuiltinMacroExpanded = 0;
701  unsigned NumFastMacroExpanded = 0;
702  unsigned NumTokenPaste = 0;
703  unsigned NumFastTokenPaste = 0;
704  unsigned NumSkipped = 0;
705 
706  /// The predefined macros that preprocessor should use from the
707  /// command line etc.
708  std::string Predefines;
709 
710  /// The file ID for the preprocessor predefines.
711  FileID PredefinesFileID;
712 
713  /// The file ID for the PCH through header.
714  FileID PCHThroughHeaderFileID;
715 
716  /// Whether tokens are being skipped until a #pragma hdrstop is seen.
717  bool SkippingUntilPragmaHdrStop = false;
718 
719  /// Whether tokens are being skipped until the through header is seen.
720  bool SkippingUntilPCHThroughHeader = false;
721 
722  /// \{
723  /// Cache of macro expanders to reduce malloc traffic.
724  enum { TokenLexerCacheSize = 8 };
725  unsigned NumCachedTokenLexers;
726  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
727  /// \}
728 
729  /// Keeps macro expanded tokens for TokenLexers.
730  //
731  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
732  /// going to lex in the cache and when it finishes the tokens are removed
733  /// from the end of the cache.
734  SmallVector<Token, 16> MacroExpandedTokens;
735  std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
736 
737  /// A record of the macro definitions and expansions that
738  /// occurred during preprocessing.
739  ///
740  /// This is an optional side structure that can be enabled with
741  /// \c createPreprocessingRecord() prior to preprocessing.
742  PreprocessingRecord *Record = nullptr;
743 
744  /// Cached tokens state.
746 
747  /// Cached tokens are stored here when we do backtracking or
748  /// lookahead. They are "lexed" by the CachingLex() method.
750 
751  /// The position of the cached token that CachingLex() should
752  /// "lex" next.
753  ///
754  /// If it points beyond the CachedTokens vector, it means that a normal
755  /// Lex() should be invoked.
756  CachedTokensTy::size_type CachedLexPos = 0;
757 
758  /// Stack of backtrack positions, allowing nested backtracks.
759  ///
760  /// The EnableBacktrackAtThisPos() method pushes a position to
761  /// indicate where CachedLexPos should be set when the BackTrack() method is
762  /// invoked (at which point the last position is popped).
763  std::vector<CachedTokensTy::size_type> BacktrackPositions;
764 
765  struct MacroInfoChain {
766  MacroInfo MI;
767  MacroInfoChain *Next;
768  };
769 
770  /// MacroInfos are managed as a chain for easy disposal. This is the head
771  /// of that list.
772  MacroInfoChain *MIChainHead = nullptr;
773 
774  void updateOutOfDateIdentifier(IdentifierInfo &II) const;
775 
776 public:
777  Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
779  MemoryBufferCache &PCMCache,
780  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
781  IdentifierInfoLookup *IILookup = nullptr,
782  bool OwnsHeaderSearch = false,
784 
785  ~Preprocessor();
786 
787  /// Initialize the preprocessor using information about the target.
788  ///
789  /// \param Target is owned by the caller and must remain valid for the
790  /// lifetime of the preprocessor.
791  /// \param AuxTarget is owned by the caller and must remain valid for
792  /// the lifetime of the preprocessor.
793  void Initialize(const TargetInfo &Target,
794  const TargetInfo *AuxTarget = nullptr);
795 
796  /// Initialize the preprocessor to parse a model file
797  ///
798  /// To parse model files the preprocessor of the original source is reused to
799  /// preserver the identifier table. However to avoid some duplicate
800  /// information in the preprocessor some cleanup is needed before it is used
801  /// to parse model files. This method does that cleanup.
802  void InitializeForModelFile();
803 
804  /// Cleanup after model file parsing
805  void FinalizeForModelFile();
806 
807  /// Retrieve the preprocessor options used to initialize this
808  /// preprocessor.
809  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
810 
811  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
812  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
813 
814  const LangOptions &getLangOpts() const { return LangOpts; }
815  const TargetInfo &getTargetInfo() const { return *Target; }
816  const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
817  FileManager &getFileManager() const { return FileMgr; }
818  SourceManager &getSourceManager() const { return SourceMgr; }
819  MemoryBufferCache &getPCMCache() const { return PCMCache; }
820  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
821 
822  IdentifierTable &getIdentifierTable() { return Identifiers; }
823  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
824  SelectorTable &getSelectorTable() { return Selectors; }
826  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
827 
829  ExternalSource = Source;
830  }
831 
833  return ExternalSource;
834  }
835 
836  /// Retrieve the module loader associated with this preprocessor.
837  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
838 
840  return TheModuleLoader.HadFatalFailure;
841  }
842 
843  /// True if we are currently preprocessing a #if or #elif directive
845  return ParsingIfOrElifDirective;
846  }
847 
848  /// Control whether the preprocessor retains comments in output.
849  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
850  this->KeepComments = KeepComments | KeepMacroComments;
851  this->KeepMacroComments = KeepMacroComments;
852  }
853 
854  bool getCommentRetentionState() const { return KeepComments; }
855 
856  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
857  bool getPragmasEnabled() const { return PragmasEnabled; }
858 
859  void SetSuppressIncludeNotFoundError(bool Suppress) {
860  SuppressIncludeNotFoundError = Suppress;
861  }
862 
864  return SuppressIncludeNotFoundError;
865  }
866 
867  /// Sets whether the preprocessor is responsible for producing output or if
868  /// it is producing tokens to be consumed by Parse and Sema.
869  void setPreprocessedOutput(bool IsPreprocessedOutput) {
870  PreprocessedOutput = IsPreprocessedOutput;
871  }
872 
873  /// Returns true if the preprocessor is responsible for generating output,
874  /// false if it is producing tokens to be consumed by Parse and Sema.
875  bool isPreprocessedOutput() const { return PreprocessedOutput; }
876 
877  /// Return true if we are lexing directly from the specified lexer.
878  bool isCurrentLexer(const PreprocessorLexer *L) const {
879  return CurPPLexer == L;
880  }
881 
882  /// Return the current lexer being lexed from.
883  ///
884  /// Note that this ignores any potentially active macro expansions and _Pragma
885  /// expansions going on at the time.
886  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
887 
888  /// Return the current file lexer being lexed from.
889  ///
890  /// Note that this ignores any potentially active macro expansions and _Pragma
891  /// expansions going on at the time.
892  PreprocessorLexer *getCurrentFileLexer() const;
893 
894  /// Return the submodule owning the file being lexed. This may not be
895  /// the current module if we have changed modules since entering the file.
896  Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
897 
898  /// Returns the FileID for the preprocessor predefines.
899  FileID getPredefinesFileID() const { return PredefinesFileID; }
900 
901  /// \{
902  /// Accessors for preprocessor callbacks.
903  ///
904  /// Note that this class takes ownership of any PPCallbacks object given to
905  /// it.
906  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
907  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
908  if (Callbacks)
909  C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
910  std::move(Callbacks));
911  Callbacks = std::move(C);
912  }
913  /// \}
914 
915  bool isMacroDefined(StringRef Id) {
916  return isMacroDefined(&Identifiers.get(Id));
917  }
918  bool isMacroDefined(const IdentifierInfo *II) {
919  return II->hasMacroDefinition() &&
920  (!getLangOpts().Modules || (bool)getMacroDefinition(II));
921  }
922 
923  /// Determine whether II is defined as a macro within the module M,
924  /// if that is a module that we've already preprocessed. Does not check for
925  /// macros imported into M.
927  if (!II->hasMacroDefinition())
928  return false;
929  auto I = Submodules.find(M);
930  if (I == Submodules.end())
931  return false;
932  auto J = I->second.Macros.find(II);
933  if (J == I->second.Macros.end())
934  return false;
935  auto *MD = J->second.getLatest();
936  return MD && MD->isDefined();
937  }
938 
940  if (!II->hasMacroDefinition())
941  return {};
942 
943  MacroState &S = CurSubmoduleState->Macros[II];
944  auto *MD = S.getLatest();
945  while (MD && isa<VisibilityMacroDirective>(MD))
946  MD = MD->getPrevious();
947  return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
948  S.getActiveModuleMacros(*this, II),
949  S.isAmbiguous(*this, II));
950  }
951 
953  SourceLocation Loc) {
954  if (!II->hadMacroDefinition())
955  return {};
956 
957  MacroState &S = CurSubmoduleState->Macros[II];
959  if (auto *MD = S.getLatest())
960  DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
961  // FIXME: Compute the set of active module macros at the specified location.
962  return MacroDefinition(DI.getDirective(),
963  S.getActiveModuleMacros(*this, II),
964  S.isAmbiguous(*this, II));
965  }
966 
967  /// Given an identifier, return its latest non-imported MacroDirective
968  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
970  if (!II->hasMacroDefinition())
971  return nullptr;
972 
973  auto *MD = getLocalMacroDirectiveHistory(II);
974  if (!MD || MD->getDefinition().isUndefined())
975  return nullptr;
976 
977  return MD;
978  }
979 
980  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
981  return const_cast<Preprocessor*>(this)->getMacroInfo(II);
982  }
983 
985  if (!II->hasMacroDefinition())
986  return nullptr;
987  if (auto MD = getMacroDefinition(II))
988  return MD.getMacroInfo();
989  return nullptr;
990  }
991 
992  /// Given an identifier, return the latest non-imported macro
993  /// directive for that identifier.
994  ///
995  /// One can iterate over all previous macro directives from the most recent
996  /// one.
997  MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const;
998 
999  /// Add a directive to the macro directive history for this identifier.
1000  void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
1002  SourceLocation Loc) {
1003  DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1004  appendMacroDirective(II, MD);
1005  return MD;
1006  }
1008  MacroInfo *MI) {
1009  return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1010  }
1011 
1012  /// Set a MacroDirective that was loaded from a PCH file.
1013  void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED,
1014  MacroDirective *MD);
1015 
1016  /// Register an exported macro for a module and identifier.
1017  ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro,
1018  ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1019  ModuleMacro *getModuleMacro(Module *Mod, IdentifierInfo *II);
1020 
1021  /// Get the list of leaf (non-overridden) module macros for a name.
1023  if (II->isOutOfDate())
1024  updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1025  auto I = LeafModuleMacros.find(II);
1026  if (I != LeafModuleMacros.end())
1027  return I->second;
1028  return None;
1029  }
1030 
1031  /// \{
1032  /// Iterators for the macro history table. Currently defined macros have
1033  /// IdentifierInfo::hasMacroDefinition() set and an empty
1034  /// MacroInfo::getUndefLoc() at the head of the list.
1035  using macro_iterator = MacroMap::const_iterator;
1036 
1037  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1038  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1039 
1040  llvm::iterator_range<macro_iterator>
1041  macros(bool IncludeExternalMacros = true) const {
1042  macro_iterator begin = macro_begin(IncludeExternalMacros);
1043  macro_iterator end = macro_end(IncludeExternalMacros);
1044  return llvm::make_range(begin, end);
1045  }
1046 
1047  /// \}
1048 
1049  /// Return the name of the macro defined before \p Loc that has
1050  /// spelling \p Tokens. If there are multiple macros with same spelling,
1051  /// return the last one defined.
1052  StringRef getLastMacroWithSpelling(SourceLocation Loc,
1053  ArrayRef<TokenValue> Tokens) const;
1054 
1055  const std::string &getPredefines() const { return Predefines; }
1056 
1057  /// Set the predefines for this Preprocessor.
1058  ///
1059  /// These predefines are automatically injected when parsing the main file.
1060  void setPredefines(const char *P) { Predefines = P; }
1061  void setPredefines(StringRef P) { Predefines = P; }
1062 
1063  /// Return information about the specified preprocessor
1064  /// identifier token.
1065  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1066  return &Identifiers.get(Name);
1067  }
1068 
1069  /// Add the specified pragma handler to this preprocessor.
1070  ///
1071  /// If \p Namespace is non-null, then it is a token required to exist on the
1072  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1073  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1075  AddPragmaHandler(StringRef(), Handler);
1076  }
1077 
1078  /// Remove the specific pragma handler from this preprocessor.
1079  ///
1080  /// If \p Namespace is non-null, then it should be the namespace that
1081  /// \p Handler was added to. It is an error to remove a handler that
1082  /// has not been registered.
1083  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1085  RemovePragmaHandler(StringRef(), Handler);
1086  }
1087 
1088  /// Install empty handlers for all pragmas (making them ignored).
1089  void IgnorePragmas();
1090 
1091  /// Add the specified comment handler to the preprocessor.
1092  void addCommentHandler(CommentHandler *Handler);
1093 
1094  /// Remove the specified comment handler.
1095  ///
1096  /// It is an error to remove a handler that has not been registered.
1097  void removeCommentHandler(CommentHandler *Handler);
1098 
1099  /// Set the code completion handler to the given object.
1101  CodeComplete = &Handler;
1102  }
1103 
1104  /// Retrieve the current code-completion handler.
1106  return CodeComplete;
1107  }
1108 
1109  /// Clear out the code completion handler.
1111  CodeComplete = nullptr;
1112  }
1113 
1114  /// Hook used by the lexer to invoke the "included file" code
1115  /// completion point.
1116  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1117 
1118  /// Hook used by the lexer to invoke the "natural language" code
1119  /// completion point.
1120  void CodeCompleteNaturalLanguage();
1121 
1122  /// Set the code completion token for filtering purposes.
1124  CodeCompletionII = Filter;
1125  }
1126 
1127  /// Set the code completion token range for detecting replacement range later
1128  /// on.
1130  const SourceLocation End) {
1131  CodeCompletionTokenRange = {Start, End};
1132  }
1134  return CodeCompletionTokenRange;
1135  }
1136 
1137  /// Get the code completion token for filtering purposes.
1139  if (CodeCompletionII)
1140  return CodeCompletionII->getName();
1141  return {};
1142  }
1143 
1144  /// Retrieve the preprocessing record, or NULL if there is no
1145  /// preprocessing record.
1146  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1147 
1148  /// Create a new preprocessing record, which will keep track of
1149  /// all macro expansions, macro definitions, etc.
1150  void createPreprocessingRecord();
1151 
1152  /// Returns true if the FileEntry is the PCH through header.
1153  bool isPCHThroughHeader(const FileEntry *FE);
1154 
1155  /// True if creating a PCH with a through header.
1156  bool creatingPCHWithThroughHeader();
1157 
1158  /// True if using a PCH with a through header.
1159  bool usingPCHWithThroughHeader();
1160 
1161  /// True if creating a PCH with a #pragma hdrstop.
1162  bool creatingPCHWithPragmaHdrStop();
1163 
1164  /// True if using a PCH with a #pragma hdrstop.
1165  bool usingPCHWithPragmaHdrStop();
1166 
1167  /// Skip tokens until after the #include of the through header or
1168  /// until after a #pragma hdrstop.
1169  void SkipTokensWhileUsingPCH();
1170 
1171  /// Process directives while skipping until the through header or
1172  /// #pragma hdrstop is found.
1173  void HandleSkippedDirectiveWhileUsingPCH(Token &Result,
1174  SourceLocation HashLoc);
1175 
1176  /// Enter the specified FileID as the main source file,
1177  /// which implicitly adds the builtin defines etc.
1178  void EnterMainSourceFile();
1179 
1180  /// Inform the preprocessor callbacks that processing is complete.
1181  void EndSourceFile();
1182 
1183  /// Add a source file to the top of the include stack and
1184  /// start lexing tokens from it instead of the current buffer.
1185  ///
1186  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1187  bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir,
1188  SourceLocation Loc);
1189 
1190  /// Add a Macro to the top of the include stack and start lexing
1191  /// tokens from it instead of the current buffer.
1192  ///
1193  /// \param Args specifies the tokens input to a function-like macro.
1194  /// \param ILEnd specifies the location of the ')' for a function-like macro
1195  /// or the identifier for an object-like macro.
1196  void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1197  MacroArgs *Args);
1198 
1199  /// Add a "macro" context to the top of the include stack,
1200  /// which will cause the lexer to start returning the specified tokens.
1201  ///
1202  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1203  /// will not be subject to further macro expansion. Otherwise, these tokens
1204  /// will be re-macro-expanded when/if expansion is enabled.
1205  ///
1206  /// If \p OwnsTokens is false, this method assumes that the specified stream
1207  /// of tokens has a permanent owner somewhere, so they do not need to be
1208  /// copied. If it is true, it assumes the array of tokens is allocated with
1209  /// \c new[] and the Preprocessor will delete[] it.
1210 private:
1211  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1212  bool DisableMacroExpansion, bool OwnsTokens);
1213 
1214 public:
1215  void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1216  bool DisableMacroExpansion) {
1217  EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true);
1218  }
1219 
1220  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion) {
1221  EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false);
1222  }
1223 
1224  /// Pop the current lexer/macro exp off the top of the lexer stack.
1225  ///
1226  /// This should only be used in situations where the current state of the
1227  /// top-of-stack lexer is known.
1228  void RemoveTopOfLexerStack();
1229 
1230  /// From the point that this method is called, and until
1231  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1232  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1233  /// make the Preprocessor re-lex the same tokens.
1234  ///
1235  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1236  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1237  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1238  ///
1239  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1240  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1241  /// tokens will continue indefinitely.
1242  ///
1243  void EnableBacktrackAtThisPos();
1244 
1245  /// Disable the last EnableBacktrackAtThisPos call.
1246  void CommitBacktrackedTokens();
1247 
1249  CachedTokensTy::size_type Begin, End;
1250  };
1251 
1252 private:
1253  /// A range of cached tokens that should be erased after lexing
1254  /// when backtracking requires the erasure of such cached tokens.
1255  Optional<CachedTokensRange> CachedTokenRangeToErase;
1256 
1257 public:
1258  /// Returns the range of cached tokens that were lexed since
1259  /// EnableBacktrackAtThisPos() was previously called.
1260  CachedTokensRange LastCachedTokenRange();
1261 
1262  /// Erase the range of cached tokens that were lexed since
1263  /// EnableBacktrackAtThisPos() was previously called.
1264  void EraseCachedTokens(CachedTokensRange TokenRange);
1265 
1266  /// Make Preprocessor re-lex the tokens that were lexed since
1267  /// EnableBacktrackAtThisPos() was previously called.
1268  void Backtrack();
1269 
1270  /// True if EnableBacktrackAtThisPos() was called and
1271  /// caching of tokens is on.
1272  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1273 
1274  /// Lex the next token for this preprocessor.
1275  void Lex(Token &Result);
1276 
1277  void LexAfterModuleImport(Token &Result);
1278 
1279  void makeModuleVisible(Module *M, SourceLocation Loc);
1280 
1282  return CurSubmoduleState->VisibleModules.getImportLoc(M);
1283  }
1284 
1285  /// Lex a string literal, which may be the concatenation of multiple
1286  /// string literals and may even come from macro expansion.
1287  /// \returns true on success, false if a error diagnostic has been generated.
1288  bool LexStringLiteral(Token &Result, std::string &String,
1289  const char *DiagnosticTag, bool AllowMacroExpansion) {
1290  if (AllowMacroExpansion)
1291  Lex(Result);
1292  else
1293  LexUnexpandedToken(Result);
1294  return FinishLexStringLiteral(Result, String, DiagnosticTag,
1295  AllowMacroExpansion);
1296  }
1297 
1298  /// Complete the lexing of a string literal where the first token has
1299  /// already been lexed (see LexStringLiteral).
1300  bool FinishLexStringLiteral(Token &Result, std::string &String,
1301  const char *DiagnosticTag,
1302  bool AllowMacroExpansion);
1303 
1304  /// Lex a token. If it's a comment, keep lexing until we get
1305  /// something not a comment.
1306  ///
1307  /// This is useful in -E -C mode where comments would foul up preprocessor
1308  /// directive handling.
1309  void LexNonComment(Token &Result) {
1310  do
1311  Lex(Result);
1312  while (Result.getKind() == tok::comment);
1313  }
1314 
1315  /// Just like Lex, but disables macro expansion of identifier tokens.
1316  void LexUnexpandedToken(Token &Result) {
1317  // Disable macro expansion.
1318  bool OldVal = DisableMacroExpansion;
1319  DisableMacroExpansion = true;
1320  // Lex the token.
1321  Lex(Result);
1322 
1323  // Reenable it.
1324  DisableMacroExpansion = OldVal;
1325  }
1326 
1327  /// Like LexNonComment, but this disables macro expansion of
1328  /// identifier tokens.
1330  do
1331  LexUnexpandedToken(Result);
1332  while (Result.getKind() == tok::comment);
1333  }
1334 
1335  /// Parses a simple integer literal to get its numeric value. Floating
1336  /// point literals and user defined literals are rejected. Used primarily to
1337  /// handle pragmas that accept integer arguments.
1338  bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1339 
1340  /// Disables macro expansion everywhere except for preprocessor directives.
1342  DisableMacroExpansion = true;
1343  MacroExpansionInDirectivesOverride = true;
1344  }
1345 
1346  /// Peeks ahead N tokens and returns that token without consuming any
1347  /// tokens.
1348  ///
1349  /// LookAhead(0) returns the next token that would be returned by Lex(),
1350  /// LookAhead(1) returns the token after it, etc. This returns normal
1351  /// tokens after phase 5. As such, it is equivalent to using
1352  /// 'Lex', not 'LexUnexpandedToken'.
1353  const Token &LookAhead(unsigned N) {
1354  if (CachedLexPos + N < CachedTokens.size())
1355  return CachedTokens[CachedLexPos+N];
1356  else
1357  return PeekAhead(N+1);
1358  }
1359 
1360  /// When backtracking is enabled and tokens are cached,
1361  /// this allows to revert a specific number of tokens.
1362  ///
1363  /// Note that the number of tokens being reverted should be up to the last
1364  /// backtrack position, not more.
1365  void RevertCachedTokens(unsigned N) {
1366  assert(isBacktrackEnabled() &&
1367  "Should only be called when tokens are cached for backtracking");
1368  assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1369  && "Should revert tokens up to the last backtrack position, not more");
1370  assert(signed(CachedLexPos) - signed(N) >= 0 &&
1371  "Corrupted backtrack positions ?");
1372  CachedLexPos -= N;
1373  }
1374 
1375  /// Enters a token in the token stream to be lexed next.
1376  ///
1377  /// If BackTrack() is called afterwards, the token will remain at the
1378  /// insertion point.
1379  void EnterToken(const Token &Tok) {
1380  EnterCachingLexMode();
1381  CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1382  }
1383 
1384  /// We notify the Preprocessor that if it is caching tokens (because
1385  /// backtrack is enabled) it should replace the most recent cached tokens
1386  /// with the given annotation token. This function has no effect if
1387  /// backtracking is not enabled.
1388  ///
1389  /// Note that the use of this function is just for optimization, so that the
1390  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1391  /// invoked.
1392  void AnnotateCachedTokens(const Token &Tok) {
1393  assert(Tok.isAnnotation() && "Expected annotation token");
1394  if (CachedLexPos != 0 && isBacktrackEnabled())
1395  AnnotatePreviousCachedTokens(Tok);
1396  }
1397 
1398  /// Get the location of the last cached token, suitable for setting the end
1399  /// location of an annotation token.
1401  assert(CachedLexPos != 0);
1402  return CachedTokens[CachedLexPos-1].getLastLoc();
1403  }
1404 
1405  /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1406  /// CachedTokens.
1407  bool IsPreviousCachedToken(const Token &Tok) const;
1408 
1409  /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1410  /// in \p NewToks.
1411  ///
1412  /// Useful when a token needs to be split in smaller ones and CachedTokens
1413  /// most recent token must to be updated to reflect that.
1414  void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1415 
1416  /// Replace the last token with an annotation token.
1417  ///
1418  /// Like AnnotateCachedTokens(), this routine replaces an
1419  /// already-parsed (and resolved) token with an annotation
1420  /// token. However, this routine only replaces the last token with
1421  /// the annotation token; it does not affect any other cached
1422  /// tokens. This function has no effect if backtracking is not
1423  /// enabled.
1425  assert(Tok.isAnnotation() && "Expected annotation token");
1426  if (CachedLexPos != 0 && isBacktrackEnabled())
1427  CachedTokens[CachedLexPos-1] = Tok;
1428  }
1429 
1430  /// Enter an annotation token into the token stream.
1431  void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1432  void *AnnotationVal);
1433 
1434  /// Update the current token to represent the provided
1435  /// identifier, in order to cache an action performed by typo correction.
1436  void TypoCorrectToken(const Token &Tok) {
1437  assert(Tok.getIdentifierInfo() && "Expected identifier token");
1438  if (CachedLexPos != 0 && isBacktrackEnabled())
1439  CachedTokens[CachedLexPos-1] = Tok;
1440  }
1441 
1442  /// Recompute the current lexer kind based on the CurLexer/
1443  /// CurTokenLexer pointers.
1444  void recomputeCurLexerKind();
1445 
1446  /// Returns true if incremental processing is enabled
1447  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1448 
1449  /// Enables the incremental processing
1450  void enableIncrementalProcessing(bool value = true) {
1451  IncrementalProcessing = value;
1452  }
1453 
1454  /// Specify the point at which code-completion will be performed.
1455  ///
1456  /// \param File the file in which code completion should occur. If
1457  /// this file is included multiple times, code-completion will
1458  /// perform completion the first time it is included. If NULL, this
1459  /// function clears out the code-completion point.
1460  ///
1461  /// \param Line the line at which code completion should occur
1462  /// (1-based).
1463  ///
1464  /// \param Column the column at which code completion should occur
1465  /// (1-based).
1466  ///
1467  /// \returns true if an error occurred, false otherwise.
1468  bool SetCodeCompletionPoint(const FileEntry *File,
1469  unsigned Line, unsigned Column);
1470 
1471  /// Determine if we are performing code completion.
1472  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1473 
1474  /// Returns the location of the code-completion point.
1475  ///
1476  /// Returns an invalid location if code-completion is not enabled or the file
1477  /// containing the code-completion point has not been lexed yet.
1478  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1479 
1480  /// Returns the start location of the file of code-completion point.
1481  ///
1482  /// Returns an invalid location if code-completion is not enabled or the file
1483  /// containing the code-completion point has not been lexed yet.
1485  return CodeCompletionFileLoc;
1486  }
1487 
1488  /// Returns true if code-completion is enabled and we have hit the
1489  /// code-completion point.
1490  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1491 
1492  /// Note that we hit the code-completion point.
1494  assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1495  CodeCompletionReached = true;
1496  // Silence any diagnostics that occur after we hit the code-completion.
1497  getDiagnostics().setSuppressAllDiagnostics(true);
1498  }
1499 
1500  /// The location of the currently-active \#pragma clang
1501  /// arc_cf_code_audited begin.
1502  ///
1503  /// Returns an invalid location if there is no such pragma active.
1505  return PragmaARCCFCodeAuditedLoc;
1506  }
1507 
1508  /// Set the location of the currently-active \#pragma clang
1509  /// arc_cf_code_audited begin. An invalid location ends the pragma.
1511  PragmaARCCFCodeAuditedLoc = Loc;
1512  }
1513 
1514  /// The location of the currently-active \#pragma clang
1515  /// assume_nonnull begin.
1516  ///
1517  /// Returns an invalid location if there is no such pragma active.
1519  return PragmaAssumeNonNullLoc;
1520  }
1521 
1522  /// Set the location of the currently-active \#pragma clang
1523  /// assume_nonnull begin. An invalid location ends the pragma.
1525  PragmaAssumeNonNullLoc = Loc;
1526  }
1527 
1528  /// Set the directory in which the main file should be considered
1529  /// to have been found, if it is not a real file.
1530  void setMainFileDir(const DirectoryEntry *Dir) {
1531  MainFileDir = Dir;
1532  }
1533 
1534  /// Instruct the preprocessor to skip part of the main source file.
1535  ///
1536  /// \param Bytes The number of bytes in the preamble to skip.
1537  ///
1538  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1539  /// start of a line.
1540  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1541  SkipMainFilePreamble.first = Bytes;
1542  SkipMainFilePreamble.second = StartOfLine;
1543  }
1544 
1545  /// Forwarding function for diagnostics. This emits a diagnostic at
1546  /// the specified Token's location, translating the token's start
1547  /// position in the current buffer into a SourcePosition object for rendering.
1548  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1549  return Diags->Report(Loc, DiagID);
1550  }
1551 
1552  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1553  return Diags->Report(Tok.getLocation(), DiagID);
1554  }
1555 
1556  /// Return the 'spelling' of the token at the given
1557  /// location; does not go up to the spelling location or down to the
1558  /// expansion location.
1559  ///
1560  /// \param buffer A buffer which will be used only if the token requires
1561  /// "cleaning", e.g. if it contains trigraphs or escaped newlines
1562  /// \param invalid If non-null, will be set \c true if an error occurs.
1564  SmallVectorImpl<char> &buffer,
1565  bool *invalid = nullptr) const {
1566  return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1567  }
1568 
1569  /// Return the 'spelling' of the Tok token.
1570  ///
1571  /// The spelling of a token is the characters used to represent the token in
1572  /// the source file after trigraph expansion and escaped-newline folding. In
1573  /// particular, this wants to get the true, uncanonicalized, spelling of
1574  /// things like digraphs, UCNs, etc.
1575  ///
1576  /// \param Invalid If non-null, will be set \c true if an error occurs.
1577  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1578  return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1579  }
1580 
1581  /// Get the spelling of a token into a preallocated buffer, instead
1582  /// of as an std::string.
1583  ///
1584  /// The caller is required to allocate enough space for the token, which is
1585  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1586  /// actual result is returned.
1587  ///
1588  /// Note that this method may do two possible things: it may either fill in
1589  /// the buffer specified with characters, or it may *change the input pointer*
1590  /// to point to a constant buffer with the data already in it (avoiding a
1591  /// copy). The caller is not allowed to modify the returned buffer pointer
1592  /// if an internal buffer is returned.
1593  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1594  bool *Invalid = nullptr) const {
1595  return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1596  }
1597 
1598  /// Get the spelling of a token into a SmallVector.
1599  ///
1600  /// Note that the returned StringRef may not point to the
1601  /// supplied buffer if a copy can be avoided.
1602  StringRef getSpelling(const Token &Tok,
1603  SmallVectorImpl<char> &Buffer,
1604  bool *Invalid = nullptr) const;
1605 
1606  /// Relex the token at the specified location.
1607  /// \returns true if there was a failure, false on success.
1608  bool getRawToken(SourceLocation Loc, Token &Result,
1609  bool IgnoreWhiteSpace = false) {
1610  return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1611  }
1612 
1613  /// Given a Token \p Tok that is a numeric constant with length 1,
1614  /// return the character.
1615  char
1617  bool *Invalid = nullptr) const {
1618  assert(Tok.is(tok::numeric_constant) &&
1619  Tok.getLength() == 1 && "Called on unsupported token");
1620  assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1621 
1622  // If the token is carrying a literal data pointer, just use it.
1623  if (const char *D = Tok.getLiteralData())
1624  return *D;
1625 
1626  // Otherwise, fall back on getCharacterData, which is slower, but always
1627  // works.
1628  return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1629  }
1630 
1631  /// Retrieve the name of the immediate macro expansion.
1632  ///
1633  /// This routine starts from a source location, and finds the name of the
1634  /// macro responsible for its immediate expansion. It looks through any
1635  /// intervening macro argument expansions to compute this. It returns a
1636  /// StringRef that refers to the SourceManager-owned buffer of the source
1637  /// where that macro name is spelled. Thus, the result shouldn't out-live
1638  /// the SourceManager.
1640  return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1641  }
1642 
1643  /// Plop the specified string into a scratch buffer and set the
1644  /// specified token's location and length to it.
1645  ///
1646  /// If specified, the source location provides a location of the expansion
1647  /// point of the token.
1648  void CreateString(StringRef Str, Token &Tok,
1649  SourceLocation ExpansionLocStart = SourceLocation(),
1650  SourceLocation ExpansionLocEnd = SourceLocation());
1651 
1652  /// Split the first Length characters out of the token starting at TokLoc
1653  /// and return a location pointing to the split token. Re-lexing from the
1654  /// split token will return the split token rather than the original.
1655  SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
1656 
1657  /// Computes the source location just past the end of the
1658  /// token at this source location.
1659  ///
1660  /// This routine can be used to produce a source location that
1661  /// points just past the end of the token referenced by \p Loc, and
1662  /// is generally used when a diagnostic needs to point just after a
1663  /// token where it expected something different that it received. If
1664  /// the returned source location would not be meaningful (e.g., if
1665  /// it points into a macro), this routine returns an invalid
1666  /// source location.
1667  ///
1668  /// \param Offset an offset from the end of the token, where the source
1669  /// location should refer to. The default offset (0) produces a source
1670  /// location pointing just past the end of the token; an offset of 1 produces
1671  /// a source location pointing to the last character in the token, etc.
1673  return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1674  }
1675 
1676  /// Returns true if the given MacroID location points at the first
1677  /// token of the macro expansion.
1678  ///
1679  /// \param MacroBegin If non-null and function returns true, it is set to
1680  /// begin location of the macro.
1682  SourceLocation *MacroBegin = nullptr) const {
1683  return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1684  MacroBegin);
1685  }
1686 
1687  /// Returns true if the given MacroID location points at the last
1688  /// token of the macro expansion.
1689  ///
1690  /// \param MacroEnd If non-null and function returns true, it is set to
1691  /// end location of the macro.
1693  SourceLocation *MacroEnd = nullptr) const {
1694  return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1695  }
1696 
1697  /// Print the token to stderr, used for debugging.
1698  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1699  void DumpLocation(SourceLocation Loc) const;
1700  void DumpMacro(const MacroInfo &MI) const;
1701  void dumpMacroInfo(const IdentifierInfo *II);
1702 
1703  /// Given a location that specifies the start of a
1704  /// token, return a new location that specifies a character within the token.
1706  unsigned Char) const {
1707  return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1708  }
1709 
1710  /// Increment the counters for the number of token paste operations
1711  /// performed.
1712  ///
1713  /// If fast was specified, this is a 'fast paste' case we handled.
1714  void IncrementPasteCounter(bool isFast) {
1715  if (isFast)
1716  ++NumFastTokenPaste;
1717  else
1718  ++NumTokenPaste;
1719  }
1720 
1721  void PrintStats();
1722 
1723  size_t getTotalMemory() const;
1724 
1725  /// When the macro expander pastes together a comment (/##/) in Microsoft
1726  /// mode, this method handles updating the current state, returning the
1727  /// token on the next source line.
1728  void HandleMicrosoftCommentPaste(Token &Tok);
1729 
1730  //===--------------------------------------------------------------------===//
1731  // Preprocessor callback methods. These are invoked by a lexer as various
1732  // directives and events are found.
1733 
1734  /// Given a tok::raw_identifier token, look up the
1735  /// identifier information for the token and install it into the token,
1736  /// updating the token kind accordingly.
1737  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1738 
1739 private:
1740  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1741 
1742 public:
1743  /// Specifies the reason for poisoning an identifier.
1744  ///
1745  /// If that identifier is accessed while poisoned, then this reason will be
1746  /// used instead of the default "poisoned" diagnostic.
1747  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1748 
1749  /// Display reason for poisoned identifier.
1750  void HandlePoisonedIdentifier(Token & Identifier);
1751 
1753  if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1754  if(II->isPoisoned()) {
1755  HandlePoisonedIdentifier(Identifier);
1756  }
1757  }
1758  }
1759 
1760 private:
1761  /// Identifiers used for SEH handling in Borland. These are only
1762  /// allowed in particular circumstances
1763  // __except block
1764  IdentifierInfo *Ident__exception_code,
1765  *Ident___exception_code,
1766  *Ident_GetExceptionCode;
1767  // __except filter expression
1768  IdentifierInfo *Ident__exception_info,
1769  *Ident___exception_info,
1770  *Ident_GetExceptionInfo;
1771  // __finally
1772  IdentifierInfo *Ident__abnormal_termination,
1773  *Ident___abnormal_termination,
1774  *Ident_AbnormalTermination;
1775 
1776  const char *getCurLexerEndPos();
1777  void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
1778 
1779 public:
1780  void PoisonSEHIdentifiers(bool Poison = true); // Borland
1781 
1782  /// Callback invoked when the lexer reads an identifier and has
1783  /// filled in the tokens IdentifierInfo member.
1784  ///
1785  /// This callback potentially macro expands it or turns it into a named
1786  /// token (like 'for').
1787  ///
1788  /// \returns true if we actually computed a token, false if we need to
1789  /// lex again.
1790  bool HandleIdentifier(Token &Identifier);
1791 
1792  /// Callback invoked when the lexer hits the end of the current file.
1793  ///
1794  /// This either returns the EOF token and returns true, or
1795  /// pops a level off the include stack and returns false, at which point the
1796  /// client should call lex again.
1797  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1798 
1799  /// Callback invoked when the current TokenLexer hits the end of its
1800  /// token stream.
1801  bool HandleEndOfTokenLexer(Token &Result);
1802 
1803  /// Callback invoked when the lexer sees a # token at the start of a
1804  /// line.
1805  ///
1806  /// This consumes the directive, modifies the lexer/preprocessor state, and
1807  /// advances the lexer(s) so that the next token read is the correct one.
1808  void HandleDirective(Token &Result);
1809 
1810  /// Ensure that the next token is a tok::eod token.
1811  ///
1812  /// If not, emit a diagnostic and consume up until the eod.
1813  /// If \p EnableMacros is true, then we consider macros that expand to zero
1814  /// tokens as being ok.
1815  void CheckEndOfDirective(const char *DirType, bool EnableMacros = false);
1816 
1817  /// Read and discard all tokens remaining on the current line until
1818  /// the tok::eod token is found. Returns the range of the skipped tokens.
1819  SourceRange DiscardUntilEndOfDirective();
1820 
1821  /// Returns true if the preprocessor has seen a use of
1822  /// __DATE__ or __TIME__ in the file so far.
1823  bool SawDateOrTime() const {
1824  return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
1825  }
1826  unsigned getCounterValue() const { return CounterValue; }
1827  void setCounterValue(unsigned V) { CounterValue = V; }
1828 
1829  /// Retrieves the module that we're currently building, if any.
1830  Module *getCurrentModule();
1831 
1832  /// Allocate a new MacroInfo object with the provided SourceLocation.
1833  MacroInfo *AllocateMacroInfo(SourceLocation L);
1834 
1835  /// Turn the specified lexer token into a fully checked and spelled
1836  /// filename, e.g. as an operand of \#include.
1837  ///
1838  /// The caller is expected to provide a buffer that is large enough to hold
1839  /// the spelling of the filename, but is also expected to handle the case
1840  /// when this method decides to use a different buffer.
1841  ///
1842  /// \returns true if the input filename was in <>'s or false if it was
1843  /// in ""'s.
1844  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
1845 
1846  /// Given a "foo" or <foo> reference, look up the indicated file.
1847  ///
1848  /// Returns null on failure. \p isAngled indicates whether the file
1849  /// reference is for system \#include's or not (i.e. using <> instead of "").
1850  const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename,
1851  bool isAngled, const DirectoryLookup *FromDir,
1852  const FileEntry *FromFile,
1853  const DirectoryLookup *&CurDir,
1854  SmallVectorImpl<char> *SearchPath,
1855  SmallVectorImpl<char> *RelativePath,
1856  ModuleMap::KnownHeader *SuggestedModule,
1857  bool *IsMapped, bool SkipCache = false);
1858 
1859  /// Get the DirectoryLookup structure used to find the current
1860  /// FileEntry, if CurLexer is non-null and if applicable.
1861  ///
1862  /// This allows us to implement \#include_next and find directory-specific
1863  /// properties.
1864  const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1865 
1866  /// Return true if we're in the top-level file, not in a \#include.
1867  bool isInPrimaryFile() const;
1868 
1869  /// Handle cases where the \#include name is expanded
1870  /// from a macro as multiple tokens, which need to be glued together.
1871  ///
1872  /// This occurs for code like:
1873  /// \code
1874  /// \#define FOO <x/y.h>
1875  /// \#include FOO
1876  /// \endcode
1877  /// because in this case, "<x/y.h>" is returned as 7 tokens, not one.
1878  ///
1879  /// This code concatenates and consumes tokens up to the '>' token. It
1880  /// returns false if the > was found, otherwise it returns true if it finds
1881  /// and consumes the EOD marker.
1882  bool ConcatenateIncludeName(SmallString<128> &FilenameBuffer,
1883  SourceLocation &End);
1884 
1885  /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1886  /// followed by EOD. Return true if the token is not a valid on-off-switch.
1887  bool LexOnOffSwitch(tok::OnOffSwitch &Result);
1888 
1889  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
1890  bool *ShadowFlag = nullptr);
1891 
1892  void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
1893  Module *LeaveSubmodule(bool ForPragma);
1894 
1895 private:
1896  friend void TokenLexer::ExpandFunctionArguments();
1897 
1898  void PushIncludeMacroStack() {
1899  assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
1900  IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
1901  std::move(CurLexer), CurPPLexer,
1902  std::move(CurTokenLexer), CurDirLookup);
1903  CurPPLexer = nullptr;
1904  }
1905 
1906  void PopIncludeMacroStack() {
1907  CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1908  CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1909  CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1910  CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1911  CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
1912  CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1913  IncludeMacroStack.pop_back();
1914  }
1915 
1916  void PropagateLineStartLeadingSpaceInfo(Token &Result);
1917 
1918  /// Determine whether we need to create module macros for #defines in the
1919  /// current context.
1920  bool needModuleMacros() const;
1921 
1922  /// Update the set of active module macros and ambiguity flag for a module
1923  /// macro name.
1924  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
1925 
1926  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
1927  SourceLocation Loc);
1928  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
1929  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
1930  bool isPublic);
1931 
1932  /// Lex and validate a macro name, which occurs after a
1933  /// \#define or \#undef.
1934  ///
1935  /// \param MacroNameTok Token that represents the name defined or undefined.
1936  /// \param IsDefineUndef Kind if preprocessor directive.
1937  /// \param ShadowFlag Points to flag that is set if macro name shadows
1938  /// a keyword.
1939  ///
1940  /// This emits a diagnostic, sets the token kind to eod,
1941  /// and discards the rest of the macro line if the macro name is invalid.
1942  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
1943  bool *ShadowFlag = nullptr);
1944 
1945  /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
1946  /// entire line) of the macro's tokens and adds them to MacroInfo, and while
1947  /// doing so performs certain validity checks including (but not limited to):
1948  /// - # (stringization) is followed by a macro parameter
1949  /// \param MacroNameTok - Token that represents the macro name
1950  /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
1951  ///
1952  /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
1953  /// returns a nullptr if an invalid sequence of tokens is encountered.
1954  MacroInfo *ReadOptionalMacroParameterListAndBody(
1955  const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
1956 
1957  /// The ( starting an argument list of a macro definition has just been read.
1958  /// Lex the rest of the parameters and the closing ), updating \p MI with
1959  /// what we learn and saving in \p LastTok the last token read.
1960  /// Return true if an error occurs parsing the arg list.
1961  bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
1962 
1963  /// We just read a \#if or related directive and decided that the
1964  /// subsequent tokens are in the \#if'd out portion of the
1965  /// file. Lex the rest of the file, until we see an \#endif. If \p
1966  /// FoundNonSkipPortion is true, then we have already emitted code for part of
1967  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
1968  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
1969  /// already seen one so a \#else directive is a duplicate. When this returns,
1970  /// the caller can lex the first valid token.
1971  void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
1972  SourceLocation IfTokenLoc,
1973  bool FoundNonSkipPortion, bool FoundElse,
1974  SourceLocation ElseLoc = SourceLocation());
1975 
1976  /// Information about the result for evaluating an expression for a
1977  /// preprocessor directive.
1978  struct DirectiveEvalResult {
1979  /// Whether the expression was evaluated as true or not.
1980  bool Conditional;
1981 
1982  /// True if the expression contained identifiers that were undefined.
1983  bool IncludedUndefinedIds;
1984 
1985  /// The source range for the expression.
1986  SourceRange ExprRange;
1987  };
1988 
1989  /// Evaluate an integer constant expression that may occur after a
1990  /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
1991  ///
1992  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
1993  DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1994 
1995  /// Install the standard preprocessor pragmas:
1996  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
1997  void RegisterBuiltinPragmas();
1998 
1999  /// Register builtin macros such as __LINE__ with the identifier table.
2000  void RegisterBuiltinMacros();
2001 
2002  /// If an identifier token is read that is to be expanded as a macro, handle
2003  /// it and return the next token as 'Tok'. If we lexed a token, return true;
2004  /// otherwise the caller should lex again.
2005  bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2006 
2007  /// Cache macro expanded tokens for TokenLexers.
2008  //
2009  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2010  /// going to lex in the cache and when it finishes the tokens are removed
2011  /// from the end of the cache.
2012  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2013  ArrayRef<Token> tokens);
2014 
2015  void removeCachedMacroExpandedTokensOfLastLexer();
2016 
2017  /// Determine whether the next preprocessor token to be
2018  /// lexed is a '('. If so, consume the token and return true, if not, this
2019  /// method should have no observable side-effect on the lexed tokens.
2020  bool isNextPPTokenLParen();
2021 
2022  /// After reading "MACRO(", this method is invoked to read all of the formal
2023  /// arguments specified for the macro invocation. Returns null on error.
2024  MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2025  SourceLocation &MacroEnd);
2026 
2027  /// If an identifier token is read that is to be expanded
2028  /// as a builtin macro, handle it and return the next token as 'Tok'.
2029  void ExpandBuiltinMacro(Token &Tok);
2030 
2031  /// Read a \c _Pragma directive, slice it up, process it, then
2032  /// return the first token after the directive.
2033  /// This assumes that the \c _Pragma token has just been read into \p Tok.
2034  void Handle_Pragma(Token &Tok);
2035 
2036  /// Like Handle_Pragma except the pragma text is not enclosed within
2037  /// a string literal.
2038  void HandleMicrosoft__pragma(Token &Tok);
2039 
2040  /// Add a lexer to the top of the include stack and
2041  /// start lexing tokens from it instead of the current buffer.
2042  void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
2043 
2044  /// Set the FileID for the preprocessor predefines.
2045  void setPredefinesFileID(FileID FID) {
2046  assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2047  PredefinesFileID = FID;
2048  }
2049 
2050  /// Set the FileID for the PCH through header.
2051  void setPCHThroughHeaderFileID(FileID FID);
2052 
2053  /// Returns true if we are lexing from a file and not a
2054  /// pragma or a macro.
2055  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2056  return L ? !L->isPragmaLexer() : P != nullptr;
2057  }
2058 
2059  static bool IsFileLexer(const IncludeStackInfo& I) {
2060  return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2061  }
2062 
2063  bool IsFileLexer() const {
2064  return IsFileLexer(CurLexer.get(), CurPPLexer);
2065  }
2066 
2067  //===--------------------------------------------------------------------===//
2068  // Caching stuff.
2069  void CachingLex(Token &Result);
2070 
2071  bool InCachingLexMode() const {
2072  // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2073  // that we are past EOF, not that we are in CachingLex mode.
2074  return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2075  }
2076 
2077  void EnterCachingLexMode();
2078 
2079  void ExitCachingLexMode() {
2080  if (InCachingLexMode())
2081  RemoveTopOfLexerStack();
2082  }
2083 
2084  const Token &PeekAhead(unsigned N);
2085  void AnnotatePreviousCachedTokens(const Token &Tok);
2086 
2087  //===--------------------------------------------------------------------===//
2088  /// Handle*Directive - implement the various preprocessor directives. These
2089  /// should side-effect the current preprocessor object so that the next call
2090  /// to Lex() will return the appropriate token next.
2091  void HandleLineDirective();
2092  void HandleDigitDirective(Token &Tok);
2093  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2094  void HandleIdentSCCSDirective(Token &Tok);
2095  void HandleMacroPublicDirective(Token &Tok);
2096  void HandleMacroPrivateDirective();
2097 
2098  // File inclusion.
2099  void HandleIncludeDirective(SourceLocation HashLoc,
2100  Token &Tok,
2101  const DirectoryLookup *LookupFrom = nullptr,
2102  const FileEntry *LookupFromFile = nullptr,
2103  bool isImport = false);
2104  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2105  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2106  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2107  void HandleMicrosoftImportDirective(Token &Tok);
2108 
2109 public:
2110  /// Check that the given module is available, producing a diagnostic if not.
2111  /// \return \c true if the check failed (because the module is not available).
2112  /// \c false if the module appears to be usable.
2113  static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2114  const TargetInfo &TargetInfo,
2115  DiagnosticsEngine &Diags, Module *M);
2116 
2117  // Module inclusion testing.
2118  /// Find the module that owns the source or header file that
2119  /// \p Loc points to. If the location is in a file that was included
2120  /// into a module, or is outside any module, returns nullptr.
2121  Module *getModuleForLocation(SourceLocation Loc);
2122 
2123  /// We want to produce a diagnostic at location IncLoc concerning a
2124  /// missing module import.
2125  ///
2126  /// \param IncLoc The location at which the missing import was detected.
2127  /// \param M The desired module.
2128  /// \param MLoc A location within the desired module at which some desired
2129  /// effect occurred (eg, where a desired entity was declared).
2130  ///
2131  /// \return A file that can be #included to import a module containing MLoc.
2132  /// Null if no such file could be determined or if a #include is not
2133  /// appropriate.
2134  const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2135  Module *M,
2136  SourceLocation MLoc);
2137 
2138  bool isRecordingPreamble() const {
2139  return PreambleConditionalStack.isRecording();
2140  }
2141 
2142  bool hasRecordedPreamble() const {
2143  return PreambleConditionalStack.hasRecordedPreamble();
2144  }
2145 
2147  return PreambleConditionalStack.getStack();
2148  }
2149 
2151  PreambleConditionalStack.setStack(s);
2152  }
2153 
2156  PreambleConditionalStack.startReplaying();
2157  PreambleConditionalStack.setStack(s);
2158  PreambleConditionalStack.SkipInfo = SkipInfo;
2159  }
2160 
2162  return PreambleConditionalStack.SkipInfo;
2163  }
2164 
2165 private:
2166  /// After processing predefined file, initialize the conditional stack from
2167  /// the preamble.
2168  void replayPreambleConditionalStack();
2169 
2170  // Macro handling.
2171  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2172  void HandleUndefDirective();
2173 
2174  // Conditional Inclusion.
2175  void HandleIfdefDirective(Token &Result, const Token &HashToken,
2176  bool isIfndef, bool ReadAnyTokensBeforeDirective);
2177  void HandleIfDirective(Token &IfToken, const Token &HashToken,
2178  bool ReadAnyTokensBeforeDirective);
2179  void HandleEndifDirective(Token &EndifToken);
2180  void HandleElseDirective(Token &Result, const Token &HashToken);
2181  void HandleElifDirective(Token &ElifToken, const Token &HashToken);
2182 
2183  // Pragmas.
2184  void HandlePragmaDirective(SourceLocation IntroducerLoc,
2185  PragmaIntroducerKind Introducer);
2186 
2187 public:
2188  void HandlePragmaOnce(Token &OnceTok);
2189  void HandlePragmaMark();
2190  void HandlePragmaPoison();
2191  void HandlePragmaSystemHeader(Token &SysHeaderTok);
2192  void HandlePragmaDependency(Token &DependencyTok);
2193  void HandlePragmaPushMacro(Token &Tok);
2194  void HandlePragmaPopMacro(Token &Tok);
2195  void HandlePragmaIncludeAlias(Token &Tok);
2196  void HandlePragmaModuleBuild(Token &Tok);
2197  void HandlePragmaHdrstop(Token &Tok);
2198  IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2199 
2200  // Return true and store the first token only if any CommentHandler
2201  // has inserted some tokens and getCommentRetentionState() is false.
2202  bool HandleComment(Token &result, SourceRange Comment);
2203 
2204  /// A macro is used, update information about macros that need unused
2205  /// warnings.
2206  void markMacroAsUsed(MacroInfo *MI);
2207 };
2208 
2209 /// Abstract base class that describes a handler that will receive
2210 /// source ranges for each of the comments encountered in the source file.
2212 public:
2213  virtual ~CommentHandler();
2214 
2215  // The handler shall return true if it has pushed any tokens
2216  // to be read using e.g. EnterToken or EnterTokenStream.
2217  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2218 };
2219 
2220 /// Registry of pragma handlers added by plugins
2221 using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2222 
2223 } // namespace clang
2224 
2225 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
Definition: Preprocessor.h:333
A set of visible modules.
Definition: Module.h:588
StringRef Identifier
Definition: Format.cpp:1635
llvm::BumpPtrAllocator & getPreprocessorAllocator()
Definition: Preprocessor.h:826
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:76
SelectorTable & getSelectorTable()
Definition: Preprocessor.h:824
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:67
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:121
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the &#39;spelling&#39; of the Tok token.
MemoryBufferCache & getPCMCache() const
Definition: Preprocessor.h:819
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:29
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:94
Defines the SourceManager interface.
static const Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:20
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
FileManager & getFileManager() const
Definition: Preprocessor.h:817
Defines the clang::Module class, which describes a module in the source code.
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
SourceLocation getPragmaARCCFCodeAuditedLoc() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
StringRef P
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:564
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1294
A directive for an undefined macro.
Definition: MacroInfo.h:429
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool hadModuleLoaderFatalFailure() const
Definition: Preprocessor.h:839
void EnterToken(const Token &Tok)
Enters a token in the token stream to be lexed next.
Manage memory buffers across multiple users.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:100
Builtin::Context & getBuiltinInfo()
Definition: Preprocessor.h:825
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:35
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:117
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Definition: MacroInfo.h:328
tok::TokenKind getKind() const
Definition: Token.h:89
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:90
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
One of these records is kept for each identifier that is lexed.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
Represents a macro directive exported by a module.
Definition: MacroInfo.h:488
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:406
This table allows us to fully hide how we implement multi-keyword caching.
LineState State
Definition: Format.h:2071
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:815
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Definition: Preprocessor.h:849
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:49
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:814
Describes a module or submodule.
Definition: Module.h:64
bool isPragmaLexer() const
isPragmaLexer - Returns true if this Lexer is being used to lex a pragma.
Definition: Lexer.h:183
A directive for setting the module visibility of a macro.
Definition: MacroInfo.h:444
bool isInvalid() const
void SetSuppressIncludeNotFoundError(bool Suppress)
Definition: Preprocessor.h:859
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:107
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
Definition: Preprocessor.h:896
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:820
const FormatToken & Tok
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void LexNonComment(Token &Result)
Lex a token.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:148
bool isMacroDefined(const IdentifierInfo *II)
Definition: Preprocessor.h:918
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we&#39;ve alread...
Definition: Preprocessor.h:926
Defines the Diagnostic-related interfaces.
const TargetInfo * getAuxTargetInfo() const
Definition: Preprocessor.h:816
TokenLexer - This implements a lexer that returns tokens from a macro body or token stream instead of...
Definition: TokenLexer.h:30
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
Definition: Preprocessor.h:869
PragmaIntroducerKind
Describes how the pragma was introduced, e.g., with #pragma, _Pragma, or __pragma.
Definition: Pragma.h:31
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:147
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:1042
void clearCodeCompletionHandler()
Clear out the code completion handler.
StringRef Filename
Definition: Format.cpp:1628
Provides lookups to, and iteration over, IdentiferInfo objects.
unsigned Offset
Definition: Format.cpp:1630
Exposes information about the current target.
Definition: TargetInfo.h:53
const IdentifierTable & getIdentifierTable() const
Definition: Preprocessor.h:823
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Abstract interface for external sources of preprocessor information.
Defines the clang::LangOptions interface.
unsigned getCounterValue() const
SourceLocation End
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:349
int Id
Definition: ASTDiff.cpp:190
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
const AnnotatedLine * Line
bool isDefined() const
Definition: MacroInfo.h:384
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Definition: Preprocessor.h:878
Implements an efficient mapping from strings to IdentifierInfo nodes.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
A class for tracking whether we&#39;re inside a VA_OPT during a traversal of the tokens of a variadic mac...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:123
#define bool
Definition: stdbool.h:31
DefInfo getDefinition()
Traverses the macro directives history and returns the next macro definition directive along with inf...
Definition: MacroInfo.cpp:178
SourceLocation Begin
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:85
llvm::Optional< PreambleSkipInfo > getPreambleSkipInfo() const
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
Definition: Preprocessor.h:980
bool getCommentRetentionState() const
Definition: Preprocessor.h:854
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:98
PPCallbacks * getPPCallbacks() const
Definition: Preprocessor.h:906
const SourceManager & SM
Definition: Format.cpp:1489
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
bool isRecordingPreamble() const
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
SourceManager & getSourceManager() const
Definition: Preprocessor.h:818
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define&#39;d and not #undef&#39;...
Definition: Preprocessor.h:969
const DefInfo findDirectiveAtLoc(SourceLocation L, const SourceManager &SM) const
Find macro definition active in the specified source location.
Definition: MacroInfo.cpp:202
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:832
llvm::cl::opt< std::string > Filter
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:290
Kind
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
void setExternalSource(ExternalPreprocessorSource *Source)
Definition: Preprocessor.h:828
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.h:93
void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:939
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:176
IdentifierTable & getIdentifierTable()
Definition: Preprocessor.h:822
void setPragmasEnabled(bool Enabled)
Definition: Preprocessor.h:856
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:85
bool isPoisoned() const
Return true if this token has been poisoned.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:24
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
Definition: Preprocessor.h:952
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
Definition: Preprocessor.h:886
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
const MacroInfo * getMacroInfo() const
Definition: MacroInfo.h:390
StringRef getName() const
Return the actual identifier string.
bool isMacroDefined(StringRef Id)
Definition: Preprocessor.h:915
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
Dataflow Directional Tag Classes.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion)
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
Definition: DeclSpec.h:1141
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion)
SourceRange getCodeCompletionTokenRange() const
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:354
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:58
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Abstract interface for a module loader.
Definition: ModuleLoader.h:73
unsigned getLength() const
Definition: Token.h:126
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:47
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:214
bool getPragmasEnabled() const
Definition: Preprocessor.h:857
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:123
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
Definition: Preprocessor.h:837
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Definition: Preprocessor.h:809
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:45
Defines the PPCallbacks interface.
Defines the clang::TokenKind enum and support functions.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
SourceLocation getModuleImportLoc(Module *M) const
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:899
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:811
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:361
bool hasRecordedPreamble() const
const std::string & getPredefines() const
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:282
void setPredefines(StringRef P)
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
The translation unit is a complete translation unit.
Definition: LangOptions.h:363
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
MacroMap::const_iterator macro_iterator
Abstract base class that describes a handler that will receive source ranges for each of the comments...
void setCounterValue(unsigned V)
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
Definition: Preprocessor.h:844
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
Definition: Preprocessor.h:863
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, llvm::Optional< PreambleSkipInfo > SkipInfo)
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Definition: Preprocessor.h:984
Callback handler that receives notifications when performing code completion within the preprocessor...
A header that is known to reside within a given module, whether it was included or excluded...
Definition: ModuleMap.h:150
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
Definition: Preprocessor.h:875
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:907
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:124
void setDiagnostics(DiagnosticsEngine &D)
Definition: Preprocessor.h:812