clang  9.0.0svn
Preprocessor.h
Go to the documentation of this file.
1 //===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines the clang::Preprocessor interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15 #define LLVM_CLANG_LEX_PREPROCESSOR_H
16 
17 #include "clang/Basic/Builtins.h"
18 #include "clang/Basic/Diagnostic.h"
20 #include "clang/Basic/LLVM.h"
22 #include "clang/Basic/Module.h"
25 #include "clang/Basic/TokenKinds.h"
26 #include "clang/Lex/Lexer.h"
27 #include "clang/Lex/MacroInfo.h"
28 #include "clang/Lex/ModuleLoader.h"
29 #include "clang/Lex/ModuleMap.h"
30 #include "clang/Lex/PPCallbacks.h"
31 #include "clang/Lex/Token.h"
32 #include "clang/Lex/TokenLexer.h"
33 #include "llvm/ADT/ArrayRef.h"
34 #include "llvm/ADT/DenseMap.h"
35 #include "llvm/ADT/FoldingSet.h"
36 #include "llvm/ADT/None.h"
37 #include "llvm/ADT/Optional.h"
38 #include "llvm/ADT/PointerUnion.h"
39 #include "llvm/ADT/STLExtras.h"
40 #include "llvm/ADT/SmallPtrSet.h"
41 #include "llvm/ADT/SmallVector.h"
42 #include "llvm/ADT/StringRef.h"
43 #include "llvm/ADT/TinyPtrVector.h"
44 #include "llvm/ADT/iterator_range.h"
45 #include "llvm/Support/Allocator.h"
46 #include "llvm/Support/Casting.h"
47 #include "llvm/Support/Registry.h"
48 #include <cassert>
49 #include <cstddef>
50 #include <cstdint>
51 #include <memory>
52 #include <map>
53 #include <string>
54 #include <utility>
55 #include <vector>
56 
57 namespace llvm {
58 
59 template<unsigned InternalLen> class SmallString;
60 
61 } // namespace llvm
62 
63 namespace clang {
64 
65 class CodeCompletionHandler;
66 class CommentHandler;
67 class DirectoryEntry;
68 class DirectoryLookup;
69 class ExternalPreprocessorSource;
70 class FileEntry;
71 class FileManager;
72 class HeaderSearch;
73 class MacroArgs;
74 class PragmaHandler;
75 class PragmaNamespace;
76 class PreprocessingRecord;
77 class PreprocessorLexer;
78 class PreprocessorOptions;
79 class ScratchBuffer;
80 class TargetInfo;
81 
82 /// Stores token information for comparing actual tokens with
83 /// predefined values. Only handles simple tokens and identifiers.
84 class TokenValue {
86  IdentifierInfo *II;
87 
88 public:
89  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
90  assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
91  assert(Kind != tok::identifier &&
92  "Identifiers should be created by TokenValue(IdentifierInfo *)");
93  assert(!tok::isLiteral(Kind) && "Literals are not supported.");
94  assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
95  }
96 
97  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
98 
99  bool operator==(const Token &Tok) const {
100  return Tok.getKind() == Kind &&
101  (!II || II == Tok.getIdentifierInfo());
102  }
103 };
104 
105 /// Context in which macro name is used.
106 enum MacroUse {
107  // other than #define or #undef
108  MU_Other = 0,
109 
110  // macro name specified in #define
112 
113  // macro name specified in #undef
115 };
116 
117 /// Engages in a tight little dance with the lexer to efficiently
118 /// preprocess tokens.
119 ///
120 /// Lexers know only about tokens within a single source file, and don't
121 /// know anything about preprocessor-level issues like the \#include stack,
122 /// token expansion, etc.
126 
127  std::shared_ptr<PreprocessorOptions> PPOpts;
128  DiagnosticsEngine *Diags;
129  LangOptions &LangOpts;
130  const TargetInfo *Target = nullptr;
131  const TargetInfo *AuxTarget = nullptr;
132  FileManager &FileMgr;
133  SourceManager &SourceMgr;
134  std::unique_ptr<ScratchBuffer> ScratchBuf;
135  HeaderSearch &HeaderInfo;
136  ModuleLoader &TheModuleLoader;
137 
138  /// External source of macros.
139  ExternalPreprocessorSource *ExternalSource;
140 
141  /// A BumpPtrAllocator object used to quickly allocate and release
142  /// objects internal to the Preprocessor.
143  llvm::BumpPtrAllocator BP;
144 
145  /// Identifiers for builtin macros and other builtins.
146  IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
147  IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
148  IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
149  IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
150  IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
151  IdentifierInfo *Ident__COUNTER__; // __COUNTER__
152  IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
153  IdentifierInfo *Ident__identifier; // __identifier
154  IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
155  IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
156  IdentifierInfo *Ident__has_feature; // __has_feature
157  IdentifierInfo *Ident__has_extension; // __has_extension
158  IdentifierInfo *Ident__has_builtin; // __has_builtin
159  IdentifierInfo *Ident__has_attribute; // __has_attribute
160  IdentifierInfo *Ident__has_include; // __has_include
161  IdentifierInfo *Ident__has_include_next; // __has_include_next
162  IdentifierInfo *Ident__has_warning; // __has_warning
163  IdentifierInfo *Ident__is_identifier; // __is_identifier
164  IdentifierInfo *Ident__building_module; // __building_module
165  IdentifierInfo *Ident__MODULE__; // __MODULE__
166  IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
167  IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
168  IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
169  IdentifierInfo *Ident__is_target_arch; // __is_target_arch
170  IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
171  IdentifierInfo *Ident__is_target_os; // __is_target_os
172  IdentifierInfo *Ident__is_target_environment; // __is_target_environment
173 
174  // Weak, only valid (and set) while InMacroArgs is true.
175  Token* ArgMacro;
176 
177  SourceLocation DATELoc, TIMELoc;
178 
179  // Next __COUNTER__ value, starts at 0.
180  unsigned CounterValue = 0;
181 
182  enum {
183  /// Maximum depth of \#includes.
184  MaxAllowedIncludeStackDepth = 200
185  };
186 
187  // State that is set before the preprocessor begins.
188  bool KeepComments : 1;
189  bool KeepMacroComments : 1;
190  bool SuppressIncludeNotFoundError : 1;
191 
192  // State that changes while the preprocessor runs:
193  bool InMacroArgs : 1; // True if parsing fn macro invocation args.
194 
195  /// Whether the preprocessor owns the header search object.
196  bool OwnsHeaderSearch : 1;
197 
198  /// True if macro expansion is disabled.
199  bool DisableMacroExpansion : 1;
200 
201  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
202  /// when parsing preprocessor directives.
203  bool MacroExpansionInDirectivesOverride : 1;
204 
206 
207  /// Whether we have already loaded macros from the external source.
208  mutable bool ReadMacrosFromExternalSource : 1;
209 
210  /// True if pragmas are enabled.
211  bool PragmasEnabled : 1;
212 
213  /// True if the current build action is a preprocessing action.
214  bool PreprocessedOutput : 1;
215 
216  /// True if we are currently preprocessing a #if or #elif directive
217  bool ParsingIfOrElifDirective;
218 
219  /// True if we are pre-expanding macro arguments.
220  bool InMacroArgPreExpansion;
221 
222  /// Mapping/lookup information for all identifiers in
223  /// the program, including program keywords.
224  mutable IdentifierTable Identifiers;
225 
226  /// This table contains all the selectors in the program.
227  ///
228  /// Unlike IdentifierTable above, this table *isn't* populated by the
229  /// preprocessor. It is declared/expanded here because its role/lifetime is
230  /// conceptually similar to the IdentifierTable. In addition, the current
231  /// control flow (in clang::ParseAST()), make it convenient to put here.
232  ///
233  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
234  /// the lifetime of the preprocessor.
235  SelectorTable Selectors;
236 
237  /// Information about builtins.
239 
240  /// Tracks all of the pragmas that the client registered
241  /// with this preprocessor.
242  std::unique_ptr<PragmaNamespace> PragmaHandlers;
243 
244  /// Pragma handlers of the original source is stored here during the
245  /// parsing of a model file.
246  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
247 
248  /// Tracks all of the comment handlers that the client registered
249  /// with this preprocessor.
250  std::vector<CommentHandler *> CommentHandlers;
251 
252  /// True if we want to ignore EOF token and continue later on (thus
253  /// avoid tearing the Lexer and etc. down).
254  bool IncrementalProcessing = false;
255 
256  /// The kind of translation unit we are processing.
257  TranslationUnitKind TUKind;
258 
259  /// The code-completion handler.
260  CodeCompletionHandler *CodeComplete = nullptr;
261 
262  /// The file that we're performing code-completion for, if any.
263  const FileEntry *CodeCompletionFile = nullptr;
264 
265  /// The offset in file for the code-completion point.
266  unsigned CodeCompletionOffset = 0;
267 
268  /// The location for the code-completion point. This gets instantiated
269  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
270  SourceLocation CodeCompletionLoc;
271 
272  /// The start location for the file of the code-completion point.
273  ///
274  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
275  /// for preprocessing.
276  SourceLocation CodeCompletionFileLoc;
277 
278  /// The source location of the \c import contextual keyword we just
279  /// lexed, if any.
280  SourceLocation ModuleImportLoc;
281 
282  /// The module import path that we're currently processing.
284 
285  /// Whether the last token we lexed was an '@'.
286  bool LastTokenWasAt = false;
287 
288  /// Whether the module import expects an identifier next. Otherwise,
289  /// it expects a '.' or ';'.
290  bool ModuleImportExpectsIdentifier = false;
291 
292  /// The source location of the currently-active
293  /// \#pragma clang arc_cf_code_audited begin.
294  SourceLocation PragmaARCCFCodeAuditedLoc;
295 
296  /// The source location of the currently-active
297  /// \#pragma clang assume_nonnull begin.
298  SourceLocation PragmaAssumeNonNullLoc;
299 
300  /// True if we hit the code-completion point.
301  bool CodeCompletionReached = false;
302 
303  /// The code completion token containing the information
304  /// on the stem that is to be code completed.
305  IdentifierInfo *CodeCompletionII = nullptr;
306 
307  /// Range for the code completion token.
308  SourceRange CodeCompletionTokenRange;
309 
310  /// The directory that the main file should be considered to occupy,
311  /// if it does not correspond to a real file (as happens when building a
312  /// module).
313  const DirectoryEntry *MainFileDir = nullptr;
314 
315  /// The number of bytes that we will initially skip when entering the
316  /// main file, along with a flag that indicates whether skipping this number
317  /// of bytes will place the lexer at the start of a line.
318  ///
319  /// This is used when loading a precompiled preamble.
320  std::pair<int, bool> SkipMainFilePreamble;
321 
322  /// Whether we hit an error due to reaching max allowed include depth. Allows
323  /// to avoid hitting the same error over and over again.
324  bool HasReachedMaxIncludeDepth = false;
325 
326 public:
331  bool FoundElse;
333 
335  bool FoundNonSkipPortion, bool FoundElse,
336  SourceLocation ElseLoc)
337  : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
338  FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
339  ElseLoc(ElseLoc) {}
340  };
341 
342 private:
343  friend class ASTReader;
344  friend class MacroArgs;
345 
346  class PreambleConditionalStackStore {
347  enum State {
348  Off = 0,
349  Recording = 1,
350  Replaying = 2,
351  };
352 
353  public:
354  PreambleConditionalStackStore() = default;
355 
356  void startRecording() { ConditionalStackState = Recording; }
357  void startReplaying() { ConditionalStackState = Replaying; }
358  bool isRecording() const { return ConditionalStackState == Recording; }
359  bool isReplaying() const { return ConditionalStackState == Replaying; }
360 
361  ArrayRef<PPConditionalInfo> getStack() const {
362  return ConditionalStack;
363  }
364 
365  void doneReplaying() {
366  ConditionalStack.clear();
367  ConditionalStackState = Off;
368  }
369 
370  void setStack(ArrayRef<PPConditionalInfo> s) {
371  if (!isRecording() && !isReplaying())
372  return;
373  ConditionalStack.clear();
374  ConditionalStack.append(s.begin(), s.end());
375  }
376 
377  bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
378 
379  bool reachedEOFWhileSkipping() const { return SkipInfo.hasValue(); }
380 
381  void clearSkipInfo() { SkipInfo.reset(); }
382 
384 
385  private:
386  SmallVector<PPConditionalInfo, 4> ConditionalStack;
387  State ConditionalStackState = Off;
388  } PreambleConditionalStack;
389 
390  /// The current top of the stack that we're lexing from if
391  /// not expanding a macro and we are lexing directly from source code.
392  ///
393  /// Only one of CurLexer, or CurTokenLexer will be non-null.
394  std::unique_ptr<Lexer> CurLexer;
395 
396  /// The current top of the stack what we're lexing from
397  /// if not expanding a macro.
398  ///
399  /// This is an alias for CurLexer.
400  PreprocessorLexer *CurPPLexer = nullptr;
401 
402  /// Used to find the current FileEntry, if CurLexer is non-null
403  /// and if applicable.
404  ///
405  /// This allows us to implement \#include_next and find directory-specific
406  /// properties.
407  const DirectoryLookup *CurDirLookup = nullptr;
408 
409  /// The current macro we are expanding, if we are expanding a macro.
410  ///
411  /// One of CurLexer and CurTokenLexer must be null.
412  std::unique_ptr<TokenLexer> CurTokenLexer;
413 
414  /// The kind of lexer we're currently working with.
415  enum CurLexerKind {
416  CLK_Lexer,
417  CLK_TokenLexer,
418  CLK_CachingLexer,
419  CLK_LexAfterModuleImport
420  } CurLexerKind = CLK_Lexer;
421 
422  /// If the current lexer is for a submodule that is being built, this
423  /// is that submodule.
424  Module *CurLexerSubmodule = nullptr;
425 
426  /// Keeps track of the stack of files currently
427  /// \#included, and macros currently being expanded from, not counting
428  /// CurLexer/CurTokenLexer.
429  struct IncludeStackInfo {
430  enum CurLexerKind CurLexerKind;
431  Module *TheSubmodule;
432  std::unique_ptr<Lexer> TheLexer;
433  PreprocessorLexer *ThePPLexer;
434  std::unique_ptr<TokenLexer> TheTokenLexer;
435  const DirectoryLookup *TheDirLookup;
436 
437  // The following constructors are completely useless copies of the default
438  // versions, only needed to pacify MSVC.
439  IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
440  std::unique_ptr<Lexer> &&TheLexer,
441  PreprocessorLexer *ThePPLexer,
442  std::unique_ptr<TokenLexer> &&TheTokenLexer,
443  const DirectoryLookup *TheDirLookup)
444  : CurLexerKind(std::move(CurLexerKind)),
445  TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
446  ThePPLexer(std::move(ThePPLexer)),
447  TheTokenLexer(std::move(TheTokenLexer)),
448  TheDirLookup(std::move(TheDirLookup)) {}
449  };
450  std::vector<IncludeStackInfo> IncludeMacroStack;
451 
452  /// Actions invoked when some preprocessor activity is
453  /// encountered (e.g. a file is \#included, etc).
454  std::unique_ptr<PPCallbacks> Callbacks;
455 
456  struct MacroExpandsInfo {
457  Token Tok;
458  MacroDefinition MD;
459  SourceRange Range;
460 
461  MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
462  : Tok(Tok), MD(MD), Range(Range) {}
463  };
464  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
465 
466  /// Information about a name that has been used to define a module macro.
467  struct ModuleMacroInfo {
468  /// The most recent macro directive for this identifier.
469  MacroDirective *MD;
470 
471  /// The active module macros for this identifier.
472  llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
473 
474  /// The generation number at which we last updated ActiveModuleMacros.
475  /// \see Preprocessor::VisibleModules.
476  unsigned ActiveModuleMacrosGeneration = 0;
477 
478  /// Whether this macro name is ambiguous.
479  bool IsAmbiguous = false;
480 
481  /// The module macros that are overridden by this macro.
482  llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
483 
484  ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
485  };
486 
487  /// The state of a macro for an identifier.
488  class MacroState {
489  mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
490 
491  ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
492  const IdentifierInfo *II) const {
493  if (II->isOutOfDate())
494  PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
495  // FIXME: Find a spare bit on IdentifierInfo and store a
496  // HasModuleMacros flag.
497  if (!II->hasMacroDefinition() ||
498  (!PP.getLangOpts().Modules &&
499  !PP.getLangOpts().ModulesLocalVisibility) ||
500  !PP.CurSubmoduleState->VisibleModules.getGeneration())
501  return nullptr;
502 
503  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
504  if (!Info) {
505  Info = new (PP.getPreprocessorAllocator())
506  ModuleMacroInfo(State.get<MacroDirective *>());
507  State = Info;
508  }
509 
510  if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
511  Info->ActiveModuleMacrosGeneration)
512  PP.updateModuleMacroInfo(II, *Info);
513  return Info;
514  }
515 
516  public:
517  MacroState() : MacroState(nullptr) {}
518  MacroState(MacroDirective *MD) : State(MD) {}
519 
520  MacroState(MacroState &&O) noexcept : State(O.State) {
521  O.State = (MacroDirective *)nullptr;
522  }
523 
524  MacroState &operator=(MacroState &&O) noexcept {
525  auto S = O.State;
526  O.State = (MacroDirective *)nullptr;
527  State = S;
528  return *this;
529  }
530 
531  ~MacroState() {
532  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
533  Info->~ModuleMacroInfo();
534  }
535 
536  MacroDirective *getLatest() const {
537  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
538  return Info->MD;
539  return State.get<MacroDirective*>();
540  }
541 
542  void setLatest(MacroDirective *MD) {
543  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
544  Info->MD = MD;
545  else
546  State = MD;
547  }
548 
549  bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
550  auto *Info = getModuleInfo(PP, II);
551  return Info ? Info->IsAmbiguous : false;
552  }
553 
555  getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
556  if (auto *Info = getModuleInfo(PP, II))
557  return Info->ActiveModuleMacros;
558  return None;
559  }
560 
561  MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
562  SourceManager &SourceMgr) const {
563  // FIXME: Incorporate module macros into the result of this.
564  if (auto *Latest = getLatest())
565  return Latest->findDirectiveAtLoc(Loc, SourceMgr);
566  return {};
567  }
568 
569  void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
570  if (auto *Info = getModuleInfo(PP, II)) {
571  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
572  Info->ActiveModuleMacros.begin(),
573  Info->ActiveModuleMacros.end());
574  Info->ActiveModuleMacros.clear();
575  Info->IsAmbiguous = false;
576  }
577  }
578 
579  ArrayRef<ModuleMacro*> getOverriddenMacros() const {
580  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
581  return Info->OverriddenMacros;
582  return None;
583  }
584 
585  void setOverriddenMacros(Preprocessor &PP,
586  ArrayRef<ModuleMacro *> Overrides) {
587  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
588  if (!Info) {
589  if (Overrides.empty())
590  return;
591  Info = new (PP.getPreprocessorAllocator())
592  ModuleMacroInfo(State.get<MacroDirective *>());
593  State = Info;
594  }
595  Info->OverriddenMacros.clear();
596  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
597  Overrides.begin(), Overrides.end());
598  Info->ActiveModuleMacrosGeneration = 0;
599  }
600  };
601 
602  /// For each IdentifierInfo that was associated with a macro, we
603  /// keep a mapping to the history of all macro definitions and #undefs in
604  /// the reverse order (the latest one is in the head of the list).
605  ///
606  /// This mapping lives within the \p CurSubmoduleState.
607  using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
608 
609  struct SubmoduleState;
610 
611  /// Information about a submodule that we're currently building.
612  struct BuildingSubmoduleInfo {
613  /// The module that we are building.
614  Module *M;
615 
616  /// The location at which the module was included.
617  SourceLocation ImportLoc;
618 
619  /// Whether we entered this submodule via a pragma.
620  bool IsPragma;
621 
622  /// The previous SubmoduleState.
623  SubmoduleState *OuterSubmoduleState;
624 
625  /// The number of pending module macro names when we started building this.
626  unsigned OuterPendingModuleMacroNames;
627 
628  BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
629  SubmoduleState *OuterSubmoduleState,
630  unsigned OuterPendingModuleMacroNames)
631  : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
632  OuterSubmoduleState(OuterSubmoduleState),
633  OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
634  };
635  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
636 
637  /// Information about a submodule's preprocessor state.
638  struct SubmoduleState {
639  /// The macros for the submodule.
640  MacroMap Macros;
641 
642  /// The set of modules that are visible within the submodule.
643  VisibleModuleSet VisibleModules;
644 
645  // FIXME: CounterValue?
646  // FIXME: PragmaPushMacroInfo?
647  };
648  std::map<Module *, SubmoduleState> Submodules;
649 
650  /// The preprocessor state for preprocessing outside of any submodule.
651  SubmoduleState NullSubmoduleState;
652 
653  /// The current submodule state. Will be \p NullSubmoduleState if we're not
654  /// in a submodule.
655  SubmoduleState *CurSubmoduleState;
656 
657  /// The set of known macros exported from modules.
658  llvm::FoldingSet<ModuleMacro> ModuleMacros;
659 
660  /// The names of potential module macros that we've not yet processed.
661  llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
662 
663  /// The list of module macros, for each identifier, that are not overridden by
664  /// any other module macro.
665  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
666  LeafModuleMacros;
667 
668  /// Macros that we want to warn because they are not used at the end
669  /// of the translation unit.
670  ///
671  /// We store just their SourceLocations instead of
672  /// something like MacroInfo*. The benefit of this is that when we are
673  /// deserializing from PCH, we don't need to deserialize identifier & macros
674  /// just so that we can report that they are unused, we just warn using
675  /// the SourceLocations of this set (that will be filled by the ASTReader).
676  /// We are using SmallPtrSet instead of a vector for faster removal.
677  using WarnUnusedMacroLocsTy = llvm::SmallPtrSet<SourceLocation, 32>;
678  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
679 
680  /// A "freelist" of MacroArg objects that can be
681  /// reused for quick allocation.
682  MacroArgs *MacroArgCache = nullptr;
683 
684  /// For each IdentifierInfo used in a \#pragma push_macro directive,
685  /// we keep a MacroInfo stack used to restore the previous macro value.
686  llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
687  PragmaPushMacroInfo;
688 
689  // Various statistics we track for performance analysis.
690  unsigned NumDirectives = 0;
691  unsigned NumDefined = 0;
692  unsigned NumUndefined = 0;
693  unsigned NumPragma = 0;
694  unsigned NumIf = 0;
695  unsigned NumElse = 0;
696  unsigned NumEndif = 0;
697  unsigned NumEnteredSourceFiles = 0;
698  unsigned MaxIncludeStackDepth = 0;
699  unsigned NumMacroExpanded = 0;
700  unsigned NumFnMacroExpanded = 0;
701  unsigned NumBuiltinMacroExpanded = 0;
702  unsigned NumFastMacroExpanded = 0;
703  unsigned NumTokenPaste = 0;
704  unsigned NumFastTokenPaste = 0;
705  unsigned NumSkipped = 0;
706 
707  /// The predefined macros that preprocessor should use from the
708  /// command line etc.
709  std::string Predefines;
710 
711  /// The file ID for the preprocessor predefines.
712  FileID PredefinesFileID;
713 
714  /// The file ID for the PCH through header.
715  FileID PCHThroughHeaderFileID;
716 
717  /// Whether tokens are being skipped until a #pragma hdrstop is seen.
718  bool SkippingUntilPragmaHdrStop = false;
719 
720  /// Whether tokens are being skipped until the through header is seen.
721  bool SkippingUntilPCHThroughHeader = false;
722 
723  /// \{
724  /// Cache of macro expanders to reduce malloc traffic.
725  enum { TokenLexerCacheSize = 8 };
726  unsigned NumCachedTokenLexers;
727  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
728  /// \}
729 
730  /// Keeps macro expanded tokens for TokenLexers.
731  //
732  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
733  /// going to lex in the cache and when it finishes the tokens are removed
734  /// from the end of the cache.
735  SmallVector<Token, 16> MacroExpandedTokens;
736  std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
737 
738  /// A record of the macro definitions and expansions that
739  /// occurred during preprocessing.
740  ///
741  /// This is an optional side structure that can be enabled with
742  /// \c createPreprocessingRecord() prior to preprocessing.
743  PreprocessingRecord *Record = nullptr;
744 
745  /// Cached tokens state.
747 
748  /// Cached tokens are stored here when we do backtracking or
749  /// lookahead. They are "lexed" by the CachingLex() method.
751 
752  /// The position of the cached token that CachingLex() should
753  /// "lex" next.
754  ///
755  /// If it points beyond the CachedTokens vector, it means that a normal
756  /// Lex() should be invoked.
757  CachedTokensTy::size_type CachedLexPos = 0;
758 
759  /// Stack of backtrack positions, allowing nested backtracks.
760  ///
761  /// The EnableBacktrackAtThisPos() method pushes a position to
762  /// indicate where CachedLexPos should be set when the BackTrack() method is
763  /// invoked (at which point the last position is popped).
764  std::vector<CachedTokensTy::size_type> BacktrackPositions;
765 
766  struct MacroInfoChain {
767  MacroInfo MI;
768  MacroInfoChain *Next;
769  };
770 
771  /// MacroInfos are managed as a chain for easy disposal. This is the head
772  /// of that list.
773  MacroInfoChain *MIChainHead = nullptr;
774 
775  void updateOutOfDateIdentifier(IdentifierInfo &II) const;
776 
777 public:
778  Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
780  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
781  IdentifierInfoLookup *IILookup = nullptr,
782  bool OwnsHeaderSearch = false,
784 
785  ~Preprocessor();
786 
787  /// Initialize the preprocessor using information about the target.
788  ///
789  /// \param Target is owned by the caller and must remain valid for the
790  /// lifetime of the preprocessor.
791  /// \param AuxTarget is owned by the caller and must remain valid for
792  /// the lifetime of the preprocessor.
793  void Initialize(const TargetInfo &Target,
794  const TargetInfo *AuxTarget = nullptr);
795 
796  /// Initialize the preprocessor to parse a model file
797  ///
798  /// To parse model files the preprocessor of the original source is reused to
799  /// preserver the identifier table. However to avoid some duplicate
800  /// information in the preprocessor some cleanup is needed before it is used
801  /// to parse model files. This method does that cleanup.
802  void InitializeForModelFile();
803 
804  /// Cleanup after model file parsing
805  void FinalizeForModelFile();
806 
807  /// Retrieve the preprocessor options used to initialize this
808  /// preprocessor.
809  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
810 
811  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
812  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
813 
814  const LangOptions &getLangOpts() const { return LangOpts; }
815  const TargetInfo &getTargetInfo() const { return *Target; }
816  const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
817  FileManager &getFileManager() const { return FileMgr; }
818  SourceManager &getSourceManager() const { return SourceMgr; }
819  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
820 
821  IdentifierTable &getIdentifierTable() { return Identifiers; }
822  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
823  SelectorTable &getSelectorTable() { return Selectors; }
825  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
826 
828  ExternalSource = Source;
829  }
830 
832  return ExternalSource;
833  }
834 
835  /// Retrieve the module loader associated with this preprocessor.
836  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
837 
839  return TheModuleLoader.HadFatalFailure;
840  }
841 
842  /// True if we are currently preprocessing a #if or #elif directive
844  return ParsingIfOrElifDirective;
845  }
846 
847  /// Control whether the preprocessor retains comments in output.
848  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
849  this->KeepComments = KeepComments | KeepMacroComments;
850  this->KeepMacroComments = KeepMacroComments;
851  }
852 
853  bool getCommentRetentionState() const { return KeepComments; }
854 
855  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
856  bool getPragmasEnabled() const { return PragmasEnabled; }
857 
858  void SetSuppressIncludeNotFoundError(bool Suppress) {
859  SuppressIncludeNotFoundError = Suppress;
860  }
861 
863  return SuppressIncludeNotFoundError;
864  }
865 
866  /// Sets whether the preprocessor is responsible for producing output or if
867  /// it is producing tokens to be consumed by Parse and Sema.
868  void setPreprocessedOutput(bool IsPreprocessedOutput) {
869  PreprocessedOutput = IsPreprocessedOutput;
870  }
871 
872  /// Returns true if the preprocessor is responsible for generating output,
873  /// false if it is producing tokens to be consumed by Parse and Sema.
874  bool isPreprocessedOutput() const { return PreprocessedOutput; }
875 
876  /// Return true if we are lexing directly from the specified lexer.
877  bool isCurrentLexer(const PreprocessorLexer *L) const {
878  return CurPPLexer == L;
879  }
880 
881  /// Return the current lexer being lexed from.
882  ///
883  /// Note that this ignores any potentially active macro expansions and _Pragma
884  /// expansions going on at the time.
885  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
886 
887  /// Return the current file lexer being lexed from.
888  ///
889  /// Note that this ignores any potentially active macro expansions and _Pragma
890  /// expansions going on at the time.
891  PreprocessorLexer *getCurrentFileLexer() const;
892 
893  /// Return the submodule owning the file being lexed. This may not be
894  /// the current module if we have changed modules since entering the file.
895  Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
896 
897  /// Returns the FileID for the preprocessor predefines.
898  FileID getPredefinesFileID() const { return PredefinesFileID; }
899 
900  /// \{
901  /// Accessors for preprocessor callbacks.
902  ///
903  /// Note that this class takes ownership of any PPCallbacks object given to
904  /// it.
905  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
906  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
907  if (Callbacks)
908  C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
909  std::move(Callbacks));
910  Callbacks = std::move(C);
911  }
912  /// \}
913 
914  bool isMacroDefined(StringRef Id) {
915  return isMacroDefined(&Identifiers.get(Id));
916  }
917  bool isMacroDefined(const IdentifierInfo *II) {
918  return II->hasMacroDefinition() &&
919  (!getLangOpts().Modules || (bool)getMacroDefinition(II));
920  }
921 
922  /// Determine whether II is defined as a macro within the module M,
923  /// if that is a module that we've already preprocessed. Does not check for
924  /// macros imported into M.
926  if (!II->hasMacroDefinition())
927  return false;
928  auto I = Submodules.find(M);
929  if (I == Submodules.end())
930  return false;
931  auto J = I->second.Macros.find(II);
932  if (J == I->second.Macros.end())
933  return false;
934  auto *MD = J->second.getLatest();
935  return MD && MD->isDefined();
936  }
937 
939  if (!II->hasMacroDefinition())
940  return {};
941 
942  MacroState &S = CurSubmoduleState->Macros[II];
943  auto *MD = S.getLatest();
944  while (MD && isa<VisibilityMacroDirective>(MD))
945  MD = MD->getPrevious();
946  return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
947  S.getActiveModuleMacros(*this, II),
948  S.isAmbiguous(*this, II));
949  }
950 
952  SourceLocation Loc) {
953  if (!II->hadMacroDefinition())
954  return {};
955 
956  MacroState &S = CurSubmoduleState->Macros[II];
958  if (auto *MD = S.getLatest())
959  DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
960  // FIXME: Compute the set of active module macros at the specified location.
961  return MacroDefinition(DI.getDirective(),
962  S.getActiveModuleMacros(*this, II),
963  S.isAmbiguous(*this, II));
964  }
965 
966  /// Given an identifier, return its latest non-imported MacroDirective
967  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
969  if (!II->hasMacroDefinition())
970  return nullptr;
971 
972  auto *MD = getLocalMacroDirectiveHistory(II);
973  if (!MD || MD->getDefinition().isUndefined())
974  return nullptr;
975 
976  return MD;
977  }
978 
979  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
980  return const_cast<Preprocessor*>(this)->getMacroInfo(II);
981  }
982 
984  if (!II->hasMacroDefinition())
985  return nullptr;
986  if (auto MD = getMacroDefinition(II))
987  return MD.getMacroInfo();
988  return nullptr;
989  }
990 
991  /// Given an identifier, return the latest non-imported macro
992  /// directive for that identifier.
993  ///
994  /// One can iterate over all previous macro directives from the most recent
995  /// one.
996  MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const;
997 
998  /// Add a directive to the macro directive history for this identifier.
999  void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
1001  SourceLocation Loc) {
1002  DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1003  appendMacroDirective(II, MD);
1004  return MD;
1005  }
1007  MacroInfo *MI) {
1008  return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1009  }
1010 
1011  /// Set a MacroDirective that was loaded from a PCH file.
1012  void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED,
1013  MacroDirective *MD);
1014 
1015  /// Register an exported macro for a module and identifier.
1016  ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro,
1017  ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1018  ModuleMacro *getModuleMacro(Module *Mod, IdentifierInfo *II);
1019 
1020  /// Get the list of leaf (non-overridden) module macros for a name.
1022  if (II->isOutOfDate())
1023  updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1024  auto I = LeafModuleMacros.find(II);
1025  if (I != LeafModuleMacros.end())
1026  return I->second;
1027  return None;
1028  }
1029 
1030  /// \{
1031  /// Iterators for the macro history table. Currently defined macros have
1032  /// IdentifierInfo::hasMacroDefinition() set and an empty
1033  /// MacroInfo::getUndefLoc() at the head of the list.
1034  using macro_iterator = MacroMap::const_iterator;
1035 
1036  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1037  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1038 
1039  llvm::iterator_range<macro_iterator>
1040  macros(bool IncludeExternalMacros = true) const {
1041  macro_iterator begin = macro_begin(IncludeExternalMacros);
1042  macro_iterator end = macro_end(IncludeExternalMacros);
1043  return llvm::make_range(begin, end);
1044  }
1045 
1046  /// \}
1047 
1048  /// Return the name of the macro defined before \p Loc that has
1049  /// spelling \p Tokens. If there are multiple macros with same spelling,
1050  /// return the last one defined.
1051  StringRef getLastMacroWithSpelling(SourceLocation Loc,
1052  ArrayRef<TokenValue> Tokens) const;
1053 
1054  const std::string &getPredefines() const { return Predefines; }
1055 
1056  /// Set the predefines for this Preprocessor.
1057  ///
1058  /// These predefines are automatically injected when parsing the main file.
1059  void setPredefines(const char *P) { Predefines = P; }
1060  void setPredefines(StringRef P) { Predefines = P; }
1061 
1062  /// Return information about the specified preprocessor
1063  /// identifier token.
1064  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1065  return &Identifiers.get(Name);
1066  }
1067 
1068  /// Add the specified pragma handler to this preprocessor.
1069  ///
1070  /// If \p Namespace is non-null, then it is a token required to exist on the
1071  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1072  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1074  AddPragmaHandler(StringRef(), Handler);
1075  }
1076 
1077  /// Remove the specific pragma handler from this preprocessor.
1078  ///
1079  /// If \p Namespace is non-null, then it should be the namespace that
1080  /// \p Handler was added to. It is an error to remove a handler that
1081  /// has not been registered.
1082  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1084  RemovePragmaHandler(StringRef(), Handler);
1085  }
1086 
1087  /// Install empty handlers for all pragmas (making them ignored).
1088  void IgnorePragmas();
1089 
1090  /// Add the specified comment handler to the preprocessor.
1091  void addCommentHandler(CommentHandler *Handler);
1092 
1093  /// Remove the specified comment handler.
1094  ///
1095  /// It is an error to remove a handler that has not been registered.
1096  void removeCommentHandler(CommentHandler *Handler);
1097 
1098  /// Set the code completion handler to the given object.
1100  CodeComplete = &Handler;
1101  }
1102 
1103  /// Retrieve the current code-completion handler.
1105  return CodeComplete;
1106  }
1107 
1108  /// Clear out the code completion handler.
1110  CodeComplete = nullptr;
1111  }
1112 
1113  /// Hook used by the lexer to invoke the "included file" code
1114  /// completion point.
1115  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1116 
1117  /// Hook used by the lexer to invoke the "natural language" code
1118  /// completion point.
1119  void CodeCompleteNaturalLanguage();
1120 
1121  /// Set the code completion token for filtering purposes.
1123  CodeCompletionII = Filter;
1124  }
1125 
1126  /// Set the code completion token range for detecting replacement range later
1127  /// on.
1129  const SourceLocation End) {
1130  CodeCompletionTokenRange = {Start, End};
1131  }
1133  return CodeCompletionTokenRange;
1134  }
1135 
1136  /// Get the code completion token for filtering purposes.
1138  if (CodeCompletionII)
1139  return CodeCompletionII->getName();
1140  return {};
1141  }
1142 
1143  /// Retrieve the preprocessing record, or NULL if there is no
1144  /// preprocessing record.
1145  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1146 
1147  /// Create a new preprocessing record, which will keep track of
1148  /// all macro expansions, macro definitions, etc.
1149  void createPreprocessingRecord();
1150 
1151  /// Returns true if the FileEntry is the PCH through header.
1152  bool isPCHThroughHeader(const FileEntry *FE);
1153 
1154  /// True if creating a PCH with a through header.
1155  bool creatingPCHWithThroughHeader();
1156 
1157  /// True if using a PCH with a through header.
1158  bool usingPCHWithThroughHeader();
1159 
1160  /// True if creating a PCH with a #pragma hdrstop.
1161  bool creatingPCHWithPragmaHdrStop();
1162 
1163  /// True if using a PCH with a #pragma hdrstop.
1164  bool usingPCHWithPragmaHdrStop();
1165 
1166  /// Skip tokens until after the #include of the through header or
1167  /// until after a #pragma hdrstop.
1168  void SkipTokensWhileUsingPCH();
1169 
1170  /// Process directives while skipping until the through header or
1171  /// #pragma hdrstop is found.
1172  void HandleSkippedDirectiveWhileUsingPCH(Token &Result,
1173  SourceLocation HashLoc);
1174 
1175  /// Enter the specified FileID as the main source file,
1176  /// which implicitly adds the builtin defines etc.
1177  void EnterMainSourceFile();
1178 
1179  /// Inform the preprocessor callbacks that processing is complete.
1180  void EndSourceFile();
1181 
1182  /// Add a source file to the top of the include stack and
1183  /// start lexing tokens from it instead of the current buffer.
1184  ///
1185  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1186  bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir,
1187  SourceLocation Loc);
1188 
1189  /// Add a Macro to the top of the include stack and start lexing
1190  /// tokens from it instead of the current buffer.
1191  ///
1192  /// \param Args specifies the tokens input to a function-like macro.
1193  /// \param ILEnd specifies the location of the ')' for a function-like macro
1194  /// or the identifier for an object-like macro.
1195  void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1196  MacroArgs *Args);
1197 
1198  /// Add a "macro" context to the top of the include stack,
1199  /// which will cause the lexer to start returning the specified tokens.
1200  ///
1201  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1202  /// will not be subject to further macro expansion. Otherwise, these tokens
1203  /// will be re-macro-expanded when/if expansion is enabled.
1204  ///
1205  /// If \p OwnsTokens is false, this method assumes that the specified stream
1206  /// of tokens has a permanent owner somewhere, so they do not need to be
1207  /// copied. If it is true, it assumes the array of tokens is allocated with
1208  /// \c new[] and the Preprocessor will delete[] it.
1209 private:
1210  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1211  bool DisableMacroExpansion, bool OwnsTokens);
1212 
1213 public:
1214  void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1215  bool DisableMacroExpansion) {
1216  EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true);
1217  }
1218 
1219  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion) {
1220  EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false);
1221  }
1222 
1223  /// Pop the current lexer/macro exp off the top of the lexer stack.
1224  ///
1225  /// This should only be used in situations where the current state of the
1226  /// top-of-stack lexer is known.
1227  void RemoveTopOfLexerStack();
1228 
1229  /// From the point that this method is called, and until
1230  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1231  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1232  /// make the Preprocessor re-lex the same tokens.
1233  ///
1234  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1235  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1236  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1237  ///
1238  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1239  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1240  /// tokens will continue indefinitely.
1241  ///
1242  void EnableBacktrackAtThisPos();
1243 
1244  /// Disable the last EnableBacktrackAtThisPos call.
1245  void CommitBacktrackedTokens();
1246 
1248  CachedTokensTy::size_type Begin, End;
1249  };
1250 
1251 private:
1252  /// A range of cached tokens that should be erased after lexing
1253  /// when backtracking requires the erasure of such cached tokens.
1254  Optional<CachedTokensRange> CachedTokenRangeToErase;
1255 
1256 public:
1257  /// Returns the range of cached tokens that were lexed since
1258  /// EnableBacktrackAtThisPos() was previously called.
1259  CachedTokensRange LastCachedTokenRange();
1260 
1261  /// Erase the range of cached tokens that were lexed since
1262  /// EnableBacktrackAtThisPos() was previously called.
1263  void EraseCachedTokens(CachedTokensRange TokenRange);
1264 
1265  /// Make Preprocessor re-lex the tokens that were lexed since
1266  /// EnableBacktrackAtThisPos() was previously called.
1267  void Backtrack();
1268 
1269  /// True if EnableBacktrackAtThisPos() was called and
1270  /// caching of tokens is on.
1271  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1272 
1273  /// Lex the next token for this preprocessor.
1274  void Lex(Token &Result);
1275 
1276  /// Lex a token, forming a header-name token if possible.
1277  bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
1278 
1279  void LexAfterModuleImport(Token &Result);
1280 
1281  void makeModuleVisible(Module *M, SourceLocation Loc);
1282 
1284  return CurSubmoduleState->VisibleModules.getImportLoc(M);
1285  }
1286 
1287  /// Lex a string literal, which may be the concatenation of multiple
1288  /// string literals and may even come from macro expansion.
1289  /// \returns true on success, false if a error diagnostic has been generated.
1290  bool LexStringLiteral(Token &Result, std::string &String,
1291  const char *DiagnosticTag, bool AllowMacroExpansion) {
1292  if (AllowMacroExpansion)
1293  Lex(Result);
1294  else
1295  LexUnexpandedToken(Result);
1296  return FinishLexStringLiteral(Result, String, DiagnosticTag,
1297  AllowMacroExpansion);
1298  }
1299 
1300  /// Complete the lexing of a string literal where the first token has
1301  /// already been lexed (see LexStringLiteral).
1302  bool FinishLexStringLiteral(Token &Result, std::string &String,
1303  const char *DiagnosticTag,
1304  bool AllowMacroExpansion);
1305 
1306  /// Lex a token. If it's a comment, keep lexing until we get
1307  /// something not a comment.
1308  ///
1309  /// This is useful in -E -C mode where comments would foul up preprocessor
1310  /// directive handling.
1311  void LexNonComment(Token &Result) {
1312  do
1313  Lex(Result);
1314  while (Result.getKind() == tok::comment);
1315  }
1316 
1317  /// Just like Lex, but disables macro expansion of identifier tokens.
1318  void LexUnexpandedToken(Token &Result) {
1319  // Disable macro expansion.
1320  bool OldVal = DisableMacroExpansion;
1321  DisableMacroExpansion = true;
1322  // Lex the token.
1323  Lex(Result);
1324 
1325  // Reenable it.
1326  DisableMacroExpansion = OldVal;
1327  }
1328 
1329  /// Like LexNonComment, but this disables macro expansion of
1330  /// identifier tokens.
1332  do
1333  LexUnexpandedToken(Result);
1334  while (Result.getKind() == tok::comment);
1335  }
1336 
1337  /// Parses a simple integer literal to get its numeric value. Floating
1338  /// point literals and user defined literals are rejected. Used primarily to
1339  /// handle pragmas that accept integer arguments.
1340  bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1341 
1342  /// Disables macro expansion everywhere except for preprocessor directives.
1344  DisableMacroExpansion = true;
1345  MacroExpansionInDirectivesOverride = true;
1346  }
1347 
1348  /// Peeks ahead N tokens and returns that token without consuming any
1349  /// tokens.
1350  ///
1351  /// LookAhead(0) returns the next token that would be returned by Lex(),
1352  /// LookAhead(1) returns the token after it, etc. This returns normal
1353  /// tokens after phase 5. As such, it is equivalent to using
1354  /// 'Lex', not 'LexUnexpandedToken'.
1355  const Token &LookAhead(unsigned N) {
1356  if (CachedLexPos + N < CachedTokens.size())
1357  return CachedTokens[CachedLexPos+N];
1358  else
1359  return PeekAhead(N+1);
1360  }
1361 
1362  /// When backtracking is enabled and tokens are cached,
1363  /// this allows to revert a specific number of tokens.
1364  ///
1365  /// Note that the number of tokens being reverted should be up to the last
1366  /// backtrack position, not more.
1367  void RevertCachedTokens(unsigned N) {
1368  assert(isBacktrackEnabled() &&
1369  "Should only be called when tokens are cached for backtracking");
1370  assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1371  && "Should revert tokens up to the last backtrack position, not more");
1372  assert(signed(CachedLexPos) - signed(N) >= 0 &&
1373  "Corrupted backtrack positions ?");
1374  CachedLexPos -= N;
1375  }
1376 
1377  /// Enters a token in the token stream to be lexed next.
1378  ///
1379  /// If BackTrack() is called afterwards, the token will remain at the
1380  /// insertion point.
1381  void EnterToken(const Token &Tok) {
1382  EnterCachingLexMode();
1383  CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1384  }
1385 
1386  /// We notify the Preprocessor that if it is caching tokens (because
1387  /// backtrack is enabled) it should replace the most recent cached tokens
1388  /// with the given annotation token. This function has no effect if
1389  /// backtracking is not enabled.
1390  ///
1391  /// Note that the use of this function is just for optimization, so that the
1392  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1393  /// invoked.
1394  void AnnotateCachedTokens(const Token &Tok) {
1395  assert(Tok.isAnnotation() && "Expected annotation token");
1396  if (CachedLexPos != 0 && isBacktrackEnabled())
1397  AnnotatePreviousCachedTokens(Tok);
1398  }
1399 
1400  /// Get the location of the last cached token, suitable for setting the end
1401  /// location of an annotation token.
1403  assert(CachedLexPos != 0);
1404  return CachedTokens[CachedLexPos-1].getLastLoc();
1405  }
1406 
1407  /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1408  /// CachedTokens.
1409  bool IsPreviousCachedToken(const Token &Tok) const;
1410 
1411  /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1412  /// in \p NewToks.
1413  ///
1414  /// Useful when a token needs to be split in smaller ones and CachedTokens
1415  /// most recent token must to be updated to reflect that.
1416  void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1417 
1418  /// Replace the last token with an annotation token.
1419  ///
1420  /// Like AnnotateCachedTokens(), this routine replaces an
1421  /// already-parsed (and resolved) token with an annotation
1422  /// token. However, this routine only replaces the last token with
1423  /// the annotation token; it does not affect any other cached
1424  /// tokens. This function has no effect if backtracking is not
1425  /// enabled.
1427  assert(Tok.isAnnotation() && "Expected annotation token");
1428  if (CachedLexPos != 0 && isBacktrackEnabled())
1429  CachedTokens[CachedLexPos-1] = Tok;
1430  }
1431 
1432  /// Enter an annotation token into the token stream.
1433  void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1434  void *AnnotationVal);
1435 
1436  /// Update the current token to represent the provided
1437  /// identifier, in order to cache an action performed by typo correction.
1438  void TypoCorrectToken(const Token &Tok) {
1439  assert(Tok.getIdentifierInfo() && "Expected identifier token");
1440  if (CachedLexPos != 0 && isBacktrackEnabled())
1441  CachedTokens[CachedLexPos-1] = Tok;
1442  }
1443 
1444  /// Recompute the current lexer kind based on the CurLexer/
1445  /// CurTokenLexer pointers.
1446  void recomputeCurLexerKind();
1447 
1448  /// Returns true if incremental processing is enabled
1449  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1450 
1451  /// Enables the incremental processing
1452  void enableIncrementalProcessing(bool value = true) {
1453  IncrementalProcessing = value;
1454  }
1455 
1456  /// Specify the point at which code-completion will be performed.
1457  ///
1458  /// \param File the file in which code completion should occur. If
1459  /// this file is included multiple times, code-completion will
1460  /// perform completion the first time it is included. If NULL, this
1461  /// function clears out the code-completion point.
1462  ///
1463  /// \param Line the line at which code completion should occur
1464  /// (1-based).
1465  ///
1466  /// \param Column the column at which code completion should occur
1467  /// (1-based).
1468  ///
1469  /// \returns true if an error occurred, false otherwise.
1470  bool SetCodeCompletionPoint(const FileEntry *File,
1471  unsigned Line, unsigned Column);
1472 
1473  /// Determine if we are performing code completion.
1474  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1475 
1476  /// Returns the location of the code-completion point.
1477  ///
1478  /// Returns an invalid location if code-completion is not enabled or the file
1479  /// containing the code-completion point has not been lexed yet.
1480  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1481 
1482  /// Returns the start location of the file of code-completion point.
1483  ///
1484  /// Returns an invalid location if code-completion is not enabled or the file
1485  /// containing the code-completion point has not been lexed yet.
1487  return CodeCompletionFileLoc;
1488  }
1489 
1490  /// Returns true if code-completion is enabled and we have hit the
1491  /// code-completion point.
1492  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1493 
1494  /// Note that we hit the code-completion point.
1496  assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1497  CodeCompletionReached = true;
1498  // Silence any diagnostics that occur after we hit the code-completion.
1499  getDiagnostics().setSuppressAllDiagnostics(true);
1500  }
1501 
1502  /// The location of the currently-active \#pragma clang
1503  /// arc_cf_code_audited begin.
1504  ///
1505  /// Returns an invalid location if there is no such pragma active.
1507  return PragmaARCCFCodeAuditedLoc;
1508  }
1509 
1510  /// Set the location of the currently-active \#pragma clang
1511  /// arc_cf_code_audited begin. An invalid location ends the pragma.
1513  PragmaARCCFCodeAuditedLoc = Loc;
1514  }
1515 
1516  /// The location of the currently-active \#pragma clang
1517  /// assume_nonnull begin.
1518  ///
1519  /// Returns an invalid location if there is no such pragma active.
1521  return PragmaAssumeNonNullLoc;
1522  }
1523 
1524  /// Set the location of the currently-active \#pragma clang
1525  /// assume_nonnull begin. An invalid location ends the pragma.
1527  PragmaAssumeNonNullLoc = Loc;
1528  }
1529 
1530  /// Set the directory in which the main file should be considered
1531  /// to have been found, if it is not a real file.
1532  void setMainFileDir(const DirectoryEntry *Dir) {
1533  MainFileDir = Dir;
1534  }
1535 
1536  /// Instruct the preprocessor to skip part of the main source file.
1537  ///
1538  /// \param Bytes The number of bytes in the preamble to skip.
1539  ///
1540  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1541  /// start of a line.
1542  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1543  SkipMainFilePreamble.first = Bytes;
1544  SkipMainFilePreamble.second = StartOfLine;
1545  }
1546 
1547  /// Forwarding function for diagnostics. This emits a diagnostic at
1548  /// the specified Token's location, translating the token's start
1549  /// position in the current buffer into a SourcePosition object for rendering.
1550  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1551  return Diags->Report(Loc, DiagID);
1552  }
1553 
1554  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1555  return Diags->Report(Tok.getLocation(), DiagID);
1556  }
1557 
1558  /// Return the 'spelling' of the token at the given
1559  /// location; does not go up to the spelling location or down to the
1560  /// expansion location.
1561  ///
1562  /// \param buffer A buffer which will be used only if the token requires
1563  /// "cleaning", e.g. if it contains trigraphs or escaped newlines
1564  /// \param invalid If non-null, will be set \c true if an error occurs.
1566  SmallVectorImpl<char> &buffer,
1567  bool *invalid = nullptr) const {
1568  return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1569  }
1570 
1571  /// Return the 'spelling' of the Tok token.
1572  ///
1573  /// The spelling of a token is the characters used to represent the token in
1574  /// the source file after trigraph expansion and escaped-newline folding. In
1575  /// particular, this wants to get the true, uncanonicalized, spelling of
1576  /// things like digraphs, UCNs, etc.
1577  ///
1578  /// \param Invalid If non-null, will be set \c true if an error occurs.
1579  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1580  return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1581  }
1582 
1583  /// Get the spelling of a token into a preallocated buffer, instead
1584  /// of as an std::string.
1585  ///
1586  /// The caller is required to allocate enough space for the token, which is
1587  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1588  /// actual result is returned.
1589  ///
1590  /// Note that this method may do two possible things: it may either fill in
1591  /// the buffer specified with characters, or it may *change the input pointer*
1592  /// to point to a constant buffer with the data already in it (avoiding a
1593  /// copy). The caller is not allowed to modify the returned buffer pointer
1594  /// if an internal buffer is returned.
1595  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1596  bool *Invalid = nullptr) const {
1597  return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1598  }
1599 
1600  /// Get the spelling of a token into a SmallVector.
1601  ///
1602  /// Note that the returned StringRef may not point to the
1603  /// supplied buffer if a copy can be avoided.
1604  StringRef getSpelling(const Token &Tok,
1605  SmallVectorImpl<char> &Buffer,
1606  bool *Invalid = nullptr) const;
1607 
1608  /// Relex the token at the specified location.
1609  /// \returns true if there was a failure, false on success.
1610  bool getRawToken(SourceLocation Loc, Token &Result,
1611  bool IgnoreWhiteSpace = false) {
1612  return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1613  }
1614 
1615  /// Given a Token \p Tok that is a numeric constant with length 1,
1616  /// return the character.
1617  char
1619  bool *Invalid = nullptr) const {
1620  assert(Tok.is(tok::numeric_constant) &&
1621  Tok.getLength() == 1 && "Called on unsupported token");
1622  assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1623 
1624  // If the token is carrying a literal data pointer, just use it.
1625  if (const char *D = Tok.getLiteralData())
1626  return *D;
1627 
1628  // Otherwise, fall back on getCharacterData, which is slower, but always
1629  // works.
1630  return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1631  }
1632 
1633  /// Retrieve the name of the immediate macro expansion.
1634  ///
1635  /// This routine starts from a source location, and finds the name of the
1636  /// macro responsible for its immediate expansion. It looks through any
1637  /// intervening macro argument expansions to compute this. It returns a
1638  /// StringRef that refers to the SourceManager-owned buffer of the source
1639  /// where that macro name is spelled. Thus, the result shouldn't out-live
1640  /// the SourceManager.
1642  return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1643  }
1644 
1645  /// Plop the specified string into a scratch buffer and set the
1646  /// specified token's location and length to it.
1647  ///
1648  /// If specified, the source location provides a location of the expansion
1649  /// point of the token.
1650  void CreateString(StringRef Str, Token &Tok,
1651  SourceLocation ExpansionLocStart = SourceLocation(),
1652  SourceLocation ExpansionLocEnd = SourceLocation());
1653 
1654  /// Split the first Length characters out of the token starting at TokLoc
1655  /// and return a location pointing to the split token. Re-lexing from the
1656  /// split token will return the split token rather than the original.
1657  SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
1658 
1659  /// Computes the source location just past the end of the
1660  /// token at this source location.
1661  ///
1662  /// This routine can be used to produce a source location that
1663  /// points just past the end of the token referenced by \p Loc, and
1664  /// is generally used when a diagnostic needs to point just after a
1665  /// token where it expected something different that it received. If
1666  /// the returned source location would not be meaningful (e.g., if
1667  /// it points into a macro), this routine returns an invalid
1668  /// source location.
1669  ///
1670  /// \param Offset an offset from the end of the token, where the source
1671  /// location should refer to. The default offset (0) produces a source
1672  /// location pointing just past the end of the token; an offset of 1 produces
1673  /// a source location pointing to the last character in the token, etc.
1675  return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1676  }
1677 
1678  /// Returns true if the given MacroID location points at the first
1679  /// token of the macro expansion.
1680  ///
1681  /// \param MacroBegin If non-null and function returns true, it is set to
1682  /// begin location of the macro.
1684  SourceLocation *MacroBegin = nullptr) const {
1685  return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1686  MacroBegin);
1687  }
1688 
1689  /// Returns true if the given MacroID location points at the last
1690  /// token of the macro expansion.
1691  ///
1692  /// \param MacroEnd If non-null and function returns true, it is set to
1693  /// end location of the macro.
1695  SourceLocation *MacroEnd = nullptr) const {
1696  return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1697  }
1698 
1699  /// Print the token to stderr, used for debugging.
1700  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1701  void DumpLocation(SourceLocation Loc) const;
1702  void DumpMacro(const MacroInfo &MI) const;
1703  void dumpMacroInfo(const IdentifierInfo *II);
1704 
1705  /// Given a location that specifies the start of a
1706  /// token, return a new location that specifies a character within the token.
1708  unsigned Char) const {
1709  return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1710  }
1711 
1712  /// Increment the counters for the number of token paste operations
1713  /// performed.
1714  ///
1715  /// If fast was specified, this is a 'fast paste' case we handled.
1716  void IncrementPasteCounter(bool isFast) {
1717  if (isFast)
1718  ++NumFastTokenPaste;
1719  else
1720  ++NumTokenPaste;
1721  }
1722 
1723  void PrintStats();
1724 
1725  size_t getTotalMemory() const;
1726 
1727  /// When the macro expander pastes together a comment (/##/) in Microsoft
1728  /// mode, this method handles updating the current state, returning the
1729  /// token on the next source line.
1730  void HandleMicrosoftCommentPaste(Token &Tok);
1731 
1732  //===--------------------------------------------------------------------===//
1733  // Preprocessor callback methods. These are invoked by a lexer as various
1734  // directives and events are found.
1735 
1736  /// Given a tok::raw_identifier token, look up the
1737  /// identifier information for the token and install it into the token,
1738  /// updating the token kind accordingly.
1739  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1740 
1741 private:
1742  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1743 
1744 public:
1745  /// Specifies the reason for poisoning an identifier.
1746  ///
1747  /// If that identifier is accessed while poisoned, then this reason will be
1748  /// used instead of the default "poisoned" diagnostic.
1749  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1750 
1751  /// Display reason for poisoned identifier.
1752  void HandlePoisonedIdentifier(Token & Identifier);
1753 
1755  if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1756  if(II->isPoisoned()) {
1757  HandlePoisonedIdentifier(Identifier);
1758  }
1759  }
1760  }
1761 
1762 private:
1763  /// Identifiers used for SEH handling in Borland. These are only
1764  /// allowed in particular circumstances
1765  // __except block
1766  IdentifierInfo *Ident__exception_code,
1767  *Ident___exception_code,
1768  *Ident_GetExceptionCode;
1769  // __except filter expression
1770  IdentifierInfo *Ident__exception_info,
1771  *Ident___exception_info,
1772  *Ident_GetExceptionInfo;
1773  // __finally
1774  IdentifierInfo *Ident__abnormal_termination,
1775  *Ident___abnormal_termination,
1776  *Ident_AbnormalTermination;
1777 
1778  const char *getCurLexerEndPos();
1779  void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
1780 
1781 public:
1782  void PoisonSEHIdentifiers(bool Poison = true); // Borland
1783 
1784  /// Callback invoked when the lexer reads an identifier and has
1785  /// filled in the tokens IdentifierInfo member.
1786  ///
1787  /// This callback potentially macro expands it or turns it into a named
1788  /// token (like 'for').
1789  ///
1790  /// \returns true if we actually computed a token, false if we need to
1791  /// lex again.
1792  bool HandleIdentifier(Token &Identifier);
1793 
1794  /// Callback invoked when the lexer hits the end of the current file.
1795  ///
1796  /// This either returns the EOF token and returns true, or
1797  /// pops a level off the include stack and returns false, at which point the
1798  /// client should call lex again.
1799  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1800 
1801  /// Callback invoked when the current TokenLexer hits the end of its
1802  /// token stream.
1803  bool HandleEndOfTokenLexer(Token &Result);
1804 
1805  /// Callback invoked when the lexer sees a # token at the start of a
1806  /// line.
1807  ///
1808  /// This consumes the directive, modifies the lexer/preprocessor state, and
1809  /// advances the lexer(s) so that the next token read is the correct one.
1810  void HandleDirective(Token &Result);
1811 
1812  /// Ensure that the next token is a tok::eod token.
1813  ///
1814  /// If not, emit a diagnostic and consume up until the eod.
1815  /// If \p EnableMacros is true, then we consider macros that expand to zero
1816  /// tokens as being ok.
1817  void CheckEndOfDirective(const char *DirType, bool EnableMacros = false);
1818 
1819  /// Read and discard all tokens remaining on the current line until
1820  /// the tok::eod token is found. Returns the range of the skipped tokens.
1821  SourceRange DiscardUntilEndOfDirective();
1822 
1823  /// Returns true if the preprocessor has seen a use of
1824  /// __DATE__ or __TIME__ in the file so far.
1825  bool SawDateOrTime() const {
1826  return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
1827  }
1828  unsigned getCounterValue() const { return CounterValue; }
1829  void setCounterValue(unsigned V) { CounterValue = V; }
1830 
1831  /// Retrieves the module that we're currently building, if any.
1832  Module *getCurrentModule();
1833 
1834  /// Allocate a new MacroInfo object with the provided SourceLocation.
1835  MacroInfo *AllocateMacroInfo(SourceLocation L);
1836 
1837  /// Turn the specified lexer token into a fully checked and spelled
1838  /// filename, e.g. as an operand of \#include.
1839  ///
1840  /// The caller is expected to provide a buffer that is large enough to hold
1841  /// the spelling of the filename, but is also expected to handle the case
1842  /// when this method decides to use a different buffer.
1843  ///
1844  /// \returns true if the input filename was in <>'s or false if it was
1845  /// in ""'s.
1846  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
1847 
1848  /// Given a "foo" or <foo> reference, look up the indicated file.
1849  ///
1850  /// Returns null on failure. \p isAngled indicates whether the file
1851  /// reference is for system \#include's or not (i.e. using <> instead of "").
1852  const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename,
1853  bool isAngled, const DirectoryLookup *FromDir,
1854  const FileEntry *FromFile,
1855  const DirectoryLookup *&CurDir,
1856  SmallVectorImpl<char> *SearchPath,
1857  SmallVectorImpl<char> *RelativePath,
1858  ModuleMap::KnownHeader *SuggestedModule,
1859  bool *IsMapped, bool *IsFrameworkFound,
1860  bool SkipCache = false);
1861 
1862  /// Get the DirectoryLookup structure used to find the current
1863  /// FileEntry, if CurLexer is non-null and if applicable.
1864  ///
1865  /// This allows us to implement \#include_next and find directory-specific
1866  /// properties.
1867  const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1868 
1869  /// Return true if we're in the top-level file, not in a \#include.
1870  bool isInPrimaryFile() const;
1871 
1872  /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1873  /// followed by EOD. Return true if the token is not a valid on-off-switch.
1874  bool LexOnOffSwitch(tok::OnOffSwitch &Result);
1875 
1876  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
1877  bool *ShadowFlag = nullptr);
1878 
1879  void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
1880  Module *LeaveSubmodule(bool ForPragma);
1881 
1882 private:
1883  friend void TokenLexer::ExpandFunctionArguments();
1884 
1885  void PushIncludeMacroStack() {
1886  assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
1887  IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
1888  std::move(CurLexer), CurPPLexer,
1889  std::move(CurTokenLexer), CurDirLookup);
1890  CurPPLexer = nullptr;
1891  }
1892 
1893  void PopIncludeMacroStack() {
1894  CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1895  CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1896  CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1897  CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1898  CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
1899  CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1900  IncludeMacroStack.pop_back();
1901  }
1902 
1903  void PropagateLineStartLeadingSpaceInfo(Token &Result);
1904 
1905  /// Determine whether we need to create module macros for #defines in the
1906  /// current context.
1907  bool needModuleMacros() const;
1908 
1909  /// Update the set of active module macros and ambiguity flag for a module
1910  /// macro name.
1911  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
1912 
1913  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
1914  SourceLocation Loc);
1915  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
1916  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
1917  bool isPublic);
1918 
1919  /// Lex and validate a macro name, which occurs after a
1920  /// \#define or \#undef.
1921  ///
1922  /// \param MacroNameTok Token that represents the name defined or undefined.
1923  /// \param IsDefineUndef Kind if preprocessor directive.
1924  /// \param ShadowFlag Points to flag that is set if macro name shadows
1925  /// a keyword.
1926  ///
1927  /// This emits a diagnostic, sets the token kind to eod,
1928  /// and discards the rest of the macro line if the macro name is invalid.
1929  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
1930  bool *ShadowFlag = nullptr);
1931 
1932  /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
1933  /// entire line) of the macro's tokens and adds them to MacroInfo, and while
1934  /// doing so performs certain validity checks including (but not limited to):
1935  /// - # (stringization) is followed by a macro parameter
1936  /// \param MacroNameTok - Token that represents the macro name
1937  /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
1938  ///
1939  /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
1940  /// returns a nullptr if an invalid sequence of tokens is encountered.
1941  MacroInfo *ReadOptionalMacroParameterListAndBody(
1942  const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
1943 
1944  /// The ( starting an argument list of a macro definition has just been read.
1945  /// Lex the rest of the parameters and the closing ), updating \p MI with
1946  /// what we learn and saving in \p LastTok the last token read.
1947  /// Return true if an error occurs parsing the arg list.
1948  bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
1949 
1950  /// We just read a \#if or related directive and decided that the
1951  /// subsequent tokens are in the \#if'd out portion of the
1952  /// file. Lex the rest of the file, until we see an \#endif. If \p
1953  /// FoundNonSkipPortion is true, then we have already emitted code for part of
1954  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
1955  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
1956  /// already seen one so a \#else directive is a duplicate. When this returns,
1957  /// the caller can lex the first valid token.
1958  void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
1959  SourceLocation IfTokenLoc,
1960  bool FoundNonSkipPortion, bool FoundElse,
1961  SourceLocation ElseLoc = SourceLocation());
1962 
1963  /// Information about the result for evaluating an expression for a
1964  /// preprocessor directive.
1965  struct DirectiveEvalResult {
1966  /// Whether the expression was evaluated as true or not.
1967  bool Conditional;
1968 
1969  /// True if the expression contained identifiers that were undefined.
1970  bool IncludedUndefinedIds;
1971 
1972  /// The source range for the expression.
1973  SourceRange ExprRange;
1974  };
1975 
1976  /// Evaluate an integer constant expression that may occur after a
1977  /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
1978  ///
1979  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
1980  DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1981 
1982  /// Install the standard preprocessor pragmas:
1983  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
1984  void RegisterBuiltinPragmas();
1985 
1986  /// Register builtin macros such as __LINE__ with the identifier table.
1987  void RegisterBuiltinMacros();
1988 
1989  /// If an identifier token is read that is to be expanded as a macro, handle
1990  /// it and return the next token as 'Tok'. If we lexed a token, return true;
1991  /// otherwise the caller should lex again.
1992  bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
1993 
1994  /// Cache macro expanded tokens for TokenLexers.
1995  //
1996  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
1997  /// going to lex in the cache and when it finishes the tokens are removed
1998  /// from the end of the cache.
1999  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2000  ArrayRef<Token> tokens);
2001 
2002  void removeCachedMacroExpandedTokensOfLastLexer();
2003 
2004  /// Determine whether the next preprocessor token to be
2005  /// lexed is a '('. If so, consume the token and return true, if not, this
2006  /// method should have no observable side-effect on the lexed tokens.
2007  bool isNextPPTokenLParen();
2008 
2009  /// After reading "MACRO(", this method is invoked to read all of the formal
2010  /// arguments specified for the macro invocation. Returns null on error.
2011  MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2012  SourceLocation &MacroEnd);
2013 
2014  /// If an identifier token is read that is to be expanded
2015  /// as a builtin macro, handle it and return the next token as 'Tok'.
2016  void ExpandBuiltinMacro(Token &Tok);
2017 
2018  /// Read a \c _Pragma directive, slice it up, process it, then
2019  /// return the first token after the directive.
2020  /// This assumes that the \c _Pragma token has just been read into \p Tok.
2021  void Handle_Pragma(Token &Tok);
2022 
2023  /// Like Handle_Pragma except the pragma text is not enclosed within
2024  /// a string literal.
2025  void HandleMicrosoft__pragma(Token &Tok);
2026 
2027  /// Add a lexer to the top of the include stack and
2028  /// start lexing tokens from it instead of the current buffer.
2029  void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
2030 
2031  /// Set the FileID for the preprocessor predefines.
2032  void setPredefinesFileID(FileID FID) {
2033  assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2034  PredefinesFileID = FID;
2035  }
2036 
2037  /// Set the FileID for the PCH through header.
2038  void setPCHThroughHeaderFileID(FileID FID);
2039 
2040  /// Returns true if we are lexing from a file and not a
2041  /// pragma or a macro.
2042  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2043  return L ? !L->isPragmaLexer() : P != nullptr;
2044  }
2045 
2046  static bool IsFileLexer(const IncludeStackInfo& I) {
2047  return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2048  }
2049 
2050  bool IsFileLexer() const {
2051  return IsFileLexer(CurLexer.get(), CurPPLexer);
2052  }
2053 
2054  //===--------------------------------------------------------------------===//
2055  // Caching stuff.
2056  void CachingLex(Token &Result);
2057 
2058  bool InCachingLexMode() const {
2059  // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2060  // that we are past EOF, not that we are in CachingLex mode.
2061  return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2062  }
2063 
2064  void EnterCachingLexMode();
2065 
2066  void ExitCachingLexMode() {
2067  if (InCachingLexMode())
2068  RemoveTopOfLexerStack();
2069  }
2070 
2071  const Token &PeekAhead(unsigned N);
2072  void AnnotatePreviousCachedTokens(const Token &Tok);
2073 
2074  //===--------------------------------------------------------------------===//
2075  /// Handle*Directive - implement the various preprocessor directives. These
2076  /// should side-effect the current preprocessor object so that the next call
2077  /// to Lex() will return the appropriate token next.
2078  void HandleLineDirective();
2079  void HandleDigitDirective(Token &Tok);
2080  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2081  void HandleIdentSCCSDirective(Token &Tok);
2082  void HandleMacroPublicDirective(Token &Tok);
2083  void HandleMacroPrivateDirective();
2084 
2085  // File inclusion.
2086  void HandleIncludeDirective(SourceLocation HashLoc,
2087  Token &Tok,
2088  const DirectoryLookup *LookupFrom = nullptr,
2089  const FileEntry *LookupFromFile = nullptr,
2090  bool isImport = false);
2091  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2092  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2093  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2094  void HandleMicrosoftImportDirective(Token &Tok);
2095 
2096 public:
2097  /// Check that the given module is available, producing a diagnostic if not.
2098  /// \return \c true if the check failed (because the module is not available).
2099  /// \c false if the module appears to be usable.
2100  static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2101  const TargetInfo &TargetInfo,
2102  DiagnosticsEngine &Diags, Module *M);
2103 
2104  // Module inclusion testing.
2105  /// Find the module that owns the source or header file that
2106  /// \p Loc points to. If the location is in a file that was included
2107  /// into a module, or is outside any module, returns nullptr.
2108  Module *getModuleForLocation(SourceLocation Loc);
2109 
2110  /// We want to produce a diagnostic at location IncLoc concerning a
2111  /// missing module import.
2112  ///
2113  /// \param IncLoc The location at which the missing import was detected.
2114  /// \param M The desired module.
2115  /// \param MLoc A location within the desired module at which some desired
2116  /// effect occurred (eg, where a desired entity was declared).
2117  ///
2118  /// \return A file that can be #included to import a module containing MLoc.
2119  /// Null if no such file could be determined or if a #include is not
2120  /// appropriate.
2121  const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2122  Module *M,
2123  SourceLocation MLoc);
2124 
2125  bool isRecordingPreamble() const {
2126  return PreambleConditionalStack.isRecording();
2127  }
2128 
2129  bool hasRecordedPreamble() const {
2130  return PreambleConditionalStack.hasRecordedPreamble();
2131  }
2132 
2134  return PreambleConditionalStack.getStack();
2135  }
2136 
2138  PreambleConditionalStack.setStack(s);
2139  }
2140 
2143  PreambleConditionalStack.startReplaying();
2144  PreambleConditionalStack.setStack(s);
2145  PreambleConditionalStack.SkipInfo = SkipInfo;
2146  }
2147 
2149  return PreambleConditionalStack.SkipInfo;
2150  }
2151 
2152 private:
2153  /// After processing predefined file, initialize the conditional stack from
2154  /// the preamble.
2155  void replayPreambleConditionalStack();
2156 
2157  // Macro handling.
2158  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2159  void HandleUndefDirective();
2160 
2161  // Conditional Inclusion.
2162  void HandleIfdefDirective(Token &Result, const Token &HashToken,
2163  bool isIfndef, bool ReadAnyTokensBeforeDirective);
2164  void HandleIfDirective(Token &IfToken, const Token &HashToken,
2165  bool ReadAnyTokensBeforeDirective);
2166  void HandleEndifDirective(Token &EndifToken);
2167  void HandleElseDirective(Token &Result, const Token &HashToken);
2168  void HandleElifDirective(Token &ElifToken, const Token &HashToken);
2169 
2170  // Pragmas.
2171  void HandlePragmaDirective(SourceLocation IntroducerLoc,
2172  PragmaIntroducerKind Introducer);
2173 
2174 public:
2175  void HandlePragmaOnce(Token &OnceTok);
2176  void HandlePragmaMark();
2177  void HandlePragmaPoison();
2178  void HandlePragmaSystemHeader(Token &SysHeaderTok);
2179  void HandlePragmaDependency(Token &DependencyTok);
2180  void HandlePragmaPushMacro(Token &Tok);
2181  void HandlePragmaPopMacro(Token &Tok);
2182  void HandlePragmaIncludeAlias(Token &Tok);
2183  void HandlePragmaModuleBuild(Token &Tok);
2184  void HandlePragmaHdrstop(Token &Tok);
2185  IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2186 
2187  // Return true and store the first token only if any CommentHandler
2188  // has inserted some tokens and getCommentRetentionState() is false.
2189  bool HandleComment(Token &result, SourceRange Comment);
2190 
2191  /// A macro is used, update information about macros that need unused
2192  /// warnings.
2193  void markMacroAsUsed(MacroInfo *MI);
2194 };
2195 
2196 /// Abstract base class that describes a handler that will receive
2197 /// source ranges for each of the comments encountered in the source file.
2199 public:
2200  virtual ~CommentHandler();
2201 
2202  // The handler shall return true if it has pushed any tokens
2203  // to be read using e.g. EnterToken or EnterTokenStream.
2204  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2205 };
2206 
2207 /// Registry of pragma handlers added by plugins
2208 using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2209 
2210 } // namespace clang
2211 
2212 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
Definition: Preprocessor.h:334
A set of visible modules.
Definition: Module.h:588
StringRef Identifier
Definition: Format.cpp:1681
llvm::BumpPtrAllocator & getPreprocessorAllocator()
Definition: Preprocessor.h:825
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:76
SelectorTable & getSelectorTable()
Definition: Preprocessor.h:823
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:67
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:116
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the &#39;spelling&#39; of the Tok token.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:29
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:94
Defines the SourceManager interface.
static const Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:20
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
FileManager & getFileManager() const
Definition: Preprocessor.h:817
Defines the clang::Module class, which describes a module in the source code.
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
SourceLocation getPragmaARCCFCodeAuditedLoc() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
StringRef P
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:564
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1296
A directive for an undefined macro.
Definition: MacroInfo.h:429
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool hadModuleLoaderFatalFailure() const
Definition: Preprocessor.h:838
void EnterToken(const Token &Tok)
Enters a token in the token stream to be lexed next.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:99
Builtin::Context & getBuiltinInfo()
Definition: Preprocessor.h:824
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:35
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:117
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Definition: MacroInfo.h:328
tok::TokenKind getKind() const
Definition: Token.h:89
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:89
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
One of these records is kept for each identifier that is lexed.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
Represents a macro directive exported by a module.
Definition: MacroInfo.h:488
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:406
This table allows us to fully hide how we implement multi-keyword caching.
LineState State
Definition: Format.h:2153
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:815
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Definition: Preprocessor.h:848
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:49
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:814
Describes a module or submodule.
Definition: Module.h:64
bool isPragmaLexer() const
isPragmaLexer - Returns true if this Lexer is being used to lex a pragma.
Definition: Lexer.h:183
A directive for setting the module visibility of a macro.
Definition: MacroInfo.h:444
bool isInvalid() const
void SetSuppressIncludeNotFoundError(bool Suppress)
Definition: Preprocessor.h:858
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:106
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
Definition: Preprocessor.h:895
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:819
const FormatToken & Tok
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void LexNonComment(Token &Result)
Lex a token.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:148
bool isMacroDefined(const IdentifierInfo *II)
Definition: Preprocessor.h:917
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we&#39;ve alread...
Definition: Preprocessor.h:925
Defines the Diagnostic-related interfaces.
const TargetInfo * getAuxTargetInfo() const
Definition: Preprocessor.h:816
TokenLexer - This implements a lexer that returns tokens from a macro body or token stream instead of...
Definition: TokenLexer.h:30
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
Definition: Preprocessor.h:868
PragmaIntroducerKind
Describes how the pragma was introduced, e.g., with #pragma, _Pragma, or __pragma.
Definition: Pragma.h:31
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:158
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:1044
void clearCodeCompletionHandler()
Clear out the code completion handler.
StringRef Filename
Definition: Format.cpp:1674
Provides lookups to, and iteration over, IdentiferInfo objects.
unsigned Offset
Definition: Format.cpp:1676
Exposes information about the current target.
Definition: TargetInfo.h:161
const IdentifierTable & getIdentifierTable() const
Definition: Preprocessor.h:822
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Abstract interface for external sources of preprocessor information.
Defines the clang::LangOptions interface.
unsigned getCounterValue() const
SourceLocation End
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:349
int Id
Definition: ASTDiff.cpp:190
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
const AnnotatedLine * Line
bool isDefined() const
Definition: MacroInfo.h:384
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Definition: Preprocessor.h:877
Implements an efficient mapping from strings to IdentifierInfo nodes.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
A class for tracking whether we&#39;re inside a VA_OPT during a traversal of the tokens of a variadic mac...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:123
#define bool
Definition: stdbool.h:31
DefInfo getDefinition()
Traverses the macro directives history and returns the next macro definition directive along with inf...
Definition: MacroInfo.cpp:178
SourceLocation Begin
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:84
llvm::Optional< PreambleSkipInfo > getPreambleSkipInfo() const
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
Definition: Preprocessor.h:979
bool getCommentRetentionState() const
Definition: Preprocessor.h:853
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:97
PPCallbacks * getPPCallbacks() const
Definition: Preprocessor.h:905
const SourceManager & SM
Definition: Format.cpp:1535
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
bool isRecordingPreamble() const
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
SourceManager & getSourceManager() const
Definition: Preprocessor.h:818
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define&#39;d and not #undef&#39;...
Definition: Preprocessor.h:968
const DefInfo findDirectiveAtLoc(SourceLocation L, const SourceManager &SM) const
Find macro definition active in the specified source location.
Definition: MacroInfo.cpp:202
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:831
llvm::cl::opt< std::string > Filter
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:290
Kind
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
void setExternalSource(ExternalPreprocessorSource *Source)
Definition: Preprocessor.h:827
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.h:93
void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:938
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:176
IdentifierTable & getIdentifierTable()
Definition: Preprocessor.h:821
void setPragmasEnabled(bool Enabled)
Definition: Preprocessor.h:855
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:85
bool isPoisoned() const
Return true if this token has been poisoned.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:24
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
Definition: Preprocessor.h:951
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
Definition: Preprocessor.h:885
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
const MacroInfo * getMacroInfo() const
Definition: MacroInfo.h:390
StringRef getName() const
Return the actual identifier string.
bool isMacroDefined(StringRef Id)
Definition: Preprocessor.h:914
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
Dataflow Directional Tag Classes.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion)
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
Definition: DeclSpec.h:1141
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion)
SourceRange getCodeCompletionTokenRange() const
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:354
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:58
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Abstract interface for a module loader.
Definition: ModuleLoader.h:73
unsigned getLength() const
Definition: Token.h:126
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:47
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:214
bool getPragmasEnabled() const
Definition: Preprocessor.h:856
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:123
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
Definition: Preprocessor.h:836
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Definition: Preprocessor.h:809
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:45
Defines the PPCallbacks interface.
Defines the clang::TokenKind enum and support functions.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
SourceLocation getModuleImportLoc(Module *M) const
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:898
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:811
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:361
bool hasRecordedPreamble() const
const std::string & getPredefines() const
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:282
void setPredefines(StringRef P)
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
The translation unit is a complete translation unit.
Definition: LangOptions.h:363
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
MacroMap::const_iterator macro_iterator
Abstract base class that describes a handler that will receive source ranges for each of the comments...
void setCounterValue(unsigned V)
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
Definition: Preprocessor.h:843
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
Definition: Preprocessor.h:862
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, llvm::Optional< PreambleSkipInfo > SkipInfo)
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Definition: Preprocessor.h:983
Callback handler that receives notifications when performing code completion within the preprocessor...
A header that is known to reside within a given module, whether it was included or excluded...
Definition: ModuleMap.h:150
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
Definition: Preprocessor.h:874
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:906
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:123
void setDiagnostics(DiagnosticsEngine &D)
Definition: Preprocessor.h:812