clang  7.0.0svn
Preprocessor.h
Go to the documentation of this file.
1 //===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// Defines the clang::Preprocessor interface.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
16 #define LLVM_CLANG_LEX_PREPROCESSOR_H
17 
18 #include "clang/Basic/Builtins.h"
19 #include "clang/Basic/Diagnostic.h"
21 #include "clang/Basic/LLVM.h"
23 #include "clang/Basic/Module.h"
26 #include "clang/Basic/TokenKinds.h"
27 #include "clang/Lex/Lexer.h"
28 #include "clang/Lex/MacroInfo.h"
29 #include "clang/Lex/ModuleLoader.h"
30 #include "clang/Lex/ModuleMap.h"
31 #include "clang/Lex/PPCallbacks.h"
32 #include "clang/Lex/PTHLexer.h"
33 #include "clang/Lex/Token.h"
34 #include "clang/Lex/TokenLexer.h"
35 #include "llvm/ADT/ArrayRef.h"
36 #include "llvm/ADT/DenseMap.h"
37 #include "llvm/ADT/FoldingSet.h"
38 #include "llvm/ADT/None.h"
39 #include "llvm/ADT/Optional.h"
40 #include "llvm/ADT/PointerUnion.h"
41 #include "llvm/ADT/STLExtras.h"
42 #include "llvm/ADT/SmallPtrSet.h"
43 #include "llvm/ADT/SmallVector.h"
44 #include "llvm/ADT/StringRef.h"
45 #include "llvm/ADT/TinyPtrVector.h"
46 #include "llvm/ADT/iterator_range.h"
47 #include "llvm/Support/Allocator.h"
48 #include "llvm/Support/Casting.h"
49 #include "llvm/Support/Registry.h"
50 #include <cassert>
51 #include <cstddef>
52 #include <cstdint>
53 #include <memory>
54 #include <map>
55 #include <string>
56 #include <utility>
57 #include <vector>
58 
59 namespace llvm {
60 
61 template<unsigned InternalLen> class SmallString;
62 
63 } // namespace llvm
64 
65 namespace clang {
66 
67 class CodeCompletionHandler;
68 class CommentHandler;
69 class DirectoryEntry;
70 class DirectoryLookup;
71 class ExternalPreprocessorSource;
72 class FileEntry;
73 class FileManager;
74 class HeaderSearch;
75 class MacroArgs;
76 class MemoryBufferCache;
77 class PragmaHandler;
78 class PragmaNamespace;
79 class PreprocessingRecord;
80 class PreprocessorLexer;
81 class PreprocessorOptions;
82 class PTHManager;
83 class ScratchBuffer;
84 class TargetInfo;
85 
86 /// Stores token information for comparing actual tokens with
87 /// predefined values. Only handles simple tokens and identifiers.
88 class TokenValue {
90  IdentifierInfo *II;
91 
92 public:
93  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
94  assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
95  assert(Kind != tok::identifier &&
96  "Identifiers should be created by TokenValue(IdentifierInfo *)");
97  assert(!tok::isLiteral(Kind) && "Literals are not supported.");
98  assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
99  }
100 
101  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
102 
103  bool operator==(const Token &Tok) const {
104  return Tok.getKind() == Kind &&
105  (!II || II == Tok.getIdentifierInfo());
106  }
107 };
108 
109 /// Context in which macro name is used.
110 enum MacroUse {
111  // other than #define or #undef
112  MU_Other = 0,
113 
114  // macro name specified in #define
116 
117  // macro name specified in #undef
119 };
120 
121 /// Engages in a tight little dance with the lexer to efficiently
122 /// preprocess tokens.
123 ///
124 /// Lexers know only about tokens within a single source file, and don't
125 /// know anything about preprocessor-level issues like the \#include stack,
126 /// token expansion, etc.
130 
131  std::shared_ptr<PreprocessorOptions> PPOpts;
132  DiagnosticsEngine *Diags;
133  LangOptions &LangOpts;
134  const TargetInfo *Target = nullptr;
135  const TargetInfo *AuxTarget = nullptr;
136  FileManager &FileMgr;
137  SourceManager &SourceMgr;
138  MemoryBufferCache &PCMCache;
139  std::unique_ptr<ScratchBuffer> ScratchBuf;
140  HeaderSearch &HeaderInfo;
141  ModuleLoader &TheModuleLoader;
142 
143  /// External source of macros.
144  ExternalPreprocessorSource *ExternalSource;
145 
146  /// An optional PTHManager object used for getting tokens from
147  /// a token cache rather than lexing the original source file.
148  std::unique_ptr<PTHManager> PTH;
149 
150  /// A BumpPtrAllocator object used to quickly allocate and release
151  /// objects internal to the Preprocessor.
152  llvm::BumpPtrAllocator BP;
153 
154  /// Identifiers for builtin macros and other builtins.
155  IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
156  IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
157  IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
158  IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
159  IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
160  IdentifierInfo *Ident__COUNTER__; // __COUNTER__
161  IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
162  IdentifierInfo *Ident__identifier; // __identifier
163  IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
164  IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
165  IdentifierInfo *Ident__has_feature; // __has_feature
166  IdentifierInfo *Ident__has_extension; // __has_extension
167  IdentifierInfo *Ident__has_builtin; // __has_builtin
168  IdentifierInfo *Ident__has_attribute; // __has_attribute
169  IdentifierInfo *Ident__has_include; // __has_include
170  IdentifierInfo *Ident__has_include_next; // __has_include_next
171  IdentifierInfo *Ident__has_warning; // __has_warning
172  IdentifierInfo *Ident__is_identifier; // __is_identifier
173  IdentifierInfo *Ident__building_module; // __building_module
174  IdentifierInfo *Ident__MODULE__; // __MODULE__
175  IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
176  IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
177  IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
178  IdentifierInfo *Ident__is_target_arch; // __is_target_arch
179  IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
180  IdentifierInfo *Ident__is_target_os; // __is_target_os
181  IdentifierInfo *Ident__is_target_environment; // __is_target_environment
182 
183  SourceLocation DATELoc, TIMELoc;
184 
185  // Next __COUNTER__ value, starts at 0.
186  unsigned CounterValue = 0;
187 
188  enum {
189  /// Maximum depth of \#includes.
190  MaxAllowedIncludeStackDepth = 200
191  };
192 
193  // State that is set before the preprocessor begins.
194  bool KeepComments : 1;
195  bool KeepMacroComments : 1;
196  bool SuppressIncludeNotFoundError : 1;
197 
198  // State that changes while the preprocessor runs:
199  bool InMacroArgs : 1; // True if parsing fn macro invocation args.
200 
201  /// Whether the preprocessor owns the header search object.
202  bool OwnsHeaderSearch : 1;
203 
204  /// True if macro expansion is disabled.
205  bool DisableMacroExpansion : 1;
206 
207  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
208  /// when parsing preprocessor directives.
209  bool MacroExpansionInDirectivesOverride : 1;
210 
212 
213  /// Whether we have already loaded macros from the external source.
214  mutable bool ReadMacrosFromExternalSource : 1;
215 
216  /// True if pragmas are enabled.
217  bool PragmasEnabled : 1;
218 
219  /// True if the current build action is a preprocessing action.
220  bool PreprocessedOutput : 1;
221 
222  /// True if we are currently preprocessing a #if or #elif directive
223  bool ParsingIfOrElifDirective;
224 
225  /// True if we are pre-expanding macro arguments.
226  bool InMacroArgPreExpansion;
227 
228  /// Mapping/lookup information for all identifiers in
229  /// the program, including program keywords.
230  mutable IdentifierTable Identifiers;
231 
232  /// This table contains all the selectors in the program.
233  ///
234  /// Unlike IdentifierTable above, this table *isn't* populated by the
235  /// preprocessor. It is declared/expanded here because its role/lifetime is
236  /// conceptually similar to the IdentifierTable. In addition, the current
237  /// control flow (in clang::ParseAST()), make it convenient to put here.
238  ///
239  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
240  /// the lifetime of the preprocessor.
241  SelectorTable Selectors;
242 
243  /// Information about builtins.
245 
246  /// Tracks all of the pragmas that the client registered
247  /// with this preprocessor.
248  std::unique_ptr<PragmaNamespace> PragmaHandlers;
249 
250  /// Pragma handlers of the original source is stored here during the
251  /// parsing of a model file.
252  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
253 
254  /// Tracks all of the comment handlers that the client registered
255  /// with this preprocessor.
256  std::vector<CommentHandler *> CommentHandlers;
257 
258  /// True if we want to ignore EOF token and continue later on (thus
259  /// avoid tearing the Lexer and etc. down).
260  bool IncrementalProcessing = false;
261 
262  /// The kind of translation unit we are processing.
263  TranslationUnitKind TUKind;
264 
265  /// The code-completion handler.
266  CodeCompletionHandler *CodeComplete = nullptr;
267 
268  /// The file that we're performing code-completion for, if any.
269  const FileEntry *CodeCompletionFile = nullptr;
270 
271  /// The offset in file for the code-completion point.
272  unsigned CodeCompletionOffset = 0;
273 
274  /// The location for the code-completion point. This gets instantiated
275  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
276  SourceLocation CodeCompletionLoc;
277 
278  /// The start location for the file of the code-completion point.
279  ///
280  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
281  /// for preprocessing.
282  SourceLocation CodeCompletionFileLoc;
283 
284  /// The source location of the \c import contextual keyword we just
285  /// lexed, if any.
286  SourceLocation ModuleImportLoc;
287 
288  /// The module import path that we're currently processing.
290 
291  /// Whether the last token we lexed was an '@'.
292  bool LastTokenWasAt = false;
293 
294  /// Whether the module import expects an identifier next. Otherwise,
295  /// it expects a '.' or ';'.
296  bool ModuleImportExpectsIdentifier = false;
297 
298  /// The source location of the currently-active
299  /// \#pragma clang arc_cf_code_audited begin.
300  SourceLocation PragmaARCCFCodeAuditedLoc;
301 
302  /// The source location of the currently-active
303  /// \#pragma clang assume_nonnull begin.
304  SourceLocation PragmaAssumeNonNullLoc;
305 
306  /// True if we hit the code-completion point.
307  bool CodeCompletionReached = false;
308 
309  /// The code completion token containing the information
310  /// on the stem that is to be code completed.
311  IdentifierInfo *CodeCompletionII = nullptr;
312 
313  /// The directory that the main file should be considered to occupy,
314  /// if it does not correspond to a real file (as happens when building a
315  /// module).
316  const DirectoryEntry *MainFileDir = nullptr;
317 
318  /// The number of bytes that we will initially skip when entering the
319  /// main file, along with a flag that indicates whether skipping this number
320  /// of bytes will place the lexer at the start of a line.
321  ///
322  /// This is used when loading a precompiled preamble.
323  std::pair<int, bool> SkipMainFilePreamble;
324 
325 public:
330  bool FoundElse;
332 
334  bool FoundNonSkipPortion, bool FoundElse,
335  SourceLocation ElseLoc)
336  : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
337  FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
338  ElseLoc(ElseLoc) {}
339  };
340 
341 private:
342  friend class ASTReader;
343  friend class MacroArgs;
344 
345  class PreambleConditionalStackStore {
346  enum State {
347  Off = 0,
348  Recording = 1,
349  Replaying = 2,
350  };
351 
352  public:
353  PreambleConditionalStackStore() = default;
354 
355  void startRecording() { ConditionalStackState = Recording; }
356  void startReplaying() { ConditionalStackState = Replaying; }
357  bool isRecording() const { return ConditionalStackState == Recording; }
358  bool isReplaying() const { return ConditionalStackState == Replaying; }
359 
360  ArrayRef<PPConditionalInfo> getStack() const {
361  return ConditionalStack;
362  }
363 
364  void doneReplaying() {
365  ConditionalStack.clear();
366  ConditionalStackState = Off;
367  }
368 
369  void setStack(ArrayRef<PPConditionalInfo> s) {
370  if (!isRecording() && !isReplaying())
371  return;
372  ConditionalStack.clear();
373  ConditionalStack.append(s.begin(), s.end());
374  }
375 
376  bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
377 
378  bool reachedEOFWhileSkipping() const { return SkipInfo.hasValue(); }
379 
380  void clearSkipInfo() { SkipInfo.reset(); }
381 
383 
384  private:
385  SmallVector<PPConditionalInfo, 4> ConditionalStack;
386  State ConditionalStackState = Off;
387  } PreambleConditionalStack;
388 
389  /// The current top of the stack that we're lexing from if
390  /// not expanding a macro and we are lexing directly from source code.
391  ///
392  /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
393  std::unique_ptr<Lexer> CurLexer;
394 
395  /// The current top of stack that we're lexing from if
396  /// not expanding from a macro and we are lexing from a PTH cache.
397  ///
398  /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
399  std::unique_ptr<PTHLexer> CurPTHLexer;
400 
401  /// The current top of the stack what we're lexing from
402  /// if not expanding a macro.
403  ///
404  /// This is an alias for either CurLexer or CurPTHLexer.
405  PreprocessorLexer *CurPPLexer = nullptr;
406 
407  /// Used to find the current FileEntry, if CurLexer is non-null
408  /// and if applicable.
409  ///
410  /// This allows us to implement \#include_next and find directory-specific
411  /// properties.
412  const DirectoryLookup *CurDirLookup = nullptr;
413 
414  /// The current macro we are expanding, if we are expanding a macro.
415  ///
416  /// One of CurLexer and CurTokenLexer must be null.
417  std::unique_ptr<TokenLexer> CurTokenLexer;
418 
419  /// The kind of lexer we're currently working with.
420  enum CurLexerKind {
421  CLK_Lexer,
422  CLK_PTHLexer,
423  CLK_TokenLexer,
424  CLK_CachingLexer,
425  CLK_LexAfterModuleImport
426  } CurLexerKind = CLK_Lexer;
427 
428  /// If the current lexer is for a submodule that is being built, this
429  /// is that submodule.
430  Module *CurLexerSubmodule = nullptr;
431 
432  /// Keeps track of the stack of files currently
433  /// \#included, and macros currently being expanded from, not counting
434  /// CurLexer/CurTokenLexer.
435  struct IncludeStackInfo {
436  enum CurLexerKind CurLexerKind;
437  Module *TheSubmodule;
438  std::unique_ptr<Lexer> TheLexer;
439  std::unique_ptr<PTHLexer> ThePTHLexer;
440  PreprocessorLexer *ThePPLexer;
441  std::unique_ptr<TokenLexer> TheTokenLexer;
442  const DirectoryLookup *TheDirLookup;
443 
444  // The following constructors are completely useless copies of the default
445  // versions, only needed to pacify MSVC.
446  IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
447  std::unique_ptr<Lexer> &&TheLexer,
448  std::unique_ptr<PTHLexer> &&ThePTHLexer,
449  PreprocessorLexer *ThePPLexer,
450  std::unique_ptr<TokenLexer> &&TheTokenLexer,
451  const DirectoryLookup *TheDirLookup)
452  : CurLexerKind(std::move(CurLexerKind)),
453  TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
454  ThePTHLexer(std::move(ThePTHLexer)),
455  ThePPLexer(std::move(ThePPLexer)),
456  TheTokenLexer(std::move(TheTokenLexer)),
457  TheDirLookup(std::move(TheDirLookup)) {}
458  };
459  std::vector<IncludeStackInfo> IncludeMacroStack;
460 
461  /// Actions invoked when some preprocessor activity is
462  /// encountered (e.g. a file is \#included, etc).
463  std::unique_ptr<PPCallbacks> Callbacks;
464 
465  struct MacroExpandsInfo {
466  Token Tok;
467  MacroDefinition MD;
468  SourceRange Range;
469 
470  MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
471  : Tok(Tok), MD(MD), Range(Range) {}
472  };
473  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
474 
475  /// Information about a name that has been used to define a module macro.
476  struct ModuleMacroInfo {
477  /// The most recent macro directive for this identifier.
478  MacroDirective *MD;
479 
480  /// The active module macros for this identifier.
481  llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
482 
483  /// The generation number at which we last updated ActiveModuleMacros.
484  /// \see Preprocessor::VisibleModules.
485  unsigned ActiveModuleMacrosGeneration = 0;
486 
487  /// Whether this macro name is ambiguous.
488  bool IsAmbiguous = false;
489 
490  /// The module macros that are overridden by this macro.
491  llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
492 
493  ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
494  };
495 
496  /// The state of a macro for an identifier.
497  class MacroState {
498  mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
499 
500  ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
501  const IdentifierInfo *II) const {
502  if (II->isOutOfDate())
503  PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
504  // FIXME: Find a spare bit on IdentifierInfo and store a
505  // HasModuleMacros flag.
506  if (!II->hasMacroDefinition() ||
507  (!PP.getLangOpts().Modules &&
508  !PP.getLangOpts().ModulesLocalVisibility) ||
509  !PP.CurSubmoduleState->VisibleModules.getGeneration())
510  return nullptr;
511 
512  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
513  if (!Info) {
514  Info = new (PP.getPreprocessorAllocator())
515  ModuleMacroInfo(State.get<MacroDirective *>());
516  State = Info;
517  }
518 
519  if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
520  Info->ActiveModuleMacrosGeneration)
521  PP.updateModuleMacroInfo(II, *Info);
522  return Info;
523  }
524 
525  public:
526  MacroState() : MacroState(nullptr) {}
527  MacroState(MacroDirective *MD) : State(MD) {}
528 
529  MacroState(MacroState &&O) noexcept : State(O.State) {
530  O.State = (MacroDirective *)nullptr;
531  }
532 
533  MacroState &operator=(MacroState &&O) noexcept {
534  auto S = O.State;
535  O.State = (MacroDirective *)nullptr;
536  State = S;
537  return *this;
538  }
539 
540  ~MacroState() {
541  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
542  Info->~ModuleMacroInfo();
543  }
544 
545  MacroDirective *getLatest() const {
546  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
547  return Info->MD;
548  return State.get<MacroDirective*>();
549  }
550 
551  void setLatest(MacroDirective *MD) {
552  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
553  Info->MD = MD;
554  else
555  State = MD;
556  }
557 
558  bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
559  auto *Info = getModuleInfo(PP, II);
560  return Info ? Info->IsAmbiguous : false;
561  }
562 
564  getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
565  if (auto *Info = getModuleInfo(PP, II))
566  return Info->ActiveModuleMacros;
567  return None;
568  }
569 
570  MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
571  SourceManager &SourceMgr) const {
572  // FIXME: Incorporate module macros into the result of this.
573  if (auto *Latest = getLatest())
574  return Latest->findDirectiveAtLoc(Loc, SourceMgr);
575  return {};
576  }
577 
578  void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
579  if (auto *Info = getModuleInfo(PP, II)) {
580  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
581  Info->ActiveModuleMacros.begin(),
582  Info->ActiveModuleMacros.end());
583  Info->ActiveModuleMacros.clear();
584  Info->IsAmbiguous = false;
585  }
586  }
587 
588  ArrayRef<ModuleMacro*> getOverriddenMacros() const {
589  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
590  return Info->OverriddenMacros;
591  return None;
592  }
593 
594  void setOverriddenMacros(Preprocessor &PP,
595  ArrayRef<ModuleMacro *> Overrides) {
596  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
597  if (!Info) {
598  if (Overrides.empty())
599  return;
600  Info = new (PP.getPreprocessorAllocator())
601  ModuleMacroInfo(State.get<MacroDirective *>());
602  State = Info;
603  }
604  Info->OverriddenMacros.clear();
605  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
606  Overrides.begin(), Overrides.end());
607  Info->ActiveModuleMacrosGeneration = 0;
608  }
609  };
610 
611  /// For each IdentifierInfo that was associated with a macro, we
612  /// keep a mapping to the history of all macro definitions and #undefs in
613  /// the reverse order (the latest one is in the head of the list).
614  ///
615  /// This mapping lives within the \p CurSubmoduleState.
616  using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
617 
618  struct SubmoduleState;
619 
620  /// Information about a submodule that we're currently building.
621  struct BuildingSubmoduleInfo {
622  /// The module that we are building.
623  Module *M;
624 
625  /// The location at which the module was included.
626  SourceLocation ImportLoc;
627 
628  /// Whether we entered this submodule via a pragma.
629  bool IsPragma;
630 
631  /// The previous SubmoduleState.
632  SubmoduleState *OuterSubmoduleState;
633 
634  /// The number of pending module macro names when we started building this.
635  unsigned OuterPendingModuleMacroNames;
636 
637  BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
638  SubmoduleState *OuterSubmoduleState,
639  unsigned OuterPendingModuleMacroNames)
640  : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
641  OuterSubmoduleState(OuterSubmoduleState),
642  OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
643  };
644  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
645 
646  /// Information about a submodule's preprocessor state.
647  struct SubmoduleState {
648  /// The macros for the submodule.
649  MacroMap Macros;
650 
651  /// The set of modules that are visible within the submodule.
652  VisibleModuleSet VisibleModules;
653 
654  // FIXME: CounterValue?
655  // FIXME: PragmaPushMacroInfo?
656  };
657  std::map<Module *, SubmoduleState> Submodules;
658 
659  /// The preprocessor state for preprocessing outside of any submodule.
660  SubmoduleState NullSubmoduleState;
661 
662  /// The current submodule state. Will be \p NullSubmoduleState if we're not
663  /// in a submodule.
664  SubmoduleState *CurSubmoduleState;
665 
666  /// The set of known macros exported from modules.
667  llvm::FoldingSet<ModuleMacro> ModuleMacros;
668 
669  /// The names of potential module macros that we've not yet processed.
670  llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
671 
672  /// The list of module macros, for each identifier, that are not overridden by
673  /// any other module macro.
674  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
675  LeafModuleMacros;
676 
677  /// Macros that we want to warn because they are not used at the end
678  /// of the translation unit.
679  ///
680  /// We store just their SourceLocations instead of
681  /// something like MacroInfo*. The benefit of this is that when we are
682  /// deserializing from PCH, we don't need to deserialize identifier & macros
683  /// just so that we can report that they are unused, we just warn using
684  /// the SourceLocations of this set (that will be filled by the ASTReader).
685  /// We are using SmallPtrSet instead of a vector for faster removal.
686  using WarnUnusedMacroLocsTy = llvm::SmallPtrSet<SourceLocation, 32>;
687  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
688 
689  /// A "freelist" of MacroArg objects that can be
690  /// reused for quick allocation.
691  MacroArgs *MacroArgCache = nullptr;
692 
693  /// For each IdentifierInfo used in a \#pragma push_macro directive,
694  /// we keep a MacroInfo stack used to restore the previous macro value.
695  llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
696  PragmaPushMacroInfo;
697 
698  // Various statistics we track for performance analysis.
699  unsigned NumDirectives = 0;
700  unsigned NumDefined = 0;
701  unsigned NumUndefined = 0;
702  unsigned NumPragma = 0;
703  unsigned NumIf = 0;
704  unsigned NumElse = 0;
705  unsigned NumEndif = 0;
706  unsigned NumEnteredSourceFiles = 0;
707  unsigned MaxIncludeStackDepth = 0;
708  unsigned NumMacroExpanded = 0;
709  unsigned NumFnMacroExpanded = 0;
710  unsigned NumBuiltinMacroExpanded = 0;
711  unsigned NumFastMacroExpanded = 0;
712  unsigned NumTokenPaste = 0;
713  unsigned NumFastTokenPaste = 0;
714  unsigned NumSkipped = 0;
715 
716  /// The predefined macros that preprocessor should use from the
717  /// command line etc.
718  std::string Predefines;
719 
720  /// The file ID for the preprocessor predefines.
721  FileID PredefinesFileID;
722 
723  /// The file ID for the PCH through header.
724  FileID PCHThroughHeaderFileID;
725 
726  /// Whether tokens are being skipped until the through header is seen.
727  bool SkippingUntilPCHThroughHeader = false;
728 
729  /// \{
730  /// Cache of macro expanders to reduce malloc traffic.
731  enum { TokenLexerCacheSize = 8 };
732  unsigned NumCachedTokenLexers;
733  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
734  /// \}
735 
736  /// Keeps macro expanded tokens for TokenLexers.
737  //
738  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
739  /// going to lex in the cache and when it finishes the tokens are removed
740  /// from the end of the cache.
741  SmallVector<Token, 16> MacroExpandedTokens;
742  std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
743 
744  /// A record of the macro definitions and expansions that
745  /// occurred during preprocessing.
746  ///
747  /// This is an optional side structure that can be enabled with
748  /// \c createPreprocessingRecord() prior to preprocessing.
749  PreprocessingRecord *Record = nullptr;
750 
751  /// Cached tokens state.
753 
754  /// Cached tokens are stored here when we do backtracking or
755  /// lookahead. They are "lexed" by the CachingLex() method.
757 
758  /// The position of the cached token that CachingLex() should
759  /// "lex" next.
760  ///
761  /// If it points beyond the CachedTokens vector, it means that a normal
762  /// Lex() should be invoked.
763  CachedTokensTy::size_type CachedLexPos = 0;
764 
765  /// Stack of backtrack positions, allowing nested backtracks.
766  ///
767  /// The EnableBacktrackAtThisPos() method pushes a position to
768  /// indicate where CachedLexPos should be set when the BackTrack() method is
769  /// invoked (at which point the last position is popped).
770  std::vector<CachedTokensTy::size_type> BacktrackPositions;
771 
772  struct MacroInfoChain {
773  MacroInfo MI;
774  MacroInfoChain *Next;
775  };
776 
777  /// MacroInfos are managed as a chain for easy disposal. This is the head
778  /// of that list.
779  MacroInfoChain *MIChainHead = nullptr;
780 
781  void updateOutOfDateIdentifier(IdentifierInfo &II) const;
782 
783 public:
784  Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
786  MemoryBufferCache &PCMCache,
787  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
788  IdentifierInfoLookup *IILookup = nullptr,
789  bool OwnsHeaderSearch = false,
791 
792  ~Preprocessor();
793 
794  /// Initialize the preprocessor using information about the target.
795  ///
796  /// \param Target is owned by the caller and must remain valid for the
797  /// lifetime of the preprocessor.
798  /// \param AuxTarget is owned by the caller and must remain valid for
799  /// the lifetime of the preprocessor.
800  void Initialize(const TargetInfo &Target,
801  const TargetInfo *AuxTarget = nullptr);
802 
803  /// Initialize the preprocessor to parse a model file
804  ///
805  /// To parse model files the preprocessor of the original source is reused to
806  /// preserver the identifier table. However to avoid some duplicate
807  /// information in the preprocessor some cleanup is needed before it is used
808  /// to parse model files. This method does that cleanup.
809  void InitializeForModelFile();
810 
811  /// Cleanup after model file parsing
812  void FinalizeForModelFile();
813 
814  /// Retrieve the preprocessor options used to initialize this
815  /// preprocessor.
816  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
817 
818  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
819  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
820 
821  const LangOptions &getLangOpts() const { return LangOpts; }
822  const TargetInfo &getTargetInfo() const { return *Target; }
823  const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
824  FileManager &getFileManager() const { return FileMgr; }
825  SourceManager &getSourceManager() const { return SourceMgr; }
826  MemoryBufferCache &getPCMCache() const { return PCMCache; }
827  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
828 
829  IdentifierTable &getIdentifierTable() { return Identifiers; }
830  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
831  SelectorTable &getSelectorTable() { return Selectors; }
833  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
834 
835  void setPTHManager(PTHManager* pm);
836 
837  PTHManager *getPTHManager() { return PTH.get(); }
838 
840  ExternalSource = Source;
841  }
842 
844  return ExternalSource;
845  }
846 
847  /// Retrieve the module loader associated with this preprocessor.
848  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
849 
851  return TheModuleLoader.HadFatalFailure;
852  }
853 
854  /// True if we are currently preprocessing a #if or #elif directive
856  return ParsingIfOrElifDirective;
857  }
858 
859  /// Control whether the preprocessor retains comments in output.
860  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
861  this->KeepComments = KeepComments | KeepMacroComments;
862  this->KeepMacroComments = KeepMacroComments;
863  }
864 
865  bool getCommentRetentionState() const { return KeepComments; }
866 
867  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
868  bool getPragmasEnabled() const { return PragmasEnabled; }
869 
870  void SetSuppressIncludeNotFoundError(bool Suppress) {
871  SuppressIncludeNotFoundError = Suppress;
872  }
873 
875  return SuppressIncludeNotFoundError;
876  }
877 
878  /// Sets whether the preprocessor is responsible for producing output or if
879  /// it is producing tokens to be consumed by Parse and Sema.
880  void setPreprocessedOutput(bool IsPreprocessedOutput) {
881  PreprocessedOutput = IsPreprocessedOutput;
882  }
883 
884  /// Returns true if the preprocessor is responsible for generating output,
885  /// false if it is producing tokens to be consumed by Parse and Sema.
886  bool isPreprocessedOutput() const { return PreprocessedOutput; }
887 
888  /// Return true if we are lexing directly from the specified lexer.
889  bool isCurrentLexer(const PreprocessorLexer *L) const {
890  return CurPPLexer == L;
891  }
892 
893  /// Return the current lexer being lexed from.
894  ///
895  /// Note that this ignores any potentially active macro expansions and _Pragma
896  /// expansions going on at the time.
897  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
898 
899  /// Return the current file lexer being lexed from.
900  ///
901  /// Note that this ignores any potentially active macro expansions and _Pragma
902  /// expansions going on at the time.
903  PreprocessorLexer *getCurrentFileLexer() const;
904 
905  /// Return the submodule owning the file being lexed. This may not be
906  /// the current module if we have changed modules since entering the file.
907  Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
908 
909  /// Returns the FileID for the preprocessor predefines.
910  FileID getPredefinesFileID() const { return PredefinesFileID; }
911 
912  /// \{
913  /// Accessors for preprocessor callbacks.
914  ///
915  /// Note that this class takes ownership of any PPCallbacks object given to
916  /// it.
917  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
918  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
919  if (Callbacks)
920  C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
921  std::move(Callbacks));
922  Callbacks = std::move(C);
923  }
924  /// \}
925 
926  bool isMacroDefined(StringRef Id) {
927  return isMacroDefined(&Identifiers.get(Id));
928  }
929  bool isMacroDefined(const IdentifierInfo *II) {
930  return II->hasMacroDefinition() &&
931  (!getLangOpts().Modules || (bool)getMacroDefinition(II));
932  }
933 
934  /// Determine whether II is defined as a macro within the module M,
935  /// if that is a module that we've already preprocessed. Does not check for
936  /// macros imported into M.
938  if (!II->hasMacroDefinition())
939  return false;
940  auto I = Submodules.find(M);
941  if (I == Submodules.end())
942  return false;
943  auto J = I->second.Macros.find(II);
944  if (J == I->second.Macros.end())
945  return false;
946  auto *MD = J->second.getLatest();
947  return MD && MD->isDefined();
948  }
949 
951  if (!II->hasMacroDefinition())
952  return {};
953 
954  MacroState &S = CurSubmoduleState->Macros[II];
955  auto *MD = S.getLatest();
956  while (MD && isa<VisibilityMacroDirective>(MD))
957  MD = MD->getPrevious();
958  return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
959  S.getActiveModuleMacros(*this, II),
960  S.isAmbiguous(*this, II));
961  }
962 
964  SourceLocation Loc) {
965  if (!II->hadMacroDefinition())
966  return {};
967 
968  MacroState &S = CurSubmoduleState->Macros[II];
970  if (auto *MD = S.getLatest())
971  DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
972  // FIXME: Compute the set of active module macros at the specified location.
973  return MacroDefinition(DI.getDirective(),
974  S.getActiveModuleMacros(*this, II),
975  S.isAmbiguous(*this, II));
976  }
977 
978  /// Given an identifier, return its latest non-imported MacroDirective
979  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
981  if (!II->hasMacroDefinition())
982  return nullptr;
983 
984  auto *MD = getLocalMacroDirectiveHistory(II);
985  if (!MD || MD->getDefinition().isUndefined())
986  return nullptr;
987 
988  return MD;
989  }
990 
991  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
992  return const_cast<Preprocessor*>(this)->getMacroInfo(II);
993  }
994 
996  if (!II->hasMacroDefinition())
997  return nullptr;
998  if (auto MD = getMacroDefinition(II))
999  return MD.getMacroInfo();
1000  return nullptr;
1001  }
1002 
1003  /// Given an identifier, return the latest non-imported macro
1004  /// directive for that identifier.
1005  ///
1006  /// One can iterate over all previous macro directives from the most recent
1007  /// one.
1008  MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const;
1009 
1010  /// Add a directive to the macro directive history for this identifier.
1011  void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
1013  SourceLocation Loc) {
1014  DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1015  appendMacroDirective(II, MD);
1016  return MD;
1017  }
1019  MacroInfo *MI) {
1020  return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1021  }
1022 
1023  /// Set a MacroDirective that was loaded from a PCH file.
1024  void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED,
1025  MacroDirective *MD);
1026 
1027  /// Register an exported macro for a module and identifier.
1028  ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro,
1029  ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1030  ModuleMacro *getModuleMacro(Module *Mod, IdentifierInfo *II);
1031 
1032  /// Get the list of leaf (non-overridden) module macros for a name.
1034  if (II->isOutOfDate())
1035  updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1036  auto I = LeafModuleMacros.find(II);
1037  if (I != LeafModuleMacros.end())
1038  return I->second;
1039  return None;
1040  }
1041 
1042  /// \{
1043  /// Iterators for the macro history table. Currently defined macros have
1044  /// IdentifierInfo::hasMacroDefinition() set and an empty
1045  /// MacroInfo::getUndefLoc() at the head of the list.
1046  using macro_iterator = MacroMap::const_iterator;
1047 
1048  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1049  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1050 
1051  llvm::iterator_range<macro_iterator>
1052  macros(bool IncludeExternalMacros = true) const {
1053  macro_iterator begin = macro_begin(IncludeExternalMacros);
1054  macro_iterator end = macro_end(IncludeExternalMacros);
1055  return llvm::make_range(begin, end);
1056  }
1057 
1058  /// \}
1059 
1060  /// Return the name of the macro defined before \p Loc that has
1061  /// spelling \p Tokens. If there are multiple macros with same spelling,
1062  /// return the last one defined.
1063  StringRef getLastMacroWithSpelling(SourceLocation Loc,
1064  ArrayRef<TokenValue> Tokens) const;
1065 
1066  const std::string &getPredefines() const { return Predefines; }
1067 
1068  /// Set the predefines for this Preprocessor.
1069  ///
1070  /// These predefines are automatically injected when parsing the main file.
1071  void setPredefines(const char *P) { Predefines = P; }
1072  void setPredefines(StringRef P) { Predefines = P; }
1073 
1074  /// Return information about the specified preprocessor
1075  /// identifier token.
1076  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1077  return &Identifiers.get(Name);
1078  }
1079 
1080  /// Add the specified pragma handler to this preprocessor.
1081  ///
1082  /// If \p Namespace is non-null, then it is a token required to exist on the
1083  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1084  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1086  AddPragmaHandler(StringRef(), Handler);
1087  }
1088 
1089  /// Remove the specific pragma handler from this preprocessor.
1090  ///
1091  /// If \p Namespace is non-null, then it should be the namespace that
1092  /// \p Handler was added to. It is an error to remove a handler that
1093  /// has not been registered.
1094  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1096  RemovePragmaHandler(StringRef(), Handler);
1097  }
1098 
1099  /// Install empty handlers for all pragmas (making them ignored).
1100  void IgnorePragmas();
1101 
1102  /// Add the specified comment handler to the preprocessor.
1103  void addCommentHandler(CommentHandler *Handler);
1104 
1105  /// Remove the specified comment handler.
1106  ///
1107  /// It is an error to remove a handler that has not been registered.
1108  void removeCommentHandler(CommentHandler *Handler);
1109 
1110  /// Set the code completion handler to the given object.
1112  CodeComplete = &Handler;
1113  }
1114 
1115  /// Retrieve the current code-completion handler.
1117  return CodeComplete;
1118  }
1119 
1120  /// Clear out the code completion handler.
1122  CodeComplete = nullptr;
1123  }
1124 
1125  /// Hook used by the lexer to invoke the "natural language" code
1126  /// completion point.
1127  void CodeCompleteNaturalLanguage();
1128 
1129  /// Set the code completion token for filtering purposes.
1131  CodeCompletionII = Filter;
1132  }
1133 
1134  /// Get the code completion token for filtering purposes.
1136  if (CodeCompletionII)
1137  return CodeCompletionII->getName();
1138  return {};
1139  }
1140 
1141  /// Retrieve the preprocessing record, or NULL if there is no
1142  /// preprocessing record.
1143  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1144 
1145  /// Create a new preprocessing record, which will keep track of
1146  /// all macro expansions, macro definitions, etc.
1147  void createPreprocessingRecord();
1148 
1149  /// Returns true if the FileEntry is the PCH through header.
1150  bool isPCHThroughHeader(const FileEntry *File);
1151 
1152  /// True if creating a PCH with a through header.
1153  bool creatingPCHWithThroughHeader();
1154 
1155  /// True if using a PCH with a through header.
1156  bool usingPCHWithThroughHeader();
1157 
1158  /// Skip tokens until after the #include of the through header.
1159  void SkipTokensUntilPCHThroughHeader();
1160 
1161  /// Process directives while skipping until the through header is found.
1162  void HandleSkippedThroughHeaderDirective(Token &Result,
1163  SourceLocation HashLoc);
1164 
1165  /// Enter the specified FileID as the main source file,
1166  /// which implicitly adds the builtin defines etc.
1167  void EnterMainSourceFile();
1168 
1169  /// Inform the preprocessor callbacks that processing is complete.
1170  void EndSourceFile();
1171 
1172  /// Add a source file to the top of the include stack and
1173  /// start lexing tokens from it instead of the current buffer.
1174  ///
1175  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1176  bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir,
1177  SourceLocation Loc);
1178 
1179  /// Add a Macro to the top of the include stack and start lexing
1180  /// tokens from it instead of the current buffer.
1181  ///
1182  /// \param Args specifies the tokens input to a function-like macro.
1183  /// \param ILEnd specifies the location of the ')' for a function-like macro
1184  /// or the identifier for an object-like macro.
1185  void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro,
1186  MacroArgs *Args);
1187 
1188  /// Add a "macro" context to the top of the include stack,
1189  /// which will cause the lexer to start returning the specified tokens.
1190  ///
1191  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1192  /// will not be subject to further macro expansion. Otherwise, these tokens
1193  /// will be re-macro-expanded when/if expansion is enabled.
1194  ///
1195  /// If \p OwnsTokens is false, this method assumes that the specified stream
1196  /// of tokens has a permanent owner somewhere, so they do not need to be
1197  /// copied. If it is true, it assumes the array of tokens is allocated with
1198  /// \c new[] and the Preprocessor will delete[] it.
1199 private:
1200  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1201  bool DisableMacroExpansion, bool OwnsTokens);
1202 
1203 public:
1204  void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1205  bool DisableMacroExpansion) {
1206  EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true);
1207  }
1208 
1209  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion) {
1210  EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false);
1211  }
1212 
1213  /// Pop the current lexer/macro exp off the top of the lexer stack.
1214  ///
1215  /// This should only be used in situations where the current state of the
1216  /// top-of-stack lexer is known.
1217  void RemoveTopOfLexerStack();
1218 
1219  /// From the point that this method is called, and until
1220  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1221  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1222  /// make the Preprocessor re-lex the same tokens.
1223  ///
1224  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1225  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1226  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1227  ///
1228  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1229  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1230  /// tokens will continue indefinitely.
1231  ///
1232  void EnableBacktrackAtThisPos();
1233 
1234  /// Disable the last EnableBacktrackAtThisPos call.
1235  void CommitBacktrackedTokens();
1236 
1238  CachedTokensTy::size_type Begin, End;
1239  };
1240 
1241 private:
1242  /// A range of cached tokens that should be erased after lexing
1243  /// when backtracking requires the erasure of such cached tokens.
1244  Optional<CachedTokensRange> CachedTokenRangeToErase;
1245 
1246 public:
1247  /// Returns the range of cached tokens that were lexed since
1248  /// EnableBacktrackAtThisPos() was previously called.
1249  CachedTokensRange LastCachedTokenRange();
1250 
1251  /// Erase the range of cached tokens that were lexed since
1252  /// EnableBacktrackAtThisPos() was previously called.
1253  void EraseCachedTokens(CachedTokensRange TokenRange);
1254 
1255  /// Make Preprocessor re-lex the tokens that were lexed since
1256  /// EnableBacktrackAtThisPos() was previously called.
1257  void Backtrack();
1258 
1259  /// True if EnableBacktrackAtThisPos() was called and
1260  /// caching of tokens is on.
1261  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1262 
1263  /// Lex the next token for this preprocessor.
1264  void Lex(Token &Result);
1265 
1266  void LexAfterModuleImport(Token &Result);
1267 
1268  void makeModuleVisible(Module *M, SourceLocation Loc);
1269 
1271  return CurSubmoduleState->VisibleModules.getImportLoc(M);
1272  }
1273 
1274  /// Lex a string literal, which may be the concatenation of multiple
1275  /// string literals and may even come from macro expansion.
1276  /// \returns true on success, false if a error diagnostic has been generated.
1277  bool LexStringLiteral(Token &Result, std::string &String,
1278  const char *DiagnosticTag, bool AllowMacroExpansion) {
1279  if (AllowMacroExpansion)
1280  Lex(Result);
1281  else
1282  LexUnexpandedToken(Result);
1283  return FinishLexStringLiteral(Result, String, DiagnosticTag,
1284  AllowMacroExpansion);
1285  }
1286 
1287  /// Complete the lexing of a string literal where the first token has
1288  /// already been lexed (see LexStringLiteral).
1289  bool FinishLexStringLiteral(Token &Result, std::string &String,
1290  const char *DiagnosticTag,
1291  bool AllowMacroExpansion);
1292 
1293  /// Lex a token. If it's a comment, keep lexing until we get
1294  /// something not a comment.
1295  ///
1296  /// This is useful in -E -C mode where comments would foul up preprocessor
1297  /// directive handling.
1298  void LexNonComment(Token &Result) {
1299  do
1300  Lex(Result);
1301  while (Result.getKind() == tok::comment);
1302  }
1303 
1304  /// Just like Lex, but disables macro expansion of identifier tokens.
1305  void LexUnexpandedToken(Token &Result) {
1306  // Disable macro expansion.
1307  bool OldVal = DisableMacroExpansion;
1308  DisableMacroExpansion = true;
1309  // Lex the token.
1310  Lex(Result);
1311 
1312  // Reenable it.
1313  DisableMacroExpansion = OldVal;
1314  }
1315 
1316  /// Like LexNonComment, but this disables macro expansion of
1317  /// identifier tokens.
1319  do
1320  LexUnexpandedToken(Result);
1321  while (Result.getKind() == tok::comment);
1322  }
1323 
1324  /// Parses a simple integer literal to get its numeric value. Floating
1325  /// point literals and user defined literals are rejected. Used primarily to
1326  /// handle pragmas that accept integer arguments.
1327  bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1328 
1329  /// Disables macro expansion everywhere except for preprocessor directives.
1331  DisableMacroExpansion = true;
1332  MacroExpansionInDirectivesOverride = true;
1333  }
1334 
1335  /// Peeks ahead N tokens and returns that token without consuming any
1336  /// tokens.
1337  ///
1338  /// LookAhead(0) returns the next token that would be returned by Lex(),
1339  /// LookAhead(1) returns the token after it, etc. This returns normal
1340  /// tokens after phase 5. As such, it is equivalent to using
1341  /// 'Lex', not 'LexUnexpandedToken'.
1342  const Token &LookAhead(unsigned N) {
1343  if (CachedLexPos + N < CachedTokens.size())
1344  return CachedTokens[CachedLexPos+N];
1345  else
1346  return PeekAhead(N+1);
1347  }
1348 
1349  /// When backtracking is enabled and tokens are cached,
1350  /// this allows to revert a specific number of tokens.
1351  ///
1352  /// Note that the number of tokens being reverted should be up to the last
1353  /// backtrack position, not more.
1354  void RevertCachedTokens(unsigned N) {
1355  assert(isBacktrackEnabled() &&
1356  "Should only be called when tokens are cached for backtracking");
1357  assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1358  && "Should revert tokens up to the last backtrack position, not more");
1359  assert(signed(CachedLexPos) - signed(N) >= 0 &&
1360  "Corrupted backtrack positions ?");
1361  CachedLexPos -= N;
1362  }
1363 
1364  /// Enters a token in the token stream to be lexed next.
1365  ///
1366  /// If BackTrack() is called afterwards, the token will remain at the
1367  /// insertion point.
1368  void EnterToken(const Token &Tok) {
1369  EnterCachingLexMode();
1370  CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1371  }
1372 
1373  /// We notify the Preprocessor that if it is caching tokens (because
1374  /// backtrack is enabled) it should replace the most recent cached tokens
1375  /// with the given annotation token. This function has no effect if
1376  /// backtracking is not enabled.
1377  ///
1378  /// Note that the use of this function is just for optimization, so that the
1379  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1380  /// invoked.
1381  void AnnotateCachedTokens(const Token &Tok) {
1382  assert(Tok.isAnnotation() && "Expected annotation token");
1383  if (CachedLexPos != 0 && isBacktrackEnabled())
1384  AnnotatePreviousCachedTokens(Tok);
1385  }
1386 
1387  /// Get the location of the last cached token, suitable for setting the end
1388  /// location of an annotation token.
1390  assert(CachedLexPos != 0);
1391  return CachedTokens[CachedLexPos-1].getLastLoc();
1392  }
1393 
1394  /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1395  /// CachedTokens.
1396  bool IsPreviousCachedToken(const Token &Tok) const;
1397 
1398  /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1399  /// in \p NewToks.
1400  ///
1401  /// Useful when a token needs to be split in smaller ones and CachedTokens
1402  /// most recent token must to be updated to reflect that.
1403  void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1404 
1405  /// Replace the last token with an annotation token.
1406  ///
1407  /// Like AnnotateCachedTokens(), this routine replaces an
1408  /// already-parsed (and resolved) token with an annotation
1409  /// token. However, this routine only replaces the last token with
1410  /// the annotation token; it does not affect any other cached
1411  /// tokens. This function has no effect if backtracking is not
1412  /// enabled.
1414  assert(Tok.isAnnotation() && "Expected annotation token");
1415  if (CachedLexPos != 0 && isBacktrackEnabled())
1416  CachedTokens[CachedLexPos-1] = Tok;
1417  }
1418 
1419  /// Enter an annotation token into the token stream.
1420  void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1421  void *AnnotationVal);
1422 
1423  /// Update the current token to represent the provided
1424  /// identifier, in order to cache an action performed by typo correction.
1425  void TypoCorrectToken(const Token &Tok) {
1426  assert(Tok.getIdentifierInfo() && "Expected identifier token");
1427  if (CachedLexPos != 0 && isBacktrackEnabled())
1428  CachedTokens[CachedLexPos-1] = Tok;
1429  }
1430 
1431  /// Recompute the current lexer kind based on the CurLexer/CurPTHLexer/
1432  /// CurTokenLexer pointers.
1433  void recomputeCurLexerKind();
1434 
1435  /// Returns true if incremental processing is enabled
1436  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1437 
1438  /// Enables the incremental processing
1439  void enableIncrementalProcessing(bool value = true) {
1440  IncrementalProcessing = value;
1441  }
1442 
1443  /// Specify the point at which code-completion will be performed.
1444  ///
1445  /// \param File the file in which code completion should occur. If
1446  /// this file is included multiple times, code-completion will
1447  /// perform completion the first time it is included. If NULL, this
1448  /// function clears out the code-completion point.
1449  ///
1450  /// \param Line the line at which code completion should occur
1451  /// (1-based).
1452  ///
1453  /// \param Column the column at which code completion should occur
1454  /// (1-based).
1455  ///
1456  /// \returns true if an error occurred, false otherwise.
1457  bool SetCodeCompletionPoint(const FileEntry *File,
1458  unsigned Line, unsigned Column);
1459 
1460  /// Determine if we are performing code completion.
1461  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1462 
1463  /// Returns the location of the code-completion point.
1464  ///
1465  /// Returns an invalid location if code-completion is not enabled or the file
1466  /// containing the code-completion point has not been lexed yet.
1467  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1468 
1469  /// Returns the start location of the file of code-completion point.
1470  ///
1471  /// Returns an invalid location if code-completion is not enabled or the file
1472  /// containing the code-completion point has not been lexed yet.
1474  return CodeCompletionFileLoc;
1475  }
1476 
1477  /// Returns true if code-completion is enabled and we have hit the
1478  /// code-completion point.
1479  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1480 
1481  /// Note that we hit the code-completion point.
1483  assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1484  CodeCompletionReached = true;
1485  // Silence any diagnostics that occur after we hit the code-completion.
1486  getDiagnostics().setSuppressAllDiagnostics(true);
1487  }
1488 
1489  /// The location of the currently-active \#pragma clang
1490  /// arc_cf_code_audited begin.
1491  ///
1492  /// Returns an invalid location if there is no such pragma active.
1494  return PragmaARCCFCodeAuditedLoc;
1495  }
1496 
1497  /// Set the location of the currently-active \#pragma clang
1498  /// arc_cf_code_audited begin. An invalid location ends the pragma.
1500  PragmaARCCFCodeAuditedLoc = Loc;
1501  }
1502 
1503  /// The location of the currently-active \#pragma clang
1504  /// assume_nonnull begin.
1505  ///
1506  /// Returns an invalid location if there is no such pragma active.
1508  return PragmaAssumeNonNullLoc;
1509  }
1510 
1511  /// Set the location of the currently-active \#pragma clang
1512  /// assume_nonnull begin. An invalid location ends the pragma.
1514  PragmaAssumeNonNullLoc = Loc;
1515  }
1516 
1517  /// Set the directory in which the main file should be considered
1518  /// to have been found, if it is not a real file.
1519  void setMainFileDir(const DirectoryEntry *Dir) {
1520  MainFileDir = Dir;
1521  }
1522 
1523  /// Instruct the preprocessor to skip part of the main source file.
1524  ///
1525  /// \param Bytes The number of bytes in the preamble to skip.
1526  ///
1527  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1528  /// start of a line.
1529  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1530  SkipMainFilePreamble.first = Bytes;
1531  SkipMainFilePreamble.second = StartOfLine;
1532  }
1533 
1534  /// Forwarding function for diagnostics. This emits a diagnostic at
1535  /// the specified Token's location, translating the token's start
1536  /// position in the current buffer into a SourcePosition object for rendering.
1537  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1538  return Diags->Report(Loc, DiagID);
1539  }
1540 
1541  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1542  return Diags->Report(Tok.getLocation(), DiagID);
1543  }
1544 
1545  /// Return the 'spelling' of the token at the given
1546  /// location; does not go up to the spelling location or down to the
1547  /// expansion location.
1548  ///
1549  /// \param buffer A buffer which will be used only if the token requires
1550  /// "cleaning", e.g. if it contains trigraphs or escaped newlines
1551  /// \param invalid If non-null, will be set \c true if an error occurs.
1553  SmallVectorImpl<char> &buffer,
1554  bool *invalid = nullptr) const {
1555  return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1556  }
1557 
1558  /// Return the 'spelling' of the Tok token.
1559  ///
1560  /// The spelling of a token is the characters used to represent the token in
1561  /// the source file after trigraph expansion and escaped-newline folding. In
1562  /// particular, this wants to get the true, uncanonicalized, spelling of
1563  /// things like digraphs, UCNs, etc.
1564  ///
1565  /// \param Invalid If non-null, will be set \c true if an error occurs.
1566  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1567  return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1568  }
1569 
1570  /// Get the spelling of a token into a preallocated buffer, instead
1571  /// of as an std::string.
1572  ///
1573  /// The caller is required to allocate enough space for the token, which is
1574  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1575  /// actual result is returned.
1576  ///
1577  /// Note that this method may do two possible things: it may either fill in
1578  /// the buffer specified with characters, or it may *change the input pointer*
1579  /// to point to a constant buffer with the data already in it (avoiding a
1580  /// copy). The caller is not allowed to modify the returned buffer pointer
1581  /// if an internal buffer is returned.
1582  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1583  bool *Invalid = nullptr) const {
1584  return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1585  }
1586 
1587  /// Get the spelling of a token into a SmallVector.
1588  ///
1589  /// Note that the returned StringRef may not point to the
1590  /// supplied buffer if a copy can be avoided.
1591  StringRef getSpelling(const Token &Tok,
1592  SmallVectorImpl<char> &Buffer,
1593  bool *Invalid = nullptr) const;
1594 
1595  /// Relex the token at the specified location.
1596  /// \returns true if there was a failure, false on success.
1597  bool getRawToken(SourceLocation Loc, Token &Result,
1598  bool IgnoreWhiteSpace = false) {
1599  return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1600  }
1601 
1602  /// Given a Token \p Tok that is a numeric constant with length 1,
1603  /// return the character.
1604  char
1606  bool *Invalid = nullptr) const {
1607  assert(Tok.is(tok::numeric_constant) &&
1608  Tok.getLength() == 1 && "Called on unsupported token");
1609  assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1610 
1611  // If the token is carrying a literal data pointer, just use it.
1612  if (const char *D = Tok.getLiteralData())
1613  return *D;
1614 
1615  // Otherwise, fall back on getCharacterData, which is slower, but always
1616  // works.
1617  return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1618  }
1619 
1620  /// Retrieve the name of the immediate macro expansion.
1621  ///
1622  /// This routine starts from a source location, and finds the name of the
1623  /// macro responsible for its immediate expansion. It looks through any
1624  /// intervening macro argument expansions to compute this. It returns a
1625  /// StringRef that refers to the SourceManager-owned buffer of the source
1626  /// where that macro name is spelled. Thus, the result shouldn't out-live
1627  /// the SourceManager.
1629  return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1630  }
1631 
1632  /// Plop the specified string into a scratch buffer and set the
1633  /// specified token's location and length to it.
1634  ///
1635  /// If specified, the source location provides a location of the expansion
1636  /// point of the token.
1637  void CreateString(StringRef Str, Token &Tok,
1638  SourceLocation ExpansionLocStart = SourceLocation(),
1639  SourceLocation ExpansionLocEnd = SourceLocation());
1640 
1641  /// Split the first Length characters out of the token starting at TokLoc
1642  /// and return a location pointing to the split token. Re-lexing from the
1643  /// split token will return the split token rather than the original.
1644  SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
1645 
1646  /// Computes the source location just past the end of the
1647  /// token at this source location.
1648  ///
1649  /// This routine can be used to produce a source location that
1650  /// points just past the end of the token referenced by \p Loc, and
1651  /// is generally used when a diagnostic needs to point just after a
1652  /// token where it expected something different that it received. If
1653  /// the returned source location would not be meaningful (e.g., if
1654  /// it points into a macro), this routine returns an invalid
1655  /// source location.
1656  ///
1657  /// \param Offset an offset from the end of the token, where the source
1658  /// location should refer to. The default offset (0) produces a source
1659  /// location pointing just past the end of the token; an offset of 1 produces
1660  /// a source location pointing to the last character in the token, etc.
1662  return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1663  }
1664 
1665  /// Returns true if the given MacroID location points at the first
1666  /// token of the macro expansion.
1667  ///
1668  /// \param MacroBegin If non-null and function returns true, it is set to
1669  /// begin location of the macro.
1671  SourceLocation *MacroBegin = nullptr) const {
1672  return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1673  MacroBegin);
1674  }
1675 
1676  /// Returns true if the given MacroID location points at the last
1677  /// token of the macro expansion.
1678  ///
1679  /// \param MacroEnd If non-null and function returns true, it is set to
1680  /// end location of the macro.
1682  SourceLocation *MacroEnd = nullptr) const {
1683  return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1684  }
1685 
1686  /// Print the token to stderr, used for debugging.
1687  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1688  void DumpLocation(SourceLocation Loc) const;
1689  void DumpMacro(const MacroInfo &MI) const;
1690  void dumpMacroInfo(const IdentifierInfo *II);
1691 
1692  /// Given a location that specifies the start of a
1693  /// token, return a new location that specifies a character within the token.
1695  unsigned Char) const {
1696  return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1697  }
1698 
1699  /// Increment the counters for the number of token paste operations
1700  /// performed.
1701  ///
1702  /// If fast was specified, this is a 'fast paste' case we handled.
1703  void IncrementPasteCounter(bool isFast) {
1704  if (isFast)
1705  ++NumFastTokenPaste;
1706  else
1707  ++NumTokenPaste;
1708  }
1709 
1710  void PrintStats();
1711 
1712  size_t getTotalMemory() const;
1713 
1714  /// When the macro expander pastes together a comment (/##/) in Microsoft
1715  /// mode, this method handles updating the current state, returning the
1716  /// token on the next source line.
1717  void HandleMicrosoftCommentPaste(Token &Tok);
1718 
1719  //===--------------------------------------------------------------------===//
1720  // Preprocessor callback methods. These are invoked by a lexer as various
1721  // directives and events are found.
1722 
1723  /// Given a tok::raw_identifier token, look up the
1724  /// identifier information for the token and install it into the token,
1725  /// updating the token kind accordingly.
1726  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1727 
1728 private:
1729  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1730 
1731 public:
1732  /// Specifies the reason for poisoning an identifier.
1733  ///
1734  /// If that identifier is accessed while poisoned, then this reason will be
1735  /// used instead of the default "poisoned" diagnostic.
1736  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1737 
1738  /// Display reason for poisoned identifier.
1739  void HandlePoisonedIdentifier(Token & Tok);
1740 
1742  if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1743  if(II->isPoisoned()) {
1744  HandlePoisonedIdentifier(Identifier);
1745  }
1746  }
1747  }
1748 
1749 private:
1750  /// Identifiers used for SEH handling in Borland. These are only
1751  /// allowed in particular circumstances
1752  // __except block
1753  IdentifierInfo *Ident__exception_code,
1754  *Ident___exception_code,
1755  *Ident_GetExceptionCode;
1756  // __except filter expression
1757  IdentifierInfo *Ident__exception_info,
1758  *Ident___exception_info,
1759  *Ident_GetExceptionInfo;
1760  // __finally
1761  IdentifierInfo *Ident__abnormal_termination,
1762  *Ident___abnormal_termination,
1763  *Ident_AbnormalTermination;
1764 
1765  const char *getCurLexerEndPos();
1766  void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
1767 
1768 public:
1769  void PoisonSEHIdentifiers(bool Poison = true); // Borland
1770 
1771  /// Callback invoked when the lexer reads an identifier and has
1772  /// filled in the tokens IdentifierInfo member.
1773  ///
1774  /// This callback potentially macro expands it or turns it into a named
1775  /// token (like 'for').
1776  ///
1777  /// \returns true if we actually computed a token, false if we need to
1778  /// lex again.
1779  bool HandleIdentifier(Token &Identifier);
1780 
1781  /// Callback invoked when the lexer hits the end of the current file.
1782  ///
1783  /// This either returns the EOF token and returns true, or
1784  /// pops a level off the include stack and returns false, at which point the
1785  /// client should call lex again.
1786  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1787 
1788  /// Callback invoked when the current TokenLexer hits the end of its
1789  /// token stream.
1790  bool HandleEndOfTokenLexer(Token &Result);
1791 
1792  /// Callback invoked when the lexer sees a # token at the start of a
1793  /// line.
1794  ///
1795  /// This consumes the directive, modifies the lexer/preprocessor state, and
1796  /// advances the lexer(s) so that the next token read is the correct one.
1797  void HandleDirective(Token &Result);
1798 
1799  /// Ensure that the next token is a tok::eod token.
1800  ///
1801  /// If not, emit a diagnostic and consume up until the eod.
1802  /// If \p EnableMacros is true, then we consider macros that expand to zero
1803  /// tokens as being ok.
1804  void CheckEndOfDirective(const char *Directive, bool EnableMacros = false);
1805 
1806  /// Read and discard all tokens remaining on the current line until
1807  /// the tok::eod token is found.
1808  void DiscardUntilEndOfDirective();
1809 
1810  /// Returns true if the preprocessor has seen a use of
1811  /// __DATE__ or __TIME__ in the file so far.
1812  bool SawDateOrTime() const {
1813  return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
1814  }
1815  unsigned getCounterValue() const { return CounterValue; }
1816  void setCounterValue(unsigned V) { CounterValue = V; }
1817 
1818  /// Retrieves the module that we're currently building, if any.
1819  Module *getCurrentModule();
1820 
1821  /// Allocate a new MacroInfo object with the provided SourceLocation.
1822  MacroInfo *AllocateMacroInfo(SourceLocation L);
1823 
1824  /// Turn the specified lexer token into a fully checked and spelled
1825  /// filename, e.g. as an operand of \#include.
1826  ///
1827  /// The caller is expected to provide a buffer that is large enough to hold
1828  /// the spelling of the filename, but is also expected to handle the case
1829  /// when this method decides to use a different buffer.
1830  ///
1831  /// \returns true if the input filename was in <>'s or false if it was
1832  /// in ""'s.
1833  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Filename);
1834 
1835  /// Given a "foo" or <foo> reference, look up the indicated file.
1836  ///
1837  /// Returns null on failure. \p isAngled indicates whether the file
1838  /// reference is for system \#include's or not (i.e. using <> instead of "").
1839  const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename,
1840  bool isAngled, const DirectoryLookup *FromDir,
1841  const FileEntry *FromFile,
1842  const DirectoryLookup *&CurDir,
1843  SmallVectorImpl<char> *SearchPath,
1844  SmallVectorImpl<char> *RelativePath,
1845  ModuleMap::KnownHeader *SuggestedModule,
1846  bool *IsMapped, bool SkipCache = false);
1847 
1848  /// Get the DirectoryLookup structure used to find the current
1849  /// FileEntry, if CurLexer is non-null and if applicable.
1850  ///
1851  /// This allows us to implement \#include_next and find directory-specific
1852  /// properties.
1853  const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1854 
1855  /// Return true if we're in the top-level file, not in a \#include.
1856  bool isInPrimaryFile() const;
1857 
1858  /// Handle cases where the \#include name is expanded
1859  /// from a macro as multiple tokens, which need to be glued together.
1860  ///
1861  /// This occurs for code like:
1862  /// \code
1863  /// \#define FOO <x/y.h>
1864  /// \#include FOO
1865  /// \endcode
1866  /// because in this case, "<x/y.h>" is returned as 7 tokens, not one.
1867  ///
1868  /// This code concatenates and consumes tokens up to the '>' token. It
1869  /// returns false if the > was found, otherwise it returns true if it finds
1870  /// and consumes the EOD marker.
1871  bool ConcatenateIncludeName(SmallString<128> &FilenameBuffer,
1872  SourceLocation &End);
1873 
1874  /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1875  /// followed by EOD. Return true if the token is not a valid on-off-switch.
1876  bool LexOnOffSwitch(tok::OnOffSwitch &OOS);
1877 
1878  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
1879  bool *ShadowFlag = nullptr);
1880 
1881  void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
1882  Module *LeaveSubmodule(bool ForPragma);
1883 
1884 private:
1885  friend void TokenLexer::ExpandFunctionArguments();
1886 
1887  void PushIncludeMacroStack() {
1888  assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
1889  IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
1890  std::move(CurLexer), std::move(CurPTHLexer),
1891  CurPPLexer, std::move(CurTokenLexer),
1892  CurDirLookup);
1893  CurPPLexer = nullptr;
1894  }
1895 
1896  void PopIncludeMacroStack() {
1897  CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1898  CurPTHLexer = std::move(IncludeMacroStack.back().ThePTHLexer);
1899  CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1900  CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1901  CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1902  CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
1903  CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1904  IncludeMacroStack.pop_back();
1905  }
1906 
1907  void PropagateLineStartLeadingSpaceInfo(Token &Result);
1908 
1909  /// Determine whether we need to create module macros for #defines in the
1910  /// current context.
1911  bool needModuleMacros() const;
1912 
1913  /// Update the set of active module macros and ambiguity flag for a module
1914  /// macro name.
1915  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
1916 
1917  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
1918  SourceLocation Loc);
1919  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
1920  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
1921  bool isPublic);
1922 
1923  /// Lex and validate a macro name, which occurs after a
1924  /// \#define or \#undef.
1925  ///
1926  /// \param MacroNameTok Token that represents the name defined or undefined.
1927  /// \param IsDefineUndef Kind if preprocessor directive.
1928  /// \param ShadowFlag Points to flag that is set if macro name shadows
1929  /// a keyword.
1930  ///
1931  /// This emits a diagnostic, sets the token kind to eod,
1932  /// and discards the rest of the macro line if the macro name is invalid.
1933  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
1934  bool *ShadowFlag = nullptr);
1935 
1936  /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
1937  /// entire line) of the macro's tokens and adds them to MacroInfo, and while
1938  /// doing so performs certain validity checks including (but not limited to):
1939  /// - # (stringization) is followed by a macro parameter
1940  /// \param MacroNameTok - Token that represents the macro name
1941  /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
1942  ///
1943  /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
1944  /// returns a nullptr if an invalid sequence of tokens is encountered.
1945  MacroInfo *ReadOptionalMacroParameterListAndBody(
1946  const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
1947 
1948  /// The ( starting an argument list of a macro definition has just been read.
1949  /// Lex the rest of the parameters and the closing ), updating \p MI with
1950  /// what we learn and saving in \p LastTok the last token read.
1951  /// Return true if an error occurs parsing the arg list.
1952  bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
1953 
1954  /// We just read a \#if or related directive and decided that the
1955  /// subsequent tokens are in the \#if'd out portion of the
1956  /// file. Lex the rest of the file, until we see an \#endif. If \p
1957  /// FoundNonSkipPortion is true, then we have already emitted code for part of
1958  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
1959  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
1960  /// already seen one so a \#else directive is a duplicate. When this returns,
1961  /// the caller can lex the first valid token.
1962  void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
1963  SourceLocation IfTokenLoc,
1964  bool FoundNonSkipPortion, bool FoundElse,
1965  SourceLocation ElseLoc = SourceLocation());
1966 
1967  /// A fast PTH version of SkipExcludedConditionalBlock.
1968  void PTHSkipExcludedConditionalBlock();
1969 
1970  /// Information about the result for evaluating an expression for a
1971  /// preprocessor directive.
1972  struct DirectiveEvalResult {
1973  /// Whether the expression was evaluated as true or not.
1974  bool Conditional;
1975 
1976  /// True if the expression contained identifiers that were undefined.
1977  bool IncludedUndefinedIds;
1978  };
1979 
1980  /// Evaluate an integer constant expression that may occur after a
1981  /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
1982  ///
1983  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
1984  DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1985 
1986  /// Install the standard preprocessor pragmas:
1987  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
1988  void RegisterBuiltinPragmas();
1989 
1990  /// Register builtin macros such as __LINE__ with the identifier table.
1991  void RegisterBuiltinMacros();
1992 
1993  /// If an identifier token is read that is to be expanded as a macro, handle
1994  /// it and return the next token as 'Tok'. If we lexed a token, return true;
1995  /// otherwise the caller should lex again.
1996  bool HandleMacroExpandedIdentifier(Token &Tok, const MacroDefinition &MD);
1997 
1998  /// Cache macro expanded tokens for TokenLexers.
1999  //
2000  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2001  /// going to lex in the cache and when it finishes the tokens are removed
2002  /// from the end of the cache.
2003  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2004  ArrayRef<Token> tokens);
2005 
2006  void removeCachedMacroExpandedTokensOfLastLexer();
2007 
2008  /// Determine whether the next preprocessor token to be
2009  /// lexed is a '('. If so, consume the token and return true, if not, this
2010  /// method should have no observable side-effect on the lexed tokens.
2011  bool isNextPPTokenLParen();
2012 
2013  /// After reading "MACRO(", this method is invoked to read all of the formal
2014  /// arguments specified for the macro invocation. Returns null on error.
2015  MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2016  SourceLocation &ExpansionEnd);
2017 
2018  /// If an identifier token is read that is to be expanded
2019  /// as a builtin macro, handle it and return the next token as 'Tok'.
2020  void ExpandBuiltinMacro(Token &Tok);
2021 
2022  /// Read a \c _Pragma directive, slice it up, process it, then
2023  /// return the first token after the directive.
2024  /// This assumes that the \c _Pragma token has just been read into \p Tok.
2025  void Handle_Pragma(Token &Tok);
2026 
2027  /// Like Handle_Pragma except the pragma text is not enclosed within
2028  /// a string literal.
2029  void HandleMicrosoft__pragma(Token &Tok);
2030 
2031  /// Add a lexer to the top of the include stack and
2032  /// start lexing tokens from it instead of the current buffer.
2033  void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
2034 
2035  /// Add a lexer to the top of the include stack and
2036  /// start getting tokens from it using the PTH cache.
2037  void EnterSourceFileWithPTH(PTHLexer *PL, const DirectoryLookup *Dir);
2038 
2039  /// Set the FileID for the preprocessor predefines.
2040  void setPredefinesFileID(FileID FID) {
2041  assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2042  PredefinesFileID = FID;
2043  }
2044 
2045  /// Set the FileID for the PCH through header.
2046  void setPCHThroughHeaderFileID(FileID FID);
2047 
2048  /// Returns true if we are lexing from a file and not a
2049  /// pragma or a macro.
2050  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2051  return L ? !L->isPragmaLexer() : P != nullptr;
2052  }
2053 
2054  static bool IsFileLexer(const IncludeStackInfo& I) {
2055  return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2056  }
2057 
2058  bool IsFileLexer() const {
2059  return IsFileLexer(CurLexer.get(), CurPPLexer);
2060  }
2061 
2062  //===--------------------------------------------------------------------===//
2063  // Caching stuff.
2064  void CachingLex(Token &Result);
2065 
2066  bool InCachingLexMode() const {
2067  // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2068  // that we are past EOF, not that we are in CachingLex mode.
2069  return !CurPPLexer && !CurTokenLexer && !CurPTHLexer &&
2070  !IncludeMacroStack.empty();
2071  }
2072 
2073  void EnterCachingLexMode();
2074 
2075  void ExitCachingLexMode() {
2076  if (InCachingLexMode())
2077  RemoveTopOfLexerStack();
2078  }
2079 
2080  const Token &PeekAhead(unsigned N);
2081  void AnnotatePreviousCachedTokens(const Token &Tok);
2082 
2083  //===--------------------------------------------------------------------===//
2084  /// Handle*Directive - implement the various preprocessor directives. These
2085  /// should side-effect the current preprocessor object so that the next call
2086  /// to Lex() will return the appropriate token next.
2087  void HandleLineDirective();
2088  void HandleDigitDirective(Token &Tok);
2089  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2090  void HandleIdentSCCSDirective(Token &Tok);
2091  void HandleMacroPublicDirective(Token &Tok);
2092  void HandleMacroPrivateDirective();
2093 
2094  // File inclusion.
2095  void HandleIncludeDirective(SourceLocation HashLoc,
2096  Token &Tok,
2097  const DirectoryLookup *LookupFrom = nullptr,
2098  const FileEntry *LookupFromFile = nullptr,
2099  bool isImport = false);
2100  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2101  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2102  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2103  void HandleMicrosoftImportDirective(Token &Tok);
2104 
2105 public:
2106  /// Check that the given module is available, producing a diagnostic if not.
2107  /// \return \c true if the check failed (because the module is not available).
2108  /// \c false if the module appears to be usable.
2109  static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2110  const TargetInfo &TargetInfo,
2111  DiagnosticsEngine &Diags, Module *M);
2112 
2113  // Module inclusion testing.
2114  /// Find the module that owns the source or header file that
2115  /// \p Loc points to. If the location is in a file that was included
2116  /// into a module, or is outside any module, returns nullptr.
2117  Module *getModuleForLocation(SourceLocation Loc);
2118 
2119  /// We want to produce a diagnostic at location IncLoc concerning a
2120  /// missing module import.
2121  ///
2122  /// \param IncLoc The location at which the missing import was detected.
2123  /// \param M The desired module.
2124  /// \param MLoc A location within the desired module at which some desired
2125  /// effect occurred (eg, where a desired entity was declared).
2126  ///
2127  /// \return A file that can be #included to import a module containing MLoc.
2128  /// Null if no such file could be determined or if a #include is not
2129  /// appropriate.
2130  const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2131  Module *M,
2132  SourceLocation MLoc);
2133 
2134  bool isRecordingPreamble() const {
2135  return PreambleConditionalStack.isRecording();
2136  }
2137 
2138  bool hasRecordedPreamble() const {
2139  return PreambleConditionalStack.hasRecordedPreamble();
2140  }
2141 
2143  return PreambleConditionalStack.getStack();
2144  }
2145 
2147  PreambleConditionalStack.setStack(s);
2148  }
2149 
2152  PreambleConditionalStack.startReplaying();
2153  PreambleConditionalStack.setStack(s);
2154  PreambleConditionalStack.SkipInfo = SkipInfo;
2155  }
2156 
2158  return PreambleConditionalStack.SkipInfo;
2159  }
2160 
2161 private:
2162  /// After processing predefined file, initialize the conditional stack from
2163  /// the preamble.
2164  void replayPreambleConditionalStack();
2165 
2166  // Macro handling.
2167  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterTopLevelIfndef);
2168  void HandleUndefDirective();
2169 
2170  // Conditional Inclusion.
2171  void HandleIfdefDirective(Token &Tok, const Token &HashToken,
2172  bool isIfndef, bool ReadAnyTokensBeforeDirective);
2173  void HandleIfDirective(Token &Tok, const Token &HashToken,
2174  bool ReadAnyTokensBeforeDirective);
2175  void HandleEndifDirective(Token &Tok);
2176  void HandleElseDirective(Token &Tok, const Token &HashToken);
2177  void HandleElifDirective(Token &Tok, const Token &HashToken);
2178 
2179  // Pragmas.
2180  void HandlePragmaDirective(SourceLocation IntroducerLoc,
2181  PragmaIntroducerKind Introducer);
2182 
2183 public:
2184  void HandlePragmaOnce(Token &OnceTok);
2185  void HandlePragmaMark();
2186  void HandlePragmaPoison();
2187  void HandlePragmaSystemHeader(Token &SysHeaderTok);
2188  void HandlePragmaDependency(Token &DependencyTok);
2189  void HandlePragmaPushMacro(Token &Tok);
2190  void HandlePragmaPopMacro(Token &Tok);
2191  void HandlePragmaIncludeAlias(Token &Tok);
2192  void HandlePragmaModuleBuild(Token &Tok);
2193  IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2194 
2195  // Return true and store the first token only if any CommentHandler
2196  // has inserted some tokens and getCommentRetentionState() is false.
2197  bool HandleComment(Token &Token, SourceRange Comment);
2198 
2199  /// A macro is used, update information about macros that need unused
2200  /// warnings.
2201  void markMacroAsUsed(MacroInfo *MI);
2202 };
2203 
2204 /// Abstract base class that describes a handler that will receive
2205 /// source ranges for each of the comments encountered in the source file.
2207 public:
2208  virtual ~CommentHandler();
2209 
2210  // The handler shall return true if it has pushed any tokens
2211  // to be read using e.g. EnterToken or EnterTokenStream.
2212  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2213 };
2214 
2215 /// Registry of pragma handlers added by plugins
2216 using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2217 
2218 } // namespace clang
2219 
2220 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H
IdentifierInfo * get(StringRef Name) override
get - Return the identifier token info for the specified named identifier.
Definition: PTHLexer.cpp:599
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
Definition: Preprocessor.h:333
A set of visible modules.
Definition: Module.h:589
llvm::BumpPtrAllocator & getPreprocessorAllocator()
Definition: Preprocessor.h:833
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:77
SelectorTable & getSelectorTable()
Definition: Preprocessor.h:831
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:68
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:116
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the &#39;spelling&#39; of the Tok token.
MemoryBufferCache & getPCMCache() const
Definition: Preprocessor.h:826
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:30
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
Defines the SourceManager interface.
static const Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:21
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
FileManager & getFileManager() const
Definition: Preprocessor.h:824
Defines the clang::Module class, which describes a module in the source code.
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
SourceLocation getPragmaARCCFCodeAuditedLoc() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
StringRef P
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:564
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1294
A directive for an undefined macro.
Definition: MacroInfo.h:429
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool hadModuleLoaderFatalFailure() const
Definition: Preprocessor.h:850
const DefInfo findDirectiveAtLoc(SourceLocation L, SourceManager &SM) const
Find macro definition active in the specified source location.
Definition: MacroInfo.cpp:203
void EnterToken(const Token &Tok)
Enters a token in the token stream to be lexed next.
Manage memory buffers across multiple users.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:103
Builtin::Context & getBuiltinInfo()
Definition: Preprocessor.h:832
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:36
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:118
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Definition: MacroInfo.h:329
tok::TokenKind getKind() const
Definition: Token.h:90
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:93
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
One of these records is kept for each identifier that is lexed.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
Represents a macro directive exported by a module.
Definition: MacroInfo.h:488
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:406
This table allows us to fully hide how we implement multi-keyword caching.
LineState State
Definition: Format.h:2031
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:822
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Definition: Preprocessor.h:860
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:50
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:821
Describes a module or submodule.
Definition: Module.h:65
bool isPragmaLexer() const
isPragmaLexer - Returns true if this Lexer is being used to lex a pragma.
Definition: Lexer.h:184
A directive for setting the module visibility of a macro.
Definition: MacroInfo.h:444
bool isInvalid() const
void SetSuppressIncludeNotFoundError(bool Suppress)
Definition: Preprocessor.h:870
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:110
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
Definition: Preprocessor.h:907
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:827
uint32_t Offset
Definition: CacheTokens.cpp:43
const FormatToken & Tok
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void LexNonComment(Token &Result)
Lex a token.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:149
bool isMacroDefined(const IdentifierInfo *II)
Definition: Preprocessor.h:929
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we&#39;ve alread...
Definition: Preprocessor.h:937
Defines the Diagnostic-related interfaces.
const TargetInfo * getAuxTargetInfo() const
Definition: Preprocessor.h:823
TokenLexer - This implements a lexer that returns tokens from a macro body or token stream instead of...
Definition: TokenLexer.h:31
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
Definition: Preprocessor.h:880
PragmaIntroducerKind
Describes how the pragma was introduced, e.g., with #pragma, _Pragma, or __pragma.
Definition: Pragma.h:32
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:148
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:1042
void clearCodeCompletionHandler()
Clear out the code completion handler.
StringRef Filename
Definition: Format.cpp:1604
Provides lookups to, and iteration over, IdentiferInfo objects.
Exposes information about the current target.
Definition: TargetInfo.h:54
const IdentifierTable & getIdentifierTable() const
Definition: Preprocessor.h:830
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Abstract interface for external sources of preprocessor information.
Defines the clang::LangOptions interface.
unsigned getCounterValue() const
SourceLocation End
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:350
int Id
Definition: ASTDiff.cpp:191
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
const AnnotatedLine * Line
bool isDefined() const
Definition: MacroInfo.h:385
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Definition: Preprocessor.h:889
Implements an efficient mapping from strings to IdentifierInfo nodes.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
A class for tracking whether we&#39;re inside a VA_OPT during a traversal of the tokens of a variadic mac...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
#define bool
Definition: stdbool.h:31
DefInfo getDefinition()
Traverses the macro directives history and returns the next macro definition directive along with inf...
Definition: MacroInfo.cpp:179
SourceLocation Begin
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:88
llvm::Optional< PreambleSkipInfo > getPreambleSkipInfo() const
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
Definition: Preprocessor.h:991
bool getCommentRetentionState() const
Definition: Preprocessor.h:865
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:101
PPCallbacks * getPPCallbacks() const
Definition: Preprocessor.h:917
const SourceManager & SM
Definition: Format.cpp:1474
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
bool isRecordingPreamble() const
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
SourceManager & getSourceManager() const
Definition: Preprocessor.h:825
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define&#39;d and not #undef&#39;...
Definition: Preprocessor.h:980
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:843
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:291
Kind
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
void setExternalSource(ExternalPreprocessorSource *Source)
Definition: Preprocessor.h:839
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.h:95
void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:950
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:177
IdentifierTable & getIdentifierTable()
Definition: Preprocessor.h:829
void setPragmasEnabled(bool Enabled)
Definition: Preprocessor.h:867
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:87
bool isPoisoned() const
Return true if this token has been poisoned.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
Definition: Preprocessor.h:963
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
Definition: Preprocessor.h:897
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
const MacroInfo * getMacroInfo() const
Definition: MacroInfo.h:391
StringRef getName() const
Return the actual identifier string.
bool isMacroDefined(StringRef Id)
Definition: Preprocessor.h:926
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
Dataflow Directional Tag Classes.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion)
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
Definition: DeclSpec.h:1128
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion)
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:354
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:59
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Abstract interface for a module loader.
Definition: ModuleLoader.h:74
unsigned getLength() const
Definition: Token.h:127
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:40
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:49
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:215
bool getPragmasEnabled() const
Definition: Preprocessor.h:868
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:124
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
Definition: Preprocessor.h:848
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Definition: Preprocessor.h:816
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:45
Defines the PPCallbacks interface.
Defines the clang::TokenKind enum and support functions.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
SourceLocation getModuleImportLoc(Module *M) const
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:910
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:818
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:301
bool hasRecordedPreamble() const
const std::string & getPredefines() const
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:283
void setPredefines(StringRef P)
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
The translation unit is a complete translation unit.
Definition: LangOptions.h:303
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
MacroMap::const_iterator macro_iterator
Abstract base class that describes a handler that will receive source ranges for each of the comments...
void setCounterValue(unsigned V)
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
Definition: Preprocessor.h:855
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
Definition: Preprocessor.h:874
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, llvm::Optional< PreambleSkipInfo > SkipInfo)
Directive - Abstract class representing a parsed verify directive.
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Definition: Preprocessor.h:995
Callback handler that receives notifications when performing code completion within the preprocessor...
A header that is known to reside within a given module, whether it was included or excluded...
Definition: ModuleMap.h:149
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
Definition: Preprocessor.h:886
PTHManager * getPTHManager()
Definition: Preprocessor.h:837
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:918
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:127
void setDiagnostics(DiagnosticsEngine &D)
Definition: Preprocessor.h:819