clang  9.0.0svn
Preprocessor.h
Go to the documentation of this file.
1 //===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines the clang::Preprocessor interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15 #define LLVM_CLANG_LEX_PREPROCESSOR_H
16 
17 #include "clang/Basic/Builtins.h"
18 #include "clang/Basic/Diagnostic.h"
20 #include "clang/Basic/LLVM.h"
22 #include "clang/Basic/Module.h"
25 #include "clang/Basic/TokenKinds.h"
26 #include "clang/Lex/Lexer.h"
27 #include "clang/Lex/MacroInfo.h"
28 #include "clang/Lex/ModuleLoader.h"
29 #include "clang/Lex/ModuleMap.h"
30 #include "clang/Lex/PPCallbacks.h"
31 #include "clang/Lex/Token.h"
32 #include "clang/Lex/TokenLexer.h"
33 #include "llvm/ADT/ArrayRef.h"
34 #include "llvm/ADT/DenseMap.h"
35 #include "llvm/ADT/FoldingSet.h"
36 #include "llvm/ADT/None.h"
37 #include "llvm/ADT/Optional.h"
38 #include "llvm/ADT/PointerUnion.h"
39 #include "llvm/ADT/STLExtras.h"
40 #include "llvm/ADT/SmallPtrSet.h"
41 #include "llvm/ADT/SmallVector.h"
42 #include "llvm/ADT/StringRef.h"
43 #include "llvm/ADT/TinyPtrVector.h"
44 #include "llvm/ADT/iterator_range.h"
45 #include "llvm/Support/Allocator.h"
46 #include "llvm/Support/Casting.h"
47 #include "llvm/Support/Registry.h"
48 #include <cassert>
49 #include <cstddef>
50 #include <cstdint>
51 #include <memory>
52 #include <map>
53 #include <string>
54 #include <utility>
55 #include <vector>
56 
57 namespace llvm {
58 
59 template<unsigned InternalLen> class SmallString;
60 
61 } // namespace llvm
62 
63 namespace clang {
64 
65 class CodeCompletionHandler;
66 class CommentHandler;
67 class DirectoryEntry;
68 class DirectoryLookup;
69 class ExternalPreprocessorSource;
70 class FileEntry;
71 class FileManager;
72 class HeaderSearch;
73 class MacroArgs;
74 class MemoryBufferCache;
75 class PragmaHandler;
76 class PragmaNamespace;
77 class PreprocessingRecord;
78 class PreprocessorLexer;
79 class PreprocessorOptions;
80 class ScratchBuffer;
81 class TargetInfo;
82 
83 /// Stores token information for comparing actual tokens with
84 /// predefined values. Only handles simple tokens and identifiers.
85 class TokenValue {
87  IdentifierInfo *II;
88 
89 public:
90  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
91  assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
92  assert(Kind != tok::identifier &&
93  "Identifiers should be created by TokenValue(IdentifierInfo *)");
94  assert(!tok::isLiteral(Kind) && "Literals are not supported.");
95  assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
96  }
97 
98  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
99 
100  bool operator==(const Token &Tok) const {
101  return Tok.getKind() == Kind &&
102  (!II || II == Tok.getIdentifierInfo());
103  }
104 };
105 
106 /// Context in which macro name is used.
107 enum MacroUse {
108  // other than #define or #undef
109  MU_Other = 0,
110 
111  // macro name specified in #define
113 
114  // macro name specified in #undef
116 };
117 
118 /// Engages in a tight little dance with the lexer to efficiently
119 /// preprocess tokens.
120 ///
121 /// Lexers know only about tokens within a single source file, and don't
122 /// know anything about preprocessor-level issues like the \#include stack,
123 /// token expansion, etc.
127 
128  std::shared_ptr<PreprocessorOptions> PPOpts;
129  DiagnosticsEngine *Diags;
130  LangOptions &LangOpts;
131  const TargetInfo *Target = nullptr;
132  const TargetInfo *AuxTarget = nullptr;
133  FileManager &FileMgr;
134  SourceManager &SourceMgr;
135  MemoryBufferCache &PCMCache;
136  std::unique_ptr<ScratchBuffer> ScratchBuf;
137  HeaderSearch &HeaderInfo;
138  ModuleLoader &TheModuleLoader;
139 
140  /// External source of macros.
141  ExternalPreprocessorSource *ExternalSource;
142 
143  /// A BumpPtrAllocator object used to quickly allocate and release
144  /// objects internal to the Preprocessor.
145  llvm::BumpPtrAllocator BP;
146 
147  /// Identifiers for builtin macros and other builtins.
148  IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
149  IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
150  IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
151  IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
152  IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
153  IdentifierInfo *Ident__COUNTER__; // __COUNTER__
154  IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
155  IdentifierInfo *Ident__identifier; // __identifier
156  IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
157  IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
158  IdentifierInfo *Ident__has_feature; // __has_feature
159  IdentifierInfo *Ident__has_extension; // __has_extension
160  IdentifierInfo *Ident__has_builtin; // __has_builtin
161  IdentifierInfo *Ident__has_attribute; // __has_attribute
162  IdentifierInfo *Ident__has_include; // __has_include
163  IdentifierInfo *Ident__has_include_next; // __has_include_next
164  IdentifierInfo *Ident__has_warning; // __has_warning
165  IdentifierInfo *Ident__is_identifier; // __is_identifier
166  IdentifierInfo *Ident__building_module; // __building_module
167  IdentifierInfo *Ident__MODULE__; // __MODULE__
168  IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
169  IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
170  IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
171  IdentifierInfo *Ident__is_target_arch; // __is_target_arch
172  IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
173  IdentifierInfo *Ident__is_target_os; // __is_target_os
174  IdentifierInfo *Ident__is_target_environment; // __is_target_environment
175 
176  // Weak, only valid (and set) while InMacroArgs is true.
177  Token* ArgMacro;
178 
179  SourceLocation DATELoc, TIMELoc;
180 
181  // Next __COUNTER__ value, starts at 0.
182  unsigned CounterValue = 0;
183 
184  enum {
185  /// Maximum depth of \#includes.
186  MaxAllowedIncludeStackDepth = 200
187  };
188 
189  // State that is set before the preprocessor begins.
190  bool KeepComments : 1;
191  bool KeepMacroComments : 1;
192  bool SuppressIncludeNotFoundError : 1;
193 
194  // State that changes while the preprocessor runs:
195  bool InMacroArgs : 1; // True if parsing fn macro invocation args.
196 
197  /// Whether the preprocessor owns the header search object.
198  bool OwnsHeaderSearch : 1;
199 
200  /// True if macro expansion is disabled.
201  bool DisableMacroExpansion : 1;
202 
203  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
204  /// when parsing preprocessor directives.
205  bool MacroExpansionInDirectivesOverride : 1;
206 
208 
209  /// Whether we have already loaded macros from the external source.
210  mutable bool ReadMacrosFromExternalSource : 1;
211 
212  /// True if pragmas are enabled.
213  bool PragmasEnabled : 1;
214 
215  /// True if the current build action is a preprocessing action.
216  bool PreprocessedOutput : 1;
217 
218  /// True if we are currently preprocessing a #if or #elif directive
219  bool ParsingIfOrElifDirective;
220 
221  /// True if we are pre-expanding macro arguments.
222  bool InMacroArgPreExpansion;
223 
224  /// Mapping/lookup information for all identifiers in
225  /// the program, including program keywords.
226  mutable IdentifierTable Identifiers;
227 
228  /// This table contains all the selectors in the program.
229  ///
230  /// Unlike IdentifierTable above, this table *isn't* populated by the
231  /// preprocessor. It is declared/expanded here because its role/lifetime is
232  /// conceptually similar to the IdentifierTable. In addition, the current
233  /// control flow (in clang::ParseAST()), make it convenient to put here.
234  ///
235  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
236  /// the lifetime of the preprocessor.
237  SelectorTable Selectors;
238 
239  /// Information about builtins.
241 
242  /// Tracks all of the pragmas that the client registered
243  /// with this preprocessor.
244  std::unique_ptr<PragmaNamespace> PragmaHandlers;
245 
246  /// Pragma handlers of the original source is stored here during the
247  /// parsing of a model file.
248  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
249 
250  /// Tracks all of the comment handlers that the client registered
251  /// with this preprocessor.
252  std::vector<CommentHandler *> CommentHandlers;
253 
254  /// True if we want to ignore EOF token and continue later on (thus
255  /// avoid tearing the Lexer and etc. down).
256  bool IncrementalProcessing = false;
257 
258  /// The kind of translation unit we are processing.
259  TranslationUnitKind TUKind;
260 
261  /// The code-completion handler.
262  CodeCompletionHandler *CodeComplete = nullptr;
263 
264  /// The file that we're performing code-completion for, if any.
265  const FileEntry *CodeCompletionFile = nullptr;
266 
267  /// The offset in file for the code-completion point.
268  unsigned CodeCompletionOffset = 0;
269 
270  /// The location for the code-completion point. This gets instantiated
271  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
272  SourceLocation CodeCompletionLoc;
273 
274  /// The start location for the file of the code-completion point.
275  ///
276  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
277  /// for preprocessing.
278  SourceLocation CodeCompletionFileLoc;
279 
280  /// The source location of the \c import contextual keyword we just
281  /// lexed, if any.
282  SourceLocation ModuleImportLoc;
283 
284  /// The module import path that we're currently processing.
286 
287  /// Whether the last token we lexed was an '@'.
288  bool LastTokenWasAt = false;
289 
290  /// Whether the module import expects an identifier next. Otherwise,
291  /// it expects a '.' or ';'.
292  bool ModuleImportExpectsIdentifier = false;
293 
294  /// The source location of the currently-active
295  /// \#pragma clang arc_cf_code_audited begin.
296  SourceLocation PragmaARCCFCodeAuditedLoc;
297 
298  /// The source location of the currently-active
299  /// \#pragma clang assume_nonnull begin.
300  SourceLocation PragmaAssumeNonNullLoc;
301 
302  /// True if we hit the code-completion point.
303  bool CodeCompletionReached = false;
304 
305  /// The code completion token containing the information
306  /// on the stem that is to be code completed.
307  IdentifierInfo *CodeCompletionII = nullptr;
308 
309  /// Range for the code completion token.
310  SourceRange CodeCompletionTokenRange;
311 
312  /// The directory that the main file should be considered to occupy,
313  /// if it does not correspond to a real file (as happens when building a
314  /// module).
315  const DirectoryEntry *MainFileDir = nullptr;
316 
317  /// The number of bytes that we will initially skip when entering the
318  /// main file, along with a flag that indicates whether skipping this number
319  /// of bytes will place the lexer at the start of a line.
320  ///
321  /// This is used when loading a precompiled preamble.
322  std::pair<int, bool> SkipMainFilePreamble;
323 
324  /// Whether we hit an error due to reaching max allowed include depth. Allows
325  /// to avoid hitting the same error over and over again.
326  bool HasReachedMaxIncludeDepth = false;
327 
328 public:
333  bool FoundElse;
335 
337  bool FoundNonSkipPortion, bool FoundElse,
338  SourceLocation ElseLoc)
339  : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
340  FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
341  ElseLoc(ElseLoc) {}
342  };
343 
344 private:
345  friend class ASTReader;
346  friend class MacroArgs;
347 
348  class PreambleConditionalStackStore {
349  enum State {
350  Off = 0,
351  Recording = 1,
352  Replaying = 2,
353  };
354 
355  public:
356  PreambleConditionalStackStore() = default;
357 
358  void startRecording() { ConditionalStackState = Recording; }
359  void startReplaying() { ConditionalStackState = Replaying; }
360  bool isRecording() const { return ConditionalStackState == Recording; }
361  bool isReplaying() const { return ConditionalStackState == Replaying; }
362 
363  ArrayRef<PPConditionalInfo> getStack() const {
364  return ConditionalStack;
365  }
366 
367  void doneReplaying() {
368  ConditionalStack.clear();
369  ConditionalStackState = Off;
370  }
371 
372  void setStack(ArrayRef<PPConditionalInfo> s) {
373  if (!isRecording() && !isReplaying())
374  return;
375  ConditionalStack.clear();
376  ConditionalStack.append(s.begin(), s.end());
377  }
378 
379  bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
380 
381  bool reachedEOFWhileSkipping() const { return SkipInfo.hasValue(); }
382 
383  void clearSkipInfo() { SkipInfo.reset(); }
384 
386 
387  private:
388  SmallVector<PPConditionalInfo, 4> ConditionalStack;
389  State ConditionalStackState = Off;
390  } PreambleConditionalStack;
391 
392  /// The current top of the stack that we're lexing from if
393  /// not expanding a macro and we are lexing directly from source code.
394  ///
395  /// Only one of CurLexer, or CurTokenLexer will be non-null.
396  std::unique_ptr<Lexer> CurLexer;
397 
398  /// The current top of the stack what we're lexing from
399  /// if not expanding a macro.
400  ///
401  /// This is an alias for CurLexer.
402  PreprocessorLexer *CurPPLexer = nullptr;
403 
404  /// Used to find the current FileEntry, if CurLexer is non-null
405  /// and if applicable.
406  ///
407  /// This allows us to implement \#include_next and find directory-specific
408  /// properties.
409  const DirectoryLookup *CurDirLookup = nullptr;
410 
411  /// The current macro we are expanding, if we are expanding a macro.
412  ///
413  /// One of CurLexer and CurTokenLexer must be null.
414  std::unique_ptr<TokenLexer> CurTokenLexer;
415 
416  /// The kind of lexer we're currently working with.
417  enum CurLexerKind {
418  CLK_Lexer,
419  CLK_TokenLexer,
420  CLK_CachingLexer,
421  CLK_LexAfterModuleImport
422  } CurLexerKind = CLK_Lexer;
423 
424  /// If the current lexer is for a submodule that is being built, this
425  /// is that submodule.
426  Module *CurLexerSubmodule = nullptr;
427 
428  /// Keeps track of the stack of files currently
429  /// \#included, and macros currently being expanded from, not counting
430  /// CurLexer/CurTokenLexer.
431  struct IncludeStackInfo {
432  enum CurLexerKind CurLexerKind;
433  Module *TheSubmodule;
434  std::unique_ptr<Lexer> TheLexer;
435  PreprocessorLexer *ThePPLexer;
436  std::unique_ptr<TokenLexer> TheTokenLexer;
437  const DirectoryLookup *TheDirLookup;
438 
439  // The following constructors are completely useless copies of the default
440  // versions, only needed to pacify MSVC.
441  IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
442  std::unique_ptr<Lexer> &&TheLexer,
443  PreprocessorLexer *ThePPLexer,
444  std::unique_ptr<TokenLexer> &&TheTokenLexer,
445  const DirectoryLookup *TheDirLookup)
446  : CurLexerKind(std::move(CurLexerKind)),
447  TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
448  ThePPLexer(std::move(ThePPLexer)),
449  TheTokenLexer(std::move(TheTokenLexer)),
450  TheDirLookup(std::move(TheDirLookup)) {}
451  };
452  std::vector<IncludeStackInfo> IncludeMacroStack;
453 
454  /// Actions invoked when some preprocessor activity is
455  /// encountered (e.g. a file is \#included, etc).
456  std::unique_ptr<PPCallbacks> Callbacks;
457 
458  struct MacroExpandsInfo {
459  Token Tok;
460  MacroDefinition MD;
461  SourceRange Range;
462 
463  MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
464  : Tok(Tok), MD(MD), Range(Range) {}
465  };
466  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
467 
468  /// Information about a name that has been used to define a module macro.
469  struct ModuleMacroInfo {
470  /// The most recent macro directive for this identifier.
471  MacroDirective *MD;
472 
473  /// The active module macros for this identifier.
474  llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
475 
476  /// The generation number at which we last updated ActiveModuleMacros.
477  /// \see Preprocessor::VisibleModules.
478  unsigned ActiveModuleMacrosGeneration = 0;
479 
480  /// Whether this macro name is ambiguous.
481  bool IsAmbiguous = false;
482 
483  /// The module macros that are overridden by this macro.
484  llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
485 
486  ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
487  };
488 
489  /// The state of a macro for an identifier.
490  class MacroState {
491  mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
492 
493  ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
494  const IdentifierInfo *II) const {
495  if (II->isOutOfDate())
496  PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
497  // FIXME: Find a spare bit on IdentifierInfo and store a
498  // HasModuleMacros flag.
499  if (!II->hasMacroDefinition() ||
500  (!PP.getLangOpts().Modules &&
501  !PP.getLangOpts().ModulesLocalVisibility) ||
502  !PP.CurSubmoduleState->VisibleModules.getGeneration())
503  return nullptr;
504 
505  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
506  if (!Info) {
507  Info = new (PP.getPreprocessorAllocator())
508  ModuleMacroInfo(State.get<MacroDirective *>());
509  State = Info;
510  }
511 
512  if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
513  Info->ActiveModuleMacrosGeneration)
514  PP.updateModuleMacroInfo(II, *Info);
515  return Info;
516  }
517 
518  public:
519  MacroState() : MacroState(nullptr) {}
520  MacroState(MacroDirective *MD) : State(MD) {}
521 
522  MacroState(MacroState &&O) noexcept : State(O.State) {
523  O.State = (MacroDirective *)nullptr;
524  }
525 
526  MacroState &operator=(MacroState &&O) noexcept {
527  auto S = O.State;
528  O.State = (MacroDirective *)nullptr;
529  State = S;
530  return *this;
531  }
532 
533  ~MacroState() {
534  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
535  Info->~ModuleMacroInfo();
536  }
537 
538  MacroDirective *getLatest() const {
539  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
540  return Info->MD;
541  return State.get<MacroDirective*>();
542  }
543 
544  void setLatest(MacroDirective *MD) {
545  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
546  Info->MD = MD;
547  else
548  State = MD;
549  }
550 
551  bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
552  auto *Info = getModuleInfo(PP, II);
553  return Info ? Info->IsAmbiguous : false;
554  }
555 
557  getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
558  if (auto *Info = getModuleInfo(PP, II))
559  return Info->ActiveModuleMacros;
560  return None;
561  }
562 
563  MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
564  SourceManager &SourceMgr) const {
565  // FIXME: Incorporate module macros into the result of this.
566  if (auto *Latest = getLatest())
567  return Latest->findDirectiveAtLoc(Loc, SourceMgr);
568  return {};
569  }
570 
571  void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
572  if (auto *Info = getModuleInfo(PP, II)) {
573  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
574  Info->ActiveModuleMacros.begin(),
575  Info->ActiveModuleMacros.end());
576  Info->ActiveModuleMacros.clear();
577  Info->IsAmbiguous = false;
578  }
579  }
580 
581  ArrayRef<ModuleMacro*> getOverriddenMacros() const {
582  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
583  return Info->OverriddenMacros;
584  return None;
585  }
586 
587  void setOverriddenMacros(Preprocessor &PP,
588  ArrayRef<ModuleMacro *> Overrides) {
589  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
590  if (!Info) {
591  if (Overrides.empty())
592  return;
593  Info = new (PP.getPreprocessorAllocator())
594  ModuleMacroInfo(State.get<MacroDirective *>());
595  State = Info;
596  }
597  Info->OverriddenMacros.clear();
598  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
599  Overrides.begin(), Overrides.end());
600  Info->ActiveModuleMacrosGeneration = 0;
601  }
602  };
603 
604  /// For each IdentifierInfo that was associated with a macro, we
605  /// keep a mapping to the history of all macro definitions and #undefs in
606  /// the reverse order (the latest one is in the head of the list).
607  ///
608  /// This mapping lives within the \p CurSubmoduleState.
609  using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
610 
611  struct SubmoduleState;
612 
613  /// Information about a submodule that we're currently building.
614  struct BuildingSubmoduleInfo {
615  /// The module that we are building.
616  Module *M;
617 
618  /// The location at which the module was included.
619  SourceLocation ImportLoc;
620 
621  /// Whether we entered this submodule via a pragma.
622  bool IsPragma;
623 
624  /// The previous SubmoduleState.
625  SubmoduleState *OuterSubmoduleState;
626 
627  /// The number of pending module macro names when we started building this.
628  unsigned OuterPendingModuleMacroNames;
629 
630  BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
631  SubmoduleState *OuterSubmoduleState,
632  unsigned OuterPendingModuleMacroNames)
633  : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
634  OuterSubmoduleState(OuterSubmoduleState),
635  OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
636  };
637  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
638 
639  /// Information about a submodule's preprocessor state.
640  struct SubmoduleState {
641  /// The macros for the submodule.
642  MacroMap Macros;
643 
644  /// The set of modules that are visible within the submodule.
645  VisibleModuleSet VisibleModules;
646 
647  // FIXME: CounterValue?
648  // FIXME: PragmaPushMacroInfo?
649  };
650  std::map<Module *, SubmoduleState> Submodules;
651 
652  /// The preprocessor state for preprocessing outside of any submodule.
653  SubmoduleState NullSubmoduleState;
654 
655  /// The current submodule state. Will be \p NullSubmoduleState if we're not
656  /// in a submodule.
657  SubmoduleState *CurSubmoduleState;
658 
659  /// The set of known macros exported from modules.
660  llvm::FoldingSet<ModuleMacro> ModuleMacros;
661 
662  /// The names of potential module macros that we've not yet processed.
663  llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
664 
665  /// The list of module macros, for each identifier, that are not overridden by
666  /// any other module macro.
667  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
668  LeafModuleMacros;
669 
670  /// Macros that we want to warn because they are not used at the end
671  /// of the translation unit.
672  ///
673  /// We store just their SourceLocations instead of
674  /// something like MacroInfo*. The benefit of this is that when we are
675  /// deserializing from PCH, we don't need to deserialize identifier & macros
676  /// just so that we can report that they are unused, we just warn using
677  /// the SourceLocations of this set (that will be filled by the ASTReader).
678  /// We are using SmallPtrSet instead of a vector for faster removal.
679  using WarnUnusedMacroLocsTy = llvm::SmallPtrSet<SourceLocation, 32>;
680  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
681 
682  /// A "freelist" of MacroArg objects that can be
683  /// reused for quick allocation.
684  MacroArgs *MacroArgCache = nullptr;
685 
686  /// For each IdentifierInfo used in a \#pragma push_macro directive,
687  /// we keep a MacroInfo stack used to restore the previous macro value.
688  llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
689  PragmaPushMacroInfo;
690 
691  // Various statistics we track for performance analysis.
692  unsigned NumDirectives = 0;
693  unsigned NumDefined = 0;
694  unsigned NumUndefined = 0;
695  unsigned NumPragma = 0;
696  unsigned NumIf = 0;
697  unsigned NumElse = 0;
698  unsigned NumEndif = 0;
699  unsigned NumEnteredSourceFiles = 0;
700  unsigned MaxIncludeStackDepth = 0;
701  unsigned NumMacroExpanded = 0;
702  unsigned NumFnMacroExpanded = 0;
703  unsigned NumBuiltinMacroExpanded = 0;
704  unsigned NumFastMacroExpanded = 0;
705  unsigned NumTokenPaste = 0;
706  unsigned NumFastTokenPaste = 0;
707  unsigned NumSkipped = 0;
708 
709  /// The predefined macros that preprocessor should use from the
710  /// command line etc.
711  std::string Predefines;
712 
713  /// The file ID for the preprocessor predefines.
714  FileID PredefinesFileID;
715 
716  /// The file ID for the PCH through header.
717  FileID PCHThroughHeaderFileID;
718 
719  /// Whether tokens are being skipped until a #pragma hdrstop is seen.
720  bool SkippingUntilPragmaHdrStop = false;
721 
722  /// Whether tokens are being skipped until the through header is seen.
723  bool SkippingUntilPCHThroughHeader = false;
724 
725  /// \{
726  /// Cache of macro expanders to reduce malloc traffic.
727  enum { TokenLexerCacheSize = 8 };
728  unsigned NumCachedTokenLexers;
729  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
730  /// \}
731 
732  /// Keeps macro expanded tokens for TokenLexers.
733  //
734  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
735  /// going to lex in the cache and when it finishes the tokens are removed
736  /// from the end of the cache.
737  SmallVector<Token, 16> MacroExpandedTokens;
738  std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
739 
740  /// A record of the macro definitions and expansions that
741  /// occurred during preprocessing.
742  ///
743  /// This is an optional side structure that can be enabled with
744  /// \c createPreprocessingRecord() prior to preprocessing.
745  PreprocessingRecord *Record = nullptr;
746 
747  /// Cached tokens state.
749 
750  /// Cached tokens are stored here when we do backtracking or
751  /// lookahead. They are "lexed" by the CachingLex() method.
753 
754  /// The position of the cached token that CachingLex() should
755  /// "lex" next.
756  ///
757  /// If it points beyond the CachedTokens vector, it means that a normal
758  /// Lex() should be invoked.
759  CachedTokensTy::size_type CachedLexPos = 0;
760 
761  /// Stack of backtrack positions, allowing nested backtracks.
762  ///
763  /// The EnableBacktrackAtThisPos() method pushes a position to
764  /// indicate where CachedLexPos should be set when the BackTrack() method is
765  /// invoked (at which point the last position is popped).
766  std::vector<CachedTokensTy::size_type> BacktrackPositions;
767 
768  struct MacroInfoChain {
769  MacroInfo MI;
770  MacroInfoChain *Next;
771  };
772 
773  /// MacroInfos are managed as a chain for easy disposal. This is the head
774  /// of that list.
775  MacroInfoChain *MIChainHead = nullptr;
776 
777  void updateOutOfDateIdentifier(IdentifierInfo &II) const;
778 
779 public:
780  Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
782  MemoryBufferCache &PCMCache,
783  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
784  IdentifierInfoLookup *IILookup = nullptr,
785  bool OwnsHeaderSearch = false,
787 
788  ~Preprocessor();
789 
790  /// Initialize the preprocessor using information about the target.
791  ///
792  /// \param Target is owned by the caller and must remain valid for the
793  /// lifetime of the preprocessor.
794  /// \param AuxTarget is owned by the caller and must remain valid for
795  /// the lifetime of the preprocessor.
796  void Initialize(const TargetInfo &Target,
797  const TargetInfo *AuxTarget = nullptr);
798 
799  /// Initialize the preprocessor to parse a model file
800  ///
801  /// To parse model files the preprocessor of the original source is reused to
802  /// preserver the identifier table. However to avoid some duplicate
803  /// information in the preprocessor some cleanup is needed before it is used
804  /// to parse model files. This method does that cleanup.
805  void InitializeForModelFile();
806 
807  /// Cleanup after model file parsing
808  void FinalizeForModelFile();
809 
810  /// Retrieve the preprocessor options used to initialize this
811  /// preprocessor.
812  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
813 
814  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
815  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
816 
817  const LangOptions &getLangOpts() const { return LangOpts; }
818  const TargetInfo &getTargetInfo() const { return *Target; }
819  const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
820  FileManager &getFileManager() const { return FileMgr; }
821  SourceManager &getSourceManager() const { return SourceMgr; }
822  MemoryBufferCache &getPCMCache() const { return PCMCache; }
823  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
824 
825  IdentifierTable &getIdentifierTable() { return Identifiers; }
826  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
827  SelectorTable &getSelectorTable() { return Selectors; }
829  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
830 
832  ExternalSource = Source;
833  }
834 
836  return ExternalSource;
837  }
838 
839  /// Retrieve the module loader associated with this preprocessor.
840  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
841 
843  return TheModuleLoader.HadFatalFailure;
844  }
845 
846  /// True if we are currently preprocessing a #if or #elif directive
848  return ParsingIfOrElifDirective;
849  }
850 
851  /// Control whether the preprocessor retains comments in output.
852  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
853  this->KeepComments = KeepComments | KeepMacroComments;
854  this->KeepMacroComments = KeepMacroComments;
855  }
856 
857  bool getCommentRetentionState() const { return KeepComments; }
858 
859  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
860  bool getPragmasEnabled() const { return PragmasEnabled; }
861 
862  void SetSuppressIncludeNotFoundError(bool Suppress) {
863  SuppressIncludeNotFoundError = Suppress;
864  }
865 
867  return SuppressIncludeNotFoundError;
868  }
869 
870  /// Sets whether the preprocessor is responsible for producing output or if
871  /// it is producing tokens to be consumed by Parse and Sema.
872  void setPreprocessedOutput(bool IsPreprocessedOutput) {
873  PreprocessedOutput = IsPreprocessedOutput;
874  }
875 
876  /// Returns true if the preprocessor is responsible for generating output,
877  /// false if it is producing tokens to be consumed by Parse and Sema.
878  bool isPreprocessedOutput() const { return PreprocessedOutput; }
879 
880  /// Return true if we are lexing directly from the specified lexer.
881  bool isCurrentLexer(const PreprocessorLexer *L) const {
882  return CurPPLexer == L;
883  }
884 
885  /// Return the current lexer being lexed from.
886  ///
887  /// Note that this ignores any potentially active macro expansions and _Pragma
888  /// expansions going on at the time.
889  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
890 
891  /// Return the current file lexer being lexed from.
892  ///
893  /// Note that this ignores any potentially active macro expansions and _Pragma
894  /// expansions going on at the time.
895  PreprocessorLexer *getCurrentFileLexer() const;
896 
897  /// Return the submodule owning the file being lexed. This may not be
898  /// the current module if we have changed modules since entering the file.
899  Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
900 
901  /// Returns the FileID for the preprocessor predefines.
902  FileID getPredefinesFileID() const { return PredefinesFileID; }
903 
904  /// \{
905  /// Accessors for preprocessor callbacks.
906  ///
907  /// Note that this class takes ownership of any PPCallbacks object given to
908  /// it.
909  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
910  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
911  if (Callbacks)
912  C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
913  std::move(Callbacks));
914  Callbacks = std::move(C);
915  }
916  /// \}
917 
918  bool isMacroDefined(StringRef Id) {
919  return isMacroDefined(&Identifiers.get(Id));
920  }
921  bool isMacroDefined(const IdentifierInfo *II) {
922  return II->hasMacroDefinition() &&
923  (!getLangOpts().Modules || (bool)getMacroDefinition(II));
924  }
925 
926  /// Determine whether II is defined as a macro within the module M,
927  /// if that is a module that we've already preprocessed. Does not check for
928  /// macros imported into M.
930  if (!II->hasMacroDefinition())
931  return false;
932  auto I = Submodules.find(M);
933  if (I == Submodules.end())
934  return false;
935  auto J = I->second.Macros.find(II);
936  if (J == I->second.Macros.end())
937  return false;
938  auto *MD = J->second.getLatest();
939  return MD && MD->isDefined();
940  }
941 
943  if (!II->hasMacroDefinition())
944  return {};
945 
946  MacroState &S = CurSubmoduleState->Macros[II];
947  auto *MD = S.getLatest();
948  while (MD && isa<VisibilityMacroDirective>(MD))
949  MD = MD->getPrevious();
950  return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
951  S.getActiveModuleMacros(*this, II),
952  S.isAmbiguous(*this, II));
953  }
954 
956  SourceLocation Loc) {
957  if (!II->hadMacroDefinition())
958  return {};
959 
960  MacroState &S = CurSubmoduleState->Macros[II];
962  if (auto *MD = S.getLatest())
963  DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
964  // FIXME: Compute the set of active module macros at the specified location.
965  return MacroDefinition(DI.getDirective(),
966  S.getActiveModuleMacros(*this, II),
967  S.isAmbiguous(*this, II));
968  }
969 
970  /// Given an identifier, return its latest non-imported MacroDirective
971  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
973  if (!II->hasMacroDefinition())
974  return nullptr;
975 
976  auto *MD = getLocalMacroDirectiveHistory(II);
977  if (!MD || MD->getDefinition().isUndefined())
978  return nullptr;
979 
980  return MD;
981  }
982 
983  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
984  return const_cast<Preprocessor*>(this)->getMacroInfo(II);
985  }
986 
988  if (!II->hasMacroDefinition())
989  return nullptr;
990  if (auto MD = getMacroDefinition(II))
991  return MD.getMacroInfo();
992  return nullptr;
993  }
994 
995  /// Given an identifier, return the latest non-imported macro
996  /// directive for that identifier.
997  ///
998  /// One can iterate over all previous macro directives from the most recent
999  /// one.
1000  MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const;
1001 
1002  /// Add a directive to the macro directive history for this identifier.
1003  void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
1005  SourceLocation Loc) {
1006  DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1007  appendMacroDirective(II, MD);
1008  return MD;
1009  }
1011  MacroInfo *MI) {
1012  return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1013  }
1014 
1015  /// Set a MacroDirective that was loaded from a PCH file.
1016  void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED,
1017  MacroDirective *MD);
1018 
1019  /// Register an exported macro for a module and identifier.
1020  ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro,
1021  ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1022  ModuleMacro *getModuleMacro(Module *Mod, IdentifierInfo *II);
1023 
1024  /// Get the list of leaf (non-overridden) module macros for a name.
1026  if (II->isOutOfDate())
1027  updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1028  auto I = LeafModuleMacros.find(II);
1029  if (I != LeafModuleMacros.end())
1030  return I->second;
1031  return None;
1032  }
1033 
1034  /// \{
1035  /// Iterators for the macro history table. Currently defined macros have
1036  /// IdentifierInfo::hasMacroDefinition() set and an empty
1037  /// MacroInfo::getUndefLoc() at the head of the list.
1038  using macro_iterator = MacroMap::const_iterator;
1039 
1040  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1041  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1042 
1043  llvm::iterator_range<macro_iterator>
1044  macros(bool IncludeExternalMacros = true) const {
1045  macro_iterator begin = macro_begin(IncludeExternalMacros);
1046  macro_iterator end = macro_end(IncludeExternalMacros);
1047  return llvm::make_range(begin, end);
1048  }
1049 
1050  /// \}
1051 
1052  /// Return the name of the macro defined before \p Loc that has
1053  /// spelling \p Tokens. If there are multiple macros with same spelling,
1054  /// return the last one defined.
1055  StringRef getLastMacroWithSpelling(SourceLocation Loc,
1056  ArrayRef<TokenValue> Tokens) const;
1057 
1058  const std::string &getPredefines() const { return Predefines; }
1059 
1060  /// Set the predefines for this Preprocessor.
1061  ///
1062  /// These predefines are automatically injected when parsing the main file.
1063  void setPredefines(const char *P) { Predefines = P; }
1064  void setPredefines(StringRef P) { Predefines = P; }
1065 
1066  /// Return information about the specified preprocessor
1067  /// identifier token.
1068  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1069  return &Identifiers.get(Name);
1070  }
1071 
1072  /// Add the specified pragma handler to this preprocessor.
1073  ///
1074  /// If \p Namespace is non-null, then it is a token required to exist on the
1075  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1076  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1078  AddPragmaHandler(StringRef(), Handler);
1079  }
1080 
1081  /// Remove the specific pragma handler from this preprocessor.
1082  ///
1083  /// If \p Namespace is non-null, then it should be the namespace that
1084  /// \p Handler was added to. It is an error to remove a handler that
1085  /// has not been registered.
1086  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1088  RemovePragmaHandler(StringRef(), Handler);
1089  }
1090 
1091  /// Install empty handlers for all pragmas (making them ignored).
1092  void IgnorePragmas();
1093 
1094  /// Add the specified comment handler to the preprocessor.
1095  void addCommentHandler(CommentHandler *Handler);
1096 
1097  /// Remove the specified comment handler.
1098  ///
1099  /// It is an error to remove a handler that has not been registered.
1100  void removeCommentHandler(CommentHandler *Handler);
1101 
1102  /// Set the code completion handler to the given object.
1104  CodeComplete = &Handler;
1105  }
1106 
1107  /// Retrieve the current code-completion handler.
1109  return CodeComplete;
1110  }
1111 
1112  /// Clear out the code completion handler.
1114  CodeComplete = nullptr;
1115  }
1116 
1117  /// Hook used by the lexer to invoke the "included file" code
1118  /// completion point.
1119  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1120 
1121  /// Hook used by the lexer to invoke the "natural language" code
1122  /// completion point.
1123  void CodeCompleteNaturalLanguage();
1124 
1125  /// Set the code completion token for filtering purposes.
1127  CodeCompletionII = Filter;
1128  }
1129 
1130  /// Set the code completion token range for detecting replacement range later
1131  /// on.
1133  const SourceLocation End) {
1134  CodeCompletionTokenRange = {Start, End};
1135  }
1137  return CodeCompletionTokenRange;
1138  }
1139 
1140  /// Get the code completion token for filtering purposes.
1142  if (CodeCompletionII)
1143  return CodeCompletionII->getName();
1144  return {};
1145  }
1146 
1147  /// Retrieve the preprocessing record, or NULL if there is no
1148  /// preprocessing record.
1149  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1150 
1151  /// Create a new preprocessing record, which will keep track of
1152  /// all macro expansions, macro definitions, etc.
1153  void createPreprocessingRecord();
1154 
1155  /// Returns true if the FileEntry is the PCH through header.
1156  bool isPCHThroughHeader(const FileEntry *FE);
1157 
1158  /// True if creating a PCH with a through header.
1159  bool creatingPCHWithThroughHeader();
1160 
1161  /// True if using a PCH with a through header.
1162  bool usingPCHWithThroughHeader();
1163 
1164  /// True if creating a PCH with a #pragma hdrstop.
1165  bool creatingPCHWithPragmaHdrStop();
1166 
1167  /// True if using a PCH with a #pragma hdrstop.
1168  bool usingPCHWithPragmaHdrStop();
1169 
1170  /// Skip tokens until after the #include of the through header or
1171  /// until after a #pragma hdrstop.
1172  void SkipTokensWhileUsingPCH();
1173 
1174  /// Process directives while skipping until the through header or
1175  /// #pragma hdrstop is found.
1176  void HandleSkippedDirectiveWhileUsingPCH(Token &Result,
1177  SourceLocation HashLoc);
1178 
1179  /// Enter the specified FileID as the main source file,
1180  /// which implicitly adds the builtin defines etc.
1181  void EnterMainSourceFile();
1182 
1183  /// Inform the preprocessor callbacks that processing is complete.
1184  void EndSourceFile();
1185 
1186  /// Add a source file to the top of the include stack and
1187  /// start lexing tokens from it instead of the current buffer.
1188  ///
1189  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1190  bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir,
1191  SourceLocation Loc);
1192 
1193  /// Add a Macro to the top of the include stack and start lexing
1194  /// tokens from it instead of the current buffer.
1195  ///
1196  /// \param Args specifies the tokens input to a function-like macro.
1197  /// \param ILEnd specifies the location of the ')' for a function-like macro
1198  /// or the identifier for an object-like macro.
1199  void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1200  MacroArgs *Args);
1201 
1202  /// Add a "macro" context to the top of the include stack,
1203  /// which will cause the lexer to start returning the specified tokens.
1204  ///
1205  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1206  /// will not be subject to further macro expansion. Otherwise, these tokens
1207  /// will be re-macro-expanded when/if expansion is enabled.
1208  ///
1209  /// If \p OwnsTokens is false, this method assumes that the specified stream
1210  /// of tokens has a permanent owner somewhere, so they do not need to be
1211  /// copied. If it is true, it assumes the array of tokens is allocated with
1212  /// \c new[] and the Preprocessor will delete[] it.
1213 private:
1214  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1215  bool DisableMacroExpansion, bool OwnsTokens);
1216 
1217 public:
1218  void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1219  bool DisableMacroExpansion) {
1220  EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true);
1221  }
1222 
1223  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion) {
1224  EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false);
1225  }
1226 
1227  /// Pop the current lexer/macro exp off the top of the lexer stack.
1228  ///
1229  /// This should only be used in situations where the current state of the
1230  /// top-of-stack lexer is known.
1231  void RemoveTopOfLexerStack();
1232 
1233  /// From the point that this method is called, and until
1234  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1235  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1236  /// make the Preprocessor re-lex the same tokens.
1237  ///
1238  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1239  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1240  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1241  ///
1242  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1243  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1244  /// tokens will continue indefinitely.
1245  ///
1246  void EnableBacktrackAtThisPos();
1247 
1248  /// Disable the last EnableBacktrackAtThisPos call.
1249  void CommitBacktrackedTokens();
1250 
1252  CachedTokensTy::size_type Begin, End;
1253  };
1254 
1255 private:
1256  /// A range of cached tokens that should be erased after lexing
1257  /// when backtracking requires the erasure of such cached tokens.
1258  Optional<CachedTokensRange> CachedTokenRangeToErase;
1259 
1260 public:
1261  /// Returns the range of cached tokens that were lexed since
1262  /// EnableBacktrackAtThisPos() was previously called.
1263  CachedTokensRange LastCachedTokenRange();
1264 
1265  /// Erase the range of cached tokens that were lexed since
1266  /// EnableBacktrackAtThisPos() was previously called.
1267  void EraseCachedTokens(CachedTokensRange TokenRange);
1268 
1269  /// Make Preprocessor re-lex the tokens that were lexed since
1270  /// EnableBacktrackAtThisPos() was previously called.
1271  void Backtrack();
1272 
1273  /// True if EnableBacktrackAtThisPos() was called and
1274  /// caching of tokens is on.
1275  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1276 
1277  /// Lex the next token for this preprocessor.
1278  void Lex(Token &Result);
1279 
1280  void LexAfterModuleImport(Token &Result);
1281 
1282  void makeModuleVisible(Module *M, SourceLocation Loc);
1283 
1285  return CurSubmoduleState->VisibleModules.getImportLoc(M);
1286  }
1287 
1288  /// Lex a string literal, which may be the concatenation of multiple
1289  /// string literals and may even come from macro expansion.
1290  /// \returns true on success, false if a error diagnostic has been generated.
1291  bool LexStringLiteral(Token &Result, std::string &String,
1292  const char *DiagnosticTag, bool AllowMacroExpansion) {
1293  if (AllowMacroExpansion)
1294  Lex(Result);
1295  else
1296  LexUnexpandedToken(Result);
1297  return FinishLexStringLiteral(Result, String, DiagnosticTag,
1298  AllowMacroExpansion);
1299  }
1300 
1301  /// Complete the lexing of a string literal where the first token has
1302  /// already been lexed (see LexStringLiteral).
1303  bool FinishLexStringLiteral(Token &Result, std::string &String,
1304  const char *DiagnosticTag,
1305  bool AllowMacroExpansion);
1306 
1307  /// Lex a token. If it's a comment, keep lexing until we get
1308  /// something not a comment.
1309  ///
1310  /// This is useful in -E -C mode where comments would foul up preprocessor
1311  /// directive handling.
1312  void LexNonComment(Token &Result) {
1313  do
1314  Lex(Result);
1315  while (Result.getKind() == tok::comment);
1316  }
1317 
1318  /// Just like Lex, but disables macro expansion of identifier tokens.
1319  void LexUnexpandedToken(Token &Result) {
1320  // Disable macro expansion.
1321  bool OldVal = DisableMacroExpansion;
1322  DisableMacroExpansion = true;
1323  // Lex the token.
1324  Lex(Result);
1325 
1326  // Reenable it.
1327  DisableMacroExpansion = OldVal;
1328  }
1329 
1330  /// Like LexNonComment, but this disables macro expansion of
1331  /// identifier tokens.
1333  do
1334  LexUnexpandedToken(Result);
1335  while (Result.getKind() == tok::comment);
1336  }
1337 
1338  /// Parses a simple integer literal to get its numeric value. Floating
1339  /// point literals and user defined literals are rejected. Used primarily to
1340  /// handle pragmas that accept integer arguments.
1341  bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1342 
1343  /// Disables macro expansion everywhere except for preprocessor directives.
1345  DisableMacroExpansion = true;
1346  MacroExpansionInDirectivesOverride = true;
1347  }
1348 
1349  /// Peeks ahead N tokens and returns that token without consuming any
1350  /// tokens.
1351  ///
1352  /// LookAhead(0) returns the next token that would be returned by Lex(),
1353  /// LookAhead(1) returns the token after it, etc. This returns normal
1354  /// tokens after phase 5. As such, it is equivalent to using
1355  /// 'Lex', not 'LexUnexpandedToken'.
1356  const Token &LookAhead(unsigned N) {
1357  if (CachedLexPos + N < CachedTokens.size())
1358  return CachedTokens[CachedLexPos+N];
1359  else
1360  return PeekAhead(N+1);
1361  }
1362 
1363  /// When backtracking is enabled and tokens are cached,
1364  /// this allows to revert a specific number of tokens.
1365  ///
1366  /// Note that the number of tokens being reverted should be up to the last
1367  /// backtrack position, not more.
1368  void RevertCachedTokens(unsigned N) {
1369  assert(isBacktrackEnabled() &&
1370  "Should only be called when tokens are cached for backtracking");
1371  assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1372  && "Should revert tokens up to the last backtrack position, not more");
1373  assert(signed(CachedLexPos) - signed(N) >= 0 &&
1374  "Corrupted backtrack positions ?");
1375  CachedLexPos -= N;
1376  }
1377 
1378  /// Enters a token in the token stream to be lexed next.
1379  ///
1380  /// If BackTrack() is called afterwards, the token will remain at the
1381  /// insertion point.
1382  void EnterToken(const Token &Tok) {
1383  EnterCachingLexMode();
1384  CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1385  }
1386 
1387  /// We notify the Preprocessor that if it is caching tokens (because
1388  /// backtrack is enabled) it should replace the most recent cached tokens
1389  /// with the given annotation token. This function has no effect if
1390  /// backtracking is not enabled.
1391  ///
1392  /// Note that the use of this function is just for optimization, so that the
1393  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1394  /// invoked.
1395  void AnnotateCachedTokens(const Token &Tok) {
1396  assert(Tok.isAnnotation() && "Expected annotation token");
1397  if (CachedLexPos != 0 && isBacktrackEnabled())
1398  AnnotatePreviousCachedTokens(Tok);
1399  }
1400 
1401  /// Get the location of the last cached token, suitable for setting the end
1402  /// location of an annotation token.
1404  assert(CachedLexPos != 0);
1405  return CachedTokens[CachedLexPos-1].getLastLoc();
1406  }
1407 
1408  /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1409  /// CachedTokens.
1410  bool IsPreviousCachedToken(const Token &Tok) const;
1411 
1412  /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1413  /// in \p NewToks.
1414  ///
1415  /// Useful when a token needs to be split in smaller ones and CachedTokens
1416  /// most recent token must to be updated to reflect that.
1417  void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1418 
1419  /// Replace the last token with an annotation token.
1420  ///
1421  /// Like AnnotateCachedTokens(), this routine replaces an
1422  /// already-parsed (and resolved) token with an annotation
1423  /// token. However, this routine only replaces the last token with
1424  /// the annotation token; it does not affect any other cached
1425  /// tokens. This function has no effect if backtracking is not
1426  /// enabled.
1428  assert(Tok.isAnnotation() && "Expected annotation token");
1429  if (CachedLexPos != 0 && isBacktrackEnabled())
1430  CachedTokens[CachedLexPos-1] = Tok;
1431  }
1432 
1433  /// Enter an annotation token into the token stream.
1434  void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1435  void *AnnotationVal);
1436 
1437  /// Update the current token to represent the provided
1438  /// identifier, in order to cache an action performed by typo correction.
1439  void TypoCorrectToken(const Token &Tok) {
1440  assert(Tok.getIdentifierInfo() && "Expected identifier token");
1441  if (CachedLexPos != 0 && isBacktrackEnabled())
1442  CachedTokens[CachedLexPos-1] = Tok;
1443  }
1444 
1445  /// Recompute the current lexer kind based on the CurLexer/
1446  /// CurTokenLexer pointers.
1447  void recomputeCurLexerKind();
1448 
1449  /// Returns true if incremental processing is enabled
1450  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1451 
1452  /// Enables the incremental processing
1453  void enableIncrementalProcessing(bool value = true) {
1454  IncrementalProcessing = value;
1455  }
1456 
1457  /// Specify the point at which code-completion will be performed.
1458  ///
1459  /// \param File the file in which code completion should occur. If
1460  /// this file is included multiple times, code-completion will
1461  /// perform completion the first time it is included. If NULL, this
1462  /// function clears out the code-completion point.
1463  ///
1464  /// \param Line the line at which code completion should occur
1465  /// (1-based).
1466  ///
1467  /// \param Column the column at which code completion should occur
1468  /// (1-based).
1469  ///
1470  /// \returns true if an error occurred, false otherwise.
1471  bool SetCodeCompletionPoint(const FileEntry *File,
1472  unsigned Line, unsigned Column);
1473 
1474  /// Determine if we are performing code completion.
1475  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1476 
1477  /// Returns the location of the code-completion point.
1478  ///
1479  /// Returns an invalid location if code-completion is not enabled or the file
1480  /// containing the code-completion point has not been lexed yet.
1481  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1482 
1483  /// Returns the start location of the file of code-completion point.
1484  ///
1485  /// Returns an invalid location if code-completion is not enabled or the file
1486  /// containing the code-completion point has not been lexed yet.
1488  return CodeCompletionFileLoc;
1489  }
1490 
1491  /// Returns true if code-completion is enabled and we have hit the
1492  /// code-completion point.
1493  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1494 
1495  /// Note that we hit the code-completion point.
1497  assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1498  CodeCompletionReached = true;
1499  // Silence any diagnostics that occur after we hit the code-completion.
1500  getDiagnostics().setSuppressAllDiagnostics(true);
1501  }
1502 
1503  /// The location of the currently-active \#pragma clang
1504  /// arc_cf_code_audited begin.
1505  ///
1506  /// Returns an invalid location if there is no such pragma active.
1508  return PragmaARCCFCodeAuditedLoc;
1509  }
1510 
1511  /// Set the location of the currently-active \#pragma clang
1512  /// arc_cf_code_audited begin. An invalid location ends the pragma.
1514  PragmaARCCFCodeAuditedLoc = Loc;
1515  }
1516 
1517  /// The location of the currently-active \#pragma clang
1518  /// assume_nonnull begin.
1519  ///
1520  /// Returns an invalid location if there is no such pragma active.
1522  return PragmaAssumeNonNullLoc;
1523  }
1524 
1525  /// Set the location of the currently-active \#pragma clang
1526  /// assume_nonnull begin. An invalid location ends the pragma.
1528  PragmaAssumeNonNullLoc = Loc;
1529  }
1530 
1531  /// Set the directory in which the main file should be considered
1532  /// to have been found, if it is not a real file.
1533  void setMainFileDir(const DirectoryEntry *Dir) {
1534  MainFileDir = Dir;
1535  }
1536 
1537  /// Instruct the preprocessor to skip part of the main source file.
1538  ///
1539  /// \param Bytes The number of bytes in the preamble to skip.
1540  ///
1541  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1542  /// start of a line.
1543  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1544  SkipMainFilePreamble.first = Bytes;
1545  SkipMainFilePreamble.second = StartOfLine;
1546  }
1547 
1548  /// Forwarding function for diagnostics. This emits a diagnostic at
1549  /// the specified Token's location, translating the token's start
1550  /// position in the current buffer into a SourcePosition object for rendering.
1551  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1552  return Diags->Report(Loc, DiagID);
1553  }
1554 
1555  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1556  return Diags->Report(Tok.getLocation(), DiagID);
1557  }
1558 
1559  /// Return the 'spelling' of the token at the given
1560  /// location; does not go up to the spelling location or down to the
1561  /// expansion location.
1562  ///
1563  /// \param buffer A buffer which will be used only if the token requires
1564  /// "cleaning", e.g. if it contains trigraphs or escaped newlines
1565  /// \param invalid If non-null, will be set \c true if an error occurs.
1567  SmallVectorImpl<char> &buffer,
1568  bool *invalid = nullptr) const {
1569  return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1570  }
1571 
1572  /// Return the 'spelling' of the Tok token.
1573  ///
1574  /// The spelling of a token is the characters used to represent the token in
1575  /// the source file after trigraph expansion and escaped-newline folding. In
1576  /// particular, this wants to get the true, uncanonicalized, spelling of
1577  /// things like digraphs, UCNs, etc.
1578  ///
1579  /// \param Invalid If non-null, will be set \c true if an error occurs.
1580  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1581  return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1582  }
1583 
1584  /// Get the spelling of a token into a preallocated buffer, instead
1585  /// of as an std::string.
1586  ///
1587  /// The caller is required to allocate enough space for the token, which is
1588  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1589  /// actual result is returned.
1590  ///
1591  /// Note that this method may do two possible things: it may either fill in
1592  /// the buffer specified with characters, or it may *change the input pointer*
1593  /// to point to a constant buffer with the data already in it (avoiding a
1594  /// copy). The caller is not allowed to modify the returned buffer pointer
1595  /// if an internal buffer is returned.
1596  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1597  bool *Invalid = nullptr) const {
1598  return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1599  }
1600 
1601  /// Get the spelling of a token into a SmallVector.
1602  ///
1603  /// Note that the returned StringRef may not point to the
1604  /// supplied buffer if a copy can be avoided.
1605  StringRef getSpelling(const Token &Tok,
1606  SmallVectorImpl<char> &Buffer,
1607  bool *Invalid = nullptr) const;
1608 
1609  /// Relex the token at the specified location.
1610  /// \returns true if there was a failure, false on success.
1611  bool getRawToken(SourceLocation Loc, Token &Result,
1612  bool IgnoreWhiteSpace = false) {
1613  return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1614  }
1615 
1616  /// Given a Token \p Tok that is a numeric constant with length 1,
1617  /// return the character.
1618  char
1620  bool *Invalid = nullptr) const {
1621  assert(Tok.is(tok::numeric_constant) &&
1622  Tok.getLength() == 1 && "Called on unsupported token");
1623  assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1624 
1625  // If the token is carrying a literal data pointer, just use it.
1626  if (const char *D = Tok.getLiteralData())
1627  return *D;
1628 
1629  // Otherwise, fall back on getCharacterData, which is slower, but always
1630  // works.
1631  return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1632  }
1633 
1634  /// Retrieve the name of the immediate macro expansion.
1635  ///
1636  /// This routine starts from a source location, and finds the name of the
1637  /// macro responsible for its immediate expansion. It looks through any
1638  /// intervening macro argument expansions to compute this. It returns a
1639  /// StringRef that refers to the SourceManager-owned buffer of the source
1640  /// where that macro name is spelled. Thus, the result shouldn't out-live
1641  /// the SourceManager.
1643  return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1644  }
1645 
1646  /// Plop the specified string into a scratch buffer and set the
1647  /// specified token's location and length to it.
1648  ///
1649  /// If specified, the source location provides a location of the expansion
1650  /// point of the token.
1651  void CreateString(StringRef Str, Token &Tok,
1652  SourceLocation ExpansionLocStart = SourceLocation(),
1653  SourceLocation ExpansionLocEnd = SourceLocation());
1654 
1655  /// Split the first Length characters out of the token starting at TokLoc
1656  /// and return a location pointing to the split token. Re-lexing from the
1657  /// split token will return the split token rather than the original.
1658  SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
1659 
1660  /// Computes the source location just past the end of the
1661  /// token at this source location.
1662  ///
1663  /// This routine can be used to produce a source location that
1664  /// points just past the end of the token referenced by \p Loc, and
1665  /// is generally used when a diagnostic needs to point just after a
1666  /// token where it expected something different that it received. If
1667  /// the returned source location would not be meaningful (e.g., if
1668  /// it points into a macro), this routine returns an invalid
1669  /// source location.
1670  ///
1671  /// \param Offset an offset from the end of the token, where the source
1672  /// location should refer to. The default offset (0) produces a source
1673  /// location pointing just past the end of the token; an offset of 1 produces
1674  /// a source location pointing to the last character in the token, etc.
1676  return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1677  }
1678 
1679  /// Returns true if the given MacroID location points at the first
1680  /// token of the macro expansion.
1681  ///
1682  /// \param MacroBegin If non-null and function returns true, it is set to
1683  /// begin location of the macro.
1685  SourceLocation *MacroBegin = nullptr) const {
1686  return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1687  MacroBegin);
1688  }
1689 
1690  /// Returns true if the given MacroID location points at the last
1691  /// token of the macro expansion.
1692  ///
1693  /// \param MacroEnd If non-null and function returns true, it is set to
1694  /// end location of the macro.
1696  SourceLocation *MacroEnd = nullptr) const {
1697  return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1698  }
1699 
1700  /// Print the token to stderr, used for debugging.
1701  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1702  void DumpLocation(SourceLocation Loc) const;
1703  void DumpMacro(const MacroInfo &MI) const;
1704  void dumpMacroInfo(const IdentifierInfo *II);
1705 
1706  /// Given a location that specifies the start of a
1707  /// token, return a new location that specifies a character within the token.
1709  unsigned Char) const {
1710  return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1711  }
1712 
1713  /// Increment the counters for the number of token paste operations
1714  /// performed.
1715  ///
1716  /// If fast was specified, this is a 'fast paste' case we handled.
1717  void IncrementPasteCounter(bool isFast) {
1718  if (isFast)
1719  ++NumFastTokenPaste;
1720  else
1721  ++NumTokenPaste;
1722  }
1723 
1724  void PrintStats();
1725 
1726  size_t getTotalMemory() const;
1727 
1728  /// When the macro expander pastes together a comment (/##/) in Microsoft
1729  /// mode, this method handles updating the current state, returning the
1730  /// token on the next source line.
1731  void HandleMicrosoftCommentPaste(Token &Tok);
1732 
1733  //===--------------------------------------------------------------------===//
1734  // Preprocessor callback methods. These are invoked by a lexer as various
1735  // directives and events are found.
1736 
1737  /// Given a tok::raw_identifier token, look up the
1738  /// identifier information for the token and install it into the token,
1739  /// updating the token kind accordingly.
1740  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1741 
1742 private:
1743  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1744 
1745 public:
1746  /// Specifies the reason for poisoning an identifier.
1747  ///
1748  /// If that identifier is accessed while poisoned, then this reason will be
1749  /// used instead of the default "poisoned" diagnostic.
1750  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1751 
1752  /// Display reason for poisoned identifier.
1753  void HandlePoisonedIdentifier(Token & Identifier);
1754 
1756  if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1757  if(II->isPoisoned()) {
1758  HandlePoisonedIdentifier(Identifier);
1759  }
1760  }
1761  }
1762 
1763 private:
1764  /// Identifiers used for SEH handling in Borland. These are only
1765  /// allowed in particular circumstances
1766  // __except block
1767  IdentifierInfo *Ident__exception_code,
1768  *Ident___exception_code,
1769  *Ident_GetExceptionCode;
1770  // __except filter expression
1771  IdentifierInfo *Ident__exception_info,
1772  *Ident___exception_info,
1773  *Ident_GetExceptionInfo;
1774  // __finally
1775  IdentifierInfo *Ident__abnormal_termination,
1776  *Ident___abnormal_termination,
1777  *Ident_AbnormalTermination;
1778 
1779  const char *getCurLexerEndPos();
1780  void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
1781 
1782 public:
1783  void PoisonSEHIdentifiers(bool Poison = true); // Borland
1784 
1785  /// Callback invoked when the lexer reads an identifier and has
1786  /// filled in the tokens IdentifierInfo member.
1787  ///
1788  /// This callback potentially macro expands it or turns it into a named
1789  /// token (like 'for').
1790  ///
1791  /// \returns true if we actually computed a token, false if we need to
1792  /// lex again.
1793  bool HandleIdentifier(Token &Identifier);
1794 
1795  /// Callback invoked when the lexer hits the end of the current file.
1796  ///
1797  /// This either returns the EOF token and returns true, or
1798  /// pops a level off the include stack and returns false, at which point the
1799  /// client should call lex again.
1800  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1801 
1802  /// Callback invoked when the current TokenLexer hits the end of its
1803  /// token stream.
1804  bool HandleEndOfTokenLexer(Token &Result);
1805 
1806  /// Callback invoked when the lexer sees a # token at the start of a
1807  /// line.
1808  ///
1809  /// This consumes the directive, modifies the lexer/preprocessor state, and
1810  /// advances the lexer(s) so that the next token read is the correct one.
1811  void HandleDirective(Token &Result);
1812 
1813  /// Ensure that the next token is a tok::eod token.
1814  ///
1815  /// If not, emit a diagnostic and consume up until the eod.
1816  /// If \p EnableMacros is true, then we consider macros that expand to zero
1817  /// tokens as being ok.
1818  void CheckEndOfDirective(const char *DirType, bool EnableMacros = false);
1819 
1820  /// Read and discard all tokens remaining on the current line until
1821  /// the tok::eod token is found. Returns the range of the skipped tokens.
1822  SourceRange DiscardUntilEndOfDirective();
1823 
1824  /// Returns true if the preprocessor has seen a use of
1825  /// __DATE__ or __TIME__ in the file so far.
1826  bool SawDateOrTime() const {
1827  return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
1828  }
1829  unsigned getCounterValue() const { return CounterValue; }
1830  void setCounterValue(unsigned V) { CounterValue = V; }
1831 
1832  /// Retrieves the module that we're currently building, if any.
1833  Module *getCurrentModule();
1834 
1835  /// Allocate a new MacroInfo object with the provided SourceLocation.
1836  MacroInfo *AllocateMacroInfo(SourceLocation L);
1837 
1838  /// Turn the specified lexer token into a fully checked and spelled
1839  /// filename, e.g. as an operand of \#include.
1840  ///
1841  /// The caller is expected to provide a buffer that is large enough to hold
1842  /// the spelling of the filename, but is also expected to handle the case
1843  /// when this method decides to use a different buffer.
1844  ///
1845  /// \returns true if the input filename was in <>'s or false if it was
1846  /// in ""'s.
1847  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
1848 
1849  /// Given a "foo" or <foo> reference, look up the indicated file.
1850  ///
1851  /// Returns null on failure. \p isAngled indicates whether the file
1852  /// reference is for system \#include's or not (i.e. using <> instead of "").
1853  const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename,
1854  bool isAngled, const DirectoryLookup *FromDir,
1855  const FileEntry *FromFile,
1856  const DirectoryLookup *&CurDir,
1857  SmallVectorImpl<char> *SearchPath,
1858  SmallVectorImpl<char> *RelativePath,
1859  ModuleMap::KnownHeader *SuggestedModule,
1860  bool *IsMapped, bool *IsFrameworkFound,
1861  bool SkipCache = false);
1862 
1863  /// Get the DirectoryLookup structure used to find the current
1864  /// FileEntry, if CurLexer is non-null and if applicable.
1865  ///
1866  /// This allows us to implement \#include_next and find directory-specific
1867  /// properties.
1868  const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1869 
1870  /// Return true if we're in the top-level file, not in a \#include.
1871  bool isInPrimaryFile() const;
1872 
1873  /// Handle cases where the \#include name is expanded
1874  /// from a macro as multiple tokens, which need to be glued together.
1875  ///
1876  /// This occurs for code like:
1877  /// \code
1878  /// \#define FOO <x/y.h>
1879  /// \#include FOO
1880  /// \endcode
1881  /// because in this case, "<x/y.h>" is returned as 7 tokens, not one.
1882  ///
1883  /// This code concatenates and consumes tokens up to the '>' token. It
1884  /// returns false if the > was found, otherwise it returns true if it finds
1885  /// and consumes the EOD marker.
1886  bool ConcatenateIncludeName(SmallString<128> &FilenameBuffer,
1887  SourceLocation &End);
1888 
1889  /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1890  /// followed by EOD. Return true if the token is not a valid on-off-switch.
1891  bool LexOnOffSwitch(tok::OnOffSwitch &Result);
1892 
1893  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
1894  bool *ShadowFlag = nullptr);
1895 
1896  void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
1897  Module *LeaveSubmodule(bool ForPragma);
1898 
1899 private:
1900  friend void TokenLexer::ExpandFunctionArguments();
1901 
1902  void PushIncludeMacroStack() {
1903  assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
1904  IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
1905  std::move(CurLexer), CurPPLexer,
1906  std::move(CurTokenLexer), CurDirLookup);
1907  CurPPLexer = nullptr;
1908  }
1909 
1910  void PopIncludeMacroStack() {
1911  CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1912  CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1913  CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1914  CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1915  CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
1916  CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1917  IncludeMacroStack.pop_back();
1918  }
1919 
1920  void PropagateLineStartLeadingSpaceInfo(Token &Result);
1921 
1922  /// Determine whether we need to create module macros for #defines in the
1923  /// current context.
1924  bool needModuleMacros() const;
1925 
1926  /// Update the set of active module macros and ambiguity flag for a module
1927  /// macro name.
1928  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
1929 
1930  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
1931  SourceLocation Loc);
1932  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
1933  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
1934  bool isPublic);
1935 
1936  /// Lex and validate a macro name, which occurs after a
1937  /// \#define or \#undef.
1938  ///
1939  /// \param MacroNameTok Token that represents the name defined or undefined.
1940  /// \param IsDefineUndef Kind if preprocessor directive.
1941  /// \param ShadowFlag Points to flag that is set if macro name shadows
1942  /// a keyword.
1943  ///
1944  /// This emits a diagnostic, sets the token kind to eod,
1945  /// and discards the rest of the macro line if the macro name is invalid.
1946  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
1947  bool *ShadowFlag = nullptr);
1948 
1949  /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
1950  /// entire line) of the macro's tokens and adds them to MacroInfo, and while
1951  /// doing so performs certain validity checks including (but not limited to):
1952  /// - # (stringization) is followed by a macro parameter
1953  /// \param MacroNameTok - Token that represents the macro name
1954  /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
1955  ///
1956  /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
1957  /// returns a nullptr if an invalid sequence of tokens is encountered.
1958  MacroInfo *ReadOptionalMacroParameterListAndBody(
1959  const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
1960 
1961  /// The ( starting an argument list of a macro definition has just been read.
1962  /// Lex the rest of the parameters and the closing ), updating \p MI with
1963  /// what we learn and saving in \p LastTok the last token read.
1964  /// Return true if an error occurs parsing the arg list.
1965  bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
1966 
1967  /// We just read a \#if or related directive and decided that the
1968  /// subsequent tokens are in the \#if'd out portion of the
1969  /// file. Lex the rest of the file, until we see an \#endif. If \p
1970  /// FoundNonSkipPortion is true, then we have already emitted code for part of
1971  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
1972  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
1973  /// already seen one so a \#else directive is a duplicate. When this returns,
1974  /// the caller can lex the first valid token.
1975  void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
1976  SourceLocation IfTokenLoc,
1977  bool FoundNonSkipPortion, bool FoundElse,
1978  SourceLocation ElseLoc = SourceLocation());
1979 
1980  /// Information about the result for evaluating an expression for a
1981  /// preprocessor directive.
1982  struct DirectiveEvalResult {
1983  /// Whether the expression was evaluated as true or not.
1984  bool Conditional;
1985 
1986  /// True if the expression contained identifiers that were undefined.
1987  bool IncludedUndefinedIds;
1988 
1989  /// The source range for the expression.
1990  SourceRange ExprRange;
1991  };
1992 
1993  /// Evaluate an integer constant expression that may occur after a
1994  /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
1995  ///
1996  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
1997  DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1998 
1999  /// Install the standard preprocessor pragmas:
2000  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
2001  void RegisterBuiltinPragmas();
2002 
2003  /// Register builtin macros such as __LINE__ with the identifier table.
2004  void RegisterBuiltinMacros();
2005 
2006  /// If an identifier token is read that is to be expanded as a macro, handle
2007  /// it and return the next token as 'Tok'. If we lexed a token, return true;
2008  /// otherwise the caller should lex again.
2009  bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2010 
2011  /// Cache macro expanded tokens for TokenLexers.
2012  //
2013  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2014  /// going to lex in the cache and when it finishes the tokens are removed
2015  /// from the end of the cache.
2016  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2017  ArrayRef<Token> tokens);
2018 
2019  void removeCachedMacroExpandedTokensOfLastLexer();
2020 
2021  /// Determine whether the next preprocessor token to be
2022  /// lexed is a '('. If so, consume the token and return true, if not, this
2023  /// method should have no observable side-effect on the lexed tokens.
2024  bool isNextPPTokenLParen();
2025 
2026  /// After reading "MACRO(", this method is invoked to read all of the formal
2027  /// arguments specified for the macro invocation. Returns null on error.
2028  MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2029  SourceLocation &MacroEnd);
2030 
2031  /// If an identifier token is read that is to be expanded
2032  /// as a builtin macro, handle it and return the next token as 'Tok'.
2033  void ExpandBuiltinMacro(Token &Tok);
2034 
2035  /// Read a \c _Pragma directive, slice it up, process it, then
2036  /// return the first token after the directive.
2037  /// This assumes that the \c _Pragma token has just been read into \p Tok.
2038  void Handle_Pragma(Token &Tok);
2039 
2040  /// Like Handle_Pragma except the pragma text is not enclosed within
2041  /// a string literal.
2042  void HandleMicrosoft__pragma(Token &Tok);
2043 
2044  /// Add a lexer to the top of the include stack and
2045  /// start lexing tokens from it instead of the current buffer.
2046  void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
2047 
2048  /// Set the FileID for the preprocessor predefines.
2049  void setPredefinesFileID(FileID FID) {
2050  assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2051  PredefinesFileID = FID;
2052  }
2053 
2054  /// Set the FileID for the PCH through header.
2055  void setPCHThroughHeaderFileID(FileID FID);
2056 
2057  /// Returns true if we are lexing from a file and not a
2058  /// pragma or a macro.
2059  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2060  return L ? !L->isPragmaLexer() : P != nullptr;
2061  }
2062 
2063  static bool IsFileLexer(const IncludeStackInfo& I) {
2064  return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2065  }
2066 
2067  bool IsFileLexer() const {
2068  return IsFileLexer(CurLexer.get(), CurPPLexer);
2069  }
2070 
2071  //===--------------------------------------------------------------------===//
2072  // Caching stuff.
2073  void CachingLex(Token &Result);
2074 
2075  bool InCachingLexMode() const {
2076  // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2077  // that we are past EOF, not that we are in CachingLex mode.
2078  return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2079  }
2080 
2081  void EnterCachingLexMode();
2082 
2083  void ExitCachingLexMode() {
2084  if (InCachingLexMode())
2085  RemoveTopOfLexerStack();
2086  }
2087 
2088  const Token &PeekAhead(unsigned N);
2089  void AnnotatePreviousCachedTokens(const Token &Tok);
2090 
2091  //===--------------------------------------------------------------------===//
2092  /// Handle*Directive - implement the various preprocessor directives. These
2093  /// should side-effect the current preprocessor object so that the next call
2094  /// to Lex() will return the appropriate token next.
2095  void HandleLineDirective();
2096  void HandleDigitDirective(Token &Tok);
2097  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2098  void HandleIdentSCCSDirective(Token &Tok);
2099  void HandleMacroPublicDirective(Token &Tok);
2100  void HandleMacroPrivateDirective();
2101 
2102  // File inclusion.
2103  void HandleIncludeDirective(SourceLocation HashLoc,
2104  Token &Tok,
2105  const DirectoryLookup *LookupFrom = nullptr,
2106  const FileEntry *LookupFromFile = nullptr,
2107  bool isImport = false);
2108  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2109  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2110  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2111  void HandleMicrosoftImportDirective(Token &Tok);
2112 
2113 public:
2114  /// Check that the given module is available, producing a diagnostic if not.
2115  /// \return \c true if the check failed (because the module is not available).
2116  /// \c false if the module appears to be usable.
2117  static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2118  const TargetInfo &TargetInfo,
2119  DiagnosticsEngine &Diags, Module *M);
2120 
2121  // Module inclusion testing.
2122  /// Find the module that owns the source or header file that
2123  /// \p Loc points to. If the location is in a file that was included
2124  /// into a module, or is outside any module, returns nullptr.
2125  Module *getModuleForLocation(SourceLocation Loc);
2126 
2127  /// We want to produce a diagnostic at location IncLoc concerning a
2128  /// missing module import.
2129  ///
2130  /// \param IncLoc The location at which the missing import was detected.
2131  /// \param M The desired module.
2132  /// \param MLoc A location within the desired module at which some desired
2133  /// effect occurred (eg, where a desired entity was declared).
2134  ///
2135  /// \return A file that can be #included to import a module containing MLoc.
2136  /// Null if no such file could be determined or if a #include is not
2137  /// appropriate.
2138  const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2139  Module *M,
2140  SourceLocation MLoc);
2141 
2142  bool isRecordingPreamble() const {
2143  return PreambleConditionalStack.isRecording();
2144  }
2145 
2146  bool hasRecordedPreamble() const {
2147  return PreambleConditionalStack.hasRecordedPreamble();
2148  }
2149 
2151  return PreambleConditionalStack.getStack();
2152  }
2153 
2155  PreambleConditionalStack.setStack(s);
2156  }
2157 
2160  PreambleConditionalStack.startReplaying();
2161  PreambleConditionalStack.setStack(s);
2162  PreambleConditionalStack.SkipInfo = SkipInfo;
2163  }
2164 
2166  return PreambleConditionalStack.SkipInfo;
2167  }
2168 
2169 private:
2170  /// After processing predefined file, initialize the conditional stack from
2171  /// the preamble.
2172  void replayPreambleConditionalStack();
2173 
2174  // Macro handling.
2175  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2176  void HandleUndefDirective();
2177 
2178  // Conditional Inclusion.
2179  void HandleIfdefDirective(Token &Result, const Token &HashToken,
2180  bool isIfndef, bool ReadAnyTokensBeforeDirective);
2181  void HandleIfDirective(Token &IfToken, const Token &HashToken,
2182  bool ReadAnyTokensBeforeDirective);
2183  void HandleEndifDirective(Token &EndifToken);
2184  void HandleElseDirective(Token &Result, const Token &HashToken);
2185  void HandleElifDirective(Token &ElifToken, const Token &HashToken);
2186 
2187  // Pragmas.
2188  void HandlePragmaDirective(SourceLocation IntroducerLoc,
2189  PragmaIntroducerKind Introducer);
2190 
2191 public:
2192  void HandlePragmaOnce(Token &OnceTok);
2193  void HandlePragmaMark();
2194  void HandlePragmaPoison();
2195  void HandlePragmaSystemHeader(Token &SysHeaderTok);
2196  void HandlePragmaDependency(Token &DependencyTok);
2197  void HandlePragmaPushMacro(Token &Tok);
2198  void HandlePragmaPopMacro(Token &Tok);
2199  void HandlePragmaIncludeAlias(Token &Tok);
2200  void HandlePragmaModuleBuild(Token &Tok);
2201  void HandlePragmaHdrstop(Token &Tok);
2202  IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2203 
2204  // Return true and store the first token only if any CommentHandler
2205  // has inserted some tokens and getCommentRetentionState() is false.
2206  bool HandleComment(Token &result, SourceRange Comment);
2207 
2208  /// A macro is used, update information about macros that need unused
2209  /// warnings.
2210  void markMacroAsUsed(MacroInfo *MI);
2211 };
2212 
2213 /// Abstract base class that describes a handler that will receive
2214 /// source ranges for each of the comments encountered in the source file.
2216 public:
2217  virtual ~CommentHandler();
2218 
2219  // The handler shall return true if it has pushed any tokens
2220  // to be read using e.g. EnterToken or EnterTokenStream.
2221  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2222 };
2223 
2224 /// Registry of pragma handlers added by plugins
2225 using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2226 
2227 } // namespace clang
2228 
2229 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
Definition: Preprocessor.h:336
A set of visible modules.
Definition: Module.h:588
StringRef Identifier
Definition: Format.cpp:1635
llvm::BumpPtrAllocator & getPreprocessorAllocator()
Definition: Preprocessor.h:829
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:76
SelectorTable & getSelectorTable()
Definition: Preprocessor.h:827
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:67
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:120
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the &#39;spelling&#39; of the Tok token.
MemoryBufferCache & getPCMCache() const
Definition: Preprocessor.h:822
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:29
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:94
Defines the SourceManager interface.
static const Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:20
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
FileManager & getFileManager() const
Definition: Preprocessor.h:820
Defines the clang::Module class, which describes a module in the source code.
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
SourceLocation getPragmaARCCFCodeAuditedLoc() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
StringRef P
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:564
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1294
A directive for an undefined macro.
Definition: MacroInfo.h:429
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool hadModuleLoaderFatalFailure() const
Definition: Preprocessor.h:842
void EnterToken(const Token &Tok)
Enters a token in the token stream to be lexed next.
Manage memory buffers across multiple users.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:100
Builtin::Context & getBuiltinInfo()
Definition: Preprocessor.h:828
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:35
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:117
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Definition: MacroInfo.h:328
tok::TokenKind getKind() const
Definition: Token.h:89
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:90
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
One of these records is kept for each identifier that is lexed.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
Represents a macro directive exported by a module.
Definition: MacroInfo.h:488
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:406
This table allows us to fully hide how we implement multi-keyword caching.
LineState State
Definition: Format.h:2071
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:818
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Definition: Preprocessor.h:852
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:49
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:817
Describes a module or submodule.
Definition: Module.h:64
bool isPragmaLexer() const
isPragmaLexer - Returns true if this Lexer is being used to lex a pragma.
Definition: Lexer.h:183
A directive for setting the module visibility of a macro.
Definition: MacroInfo.h:444
bool isInvalid() const
void SetSuppressIncludeNotFoundError(bool Suppress)
Definition: Preprocessor.h:862
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:107
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
Definition: Preprocessor.h:899
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:823
const FormatToken & Tok
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void LexNonComment(Token &Result)
Lex a token.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:148
bool isMacroDefined(const IdentifierInfo *II)
Definition: Preprocessor.h:921
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we&#39;ve alread...
Definition: Preprocessor.h:929
Defines the Diagnostic-related interfaces.
const TargetInfo * getAuxTargetInfo() const
Definition: Preprocessor.h:819
TokenLexer - This implements a lexer that returns tokens from a macro body or token stream instead of...
Definition: TokenLexer.h:30
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
Definition: Preprocessor.h:872
PragmaIntroducerKind
Describes how the pragma was introduced, e.g., with #pragma, _Pragma, or __pragma.
Definition: Pragma.h:31
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:158
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:1042
void clearCodeCompletionHandler()
Clear out the code completion handler.
StringRef Filename
Definition: Format.cpp:1628
Provides lookups to, and iteration over, IdentiferInfo objects.
unsigned Offset
Definition: Format.cpp:1630
Exposes information about the current target.
Definition: TargetInfo.h:161
const IdentifierTable & getIdentifierTable() const
Definition: Preprocessor.h:826
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Abstract interface for external sources of preprocessor information.
Defines the clang::LangOptions interface.
unsigned getCounterValue() const
SourceLocation End
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:349
int Id
Definition: ASTDiff.cpp:190
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
const AnnotatedLine * Line
bool isDefined() const
Definition: MacroInfo.h:384
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Definition: Preprocessor.h:881
Implements an efficient mapping from strings to IdentifierInfo nodes.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
A class for tracking whether we&#39;re inside a VA_OPT during a traversal of the tokens of a variadic mac...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:123
#define bool
Definition: stdbool.h:31
DefInfo getDefinition()
Traverses the macro directives history and returns the next macro definition directive along with inf...
Definition: MacroInfo.cpp:178
SourceLocation Begin
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:85
llvm::Optional< PreambleSkipInfo > getPreambleSkipInfo() const
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
Definition: Preprocessor.h:983
bool getCommentRetentionState() const
Definition: Preprocessor.h:857
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:98
PPCallbacks * getPPCallbacks() const
Definition: Preprocessor.h:909
const SourceManager & SM
Definition: Format.cpp:1489
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
bool isRecordingPreamble() const
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
SourceManager & getSourceManager() const
Definition: Preprocessor.h:821
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define&#39;d and not #undef&#39;...
Definition: Preprocessor.h:972
const DefInfo findDirectiveAtLoc(SourceLocation L, const SourceManager &SM) const
Find macro definition active in the specified source location.
Definition: MacroInfo.cpp:202
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:835
llvm::cl::opt< std::string > Filter
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:290
Kind
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
void setExternalSource(ExternalPreprocessorSource *Source)
Definition: Preprocessor.h:831
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.h:93
void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:942
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:176
IdentifierTable & getIdentifierTable()
Definition: Preprocessor.h:825
void setPragmasEnabled(bool Enabled)
Definition: Preprocessor.h:859
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:85
bool isPoisoned() const
Return true if this token has been poisoned.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:24
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
Definition: Preprocessor.h:955
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
Definition: Preprocessor.h:889
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
const MacroInfo * getMacroInfo() const
Definition: MacroInfo.h:390
StringRef getName() const
Return the actual identifier string.
bool isMacroDefined(StringRef Id)
Definition: Preprocessor.h:918
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
Dataflow Directional Tag Classes.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion)
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
Definition: DeclSpec.h:1141
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion)
SourceRange getCodeCompletionTokenRange() const
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:354
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:58
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Abstract interface for a module loader.
Definition: ModuleLoader.h:73
unsigned getLength() const
Definition: Token.h:126
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:47
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:214
bool getPragmasEnabled() const
Definition: Preprocessor.h:860
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:123
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
Definition: Preprocessor.h:840
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Definition: Preprocessor.h:812
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:45
Defines the PPCallbacks interface.
Defines the clang::TokenKind enum and support functions.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
SourceLocation getModuleImportLoc(Module *M) const
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:902
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:814
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:361
bool hasRecordedPreamble() const
const std::string & getPredefines() const
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:282
void setPredefines(StringRef P)
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
The translation unit is a complete translation unit.
Definition: LangOptions.h:363
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
MacroMap::const_iterator macro_iterator
Abstract base class that describes a handler that will receive source ranges for each of the comments...
void setCounterValue(unsigned V)
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
Definition: Preprocessor.h:847
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
Definition: Preprocessor.h:866
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, llvm::Optional< PreambleSkipInfo > SkipInfo)
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Definition: Preprocessor.h:987
Callback handler that receives notifications when performing code completion within the preprocessor...
A header that is known to reside within a given module, whether it was included or excluded...
Definition: ModuleMap.h:150
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
Definition: Preprocessor.h:878
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:910
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:124
void setDiagnostics(DiagnosticsEngine &D)
Definition: Preprocessor.h:815