clang  10.0.0svn
Preprocessor.h
Go to the documentation of this file.
1 //===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines the clang::Preprocessor interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15 #define LLVM_CLANG_LEX_PREPROCESSOR_H
16 
17 #include "clang/Basic/Builtins.h"
18 #include "clang/Basic/Diagnostic.h"
20 #include "clang/Basic/LLVM.h"
22 #include "clang/Basic/Module.h"
25 #include "clang/Basic/TokenKinds.h"
26 #include "clang/Lex/Lexer.h"
27 #include "clang/Lex/MacroInfo.h"
28 #include "clang/Lex/ModuleLoader.h"
29 #include "clang/Lex/ModuleMap.h"
30 #include "clang/Lex/PPCallbacks.h"
32 #include "clang/Lex/Token.h"
33 #include "clang/Lex/TokenLexer.h"
34 #include "llvm/ADT/ArrayRef.h"
35 #include "llvm/ADT/DenseMap.h"
36 #include "llvm/ADT/FoldingSet.h"
37 #include "llvm/ADT/FunctionExtras.h"
38 #include "llvm/ADT/None.h"
39 #include "llvm/ADT/Optional.h"
40 #include "llvm/ADT/PointerUnion.h"
41 #include "llvm/ADT/STLExtras.h"
42 #include "llvm/ADT/SmallPtrSet.h"
43 #include "llvm/ADT/SmallVector.h"
44 #include "llvm/ADT/StringRef.h"
45 #include "llvm/ADT/TinyPtrVector.h"
46 #include "llvm/ADT/iterator_range.h"
47 #include "llvm/Support/Allocator.h"
48 #include "llvm/Support/Casting.h"
49 #include "llvm/Support/Registry.h"
50 #include <cassert>
51 #include <cstddef>
52 #include <cstdint>
53 #include <map>
54 #include <memory>
55 #include <string>
56 #include <utility>
57 #include <vector>
58 
59 namespace llvm {
60 
61 template<unsigned InternalLen> class SmallString;
62 
63 } // namespace llvm
64 
65 namespace clang {
66 
67 class CodeCompletionHandler;
68 class CommentHandler;
69 class DirectoryEntry;
70 class DirectoryLookup;
71 class ExternalPreprocessorSource;
72 class FileEntry;
73 class FileManager;
74 class HeaderSearch;
75 class MacroArgs;
76 class PragmaHandler;
77 class PragmaNamespace;
78 class PreprocessingRecord;
79 class PreprocessorLexer;
80 class PreprocessorOptions;
81 class ScratchBuffer;
82 class TargetInfo;
83 
84 /// Stores token information for comparing actual tokens with
85 /// predefined values. Only handles simple tokens and identifiers.
86 class TokenValue {
88  IdentifierInfo *II;
89 
90 public:
91  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
92  assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
93  assert(Kind != tok::identifier &&
94  "Identifiers should be created by TokenValue(IdentifierInfo *)");
95  assert(!tok::isLiteral(Kind) && "Literals are not supported.");
96  assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
97  }
98 
99  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
100 
101  bool operator==(const Token &Tok) const {
102  return Tok.getKind() == Kind &&
103  (!II || II == Tok.getIdentifierInfo());
104  }
105 };
106 
107 /// Context in which macro name is used.
108 enum MacroUse {
109  // other than #define or #undef
110  MU_Other = 0,
111 
112  // macro name specified in #define
114 
115  // macro name specified in #undef
117 };
118 
119 /// Engages in a tight little dance with the lexer to efficiently
120 /// preprocess tokens.
121 ///
122 /// Lexers know only about tokens within a single source file, and don't
123 /// know anything about preprocessor-level issues like the \#include stack,
124 /// token expansion, etc.
128 
129  llvm::unique_function<void(const clang::Token &)> OnToken;
130  std::shared_ptr<PreprocessorOptions> PPOpts;
131  DiagnosticsEngine *Diags;
132  LangOptions &LangOpts;
133  const TargetInfo *Target = nullptr;
134  const TargetInfo *AuxTarget = nullptr;
135  FileManager &FileMgr;
136  SourceManager &SourceMgr;
137  std::unique_ptr<ScratchBuffer> ScratchBuf;
138  HeaderSearch &HeaderInfo;
139  ModuleLoader &TheModuleLoader;
140 
141  /// External source of macros.
142  ExternalPreprocessorSource *ExternalSource;
143 
144  /// A BumpPtrAllocator object used to quickly allocate and release
145  /// objects internal to the Preprocessor.
146  llvm::BumpPtrAllocator BP;
147 
148  /// Identifiers for builtin macros and other builtins.
149  IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
150  IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
151  IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
152  IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
153  IdentifierInfo *Ident__FILE_NAME__; // __FILE_NAME__
154  IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
155  IdentifierInfo *Ident__COUNTER__; // __COUNTER__
156  IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
157  IdentifierInfo *Ident__identifier; // __identifier
158  IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
159  IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
160  IdentifierInfo *Ident__has_feature; // __has_feature
161  IdentifierInfo *Ident__has_extension; // __has_extension
162  IdentifierInfo *Ident__has_builtin; // __has_builtin
163  IdentifierInfo *Ident__has_attribute; // __has_attribute
164  IdentifierInfo *Ident__has_include; // __has_include
165  IdentifierInfo *Ident__has_include_next; // __has_include_next
166  IdentifierInfo *Ident__has_warning; // __has_warning
167  IdentifierInfo *Ident__is_identifier; // __is_identifier
168  IdentifierInfo *Ident__building_module; // __building_module
169  IdentifierInfo *Ident__MODULE__; // __MODULE__
170  IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
171  IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
172  IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
173  IdentifierInfo *Ident__is_target_arch; // __is_target_arch
174  IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
175  IdentifierInfo *Ident__is_target_os; // __is_target_os
176  IdentifierInfo *Ident__is_target_environment; // __is_target_environment
177 
178  // Weak, only valid (and set) while InMacroArgs is true.
179  Token* ArgMacro;
180 
181  SourceLocation DATELoc, TIMELoc;
182 
183  // Next __COUNTER__ value, starts at 0.
184  unsigned CounterValue = 0;
185 
186  enum {
187  /// Maximum depth of \#includes.
188  MaxAllowedIncludeStackDepth = 200
189  };
190 
191  // State that is set before the preprocessor begins.
192  bool KeepComments : 1;
193  bool KeepMacroComments : 1;
194  bool SuppressIncludeNotFoundError : 1;
195 
196  // State that changes while the preprocessor runs:
197  bool InMacroArgs : 1; // True if parsing fn macro invocation args.
198 
199  /// Whether the preprocessor owns the header search object.
200  bool OwnsHeaderSearch : 1;
201 
202  /// True if macro expansion is disabled.
203  bool DisableMacroExpansion : 1;
204 
205  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
206  /// when parsing preprocessor directives.
207  bool MacroExpansionInDirectivesOverride : 1;
208 
210 
211  /// Whether we have already loaded macros from the external source.
212  mutable bool ReadMacrosFromExternalSource : 1;
213 
214  /// True if pragmas are enabled.
215  bool PragmasEnabled : 1;
216 
217  /// True if the current build action is a preprocessing action.
218  bool PreprocessedOutput : 1;
219 
220  /// True if we are currently preprocessing a #if or #elif directive
221  bool ParsingIfOrElifDirective;
222 
223  /// True if we are pre-expanding macro arguments.
224  bool InMacroArgPreExpansion;
225 
226  /// Mapping/lookup information for all identifiers in
227  /// the program, including program keywords.
228  mutable IdentifierTable Identifiers;
229 
230  /// This table contains all the selectors in the program.
231  ///
232  /// Unlike IdentifierTable above, this table *isn't* populated by the
233  /// preprocessor. It is declared/expanded here because its role/lifetime is
234  /// conceptually similar to the IdentifierTable. In addition, the current
235  /// control flow (in clang::ParseAST()), make it convenient to put here.
236  ///
237  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
238  /// the lifetime of the preprocessor.
239  SelectorTable Selectors;
240 
241  /// Information about builtins.
243 
244  /// Tracks all of the pragmas that the client registered
245  /// with this preprocessor.
246  std::unique_ptr<PragmaNamespace> PragmaHandlers;
247 
248  /// Pragma handlers of the original source is stored here during the
249  /// parsing of a model file.
250  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
251 
252  /// Tracks all of the comment handlers that the client registered
253  /// with this preprocessor.
254  std::vector<CommentHandler *> CommentHandlers;
255 
256  /// True if we want to ignore EOF token and continue later on (thus
257  /// avoid tearing the Lexer and etc. down).
258  bool IncrementalProcessing = false;
259 
260  /// The kind of translation unit we are processing.
261  TranslationUnitKind TUKind;
262 
263  /// The code-completion handler.
264  CodeCompletionHandler *CodeComplete = nullptr;
265 
266  /// The file that we're performing code-completion for, if any.
267  const FileEntry *CodeCompletionFile = nullptr;
268 
269  /// The offset in file for the code-completion point.
270  unsigned CodeCompletionOffset = 0;
271 
272  /// The location for the code-completion point. This gets instantiated
273  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
274  SourceLocation CodeCompletionLoc;
275 
276  /// The start location for the file of the code-completion point.
277  ///
278  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
279  /// for preprocessing.
280  SourceLocation CodeCompletionFileLoc;
281 
282  /// The source location of the \c import contextual keyword we just
283  /// lexed, if any.
284  SourceLocation ModuleImportLoc;
285 
286  /// The module import path that we're currently processing.
288 
289  /// Whether the last token we lexed was an '@'.
290  bool LastTokenWasAt = false;
291 
292  /// A position within a C++20 import-seq.
293  class ImportSeq {
294  public:
295  enum State : int {
296  // Positive values represent a number of unclosed brackets.
297  AtTopLevel = 0,
298  AfterTopLevelTokenSeq = -1,
299  AfterExport = -2,
300  AfterImportSeq = -3,
301  };
302 
303  ImportSeq(State S) : S(S) {}
304 
305  /// Saw any kind of open bracket.
306  void handleOpenBracket() {
307  S = static_cast<State>(std::max<int>(S, 0) + 1);
308  }
309  /// Saw any kind of close bracket other than '}'.
310  void handleCloseBracket() {
311  S = static_cast<State>(std::max<int>(S, 1) - 1);
312  }
313  /// Saw a close brace.
314  void handleCloseBrace() {
315  handleCloseBracket();
316  if (S == AtTopLevel && !AfterHeaderName)
317  S = AfterTopLevelTokenSeq;
318  }
319  /// Saw a semicolon.
320  void handleSemi() {
321  if (atTopLevel()) {
322  S = AfterTopLevelTokenSeq;
323  AfterHeaderName = false;
324  }
325  }
326 
327  /// Saw an 'export' identifier.
328  void handleExport() {
329  if (S == AfterTopLevelTokenSeq)
330  S = AfterExport;
331  else if (S <= 0)
332  S = AtTopLevel;
333  }
334  /// Saw an 'import' identifier.
335  void handleImport() {
336  if (S == AfterTopLevelTokenSeq || S == AfterExport)
337  S = AfterImportSeq;
338  else if (S <= 0)
339  S = AtTopLevel;
340  }
341 
342  /// Saw a 'header-name' token; do not recognize any more 'import' tokens
343  /// until we reach a top-level semicolon.
344  void handleHeaderName() {
345  if (S == AfterImportSeq)
346  AfterHeaderName = true;
347  handleMisc();
348  }
349 
350  /// Saw any other token.
351  void handleMisc() {
352  if (S <= 0)
353  S = AtTopLevel;
354  }
355 
356  bool atTopLevel() { return S <= 0; }
357  bool afterImportSeq() { return S == AfterImportSeq; }
358 
359  private:
360  State S;
361  /// Whether we're in the pp-import-suffix following the header-name in a
362  /// pp-import. If so, a close-brace is not sufficient to end the
363  /// top-level-token-seq of an import-seq.
364  bool AfterHeaderName = false;
365  };
366 
367  /// Our current position within a C++20 import-seq.
368  ImportSeq ImportSeqState = ImportSeq::AfterTopLevelTokenSeq;
369 
370  /// Whether the module import expects an identifier next. Otherwise,
371  /// it expects a '.' or ';'.
372  bool ModuleImportExpectsIdentifier = false;
373 
374  /// The identifier and source location of the currently-active
375  /// \#pragma clang arc_cf_code_audited begin.
376  std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
377 
378  /// The source location of the currently-active
379  /// \#pragma clang assume_nonnull begin.
380  SourceLocation PragmaAssumeNonNullLoc;
381 
382  /// True if we hit the code-completion point.
383  bool CodeCompletionReached = false;
384 
385  /// The code completion token containing the information
386  /// on the stem that is to be code completed.
387  IdentifierInfo *CodeCompletionII = nullptr;
388 
389  /// Range for the code completion token.
390  SourceRange CodeCompletionTokenRange;
391 
392  /// The directory that the main file should be considered to occupy,
393  /// if it does not correspond to a real file (as happens when building a
394  /// module).
395  const DirectoryEntry *MainFileDir = nullptr;
396 
397  /// The number of bytes that we will initially skip when entering the
398  /// main file, along with a flag that indicates whether skipping this number
399  /// of bytes will place the lexer at the start of a line.
400  ///
401  /// This is used when loading a precompiled preamble.
402  std::pair<int, bool> SkipMainFilePreamble;
403 
404  /// Whether we hit an error due to reaching max allowed include depth. Allows
405  /// to avoid hitting the same error over and over again.
406  bool HasReachedMaxIncludeDepth = false;
407 
408  /// The number of currently-active calls to Lex.
409  ///
410  /// Lex is reentrant, and asking for an (end-of-phase-4) token can often
411  /// require asking for multiple additional tokens. This counter makes it
412  /// possible for Lex to detect whether it's producing a token for the end
413  /// of phase 4 of translation or for some other situation.
414  unsigned LexLevel = 0;
415 
416 public:
421  bool FoundElse;
423 
425  bool FoundNonSkipPortion, bool FoundElse,
426  SourceLocation ElseLoc)
427  : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
428  FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
429  ElseLoc(ElseLoc) {}
430  };
431 
432 private:
433  friend class ASTReader;
434  friend class MacroArgs;
435 
436  class PreambleConditionalStackStore {
437  enum State {
438  Off = 0,
439  Recording = 1,
440  Replaying = 2,
441  };
442 
443  public:
444  PreambleConditionalStackStore() = default;
445 
446  void startRecording() { ConditionalStackState = Recording; }
447  void startReplaying() { ConditionalStackState = Replaying; }
448  bool isRecording() const { return ConditionalStackState == Recording; }
449  bool isReplaying() const { return ConditionalStackState == Replaying; }
450 
451  ArrayRef<PPConditionalInfo> getStack() const {
452  return ConditionalStack;
453  }
454 
455  void doneReplaying() {
456  ConditionalStack.clear();
457  ConditionalStackState = Off;
458  }
459 
460  void setStack(ArrayRef<PPConditionalInfo> s) {
461  if (!isRecording() && !isReplaying())
462  return;
463  ConditionalStack.clear();
464  ConditionalStack.append(s.begin(), s.end());
465  }
466 
467  bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
468 
469  bool reachedEOFWhileSkipping() const { return SkipInfo.hasValue(); }
470 
471  void clearSkipInfo() { SkipInfo.reset(); }
472 
474 
475  private:
476  SmallVector<PPConditionalInfo, 4> ConditionalStack;
477  State ConditionalStackState = Off;
478  } PreambleConditionalStack;
479 
480  /// The current top of the stack that we're lexing from if
481  /// not expanding a macro and we are lexing directly from source code.
482  ///
483  /// Only one of CurLexer, or CurTokenLexer will be non-null.
484  std::unique_ptr<Lexer> CurLexer;
485 
486  /// The current top of the stack what we're lexing from
487  /// if not expanding a macro.
488  ///
489  /// This is an alias for CurLexer.
490  PreprocessorLexer *CurPPLexer = nullptr;
491 
492  /// Used to find the current FileEntry, if CurLexer is non-null
493  /// and if applicable.
494  ///
495  /// This allows us to implement \#include_next and find directory-specific
496  /// properties.
497  const DirectoryLookup *CurDirLookup = nullptr;
498 
499  /// The current macro we are expanding, if we are expanding a macro.
500  ///
501  /// One of CurLexer and CurTokenLexer must be null.
502  std::unique_ptr<TokenLexer> CurTokenLexer;
503 
504  /// The kind of lexer we're currently working with.
505  enum CurLexerKind {
506  CLK_Lexer,
507  CLK_TokenLexer,
508  CLK_CachingLexer,
509  CLK_LexAfterModuleImport
510  } CurLexerKind = CLK_Lexer;
511 
512  /// If the current lexer is for a submodule that is being built, this
513  /// is that submodule.
514  Module *CurLexerSubmodule = nullptr;
515 
516  /// Keeps track of the stack of files currently
517  /// \#included, and macros currently being expanded from, not counting
518  /// CurLexer/CurTokenLexer.
519  struct IncludeStackInfo {
520  enum CurLexerKind CurLexerKind;
521  Module *TheSubmodule;
522  std::unique_ptr<Lexer> TheLexer;
523  PreprocessorLexer *ThePPLexer;
524  std::unique_ptr<TokenLexer> TheTokenLexer;
525  const DirectoryLookup *TheDirLookup;
526 
527  // The following constructors are completely useless copies of the default
528  // versions, only needed to pacify MSVC.
529  IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
530  std::unique_ptr<Lexer> &&TheLexer,
531  PreprocessorLexer *ThePPLexer,
532  std::unique_ptr<TokenLexer> &&TheTokenLexer,
533  const DirectoryLookup *TheDirLookup)
534  : CurLexerKind(std::move(CurLexerKind)),
535  TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
536  ThePPLexer(std::move(ThePPLexer)),
537  TheTokenLexer(std::move(TheTokenLexer)),
538  TheDirLookup(std::move(TheDirLookup)) {}
539  };
540  std::vector<IncludeStackInfo> IncludeMacroStack;
541 
542  /// Actions invoked when some preprocessor activity is
543  /// encountered (e.g. a file is \#included, etc).
544  std::unique_ptr<PPCallbacks> Callbacks;
545 
546  struct MacroExpandsInfo {
547  Token Tok;
548  MacroDefinition MD;
549  SourceRange Range;
550 
551  MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
552  : Tok(Tok), MD(MD), Range(Range) {}
553  };
554  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
555 
556  /// Information about a name that has been used to define a module macro.
557  struct ModuleMacroInfo {
558  /// The most recent macro directive for this identifier.
559  MacroDirective *MD;
560 
561  /// The active module macros for this identifier.
562  llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
563 
564  /// The generation number at which we last updated ActiveModuleMacros.
565  /// \see Preprocessor::VisibleModules.
566  unsigned ActiveModuleMacrosGeneration = 0;
567 
568  /// Whether this macro name is ambiguous.
569  bool IsAmbiguous = false;
570 
571  /// The module macros that are overridden by this macro.
572  llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
573 
574  ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
575  };
576 
577  /// The state of a macro for an identifier.
578  class MacroState {
579  mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
580 
581  ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
582  const IdentifierInfo *II) const {
583  if (II->isOutOfDate())
584  PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
585  // FIXME: Find a spare bit on IdentifierInfo and store a
586  // HasModuleMacros flag.
587  if (!II->hasMacroDefinition() ||
588  (!PP.getLangOpts().Modules &&
589  !PP.getLangOpts().ModulesLocalVisibility) ||
590  !PP.CurSubmoduleState->VisibleModules.getGeneration())
591  return nullptr;
592 
593  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
594  if (!Info) {
595  Info = new (PP.getPreprocessorAllocator())
596  ModuleMacroInfo(State.get<MacroDirective *>());
597  State = Info;
598  }
599 
600  if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
601  Info->ActiveModuleMacrosGeneration)
602  PP.updateModuleMacroInfo(II, *Info);
603  return Info;
604  }
605 
606  public:
607  MacroState() : MacroState(nullptr) {}
608  MacroState(MacroDirective *MD) : State(MD) {}
609 
610  MacroState(MacroState &&O) noexcept : State(O.State) {
611  O.State = (MacroDirective *)nullptr;
612  }
613 
614  MacroState &operator=(MacroState &&O) noexcept {
615  auto S = O.State;
616  O.State = (MacroDirective *)nullptr;
617  State = S;
618  return *this;
619  }
620 
621  ~MacroState() {
622  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
623  Info->~ModuleMacroInfo();
624  }
625 
626  MacroDirective *getLatest() const {
627  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
628  return Info->MD;
629  return State.get<MacroDirective*>();
630  }
631 
632  void setLatest(MacroDirective *MD) {
633  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
634  Info->MD = MD;
635  else
636  State = MD;
637  }
638 
639  bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
640  auto *Info = getModuleInfo(PP, II);
641  return Info ? Info->IsAmbiguous : false;
642  }
643 
645  getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
646  if (auto *Info = getModuleInfo(PP, II))
647  return Info->ActiveModuleMacros;
648  return None;
649  }
650 
651  MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
652  SourceManager &SourceMgr) const {
653  // FIXME: Incorporate module macros into the result of this.
654  if (auto *Latest = getLatest())
655  return Latest->findDirectiveAtLoc(Loc, SourceMgr);
656  return {};
657  }
658 
659  void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
660  if (auto *Info = getModuleInfo(PP, II)) {
661  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
662  Info->ActiveModuleMacros.begin(),
663  Info->ActiveModuleMacros.end());
664  Info->ActiveModuleMacros.clear();
665  Info->IsAmbiguous = false;
666  }
667  }
668 
669  ArrayRef<ModuleMacro*> getOverriddenMacros() const {
670  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
671  return Info->OverriddenMacros;
672  return None;
673  }
674 
675  void setOverriddenMacros(Preprocessor &PP,
676  ArrayRef<ModuleMacro *> Overrides) {
677  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
678  if (!Info) {
679  if (Overrides.empty())
680  return;
681  Info = new (PP.getPreprocessorAllocator())
682  ModuleMacroInfo(State.get<MacroDirective *>());
683  State = Info;
684  }
685  Info->OverriddenMacros.clear();
686  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
687  Overrides.begin(), Overrides.end());
688  Info->ActiveModuleMacrosGeneration = 0;
689  }
690  };
691 
692  /// For each IdentifierInfo that was associated with a macro, we
693  /// keep a mapping to the history of all macro definitions and #undefs in
694  /// the reverse order (the latest one is in the head of the list).
695  ///
696  /// This mapping lives within the \p CurSubmoduleState.
697  using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
698 
699  struct SubmoduleState;
700 
701  /// Information about a submodule that we're currently building.
702  struct BuildingSubmoduleInfo {
703  /// The module that we are building.
704  Module *M;
705 
706  /// The location at which the module was included.
707  SourceLocation ImportLoc;
708 
709  /// Whether we entered this submodule via a pragma.
710  bool IsPragma;
711 
712  /// The previous SubmoduleState.
713  SubmoduleState *OuterSubmoduleState;
714 
715  /// The number of pending module macro names when we started building this.
716  unsigned OuterPendingModuleMacroNames;
717 
718  BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
719  SubmoduleState *OuterSubmoduleState,
720  unsigned OuterPendingModuleMacroNames)
721  : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
722  OuterSubmoduleState(OuterSubmoduleState),
723  OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
724  };
725  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
726 
727  /// Information about a submodule's preprocessor state.
728  struct SubmoduleState {
729  /// The macros for the submodule.
730  MacroMap Macros;
731 
732  /// The set of modules that are visible within the submodule.
733  VisibleModuleSet VisibleModules;
734 
735  // FIXME: CounterValue?
736  // FIXME: PragmaPushMacroInfo?
737  };
738  std::map<Module *, SubmoduleState> Submodules;
739 
740  /// The preprocessor state for preprocessing outside of any submodule.
741  SubmoduleState NullSubmoduleState;
742 
743  /// The current submodule state. Will be \p NullSubmoduleState if we're not
744  /// in a submodule.
745  SubmoduleState *CurSubmoduleState;
746 
747  /// The set of known macros exported from modules.
748  llvm::FoldingSet<ModuleMacro> ModuleMacros;
749 
750  /// The names of potential module macros that we've not yet processed.
751  llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
752 
753  /// The list of module macros, for each identifier, that are not overridden by
754  /// any other module macro.
755  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
756  LeafModuleMacros;
757 
758  /// Macros that we want to warn because they are not used at the end
759  /// of the translation unit.
760  ///
761  /// We store just their SourceLocations instead of
762  /// something like MacroInfo*. The benefit of this is that when we are
763  /// deserializing from PCH, we don't need to deserialize identifier & macros
764  /// just so that we can report that they are unused, we just warn using
765  /// the SourceLocations of this set (that will be filled by the ASTReader).
766  /// We are using SmallPtrSet instead of a vector for faster removal.
767  using WarnUnusedMacroLocsTy = llvm::SmallPtrSet<SourceLocation, 32>;
768  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
769 
770  /// A "freelist" of MacroArg objects that can be
771  /// reused for quick allocation.
772  MacroArgs *MacroArgCache = nullptr;
773 
774  /// For each IdentifierInfo used in a \#pragma push_macro directive,
775  /// we keep a MacroInfo stack used to restore the previous macro value.
776  llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
777  PragmaPushMacroInfo;
778 
779  // Various statistics we track for performance analysis.
780  unsigned NumDirectives = 0;
781  unsigned NumDefined = 0;
782  unsigned NumUndefined = 0;
783  unsigned NumPragma = 0;
784  unsigned NumIf = 0;
785  unsigned NumElse = 0;
786  unsigned NumEndif = 0;
787  unsigned NumEnteredSourceFiles = 0;
788  unsigned MaxIncludeStackDepth = 0;
789  unsigned NumMacroExpanded = 0;
790  unsigned NumFnMacroExpanded = 0;
791  unsigned NumBuiltinMacroExpanded = 0;
792  unsigned NumFastMacroExpanded = 0;
793  unsigned NumTokenPaste = 0;
794  unsigned NumFastTokenPaste = 0;
795  unsigned NumSkipped = 0;
796 
797  /// The predefined macros that preprocessor should use from the
798  /// command line etc.
799  std::string Predefines;
800 
801  /// The file ID for the preprocessor predefines.
802  FileID PredefinesFileID;
803 
804  /// The file ID for the PCH through header.
805  FileID PCHThroughHeaderFileID;
806 
807  /// Whether tokens are being skipped until a #pragma hdrstop is seen.
808  bool SkippingUntilPragmaHdrStop = false;
809 
810  /// Whether tokens are being skipped until the through header is seen.
811  bool SkippingUntilPCHThroughHeader = false;
812 
813  /// \{
814  /// Cache of macro expanders to reduce malloc traffic.
815  enum { TokenLexerCacheSize = 8 };
816  unsigned NumCachedTokenLexers;
817  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
818  /// \}
819 
820  /// Keeps macro expanded tokens for TokenLexers.
821  //
822  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
823  /// going to lex in the cache and when it finishes the tokens are removed
824  /// from the end of the cache.
825  SmallVector<Token, 16> MacroExpandedTokens;
826  std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
827 
828  /// A record of the macro definitions and expansions that
829  /// occurred during preprocessing.
830  ///
831  /// This is an optional side structure that can be enabled with
832  /// \c createPreprocessingRecord() prior to preprocessing.
833  PreprocessingRecord *Record = nullptr;
834 
835  /// Cached tokens state.
837 
838  /// Cached tokens are stored here when we do backtracking or
839  /// lookahead. They are "lexed" by the CachingLex() method.
841 
842  /// The position of the cached token that CachingLex() should
843  /// "lex" next.
844  ///
845  /// If it points beyond the CachedTokens vector, it means that a normal
846  /// Lex() should be invoked.
847  CachedTokensTy::size_type CachedLexPos = 0;
848 
849  /// Stack of backtrack positions, allowing nested backtracks.
850  ///
851  /// The EnableBacktrackAtThisPos() method pushes a position to
852  /// indicate where CachedLexPos should be set when the BackTrack() method is
853  /// invoked (at which point the last position is popped).
854  std::vector<CachedTokensTy::size_type> BacktrackPositions;
855 
856  struct MacroInfoChain {
857  MacroInfo MI;
858  MacroInfoChain *Next;
859  };
860 
861  /// MacroInfos are managed as a chain for easy disposal. This is the head
862  /// of that list.
863  MacroInfoChain *MIChainHead = nullptr;
864 
865  void updateOutOfDateIdentifier(IdentifierInfo &II) const;
866 
867 public:
868  Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
870  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
871  IdentifierInfoLookup *IILookup = nullptr,
872  bool OwnsHeaderSearch = false,
874 
875  ~Preprocessor();
876 
877  /// Initialize the preprocessor using information about the target.
878  ///
879  /// \param Target is owned by the caller and must remain valid for the
880  /// lifetime of the preprocessor.
881  /// \param AuxTarget is owned by the caller and must remain valid for
882  /// the lifetime of the preprocessor.
883  void Initialize(const TargetInfo &Target,
884  const TargetInfo *AuxTarget = nullptr);
885 
886  /// Initialize the preprocessor to parse a model file
887  ///
888  /// To parse model files the preprocessor of the original source is reused to
889  /// preserver the identifier table. However to avoid some duplicate
890  /// information in the preprocessor some cleanup is needed before it is used
891  /// to parse model files. This method does that cleanup.
892  void InitializeForModelFile();
893 
894  /// Cleanup after model file parsing
895  void FinalizeForModelFile();
896 
897  /// Retrieve the preprocessor options used to initialize this
898  /// preprocessor.
899  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
900 
901  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
902  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
903 
904  const LangOptions &getLangOpts() const { return LangOpts; }
905  const TargetInfo &getTargetInfo() const { return *Target; }
906  const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
907  FileManager &getFileManager() const { return FileMgr; }
908  SourceManager &getSourceManager() const { return SourceMgr; }
909  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
910 
911  IdentifierTable &getIdentifierTable() { return Identifiers; }
912  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
913  SelectorTable &getSelectorTable() { return Selectors; }
915  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
916 
918  ExternalSource = Source;
919  }
920 
922  return ExternalSource;
923  }
924 
925  /// Retrieve the module loader associated with this preprocessor.
926  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
927 
929  return TheModuleLoader.HadFatalFailure;
930  }
931 
932  /// True if we are currently preprocessing a #if or #elif directive
934  return ParsingIfOrElifDirective;
935  }
936 
937  /// Control whether the preprocessor retains comments in output.
938  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
939  this->KeepComments = KeepComments | KeepMacroComments;
940  this->KeepMacroComments = KeepMacroComments;
941  }
942 
943  bool getCommentRetentionState() const { return KeepComments; }
944 
945  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
946  bool getPragmasEnabled() const { return PragmasEnabled; }
947 
948  void SetSuppressIncludeNotFoundError(bool Suppress) {
949  SuppressIncludeNotFoundError = Suppress;
950  }
951 
953  return SuppressIncludeNotFoundError;
954  }
955 
956  /// Sets whether the preprocessor is responsible for producing output or if
957  /// it is producing tokens to be consumed by Parse and Sema.
958  void setPreprocessedOutput(bool IsPreprocessedOutput) {
959  PreprocessedOutput = IsPreprocessedOutput;
960  }
961 
962  /// Returns true if the preprocessor is responsible for generating output,
963  /// false if it is producing tokens to be consumed by Parse and Sema.
964  bool isPreprocessedOutput() const { return PreprocessedOutput; }
965 
966  /// Return true if we are lexing directly from the specified lexer.
967  bool isCurrentLexer(const PreprocessorLexer *L) const {
968  return CurPPLexer == L;
969  }
970 
971  /// Return the current lexer being lexed from.
972  ///
973  /// Note that this ignores any potentially active macro expansions and _Pragma
974  /// expansions going on at the time.
975  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
976 
977  /// Return the current file lexer being lexed from.
978  ///
979  /// Note that this ignores any potentially active macro expansions and _Pragma
980  /// expansions going on at the time.
981  PreprocessorLexer *getCurrentFileLexer() const;
982 
983  /// Return the submodule owning the file being lexed. This may not be
984  /// the current module if we have changed modules since entering the file.
985  Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
986 
987  /// Returns the FileID for the preprocessor predefines.
988  FileID getPredefinesFileID() const { return PredefinesFileID; }
989 
990  /// \{
991  /// Accessors for preprocessor callbacks.
992  ///
993  /// Note that this class takes ownership of any PPCallbacks object given to
994  /// it.
995  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
996  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
997  if (Callbacks)
998  C = std::make_unique<PPChainedCallbacks>(std::move(C),
999  std::move(Callbacks));
1000  Callbacks = std::move(C);
1001  }
1002  /// \}
1003 
1004  /// Register a function that would be called on each token in the final
1005  /// expanded token stream.
1006  /// This also reports annotation tokens produced by the parser.
1007  void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
1008  OnToken = std::move(F);
1009  }
1010 
1011  bool isMacroDefined(StringRef Id) {
1012  return isMacroDefined(&Identifiers.get(Id));
1013  }
1014  bool isMacroDefined(const IdentifierInfo *II) {
1015  return II->hasMacroDefinition() &&
1016  (!getLangOpts().Modules || (bool)getMacroDefinition(II));
1017  }
1018 
1019  /// Determine whether II is defined as a macro within the module M,
1020  /// if that is a module that we've already preprocessed. Does not check for
1021  /// macros imported into M.
1023  if (!II->hasMacroDefinition())
1024  return false;
1025  auto I = Submodules.find(M);
1026  if (I == Submodules.end())
1027  return false;
1028  auto J = I->second.Macros.find(II);
1029  if (J == I->second.Macros.end())
1030  return false;
1031  auto *MD = J->second.getLatest();
1032  return MD && MD->isDefined();
1033  }
1034 
1036  if (!II->hasMacroDefinition())
1037  return {};
1038 
1039  MacroState &S = CurSubmoduleState->Macros[II];
1040  auto *MD = S.getLatest();
1041  while (MD && isa<VisibilityMacroDirective>(MD))
1042  MD = MD->getPrevious();
1043  return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
1044  S.getActiveModuleMacros(*this, II),
1045  S.isAmbiguous(*this, II));
1046  }
1047 
1049  SourceLocation Loc) {
1050  if (!II->hadMacroDefinition())
1051  return {};
1052 
1053  MacroState &S = CurSubmoduleState->Macros[II];
1055  if (auto *MD = S.getLatest())
1056  DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
1057  // FIXME: Compute the set of active module macros at the specified location.
1058  return MacroDefinition(DI.getDirective(),
1059  S.getActiveModuleMacros(*this, II),
1060  S.isAmbiguous(*this, II));
1061  }
1062 
1063  /// Given an identifier, return its latest non-imported MacroDirective
1064  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
1066  if (!II->hasMacroDefinition())
1067  return nullptr;
1068 
1069  auto *MD = getLocalMacroDirectiveHistory(II);
1070  if (!MD || MD->getDefinition().isUndefined())
1071  return nullptr;
1072 
1073  return MD;
1074  }
1075 
1076  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
1077  return const_cast<Preprocessor*>(this)->getMacroInfo(II);
1078  }
1079 
1081  if (!II->hasMacroDefinition())
1082  return nullptr;
1083  if (auto MD = getMacroDefinition(II))
1084  return MD.getMacroInfo();
1085  return nullptr;
1086  }
1087 
1088  /// Given an identifier, return the latest non-imported macro
1089  /// directive for that identifier.
1090  ///
1091  /// One can iterate over all previous macro directives from the most recent
1092  /// one.
1093  MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const;
1094 
1095  /// Add a directive to the macro directive history for this identifier.
1096  void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
1098  SourceLocation Loc) {
1099  DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1100  appendMacroDirective(II, MD);
1101  return MD;
1102  }
1104  MacroInfo *MI) {
1105  return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1106  }
1107 
1108  /// Set a MacroDirective that was loaded from a PCH file.
1109  void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED,
1110  MacroDirective *MD);
1111 
1112  /// Register an exported macro for a module and identifier.
1113  ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro,
1114  ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1115  ModuleMacro *getModuleMacro(Module *Mod, IdentifierInfo *II);
1116 
1117  /// Get the list of leaf (non-overridden) module macros for a name.
1119  if (II->isOutOfDate())
1120  updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1121  auto I = LeafModuleMacros.find(II);
1122  if (I != LeafModuleMacros.end())
1123  return I->second;
1124  return None;
1125  }
1126 
1127  /// \{
1128  /// Iterators for the macro history table. Currently defined macros have
1129  /// IdentifierInfo::hasMacroDefinition() set and an empty
1130  /// MacroInfo::getUndefLoc() at the head of the list.
1131  using macro_iterator = MacroMap::const_iterator;
1132 
1133  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1134  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1135 
1136  llvm::iterator_range<macro_iterator>
1137  macros(bool IncludeExternalMacros = true) const {
1138  macro_iterator begin = macro_begin(IncludeExternalMacros);
1139  macro_iterator end = macro_end(IncludeExternalMacros);
1140  return llvm::make_range(begin, end);
1141  }
1142 
1143  /// \}
1144 
1145  /// Return the name of the macro defined before \p Loc that has
1146  /// spelling \p Tokens. If there are multiple macros with same spelling,
1147  /// return the last one defined.
1148  StringRef getLastMacroWithSpelling(SourceLocation Loc,
1149  ArrayRef<TokenValue> Tokens) const;
1150 
1151  const std::string &getPredefines() const { return Predefines; }
1152 
1153  /// Set the predefines for this Preprocessor.
1154  ///
1155  /// These predefines are automatically injected when parsing the main file.
1156  void setPredefines(const char *P) { Predefines = P; }
1157  void setPredefines(StringRef P) { Predefines = P; }
1158 
1159  /// Return information about the specified preprocessor
1160  /// identifier token.
1161  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1162  return &Identifiers.get(Name);
1163  }
1164 
1165  /// Add the specified pragma handler to this preprocessor.
1166  ///
1167  /// If \p Namespace is non-null, then it is a token required to exist on the
1168  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1169  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1171  AddPragmaHandler(StringRef(), Handler);
1172  }
1173 
1174  /// Remove the specific pragma handler from this preprocessor.
1175  ///
1176  /// If \p Namespace is non-null, then it should be the namespace that
1177  /// \p Handler was added to. It is an error to remove a handler that
1178  /// has not been registered.
1179  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1181  RemovePragmaHandler(StringRef(), Handler);
1182  }
1183 
1184  /// Install empty handlers for all pragmas (making them ignored).
1185  void IgnorePragmas();
1186 
1187  /// Add the specified comment handler to the preprocessor.
1188  void addCommentHandler(CommentHandler *Handler);
1189 
1190  /// Remove the specified comment handler.
1191  ///
1192  /// It is an error to remove a handler that has not been registered.
1193  void removeCommentHandler(CommentHandler *Handler);
1194 
1195  /// Set the code completion handler to the given object.
1197  CodeComplete = &Handler;
1198  }
1199 
1200  /// Retrieve the current code-completion handler.
1202  return CodeComplete;
1203  }
1204 
1205  /// Clear out the code completion handler.
1207  CodeComplete = nullptr;
1208  }
1209 
1210  /// Hook used by the lexer to invoke the "included file" code
1211  /// completion point.
1212  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1213 
1214  /// Hook used by the lexer to invoke the "natural language" code
1215  /// completion point.
1216  void CodeCompleteNaturalLanguage();
1217 
1218  /// Set the code completion token for filtering purposes.
1220  CodeCompletionII = Filter;
1221  }
1222 
1223  /// Set the code completion token range for detecting replacement range later
1224  /// on.
1226  const SourceLocation End) {
1227  CodeCompletionTokenRange = {Start, End};
1228  }
1230  return CodeCompletionTokenRange;
1231  }
1232 
1233  /// Get the code completion token for filtering purposes.
1235  if (CodeCompletionII)
1236  return CodeCompletionII->getName();
1237  return {};
1238  }
1239 
1240  /// Retrieve the preprocessing record, or NULL if there is no
1241  /// preprocessing record.
1242  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1243 
1244  /// Create a new preprocessing record, which will keep track of
1245  /// all macro expansions, macro definitions, etc.
1246  void createPreprocessingRecord();
1247 
1248  /// Returns true if the FileEntry is the PCH through header.
1249  bool isPCHThroughHeader(const FileEntry *FE);
1250 
1251  /// True if creating a PCH with a through header.
1252  bool creatingPCHWithThroughHeader();
1253 
1254  /// True if using a PCH with a through header.
1255  bool usingPCHWithThroughHeader();
1256 
1257  /// True if creating a PCH with a #pragma hdrstop.
1258  bool creatingPCHWithPragmaHdrStop();
1259 
1260  /// True if using a PCH with a #pragma hdrstop.
1261  bool usingPCHWithPragmaHdrStop();
1262 
1263  /// Skip tokens until after the #include of the through header or
1264  /// until after a #pragma hdrstop.
1265  void SkipTokensWhileUsingPCH();
1266 
1267  /// Process directives while skipping until the through header or
1268  /// #pragma hdrstop is found.
1269  void HandleSkippedDirectiveWhileUsingPCH(Token &Result,
1270  SourceLocation HashLoc);
1271 
1272  /// Enter the specified FileID as the main source file,
1273  /// which implicitly adds the builtin defines etc.
1274  void EnterMainSourceFile();
1275 
1276  /// Inform the preprocessor callbacks that processing is complete.
1277  void EndSourceFile();
1278 
1279  /// Add a source file to the top of the include stack and
1280  /// start lexing tokens from it instead of the current buffer.
1281  ///
1282  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1283  bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir,
1284  SourceLocation Loc);
1285 
1286  /// Add a Macro to the top of the include stack and start lexing
1287  /// tokens from it instead of the current buffer.
1288  ///
1289  /// \param Args specifies the tokens input to a function-like macro.
1290  /// \param ILEnd specifies the location of the ')' for a function-like macro
1291  /// or the identifier for an object-like macro.
1292  void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1293  MacroArgs *Args);
1294 
1295 private:
1296  /// Add a "macro" context to the top of the include stack,
1297  /// which will cause the lexer to start returning the specified tokens.
1298  ///
1299  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1300  /// will not be subject to further macro expansion. Otherwise, these tokens
1301  /// will be re-macro-expanded when/if expansion is enabled.
1302  ///
1303  /// If \p OwnsTokens is false, this method assumes that the specified stream
1304  /// of tokens has a permanent owner somewhere, so they do not need to be
1305  /// copied. If it is true, it assumes the array of tokens is allocated with
1306  /// \c new[] and the Preprocessor will delete[] it.
1307  ///
1308  /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
1309  /// set, see the flag documentation for details.
1310  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1311  bool DisableMacroExpansion, bool OwnsTokens,
1312  bool IsReinject);
1313 
1314 public:
1315  void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1316  bool DisableMacroExpansion, bool IsReinject) {
1317  EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true,
1318  IsReinject);
1319  }
1320 
1321  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
1322  bool IsReinject) {
1323  EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false,
1324  IsReinject);
1325  }
1326 
1327  /// Pop the current lexer/macro exp off the top of the lexer stack.
1328  ///
1329  /// This should only be used in situations where the current state of the
1330  /// top-of-stack lexer is known.
1331  void RemoveTopOfLexerStack();
1332 
1333  /// From the point that this method is called, and until
1334  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1335  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1336  /// make the Preprocessor re-lex the same tokens.
1337  ///
1338  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1339  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1340  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1341  ///
1342  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1343  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1344  /// tokens will continue indefinitely.
1345  ///
1346  void EnableBacktrackAtThisPos();
1347 
1348  /// Disable the last EnableBacktrackAtThisPos call.
1349  void CommitBacktrackedTokens();
1350 
1351  /// Make Preprocessor re-lex the tokens that were lexed since
1352  /// EnableBacktrackAtThisPos() was previously called.
1353  void Backtrack();
1354 
1355  /// True if EnableBacktrackAtThisPos() was called and
1356  /// caching of tokens is on.
1357  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1358 
1359  /// Lex the next token for this preprocessor.
1360  void Lex(Token &Result);
1361 
1362  /// Lex a token, forming a header-name token if possible.
1363  bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
1364 
1365  bool LexAfterModuleImport(Token &Result);
1366  void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks);
1367 
1368  void makeModuleVisible(Module *M, SourceLocation Loc);
1369 
1371  return CurSubmoduleState->VisibleModules.getImportLoc(M);
1372  }
1373 
1374  /// Lex a string literal, which may be the concatenation of multiple
1375  /// string literals and may even come from macro expansion.
1376  /// \returns true on success, false if a error diagnostic has been generated.
1377  bool LexStringLiteral(Token &Result, std::string &String,
1378  const char *DiagnosticTag, bool AllowMacroExpansion) {
1379  if (AllowMacroExpansion)
1380  Lex(Result);
1381  else
1382  LexUnexpandedToken(Result);
1383  return FinishLexStringLiteral(Result, String, DiagnosticTag,
1384  AllowMacroExpansion);
1385  }
1386 
1387  /// Complete the lexing of a string literal where the first token has
1388  /// already been lexed (see LexStringLiteral).
1389  bool FinishLexStringLiteral(Token &Result, std::string &String,
1390  const char *DiagnosticTag,
1391  bool AllowMacroExpansion);
1392 
1393  /// Lex a token. If it's a comment, keep lexing until we get
1394  /// something not a comment.
1395  ///
1396  /// This is useful in -E -C mode where comments would foul up preprocessor
1397  /// directive handling.
1398  void LexNonComment(Token &Result) {
1399  do
1400  Lex(Result);
1401  while (Result.getKind() == tok::comment);
1402  }
1403 
1404  /// Just like Lex, but disables macro expansion of identifier tokens.
1405  void LexUnexpandedToken(Token &Result) {
1406  // Disable macro expansion.
1407  bool OldVal = DisableMacroExpansion;
1408  DisableMacroExpansion = true;
1409  // Lex the token.
1410  Lex(Result);
1411 
1412  // Reenable it.
1413  DisableMacroExpansion = OldVal;
1414  }
1415 
1416  /// Like LexNonComment, but this disables macro expansion of
1417  /// identifier tokens.
1419  do
1420  LexUnexpandedToken(Result);
1421  while (Result.getKind() == tok::comment);
1422  }
1423 
1424  /// Parses a simple integer literal to get its numeric value. Floating
1425  /// point literals and user defined literals are rejected. Used primarily to
1426  /// handle pragmas that accept integer arguments.
1427  bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1428 
1429  /// Disables macro expansion everywhere except for preprocessor directives.
1431  DisableMacroExpansion = true;
1432  MacroExpansionInDirectivesOverride = true;
1433  }
1434 
1435  /// Peeks ahead N tokens and returns that token without consuming any
1436  /// tokens.
1437  ///
1438  /// LookAhead(0) returns the next token that would be returned by Lex(),
1439  /// LookAhead(1) returns the token after it, etc. This returns normal
1440  /// tokens after phase 5. As such, it is equivalent to using
1441  /// 'Lex', not 'LexUnexpandedToken'.
1442  const Token &LookAhead(unsigned N) {
1443  assert(LexLevel == 0 && "cannot use lookahead while lexing");
1444  if (CachedLexPos + N < CachedTokens.size())
1445  return CachedTokens[CachedLexPos+N];
1446  else
1447  return PeekAhead(N+1);
1448  }
1449 
1450  /// When backtracking is enabled and tokens are cached,
1451  /// this allows to revert a specific number of tokens.
1452  ///
1453  /// Note that the number of tokens being reverted should be up to the last
1454  /// backtrack position, not more.
1455  void RevertCachedTokens(unsigned N) {
1456  assert(isBacktrackEnabled() &&
1457  "Should only be called when tokens are cached for backtracking");
1458  assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1459  && "Should revert tokens up to the last backtrack position, not more");
1460  assert(signed(CachedLexPos) - signed(N) >= 0 &&
1461  "Corrupted backtrack positions ?");
1462  CachedLexPos -= N;
1463  }
1464 
1465  /// Enters a token in the token stream to be lexed next.
1466  ///
1467  /// If BackTrack() is called afterwards, the token will remain at the
1468  /// insertion point.
1469  /// If \p IsReinject is true, resulting token will have Token::IsReinjected
1470  /// flag set. See the flag documentation for details.
1471  void EnterToken(const Token &Tok, bool IsReinject) {
1472  if (LexLevel) {
1473  // It's not correct in general to enter caching lex mode while in the
1474  // middle of a nested lexing action.
1475  auto TokCopy = std::make_unique<Token[]>(1);
1476  TokCopy[0] = Tok;
1477  EnterTokenStream(std::move(TokCopy), 1, true, IsReinject);
1478  } else {
1479  EnterCachingLexMode();
1480  assert(IsReinject && "new tokens in the middle of cached stream");
1481  CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1482  }
1483  }
1484 
1485  /// We notify the Preprocessor that if it is caching tokens (because
1486  /// backtrack is enabled) it should replace the most recent cached tokens
1487  /// with the given annotation token. This function has no effect if
1488  /// backtracking is not enabled.
1489  ///
1490  /// Note that the use of this function is just for optimization, so that the
1491  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1492  /// invoked.
1493  void AnnotateCachedTokens(const Token &Tok) {
1494  assert(Tok.isAnnotation() && "Expected annotation token");
1495  if (CachedLexPos != 0 && isBacktrackEnabled())
1496  AnnotatePreviousCachedTokens(Tok);
1497  }
1498 
1499  /// Get the location of the last cached token, suitable for setting the end
1500  /// location of an annotation token.
1502  assert(CachedLexPos != 0);
1503  return CachedTokens[CachedLexPos-1].getLastLoc();
1504  }
1505 
1506  /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1507  /// CachedTokens.
1508  bool IsPreviousCachedToken(const Token &Tok) const;
1509 
1510  /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1511  /// in \p NewToks.
1512  ///
1513  /// Useful when a token needs to be split in smaller ones and CachedTokens
1514  /// most recent token must to be updated to reflect that.
1515  void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1516 
1517  /// Replace the last token with an annotation token.
1518  ///
1519  /// Like AnnotateCachedTokens(), this routine replaces an
1520  /// already-parsed (and resolved) token with an annotation
1521  /// token. However, this routine only replaces the last token with
1522  /// the annotation token; it does not affect any other cached
1523  /// tokens. This function has no effect if backtracking is not
1524  /// enabled.
1526  assert(Tok.isAnnotation() && "Expected annotation token");
1527  if (CachedLexPos != 0 && isBacktrackEnabled())
1528  CachedTokens[CachedLexPos-1] = Tok;
1529  }
1530 
1531  /// Enter an annotation token into the token stream.
1532  void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1533  void *AnnotationVal);
1534 
1535  /// Update the current token to represent the provided
1536  /// identifier, in order to cache an action performed by typo correction.
1537  void TypoCorrectToken(const Token &Tok) {
1538  assert(Tok.getIdentifierInfo() && "Expected identifier token");
1539  if (CachedLexPos != 0 && isBacktrackEnabled())
1540  CachedTokens[CachedLexPos-1] = Tok;
1541  }
1542 
1543  /// Recompute the current lexer kind based on the CurLexer/
1544  /// CurTokenLexer pointers.
1545  void recomputeCurLexerKind();
1546 
1547  /// Returns true if incremental processing is enabled
1548  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1549 
1550  /// Enables the incremental processing
1551  void enableIncrementalProcessing(bool value = true) {
1552  IncrementalProcessing = value;
1553  }
1554 
1555  /// Specify the point at which code-completion will be performed.
1556  ///
1557  /// \param File the file in which code completion should occur. If
1558  /// this file is included multiple times, code-completion will
1559  /// perform completion the first time it is included. If NULL, this
1560  /// function clears out the code-completion point.
1561  ///
1562  /// \param Line the line at which code completion should occur
1563  /// (1-based).
1564  ///
1565  /// \param Column the column at which code completion should occur
1566  /// (1-based).
1567  ///
1568  /// \returns true if an error occurred, false otherwise.
1569  bool SetCodeCompletionPoint(const FileEntry *File,
1570  unsigned Line, unsigned Column);
1571 
1572  /// Determine if we are performing code completion.
1573  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1574 
1575  /// Returns the location of the code-completion point.
1576  ///
1577  /// Returns an invalid location if code-completion is not enabled or the file
1578  /// containing the code-completion point has not been lexed yet.
1579  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1580 
1581  /// Returns the start location of the file of code-completion point.
1582  ///
1583  /// Returns an invalid location if code-completion is not enabled or the file
1584  /// containing the code-completion point has not been lexed yet.
1586  return CodeCompletionFileLoc;
1587  }
1588 
1589  /// Returns true if code-completion is enabled and we have hit the
1590  /// code-completion point.
1591  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1592 
1593  /// Note that we hit the code-completion point.
1595  assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1596  CodeCompletionReached = true;
1597  // Silence any diagnostics that occur after we hit the code-completion.
1598  getDiagnostics().setSuppressAllDiagnostics(true);
1599  }
1600 
1601  /// The location of the currently-active \#pragma clang
1602  /// arc_cf_code_audited begin.
1603  ///
1604  /// Returns an invalid location if there is no such pragma active.
1605  std::pair<IdentifierInfo *, SourceLocation>
1607  return PragmaARCCFCodeAuditedInfo;
1608  }
1609 
1610  /// Set the location of the currently-active \#pragma clang
1611  /// arc_cf_code_audited begin. An invalid location ends the pragma.
1613  SourceLocation Loc) {
1614  PragmaARCCFCodeAuditedInfo = {Ident, Loc};
1615  }
1616 
1617  /// The location of the currently-active \#pragma clang
1618  /// assume_nonnull begin.
1619  ///
1620  /// Returns an invalid location if there is no such pragma active.
1622  return PragmaAssumeNonNullLoc;
1623  }
1624 
1625  /// Set the location of the currently-active \#pragma clang
1626  /// assume_nonnull begin. An invalid location ends the pragma.
1628  PragmaAssumeNonNullLoc = Loc;
1629  }
1630 
1631  /// Set the directory in which the main file should be considered
1632  /// to have been found, if it is not a real file.
1633  void setMainFileDir(const DirectoryEntry *Dir) {
1634  MainFileDir = Dir;
1635  }
1636 
1637  /// Instruct the preprocessor to skip part of the main source file.
1638  ///
1639  /// \param Bytes The number of bytes in the preamble to skip.
1640  ///
1641  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1642  /// start of a line.
1643  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1644  SkipMainFilePreamble.first = Bytes;
1645  SkipMainFilePreamble.second = StartOfLine;
1646  }
1647 
1648  /// Forwarding function for diagnostics. This emits a diagnostic at
1649  /// the specified Token's location, translating the token's start
1650  /// position in the current buffer into a SourcePosition object for rendering.
1651  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1652  return Diags->Report(Loc, DiagID);
1653  }
1654 
1655  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1656  return Diags->Report(Tok.getLocation(), DiagID);
1657  }
1658 
1659  /// Return the 'spelling' of the token at the given
1660  /// location; does not go up to the spelling location or down to the
1661  /// expansion location.
1662  ///
1663  /// \param buffer A buffer which will be used only if the token requires
1664  /// "cleaning", e.g. if it contains trigraphs or escaped newlines
1665  /// \param invalid If non-null, will be set \c true if an error occurs.
1667  SmallVectorImpl<char> &buffer,
1668  bool *invalid = nullptr) const {
1669  return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1670  }
1671 
1672  /// Return the 'spelling' of the Tok token.
1673  ///
1674  /// The spelling of a token is the characters used to represent the token in
1675  /// the source file after trigraph expansion and escaped-newline folding. In
1676  /// particular, this wants to get the true, uncanonicalized, spelling of
1677  /// things like digraphs, UCNs, etc.
1678  ///
1679  /// \param Invalid If non-null, will be set \c true if an error occurs.
1680  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1681  return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1682  }
1683 
1684  /// Get the spelling of a token into a preallocated buffer, instead
1685  /// of as an std::string.
1686  ///
1687  /// The caller is required to allocate enough space for the token, which is
1688  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1689  /// actual result is returned.
1690  ///
1691  /// Note that this method may do two possible things: it may either fill in
1692  /// the buffer specified with characters, or it may *change the input pointer*
1693  /// to point to a constant buffer with the data already in it (avoiding a
1694  /// copy). The caller is not allowed to modify the returned buffer pointer
1695  /// if an internal buffer is returned.
1696  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1697  bool *Invalid = nullptr) const {
1698  return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1699  }
1700 
1701  /// Get the spelling of a token into a SmallVector.
1702  ///
1703  /// Note that the returned StringRef may not point to the
1704  /// supplied buffer if a copy can be avoided.
1705  StringRef getSpelling(const Token &Tok,
1706  SmallVectorImpl<char> &Buffer,
1707  bool *Invalid = nullptr) const;
1708 
1709  /// Relex the token at the specified location.
1710  /// \returns true if there was a failure, false on success.
1711  bool getRawToken(SourceLocation Loc, Token &Result,
1712  bool IgnoreWhiteSpace = false) {
1713  return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1714  }
1715 
1716  /// Given a Token \p Tok that is a numeric constant with length 1,
1717  /// return the character.
1718  char
1720  bool *Invalid = nullptr) const {
1721  assert(Tok.is(tok::numeric_constant) &&
1722  Tok.getLength() == 1 && "Called on unsupported token");
1723  assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1724 
1725  // If the token is carrying a literal data pointer, just use it.
1726  if (const char *D = Tok.getLiteralData())
1727  return *D;
1728 
1729  // Otherwise, fall back on getCharacterData, which is slower, but always
1730  // works.
1731  return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1732  }
1733 
1734  /// Retrieve the name of the immediate macro expansion.
1735  ///
1736  /// This routine starts from a source location, and finds the name of the
1737  /// macro responsible for its immediate expansion. It looks through any
1738  /// intervening macro argument expansions to compute this. It returns a
1739  /// StringRef that refers to the SourceManager-owned buffer of the source
1740  /// where that macro name is spelled. Thus, the result shouldn't out-live
1741  /// the SourceManager.
1743  return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1744  }
1745 
1746  /// Plop the specified string into a scratch buffer and set the
1747  /// specified token's location and length to it.
1748  ///
1749  /// If specified, the source location provides a location of the expansion
1750  /// point of the token.
1751  void CreateString(StringRef Str, Token &Tok,
1752  SourceLocation ExpansionLocStart = SourceLocation(),
1753  SourceLocation ExpansionLocEnd = SourceLocation());
1754 
1755  /// Split the first Length characters out of the token starting at TokLoc
1756  /// and return a location pointing to the split token. Re-lexing from the
1757  /// split token will return the split token rather than the original.
1758  SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
1759 
1760  /// Computes the source location just past the end of the
1761  /// token at this source location.
1762  ///
1763  /// This routine can be used to produce a source location that
1764  /// points just past the end of the token referenced by \p Loc, and
1765  /// is generally used when a diagnostic needs to point just after a
1766  /// token where it expected something different that it received. If
1767  /// the returned source location would not be meaningful (e.g., if
1768  /// it points into a macro), this routine returns an invalid
1769  /// source location.
1770  ///
1771  /// \param Offset an offset from the end of the token, where the source
1772  /// location should refer to. The default offset (0) produces a source
1773  /// location pointing just past the end of the token; an offset of 1 produces
1774  /// a source location pointing to the last character in the token, etc.
1776  return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1777  }
1778 
1779  /// Returns true if the given MacroID location points at the first
1780  /// token of the macro expansion.
1781  ///
1782  /// \param MacroBegin If non-null and function returns true, it is set to
1783  /// begin location of the macro.
1785  SourceLocation *MacroBegin = nullptr) const {
1786  return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1787  MacroBegin);
1788  }
1789 
1790  /// Returns true if the given MacroID location points at the last
1791  /// token of the macro expansion.
1792  ///
1793  /// \param MacroEnd If non-null and function returns true, it is set to
1794  /// end location of the macro.
1796  SourceLocation *MacroEnd = nullptr) const {
1797  return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1798  }
1799 
1800  /// Print the token to stderr, used for debugging.
1801  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1802  void DumpLocation(SourceLocation Loc) const;
1803  void DumpMacro(const MacroInfo &MI) const;
1804  void dumpMacroInfo(const IdentifierInfo *II);
1805 
1806  /// Given a location that specifies the start of a
1807  /// token, return a new location that specifies a character within the token.
1809  unsigned Char) const {
1810  return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1811  }
1812 
1813  /// Increment the counters for the number of token paste operations
1814  /// performed.
1815  ///
1816  /// If fast was specified, this is a 'fast paste' case we handled.
1817  void IncrementPasteCounter(bool isFast) {
1818  if (isFast)
1819  ++NumFastTokenPaste;
1820  else
1821  ++NumTokenPaste;
1822  }
1823 
1824  void PrintStats();
1825 
1826  size_t getTotalMemory() const;
1827 
1828  /// When the macro expander pastes together a comment (/##/) in Microsoft
1829  /// mode, this method handles updating the current state, returning the
1830  /// token on the next source line.
1831  void HandleMicrosoftCommentPaste(Token &Tok);
1832 
1833  //===--------------------------------------------------------------------===//
1834  // Preprocessor callback methods. These are invoked by a lexer as various
1835  // directives and events are found.
1836 
1837  /// Given a tok::raw_identifier token, look up the
1838  /// identifier information for the token and install it into the token,
1839  /// updating the token kind accordingly.
1840  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1841 
1842 private:
1843  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1844 
1845 public:
1846  /// Specifies the reason for poisoning an identifier.
1847  ///
1848  /// If that identifier is accessed while poisoned, then this reason will be
1849  /// used instead of the default "poisoned" diagnostic.
1850  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1851 
1852  /// Display reason for poisoned identifier.
1853  void HandlePoisonedIdentifier(Token & Identifier);
1854 
1856  if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1857  if(II->isPoisoned()) {
1858  HandlePoisonedIdentifier(Identifier);
1859  }
1860  }
1861  }
1862 
1863 private:
1864  /// Identifiers used for SEH handling in Borland. These are only
1865  /// allowed in particular circumstances
1866  // __except block
1867  IdentifierInfo *Ident__exception_code,
1868  *Ident___exception_code,
1869  *Ident_GetExceptionCode;
1870  // __except filter expression
1871  IdentifierInfo *Ident__exception_info,
1872  *Ident___exception_info,
1873  *Ident_GetExceptionInfo;
1874  // __finally
1875  IdentifierInfo *Ident__abnormal_termination,
1876  *Ident___abnormal_termination,
1877  *Ident_AbnormalTermination;
1878 
1879  const char *getCurLexerEndPos();
1880  void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
1881 
1882 public:
1883  void PoisonSEHIdentifiers(bool Poison = true); // Borland
1884 
1885  /// Callback invoked when the lexer reads an identifier and has
1886  /// filled in the tokens IdentifierInfo member.
1887  ///
1888  /// This callback potentially macro expands it or turns it into a named
1889  /// token (like 'for').
1890  ///
1891  /// \returns true if we actually computed a token, false if we need to
1892  /// lex again.
1893  bool HandleIdentifier(Token &Identifier);
1894 
1895  /// Callback invoked when the lexer hits the end of the current file.
1896  ///
1897  /// This either returns the EOF token and returns true, or
1898  /// pops a level off the include stack and returns false, at which point the
1899  /// client should call lex again.
1900  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1901 
1902  /// Callback invoked when the current TokenLexer hits the end of its
1903  /// token stream.
1904  bool HandleEndOfTokenLexer(Token &Result);
1905 
1906  /// Callback invoked when the lexer sees a # token at the start of a
1907  /// line.
1908  ///
1909  /// This consumes the directive, modifies the lexer/preprocessor state, and
1910  /// advances the lexer(s) so that the next token read is the correct one.
1911  void HandleDirective(Token &Result);
1912 
1913  /// Ensure that the next token is a tok::eod token.
1914  ///
1915  /// If not, emit a diagnostic and consume up until the eod.
1916  /// If \p EnableMacros is true, then we consider macros that expand to zero
1917  /// tokens as being ok.
1918  ///
1919  /// \return The location of the end of the directive (the terminating
1920  /// newline).
1921  SourceLocation CheckEndOfDirective(const char *DirType,
1922  bool EnableMacros = false);
1923 
1924  /// Read and discard all tokens remaining on the current line until
1925  /// the tok::eod token is found. Returns the range of the skipped tokens.
1926  SourceRange DiscardUntilEndOfDirective();
1927 
1928  /// Returns true if the preprocessor has seen a use of
1929  /// __DATE__ or __TIME__ in the file so far.
1930  bool SawDateOrTime() const {
1931  return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
1932  }
1933  unsigned getCounterValue() const { return CounterValue; }
1934  void setCounterValue(unsigned V) { CounterValue = V; }
1935 
1936  /// Retrieves the module that we're currently building, if any.
1937  Module *getCurrentModule();
1938 
1939  /// Allocate a new MacroInfo object with the provided SourceLocation.
1940  MacroInfo *AllocateMacroInfo(SourceLocation L);
1941 
1942  /// Turn the specified lexer token into a fully checked and spelled
1943  /// filename, e.g. as an operand of \#include.
1944  ///
1945  /// The caller is expected to provide a buffer that is large enough to hold
1946  /// the spelling of the filename, but is also expected to handle the case
1947  /// when this method decides to use a different buffer.
1948  ///
1949  /// \returns true if the input filename was in <>'s or false if it was
1950  /// in ""'s.
1951  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
1952 
1953  /// Given a "foo" or <foo> reference, look up the indicated file.
1954  ///
1955  /// Returns None on failure. \p isAngled indicates whether the file
1956  /// reference is for system \#include's or not (i.e. using <> instead of "").
1958  LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled,
1959  const DirectoryLookup *FromDir, const FileEntry *FromFile,
1960  const DirectoryLookup *&CurDir, SmallVectorImpl<char> *SearchPath,
1961  SmallVectorImpl<char> *RelativePath,
1962  ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped,
1963  bool *IsFrameworkFound, bool SkipCache = false);
1964 
1965  /// Get the DirectoryLookup structure used to find the current
1966  /// FileEntry, if CurLexer is non-null and if applicable.
1967  ///
1968  /// This allows us to implement \#include_next and find directory-specific
1969  /// properties.
1970  const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1971 
1972  /// Return true if we're in the top-level file, not in a \#include.
1973  bool isInPrimaryFile() const;
1974 
1975  /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1976  /// followed by EOD. Return true if the token is not a valid on-off-switch.
1977  bool LexOnOffSwitch(tok::OnOffSwitch &Result);
1978 
1979  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
1980  bool *ShadowFlag = nullptr);
1981 
1982  void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
1983  Module *LeaveSubmodule(bool ForPragma);
1984 
1985 private:
1986  friend void TokenLexer::ExpandFunctionArguments();
1987 
1988  void PushIncludeMacroStack() {
1989  assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
1990  IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
1991  std::move(CurLexer), CurPPLexer,
1992  std::move(CurTokenLexer), CurDirLookup);
1993  CurPPLexer = nullptr;
1994  }
1995 
1996  void PopIncludeMacroStack() {
1997  CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1998  CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1999  CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2000  CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2001  CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2002  CurLexerKind = IncludeMacroStack.back().CurLexerKind;
2003  IncludeMacroStack.pop_back();
2004  }
2005 
2006  void PropagateLineStartLeadingSpaceInfo(Token &Result);
2007 
2008  /// Determine whether we need to create module macros for #defines in the
2009  /// current context.
2010  bool needModuleMacros() const;
2011 
2012  /// Update the set of active module macros and ambiguity flag for a module
2013  /// macro name.
2014  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
2015 
2016  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2017  SourceLocation Loc);
2018  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2019  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2020  bool isPublic);
2021 
2022  /// Lex and validate a macro name, which occurs after a
2023  /// \#define or \#undef.
2024  ///
2025  /// \param MacroNameTok Token that represents the name defined or undefined.
2026  /// \param IsDefineUndef Kind if preprocessor directive.
2027  /// \param ShadowFlag Points to flag that is set if macro name shadows
2028  /// a keyword.
2029  ///
2030  /// This emits a diagnostic, sets the token kind to eod,
2031  /// and discards the rest of the macro line if the macro name is invalid.
2032  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
2033  bool *ShadowFlag = nullptr);
2034 
2035  /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
2036  /// entire line) of the macro's tokens and adds them to MacroInfo, and while
2037  /// doing so performs certain validity checks including (but not limited to):
2038  /// - # (stringization) is followed by a macro parameter
2039  /// \param MacroNameTok - Token that represents the macro name
2040  /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
2041  ///
2042  /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
2043  /// returns a nullptr if an invalid sequence of tokens is encountered.
2044  MacroInfo *ReadOptionalMacroParameterListAndBody(
2045  const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
2046 
2047  /// The ( starting an argument list of a macro definition has just been read.
2048  /// Lex the rest of the parameters and the closing ), updating \p MI with
2049  /// what we learn and saving in \p LastTok the last token read.
2050  /// Return true if an error occurs parsing the arg list.
2051  bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2052 
2053  /// We just read a \#if or related directive and decided that the
2054  /// subsequent tokens are in the \#if'd out portion of the
2055  /// file. Lex the rest of the file, until we see an \#endif. If \p
2056  /// FoundNonSkipPortion is true, then we have already emitted code for part of
2057  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
2058  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
2059  /// already seen one so a \#else directive is a duplicate. When this returns,
2060  /// the caller can lex the first valid token.
2061  void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2062  SourceLocation IfTokenLoc,
2063  bool FoundNonSkipPortion, bool FoundElse,
2064  SourceLocation ElseLoc = SourceLocation());
2065 
2066  /// Information about the result for evaluating an expression for a
2067  /// preprocessor directive.
2068  struct DirectiveEvalResult {
2069  /// Whether the expression was evaluated as true or not.
2070  bool Conditional;
2071 
2072  /// True if the expression contained identifiers that were undefined.
2073  bool IncludedUndefinedIds;
2074 
2075  /// The source range for the expression.
2076  SourceRange ExprRange;
2077  };
2078 
2079  /// Evaluate an integer constant expression that may occur after a
2080  /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2081  ///
2082  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2083  DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
2084 
2085  /// Install the standard preprocessor pragmas:
2086  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
2087  void RegisterBuiltinPragmas();
2088 
2089  /// Register builtin macros such as __LINE__ with the identifier table.
2090  void RegisterBuiltinMacros();
2091 
2092  /// If an identifier token is read that is to be expanded as a macro, handle
2093  /// it and return the next token as 'Tok'. If we lexed a token, return true;
2094  /// otherwise the caller should lex again.
2095  bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2096 
2097  /// Cache macro expanded tokens for TokenLexers.
2098  //
2099  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2100  /// going to lex in the cache and when it finishes the tokens are removed
2101  /// from the end of the cache.
2102  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2103  ArrayRef<Token> tokens);
2104 
2105  void removeCachedMacroExpandedTokensOfLastLexer();
2106 
2107  /// Determine whether the next preprocessor token to be
2108  /// lexed is a '('. If so, consume the token and return true, if not, this
2109  /// method should have no observable side-effect on the lexed tokens.
2110  bool isNextPPTokenLParen();
2111 
2112  /// After reading "MACRO(", this method is invoked to read all of the formal
2113  /// arguments specified for the macro invocation. Returns null on error.
2114  MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2115  SourceLocation &MacroEnd);
2116 
2117  /// If an identifier token is read that is to be expanded
2118  /// as a builtin macro, handle it and return the next token as 'Tok'.
2119  void ExpandBuiltinMacro(Token &Tok);
2120 
2121  /// Read a \c _Pragma directive, slice it up, process it, then
2122  /// return the first token after the directive.
2123  /// This assumes that the \c _Pragma token has just been read into \p Tok.
2124  void Handle_Pragma(Token &Tok);
2125 
2126  /// Like Handle_Pragma except the pragma text is not enclosed within
2127  /// a string literal.
2128  void HandleMicrosoft__pragma(Token &Tok);
2129 
2130  /// Add a lexer to the top of the include stack and
2131  /// start lexing tokens from it instead of the current buffer.
2132  void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
2133 
2134  /// Set the FileID for the preprocessor predefines.
2135  void setPredefinesFileID(FileID FID) {
2136  assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2137  PredefinesFileID = FID;
2138  }
2139 
2140  /// Set the FileID for the PCH through header.
2141  void setPCHThroughHeaderFileID(FileID FID);
2142 
2143  /// Returns true if we are lexing from a file and not a
2144  /// pragma or a macro.
2145  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2146  return L ? !L->isPragmaLexer() : P != nullptr;
2147  }
2148 
2149  static bool IsFileLexer(const IncludeStackInfo& I) {
2150  return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2151  }
2152 
2153  bool IsFileLexer() const {
2154  return IsFileLexer(CurLexer.get(), CurPPLexer);
2155  }
2156 
2157  //===--------------------------------------------------------------------===//
2158  // Caching stuff.
2159  void CachingLex(Token &Result);
2160 
2161  bool InCachingLexMode() const {
2162  // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2163  // that we are past EOF, not that we are in CachingLex mode.
2164  return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2165  }
2166 
2167  void EnterCachingLexMode();
2168  void EnterCachingLexModeUnchecked();
2169 
2170  void ExitCachingLexMode() {
2171  if (InCachingLexMode())
2172  RemoveTopOfLexerStack();
2173  }
2174 
2175  const Token &PeekAhead(unsigned N);
2176  void AnnotatePreviousCachedTokens(const Token &Tok);
2177 
2178  //===--------------------------------------------------------------------===//
2179  /// Handle*Directive - implement the various preprocessor directives. These
2180  /// should side-effect the current preprocessor object so that the next call
2181  /// to Lex() will return the appropriate token next.
2182  void HandleLineDirective();
2183  void HandleDigitDirective(Token &Tok);
2184  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2185  void HandleIdentSCCSDirective(Token &Tok);
2186  void HandleMacroPublicDirective(Token &Tok);
2187  void HandleMacroPrivateDirective();
2188 
2189  /// An additional notification that can be produced by a header inclusion or
2190  /// import to tell the parser what happened.
2191  struct ImportAction {
2192  enum ActionKind {
2193  None,
2194  ModuleBegin,
2195  ModuleImport,
2196  SkippedModuleImport,
2197  } Kind;
2198  Module *ModuleForHeader = nullptr;
2199 
2200  ImportAction(ActionKind AK, Module *Mod = nullptr)
2201  : Kind(AK), ModuleForHeader(Mod) {
2202  assert((AK == None || Mod) && "no module for module action");
2203  }
2204  };
2205 
2206  Optional<FileEntryRef> LookupHeaderIncludeOrImport(
2207  const DirectoryLookup *&CurDir, StringRef Filename,
2208  SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2209  const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
2210  bool &IsMapped, const DirectoryLookup *LookupFrom,
2211  const FileEntry *LookupFromFile, StringRef LookupFilename,
2212  SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2213  ModuleMap::KnownHeader &SuggestedModule, bool isAngled);
2214 
2215  // File inclusion.
2216  void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2217  const DirectoryLookup *LookupFrom = nullptr,
2218  const FileEntry *LookupFromFile = nullptr);
2219  ImportAction
2220  HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2221  Token &FilenameTok, SourceLocation EndLoc,
2222  const DirectoryLookup *LookupFrom = nullptr,
2223  const FileEntry *LookupFromFile = nullptr);
2224  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2225  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2226  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2227  void HandleMicrosoftImportDirective(Token &Tok);
2228 
2229 public:
2230  /// Check that the given module is available, producing a diagnostic if not.
2231  /// \return \c true if the check failed (because the module is not available).
2232  /// \c false if the module appears to be usable.
2233  static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2234  const TargetInfo &TargetInfo,
2235  DiagnosticsEngine &Diags, Module *M);
2236 
2237  // Module inclusion testing.
2238  /// Find the module that owns the source or header file that
2239  /// \p Loc points to. If the location is in a file that was included
2240  /// into a module, or is outside any module, returns nullptr.
2241  Module *getModuleForLocation(SourceLocation Loc);
2242 
2243  /// We want to produce a diagnostic at location IncLoc concerning a
2244  /// missing module import.
2245  ///
2246  /// \param IncLoc The location at which the missing import was detected.
2247  /// \param M The desired module.
2248  /// \param MLoc A location within the desired module at which some desired
2249  /// effect occurred (eg, where a desired entity was declared).
2250  ///
2251  /// \return A file that can be #included to import a module containing MLoc.
2252  /// Null if no such file could be determined or if a #include is not
2253  /// appropriate.
2254  const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2255  Module *M,
2256  SourceLocation MLoc);
2257 
2258  bool isRecordingPreamble() const {
2259  return PreambleConditionalStack.isRecording();
2260  }
2261 
2262  bool hasRecordedPreamble() const {
2263  return PreambleConditionalStack.hasRecordedPreamble();
2264  }
2265 
2267  return PreambleConditionalStack.getStack();
2268  }
2269 
2271  PreambleConditionalStack.setStack(s);
2272  }
2273 
2276  PreambleConditionalStack.startReplaying();
2277  PreambleConditionalStack.setStack(s);
2278  PreambleConditionalStack.SkipInfo = SkipInfo;
2279  }
2280 
2282  return PreambleConditionalStack.SkipInfo;
2283  }
2284 
2285 private:
2286  /// After processing predefined file, initialize the conditional stack from
2287  /// the preamble.
2288  void replayPreambleConditionalStack();
2289 
2290  // Macro handling.
2291  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2292  void HandleUndefDirective();
2293 
2294  // Conditional Inclusion.
2295  void HandleIfdefDirective(Token &Result, const Token &HashToken,
2296  bool isIfndef, bool ReadAnyTokensBeforeDirective);
2297  void HandleIfDirective(Token &IfToken, const Token &HashToken,
2298  bool ReadAnyTokensBeforeDirective);
2299  void HandleEndifDirective(Token &EndifToken);
2300  void HandleElseDirective(Token &Result, const Token &HashToken);
2301  void HandleElifDirective(Token &ElifToken, const Token &HashToken);
2302 
2303  // Pragmas.
2304  void HandlePragmaDirective(PragmaIntroducer Introducer);
2305 
2306 public:
2307  void HandlePragmaOnce(Token &OnceTok);
2308  void HandlePragmaMark();
2309  void HandlePragmaPoison();
2310  void HandlePragmaSystemHeader(Token &SysHeaderTok);
2311  void HandlePragmaDependency(Token &DependencyTok);
2312  void HandlePragmaPushMacro(Token &Tok);
2313  void HandlePragmaPopMacro(Token &Tok);
2314  void HandlePragmaIncludeAlias(Token &Tok);
2315  void HandlePragmaModuleBuild(Token &Tok);
2316  void HandlePragmaHdrstop(Token &Tok);
2317  IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2318 
2319  // Return true and store the first token only if any CommentHandler
2320  // has inserted some tokens and getCommentRetentionState() is false.
2321  bool HandleComment(Token &result, SourceRange Comment);
2322 
2323  /// A macro is used, update information about macros that need unused
2324  /// warnings.
2325  void markMacroAsUsed(MacroInfo *MI);
2326 
2327 private:
2329  getSkippedRangeForExcludedConditionalBlock(SourceLocation HashLoc);
2330 
2331  /// Contains the currently active skipped range mappings for skipping excluded
2332  /// conditional directives.
2334  *ExcludedConditionalDirectiveSkipMappings;
2335 };
2336 
2337 /// Abstract base class that describes a handler that will receive
2338 /// source ranges for each of the comments encountered in the source file.
2340 public:
2341  virtual ~CommentHandler();
2342 
2343  // The handler shall return true if it has pushed any tokens
2344  // to be read using e.g. EnterToken or EnterTokenStream.
2345  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2346 };
2347 
2348 /// Registry of pragma handlers added by plugins
2349 using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2350 
2351 } // namespace clang
2352 
2353 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
Definition: Preprocessor.h:424
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
A set of visible modules.
Definition: Module.h:596
StringRef Identifier
Definition: Format.cpp:1815
llvm::BumpPtrAllocator & getPreprocessorAllocator()
Definition: Preprocessor.h:915
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:76
SelectorTable & getSelectorTable()
Definition: Preprocessor.h:913
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:67
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:171
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the &#39;spelling&#39; of the Tok token.
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
Definition: Dominators.h:30
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:97
Defines the SourceManager interface.
static const Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:20
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
FileManager & getFileManager() const
Definition: Preprocessor.h:907
Defines the clang::Module class, which describes a module in the source code.
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
StringRef P
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:564
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1290
A directive for an undefined macro.
Definition: MacroInfo.h:429
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool hadModuleLoaderFatalFailure() const
Definition: Preprocessor.h:928
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:101
Builtin::Context & getBuiltinInfo()
Definition: Preprocessor.h:914
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:35
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:120
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Definition: MacroInfo.h:328
tok::TokenKind getKind() const
Definition: Token.h:92
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:91
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
One of these records is kept for each identifier that is lexed.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
Represents a macro directive exported by a module.
Definition: MacroInfo.h:488
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:406
This table allows us to fully hide how we implement multi-keyword caching.
LineState State
Definition: Format.h:2392
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:905
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Definition: Preprocessor.h:938
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:49
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:904
Describes a module or submodule.
Definition: Module.h:64
bool isPragmaLexer() const
isPragmaLexer - Returns true if this Lexer is being used to lex a pragma.
Definition: Lexer.h:183
A directive for setting the module visibility of a macro.
Definition: MacroInfo.h:444
bool isInvalid() const
void SetSuppressIncludeNotFoundError(bool Suppress)
Definition: Preprocessor.h:948
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:108
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
Definition: Preprocessor.h:985
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:909
const FormatToken & Tok
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void LexNonComment(Token &Result)
Lex a token.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:149
bool isMacroDefined(const IdentifierInfo *II)
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we&#39;ve alread...
Defines the Diagnostic-related interfaces.
const TargetInfo * getAuxTargetInfo() const
Definition: Preprocessor.h:906
TokenLexer - This implements a lexer that returns tokens from a macro body or token stream instead of...
Definition: TokenLexer.h:30
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream...
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
Definition: Preprocessor.h:958
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:158
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:1043
void clearCodeCompletionHandler()
Clear out the code completion handler.
StringRef Filename
Definition: Format.cpp:1807
Provides lookups to, and iteration over, IdentiferInfo objects.
unsigned Offset
Definition: Format.cpp:1809
Exposes information about the current target.
Definition: TargetInfo.h:163
const IdentifierTable & getIdentifierTable() const
Definition: Preprocessor.h:912
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Abstract interface for external sources of preprocessor information.
Defines the clang::LangOptions interface.
unsigned getCounterValue() const
SourceLocation End
Represents a character-granular source range.
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:349
int Id
Definition: ASTDiff.cpp:190
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
const AnnotatedLine * Line
bool isDefined() const
Definition: MacroInfo.h:384
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Definition: Preprocessor.h:967
Implements an efficient mapping from strings to IdentifierInfo nodes.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
#define V(N, I)
Definition: ASTContext.h:2921
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
A class for tracking whether we&#39;re inside a VA_OPT during a traversal of the tokens of a variadic mac...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:126
#define bool
Definition: stdbool.h:15
DefInfo getDefinition()
Traverses the macro directives history and returns the next macro definition directive along with inf...
Definition: MacroInfo.cpp:178
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:86
llvm::Optional< PreambleSkipInfo > getPreambleSkipInfo() const
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
bool getCommentRetentionState() const
Definition: Preprocessor.h:943
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:99
PPCallbacks * getPPCallbacks() const
Definition: Preprocessor.h:995
const SourceManager & SM
Definition: Format.cpp:1667
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
bool isRecordingPreamble() const
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
SourceManager & getSourceManager() const
Definition: Preprocessor.h:908
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define&#39;d and not #undef&#39;...
const DefInfo findDirectiveAtLoc(SourceLocation L, const SourceManager &SM) const
Find macro definition active in the specified source location.
Definition: MacroInfo.cpp:202
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:921
llvm::cl::opt< std::string > Filter
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:290
Kind
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
void setExternalSource(ExternalPreprocessorSource *Source)
Definition: Preprocessor.h:917
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.cpp:49
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:179
IdentifierTable & getIdentifierTable()
Definition: Preprocessor.h:911
void setPragmasEnabled(bool Enabled)
Definition: Preprocessor.h:945
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:78
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:85
bool isPoisoned() const
Return true if this token has been poisoned.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:24
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
Definition: Preprocessor.h:975
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
const MacroInfo * getMacroInfo() const
Definition: MacroInfo.h:390
StringRef getName() const
Return the actual identifier string.
bool isMacroDefined(StringRef Id)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
Dataflow Directional Tag Classes.
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
Definition: DeclSpec.h:1152
SourceRange getCodeCompletionTokenRange() const
std::pair< IdentifierInfo *, SourceLocation > getPragmaARCCFCodeAuditedInfo() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:354
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:65
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Abstract interface for a module loader.
Definition: ModuleLoader.h:73
unsigned getLength() const
Definition: Token.h:129
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:47
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:217
void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident, SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
bool getPragmasEnabled() const
Definition: Preprocessor.h:946
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:123
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
Definition: Preprocessor.h:926
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Definition: Preprocessor.h:899
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:45
Defines the PPCallbacks interface.
Defines the clang::TokenKind enum and support functions.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
SourceLocation getModuleImportLoc(Module *M) const
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:988
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:901
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:378
bool hasRecordedPreamble() const
const std::string & getPredefines() const
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:285
void setPredefines(StringRef P)
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
The translation unit is a complete translation unit.
Definition: LangOptions.h:380
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
llvm::DenseMap< const llvm::MemoryBuffer *, const PreprocessorSkippedRangeMapping * > ExcludedPreprocessorDirectiveSkipMapping
The datastructure that holds the mapping between the active memory buffers and the individual skip ma...
MacroMap::const_iterator macro_iterator
Abstract base class that describes a handler that will receive source ranges for each of the comments...
Describes how and where the pragma was introduced.
Definition: Pragma.h:51
void setCounterValue(unsigned V)
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
Definition: Preprocessor.h:933
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
Definition: Preprocessor.h:952
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, llvm::Optional< PreambleSkipInfo > SkipInfo)
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Callback handler that receives notifications when performing code completion within the preprocessor...
A header that is known to reside within a given module, whether it was included or excluded...
Definition: ModuleMap.h:150
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
Definition: Preprocessor.h:964
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:996
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:125
void setDiagnostics(DiagnosticsEngine &D)
Definition: Preprocessor.h:902