clang  15.0.0git
Preprocessor.h
Go to the documentation of this file.
1 //===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines the clang::Preprocessor interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15 #define LLVM_CLANG_LEX_PREPROCESSOR_H
16 
17 #include "clang/Basic/Diagnostic.h"
20 #include "clang/Basic/LLVM.h"
22 #include "clang/Basic/Module.h"
25 #include "clang/Basic/TokenKinds.h"
26 #include "clang/Lex/HeaderSearch.h"
27 #include "clang/Lex/Lexer.h"
28 #include "clang/Lex/MacroInfo.h"
29 #include "clang/Lex/ModuleLoader.h"
30 #include "clang/Lex/ModuleMap.h"
31 #include "clang/Lex/PPCallbacks.h"
32 #include "clang/Lex/Token.h"
33 #include "clang/Lex/TokenLexer.h"
34 #include "llvm/ADT/ArrayRef.h"
35 #include "llvm/ADT/DenseMap.h"
36 #include "llvm/ADT/FoldingSet.h"
37 #include "llvm/ADT/FunctionExtras.h"
38 #include "llvm/ADT/None.h"
39 #include "llvm/ADT/Optional.h"
40 #include "llvm/ADT/PointerUnion.h"
41 #include "llvm/ADT/STLExtras.h"
42 #include "llvm/ADT/SmallPtrSet.h"
43 #include "llvm/ADT/SmallVector.h"
44 #include "llvm/ADT/StringRef.h"
45 #include "llvm/ADT/TinyPtrVector.h"
46 #include "llvm/ADT/iterator_range.h"
47 #include "llvm/Support/Allocator.h"
48 #include "llvm/Support/Casting.h"
49 #include "llvm/Support/Registry.h"
50 #include <cassert>
51 #include <cstddef>
52 #include <cstdint>
53 #include <map>
54 #include <memory>
55 #include <string>
56 #include <utility>
57 #include <vector>
58 
59 namespace llvm {
60 
61 template<unsigned InternalLen> class SmallString;
62 
63 } // namespace llvm
64 
65 namespace clang {
66 
67 class CodeCompletionHandler;
68 class CommentHandler;
69 class DirectoryEntry;
70 class DirectoryLookup;
71 class EmptylineHandler;
72 class ExternalPreprocessorSource;
73 class FileEntry;
74 class FileManager;
75 class HeaderSearch;
76 class MacroArgs;
77 class PragmaHandler;
78 class PragmaNamespace;
79 class PreprocessingRecord;
80 class PreprocessorLexer;
81 class PreprocessorOptions;
82 class ScratchBuffer;
83 class TargetInfo;
84 
85 namespace Builtin {
86 class Context;
87 }
88 
89 /// Stores token information for comparing actual tokens with
90 /// predefined values. Only handles simple tokens and identifiers.
91 class TokenValue {
92  tok::TokenKind Kind;
93  IdentifierInfo *II;
94 
95 public:
96  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
97  assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
98  assert(Kind != tok::identifier &&
99  "Identifiers should be created by TokenValue(IdentifierInfo *)");
100  assert(!tok::isLiteral(Kind) && "Literals are not supported.");
101  assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
102  }
103 
104  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
105 
106  bool operator==(const Token &Tok) const {
107  return Tok.getKind() == Kind &&
108  (!II || II == Tok.getIdentifierInfo());
109  }
110 };
111 
112 /// Context in which macro name is used.
113 enum MacroUse {
114  // other than #define or #undef
115  MU_Other = 0,
116 
117  // macro name specified in #define
119 
120  // macro name specified in #undef
122 };
123 
124 /// Engages in a tight little dance with the lexer to efficiently
125 /// preprocess tokens.
126 ///
127 /// Lexers know only about tokens within a single source file, and don't
128 /// know anything about preprocessor-level issues like the \#include stack,
129 /// token expansion, etc.
133 
134  llvm::unique_function<void(const clang::Token &)> OnToken;
135  std::shared_ptr<PreprocessorOptions> PPOpts;
136  DiagnosticsEngine *Diags;
137  LangOptions &LangOpts;
138  const TargetInfo *Target = nullptr;
139  const TargetInfo *AuxTarget = nullptr;
140  FileManager &FileMgr;
141  SourceManager &SourceMgr;
142  std::unique_ptr<ScratchBuffer> ScratchBuf;
143  HeaderSearch &HeaderInfo;
144  ModuleLoader &TheModuleLoader;
145 
146  /// External source of macros.
147  ExternalPreprocessorSource *ExternalSource;
148 
149  /// A BumpPtrAllocator object used to quickly allocate and release
150  /// objects internal to the Preprocessor.
151  llvm::BumpPtrAllocator BP;
152 
153  /// Identifiers for builtin macros and other builtins.
154  IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
155  IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
156  IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
157  IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
158  IdentifierInfo *Ident__FILE_NAME__; // __FILE_NAME__
159  IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
160  IdentifierInfo *Ident__COUNTER__; // __COUNTER__
161  IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
162  IdentifierInfo *Ident__identifier; // __identifier
163  IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
164  IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
165  IdentifierInfo *Ident__has_feature; // __has_feature
166  IdentifierInfo *Ident__has_extension; // __has_extension
167  IdentifierInfo *Ident__has_builtin; // __has_builtin
168  IdentifierInfo *Ident__has_attribute; // __has_attribute
169  IdentifierInfo *Ident__has_include; // __has_include
170  IdentifierInfo *Ident__has_include_next; // __has_include_next
171  IdentifierInfo *Ident__has_warning; // __has_warning
172  IdentifierInfo *Ident__is_identifier; // __is_identifier
173  IdentifierInfo *Ident__building_module; // __building_module
174  IdentifierInfo *Ident__MODULE__; // __MODULE__
175  IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
176  IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
177  IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
178  IdentifierInfo *Ident__is_target_arch; // __is_target_arch
179  IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
180  IdentifierInfo *Ident__is_target_os; // __is_target_os
181  IdentifierInfo *Ident__is_target_environment; // __is_target_environment
182  IdentifierInfo *Ident__FLT_EVAL_METHOD__; // __FLT_EVAL_METHOD
183 
184  // Weak, only valid (and set) while InMacroArgs is true.
185  Token* ArgMacro;
186 
187  SourceLocation DATELoc, TIMELoc;
188 
189  // FEM_UnsetOnCommandLine means that an explicit evaluation method was
190  // not specified on the command line. The target is queried to set the
191  // default evaluation method.
192  LangOptions::FPEvalMethodKind CurrentFPEvalMethod =
193  LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine;
194 
195  // Keeps the value of the last evaluation method before a
196  // `pragma float_control (precise,off) is applied.
197  LangOptions::FPEvalMethodKind LastFPEvalMethod =
198  LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine;
199 
200  // The most recent pragma location where the floating point evaluation
201  // method was modified. This is used to determine whether the
202  // 'pragma clang fp eval_method' was used whithin the current scope.
203  SourceLocation LastFPEvalPragmaLocation;
204 
205  LangOptions::FPEvalMethodKind TUFPEvalMethod =
206  LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine;
207 
208  // Next __COUNTER__ value, starts at 0.
209  unsigned CounterValue = 0;
210 
211  enum {
212  /// Maximum depth of \#includes.
213  MaxAllowedIncludeStackDepth = 200
214  };
215 
216  // State that is set before the preprocessor begins.
217  bool KeepComments : 1;
218  bool KeepMacroComments : 1;
219  bool SuppressIncludeNotFoundError : 1;
220 
221  // State that changes while the preprocessor runs:
222  bool InMacroArgs : 1; // True if parsing fn macro invocation args.
223 
224  /// Whether the preprocessor owns the header search object.
225  bool OwnsHeaderSearch : 1;
226 
227  /// True if macro expansion is disabled.
228  bool DisableMacroExpansion : 1;
229 
230  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
231  /// when parsing preprocessor directives.
232  bool MacroExpansionInDirectivesOverride : 1;
233 
234  class ResetMacroExpansionHelper;
235 
236  /// Whether we have already loaded macros from the external source.
237  mutable bool ReadMacrosFromExternalSource : 1;
238 
239  /// True if pragmas are enabled.
240  bool PragmasEnabled : 1;
241 
242  /// True if the current build action is a preprocessing action.
243  bool PreprocessedOutput : 1;
244 
245  /// True if we are currently preprocessing a #if or #elif directive
246  bool ParsingIfOrElifDirective;
247 
248  /// True if we are pre-expanding macro arguments.
249  bool InMacroArgPreExpansion;
250 
251  /// Mapping/lookup information for all identifiers in
252  /// the program, including program keywords.
253  mutable IdentifierTable Identifiers;
254 
255  /// This table contains all the selectors in the program.
256  ///
257  /// Unlike IdentifierTable above, this table *isn't* populated by the
258  /// preprocessor. It is declared/expanded here because its role/lifetime is
259  /// conceptually similar to the IdentifierTable. In addition, the current
260  /// control flow (in clang::ParseAST()), make it convenient to put here.
261  ///
262  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
263  /// the lifetime of the preprocessor.
264  SelectorTable Selectors;
265 
266  /// Information about builtins.
267  std::unique_ptr<Builtin::Context> BuiltinInfo;
268 
269  /// Tracks all of the pragmas that the client registered
270  /// with this preprocessor.
271  std::unique_ptr<PragmaNamespace> PragmaHandlers;
272 
273  /// Pragma handlers of the original source is stored here during the
274  /// parsing of a model file.
275  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
276 
277  /// Tracks all of the comment handlers that the client registered
278  /// with this preprocessor.
279  std::vector<CommentHandler *> CommentHandlers;
280 
281  /// Empty line handler.
282  EmptylineHandler *Emptyline = nullptr;
283 
284  /// True if we want to ignore EOF token and continue later on (thus
285  /// avoid tearing the Lexer and etc. down).
286  bool IncrementalProcessing = false;
287 
288 public:
289  /// The kind of translation unit we are processing.
291 
292 private:
293  /// The code-completion handler.
294  CodeCompletionHandler *CodeComplete = nullptr;
295 
296  /// The file that we're performing code-completion for, if any.
297  const FileEntry *CodeCompletionFile = nullptr;
298 
299  /// The offset in file for the code-completion point.
300  unsigned CodeCompletionOffset = 0;
301 
302  /// The location for the code-completion point. This gets instantiated
303  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
304  SourceLocation CodeCompletionLoc;
305 
306  /// The start location for the file of the code-completion point.
307  ///
308  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
309  /// for preprocessing.
310  SourceLocation CodeCompletionFileLoc;
311 
312  /// The source location of the \c import contextual keyword we just
313  /// lexed, if any.
314  SourceLocation ModuleImportLoc;
315 
316  /// The module import path that we're currently processing.
318 
319  /// Whether the last token we lexed was an '@'.
320  bool LastTokenWasAt = false;
321 
322  /// A position within a C++20 import-seq.
323  class ImportSeq {
324  public:
325  enum State : int {
326  // Positive values represent a number of unclosed brackets.
327  AtTopLevel = 0,
328  AfterTopLevelTokenSeq = -1,
329  AfterExport = -2,
330  AfterImportSeq = -3,
331  };
332 
333  ImportSeq(State S) : S(S) {}
334 
335  /// Saw any kind of open bracket.
336  void handleOpenBracket() {
337  S = static_cast<State>(std::max<int>(S, 0) + 1);
338  }
339  /// Saw any kind of close bracket other than '}'.
340  void handleCloseBracket() {
341  S = static_cast<State>(std::max<int>(S, 1) - 1);
342  }
343  /// Saw a close brace.
344  void handleCloseBrace() {
345  handleCloseBracket();
346  if (S == AtTopLevel && !AfterHeaderName)
347  S = AfterTopLevelTokenSeq;
348  }
349  /// Saw a semicolon.
350  void handleSemi() {
351  if (atTopLevel()) {
352  S = AfterTopLevelTokenSeq;
353  AfterHeaderName = false;
354  }
355  }
356 
357  /// Saw an 'export' identifier.
358  void handleExport() {
359  if (S == AfterTopLevelTokenSeq)
360  S = AfterExport;
361  else if (S <= 0)
362  S = AtTopLevel;
363  }
364  /// Saw an 'import' identifier.
365  void handleImport() {
366  if (S == AfterTopLevelTokenSeq || S == AfterExport)
367  S = AfterImportSeq;
368  else if (S <= 0)
369  S = AtTopLevel;
370  }
371 
372  /// Saw a 'header-name' token; do not recognize any more 'import' tokens
373  /// until we reach a top-level semicolon.
374  void handleHeaderName() {
375  if (S == AfterImportSeq)
376  AfterHeaderName = true;
377  handleMisc();
378  }
379 
380  /// Saw any other token.
381  void handleMisc() {
382  if (S <= 0)
383  S = AtTopLevel;
384  }
385 
386  bool atTopLevel() { return S <= 0; }
387  bool afterImportSeq() { return S == AfterImportSeq; }
388 
389  private:
390  State S;
391  /// Whether we're in the pp-import-suffix following the header-name in a
392  /// pp-import. If so, a close-brace is not sufficient to end the
393  /// top-level-token-seq of an import-seq.
394  bool AfterHeaderName = false;
395  };
396 
397  /// Our current position within a C++20 import-seq.
398  ImportSeq ImportSeqState = ImportSeq::AfterTopLevelTokenSeq;
399 
400  /// Whether the module import expects an identifier next. Otherwise,
401  /// it expects a '.' or ';'.
402  bool ModuleImportExpectsIdentifier = false;
403 
404  /// The identifier and source location of the currently-active
405  /// \#pragma clang arc_cf_code_audited begin.
406  std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
407 
408  /// The source location of the currently-active
409  /// \#pragma clang assume_nonnull begin.
410  SourceLocation PragmaAssumeNonNullLoc;
411 
412  /// Set only for preambles which end with an active
413  /// \#pragma clang assume_nonnull begin.
414  ///
415  /// When the preamble is loaded into the main file,
416  /// `PragmaAssumeNonNullLoc` will be set to this to
417  /// replay the unterminated assume_nonnull.
418  SourceLocation PreambleRecordedPragmaAssumeNonNullLoc;
419 
420  /// True if we hit the code-completion point.
421  bool CodeCompletionReached = false;
422 
423  /// The code completion token containing the information
424  /// on the stem that is to be code completed.
425  IdentifierInfo *CodeCompletionII = nullptr;
426 
427  /// Range for the code completion token.
428  SourceRange CodeCompletionTokenRange;
429 
430  /// The directory that the main file should be considered to occupy,
431  /// if it does not correspond to a real file (as happens when building a
432  /// module).
433  const DirectoryEntry *MainFileDir = nullptr;
434 
435  /// The number of bytes that we will initially skip when entering the
436  /// main file, along with a flag that indicates whether skipping this number
437  /// of bytes will place the lexer at the start of a line.
438  ///
439  /// This is used when loading a precompiled preamble.
440  std::pair<int, bool> SkipMainFilePreamble;
441 
442  /// Whether we hit an error due to reaching max allowed include depth. Allows
443  /// to avoid hitting the same error over and over again.
444  bool HasReachedMaxIncludeDepth = false;
445 
446  /// The number of currently-active calls to Lex.
447  ///
448  /// Lex is reentrant, and asking for an (end-of-phase-4) token can often
449  /// require asking for multiple additional tokens. This counter makes it
450  /// possible for Lex to detect whether it's producing a token for the end
451  /// of phase 4 of translation or for some other situation.
452  unsigned LexLevel = 0;
453 
454  /// The number of (LexLevel 0) preprocessor tokens.
455  unsigned TokenCount = 0;
456 
457  /// Preprocess every token regardless of LexLevel.
458  bool PreprocessToken = false;
459 
460  /// The maximum number of (LexLevel 0) tokens before issuing a -Wmax-tokens
461  /// warning, or zero for unlimited.
462  unsigned MaxTokens = 0;
463  SourceLocation MaxTokensOverrideLoc;
464 
465 public:
470  bool FoundElse;
472 
474  bool FoundNonSkipPortion, bool FoundElse,
478  ElseLoc(ElseLoc) {}
479  };
480 
482 
483 private:
484  friend class ASTReader;
485  friend class MacroArgs;
486 
487  class PreambleConditionalStackStore {
488  enum State {
489  Off = 0,
490  Recording = 1,
491  Replaying = 2,
492  };
493 
494  public:
495  PreambleConditionalStackStore() = default;
496 
497  void startRecording() { ConditionalStackState = Recording; }
498  void startReplaying() { ConditionalStackState = Replaying; }
499  bool isRecording() const { return ConditionalStackState == Recording; }
500  bool isReplaying() const { return ConditionalStackState == Replaying; }
501 
502  ArrayRef<PPConditionalInfo> getStack() const {
503  return ConditionalStack;
504  }
505 
506  void doneReplaying() {
507  ConditionalStack.clear();
508  ConditionalStackState = Off;
509  }
510 
511  void setStack(ArrayRef<PPConditionalInfo> s) {
512  if (!isRecording() && !isReplaying())
513  return;
514  ConditionalStack.clear();
515  ConditionalStack.append(s.begin(), s.end());
516  }
517 
518  bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
519 
520  bool reachedEOFWhileSkipping() const { return SkipInfo.has_value(); }
521 
522  void clearSkipInfo() { SkipInfo.reset(); }
523 
525 
526  private:
527  SmallVector<PPConditionalInfo, 4> ConditionalStack;
528  State ConditionalStackState = Off;
529  } PreambleConditionalStack;
530 
531  /// The current top of the stack that we're lexing from if
532  /// not expanding a macro and we are lexing directly from source code.
533  ///
534  /// Only one of CurLexer, or CurTokenLexer will be non-null.
535  std::unique_ptr<Lexer> CurLexer;
536 
537  /// The current top of the stack what we're lexing from
538  /// if not expanding a macro.
539  ///
540  /// This is an alias for CurLexer.
541  PreprocessorLexer *CurPPLexer = nullptr;
542 
543  /// Used to find the current FileEntry, if CurLexer is non-null
544  /// and if applicable.
545  ///
546  /// This allows us to implement \#include_next and find directory-specific
547  /// properties.
548  ConstSearchDirIterator CurDirLookup = nullptr;
549 
550  /// The current macro we are expanding, if we are expanding a macro.
551  ///
552  /// One of CurLexer and CurTokenLexer must be null.
553  std::unique_ptr<TokenLexer> CurTokenLexer;
554 
555  /// The kind of lexer we're currently working with.
556  enum CurLexerKind {
557  CLK_Lexer,
558  CLK_TokenLexer,
559  CLK_CachingLexer,
560  CLK_DependencyDirectivesLexer,
561  CLK_LexAfterModuleImport
562  } CurLexerKind = CLK_Lexer;
563 
564  /// If the current lexer is for a submodule that is being built, this
565  /// is that submodule.
566  Module *CurLexerSubmodule = nullptr;
567 
568  /// Keeps track of the stack of files currently
569  /// \#included, and macros currently being expanded from, not counting
570  /// CurLexer/CurTokenLexer.
571  struct IncludeStackInfo {
572  enum CurLexerKind CurLexerKind;
573  Module *TheSubmodule;
574  std::unique_ptr<Lexer> TheLexer;
575  PreprocessorLexer *ThePPLexer;
576  std::unique_ptr<TokenLexer> TheTokenLexer;
577  ConstSearchDirIterator TheDirLookup;
578 
579  // The following constructors are completely useless copies of the default
580  // versions, only needed to pacify MSVC.
581  IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
582  std::unique_ptr<Lexer> &&TheLexer,
583  PreprocessorLexer *ThePPLexer,
584  std::unique_ptr<TokenLexer> &&TheTokenLexer,
585  ConstSearchDirIterator TheDirLookup)
586  : CurLexerKind(std::move(CurLexerKind)),
587  TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
588  ThePPLexer(std::move(ThePPLexer)),
589  TheTokenLexer(std::move(TheTokenLexer)),
590  TheDirLookup(std::move(TheDirLookup)) {}
591  };
592  std::vector<IncludeStackInfo> IncludeMacroStack;
593 
594  /// Actions invoked when some preprocessor activity is
595  /// encountered (e.g. a file is \#included, etc).
596  std::unique_ptr<PPCallbacks> Callbacks;
597 
598  struct MacroExpandsInfo {
599  Token Tok;
600  MacroDefinition MD;
601  SourceRange Range;
602 
603  MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
604  : Tok(Tok), MD(MD), Range(Range) {}
605  };
606  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
607 
608  /// Information about a name that has been used to define a module macro.
609  struct ModuleMacroInfo {
610  /// The most recent macro directive for this identifier.
611  MacroDirective *MD;
612 
613  /// The active module macros for this identifier.
614  llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
615 
616  /// The generation number at which we last updated ActiveModuleMacros.
617  /// \see Preprocessor::VisibleModules.
618  unsigned ActiveModuleMacrosGeneration = 0;
619 
620  /// Whether this macro name is ambiguous.
621  bool IsAmbiguous = false;
622 
623  /// The module macros that are overridden by this macro.
624  llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
625 
626  ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
627  };
628 
629  /// The state of a macro for an identifier.
630  class MacroState {
631  mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
632 
633  ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
634  const IdentifierInfo *II) const {
635  if (II->isOutOfDate())
636  PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
637  // FIXME: Find a spare bit on IdentifierInfo and store a
638  // HasModuleMacros flag.
639  if (!II->hasMacroDefinition() ||
640  (!PP.getLangOpts().Modules &&
641  !PP.getLangOpts().ModulesLocalVisibility) ||
642  !PP.CurSubmoduleState->VisibleModules.getGeneration())
643  return nullptr;
644 
645  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
646  if (!Info) {
647  Info = new (PP.getPreprocessorAllocator())
648  ModuleMacroInfo(State.get<MacroDirective *>());
649  State = Info;
650  }
651 
652  if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
653  Info->ActiveModuleMacrosGeneration)
654  PP.updateModuleMacroInfo(II, *Info);
655  return Info;
656  }
657 
658  public:
659  MacroState() : MacroState(nullptr) {}
660  MacroState(MacroDirective *MD) : State(MD) {}
661 
662  MacroState(MacroState &&O) noexcept : State(O.State) {
663  O.State = (MacroDirective *)nullptr;
664  }
665 
666  MacroState &operator=(MacroState &&O) noexcept {
667  auto S = O.State;
668  O.State = (MacroDirective *)nullptr;
669  State = S;
670  return *this;
671  }
672 
673  ~MacroState() {
674  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
675  Info->~ModuleMacroInfo();
676  }
677 
678  MacroDirective *getLatest() const {
679  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
680  return Info->MD;
681  return State.get<MacroDirective*>();
682  }
683 
684  void setLatest(MacroDirective *MD) {
685  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
686  Info->MD = MD;
687  else
688  State = MD;
689  }
690 
691  bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
692  auto *Info = getModuleInfo(PP, II);
693  return Info ? Info->IsAmbiguous : false;
694  }
695 
696  ArrayRef<ModuleMacro *>
697  getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
698  if (auto *Info = getModuleInfo(PP, II))
699  return Info->ActiveModuleMacros;
700  return None;
701  }
702 
703  MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
704  SourceManager &SourceMgr) const {
705  // FIXME: Incorporate module macros into the result of this.
706  if (auto *Latest = getLatest())
707  return Latest->findDirectiveAtLoc(Loc, SourceMgr);
708  return {};
709  }
710 
711  void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
712  if (auto *Info = getModuleInfo(PP, II)) {
713  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
714  Info->ActiveModuleMacros.begin(),
715  Info->ActiveModuleMacros.end());
716  Info->ActiveModuleMacros.clear();
717  Info->IsAmbiguous = false;
718  }
719  }
720 
721  ArrayRef<ModuleMacro*> getOverriddenMacros() const {
722  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
723  return Info->OverriddenMacros;
724  return None;
725  }
726 
727  void setOverriddenMacros(Preprocessor &PP,
728  ArrayRef<ModuleMacro *> Overrides) {
729  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
730  if (!Info) {
731  if (Overrides.empty())
732  return;
733  Info = new (PP.getPreprocessorAllocator())
734  ModuleMacroInfo(State.get<MacroDirective *>());
735  State = Info;
736  }
737  Info->OverriddenMacros.clear();
738  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
739  Overrides.begin(), Overrides.end());
740  Info->ActiveModuleMacrosGeneration = 0;
741  }
742  };
743 
744  /// For each IdentifierInfo that was associated with a macro, we
745  /// keep a mapping to the history of all macro definitions and #undefs in
746  /// the reverse order (the latest one is in the head of the list).
747  ///
748  /// This mapping lives within the \p CurSubmoduleState.
749  using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
750 
751  struct SubmoduleState;
752 
753  /// Information about a submodule that we're currently building.
754  struct BuildingSubmoduleInfo {
755  /// The module that we are building.
756  Module *M;
757 
758  /// The location at which the module was included.
759  SourceLocation ImportLoc;
760 
761  /// Whether we entered this submodule via a pragma.
762  bool IsPragma;
763 
764  /// The previous SubmoduleState.
765  SubmoduleState *OuterSubmoduleState;
766 
767  /// The number of pending module macro names when we started building this.
768  unsigned OuterPendingModuleMacroNames;
769 
770  BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
771  SubmoduleState *OuterSubmoduleState,
772  unsigned OuterPendingModuleMacroNames)
773  : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
774  OuterSubmoduleState(OuterSubmoduleState),
775  OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
776  };
777  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
778 
779  /// Information about a submodule's preprocessor state.
780  struct SubmoduleState {
781  /// The macros for the submodule.
782  MacroMap Macros;
783 
784  /// The set of modules that are visible within the submodule.
785  VisibleModuleSet VisibleModules;
786 
787  // FIXME: CounterValue?
788  // FIXME: PragmaPushMacroInfo?
789  };
790  std::map<Module *, SubmoduleState> Submodules;
791 
792  /// The preprocessor state for preprocessing outside of any submodule.
793  SubmoduleState NullSubmoduleState;
794 
795  /// The current submodule state. Will be \p NullSubmoduleState if we're not
796  /// in a submodule.
797  SubmoduleState *CurSubmoduleState;
798 
799  /// The files that have been included.
800  IncludedFilesSet IncludedFiles;
801 
802  /// The set of known macros exported from modules.
803  llvm::FoldingSet<ModuleMacro> ModuleMacros;
804 
805  /// The names of potential module macros that we've not yet processed.
806  llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
807 
808  /// The list of module macros, for each identifier, that are not overridden by
809  /// any other module macro.
810  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
811  LeafModuleMacros;
812 
813  /// Macros that we want to warn because they are not used at the end
814  /// of the translation unit.
815  ///
816  /// We store just their SourceLocations instead of
817  /// something like MacroInfo*. The benefit of this is that when we are
818  /// deserializing from PCH, we don't need to deserialize identifier & macros
819  /// just so that we can report that they are unused, we just warn using
820  /// the SourceLocations of this set (that will be filled by the ASTReader).
821  using WarnUnusedMacroLocsTy = llvm::SmallDenseSet<SourceLocation, 32>;
822  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
823 
824  /// This is a pair of an optional message and source location used for pragmas
825  /// that annotate macros like pragma clang restrict_expansion and pragma clang
826  /// deprecated. This pair stores the optional message and the location of the
827  /// annotation pragma for use producing diagnostics and notes.
828  using MsgLocationPair = std::pair<std::string, SourceLocation>;
829 
830  struct MacroAnnotationInfo {
831  SourceLocation Location;
833  };
834 
835  struct MacroAnnotations {
836  llvm::Optional<MacroAnnotationInfo> DeprecationInfo;
837  llvm::Optional<MacroAnnotationInfo> RestrictExpansionInfo;
838  llvm::Optional<SourceLocation> FinalAnnotationLoc;
839 
840  static MacroAnnotations makeDeprecation(SourceLocation Loc,
841  std::string Msg) {
842  return MacroAnnotations{MacroAnnotationInfo{Loc, std::move(Msg)},
843  llvm::None, llvm::None};
844  }
845 
846  static MacroAnnotations makeRestrictExpansion(SourceLocation Loc,
847  std::string Msg) {
848  return MacroAnnotations{
849  llvm::None, MacroAnnotationInfo{Loc, std::move(Msg)}, llvm::None};
850  }
851 
852  static MacroAnnotations makeFinal(SourceLocation Loc) {
853  return MacroAnnotations{llvm::None, llvm::None, Loc};
854  }
855  };
856 
857  /// Warning information for macro annotations.
858  llvm::DenseMap<const IdentifierInfo *, MacroAnnotations> AnnotationInfos;
859 
860  /// A "freelist" of MacroArg objects that can be
861  /// reused for quick allocation.
862  MacroArgs *MacroArgCache = nullptr;
863 
864  /// For each IdentifierInfo used in a \#pragma push_macro directive,
865  /// we keep a MacroInfo stack used to restore the previous macro value.
866  llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
867  PragmaPushMacroInfo;
868 
869  // Various statistics we track for performance analysis.
870  unsigned NumDirectives = 0;
871  unsigned NumDefined = 0;
872  unsigned NumUndefined = 0;
873  unsigned NumPragma = 0;
874  unsigned NumIf = 0;
875  unsigned NumElse = 0;
876  unsigned NumEndif = 0;
877  unsigned NumEnteredSourceFiles = 0;
878  unsigned MaxIncludeStackDepth = 0;
879  unsigned NumMacroExpanded = 0;
880  unsigned NumFnMacroExpanded = 0;
881  unsigned NumBuiltinMacroExpanded = 0;
882  unsigned NumFastMacroExpanded = 0;
883  unsigned NumTokenPaste = 0;
884  unsigned NumFastTokenPaste = 0;
885  unsigned NumSkipped = 0;
886 
887  /// The predefined macros that preprocessor should use from the
888  /// command line etc.
889  std::string Predefines;
890 
891  /// The file ID for the preprocessor predefines.
892  FileID PredefinesFileID;
893 
894  /// The file ID for the PCH through header.
895  FileID PCHThroughHeaderFileID;
896 
897  /// Whether tokens are being skipped until a #pragma hdrstop is seen.
898  bool SkippingUntilPragmaHdrStop = false;
899 
900  /// Whether tokens are being skipped until the through header is seen.
901  bool SkippingUntilPCHThroughHeader = false;
902 
903  /// \{
904  /// Cache of macro expanders to reduce malloc traffic.
905  enum { TokenLexerCacheSize = 8 };
906  unsigned NumCachedTokenLexers;
907  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
908  /// \}
909 
910  /// Keeps macro expanded tokens for TokenLexers.
911  //
912  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
913  /// going to lex in the cache and when it finishes the tokens are removed
914  /// from the end of the cache.
915  SmallVector<Token, 16> MacroExpandedTokens;
916  std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
917 
918  /// A record of the macro definitions and expansions that
919  /// occurred during preprocessing.
920  ///
921  /// This is an optional side structure that can be enabled with
922  /// \c createPreprocessingRecord() prior to preprocessing.
923  PreprocessingRecord *Record = nullptr;
924 
925  /// Cached tokens state.
926  using CachedTokensTy = SmallVector<Token, 1>;
927 
928  /// Cached tokens are stored here when we do backtracking or
929  /// lookahead. They are "lexed" by the CachingLex() method.
930  CachedTokensTy CachedTokens;
931 
932  /// The position of the cached token that CachingLex() should
933  /// "lex" next.
934  ///
935  /// If it points beyond the CachedTokens vector, it means that a normal
936  /// Lex() should be invoked.
937  CachedTokensTy::size_type CachedLexPos = 0;
938 
939  /// Stack of backtrack positions, allowing nested backtracks.
940  ///
941  /// The EnableBacktrackAtThisPos() method pushes a position to
942  /// indicate where CachedLexPos should be set when the BackTrack() method is
943  /// invoked (at which point the last position is popped).
944  std::vector<CachedTokensTy::size_type> BacktrackPositions;
945 
946  struct MacroInfoChain {
947  MacroInfo MI;
948  MacroInfoChain *Next;
949  };
950 
951  /// MacroInfos are managed as a chain for easy disposal. This is the head
952  /// of that list.
953  MacroInfoChain *MIChainHead = nullptr;
954 
955  /// True if \p Preprocessor::SkipExcludedConditionalBlock() is running.
956  /// This is used to guard against calling this function recursively.
957  ///
958  /// See comments at the use-site for more context about why it is needed.
959  bool SkippingExcludedConditionalBlock = false;
960 
961  /// Keeps track of skipped range mappings that were recorded while skipping
962  /// excluded conditional directives. It maps the source buffer pointer at
963  /// the beginning of a skipped block, to the number of bytes that should be
964  /// skipped.
965  llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges;
966 
967  void updateOutOfDateIdentifier(IdentifierInfo &II) const;
968 
969 public:
970  Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
971  DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM,
972  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
973  IdentifierInfoLookup *IILookup = nullptr,
974  bool OwnsHeaderSearch = false,
976 
977  ~Preprocessor();
978 
979  /// Initialize the preprocessor using information about the target.
980  ///
981  /// \param Target is owned by the caller and must remain valid for the
982  /// lifetime of the preprocessor.
983  /// \param AuxTarget is owned by the caller and must remain valid for
984  /// the lifetime of the preprocessor.
985  void Initialize(const TargetInfo &Target,
986  const TargetInfo *AuxTarget = nullptr);
987 
988  /// Initialize the preprocessor to parse a model file
989  ///
990  /// To parse model files the preprocessor of the original source is reused to
991  /// preserver the identifier table. However to avoid some duplicate
992  /// information in the preprocessor some cleanup is needed before it is used
993  /// to parse model files. This method does that cleanup.
994  void InitializeForModelFile();
995 
996  /// Cleanup after model file parsing
997  void FinalizeForModelFile();
998 
999  /// Retrieve the preprocessor options used to initialize this
1000  /// preprocessor.
1001  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
1002 
1003  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
1004  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
1005 
1006  const LangOptions &getLangOpts() const { return LangOpts; }
1007  const TargetInfo &getTargetInfo() const { return *Target; }
1008  const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
1009  FileManager &getFileManager() const { return FileMgr; }
1010  SourceManager &getSourceManager() const { return SourceMgr; }
1011  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
1012 
1013  IdentifierTable &getIdentifierTable() { return Identifiers; }
1014  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
1015  SelectorTable &getSelectorTable() { return Selectors; }
1016  Builtin::Context &getBuiltinInfo() { return *BuiltinInfo; }
1017  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
1018 
1020  ExternalSource = Source;
1021  }
1022 
1024  return ExternalSource;
1025  }
1026 
1027  /// Retrieve the module loader associated with this preprocessor.
1028  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
1029 
1031  return TheModuleLoader.HadFatalFailure;
1032  }
1033 
1034  /// Retrieve the number of Directives that have been processed by the
1035  /// Preprocessor.
1036  unsigned getNumDirectives() const {
1037  return NumDirectives;
1038  }
1039 
1040  /// True if we are currently preprocessing a #if or #elif directive
1042  return ParsingIfOrElifDirective;
1043  }
1044 
1045  /// Control whether the preprocessor retains comments in output.
1046  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
1047  this->KeepComments = KeepComments | KeepMacroComments;
1048  this->KeepMacroComments = KeepMacroComments;
1049  }
1050 
1051  bool getCommentRetentionState() const { return KeepComments; }
1052 
1053  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
1054  bool getPragmasEnabled() const { return PragmasEnabled; }
1055 
1056  void SetSuppressIncludeNotFoundError(bool Suppress) {
1057  SuppressIncludeNotFoundError = Suppress;
1058  }
1059 
1061  return SuppressIncludeNotFoundError;
1062  }
1063 
1064  /// Sets whether the preprocessor is responsible for producing output or if
1065  /// it is producing tokens to be consumed by Parse and Sema.
1066  void setPreprocessedOutput(bool IsPreprocessedOutput) {
1067  PreprocessedOutput = IsPreprocessedOutput;
1068  }
1069 
1070  /// Returns true if the preprocessor is responsible for generating output,
1071  /// false if it is producing tokens to be consumed by Parse and Sema.
1072  bool isPreprocessedOutput() const { return PreprocessedOutput; }
1073 
1074  /// Return true if we are lexing directly from the specified lexer.
1075  bool isCurrentLexer(const PreprocessorLexer *L) const {
1076  return CurPPLexer == L;
1077  }
1078 
1079  /// Return the current lexer being lexed from.
1080  ///
1081  /// Note that this ignores any potentially active macro expansions and _Pragma
1082  /// expansions going on at the time.
1083  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
1084 
1085  /// Return the current file lexer being lexed from.
1086  ///
1087  /// Note that this ignores any potentially active macro expansions and _Pragma
1088  /// expansions going on at the time.
1090 
1091  /// Return the submodule owning the file being lexed. This may not be
1092  /// the current module if we have changed modules since entering the file.
1093  Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
1094 
1095  /// Returns the FileID for the preprocessor predefines.
1096  FileID getPredefinesFileID() const { return PredefinesFileID; }
1097 
1098  /// \{
1099  /// Accessors for preprocessor callbacks.
1100  ///
1101  /// Note that this class takes ownership of any PPCallbacks object given to
1102  /// it.
1103  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
1104  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
1105  if (Callbacks)
1106  C = std::make_unique<PPChainedCallbacks>(std::move(C),
1107  std::move(Callbacks));
1108  Callbacks = std::move(C);
1109  }
1110  /// \}
1111 
1112  /// Get the number of tokens processed so far.
1113  unsigned getTokenCount() const { return TokenCount; }
1114 
1115  /// Get the max number of tokens before issuing a -Wmax-tokens warning.
1116  unsigned getMaxTokens() const { return MaxTokens; }
1117 
1119  MaxTokens = Value;
1120  MaxTokensOverrideLoc = Loc;
1121  };
1122 
1123  SourceLocation getMaxTokensOverrideLoc() const { return MaxTokensOverrideLoc; }
1124 
1125  /// Register a function that would be called on each token in the final
1126  /// expanded token stream.
1127  /// This also reports annotation tokens produced by the parser.
1128  void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
1129  OnToken = std::move(F);
1130  }
1131 
1132  void setPreprocessToken(bool Preprocess) { PreprocessToken = Preprocess; }
1133 
1134  bool isMacroDefined(StringRef Id) {
1135  return isMacroDefined(&Identifiers.get(Id));
1136  }
1137  bool isMacroDefined(const IdentifierInfo *II) {
1138  return II->hasMacroDefinition() &&
1139  (!getLangOpts().Modules || (bool)getMacroDefinition(II));
1140  }
1141 
1142  /// Determine whether II is defined as a macro within the module M,
1143  /// if that is a module that we've already preprocessed. Does not check for
1144  /// macros imported into M.
1146  if (!II->hasMacroDefinition())
1147  return false;
1148  auto I = Submodules.find(M);
1149  if (I == Submodules.end())
1150  return false;
1151  auto J = I->second.Macros.find(II);
1152  if (J == I->second.Macros.end())
1153  return false;
1154  auto *MD = J->second.getLatest();
1155  return MD && MD->isDefined();
1156  }
1157 
1159  if (!II->hasMacroDefinition())
1160  return {};
1161 
1162  MacroState &S = CurSubmoduleState->Macros[II];
1163  auto *MD = S.getLatest();
1164  while (MD && isa<VisibilityMacroDirective>(MD))
1165  MD = MD->getPrevious();
1166  return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
1167  S.getActiveModuleMacros(*this, II),
1168  S.isAmbiguous(*this, II));
1169  }
1170 
1172  SourceLocation Loc) {
1173  if (!II->hadMacroDefinition())
1174  return {};
1175 
1176  MacroState &S = CurSubmoduleState->Macros[II];
1178  if (auto *MD = S.getLatest())
1179  DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
1180  // FIXME: Compute the set of active module macros at the specified location.
1181  return MacroDefinition(DI.getDirective(),
1182  S.getActiveModuleMacros(*this, II),
1183  S.isAmbiguous(*this, II));
1184  }
1185 
1186  /// Given an identifier, return its latest non-imported MacroDirective
1187  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
1189  if (!II->hasMacroDefinition())
1190  return nullptr;
1191 
1192  auto *MD = getLocalMacroDirectiveHistory(II);
1193  if (!MD || MD->getDefinition().isUndefined())
1194  return nullptr;
1195 
1196  return MD;
1197  }
1198 
1199  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
1200  return const_cast<Preprocessor*>(this)->getMacroInfo(II);
1201  }
1202 
1204  if (!II->hasMacroDefinition())
1205  return nullptr;
1206  if (auto MD = getMacroDefinition(II))
1207  return MD.getMacroInfo();
1208  return nullptr;
1209  }
1210 
1211  /// Given an identifier, return the latest non-imported macro
1212  /// directive for that identifier.
1213  ///
1214  /// One can iterate over all previous macro directives from the most recent
1215  /// one.
1217 
1218  /// Add a directive to the macro directive history for this identifier.
1221  SourceLocation Loc) {
1222  DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1223  appendMacroDirective(II, MD);
1224  return MD;
1225  }
1227  MacroInfo *MI) {
1228  return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1229  }
1230 
1231  /// Set a MacroDirective that was loaded from a PCH file.
1233  MacroDirective *MD);
1234 
1235  /// Register an exported macro for a module and identifier.
1237  ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1238  ModuleMacro *getModuleMacro(Module *Mod, const IdentifierInfo *II);
1239 
1240  /// Get the list of leaf (non-overridden) module macros for a name.
1242  if (II->isOutOfDate())
1243  updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1244  auto I = LeafModuleMacros.find(II);
1245  if (I != LeafModuleMacros.end())
1246  return I->second;
1247  return None;
1248  }
1249 
1250  /// Get the list of submodules that we're currently building.
1252  return BuildingSubmoduleStack;
1253  }
1254 
1255  /// \{
1256  /// Iterators for the macro history table. Currently defined macros have
1257  /// IdentifierInfo::hasMacroDefinition() set and an empty
1258  /// MacroInfo::getUndefLoc() at the head of the list.
1259  using macro_iterator = MacroMap::const_iterator;
1260 
1261  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1262  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1263 
1264  llvm::iterator_range<macro_iterator>
1265  macros(bool IncludeExternalMacros = true) const {
1266  macro_iterator begin = macro_begin(IncludeExternalMacros);
1267  macro_iterator end = macro_end(IncludeExternalMacros);
1268  return llvm::make_range(begin, end);
1269  }
1270 
1271  /// \}
1272 
1273  /// Mark the file as included.
1274  /// Returns true if this is the first time the file was included.
1275  bool markIncluded(const FileEntry *File) {
1276  HeaderInfo.getFileInfo(File);
1277  return IncludedFiles.insert(File).second;
1278  }
1279 
1280  /// Return true if this header has already been included.
1281  bool alreadyIncluded(const FileEntry *File) const {
1282  return IncludedFiles.count(File);
1283  }
1284 
1285  /// Get the set of included files.
1286  IncludedFilesSet &getIncludedFiles() { return IncludedFiles; }
1287  const IncludedFilesSet &getIncludedFiles() const { return IncludedFiles; }
1288 
1289  /// Return the name of the macro defined before \p Loc that has
1290  /// spelling \p Tokens. If there are multiple macros with same spelling,
1291  /// return the last one defined.
1293  ArrayRef<TokenValue> Tokens) const;
1294 
1295  /// Set the predefines for this Preprocessor.
1296  ///
1297  /// These predefines are automatically injected when parsing the main file.
1298  void setPredefines(std::string P) { Predefines = std::move(P); }
1299 
1300  /// Return information about the specified preprocessor
1301  /// identifier token.
1302  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1303  return &Identifiers.get(Name);
1304  }
1305 
1306  /// Add the specified pragma handler to this preprocessor.
1307  ///
1308  /// If \p Namespace is non-null, then it is a token required to exist on the
1309  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1310  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1312  AddPragmaHandler(StringRef(), Handler);
1313  }
1314 
1315  /// Remove the specific pragma handler from this preprocessor.
1316  ///
1317  /// If \p Namespace is non-null, then it should be the namespace that
1318  /// \p Handler was added to. It is an error to remove a handler that
1319  /// has not been registered.
1320  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1322  RemovePragmaHandler(StringRef(), Handler);
1323  }
1324 
1325  /// Install empty handlers for all pragmas (making them ignored).
1326  void IgnorePragmas();
1327 
1328  /// Set empty line handler.
1329  void setEmptylineHandler(EmptylineHandler *Handler) { Emptyline = Handler; }
1330 
1331  EmptylineHandler *getEmptylineHandler() const { return Emptyline; }
1332 
1333  /// Add the specified comment handler to the preprocessor.
1334  void addCommentHandler(CommentHandler *Handler);
1335 
1336  /// Remove the specified comment handler.
1337  ///
1338  /// It is an error to remove a handler that has not been registered.
1339  void removeCommentHandler(CommentHandler *Handler);
1340 
1341  /// Set the code completion handler to the given object.
1343  CodeComplete = &Handler;
1344  }
1345 
1346  /// Retrieve the current code-completion handler.
1348  return CodeComplete;
1349  }
1350 
1351  /// Clear out the code completion handler.
1353  CodeComplete = nullptr;
1354  }
1355 
1356  /// Hook used by the lexer to invoke the "included file" code
1357  /// completion point.
1358  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1359 
1360  /// Hook used by the lexer to invoke the "natural language" code
1361  /// completion point.
1363 
1364  /// Set the code completion token for filtering purposes.
1366  CodeCompletionII = Filter;
1367  }
1368 
1369  /// Set the code completion token range for detecting replacement range later
1370  /// on.
1372  const SourceLocation End) {
1373  CodeCompletionTokenRange = {Start, End};
1374  }
1376  return CodeCompletionTokenRange;
1377  }
1378 
1379  /// Get the code completion token for filtering purposes.
1381  if (CodeCompletionII)
1382  return CodeCompletionII->getName();
1383  return {};
1384  }
1385 
1386  /// Retrieve the preprocessing record, or NULL if there is no
1387  /// preprocessing record.
1388  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1389 
1390  /// Create a new preprocessing record, which will keep track of
1391  /// all macro expansions, macro definitions, etc.
1393 
1394  /// Returns true if the FileEntry is the PCH through header.
1395  bool isPCHThroughHeader(const FileEntry *FE);
1396 
1397  /// True if creating a PCH with a through header.
1399 
1400  /// True if using a PCH with a through header.
1402 
1403  /// True if creating a PCH with a #pragma hdrstop.
1405 
1406  /// True if using a PCH with a #pragma hdrstop.
1408 
1409  /// Skip tokens until after the #include of the through header or
1410  /// until after a #pragma hdrstop.
1411  void SkipTokensWhileUsingPCH();
1412 
1413  /// Process directives while skipping until the through header or
1414  /// #pragma hdrstop is found.
1416  SourceLocation HashLoc);
1417 
1418  /// Enter the specified FileID as the main source file,
1419  /// which implicitly adds the builtin defines etc.
1420  void EnterMainSourceFile();
1421 
1422  /// Inform the preprocessor callbacks that processing is complete.
1423  void EndSourceFile();
1424 
1425  /// Add a source file to the top of the include stack and
1426  /// start lexing tokens from it instead of the current buffer.
1427  ///
1428  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1430  SourceLocation Loc, bool IsFirstIncludeOfFile = true);
1431 
1432  /// Add a Macro to the top of the include stack and start lexing
1433  /// tokens from it instead of the current buffer.
1434  ///
1435  /// \param Args specifies the tokens input to a function-like macro.
1436  /// \param ILEnd specifies the location of the ')' for a function-like macro
1437  /// or the identifier for an object-like macro.
1438  void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1439  MacroArgs *Args);
1440 
1441 private:
1442  /// Add a "macro" context to the top of the include stack,
1443  /// which will cause the lexer to start returning the specified tokens.
1444  ///
1445  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1446  /// will not be subject to further macro expansion. Otherwise, these tokens
1447  /// will be re-macro-expanded when/if expansion is enabled.
1448  ///
1449  /// If \p OwnsTokens is false, this method assumes that the specified stream
1450  /// of tokens has a permanent owner somewhere, so they do not need to be
1451  /// copied. If it is true, it assumes the array of tokens is allocated with
1452  /// \c new[] and the Preprocessor will delete[] it.
1453  ///
1454  /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
1455  /// set, see the flag documentation for details.
1456  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1457  bool DisableMacroExpansion, bool OwnsTokens,
1458  bool IsReinject);
1459 
1460 public:
1461  void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1462  bool DisableMacroExpansion, bool IsReinject) {
1463  EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true,
1464  IsReinject);
1465  }
1466 
1467  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
1468  bool IsReinject) {
1469  EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false,
1470  IsReinject);
1471  }
1472 
1473  /// Pop the current lexer/macro exp off the top of the lexer stack.
1474  ///
1475  /// This should only be used in situations where the current state of the
1476  /// top-of-stack lexer is known.
1477  void RemoveTopOfLexerStack();
1478 
1479  /// From the point that this method is called, and until
1480  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1481  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1482  /// make the Preprocessor re-lex the same tokens.
1483  ///
1484  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1485  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1486  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1487  ///
1488  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1489  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1490  /// tokens will continue indefinitely.
1491  ///
1492  void EnableBacktrackAtThisPos();
1493 
1494  /// Disable the last EnableBacktrackAtThisPos call.
1495  void CommitBacktrackedTokens();
1496 
1497  /// Make Preprocessor re-lex the tokens that were lexed since
1498  /// EnableBacktrackAtThisPos() was previously called.
1499  void Backtrack();
1500 
1501  /// True if EnableBacktrackAtThisPos() was called and
1502  /// caching of tokens is on.
1503  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1504 
1505  /// Lex the next token for this preprocessor.
1506  void Lex(Token &Result);
1507 
1508  /// Lex a token, forming a header-name token if possible.
1509  bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
1510 
1511  bool LexAfterModuleImport(Token &Result);
1513 
1514  void makeModuleVisible(Module *M, SourceLocation Loc);
1515 
1517  return CurSubmoduleState->VisibleModules.getImportLoc(M);
1518  }
1519 
1520  /// Lex a string literal, which may be the concatenation of multiple
1521  /// string literals and may even come from macro expansion.
1522  /// \returns true on success, false if a error diagnostic has been generated.
1523  bool LexStringLiteral(Token &Result, std::string &String,
1524  const char *DiagnosticTag, bool AllowMacroExpansion) {
1525  if (AllowMacroExpansion)
1526  Lex(Result);
1527  else
1528  LexUnexpandedToken(Result);
1529  return FinishLexStringLiteral(Result, String, DiagnosticTag,
1530  AllowMacroExpansion);
1531  }
1532 
1533  /// Complete the lexing of a string literal where the first token has
1534  /// already been lexed (see LexStringLiteral).
1535  bool FinishLexStringLiteral(Token &Result, std::string &String,
1536  const char *DiagnosticTag,
1537  bool AllowMacroExpansion);
1538 
1539  /// Lex a token. If it's a comment, keep lexing until we get
1540  /// something not a comment.
1541  ///
1542  /// This is useful in -E -C mode where comments would foul up preprocessor
1543  /// directive handling.
1544  void LexNonComment(Token &Result) {
1545  do
1546  Lex(Result);
1547  while (Result.getKind() == tok::comment);
1548  }
1549 
1550  /// Just like Lex, but disables macro expansion of identifier tokens.
1551  void LexUnexpandedToken(Token &Result) {
1552  // Disable macro expansion.
1553  bool OldVal = DisableMacroExpansion;
1554  DisableMacroExpansion = true;
1555  // Lex the token.
1556  Lex(Result);
1557 
1558  // Reenable it.
1559  DisableMacroExpansion = OldVal;
1560  }
1561 
1562  /// Like LexNonComment, but this disables macro expansion of
1563  /// identifier tokens.
1565  do
1566  LexUnexpandedToken(Result);
1567  while (Result.getKind() == tok::comment);
1568  }
1569 
1570  /// Parses a simple integer literal to get its numeric value. Floating
1571  /// point literals and user defined literals are rejected. Used primarily to
1572  /// handle pragmas that accept integer arguments.
1574 
1575  /// Disables macro expansion everywhere except for preprocessor directives.
1577  DisableMacroExpansion = true;
1578  MacroExpansionInDirectivesOverride = true;
1579  }
1580 
1581  /// Peeks ahead N tokens and returns that token without consuming any
1582  /// tokens.
1583  ///
1584  /// LookAhead(0) returns the next token that would be returned by Lex(),
1585  /// LookAhead(1) returns the token after it, etc. This returns normal
1586  /// tokens after phase 5. As such, it is equivalent to using
1587  /// 'Lex', not 'LexUnexpandedToken'.
1588  const Token &LookAhead(unsigned N) {
1589  assert(LexLevel == 0 && "cannot use lookahead while lexing");
1590  if (CachedLexPos + N < CachedTokens.size())
1591  return CachedTokens[CachedLexPos+N];
1592  else
1593  return PeekAhead(N+1);
1594  }
1595 
1596  /// When backtracking is enabled and tokens are cached,
1597  /// this allows to revert a specific number of tokens.
1598  ///
1599  /// Note that the number of tokens being reverted should be up to the last
1600  /// backtrack position, not more.
1601  void RevertCachedTokens(unsigned N) {
1602  assert(isBacktrackEnabled() &&
1603  "Should only be called when tokens are cached for backtracking");
1604  assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1605  && "Should revert tokens up to the last backtrack position, not more");
1606  assert(signed(CachedLexPos) - signed(N) >= 0 &&
1607  "Corrupted backtrack positions ?");
1608  CachedLexPos -= N;
1609  }
1610 
1611  /// Enters a token in the token stream to be lexed next.
1612  ///
1613  /// If BackTrack() is called afterwards, the token will remain at the
1614  /// insertion point.
1615  /// If \p IsReinject is true, resulting token will have Token::IsReinjected
1616  /// flag set. See the flag documentation for details.
1617  void EnterToken(const Token &Tok, bool IsReinject) {
1618  if (LexLevel) {
1619  // It's not correct in general to enter caching lex mode while in the
1620  // middle of a nested lexing action.
1621  auto TokCopy = std::make_unique<Token[]>(1);
1622  TokCopy[0] = Tok;
1623  EnterTokenStream(std::move(TokCopy), 1, true, IsReinject);
1624  } else {
1625  EnterCachingLexMode();
1626  assert(IsReinject && "new tokens in the middle of cached stream");
1627  CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1628  }
1629  }
1630 
1631  /// We notify the Preprocessor that if it is caching tokens (because
1632  /// backtrack is enabled) it should replace the most recent cached tokens
1633  /// with the given annotation token. This function has no effect if
1634  /// backtracking is not enabled.
1635  ///
1636  /// Note that the use of this function is just for optimization, so that the
1637  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1638  /// invoked.
1639  void AnnotateCachedTokens(const Token &Tok) {
1640  assert(Tok.isAnnotation() && "Expected annotation token");
1641  if (CachedLexPos != 0 && isBacktrackEnabled())
1642  AnnotatePreviousCachedTokens(Tok);
1643  }
1644 
1645  /// Get the location of the last cached token, suitable for setting the end
1646  /// location of an annotation token.
1648  assert(CachedLexPos != 0);
1649  return CachedTokens[CachedLexPos-1].getLastLoc();
1650  }
1651 
1652  /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1653  /// CachedTokens.
1654  bool IsPreviousCachedToken(const Token &Tok) const;
1655 
1656  /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1657  /// in \p NewToks.
1658  ///
1659  /// Useful when a token needs to be split in smaller ones and CachedTokens
1660  /// most recent token must to be updated to reflect that.
1662 
1663  /// Replace the last token with an annotation token.
1664  ///
1665  /// Like AnnotateCachedTokens(), this routine replaces an
1666  /// already-parsed (and resolved) token with an annotation
1667  /// token. However, this routine only replaces the last token with
1668  /// the annotation token; it does not affect any other cached
1669  /// tokens. This function has no effect if backtracking is not
1670  /// enabled.
1672  assert(Tok.isAnnotation() && "Expected annotation token");
1673  if (CachedLexPos != 0 && isBacktrackEnabled())
1674  CachedTokens[CachedLexPos-1] = Tok;
1675  }
1676 
1677  /// Enter an annotation token into the token stream.
1679  void *AnnotationVal);
1680 
1681  /// Determine whether it's possible for a future call to Lex to produce an
1682  /// annotation token created by a previous call to EnterAnnotationToken.
1684  return CurLexerKind != CLK_Lexer;
1685  }
1686 
1687  /// Update the current token to represent the provided
1688  /// identifier, in order to cache an action performed by typo correction.
1689  void TypoCorrectToken(const Token &Tok) {
1690  assert(Tok.getIdentifierInfo() && "Expected identifier token");
1691  if (CachedLexPos != 0 && isBacktrackEnabled())
1692  CachedTokens[CachedLexPos-1] = Tok;
1693  }
1694 
1695  /// Recompute the current lexer kind based on the CurLexer/
1696  /// CurTokenLexer pointers.
1697  void recomputeCurLexerKind();
1698 
1699  /// Returns true if incremental processing is enabled
1700  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1701 
1702  /// Enables the incremental processing
1703  void enableIncrementalProcessing(bool value = true) {
1704  IncrementalProcessing = value;
1705  }
1706 
1707  /// Specify the point at which code-completion will be performed.
1708  ///
1709  /// \param File the file in which code completion should occur. If
1710  /// this file is included multiple times, code-completion will
1711  /// perform completion the first time it is included. If NULL, this
1712  /// function clears out the code-completion point.
1713  ///
1714  /// \param Line the line at which code completion should occur
1715  /// (1-based).
1716  ///
1717  /// \param Column the column at which code completion should occur
1718  /// (1-based).
1719  ///
1720  /// \returns true if an error occurred, false otherwise.
1721  bool SetCodeCompletionPoint(const FileEntry *File,
1722  unsigned Line, unsigned Column);
1723 
1724  /// Determine if we are performing code completion.
1725  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1726 
1727  /// Returns the location of the code-completion point.
1728  ///
1729  /// Returns an invalid location if code-completion is not enabled or the file
1730  /// containing the code-completion point has not been lexed yet.
1731  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1732 
1733  /// Returns the start location of the file of code-completion point.
1734  ///
1735  /// Returns an invalid location if code-completion is not enabled or the file
1736  /// containing the code-completion point has not been lexed yet.
1738  return CodeCompletionFileLoc;
1739  }
1740 
1741  /// Returns true if code-completion is enabled and we have hit the
1742  /// code-completion point.
1743  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1744 
1745  /// Note that we hit the code-completion point.
1747  assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1748  CodeCompletionReached = true;
1749  // Silence any diagnostics that occur after we hit the code-completion.
1751  }
1752 
1753  /// The location of the currently-active \#pragma clang
1754  /// arc_cf_code_audited begin.
1755  ///
1756  /// Returns an invalid location if there is no such pragma active.
1757  std::pair<IdentifierInfo *, SourceLocation>
1759  return PragmaARCCFCodeAuditedInfo;
1760  }
1761 
1762  /// Set the location of the currently-active \#pragma clang
1763  /// arc_cf_code_audited begin. An invalid location ends the pragma.
1765  SourceLocation Loc) {
1766  PragmaARCCFCodeAuditedInfo = {Ident, Loc};
1767  }
1768 
1769  /// The location of the currently-active \#pragma clang
1770  /// assume_nonnull begin.
1771  ///
1772  /// Returns an invalid location if there is no such pragma active.
1774  return PragmaAssumeNonNullLoc;
1775  }
1776 
1777  /// Set the location of the currently-active \#pragma clang
1778  /// assume_nonnull begin. An invalid location ends the pragma.
1780  PragmaAssumeNonNullLoc = Loc;
1781  }
1782 
1783  /// Get the location of the recorded unterminated \#pragma clang
1784  /// assume_nonnull begin in the preamble, if one exists.
1785  ///
1786  /// Returns an invalid location if the premable did not end with
1787  /// such a pragma active or if there is no recorded preamble.
1789  return PreambleRecordedPragmaAssumeNonNullLoc;
1790  }
1791 
1792  /// Record the location of the unterminated \#pragma clang
1793  /// assume_nonnull begin in the preamble.
1795  PreambleRecordedPragmaAssumeNonNullLoc = Loc;
1796  }
1797 
1798  /// Set the directory in which the main file should be considered
1799  /// to have been found, if it is not a real file.
1800  void setMainFileDir(const DirectoryEntry *Dir) {
1801  MainFileDir = Dir;
1802  }
1803 
1804  /// Instruct the preprocessor to skip part of the main source file.
1805  ///
1806  /// \param Bytes The number of bytes in the preamble to skip.
1807  ///
1808  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1809  /// start of a line.
1810  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1811  SkipMainFilePreamble.first = Bytes;
1812  SkipMainFilePreamble.second = StartOfLine;
1813  }
1814 
1815  /// Forwarding function for diagnostics. This emits a diagnostic at
1816  /// the specified Token's location, translating the token's start
1817  /// position in the current buffer into a SourcePosition object for rendering.
1818  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1819  return Diags->Report(Loc, DiagID);
1820  }
1821 
1822  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1823  return Diags->Report(Tok.getLocation(), DiagID);
1824  }
1825 
1826  /// Return the 'spelling' of the token at the given
1827  /// location; does not go up to the spelling location or down to the
1828  /// expansion location.
1829  ///
1830  /// \param buffer A buffer which will be used only if the token requires
1831  /// "cleaning", e.g. if it contains trigraphs or escaped newlines
1832  /// \param invalid If non-null, will be set \c true if an error occurs.
1834  SmallVectorImpl<char> &buffer,
1835  bool *invalid = nullptr) const {
1836  return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1837  }
1838 
1839  /// Return the 'spelling' of the Tok token.
1840  ///
1841  /// The spelling of a token is the characters used to represent the token in
1842  /// the source file after trigraph expansion and escaped-newline folding. In
1843  /// particular, this wants to get the true, uncanonicalized, spelling of
1844  /// things like digraphs, UCNs, etc.
1845  ///
1846  /// \param Invalid If non-null, will be set \c true if an error occurs.
1847  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1848  return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1849  }
1850 
1851  /// Get the spelling of a token into a preallocated buffer, instead
1852  /// of as an std::string.
1853  ///
1854  /// The caller is required to allocate enough space for the token, which is
1855  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1856  /// actual result is returned.
1857  ///
1858  /// Note that this method may do two possible things: it may either fill in
1859  /// the buffer specified with characters, or it may *change the input pointer*
1860  /// to point to a constant buffer with the data already in it (avoiding a
1861  /// copy). The caller is not allowed to modify the returned buffer pointer
1862  /// if an internal buffer is returned.
1863  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1864  bool *Invalid = nullptr) const {
1865  return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1866  }
1867 
1868  /// Get the spelling of a token into a SmallVector.
1869  ///
1870  /// Note that the returned StringRef may not point to the
1871  /// supplied buffer if a copy can be avoided.
1872  StringRef getSpelling(const Token &Tok,
1873  SmallVectorImpl<char> &Buffer,
1874  bool *Invalid = nullptr) const;
1875 
1876  /// Relex the token at the specified location.
1877  /// \returns true if there was a failure, false on success.
1878  bool getRawToken(SourceLocation Loc, Token &Result,
1879  bool IgnoreWhiteSpace = false) {
1880  return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1881  }
1882 
1883  /// Given a Token \p Tok that is a numeric constant with length 1,
1884  /// return the character.
1885  char
1887  bool *Invalid = nullptr) const {
1888  assert(Tok.is(tok::numeric_constant) &&
1889  Tok.getLength() == 1 && "Called on unsupported token");
1890  assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1891 
1892  // If the token is carrying a literal data pointer, just use it.
1893  if (const char *D = Tok.getLiteralData())
1894  return *D;
1895 
1896  // Otherwise, fall back on getCharacterData, which is slower, but always
1897  // works.
1898  return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1899  }
1900 
1901  /// Retrieve the name of the immediate macro expansion.
1902  ///
1903  /// This routine starts from a source location, and finds the name of the
1904  /// macro responsible for its immediate expansion. It looks through any
1905  /// intervening macro argument expansions to compute this. It returns a
1906  /// StringRef that refers to the SourceManager-owned buffer of the source
1907  /// where that macro name is spelled. Thus, the result shouldn't out-live
1908  /// the SourceManager.
1910  return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1911  }
1912 
1913  /// Plop the specified string into a scratch buffer and set the
1914  /// specified token's location and length to it.
1915  ///
1916  /// If specified, the source location provides a location of the expansion
1917  /// point of the token.
1918  void CreateString(StringRef Str, Token &Tok,
1919  SourceLocation ExpansionLocStart = SourceLocation(),
1920  SourceLocation ExpansionLocEnd = SourceLocation());
1921 
1922  /// Split the first Length characters out of the token starting at TokLoc
1923  /// and return a location pointing to the split token. Re-lexing from the
1924  /// split token will return the split token rather than the original.
1925  SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
1926 
1927  /// Computes the source location just past the end of the
1928  /// token at this source location.
1929  ///
1930  /// This routine can be used to produce a source location that
1931  /// points just past the end of the token referenced by \p Loc, and
1932  /// is generally used when a diagnostic needs to point just after a
1933  /// token where it expected something different that it received. If
1934  /// the returned source location would not be meaningful (e.g., if
1935  /// it points into a macro), this routine returns an invalid
1936  /// source location.
1937  ///
1938  /// \param Offset an offset from the end of the token, where the source
1939  /// location should refer to. The default offset (0) produces a source
1940  /// location pointing just past the end of the token; an offset of 1 produces
1941  /// a source location pointing to the last character in the token, etc.
1943  return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1944  }
1945 
1946  /// Returns true if the given MacroID location points at the first
1947  /// token of the macro expansion.
1948  ///
1949  /// \param MacroBegin If non-null and function returns true, it is set to
1950  /// begin location of the macro.
1952  SourceLocation *MacroBegin = nullptr) const {
1953  return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1954  MacroBegin);
1955  }
1956 
1957  /// Returns true if the given MacroID location points at the last
1958  /// token of the macro expansion.
1959  ///
1960  /// \param MacroEnd If non-null and function returns true, it is set to
1961  /// end location of the macro.
1963  SourceLocation *MacroEnd = nullptr) const {
1964  return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1965  }
1966 
1967  /// Print the token to stderr, used for debugging.
1968  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1969  void DumpLocation(SourceLocation Loc) const;
1970  void DumpMacro(const MacroInfo &MI) const;
1971  void dumpMacroInfo(const IdentifierInfo *II);
1972 
1973  /// Given a location that specifies the start of a
1974  /// token, return a new location that specifies a character within the token.
1976  unsigned Char) const {
1977  return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1978  }
1979 
1980  /// Increment the counters for the number of token paste operations
1981  /// performed.
1982  ///
1983  /// If fast was specified, this is a 'fast paste' case we handled.
1984  void IncrementPasteCounter(bool isFast) {
1985  if (isFast)
1986  ++NumFastTokenPaste;
1987  else
1988  ++NumTokenPaste;
1989  }
1990 
1991  void PrintStats();
1992 
1993  size_t getTotalMemory() const;
1994 
1995  /// When the macro expander pastes together a comment (/##/) in Microsoft
1996  /// mode, this method handles updating the current state, returning the
1997  /// token on the next source line.
1999 
2000  //===--------------------------------------------------------------------===//
2001  // Preprocessor callback methods. These are invoked by a lexer as various
2002  // directives and events are found.
2003 
2004  /// Given a tok::raw_identifier token, look up the
2005  /// identifier information for the token and install it into the token,
2006  /// updating the token kind accordingly.
2008 
2009 private:
2010  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
2011 
2012 public:
2013  /// Specifies the reason for poisoning an identifier.
2014  ///
2015  /// If that identifier is accessed while poisoned, then this reason will be
2016  /// used instead of the default "poisoned" diagnostic.
2017  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
2018 
2019  /// Display reason for poisoned identifier.
2021 
2023  if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
2024  if(II->isPoisoned()) {
2026  }
2027  }
2028  }
2029 
2030 private:
2031  /// Identifiers used for SEH handling in Borland. These are only
2032  /// allowed in particular circumstances
2033  // __except block
2034  IdentifierInfo *Ident__exception_code,
2035  *Ident___exception_code,
2036  *Ident_GetExceptionCode;
2037  // __except filter expression
2038  IdentifierInfo *Ident__exception_info,
2039  *Ident___exception_info,
2040  *Ident_GetExceptionInfo;
2041  // __finally
2042  IdentifierInfo *Ident__abnormal_termination,
2043  *Ident___abnormal_termination,
2044  *Ident_AbnormalTermination;
2045 
2046  const char *getCurLexerEndPos();
2047  void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
2048 
2049 public:
2050  void PoisonSEHIdentifiers(bool Poison = true); // Borland
2051 
2052  /// Callback invoked when the lexer reads an identifier and has
2053  /// filled in the tokens IdentifierInfo member.
2054  ///
2055  /// This callback potentially macro expands it or turns it into a named
2056  /// token (like 'for').
2057  ///
2058  /// \returns true if we actually computed a token, false if we need to
2059  /// lex again.
2061 
2062  /// Callback invoked when the lexer hits the end of the current file.
2063  ///
2064  /// This either returns the EOF token and returns true, or
2065  /// pops a level off the include stack and returns false, at which point the
2066  /// client should call lex again.
2067  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
2068 
2069  /// Callback invoked when the current TokenLexer hits the end of its
2070  /// token stream.
2071  bool HandleEndOfTokenLexer(Token &Result);
2072 
2073  /// Callback invoked when the lexer sees a # token at the start of a
2074  /// line.
2075  ///
2076  /// This consumes the directive, modifies the lexer/preprocessor state, and
2077  /// advances the lexer(s) so that the next token read is the correct one.
2078  void HandleDirective(Token &Result);
2079 
2080  /// Ensure that the next token is a tok::eod token.
2081  ///
2082  /// If not, emit a diagnostic and consume up until the eod.
2083  /// If \p EnableMacros is true, then we consider macros that expand to zero
2084  /// tokens as being ok.
2085  ///
2086  /// \return The location of the end of the directive (the terminating
2087  /// newline).
2088  SourceLocation CheckEndOfDirective(const char *DirType,
2089  bool EnableMacros = false);
2090 
2091  /// Read and discard all tokens remaining on the current line until
2092  /// the tok::eod token is found. Returns the range of the skipped tokens.
2094 
2095  /// Returns true if the preprocessor has seen a use of
2096  /// __DATE__ or __TIME__ in the file so far.
2097  bool SawDateOrTime() const {
2098  return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
2099  }
2100  unsigned getCounterValue() const { return CounterValue; }
2101  void setCounterValue(unsigned V) { CounterValue = V; }
2102 
2104  assert(CurrentFPEvalMethod != LangOptions::FEM_UnsetOnCommandLine &&
2105  "FPEvalMethod should be set either from command line or from the "
2106  "target info");
2107  return CurrentFPEvalMethod;
2108  }
2109 
2111  return TUFPEvalMethod;
2112  }
2113 
2115  return LastFPEvalPragmaLocation;
2116  }
2117 
2119  return LastFPEvalMethod;
2120  }
2121 
2123  LastFPEvalMethod = Val;
2124  }
2125 
2128  assert(Val != LangOptions::FEM_UnsetOnCommandLine &&
2129  "FPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2130  // This is the location of the '#pragma float_control" where the
2131  // execution state is modifed.
2132  LastFPEvalPragmaLocation = PragmaLoc;
2133  CurrentFPEvalMethod = Val;
2134  TUFPEvalMethod = Val;
2135  }
2136 
2138  assert(Val != LangOptions::FEM_UnsetOnCommandLine &&
2139  "TUPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2140  TUFPEvalMethod = Val;
2141  }
2142 
2143  /// Retrieves the module that we're currently building, if any.
2145 
2146  /// Allocate a new MacroInfo object with the provided SourceLocation.
2148 
2149  /// Turn the specified lexer token into a fully checked and spelled
2150  /// filename, e.g. as an operand of \#include.
2151  ///
2152  /// The caller is expected to provide a buffer that is large enough to hold
2153  /// the spelling of the filename, but is also expected to handle the case
2154  /// when this method decides to use a different buffer.
2155  ///
2156  /// \returns true if the input filename was in <>'s or false if it was
2157  /// in ""'s.
2158  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
2159 
2160  /// Given a "foo" or <foo> reference, look up the indicated file.
2161  ///
2162  /// Returns None on failure. \p isAngled indicates whether the file
2163  /// reference is for system \#include's or not (i.e. using <> instead of "").
2165  LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled,
2166  ConstSearchDirIterator FromDir, const FileEntry *FromFile,
2167  ConstSearchDirIterator *CurDir, SmallVectorImpl<char> *SearchPath,
2168  SmallVectorImpl<char> *RelativePath,
2169  ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped,
2170  bool *IsFrameworkFound, bool SkipCache = false);
2171 
2172  /// Return true if we're in the top-level file, not in a \#include.
2173  bool isInPrimaryFile() const;
2174 
2175  /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
2176  /// followed by EOD. Return true if the token is not a valid on-off-switch.
2177  bool LexOnOffSwitch(tok::OnOffSwitch &Result);
2178 
2179  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
2180  bool *ShadowFlag = nullptr);
2181 
2182  void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
2183  Module *LeaveSubmodule(bool ForPragma);
2184 
2185 private:
2186  friend void TokenLexer::ExpandFunctionArguments();
2187 
2188  void PushIncludeMacroStack() {
2189  assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
2190  IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
2191  std::move(CurLexer), CurPPLexer,
2192  std::move(CurTokenLexer), CurDirLookup);
2193  CurPPLexer = nullptr;
2194  }
2195 
2196  void PopIncludeMacroStack() {
2197  CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2198  CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2199  CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2200  CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2201  CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2202  CurLexerKind = IncludeMacroStack.back().CurLexerKind;
2203  IncludeMacroStack.pop_back();
2204  }
2205 
2206  void PropagateLineStartLeadingSpaceInfo(Token &Result);
2207 
2208  /// Determine whether we need to create module macros for #defines in the
2209  /// current context.
2210  bool needModuleMacros() const;
2211 
2212  /// Update the set of active module macros and ambiguity flag for a module
2213  /// macro name.
2214  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
2215 
2216  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2217  SourceLocation Loc);
2218  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2219  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2220  bool isPublic);
2221 
2222  /// Lex and validate a macro name, which occurs after a
2223  /// \#define or \#undef.
2224  ///
2225  /// \param MacroNameTok Token that represents the name defined or undefined.
2226  /// \param IsDefineUndef Kind if preprocessor directive.
2227  /// \param ShadowFlag Points to flag that is set if macro name shadows
2228  /// a keyword.
2229  ///
2230  /// This emits a diagnostic, sets the token kind to eod,
2231  /// and discards the rest of the macro line if the macro name is invalid.
2232  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
2233  bool *ShadowFlag = nullptr);
2234 
2235  /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
2236  /// entire line) of the macro's tokens and adds them to MacroInfo, and while
2237  /// doing so performs certain validity checks including (but not limited to):
2238  /// - # (stringization) is followed by a macro parameter
2239  /// \param MacroNameTok - Token that represents the macro name
2240  /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
2241  ///
2242  /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
2243  /// returns a nullptr if an invalid sequence of tokens is encountered.
2244  MacroInfo *ReadOptionalMacroParameterListAndBody(
2245  const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
2246 
2247  /// The ( starting an argument list of a macro definition has just been read.
2248  /// Lex the rest of the parameters and the closing ), updating \p MI with
2249  /// what we learn and saving in \p LastTok the last token read.
2250  /// Return true if an error occurs parsing the arg list.
2251  bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2252 
2253  /// Provide a suggestion for a typoed directive. If there is no typo, then
2254  /// just skip suggesting.
2255  ///
2256  /// \param Tok - Token that represents the directive
2257  /// \param Directive - String reference for the directive name
2258  void SuggestTypoedDirective(const Token &Tok, StringRef Directive) const;
2259 
2260  /// We just read a \#if or related directive and decided that the
2261  /// subsequent tokens are in the \#if'd out portion of the
2262  /// file. Lex the rest of the file, until we see an \#endif. If \p
2263  /// FoundNonSkipPortion is true, then we have already emitted code for part of
2264  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
2265  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
2266  /// already seen one so a \#else directive is a duplicate. When this returns,
2267  /// the caller can lex the first valid token.
2268  void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2269  SourceLocation IfTokenLoc,
2270  bool FoundNonSkipPortion, bool FoundElse,
2271  SourceLocation ElseLoc = SourceLocation());
2272 
2273  /// Information about the result for evaluating an expression for a
2274  /// preprocessor directive.
2275  struct DirectiveEvalResult {
2276  /// Whether the expression was evaluated as true or not.
2277  bool Conditional;
2278 
2279  /// True if the expression contained identifiers that were undefined.
2280  bool IncludedUndefinedIds;
2281 
2282  /// The source range for the expression.
2283  SourceRange ExprRange;
2284  };
2285 
2286  /// Evaluate an integer constant expression that may occur after a
2287  /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2288  ///
2289  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2290  DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
2291 
2292  /// Process a '__has_include("path")' expression.
2293  ///
2294  /// Returns true if successful.
2295  bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II);
2296 
2297  /// Process '__has_include_next("path")' expression.
2298  ///
2299  /// Returns true if successful.
2300  bool EvaluateHasIncludeNext(Token &Tok, IdentifierInfo *II);
2301 
2302  /// Get the directory and file from which to start \#include_next lookup.
2303  std::pair<ConstSearchDirIterator, const FileEntry *>
2304  getIncludeNextStart(const Token &IncludeNextTok) const;
2305 
2306  /// Install the standard preprocessor pragmas:
2307  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
2308  void RegisterBuiltinPragmas();
2309 
2310  /// Register builtin macros such as __LINE__ with the identifier table.
2311  void RegisterBuiltinMacros();
2312 
2313  /// If an identifier token is read that is to be expanded as a macro, handle
2314  /// it and return the next token as 'Tok'. If we lexed a token, return true;
2315  /// otherwise the caller should lex again.
2316  bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2317 
2318  /// Cache macro expanded tokens for TokenLexers.
2319  //
2320  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2321  /// going to lex in the cache and when it finishes the tokens are removed
2322  /// from the end of the cache.
2323  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2324  ArrayRef<Token> tokens);
2325 
2326  void removeCachedMacroExpandedTokensOfLastLexer();
2327 
2328  /// Determine whether the next preprocessor token to be
2329  /// lexed is a '('. If so, consume the token and return true, if not, this
2330  /// method should have no observable side-effect on the lexed tokens.
2331  bool isNextPPTokenLParen();
2332 
2333  /// After reading "MACRO(", this method is invoked to read all of the formal
2334  /// arguments specified for the macro invocation. Returns null on error.
2335  MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2336  SourceLocation &MacroEnd);
2337 
2338  /// If an identifier token is read that is to be expanded
2339  /// as a builtin macro, handle it and return the next token as 'Tok'.
2340  void ExpandBuiltinMacro(Token &Tok);
2341 
2342  /// Read a \c _Pragma directive, slice it up, process it, then
2343  /// return the first token after the directive.
2344  /// This assumes that the \c _Pragma token has just been read into \p Tok.
2345  void Handle_Pragma(Token &Tok);
2346 
2347  /// Like Handle_Pragma except the pragma text is not enclosed within
2348  /// a string literal.
2349  void HandleMicrosoft__pragma(Token &Tok);
2350 
2351  /// Add a lexer to the top of the include stack and
2352  /// start lexing tokens from it instead of the current buffer.
2353  void EnterSourceFileWithLexer(Lexer *TheLexer, ConstSearchDirIterator Dir);
2354 
2355  /// Set the FileID for the preprocessor predefines.
2356  void setPredefinesFileID(FileID FID) {
2357  assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2358  PredefinesFileID = FID;
2359  }
2360 
2361  /// Set the FileID for the PCH through header.
2362  void setPCHThroughHeaderFileID(FileID FID);
2363 
2364  /// Returns true if we are lexing from a file and not a
2365  /// pragma or a macro.
2366  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2367  return L ? !L->isPragmaLexer() : P != nullptr;
2368  }
2369 
2370  static bool IsFileLexer(const IncludeStackInfo& I) {
2371  return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2372  }
2373 
2374  bool IsFileLexer() const {
2375  return IsFileLexer(CurLexer.get(), CurPPLexer);
2376  }
2377 
2378  //===--------------------------------------------------------------------===//
2379  // Caching stuff.
2380  void CachingLex(Token &Result);
2381 
2382  bool InCachingLexMode() const {
2383  // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2384  // that we are past EOF, not that we are in CachingLex mode.
2385  return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2386  }
2387 
2388  void EnterCachingLexMode();
2389  void EnterCachingLexModeUnchecked();
2390 
2391  void ExitCachingLexMode() {
2392  if (InCachingLexMode())
2394  }
2395 
2396  const Token &PeekAhead(unsigned N);
2397  void AnnotatePreviousCachedTokens(const Token &Tok);
2398 
2399  //===--------------------------------------------------------------------===//
2400  /// Handle*Directive - implement the various preprocessor directives. These
2401  /// should side-effect the current preprocessor object so that the next call
2402  /// to Lex() will return the appropriate token next.
2403  void HandleLineDirective();
2404  void HandleDigitDirective(Token &Tok);
2405  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2406  void HandleIdentSCCSDirective(Token &Tok);
2407  void HandleMacroPublicDirective(Token &Tok);
2408  void HandleMacroPrivateDirective();
2409 
2410  /// An additional notification that can be produced by a header inclusion or
2411  /// import to tell the parser what happened.
2412  struct ImportAction {
2413  enum ActionKind {
2414  None,
2415  ModuleBegin,
2416  ModuleImport,
2417  SkippedModuleImport,
2418  Failure,
2419  } Kind;
2420  Module *ModuleForHeader = nullptr;
2421 
2422  ImportAction(ActionKind AK, Module *Mod = nullptr)
2423  : Kind(AK), ModuleForHeader(Mod) {
2424  assert((AK == None || Mod || AK == Failure) &&
2425  "no module for module action");
2426  }
2427  };
2428 
2429  Optional<FileEntryRef> LookupHeaderIncludeOrImport(
2430  ConstSearchDirIterator *CurDir, StringRef &Filename,
2431  SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2432  const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
2433  bool &IsMapped, ConstSearchDirIterator LookupFrom,
2434  const FileEntry *LookupFromFile, StringRef &LookupFilename,
2435  SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2436  ModuleMap::KnownHeader &SuggestedModule, bool isAngled);
2437 
2438  // File inclusion.
2439  void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2440  ConstSearchDirIterator LookupFrom = nullptr,
2441  const FileEntry *LookupFromFile = nullptr);
2442  ImportAction
2443  HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2444  Token &FilenameTok, SourceLocation EndLoc,
2445  ConstSearchDirIterator LookupFrom = nullptr,
2446  const FileEntry *LookupFromFile = nullptr);
2447  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2448  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2449  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2450  void HandleMicrosoftImportDirective(Token &Tok);
2451 
2452 public:
2453  /// Check that the given module is available, producing a diagnostic if not.
2454  /// \return \c true if the check failed (because the module is not available).
2455  /// \c false if the module appears to be usable.
2456  static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2457  const TargetInfo &TargetInfo,
2458  DiagnosticsEngine &Diags, Module *M);
2459 
2460  // Module inclusion testing.
2461  /// Find the module that owns the source or header file that
2462  /// \p Loc points to. If the location is in a file that was included
2463  /// into a module, or is outside any module, returns nullptr.
2464  Module *getModuleForLocation(SourceLocation Loc);
2465 
2466  /// We want to produce a diagnostic at location IncLoc concerning an
2467  /// unreachable effect at location MLoc (eg, where a desired entity was
2468  /// declared or defined). Determine whether the right way to make MLoc
2469  /// reachable is by #include, and if so, what header should be included.
2470  ///
2471  /// This is not necessarily fast, and might load unexpected module maps, so
2472  /// should only be called by code that intends to produce an error.
2473  ///
2474  /// \param IncLoc The location at which the missing effect was detected.
2475  /// \param MLoc A location within an unimported module at which the desired
2476  /// effect occurred.
2477  /// \return A file that can be #included to provide the desired effect. Null
2478  /// if no such file could be determined or if a #include is not
2479  /// appropriate (eg, if a module should be imported instead).
2480  const FileEntry *getHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2481  SourceLocation MLoc);
2482 
2483  bool isRecordingPreamble() const {
2484  return PreambleConditionalStack.isRecording();
2485  }
2486 
2487  bool hasRecordedPreamble() const {
2488  return PreambleConditionalStack.hasRecordedPreamble();
2489  }
2490 
2492  return PreambleConditionalStack.getStack();
2493  }
2494 
2496  PreambleConditionalStack.setStack(s);
2497  }
2498 
2501  PreambleConditionalStack.startReplaying();
2502  PreambleConditionalStack.setStack(s);
2503  PreambleConditionalStack.SkipInfo = SkipInfo;
2504  }
2505 
2507  return PreambleConditionalStack.SkipInfo;
2508  }
2509 
2510 private:
2511  /// After processing predefined file, initialize the conditional stack from
2512  /// the preamble.
2513  void replayPreambleConditionalStack();
2514 
2515  // Macro handling.
2516  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2517  void HandleUndefDirective();
2518 
2519  // Conditional Inclusion.
2520  void HandleIfdefDirective(Token &Result, const Token &HashToken,
2521  bool isIfndef, bool ReadAnyTokensBeforeDirective);
2522  void HandleIfDirective(Token &IfToken, const Token &HashToken,
2523  bool ReadAnyTokensBeforeDirective);
2524  void HandleEndifDirective(Token &EndifToken);
2525  void HandleElseDirective(Token &Result, const Token &HashToken);
2526  void HandleElifFamilyDirective(Token &ElifToken, const Token &HashToken,
2528 
2529  // Pragmas.
2530  void HandlePragmaDirective(PragmaIntroducer Introducer);
2531 
2532 public:
2533  void HandlePragmaOnce(Token &OnceTok);
2534  void HandlePragmaMark(Token &MarkTok);
2535  void HandlePragmaPoison();
2536  void HandlePragmaSystemHeader(Token &SysHeaderTok);
2537  void HandlePragmaDependency(Token &DependencyTok);
2538  void HandlePragmaPushMacro(Token &Tok);
2539  void HandlePragmaPopMacro(Token &Tok);
2540  void HandlePragmaIncludeAlias(Token &Tok);
2541  void HandlePragmaModuleBuild(Token &Tok);
2542  void HandlePragmaHdrstop(Token &Tok);
2544 
2545  // Return true and store the first token only if any CommentHandler
2546  // has inserted some tokens and getCommentRetentionState() is false.
2547  bool HandleComment(Token &result, SourceRange Comment);
2548 
2549  /// A macro is used, update information about macros that need unused
2550  /// warnings.
2551  void markMacroAsUsed(MacroInfo *MI);
2552 
2554  SourceLocation AnnotationLoc) {
2555  auto Annotations = AnnotationInfos.find(II);
2556  if (Annotations == AnnotationInfos.end())
2557  AnnotationInfos.insert(std::make_pair(
2558  II,
2559  MacroAnnotations::makeDeprecation(AnnotationLoc, std::move(Msg))));
2560  else
2561  Annotations->second.DeprecationInfo =
2562  MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2563  }
2564 
2566  SourceLocation AnnotationLoc) {
2567  auto Annotations = AnnotationInfos.find(II);
2568  if (Annotations == AnnotationInfos.end())
2569  AnnotationInfos.insert(
2570  std::make_pair(II, MacroAnnotations::makeRestrictExpansion(
2571  AnnotationLoc, std::move(Msg))));
2572  else
2573  Annotations->second.RestrictExpansionInfo =
2574  MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2575  }
2576 
2577  void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc) {
2578  auto Annotations = AnnotationInfos.find(II);
2579  if (Annotations == AnnotationInfos.end())
2580  AnnotationInfos.insert(
2581  std::make_pair(II, MacroAnnotations::makeFinal(AnnotationLoc)));
2582  else
2583  Annotations->second.FinalAnnotationLoc = AnnotationLoc;
2584  }
2585 
2586  const MacroAnnotations &getMacroAnnotations(const IdentifierInfo *II) const {
2587  return AnnotationInfos.find(II)->second;
2588  }
2589 
2591  if (Identifier.getIdentifierInfo()->isDeprecatedMacro())
2592  emitMacroDeprecationWarning(Identifier);
2593 
2594  if (Identifier.getIdentifierInfo()->isRestrictExpansion() &&
2595  !SourceMgr.isInMainFile(Identifier.getLocation()))
2596  emitRestrictExpansionWarning(Identifier);
2597  }
2598 
2600  const LangOptions &LangOpts,
2601  const TargetInfo &TI);
2602 
2603 private:
2604  void emitMacroDeprecationWarning(const Token &Identifier) const;
2605  void emitRestrictExpansionWarning(const Token &Identifier) const;
2606  void emitFinalMacroWarning(const Token &Identifier, bool IsUndef) const;
2607 };
2608 
2609 /// Abstract base class that describes a handler that will receive
2610 /// source ranges for each of the comments encountered in the source file.
2612 public:
2613  virtual ~CommentHandler();
2614 
2615  // The handler shall return true if it has pushed any tokens
2616  // to be read using e.g. EnterToken or EnterTokenStream.
2617  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2618 };
2619 
2620 /// Abstract base class that describes a handler that will receive
2621 /// source ranges for empty lines encountered in the source file.
2623 public:
2624  virtual ~EmptylineHandler();
2625 
2626  // The handler handles empty lines.
2627  virtual void HandleEmptyline(SourceRange Range) = 0;
2628 };
2629 
2630 /// Registry of pragma handlers added by plugins
2631 using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2632 
2633 } // namespace clang
2634 
2635 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H
clang::MacroDirective::DefInfo::getDirective
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:373
clang::Preprocessor::isParsingIfOrElifDirective
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
Definition: Preprocessor.h:1041
clang::Preprocessor::PreambleSkipInfo::ElseLoc
SourceLocation ElseLoc
Definition: Preprocessor.h:471
clang::Preprocessor::MaybeHandlePoisonedIdentifier
void MaybeHandlePoisonedIdentifier(Token &Identifier)
Definition: Preprocessor.h:2022
clang::VariadicMacroScopeGuard
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
Definition: VariadicMacroSupport.h:30
clang::Lexer::isAtEndOfMacroExpansion
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
Definition: Lexer.cpp:830
clang::Preprocessor::HandlePragmaPopMacro
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
Definition: Pragma.cpp:638
clang::IdentifierInfo::hasMacroDefinition
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
Definition: IdentifierTable.h:201
clang::Preprocessor::HandlePragmaSystemHeader
void HandlePragmaSystemHeader(Token &SysHeaderTok)
HandlePragmaSystemHeader - Implement #pragma GCC system_header.
Definition: Pragma.cpp:472
clang::TokenValue
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:91
clang::Preprocessor::setSkipMainFilePreamble
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
Definition: Preprocessor.h:1810
clang::prec::Conditional
@ Conditional
Definition: OperatorPrecedence.h:30
clang::Preprocessor::CreateString
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
Definition: Preprocessor.cpp:483
clang::Preprocessor::enableIncrementalProcessing
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
Definition: Preprocessor.h:1703
clang::Preprocessor::getCurrentFPEvalMethod
LangOptions::FPEvalMethodKind getCurrentFPEvalMethod() const
Definition: Preprocessor.h:2103
clang::Preprocessor::RemovePragmaHandler
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
Definition: Pragma.cpp:934
clang::Preprocessor::getSpelling
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
Definition: Preprocessor.h:1847
llvm
YAML serialization mapping.
Definition: Dominators.h:30
clang::HeaderSearch::getFileInfo
HeaderFileInfo & getFileInfo(const FileEntry *FE)
Return the HeaderFileInfo structure for the specified FileEntry, in preparation for updating it in so...
Definition: HeaderSearch.cpp:1280
clang::Preprocessor::AdvanceToTokenCharacter
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
Definition: Preprocessor.h:1975
clang::Preprocessor::CheckMacroName
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
Definition: PPDirectives.cpp:315
clang::Preprocessor::getPreprocessorAllocator
llvm::BumpPtrAllocator & getPreprocessorAllocator()
Definition: Preprocessor.h:1017
clang::ExternalPreprocessorSource
Abstract interface for external sources of preprocessor information.
Definition: ExternalPreprocessorSource.h:26
clang::Preprocessor::PrintStats
void PrintStats()
Definition: Preprocessor.cpp:268
clang::Preprocessor::FinalizeForModelFile
void FinalizeForModelFile()
Cleanup after model file parsing.
Definition: Preprocessor.cpp:223
clang::TokenValue::TokenValue
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:96
clang::DiagnosticBuilder
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:1265
clang::Preprocessor::setPragmaAssumeNonNullLoc
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
Definition: Preprocessor.h:1779
clang::Preprocessor::FinishLexStringLiteral
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
Definition: Preprocessor.cpp:1330
clang::Preprocessor::setCodeCompletionReached
void setCodeCompletionReached()
Note that we hit the code-completion point.
Definition: Preprocessor.h:1746
clang::ModuleMap::KnownHeader
A header that is known to reside within a given module, whether it was included or excluded.
Definition: ModuleMap.h:155
clang::Preprocessor::EnterTokenStream
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
Definition: Preprocessor.h:1467
clang::Preprocessor::LookAhead
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
Definition: Preprocessor.h:1588
clang::Preprocessor::setRecordedPreambleConditionalStack
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
Definition: Preprocessor.h:2495
clang::Preprocessor::getMacroDefinition
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:1158
clang::DeclaratorContext::File
@ File
clang::Preprocessor::SetCodeCompletionPoint
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
Definition: Preprocessor.cpp:389
clang::SourceRange
A trivial tuple used to represent a source range.
Definition: SourceLocation.h:210
string
string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} 0 ${PATH_LIB_START} PATH_HEAD) string(SUBSTRING $
Definition: CMakeLists.txt:22
clang::PreprocessorOptions
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
Definition: PreprocessorOptions.h:65
clang::Preprocessor::getModuleMacro
ModuleMacro * getModuleMacro(Module *Mod, const IdentifierInfo *II)
Definition: PPMacroExpansion.cpp:172
clang::Preprocessor::HandlePragmaMark
void HandlePragmaMark(Token &MarkTok)
Definition: Pragma.cpp:420
clang::Preprocessor::setCodeCompletionHandler
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
Definition: Preprocessor.h:1342
clang::Preprocessor::HandleEndOfTokenLexer
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
Definition: PPLexerChange.cpp:573
clang::Preprocessor::addModuleMacro
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
Definition: PPMacroExpansion.cpp:133
clang::Preprocessor::getLastMacroWithSpelling
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
Definition: Preprocessor.cpp:351
clang::Lexer::getRawToken
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
Definition: Lexer.cpp:461
clang::Preprocessor::getPPCallbacks
PPCallbacks * getPPCallbacks() const
Definition: Preprocessor.h:1103
clang::Preprocessor::setCodeCompletionTokenRange
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
Definition: Preprocessor.h:1371
clang::Preprocessor::isIncrementalProcessingEnabled
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
Definition: Preprocessor.h:1700
clang::Preprocessor::Lex
void Lex(Token &Result)
Lex the next token for this preprocessor.
Definition: Preprocessor.cpp:892
clang::Preprocessor::alreadyIncluded
bool alreadyIncluded(const FileEntry *File) const
Return true if this header has already been included.
Definition: Preprocessor.h:1281
clang::Preprocessor::HandleIdentifier
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
Definition: Preprocessor.cpp:800
clang::Preprocessor::getLocalMacroDirectiveHistory
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
Definition: PPMacroExpansion.cpp:64
ModuleMap.h
clang::Preprocessor::isAtEndOfMacroExpansion
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
Definition: Preprocessor.h:1962
clang::Preprocessor::setPreambleRecordedPragmaAssumeNonNullLoc
void setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc)
Record the location of the unterminated #pragma clang assume_nonnull begin in the preamble.
Definition: Preprocessor.h:1794
clang::Preprocessor::getCodeCompletionTokenRange
SourceRange getCodeCompletionTokenRange() const
Definition: Preprocessor.h:1375
clang::Preprocessor::macro_iterator
MacroMap::const_iterator macro_iterator
Definition: Preprocessor.h:1259
clang::Token::getLiteralData
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:216
clang::Preprocessor::emitMacroExpansionWarnings
void emitMacroExpansionWarnings(const Token &Identifier) const
Definition: Preprocessor.h:2590
clang::Lexer::getSpelling
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer,...
Definition: Lexer.cpp:403
clang::MacroDefinition
A description of the current definition of a macro.
Definition: MacroInfo.h:588
clang::Preprocessor::LexHeaderName
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
Definition: Preprocessor.cpp:995
clang::CommentHandler::HandleComment
virtual bool HandleComment(Preprocessor &PP, SourceRange Comment)=0
llvm::SmallVector
Definition: LLVM.h:38
clang::IdentifierTable::get
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
Definition: IdentifierTable.h:596
clang::Preprocessor::getMaxTokens
unsigned getMaxTokens() const
Get the max number of tokens before issuing a -Wmax-tokens warning.
Definition: Preprocessor.h:1116
clang::SourceLocation
Encodes a location in the source.
Definition: SourceLocation.h:86
clang::Preprocessor::TUKind
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
Definition: Preprocessor.h:290
clang::PPCallbacks
This interface provides a way to observe the actions of the preprocessor as it does its thing.
Definition: PPCallbacks.h:35
clang::Preprocessor::getCommentRetentionState
bool getCommentRetentionState() const
Definition: Preprocessor.h:1051
clang::driver::phases::Preprocess
@ Preprocess
Definition: Phases.h:18
clang::Preprocessor::RemoveTopOfLexerStack
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
Definition: PPLexerChange.cpp:594
clang::Preprocessor::IgnorePragmas
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
Definition: Pragma.cpp:2126
clang::MultiVersionKind::Target
@ Target
clang::Preprocessor::getModuleForLocation
Module * getModuleForLocation(SourceLocation Loc)
Find the module that owns the source or header file that Loc points to.
Definition: PPDirectives.cpp:859
clang::PreprocessorLexer
Definition: PreprocessorLexer.h:30
clang::tooling::Filter
llvm::cl::opt< std::string > Filter
clang::Token::getIdentifierInfo
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:178
clang::MacroArgs
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
Filename
StringRef Filename
Definition: Format.cpp:2572
clang::Preprocessor::setExternalSource
void setExternalSource(ExternalPreprocessorSource *Source)
Definition: Preprocessor.h:1019
clang::DiagnosticsEngine
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
clang::Preprocessor::LookupFile
Optional< FileEntryRef > LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
Definition: PPDirectives.cpp:946
clang::TargetInfo
Exposes information about the current target.
Definition: TargetInfo.h:189
clang::PreprocessingRecord
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
Definition: PreprocessingRecord.h:307
clang::Preprocessor::setLastFPEvalMethod
void setLastFPEvalMethod(LangOptions::FPEvalMethodKind Val)
Definition: Preprocessor.h:2122
Module.h
clang::IdentifierInfo::isOutOfDate
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
Definition: IdentifierTable.h:414
clang::Preprocessor::CodeCompleteIncludedFile
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
Definition: Preprocessor.cpp:445
llvm::Optional
Definition: LLVM.h:40
clang::Preprocessor::macro_end
macro_iterator macro_end(bool IncludeExternalMacros=true) const
Definition: Preprocessor.cpp:334
clang::ModuleLoader::HadFatalFailure
bool HadFatalFailure
Definition: ModuleLoader.h:157
clang::MacroInfo
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
SourceManager.h
clang::Preprocessor::getCurrentLexer
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
Definition: Preprocessor.h:1083
clang::Preprocessor::appendDefMacroDirective
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
Definition: Preprocessor.h:1226
clang::Preprocessor::setReplayablePreambleConditionalStack
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, llvm::Optional< PreambleSkipInfo > SkipInfo)
Definition: Preprocessor.h:2499
clang::TokenValue::operator==
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:106
clang::Preprocessor::EnterToken
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
Definition: Preprocessor.h:1617
clang::Preprocessor::HandleMicrosoftCommentPaste
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
Definition: PPLexerChange.cpp:611
clang::DiagnosticsEngine::setSuppressAllDiagnostics
void setSuppressAllDiagnostics(bool Val)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
Definition: Diagnostic.h:691
DiagnosticIDs.h
clang::Preprocessor::getCurrentFileLexer
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
Definition: PPLexerChange.cpp:49
clang::Preprocessor::IncrementPasteCounter
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Definition: Preprocessor.h:1984
clang::Preprocessor::MacroArgs
friend class MacroArgs
Definition: Preprocessor.h:485
Identifier
StringRef Identifier
Definition: Format.cpp:2580
clang::Token
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
clang::CommentHandler
Abstract base class that describes a handler that will receive source ranges for each of the comments...
Definition: Preprocessor.h:2611
clang::Preprocessor::AddPragmaHandler
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
Definition: Pragma.cpp:903
clang::Preprocessor::ReplaceLastTokenWithAnnotation
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
Definition: Preprocessor.h:1671
End
SourceLocation End
Definition: USRLocFinder.cpp:167
TokenLexer.h
clang::Preprocessor::ReplacePreviousCachedToken
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
Definition: PPCaching.cpp:157
clang::Preprocessor::setPredefines
void setPredefines(std::string P)
Set the predefines for this Preprocessor.
Definition: Preprocessor.h:1298
clang::Preprocessor::isCodeCompletionReached
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
Definition: Preprocessor.h:1743
clang::CodeCompletionHandler
Callback handler that receives notifications when performing code completion within the preprocessor.
Definition: CodeCompletionHandler.h:25
clang::Preprocessor::getIdentifierTable
IdentifierTable & getIdentifierTable()
Definition: Preprocessor.h:1013
clang::Preprocessor::addFinalLoc
void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc)
Definition: Preprocessor.h:2577
clang::Preprocessor::PreambleSkipInfo::PreambleSkipInfo
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
Definition: Preprocessor.h:473
clang::SourceManager
This class handles loading and caching of source files into memory.
Definition: SourceManager.h:627
clang::Preprocessor::setPragmasEnabled
void setPragmasEnabled(bool Enabled)
Definition: Preprocessor.h:1053
clang::Preprocessor::processPathForFileMacro
static void processPathForFileMacro(SmallVectorImpl< char > &Path, const LangOptions &LangOpts, const TargetInfo &TI)
Definition: PPMacroExpansion.cpp:1896
clang::Preprocessor::getLangOpts
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:1006
clang::Preprocessor::getImmediateMacroName
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
Definition: Preprocessor.h:1909
clang::HeaderSearch
Encapsulates the information needed to find the file referenced by a #include or #include_next,...
Definition: HeaderSearch.h:223
clang::Preprocessor::creatingPCHWithPragmaHdrStop
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
Definition: Preprocessor.cpp:614
clang::Preprocessor::AnnotateCachedTokens
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
Definition: Preprocessor.h:1639
Offset
unsigned Offset
Definition: Format.cpp:2574
uint64_t
unsigned long uint64_t
Definition: hlsl_basic_types.h:24
clang::Preprocessor::getPragmasEnabled
bool getPragmasEnabled() const
Definition: Preprocessor.h:1054
clang::Preprocessor::usingPCHWithThroughHeader
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
Definition: Preprocessor.cpp:609
BuiltinInfo
static const Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:21
TokenKinds.h
clang::Preprocessor::GetIncludeFilenameSpelling
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Buffer)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
Definition: PPDirectives.cpp:1741
clang::Preprocessor::getMacroDefinitionAtLoc
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
Definition: Preprocessor.h:1171
clang::Preprocessor::PreambleSkipInfo::HashTokenLoc
SourceLocation HashTokenLoc
Definition: Preprocessor.h:467
V
#define V(N, I)
Definition: ASTContext.h:3176
clang::Preprocessor::HandlePragmaOnce
void HandlePragmaOnce(Token &OnceTok)
HandlePragmaOnce - Handle #pragma once. OnceTok is the 'once'.
Definition: Pragma.cpp:405
clang::Preprocessor::AllocateMacroInfo
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
Definition: PPDirectives.cpp:59
clang::Preprocessor::SetPoisonReason
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
Definition: Preprocessor.cpp:735
clang::Module
Describes a module or submodule.
Definition: Module.h:96
clang::Preprocessor::TypoCorrectToken
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
Definition: Preprocessor.h:1689
clang::Preprocessor::EnterSourceFile
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
Definition: PPLexerChange.cpp:68
clang::Preprocessor::getBuildingSubmodules
ArrayRef< BuildingSubmoduleInfo > getBuildingSubmodules() const
Get the list of submodules that we're currently building.
Definition: Preprocessor.h:1251
clang::Preprocessor::getPreambleSkipInfo
llvm::Optional< PreambleSkipInfo > getPreambleSkipInfo() const
Definition: Preprocessor.h:2506
clang::Preprocessor::SetCommentRetentionState
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Definition: Preprocessor.h:1046
clang::Preprocessor::getIdentifierTable
const IdentifierTable & getIdentifierTable() const
Definition: Preprocessor.h:1014
clang::frontend::ActionKind
ActionKind
Definition: FrontendOptions.h:35
clang::Preprocessor::EndSourceFile
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
Definition: Preprocessor.cpp:687
clang::Preprocessor::ParsePragmaPushOrPopMacro
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.
Definition: Pragma.cpp:560
clang::Preprocessor::getCounterValue
unsigned getCounterValue() const
Definition: Preprocessor.h:2100
clang::Preprocessor::setLoadedMacroDirective
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
Definition: PPMacroExpansion.cpp:97
clang::DirectoryEntry
Cached information about one directory (either on disk or in the virtual file system).
Definition: DirectoryEntry.h:34
Id
int Id
Definition: ASTDiff.cpp:191
clang::Preprocessor::getSpellingOfSingleCharacterNumericConstant
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character.
Definition: Preprocessor.h:1886
clang::XRayInstrKind::None
constexpr XRayInstrMask None
Definition: XRayInstr.h:38
clang::Preprocessor::PreambleSkipInfo::FoundNonSkipPortion
bool FoundNonSkipPortion
Definition: Preprocessor.h:469
clang::Preprocessor::InitializeForModelFile
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
Definition: Preprocessor.cpp:211
clang::Preprocessor::setPreprocessToken
void setPreprocessToken(bool Preprocess)
Definition: Preprocessor.h:1132
clang::Preprocessor::addCommentHandler
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
Definition: Preprocessor.cpp:1391
clang::Token::getKind
tok::TokenKind getKind() const
Definition: Token.h:92
clang::index::SymbolKind::Module
@ Module
clang::Preprocessor::isAtStartOfMacroExpansion
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion.
Definition: Preprocessor.h:1951
clang::Preprocessor::getLastFPEvalPragmaLocation
SourceLocation getLastFPEvalPragmaLocation() const
Definition: Preprocessor.h:2114
LangOptions.h
clang::Preprocessor::isPCHThroughHeader
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
Definition: Preprocessor.cpp:598
Diagnostic.h
clang::Preprocessor::setTUFPEvalMethod
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
Definition: Preprocessor.h:2137
clang::LangOptions::FPEvalMethodKind
FPEvalMethodKind
Possible float expression evaluation method choices.
Definition: LangOptions.h:275
clang::Preprocessor::getSelectorTable
SelectorTable & getSelectorTable()
Definition: Preprocessor.h:1015
clang::Preprocessor::Diag
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
Definition: Preprocessor.h:1818
clang::Preprocessor::Diag
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
Definition: Preprocessor.h:1822
ModuleLoader.h
clang::Token::is
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:97
clang::Preprocessor::getTargetInfo
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:1007
clang::Preprocessor::createPreprocessingRecord
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
Definition: Preprocessor.cpp:1469
clang::Preprocessor::isMacroDefined
bool isMacroDefined(StringRef Id)
Definition: Preprocessor.h:1134
IdentifierTable.h
clang::Preprocessor::isBacktrackEnabled
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
Definition: Preprocessor.h:1503
clang::Preprocessor::EnterSubmodule
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
Definition: PPLexerChange.cpp:671
clang::Token::needsCleaning
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:284
clang::SelectorTable
This table allows us to fully hide how we implement multi-keyword caching.
Definition: IdentifierTable.h:897
bool
#define bool
Definition: stdbool.h:20
clang::Preprocessor::AddPragmaHandler
void AddPragmaHandler(PragmaHandler *Handler)
Definition: Preprocessor.h:1311
clang::Preprocessor::HandlePragmaPushMacro
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
Definition: Pragma.cpp:615
clang::Preprocessor::getCodeCompletionHandler
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
Definition: Preprocessor.h:1347
clang::Preprocessor::getIncludedFiles
IncludedFilesSet & getIncludedFiles()
Get the set of included files.
Definition: Preprocessor.h:1286
clang::Preprocessor::LexAfterModuleImport
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
Definition: Preprocessor.cpp:1138
clang::Preprocessor::HandleSkippedDirectiveWhileUsingPCH
void HandleSkippedDirectiveWhileUsingPCH(Token &Result, SourceLocation HashLoc)
Process directives while skipping until the through header or #pragma hdrstop is found.
Definition: PPDirectives.cpp:1105
llvm::DenseSet< const FileEntry * >
clang::Preprocessor::getMacroInfo
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
Definition: Preprocessor.h:1199
clang::PragmaIntroducer
Describes how and where the pragma was introduced.
Definition: Pragma.h:51
clang::EmptylineHandler
Abstract base class that describes a handler that will receive source ranges for empty lines encounte...
Definition: Preprocessor.h:2622
clang::CommentHandler::~CommentHandler
virtual ~CommentHandler()
clang::TokenValue::TokenValue
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:104
clang::Preprocessor::setCounterValue
void setCounterValue(unsigned V)
Definition: Preprocessor.h:2101
clang::Preprocessor::EnterMainSourceFile
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
Definition: Preprocessor.cpp:532
clang::Preprocessor::GetSuppressIncludeNotFoundError
bool GetSuppressIncludeNotFoundError()
Definition: Preprocessor.h:1060
clang::Lexer::getImmediateMacroName
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
Definition: Lexer.cpp:996
clang::EmptylineHandler::HandleEmptyline
virtual void HandleEmptyline(SourceRange Range)=0
clang::Preprocessor::getPreambleConditionalStack
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
Definition: Preprocessor.h:2491
clang::Preprocessor::getCodeCompletionFilter
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Definition: Preprocessor.h:1380
Line
const AnnotatedLine * Line
Definition: UsingDeclarationsSorter.cpp:68
clang::Preprocessor::getLastCachedTokenLocation
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
Definition: Preprocessor.h:1647
clang::Preprocessor::getRawToken
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
Definition: Preprocessor.h:1878
clang::FileEntry
Cached information about one file (either on disk or in the virtual file system).
Definition: FileEntry.h:330
clang::Preprocessor::SplitToken
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Definition: Preprocessor.cpp:503
clang::MU_Define
@ MU_Define
Definition: Preprocessor.h:118
clang::ConstSearchDirIterator
detail::SearchDirIteratorImpl< true > ConstSearchDirIterator
Definition: HeaderSearch.h:215
clang::Preprocessor::getLocalMacroDirective
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
Definition: Preprocessor.h:1188
clang::Preprocessor::LexUnexpandedNonComment
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
Definition: Preprocessor.h:1564
clang::Token::getLength
unsigned getLength() const
Definition: Token.h:128
clang::Preprocessor::getIncludedFiles
const IncludedFilesSet & getIncludedFiles() const
Definition: Preprocessor.h:1287
clang::PragmaHandler
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:65
clang::DefMacroDirective
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:430
clang::MU_Undef
@ MU_Undef
Definition: Preprocessor.h:121
clang::Preprocessor::getSourceManager
SourceManager & getSourceManager() const
Definition: Preprocessor.h:1010
clang::Preprocessor::HandlePragmaModuleBuild
void HandlePragmaModuleBuild(Token &Tok)
Definition: Pragma.cpp:795
clang::LangOptions::FEM_UnsetOnCommandLine
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
Definition: LangOptions.h:289
clang::transformer::EditKind::Range
@ Range
clang::Preprocessor::HandleComment
bool HandleComment(Token &result, SourceRange Comment)
Definition: Preprocessor.cpp:1405
clang::Preprocessor::LexOnOffSwitch
bool LexOnOffSwitch(tok::OnOffSwitch &Result)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD.
Definition: Pragma.cpp:956
clang::Preprocessor::getSpelling
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
Definition: Preprocessor.h:1833
clang::Preprocessor::PoisonSEHIdentifiers
void PoisonSEHIdentifiers(bool Poison=true)
Definition: Preprocessor.cpp:739
clang::Preprocessor::getTokenCount
unsigned getTokenCount() const
Get the number of tokens processed so far.
Definition: Preprocessor.h:1113
clang::Preprocessor::SetMacroExpansionOnlyInDirectives
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
Definition: Preprocessor.h:1576
clang::Preprocessor::PreambleSkipInfo::FoundElse
bool FoundElse
Definition: Preprocessor.h:470
clang::Preprocessor::CollectPpImportSuffix
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
Definition: Preprocessor.cpp:1084
clang::Preprocessor::getAuxTargetInfo
const TargetInfo * getAuxTargetInfo() const
Definition: Preprocessor.h:1008
SourceLocation.h
clang::Preprocessor::SkipTokensWhileUsingPCH
void SkipTokensWhileUsingPCH()
Skip tokens until after the include of the through header or until after a #pragma hdrstop.
Definition: Preprocessor.cpp:627
clang::Preprocessor::getTotalMemory
size_t getTotalMemory() const
Definition: Preprocessor.cpp:321
clang::tok::TokenKind
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
clang::Preprocessor::CodeCompleteNaturalLanguage
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
Definition: Preprocessor.cpp:452
P
StringRef P
Definition: ASTMatchersInternal.cpp:563
clang::detail::SearchDirIteratorImpl< true >
clang::Preprocessor::HandlePragmaPoison
void HandlePragmaPoison()
HandlePragmaPoison - Handle #pragma GCC poison. PoisonTok is the 'poison'.
Definition: Pragma.cpp:430
clang::Preprocessor::HandleDirective
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
Definition: PPDirectives.cpp:1130
clang::Preprocessor::getBuiltinInfo
Builtin::Context & getBuiltinInfo()
Definition: Preprocessor.h:1016
clang::Preprocessor::hadModuleLoaderFatalFailure
bool hadModuleLoaderFatalFailure() const
Definition: Preprocessor.h:1030
clang::Preprocessor::getCodeCompletionFileLoc
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
Definition: Preprocessor.h:1737
clang::Preprocessor::DumpLocation
void DumpLocation(SourceLocation Loc) const
Definition: Preprocessor.cpp:255
clang::Preprocessor::setPreprocessedOutput
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
Definition: Preprocessor.h:1066
clang::Preprocessor::usingPCHWithPragmaHdrStop
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
Definition: Preprocessor.cpp:618
clang::Preprocessor::isCodeCompletionEnabled
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Definition: Preprocessor.h:1725
clang::Preprocessor::getPragmaARCCFCodeAuditedInfo
std::pair< IdentifierInfo *, SourceLocation > getPragmaARCCFCodeAuditedInfo() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
Definition: Preprocessor.h:1758
clang::Preprocessor::dumpMacroInfo
void dumpMacroInfo(const IdentifierInfo *II)
Definition: PPMacroExpansion.cpp:259
llvm::ArrayRef
Definition: LLVM.h:34
Lexer.h
Value
Value
Definition: UninitializedValues.cpp:102
clang::Preprocessor::hasRecordedPreamble
bool hasRecordedPreamble() const
Definition: Preprocessor.h:2487
clang::Preprocessor::setCodeCompletionIdentifierInfo
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
Definition: Preprocessor.h:1365
clang::Preprocessor::setEmptylineHandler
void setEmptylineHandler(EmptylineHandler *Handler)
Set empty line handler.
Definition: Preprocessor.h:1329
clang::Preprocessor::EnableBacktrackAtThisPos
void EnableBacktrackAtThisPos()
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
Definition: PPCaching.cpp:25
clang::tok::PPKeywordKind
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a '#' at the beginning of the line.
Definition: TokenKinds.h:33
HeaderSearch.h
clang::Preprocessor::getPragmaAssumeNonNullLoc
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
Definition: Preprocessor.h:1773
clang::Preprocessor::PreambleSkipInfo::IfTokenLoc
SourceLocation IfTokenLoc
Definition: Preprocessor.h:468
clang::Preprocessor::getCurrentLexerSubmodule
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
Definition: Preprocessor.h:1093
clang::syntax::NodeRole::Message
@ Message
clang::Preprocessor::appendDefMacroDirective
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
Definition: Preprocessor.h:1220
LLVM.h
clang::Preprocessor::DumpToken
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
Definition: Preprocessor.cpp:229
clang::Preprocessor::markMacroAsUsed
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
Definition: PPMacroExpansion.cpp:1888
clang::Preprocessor::setCurrentFPEvalMethod
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
Definition: Preprocessor.h:2126
State
LineState State
Definition: UnwrappedLineFormatter.cpp:1126
clang::Preprocessor::isMacroDefined
bool isMacroDefined(const IdentifierInfo *II)
Definition: Preprocessor.h:1137
clang::Preprocessor::getFileManager
FileManager & getFileManager() const
Definition: Preprocessor.h:1009
clang::VAOptDefinitionContext
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a variadic mac...
Definition: VariadicMacroSupport.h:66
clang::Preprocessor::getCurrentModule
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
Definition: Preprocessor.cpp:519
clang::Preprocessor::Backtrack
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
Definition: PPCaching.cpp:40
clang::Preprocessor::isPreprocessedOutput
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
Definition: Preprocessor.h:1072
clang::Preprocessor::IsPreviousCachedToken
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
Definition: PPCaching.cpp:139
clang::ModuleLoader
Abstract interface for a module loader.
Definition: ModuleLoader.h:77
clang::IdentifierInfo
One of these records is kept for each identifier that is lexed.
Definition: IdentifierTable.h:84
clang::Token::getLocation
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:125
clang::Preprocessor::LexStringLiteral
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
Definition: Preprocessor.h:1523
clang::LangOptions
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:78
clang::ObjCPropertyAttribute::Kind
Kind
Definition: DeclObjCCommon.h:22
clang::Token::isAnnotation
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:119
clang::Preprocessor::removeCommentHandler
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Definition: Preprocessor.cpp:1398
clang::Preprocessor::getModuleImportLoc
SourceLocation getModuleImportLoc(Module *M) const
Definition: Preprocessor.h:1516
clang::Preprocessor::getPreprocessorOpts
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Definition: Preprocessor.h:1001
clang::ASTReader
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:341
clang::Preprocessor::IncludedFilesSet
llvm::DenseSet< const FileEntry * > IncludedFilesSet
Definition: Preprocessor.h:481
std
Definition: Format.h:4296
clang::Preprocessor::getModuleLoader
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
Definition: Preprocessor.h:1028
clang::TranslationUnitKind
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:874
clang::Preprocessor::EnterMacro
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
Definition: PPLexerChange.cpp:140
clang::Preprocessor::getSpelling
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
Definition: Preprocessor.h:1863
clang::IdentifierInfo::getName
StringRef getName() const
Return the actual identifier string.
Definition: IdentifierTable.h:195
clang::Preprocessor::getCodeCompletionLoc
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
Definition: Preprocessor.h:1731
clang::Preprocessor::CheckEndOfDirective
SourceLocation CheckEndOfDirective(const char *DirType, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
Definition: PPDirectives.cpp:416
clang::Preprocessor::clearCodeCompletionHandler
void clearCodeCompletionHandler()
Clear out the code completion handler.
Definition: Preprocessor.h:1352
clang
Definition: CalledOnceCheck.h:17
clang::FileManager
Implements support for file system lookup, file system caching, and directory search management.
Definition: FileManager.h:53
clang::Preprocessor::addPPCallbacks
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:1104
clang::Lexer::isAtStartOfMacroExpansion
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion.
Definition: Lexer.cpp:808
clang::Preprocessor::overrideMaxTokens
void overrideMaxTokens(unsigned Value, SourceLocation Loc)
Definition: Preprocessor.h:1118
clang::Preprocessor::getMaxTokensOverrideLoc
SourceLocation getMaxTokensOverrideLoc() const
Definition: Preprocessor.h:1123
PPCallbacks.h
clang::Preprocessor::markIncluded
bool markIncluded(const FileEntry *File)
Mark the file as included.
Definition: Preprocessor.h:1275
clang::Preprocessor::creatingPCHWithThroughHeader
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
Definition: Preprocessor.cpp:604
clang::Preprocessor::LookUpIdentifierInfo
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Definition: Preprocessor.cpp:700
clang::Builtin::Context
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:71
clang::IdentifierInfo::hadMacroDefinition
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
Definition: IdentifierTable.h:226
clang::PragmaHandlerRegistry
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Definition: Preprocessor.h:2631
MacroInfo.h
clang::MacroDirective
Encapsulates changes to the "macros namespace" (the location where the macro name became active,...
Definition: MacroInfo.h:314
clang::Preprocessor::HandlePragmaHdrstop
void HandlePragmaHdrstop(Token &Tok)
Definition: Pragma.cpp:869
clang::Preprocessor::getDiagnostics
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:1003
clang::Preprocessor::Preprocessor
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
Definition: Preprocessor.cpp:79
clang::EmptylineHandler::~EmptylineHandler
virtual ~EmptylineHandler()
clang::Preprocessor::getLastFPEvalMethod
LangOptions::FPEvalMethodKind getLastFPEvalMethod() const
Definition: Preprocessor.h:2118
clang::Preprocessor::getNumDirectives
unsigned getNumDirectives() const
Retrieve the number of Directives that have been processed by the Preprocessor.
Definition: Preprocessor.h:1036
clang::Preprocessor::SetSuppressIncludeNotFoundError
void SetSuppressIncludeNotFoundError(bool Suppress)
Definition: Preprocessor.h:1056
clang::Preprocessor::addRestrictExpansionMsg
void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
Definition: Preprocessor.h:2565
clang::Preprocessor::DiscardUntilEndOfDirective
SourceRange DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found.
Definition: PPDirectives.cpp:83
clang::Preprocessor::getLeafModuleMacros
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
Definition: Preprocessor.h:1241
clang::tok::OnOffSwitch
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:48
clang::FileID
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Definition: SourceLocation.h:38
clang::tok::isAnnotation
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.cpp:58
clang::tok::isLiteral
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:89
clang::Preprocessor::setDiagnostics
void setDiagnostics(DiagnosticsEngine &D)
Definition: Preprocessor.h:1004
clang::IdentifierTable
Implements an efficient mapping from strings to IdentifierInfo nodes.
Definition: IdentifierTable.h:563
s
__device__ __2f16 float bool s
Definition: __clang_hip_libdevice_declares.h:315
clang::Preprocessor::isInPrimaryFile
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
Definition: PPLexerChange.cpp:34
clang::Preprocessor::getLocForEndOfToken
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
Definition: Preprocessor.h:1942
clang::Preprocessor::makeModuleVisible
void makeModuleVisible(Module *M, SourceLocation Loc)
Definition: Preprocessor.cpp:1313
clang::Preprocessor::isCurrentLexer
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Definition: Preprocessor.h:1075
clang::Preprocessor::recomputeCurLexerKind
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Definition: Preprocessor.cpp:378
clang::Preprocessor::DumpMacro
void DumpMacro(const MacroInfo &MI) const
Definition: Preprocessor.cpp:259
clang::Preprocessor::addMacroDeprecationMsg
void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
Definition: Preprocessor.h:2553
clang::Preprocessor::HandlePragmaDependency
void HandlePragmaDependency(Token &DependencyTok)
HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
Definition: Pragma.cpp:504
clang::Preprocessor::macros
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
Definition: Preprocessor.h:1265
clang::VerifyDiagnosticConsumer::Directive
Directive - Abstract class representing a parsed verify directive.
Definition: VerifyDiagnosticConsumer.h:191
clang::Preprocessor::HandlePragmaIncludeAlias
void HandlePragmaIncludeAlias(Token &Tok)
Definition: Pragma.cpp:673
clang::Preprocessor::setPragmaARCCFCodeAuditedInfo
void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident, SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin.
Definition: Preprocessor.h:1764
clang::TU_Complete
@ TU_Complete
The translation unit is a complete translation unit.
Definition: LangOptions.h:876
clang::MU_Other
@ MU_Other
Definition: Preprocessor.h:115
clang::Preprocessor::RemovePragmaHandler
void RemovePragmaHandler(PragmaHandler *Handler)
Definition: Preprocessor.h:1321
clang::Preprocessor::LeaveSubmodule
Module * LeaveSubmodule(bool ForPragma)
Definition: PPLexerChange.cpp:748
clang::Preprocessor::getPreambleRecordedPragmaAssumeNonNullLoc
SourceLocation getPreambleRecordedPragmaAssumeNonNullLoc() const
Get the location of the recorded unterminated #pragma clang assume_nonnull begin in the preamble,...
Definition: Preprocessor.h:1788
clang::Preprocessor::setTokenWatcher
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream.
Definition: Preprocessor.h:1128
clang::Preprocessor::PreambleSkipInfo
Definition: Preprocessor.h:466
clang::Lexer::AdvanceToTokenCharacter
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Characters, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token,...
Definition: Lexer.h:399
clang::Lexer::getLocForEndOfToken
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
Definition: Lexer.cpp:786
clang::Preprocessor::checkModuleIsAvailable
static bool checkModuleIsAvailable(const LangOptions &LangOpts, const TargetInfo &TargetInfo, DiagnosticsEngine &Diags, Module *M)
Check that the given module is available, producing a diagnostic if not.
Definition: PPDirectives.cpp:1888
llvm::SmallVectorImpl
Definition: Randstruct.h:18
clang::Preprocessor::LexUnexpandedToken
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
Definition: Preprocessor.h:1551
clang::Preprocessor
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:130
clang::Preprocessor::isRecordingPreamble
bool isRecordingPreamble() const
Definition: Preprocessor.h:2483
SM
#define SM(sm)
Definition: Cuda.cpp:81
clang::Preprocessor::HandlePoisonedIdentifier
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
Definition: Preprocessor.cpp:753
clang::Preprocessor::appendMacroDirective
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
Definition: PPMacroExpansion.cpp:72
clang::Preprocessor::EnterTokenStream
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
Definition: Preprocessor.h:1461
clang::Preprocessor::getHeaderSearchInfo
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:1011
Token.h
clang::MacroDirective::DefInfo
Definition: MacroInfo.h:362
clang::Preprocessor::getEmptylineHandler
EmptylineHandler * getEmptylineHandler() const
Definition: Preprocessor.h:1331
clang::Preprocessor::SawDateOrTime
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
Definition: Preprocessor.h:2097
clang::Preprocessor::EnterAnnotationToken
void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind, void *AnnotationVal)
Enter an annotation token into the token stream.
Definition: PPDirectives.cpp:1789
clang::Preprocessor::getExternalSource
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:1023
clang::Preprocessor::mightHavePendingAnnotationTokens
bool mightHavePendingAnnotationTokens()
Determine whether it's possible for a future call to Lex to produce an annotation token created by a ...
Definition: Preprocessor.h:1683
clang::Preprocessor::getMacroAnnotations
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
Definition: Preprocessor.h:2586
clang::Preprocessor::~Preprocessor
~Preprocessor()
Definition: Preprocessor.cpp:164
clang::Preprocessor::LexNonComment
void LexNonComment(Token &Result)
Lex a token.
Definition: Preprocessor.h:1544
clang::Preprocessor::getPredefinesFileID
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:1096
clang::Preprocessor::getMacroInfo
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Definition: Preprocessor.h:1203
clang::MacroUse
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:113
clang::Preprocessor::RevertCachedTokens
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
Definition: Preprocessor.h:1601
clang::ModuleMacro
Represents a macro directive exported by a module.
Definition: MacroInfo.h:512
clang::Preprocessor::isMacroDefinedInLocalModule
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
Definition: Preprocessor.h:1145
clang::Preprocessor::setMainFileDir
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
Definition: Preprocessor.h:1800
clang::Preprocessor::getIdentifierInfo
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
Definition: Preprocessor.h:1302
clang::Preprocessor::macro_begin
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
Definition: Preprocessor.cpp:307
clang::Preprocessor::getHeaderToIncludeForDiagnostics
const FileEntry * getHeaderToIncludeForDiagnostics(SourceLocation IncLoc, SourceLocation MLoc)
We want to produce a diagnostic at location IncLoc concerning an unreachable effect at location MLoc ...
Definition: PPDirectives.cpp:880
clang::Preprocessor::parseSimpleIntegerLiteral
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
Definition: Preprocessor.cpp:1371
clang::Preprocessor::Initialize
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
Definition: Preprocessor.cpp:190
clang::DiagnosticsEngine::Report
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1536
clang::Preprocessor::CommitBacktrackedTokens
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
Definition: PPCaching.cpp:32
clang::Preprocessor::HandleEndOfFile
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
Definition: PPLexerChange.cpp:321
clang::Preprocessor::getTUFPEvalMethod
LangOptions::FPEvalMethodKind getTUFPEvalMethod() const
Definition: Preprocessor.h:2110
clang::Preprocessor::getPreprocessingRecord
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Definition: Preprocessor.h:1388
clang::MacroInfo::getDefinitionLoc
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:126