clang  16.0.0git
Preprocessor.h
Go to the documentation of this file.
1 //===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines the clang::Preprocessor interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15 #define LLVM_CLANG_LEX_PREPROCESSOR_H
16 
17 #include "clang/Basic/Diagnostic.h"
20 #include "clang/Basic/LLVM.h"
22 #include "clang/Basic/Module.h"
25 #include "clang/Basic/TokenKinds.h"
26 #include "clang/Lex/HeaderSearch.h"
27 #include "clang/Lex/Lexer.h"
28 #include "clang/Lex/MacroInfo.h"
29 #include "clang/Lex/ModuleLoader.h"
30 #include "clang/Lex/ModuleMap.h"
31 #include "clang/Lex/PPCallbacks.h"
32 #include "clang/Lex/Token.h"
33 #include "clang/Lex/TokenLexer.h"
34 #include "llvm/ADT/ArrayRef.h"
35 #include "llvm/ADT/DenseMap.h"
36 #include "llvm/ADT/FoldingSet.h"
37 #include "llvm/ADT/FunctionExtras.h"
38 #include "llvm/ADT/None.h"
39 #include "llvm/ADT/Optional.h"
40 #include "llvm/ADT/PointerUnion.h"
41 #include "llvm/ADT/STLExtras.h"
42 #include "llvm/ADT/SmallPtrSet.h"
43 #include "llvm/ADT/SmallVector.h"
44 #include "llvm/ADT/StringRef.h"
45 #include "llvm/ADT/TinyPtrVector.h"
46 #include "llvm/ADT/iterator_range.h"
47 #include "llvm/Support/Allocator.h"
48 #include "llvm/Support/Casting.h"
49 #include "llvm/Support/Registry.h"
50 #include <cassert>
51 #include <cstddef>
52 #include <cstdint>
53 #include <map>
54 #include <memory>
55 #include <string>
56 #include <utility>
57 #include <vector>
58 
59 namespace llvm {
60 
61 template<unsigned InternalLen> class SmallString;
62 
63 } // namespace llvm
64 
65 namespace clang {
66 
67 class CodeCompletionHandler;
68 class CommentHandler;
69 class DirectoryEntry;
70 class EmptylineHandler;
71 class ExternalPreprocessorSource;
72 class FileEntry;
73 class FileManager;
74 class HeaderSearch;
75 class MacroArgs;
76 class PragmaHandler;
77 class PragmaNamespace;
78 class PreprocessingRecord;
79 class PreprocessorLexer;
80 class PreprocessorOptions;
81 class ScratchBuffer;
82 class TargetInfo;
83 
84 namespace Builtin {
85 class Context;
86 }
87 
88 /// Stores token information for comparing actual tokens with
89 /// predefined values. Only handles simple tokens and identifiers.
90 class TokenValue {
91  tok::TokenKind Kind;
92  IdentifierInfo *II;
93 
94 public:
95  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
96  assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
97  assert(Kind != tok::identifier &&
98  "Identifiers should be created by TokenValue(IdentifierInfo *)");
99  assert(!tok::isLiteral(Kind) && "Literals are not supported.");
100  assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
101  }
102 
103  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
104 
105  bool operator==(const Token &Tok) const {
106  return Tok.getKind() == Kind &&
107  (!II || II == Tok.getIdentifierInfo());
108  }
109 };
110 
111 /// Context in which macro name is used.
112 enum MacroUse {
113  // other than #define or #undef
114  MU_Other = 0,
115 
116  // macro name specified in #define
118 
119  // macro name specified in #undef
121 };
122 
123 /// Engages in a tight little dance with the lexer to efficiently
124 /// preprocess tokens.
125 ///
126 /// Lexers know only about tokens within a single source file, and don't
127 /// know anything about preprocessor-level issues like the \#include stack,
128 /// token expansion, etc.
132 
133  llvm::unique_function<void(const clang::Token &)> OnToken;
134  std::shared_ptr<PreprocessorOptions> PPOpts;
135  DiagnosticsEngine *Diags;
136  LangOptions &LangOpts;
137  const TargetInfo *Target = nullptr;
138  const TargetInfo *AuxTarget = nullptr;
139  FileManager &FileMgr;
140  SourceManager &SourceMgr;
141  std::unique_ptr<ScratchBuffer> ScratchBuf;
142  HeaderSearch &HeaderInfo;
143  ModuleLoader &TheModuleLoader;
144 
145  /// External source of macros.
146  ExternalPreprocessorSource *ExternalSource;
147 
148  /// A BumpPtrAllocator object used to quickly allocate and release
149  /// objects internal to the Preprocessor.
150  llvm::BumpPtrAllocator BP;
151 
152  /// Identifiers for builtin macros and other builtins.
153  IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
154  IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
155  IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
156  IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
157  IdentifierInfo *Ident__FILE_NAME__; // __FILE_NAME__
158  IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
159  IdentifierInfo *Ident__COUNTER__; // __COUNTER__
160  IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
161  IdentifierInfo *Ident__identifier; // __identifier
162  IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
163  IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
164  IdentifierInfo *Ident__has_feature; // __has_feature
165  IdentifierInfo *Ident__has_extension; // __has_extension
166  IdentifierInfo *Ident__has_builtin; // __has_builtin
167  IdentifierInfo *Ident__has_constexpr_builtin; // __has_constexpr_builtin
168  IdentifierInfo *Ident__has_attribute; // __has_attribute
169  IdentifierInfo *Ident__has_include; // __has_include
170  IdentifierInfo *Ident__has_include_next; // __has_include_next
171  IdentifierInfo *Ident__has_warning; // __has_warning
172  IdentifierInfo *Ident__is_identifier; // __is_identifier
173  IdentifierInfo *Ident__building_module; // __building_module
174  IdentifierInfo *Ident__MODULE__; // __MODULE__
175  IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
176  IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
177  IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
178  IdentifierInfo *Ident__is_target_arch; // __is_target_arch
179  IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
180  IdentifierInfo *Ident__is_target_os; // __is_target_os
181  IdentifierInfo *Ident__is_target_environment; // __is_target_environment
182  IdentifierInfo *Ident__is_target_variant_os;
183  IdentifierInfo *Ident__is_target_variant_environment;
184  IdentifierInfo *Ident__FLT_EVAL_METHOD__; // __FLT_EVAL_METHOD
185 
186  // Weak, only valid (and set) while InMacroArgs is true.
187  Token* ArgMacro;
188 
189  SourceLocation DATELoc, TIMELoc;
190 
191  // FEM_UnsetOnCommandLine means that an explicit evaluation method was
192  // not specified on the command line. The target is queried to set the
193  // default evaluation method.
194  LangOptions::FPEvalMethodKind CurrentFPEvalMethod =
195  LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine;
196 
197  // Keeps the value of the last evaluation method before a
198  // `pragma float_control (precise,off) is applied.
199  LangOptions::FPEvalMethodKind LastFPEvalMethod =
200  LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine;
201 
202  // The most recent pragma location where the floating point evaluation
203  // method was modified. This is used to determine whether the
204  // 'pragma clang fp eval_method' was used whithin the current scope.
205  SourceLocation LastFPEvalPragmaLocation;
206 
207  LangOptions::FPEvalMethodKind TUFPEvalMethod =
208  LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine;
209 
210  // Next __COUNTER__ value, starts at 0.
211  unsigned CounterValue = 0;
212 
213  enum {
214  /// Maximum depth of \#includes.
215  MaxAllowedIncludeStackDepth = 200
216  };
217 
218  // State that is set before the preprocessor begins.
219  bool KeepComments : 1;
220  bool KeepMacroComments : 1;
221  bool SuppressIncludeNotFoundError : 1;
222 
223  // State that changes while the preprocessor runs:
224  bool InMacroArgs : 1; // True if parsing fn macro invocation args.
225 
226  /// Whether the preprocessor owns the header search object.
227  bool OwnsHeaderSearch : 1;
228 
229  /// True if macro expansion is disabled.
230  bool DisableMacroExpansion : 1;
231 
232  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
233  /// when parsing preprocessor directives.
234  bool MacroExpansionInDirectivesOverride : 1;
235 
236  class ResetMacroExpansionHelper;
237 
238  /// Whether we have already loaded macros from the external source.
239  mutable bool ReadMacrosFromExternalSource : 1;
240 
241  /// True if pragmas are enabled.
242  bool PragmasEnabled : 1;
243 
244  /// True if the current build action is a preprocessing action.
245  bool PreprocessedOutput : 1;
246 
247  /// True if we are currently preprocessing a #if or #elif directive
248  bool ParsingIfOrElifDirective;
249 
250  /// True if we are pre-expanding macro arguments.
251  bool InMacroArgPreExpansion;
252 
253  /// Mapping/lookup information for all identifiers in
254  /// the program, including program keywords.
255  mutable IdentifierTable Identifiers;
256 
257  /// This table contains all the selectors in the program.
258  ///
259  /// Unlike IdentifierTable above, this table *isn't* populated by the
260  /// preprocessor. It is declared/expanded here because its role/lifetime is
261  /// conceptually similar to the IdentifierTable. In addition, the current
262  /// control flow (in clang::ParseAST()), make it convenient to put here.
263  ///
264  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
265  /// the lifetime of the preprocessor.
266  SelectorTable Selectors;
267 
268  /// Information about builtins.
269  std::unique_ptr<Builtin::Context> BuiltinInfo;
270 
271  /// Tracks all of the pragmas that the client registered
272  /// with this preprocessor.
273  std::unique_ptr<PragmaNamespace> PragmaHandlers;
274 
275  /// Pragma handlers of the original source is stored here during the
276  /// parsing of a model file.
277  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
278 
279  /// Tracks all of the comment handlers that the client registered
280  /// with this preprocessor.
281  std::vector<CommentHandler *> CommentHandlers;
282 
283  /// Empty line handler.
284  EmptylineHandler *Emptyline = nullptr;
285 
286  /// True if we want to ignore EOF token and continue later on (thus
287  /// avoid tearing the Lexer and etc. down).
288  bool IncrementalProcessing = false;
289 
290 public:
291  /// The kind of translation unit we are processing.
293 
294 private:
295  /// The code-completion handler.
296  CodeCompletionHandler *CodeComplete = nullptr;
297 
298  /// The file that we're performing code-completion for, if any.
299  const FileEntry *CodeCompletionFile = nullptr;
300 
301  /// The offset in file for the code-completion point.
302  unsigned CodeCompletionOffset = 0;
303 
304  /// The location for the code-completion point. This gets instantiated
305  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
306  SourceLocation CodeCompletionLoc;
307 
308  /// The start location for the file of the code-completion point.
309  ///
310  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
311  /// for preprocessing.
312  SourceLocation CodeCompletionFileLoc;
313 
314  /// The source location of the \c import contextual keyword we just
315  /// lexed, if any.
316  SourceLocation ModuleImportLoc;
317 
318  /// The import path for named module that we're currently processing.
320 
321  /// Whether the last token we lexed was an '@'.
322  bool LastTokenWasAt = false;
323 
324  /// A position within a C++20 import-seq.
325  class StdCXXImportSeq {
326  public:
327  enum State : int {
328  // Positive values represent a number of unclosed brackets.
329  AtTopLevel = 0,
330  AfterTopLevelTokenSeq = -1,
331  AfterExport = -2,
332  AfterImportSeq = -3,
333  };
334 
335  StdCXXImportSeq(State S) : S(S) {}
336 
337  /// Saw any kind of open bracket.
338  void handleOpenBracket() {
339  S = static_cast<State>(std::max<int>(S, 0) + 1);
340  }
341  /// Saw any kind of close bracket other than '}'.
342  void handleCloseBracket() {
343  S = static_cast<State>(std::max<int>(S, 1) - 1);
344  }
345  /// Saw a close brace.
346  void handleCloseBrace() {
347  handleCloseBracket();
348  if (S == AtTopLevel && !AfterHeaderName)
349  S = AfterTopLevelTokenSeq;
350  }
351  /// Saw a semicolon.
352  void handleSemi() {
353  if (atTopLevel()) {
354  S = AfterTopLevelTokenSeq;
355  AfterHeaderName = false;
356  }
357  }
358 
359  /// Saw an 'export' identifier.
360  void handleExport() {
361  if (S == AfterTopLevelTokenSeq)
362  S = AfterExport;
363  else if (S <= 0)
364  S = AtTopLevel;
365  }
366  /// Saw an 'import' identifier.
367  void handleImport() {
368  if (S == AfterTopLevelTokenSeq || S == AfterExport)
369  S = AfterImportSeq;
370  else if (S <= 0)
371  S = AtTopLevel;
372  }
373 
374  /// Saw a 'header-name' token; do not recognize any more 'import' tokens
375  /// until we reach a top-level semicolon.
376  void handleHeaderName() {
377  if (S == AfterImportSeq)
378  AfterHeaderName = true;
379  handleMisc();
380  }
381 
382  /// Saw any other token.
383  void handleMisc() {
384  if (S <= 0)
385  S = AtTopLevel;
386  }
387 
388  bool atTopLevel() { return S <= 0; }
389  bool afterImportSeq() { return S == AfterImportSeq; }
390  bool afterTopLevelSeq() { return S == AfterTopLevelTokenSeq; }
391 
392  private:
393  State S;
394  /// Whether we're in the pp-import-suffix following the header-name in a
395  /// pp-import. If so, a close-brace is not sufficient to end the
396  /// top-level-token-seq of an import-seq.
397  bool AfterHeaderName = false;
398  };
399 
400  /// Our current position within a C++20 import-seq.
401  StdCXXImportSeq StdCXXImportSeqState = StdCXXImportSeq::AfterTopLevelTokenSeq;
402 
403  /// Track whether we are in a Global Module Fragment
404  class TrackGMF {
405  public:
406  enum GMFState : int {
407  GMFActive = 1,
408  MaybeGMF = 0,
409  BeforeGMFIntroducer = -1,
410  GMFAbsentOrEnded = -2,
411  };
412 
413  TrackGMF(GMFState S) : S(S) {}
414 
415  /// Saw a semicolon.
416  void handleSemi() {
417  // If it is immediately after the first instance of the module keyword,
418  // then that introduces the GMF.
419  if (S == MaybeGMF)
420  S = GMFActive;
421  }
422 
423  /// Saw an 'export' identifier.
424  void handleExport() {
425  // The presence of an 'export' keyword always ends or excludes a GMF.
426  S = GMFAbsentOrEnded;
427  }
428 
429  /// Saw an 'import' identifier.
430  void handleImport(bool AfterTopLevelTokenSeq) {
431  // If we see this before any 'module' kw, then we have no GMF.
432  if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
433  S = GMFAbsentOrEnded;
434  }
435 
436  /// Saw a 'module' identifier.
437  void handleModule(bool AfterTopLevelTokenSeq) {
438  // This was the first module identifier and not preceded by any token
439  // that would exclude a GMF. It could begin a GMF, but only if directly
440  // followed by a semicolon.
441  if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
442  S = MaybeGMF;
443  else
444  S = GMFAbsentOrEnded;
445  }
446 
447  /// Saw any other token.
448  void handleMisc() {
449  // We saw something other than ; after the 'module' kw, so not a GMF.
450  if (S == MaybeGMF)
451  S = GMFAbsentOrEnded;
452  }
453 
454  bool inGMF() { return S == GMFActive; }
455 
456  private:
457  /// Track the transitions into and out of a Global Module Fragment,
458  /// if one is present.
459  GMFState S;
460  };
461 
462  TrackGMF TrackGMFState = TrackGMF::BeforeGMFIntroducer;
463 
464  /// Whether the module import expects an identifier next. Otherwise,
465  /// it expects a '.' or ';'.
466  bool ModuleImportExpectsIdentifier = false;
467 
468  /// The identifier and source location of the currently-active
469  /// \#pragma clang arc_cf_code_audited begin.
470  std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
471 
472  /// The source location of the currently-active
473  /// \#pragma clang assume_nonnull begin.
474  SourceLocation PragmaAssumeNonNullLoc;
475 
476  /// Set only for preambles which end with an active
477  /// \#pragma clang assume_nonnull begin.
478  ///
479  /// When the preamble is loaded into the main file,
480  /// `PragmaAssumeNonNullLoc` will be set to this to
481  /// replay the unterminated assume_nonnull.
482  SourceLocation PreambleRecordedPragmaAssumeNonNullLoc;
483 
484  /// True if we hit the code-completion point.
485  bool CodeCompletionReached = false;
486 
487  /// The code completion token containing the information
488  /// on the stem that is to be code completed.
489  IdentifierInfo *CodeCompletionII = nullptr;
490 
491  /// Range for the code completion token.
492  SourceRange CodeCompletionTokenRange;
493 
494  /// The directory that the main file should be considered to occupy,
495  /// if it does not correspond to a real file (as happens when building a
496  /// module).
497  const DirectoryEntry *MainFileDir = nullptr;
498 
499  /// The number of bytes that we will initially skip when entering the
500  /// main file, along with a flag that indicates whether skipping this number
501  /// of bytes will place the lexer at the start of a line.
502  ///
503  /// This is used when loading a precompiled preamble.
504  std::pair<int, bool> SkipMainFilePreamble;
505 
506  /// Whether we hit an error due to reaching max allowed include depth. Allows
507  /// to avoid hitting the same error over and over again.
508  bool HasReachedMaxIncludeDepth = false;
509 
510  /// The number of currently-active calls to Lex.
511  ///
512  /// Lex is reentrant, and asking for an (end-of-phase-4) token can often
513  /// require asking for multiple additional tokens. This counter makes it
514  /// possible for Lex to detect whether it's producing a token for the end
515  /// of phase 4 of translation or for some other situation.
516  unsigned LexLevel = 0;
517 
518  /// The number of (LexLevel 0) preprocessor tokens.
519  unsigned TokenCount = 0;
520 
521  /// Preprocess every token regardless of LexLevel.
522  bool PreprocessToken = false;
523 
524  /// The maximum number of (LexLevel 0) tokens before issuing a -Wmax-tokens
525  /// warning, or zero for unlimited.
526  unsigned MaxTokens = 0;
527  SourceLocation MaxTokensOverrideLoc;
528 
529 public:
534  bool FoundElse;
536 
538  bool FoundNonSkipPortion, bool FoundElse,
542  ElseLoc(ElseLoc) {}
543  };
544 
546 
547 private:
548  friend class ASTReader;
549  friend class MacroArgs;
550 
551  class PreambleConditionalStackStore {
552  enum State {
553  Off = 0,
554  Recording = 1,
555  Replaying = 2,
556  };
557 
558  public:
559  PreambleConditionalStackStore() = default;
560 
561  void startRecording() { ConditionalStackState = Recording; }
562  void startReplaying() { ConditionalStackState = Replaying; }
563  bool isRecording() const { return ConditionalStackState == Recording; }
564  bool isReplaying() const { return ConditionalStackState == Replaying; }
565 
566  ArrayRef<PPConditionalInfo> getStack() const {
567  return ConditionalStack;
568  }
569 
570  void doneReplaying() {
571  ConditionalStack.clear();
572  ConditionalStackState = Off;
573  }
574 
575  void setStack(ArrayRef<PPConditionalInfo> s) {
576  if (!isRecording() && !isReplaying())
577  return;
578  ConditionalStack.clear();
579  ConditionalStack.append(s.begin(), s.end());
580  }
581 
582  bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
583 
584  bool reachedEOFWhileSkipping() const { return SkipInfo.has_value(); }
585 
586  void clearSkipInfo() { SkipInfo.reset(); }
587 
589 
590  private:
591  SmallVector<PPConditionalInfo, 4> ConditionalStack;
592  State ConditionalStackState = Off;
593  } PreambleConditionalStack;
594 
595  /// The current top of the stack that we're lexing from if
596  /// not expanding a macro and we are lexing directly from source code.
597  ///
598  /// Only one of CurLexer, or CurTokenLexer will be non-null.
599  std::unique_ptr<Lexer> CurLexer;
600 
601  /// The current top of the stack what we're lexing from
602  /// if not expanding a macro.
603  ///
604  /// This is an alias for CurLexer.
605  PreprocessorLexer *CurPPLexer = nullptr;
606 
607  /// Used to find the current FileEntry, if CurLexer is non-null
608  /// and if applicable.
609  ///
610  /// This allows us to implement \#include_next and find directory-specific
611  /// properties.
612  ConstSearchDirIterator CurDirLookup = nullptr;
613 
614  /// The current macro we are expanding, if we are expanding a macro.
615  ///
616  /// One of CurLexer and CurTokenLexer must be null.
617  std::unique_ptr<TokenLexer> CurTokenLexer;
618 
619  /// The kind of lexer we're currently working with.
620  enum CurLexerKind {
621  CLK_Lexer,
622  CLK_TokenLexer,
623  CLK_CachingLexer,
624  CLK_DependencyDirectivesLexer,
625  CLK_LexAfterModuleImport
626  } CurLexerKind = CLK_Lexer;
627 
628  /// If the current lexer is for a submodule that is being built, this
629  /// is that submodule.
630  Module *CurLexerSubmodule = nullptr;
631 
632  /// Keeps track of the stack of files currently
633  /// \#included, and macros currently being expanded from, not counting
634  /// CurLexer/CurTokenLexer.
635  struct IncludeStackInfo {
636  enum CurLexerKind CurLexerKind;
637  Module *TheSubmodule;
638  std::unique_ptr<Lexer> TheLexer;
639  PreprocessorLexer *ThePPLexer;
640  std::unique_ptr<TokenLexer> TheTokenLexer;
641  ConstSearchDirIterator TheDirLookup;
642 
643  // The following constructors are completely useless copies of the default
644  // versions, only needed to pacify MSVC.
645  IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
646  std::unique_ptr<Lexer> &&TheLexer,
647  PreprocessorLexer *ThePPLexer,
648  std::unique_ptr<TokenLexer> &&TheTokenLexer,
649  ConstSearchDirIterator TheDirLookup)
650  : CurLexerKind(std::move(CurLexerKind)),
651  TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
652  ThePPLexer(std::move(ThePPLexer)),
653  TheTokenLexer(std::move(TheTokenLexer)),
654  TheDirLookup(std::move(TheDirLookup)) {}
655  };
656  std::vector<IncludeStackInfo> IncludeMacroStack;
657 
658  /// Actions invoked when some preprocessor activity is
659  /// encountered (e.g. a file is \#included, etc).
660  std::unique_ptr<PPCallbacks> Callbacks;
661 
662  struct MacroExpandsInfo {
663  Token Tok;
664  MacroDefinition MD;
665  SourceRange Range;
666 
667  MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
668  : Tok(Tok), MD(MD), Range(Range) {}
669  };
670  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
671 
672  /// Information about a name that has been used to define a module macro.
673  struct ModuleMacroInfo {
674  /// The most recent macro directive for this identifier.
675  MacroDirective *MD;
676 
677  /// The active module macros for this identifier.
678  llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
679 
680  /// The generation number at which we last updated ActiveModuleMacros.
681  /// \see Preprocessor::VisibleModules.
682  unsigned ActiveModuleMacrosGeneration = 0;
683 
684  /// Whether this macro name is ambiguous.
685  bool IsAmbiguous = false;
686 
687  /// The module macros that are overridden by this macro.
688  llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
689 
690  ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
691  };
692 
693  /// The state of a macro for an identifier.
694  class MacroState {
695  mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
696 
697  ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
698  const IdentifierInfo *II) const {
699  if (II->isOutOfDate())
700  PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
701  // FIXME: Find a spare bit on IdentifierInfo and store a
702  // HasModuleMacros flag.
703  if (!II->hasMacroDefinition() ||
704  (!PP.getLangOpts().Modules &&
705  !PP.getLangOpts().ModulesLocalVisibility) ||
706  !PP.CurSubmoduleState->VisibleModules.getGeneration())
707  return nullptr;
708 
709  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
710  if (!Info) {
711  Info = new (PP.getPreprocessorAllocator())
712  ModuleMacroInfo(State.get<MacroDirective *>());
713  State = Info;
714  }
715 
716  if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
717  Info->ActiveModuleMacrosGeneration)
718  PP.updateModuleMacroInfo(II, *Info);
719  return Info;
720  }
721 
722  public:
723  MacroState() : MacroState(nullptr) {}
724  MacroState(MacroDirective *MD) : State(MD) {}
725 
726  MacroState(MacroState &&O) noexcept : State(O.State) {
727  O.State = (MacroDirective *)nullptr;
728  }
729 
730  MacroState &operator=(MacroState &&O) noexcept {
731  auto S = O.State;
732  O.State = (MacroDirective *)nullptr;
733  State = S;
734  return *this;
735  }
736 
737  ~MacroState() {
738  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
739  Info->~ModuleMacroInfo();
740  }
741 
742  MacroDirective *getLatest() const {
743  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
744  return Info->MD;
745  return State.get<MacroDirective*>();
746  }
747 
748  void setLatest(MacroDirective *MD) {
749  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
750  Info->MD = MD;
751  else
752  State = MD;
753  }
754 
755  bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
756  auto *Info = getModuleInfo(PP, II);
757  return Info ? Info->IsAmbiguous : false;
758  }
759 
760  ArrayRef<ModuleMacro *>
761  getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
762  if (auto *Info = getModuleInfo(PP, II))
763  return Info->ActiveModuleMacros;
764  return None;
765  }
766 
767  MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
768  SourceManager &SourceMgr) const {
769  // FIXME: Incorporate module macros into the result of this.
770  if (auto *Latest = getLatest())
771  return Latest->findDirectiveAtLoc(Loc, SourceMgr);
772  return {};
773  }
774 
775  void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
776  if (auto *Info = getModuleInfo(PP, II)) {
777  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
778  Info->ActiveModuleMacros.begin(),
779  Info->ActiveModuleMacros.end());
780  Info->ActiveModuleMacros.clear();
781  Info->IsAmbiguous = false;
782  }
783  }
784 
785  ArrayRef<ModuleMacro*> getOverriddenMacros() const {
786  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
787  return Info->OverriddenMacros;
788  return None;
789  }
790 
791  void setOverriddenMacros(Preprocessor &PP,
792  ArrayRef<ModuleMacro *> Overrides) {
793  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
794  if (!Info) {
795  if (Overrides.empty())
796  return;
797  Info = new (PP.getPreprocessorAllocator())
798  ModuleMacroInfo(State.get<MacroDirective *>());
799  State = Info;
800  }
801  Info->OverriddenMacros.clear();
802  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
803  Overrides.begin(), Overrides.end());
804  Info->ActiveModuleMacrosGeneration = 0;
805  }
806  };
807 
808  /// For each IdentifierInfo that was associated with a macro, we
809  /// keep a mapping to the history of all macro definitions and #undefs in
810  /// the reverse order (the latest one is in the head of the list).
811  ///
812  /// This mapping lives within the \p CurSubmoduleState.
813  using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
814 
815  struct SubmoduleState;
816 
817  /// Information about a submodule that we're currently building.
818  struct BuildingSubmoduleInfo {
819  /// The module that we are building.
820  Module *M;
821 
822  /// The location at which the module was included.
823  SourceLocation ImportLoc;
824 
825  /// Whether we entered this submodule via a pragma.
826  bool IsPragma;
827 
828  /// The previous SubmoduleState.
829  SubmoduleState *OuterSubmoduleState;
830 
831  /// The number of pending module macro names when we started building this.
832  unsigned OuterPendingModuleMacroNames;
833 
834  BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
835  SubmoduleState *OuterSubmoduleState,
836  unsigned OuterPendingModuleMacroNames)
837  : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
838  OuterSubmoduleState(OuterSubmoduleState),
839  OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
840  };
841  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
842 
843  /// Information about a submodule's preprocessor state.
844  struct SubmoduleState {
845  /// The macros for the submodule.
846  MacroMap Macros;
847 
848  /// The set of modules that are visible within the submodule.
849  VisibleModuleSet VisibleModules;
850 
851  // FIXME: CounterValue?
852  // FIXME: PragmaPushMacroInfo?
853  };
854  std::map<Module *, SubmoduleState> Submodules;
855 
856  /// The preprocessor state for preprocessing outside of any submodule.
857  SubmoduleState NullSubmoduleState;
858 
859  /// The current submodule state. Will be \p NullSubmoduleState if we're not
860  /// in a submodule.
861  SubmoduleState *CurSubmoduleState;
862 
863  /// The files that have been included.
864  IncludedFilesSet IncludedFiles;
865 
866  /// The set of top-level modules that affected preprocessing, but were not
867  /// imported.
868  llvm::SmallSetVector<Module *, 2> AffectingClangModules;
869 
870  /// The set of known macros exported from modules.
871  llvm::FoldingSet<ModuleMacro> ModuleMacros;
872 
873  /// The names of potential module macros that we've not yet processed.
874  llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
875 
876  /// The list of module macros, for each identifier, that are not overridden by
877  /// any other module macro.
878  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
879  LeafModuleMacros;
880 
881  /// Macros that we want to warn because they are not used at the end
882  /// of the translation unit.
883  ///
884  /// We store just their SourceLocations instead of
885  /// something like MacroInfo*. The benefit of this is that when we are
886  /// deserializing from PCH, we don't need to deserialize identifier & macros
887  /// just so that we can report that they are unused, we just warn using
888  /// the SourceLocations of this set (that will be filled by the ASTReader).
889  using WarnUnusedMacroLocsTy = llvm::SmallDenseSet<SourceLocation, 32>;
890  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
891 
892  /// This is a pair of an optional message and source location used for pragmas
893  /// that annotate macros like pragma clang restrict_expansion and pragma clang
894  /// deprecated. This pair stores the optional message and the location of the
895  /// annotation pragma for use producing diagnostics and notes.
896  using MsgLocationPair = std::pair<std::string, SourceLocation>;
897 
898  struct MacroAnnotationInfo {
899  SourceLocation Location;
901  };
902 
903  struct MacroAnnotations {
904  llvm::Optional<MacroAnnotationInfo> DeprecationInfo;
905  llvm::Optional<MacroAnnotationInfo> RestrictExpansionInfo;
906  llvm::Optional<SourceLocation> FinalAnnotationLoc;
907 
908  static MacroAnnotations makeDeprecation(SourceLocation Loc,
909  std::string Msg) {
910  return MacroAnnotations{MacroAnnotationInfo{Loc, std::move(Msg)},
911  llvm::None, llvm::None};
912  }
913 
914  static MacroAnnotations makeRestrictExpansion(SourceLocation Loc,
915  std::string Msg) {
916  return MacroAnnotations{
917  llvm::None, MacroAnnotationInfo{Loc, std::move(Msg)}, llvm::None};
918  }
919 
920  static MacroAnnotations makeFinal(SourceLocation Loc) {
921  return MacroAnnotations{llvm::None, llvm::None, Loc};
922  }
923  };
924 
925  /// Warning information for macro annotations.
926  llvm::DenseMap<const IdentifierInfo *, MacroAnnotations> AnnotationInfos;
927 
928  /// A "freelist" of MacroArg objects that can be
929  /// reused for quick allocation.
930  MacroArgs *MacroArgCache = nullptr;
931 
932  /// For each IdentifierInfo used in a \#pragma push_macro directive,
933  /// we keep a MacroInfo stack used to restore the previous macro value.
934  llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
935  PragmaPushMacroInfo;
936 
937  // Various statistics we track for performance analysis.
938  unsigned NumDirectives = 0;
939  unsigned NumDefined = 0;
940  unsigned NumUndefined = 0;
941  unsigned NumPragma = 0;
942  unsigned NumIf = 0;
943  unsigned NumElse = 0;
944  unsigned NumEndif = 0;
945  unsigned NumEnteredSourceFiles = 0;
946  unsigned MaxIncludeStackDepth = 0;
947  unsigned NumMacroExpanded = 0;
948  unsigned NumFnMacroExpanded = 0;
949  unsigned NumBuiltinMacroExpanded = 0;
950  unsigned NumFastMacroExpanded = 0;
951  unsigned NumTokenPaste = 0;
952  unsigned NumFastTokenPaste = 0;
953  unsigned NumSkipped = 0;
954 
955  /// The predefined macros that preprocessor should use from the
956  /// command line etc.
957  std::string Predefines;
958 
959  /// The file ID for the preprocessor predefines.
960  FileID PredefinesFileID;
961 
962  /// The file ID for the PCH through header.
963  FileID PCHThroughHeaderFileID;
964 
965  /// Whether tokens are being skipped until a #pragma hdrstop is seen.
966  bool SkippingUntilPragmaHdrStop = false;
967 
968  /// Whether tokens are being skipped until the through header is seen.
969  bool SkippingUntilPCHThroughHeader = false;
970 
971  /// \{
972  /// Cache of macro expanders to reduce malloc traffic.
973  enum { TokenLexerCacheSize = 8 };
974  unsigned NumCachedTokenLexers;
975  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
976  /// \}
977 
978  /// Keeps macro expanded tokens for TokenLexers.
979  //
980  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
981  /// going to lex in the cache and when it finishes the tokens are removed
982  /// from the end of the cache.
983  SmallVector<Token, 16> MacroExpandedTokens;
984  std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
985 
986  /// A record of the macro definitions and expansions that
987  /// occurred during preprocessing.
988  ///
989  /// This is an optional side structure that can be enabled with
990  /// \c createPreprocessingRecord() prior to preprocessing.
991  PreprocessingRecord *Record = nullptr;
992 
993  /// Cached tokens state.
994  using CachedTokensTy = SmallVector<Token, 1>;
995 
996  /// Cached tokens are stored here when we do backtracking or
997  /// lookahead. They are "lexed" by the CachingLex() method.
998  CachedTokensTy CachedTokens;
999 
1000  /// The position of the cached token that CachingLex() should
1001  /// "lex" next.
1002  ///
1003  /// If it points beyond the CachedTokens vector, it means that a normal
1004  /// Lex() should be invoked.
1005  CachedTokensTy::size_type CachedLexPos = 0;
1006 
1007  /// Stack of backtrack positions, allowing nested backtracks.
1008  ///
1009  /// The EnableBacktrackAtThisPos() method pushes a position to
1010  /// indicate where CachedLexPos should be set when the BackTrack() method is
1011  /// invoked (at which point the last position is popped).
1012  std::vector<CachedTokensTy::size_type> BacktrackPositions;
1013 
1014  /// True if \p Preprocessor::SkipExcludedConditionalBlock() is running.
1015  /// This is used to guard against calling this function recursively.
1016  ///
1017  /// See comments at the use-site for more context about why it is needed.
1018  bool SkippingExcludedConditionalBlock = false;
1019 
1020  /// Keeps track of skipped range mappings that were recorded while skipping
1021  /// excluded conditional directives. It maps the source buffer pointer at
1022  /// the beginning of a skipped block, to the number of bytes that should be
1023  /// skipped.
1024  llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges;
1025 
1026  void updateOutOfDateIdentifier(IdentifierInfo &II) const;
1027 
1028 public:
1029  Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
1030  DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM,
1031  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
1032  IdentifierInfoLookup *IILookup = nullptr,
1033  bool OwnsHeaderSearch = false,
1035 
1036  ~Preprocessor();
1037 
1038  /// Initialize the preprocessor using information about the target.
1039  ///
1040  /// \param Target is owned by the caller and must remain valid for the
1041  /// lifetime of the preprocessor.
1042  /// \param AuxTarget is owned by the caller and must remain valid for
1043  /// the lifetime of the preprocessor.
1044  void Initialize(const TargetInfo &Target,
1045  const TargetInfo *AuxTarget = nullptr);
1046 
1047  /// Initialize the preprocessor to parse a model file
1048  ///
1049  /// To parse model files the preprocessor of the original source is reused to
1050  /// preserver the identifier table. However to avoid some duplicate
1051  /// information in the preprocessor some cleanup is needed before it is used
1052  /// to parse model files. This method does that cleanup.
1053  void InitializeForModelFile();
1054 
1055  /// Cleanup after model file parsing
1056  void FinalizeForModelFile();
1057 
1058  /// Retrieve the preprocessor options used to initialize this
1059  /// preprocessor.
1060  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
1061 
1062  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
1063  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
1064 
1065  const LangOptions &getLangOpts() const { return LangOpts; }
1066  const TargetInfo &getTargetInfo() const { return *Target; }
1067  const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
1068  FileManager &getFileManager() const { return FileMgr; }
1069  SourceManager &getSourceManager() const { return SourceMgr; }
1070  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
1071 
1072  IdentifierTable &getIdentifierTable() { return Identifiers; }
1073  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
1074  SelectorTable &getSelectorTable() { return Selectors; }
1075  Builtin::Context &getBuiltinInfo() { return *BuiltinInfo; }
1076  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
1077 
1079  ExternalSource = Source;
1080  }
1081 
1083  return ExternalSource;
1084  }
1085 
1086  /// Retrieve the module loader associated with this preprocessor.
1087  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
1088 
1090  return TheModuleLoader.HadFatalFailure;
1091  }
1092 
1093  /// Retrieve the number of Directives that have been processed by the
1094  /// Preprocessor.
1095  unsigned getNumDirectives() const {
1096  return NumDirectives;
1097  }
1098 
1099  /// True if we are currently preprocessing a #if or #elif directive
1101  return ParsingIfOrElifDirective;
1102  }
1103 
1104  /// Control whether the preprocessor retains comments in output.
1105  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
1106  this->KeepComments = KeepComments | KeepMacroComments;
1107  this->KeepMacroComments = KeepMacroComments;
1108  }
1109 
1110  bool getCommentRetentionState() const { return KeepComments; }
1111 
1112  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
1113  bool getPragmasEnabled() const { return PragmasEnabled; }
1114 
1115  void SetSuppressIncludeNotFoundError(bool Suppress) {
1116  SuppressIncludeNotFoundError = Suppress;
1117  }
1118 
1120  return SuppressIncludeNotFoundError;
1121  }
1122 
1123  /// Sets whether the preprocessor is responsible for producing output or if
1124  /// it is producing tokens to be consumed by Parse and Sema.
1125  void setPreprocessedOutput(bool IsPreprocessedOutput) {
1126  PreprocessedOutput = IsPreprocessedOutput;
1127  }
1128 
1129  /// Returns true if the preprocessor is responsible for generating output,
1130  /// false if it is producing tokens to be consumed by Parse and Sema.
1131  bool isPreprocessedOutput() const { return PreprocessedOutput; }
1132 
1133  /// Return true if we are lexing directly from the specified lexer.
1134  bool isCurrentLexer(const PreprocessorLexer *L) const {
1135  return CurPPLexer == L;
1136  }
1137 
1138  /// Return the current lexer being lexed from.
1139  ///
1140  /// Note that this ignores any potentially active macro expansions and _Pragma
1141  /// expansions going on at the time.
1142  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
1143 
1144  /// Return the current file lexer being lexed from.
1145  ///
1146  /// Note that this ignores any potentially active macro expansions and _Pragma
1147  /// expansions going on at the time.
1149 
1150  /// Return the submodule owning the file being lexed. This may not be
1151  /// the current module if we have changed modules since entering the file.
1152  Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
1153 
1154  /// Returns the FileID for the preprocessor predefines.
1155  FileID getPredefinesFileID() const { return PredefinesFileID; }
1156 
1157  /// \{
1158  /// Accessors for preprocessor callbacks.
1159  ///
1160  /// Note that this class takes ownership of any PPCallbacks object given to
1161  /// it.
1162  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
1163  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
1164  if (Callbacks)
1165  C = std::make_unique<PPChainedCallbacks>(std::move(C),
1166  std::move(Callbacks));
1167  Callbacks = std::move(C);
1168  }
1169  /// \}
1170 
1171  /// Get the number of tokens processed so far.
1172  unsigned getTokenCount() const { return TokenCount; }
1173 
1174  /// Get the max number of tokens before issuing a -Wmax-tokens warning.
1175  unsigned getMaxTokens() const { return MaxTokens; }
1176 
1178  MaxTokens = Value;
1179  MaxTokensOverrideLoc = Loc;
1180  };
1181 
1182  SourceLocation getMaxTokensOverrideLoc() const { return MaxTokensOverrideLoc; }
1183 
1184  /// Register a function that would be called on each token in the final
1185  /// expanded token stream.
1186  /// This also reports annotation tokens produced by the parser.
1187  void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
1188  OnToken = std::move(F);
1189  }
1190 
1191  void setPreprocessToken(bool Preprocess) { PreprocessToken = Preprocess; }
1192 
1193  bool isMacroDefined(StringRef Id) {
1194  return isMacroDefined(&Identifiers.get(Id));
1195  }
1196  bool isMacroDefined(const IdentifierInfo *II) {
1197  return II->hasMacroDefinition() &&
1198  (!getLangOpts().Modules || (bool)getMacroDefinition(II));
1199  }
1200 
1201  /// Determine whether II is defined as a macro within the module M,
1202  /// if that is a module that we've already preprocessed. Does not check for
1203  /// macros imported into M.
1205  if (!II->hasMacroDefinition())
1206  return false;
1207  auto I = Submodules.find(M);
1208  if (I == Submodules.end())
1209  return false;
1210  auto J = I->second.Macros.find(II);
1211  if (J == I->second.Macros.end())
1212  return false;
1213  auto *MD = J->second.getLatest();
1214  return MD && MD->isDefined();
1215  }
1216 
1218  if (!II->hasMacroDefinition())
1219  return {};
1220 
1221  MacroState &S = CurSubmoduleState->Macros[II];
1222  auto *MD = S.getLatest();
1223  while (MD && isa<VisibilityMacroDirective>(MD))
1224  MD = MD->getPrevious();
1225  return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
1226  S.getActiveModuleMacros(*this, II),
1227  S.isAmbiguous(*this, II));
1228  }
1229 
1231  SourceLocation Loc) {
1232  if (!II->hadMacroDefinition())
1233  return {};
1234 
1235  MacroState &S = CurSubmoduleState->Macros[II];
1237  if (auto *MD = S.getLatest())
1238  DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
1239  // FIXME: Compute the set of active module macros at the specified location.
1240  return MacroDefinition(DI.getDirective(),
1241  S.getActiveModuleMacros(*this, II),
1242  S.isAmbiguous(*this, II));
1243  }
1244 
1245  /// Given an identifier, return its latest non-imported MacroDirective
1246  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
1248  if (!II->hasMacroDefinition())
1249  return nullptr;
1250 
1251  auto *MD = getLocalMacroDirectiveHistory(II);
1252  if (!MD || MD->getDefinition().isUndefined())
1253  return nullptr;
1254 
1255  return MD;
1256  }
1257 
1258  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
1259  return const_cast<Preprocessor*>(this)->getMacroInfo(II);
1260  }
1261 
1263  if (!II->hasMacroDefinition())
1264  return nullptr;
1265  if (auto MD = getMacroDefinition(II))
1266  return MD.getMacroInfo();
1267  return nullptr;
1268  }
1269 
1270  /// Given an identifier, return the latest non-imported macro
1271  /// directive for that identifier.
1272  ///
1273  /// One can iterate over all previous macro directives from the most recent
1274  /// one.
1276 
1277  /// Add a directive to the macro directive history for this identifier.
1280  SourceLocation Loc) {
1281  DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1282  appendMacroDirective(II, MD);
1283  return MD;
1284  }
1286  MacroInfo *MI) {
1287  return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1288  }
1289 
1290  /// Set a MacroDirective that was loaded from a PCH file.
1292  MacroDirective *MD);
1293 
1294  /// Register an exported macro for a module and identifier.
1296  ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1297  ModuleMacro *getModuleMacro(Module *Mod, const IdentifierInfo *II);
1298 
1299  /// Get the list of leaf (non-overridden) module macros for a name.
1301  if (II->isOutOfDate())
1302  updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1303  auto I = LeafModuleMacros.find(II);
1304  if (I != LeafModuleMacros.end())
1305  return I->second;
1306  return None;
1307  }
1308 
1309  /// Get the list of submodules that we're currently building.
1311  return BuildingSubmoduleStack;
1312  }
1313 
1314  /// \{
1315  /// Iterators for the macro history table. Currently defined macros have
1316  /// IdentifierInfo::hasMacroDefinition() set and an empty
1317  /// MacroInfo::getUndefLoc() at the head of the list.
1318  using macro_iterator = MacroMap::const_iterator;
1319 
1320  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1321  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1322 
1323  llvm::iterator_range<macro_iterator>
1324  macros(bool IncludeExternalMacros = true) const {
1325  macro_iterator begin = macro_begin(IncludeExternalMacros);
1326  macro_iterator end = macro_end(IncludeExternalMacros);
1327  return llvm::make_range(begin, end);
1328  }
1329 
1330  /// \}
1331 
1332  /// Mark the given clang module as affecting the current clang module or translation unit.
1334  assert(M->isModuleMapModule());
1335  if (!BuildingSubmoduleStack.empty()) {
1336  if (M != BuildingSubmoduleStack.back().M)
1337  BuildingSubmoduleStack.back().M->AffectingClangModules.insert(M);
1338  } else {
1339  AffectingClangModules.insert(M);
1340  }
1341  }
1342 
1343  /// Get the set of top-level clang modules that affected preprocessing, but were not
1344  /// imported.
1346  return AffectingClangModules;
1347  }
1348 
1349  /// Mark the file as included.
1350  /// Returns true if this is the first time the file was included.
1351  bool markIncluded(const FileEntry *File) {
1352  HeaderInfo.getFileInfo(File);
1353  return IncludedFiles.insert(File).second;
1354  }
1355 
1356  /// Return true if this header has already been included.
1357  bool alreadyIncluded(const FileEntry *File) const {
1358  return IncludedFiles.count(File);
1359  }
1360 
1361  /// Get the set of included files.
1362  IncludedFilesSet &getIncludedFiles() { return IncludedFiles; }
1363  const IncludedFilesSet &getIncludedFiles() const { return IncludedFiles; }
1364 
1365  /// Return the name of the macro defined before \p Loc that has
1366  /// spelling \p Tokens. If there are multiple macros with same spelling,
1367  /// return the last one defined.
1369  ArrayRef<TokenValue> Tokens) const;
1370 
1371  /// Get the predefines for this processor.
1372  /// Used by some third-party tools to inspect and add predefines (see
1373  /// https://github.com/llvm/llvm-project/issues/57483).
1374  const std::string &getPredefines() const { return Predefines; }
1375 
1376  /// Set the predefines for this Preprocessor.
1377  ///
1378  /// These predefines are automatically injected when parsing the main file.
1379  void setPredefines(std::string P) { Predefines = std::move(P); }
1380 
1381  /// Return information about the specified preprocessor
1382  /// identifier token.
1383  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1384  return &Identifiers.get(Name);
1385  }
1386 
1387  /// Add the specified pragma handler to this preprocessor.
1388  ///
1389  /// If \p Namespace is non-null, then it is a token required to exist on the
1390  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1391  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1393  AddPragmaHandler(StringRef(), Handler);
1394  }
1395 
1396  /// Remove the specific pragma handler from this preprocessor.
1397  ///
1398  /// If \p Namespace is non-null, then it should be the namespace that
1399  /// \p Handler was added to. It is an error to remove a handler that
1400  /// has not been registered.
1401  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1403  RemovePragmaHandler(StringRef(), Handler);
1404  }
1405 
1406  /// Install empty handlers for all pragmas (making them ignored).
1407  void IgnorePragmas();
1408 
1409  /// Set empty line handler.
1410  void setEmptylineHandler(EmptylineHandler *Handler) { Emptyline = Handler; }
1411 
1412  EmptylineHandler *getEmptylineHandler() const { return Emptyline; }
1413 
1414  /// Add the specified comment handler to the preprocessor.
1415  void addCommentHandler(CommentHandler *Handler);
1416 
1417  /// Remove the specified comment handler.
1418  ///
1419  /// It is an error to remove a handler that has not been registered.
1420  void removeCommentHandler(CommentHandler *Handler);
1421 
1422  /// Set the code completion handler to the given object.
1424  CodeComplete = &Handler;
1425  }
1426 
1427  /// Retrieve the current code-completion handler.
1429  return CodeComplete;
1430  }
1431 
1432  /// Clear out the code completion handler.
1434  CodeComplete = nullptr;
1435  }
1436 
1437  /// Hook used by the lexer to invoke the "included file" code
1438  /// completion point.
1439  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1440 
1441  /// Hook used by the lexer to invoke the "natural language" code
1442  /// completion point.
1444 
1445  /// Set the code completion token for filtering purposes.
1447  CodeCompletionII = Filter;
1448  }
1449 
1450  /// Set the code completion token range for detecting replacement range later
1451  /// on.
1453  const SourceLocation End) {
1454  CodeCompletionTokenRange = {Start, End};
1455  }
1457  return CodeCompletionTokenRange;
1458  }
1459 
1460  /// Get the code completion token for filtering purposes.
1462  if (CodeCompletionII)
1463  return CodeCompletionII->getName();
1464  return {};
1465  }
1466 
1467  /// Retrieve the preprocessing record, or NULL if there is no
1468  /// preprocessing record.
1469  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1470 
1471  /// Create a new preprocessing record, which will keep track of
1472  /// all macro expansions, macro definitions, etc.
1474 
1475  /// Returns true if the FileEntry is the PCH through header.
1476  bool isPCHThroughHeader(const FileEntry *FE);
1477 
1478  /// True if creating a PCH with a through header.
1480 
1481  /// True if using a PCH with a through header.
1483 
1484  /// True if creating a PCH with a #pragma hdrstop.
1486 
1487  /// True if using a PCH with a #pragma hdrstop.
1489 
1490  /// Skip tokens until after the #include of the through header or
1491  /// until after a #pragma hdrstop.
1492  void SkipTokensWhileUsingPCH();
1493 
1494  /// Process directives while skipping until the through header or
1495  /// #pragma hdrstop is found.
1497  SourceLocation HashLoc);
1498 
1499  /// Enter the specified FileID as the main source file,
1500  /// which implicitly adds the builtin defines etc.
1501  void EnterMainSourceFile();
1502 
1503  /// Inform the preprocessor callbacks that processing is complete.
1504  void EndSourceFile();
1505 
1506  /// Add a source file to the top of the include stack and
1507  /// start lexing tokens from it instead of the current buffer.
1508  ///
1509  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1511  SourceLocation Loc, bool IsFirstIncludeOfFile = true);
1512 
1513  /// Add a Macro to the top of the include stack and start lexing
1514  /// tokens from it instead of the current buffer.
1515  ///
1516  /// \param Args specifies the tokens input to a function-like macro.
1517  /// \param ILEnd specifies the location of the ')' for a function-like macro
1518  /// or the identifier for an object-like macro.
1519  void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1520  MacroArgs *Args);
1521 
1522 private:
1523  /// Add a "macro" context to the top of the include stack,
1524  /// which will cause the lexer to start returning the specified tokens.
1525  ///
1526  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1527  /// will not be subject to further macro expansion. Otherwise, these tokens
1528  /// will be re-macro-expanded when/if expansion is enabled.
1529  ///
1530  /// If \p OwnsTokens is false, this method assumes that the specified stream
1531  /// of tokens has a permanent owner somewhere, so they do not need to be
1532  /// copied. If it is true, it assumes the array of tokens is allocated with
1533  /// \c new[] and the Preprocessor will delete[] it.
1534  ///
1535  /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
1536  /// set, see the flag documentation for details.
1537  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1538  bool DisableMacroExpansion, bool OwnsTokens,
1539  bool IsReinject);
1540 
1541 public:
1542  void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1543  bool DisableMacroExpansion, bool IsReinject) {
1544  EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true,
1545  IsReinject);
1546  }
1547 
1548  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
1549  bool IsReinject) {
1550  EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false,
1551  IsReinject);
1552  }
1553 
1554  /// Pop the current lexer/macro exp off the top of the lexer stack.
1555  ///
1556  /// This should only be used in situations where the current state of the
1557  /// top-of-stack lexer is known.
1558  void RemoveTopOfLexerStack();
1559 
1560  /// From the point that this method is called, and until
1561  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1562  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1563  /// make the Preprocessor re-lex the same tokens.
1564  ///
1565  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1566  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1567  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1568  ///
1569  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1570  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1571  /// tokens will continue indefinitely.
1572  ///
1573  void EnableBacktrackAtThisPos();
1574 
1575  /// Disable the last EnableBacktrackAtThisPos call.
1576  void CommitBacktrackedTokens();
1577 
1578  /// Make Preprocessor re-lex the tokens that were lexed since
1579  /// EnableBacktrackAtThisPos() was previously called.
1580  void Backtrack();
1581 
1582  /// True if EnableBacktrackAtThisPos() was called and
1583  /// caching of tokens is on.
1584  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1585 
1586  /// Lex the next token for this preprocessor.
1587  void Lex(Token &Result);
1588 
1589  /// Lex a token, forming a header-name token if possible.
1590  bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
1591 
1592  bool LexAfterModuleImport(Token &Result);
1594 
1595  void makeModuleVisible(Module *M, SourceLocation Loc);
1596 
1598  return CurSubmoduleState->VisibleModules.getImportLoc(M);
1599  }
1600 
1601  /// Lex a string literal, which may be the concatenation of multiple
1602  /// string literals and may even come from macro expansion.
1603  /// \returns true on success, false if a error diagnostic has been generated.
1604  bool LexStringLiteral(Token &Result, std::string &String,
1605  const char *DiagnosticTag, bool AllowMacroExpansion) {
1606  if (AllowMacroExpansion)
1607  Lex(Result);
1608  else
1609  LexUnexpandedToken(Result);
1610  return FinishLexStringLiteral(Result, String, DiagnosticTag,
1611  AllowMacroExpansion);
1612  }
1613 
1614  /// Complete the lexing of a string literal where the first token has
1615  /// already been lexed (see LexStringLiteral).
1616  bool FinishLexStringLiteral(Token &Result, std::string &String,
1617  const char *DiagnosticTag,
1618  bool AllowMacroExpansion);
1619 
1620  /// Lex a token. If it's a comment, keep lexing until we get
1621  /// something not a comment.
1622  ///
1623  /// This is useful in -E -C mode where comments would foul up preprocessor
1624  /// directive handling.
1625  void LexNonComment(Token &Result) {
1626  do
1627  Lex(Result);
1628  while (Result.getKind() == tok::comment);
1629  }
1630 
1631  /// Just like Lex, but disables macro expansion of identifier tokens.
1632  void LexUnexpandedToken(Token &Result) {
1633  // Disable macro expansion.
1634  bool OldVal = DisableMacroExpansion;
1635  DisableMacroExpansion = true;
1636  // Lex the token.
1637  Lex(Result);
1638 
1639  // Reenable it.
1640  DisableMacroExpansion = OldVal;
1641  }
1642 
1643  /// Like LexNonComment, but this disables macro expansion of
1644  /// identifier tokens.
1646  do
1647  LexUnexpandedToken(Result);
1648  while (Result.getKind() == tok::comment);
1649  }
1650 
1651  /// Parses a simple integer literal to get its numeric value. Floating
1652  /// point literals and user defined literals are rejected. Used primarily to
1653  /// handle pragmas that accept integer arguments.
1655 
1656  /// Disables macro expansion everywhere except for preprocessor directives.
1658  DisableMacroExpansion = true;
1659  MacroExpansionInDirectivesOverride = true;
1660  }
1661 
1662  /// Peeks ahead N tokens and returns that token without consuming any
1663  /// tokens.
1664  ///
1665  /// LookAhead(0) returns the next token that would be returned by Lex(),
1666  /// LookAhead(1) returns the token after it, etc. This returns normal
1667  /// tokens after phase 5. As such, it is equivalent to using
1668  /// 'Lex', not 'LexUnexpandedToken'.
1669  const Token &LookAhead(unsigned N) {
1670  assert(LexLevel == 0 && "cannot use lookahead while lexing");
1671  if (CachedLexPos + N < CachedTokens.size())
1672  return CachedTokens[CachedLexPos+N];
1673  else
1674  return PeekAhead(N+1);
1675  }
1676 
1677  /// When backtracking is enabled and tokens are cached,
1678  /// this allows to revert a specific number of tokens.
1679  ///
1680  /// Note that the number of tokens being reverted should be up to the last
1681  /// backtrack position, not more.
1682  void RevertCachedTokens(unsigned N) {
1683  assert(isBacktrackEnabled() &&
1684  "Should only be called when tokens are cached for backtracking");
1685  assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1686  && "Should revert tokens up to the last backtrack position, not more");
1687  assert(signed(CachedLexPos) - signed(N) >= 0 &&
1688  "Corrupted backtrack positions ?");
1689  CachedLexPos -= N;
1690  }
1691 
1692  /// Enters a token in the token stream to be lexed next.
1693  ///
1694  /// If BackTrack() is called afterwards, the token will remain at the
1695  /// insertion point.
1696  /// If \p IsReinject is true, resulting token will have Token::IsReinjected
1697  /// flag set. See the flag documentation for details.
1698  void EnterToken(const Token &Tok, bool IsReinject) {
1699  if (LexLevel) {
1700  // It's not correct in general to enter caching lex mode while in the
1701  // middle of a nested lexing action.
1702  auto TokCopy = std::make_unique<Token[]>(1);
1703  TokCopy[0] = Tok;
1704  EnterTokenStream(std::move(TokCopy), 1, true, IsReinject);
1705  } else {
1706  EnterCachingLexMode();
1707  assert(IsReinject && "new tokens in the middle of cached stream");
1708  CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1709  }
1710  }
1711 
1712  /// We notify the Preprocessor that if it is caching tokens (because
1713  /// backtrack is enabled) it should replace the most recent cached tokens
1714  /// with the given annotation token. This function has no effect if
1715  /// backtracking is not enabled.
1716  ///
1717  /// Note that the use of this function is just for optimization, so that the
1718  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1719  /// invoked.
1720  void AnnotateCachedTokens(const Token &Tok) {
1721  assert(Tok.isAnnotation() && "Expected annotation token");
1722  if (CachedLexPos != 0 && isBacktrackEnabled())
1723  AnnotatePreviousCachedTokens(Tok);
1724  }
1725 
1726  /// Get the location of the last cached token, suitable for setting the end
1727  /// location of an annotation token.
1729  assert(CachedLexPos != 0);
1730  return CachedTokens[CachedLexPos-1].getLastLoc();
1731  }
1732 
1733  /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1734  /// CachedTokens.
1735  bool IsPreviousCachedToken(const Token &Tok) const;
1736 
1737  /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1738  /// in \p NewToks.
1739  ///
1740  /// Useful when a token needs to be split in smaller ones and CachedTokens
1741  /// most recent token must to be updated to reflect that.
1743 
1744  /// Replace the last token with an annotation token.
1745  ///
1746  /// Like AnnotateCachedTokens(), this routine replaces an
1747  /// already-parsed (and resolved) token with an annotation
1748  /// token. However, this routine only replaces the last token with
1749  /// the annotation token; it does not affect any other cached
1750  /// tokens. This function has no effect if backtracking is not
1751  /// enabled.
1753  assert(Tok.isAnnotation() && "Expected annotation token");
1754  if (CachedLexPos != 0 && isBacktrackEnabled())
1755  CachedTokens[CachedLexPos-1] = Tok;
1756  }
1757 
1758  /// Enter an annotation token into the token stream.
1760  void *AnnotationVal);
1761 
1762  /// Determine whether it's possible for a future call to Lex to produce an
1763  /// annotation token created by a previous call to EnterAnnotationToken.
1765  return CurLexerKind != CLK_Lexer;
1766  }
1767 
1768  /// Update the current token to represent the provided
1769  /// identifier, in order to cache an action performed by typo correction.
1770  void TypoCorrectToken(const Token &Tok) {
1771  assert(Tok.getIdentifierInfo() && "Expected identifier token");
1772  if (CachedLexPos != 0 && isBacktrackEnabled())
1773  CachedTokens[CachedLexPos-1] = Tok;
1774  }
1775 
1776  /// Recompute the current lexer kind based on the CurLexer/
1777  /// CurTokenLexer pointers.
1778  void recomputeCurLexerKind();
1779 
1780  /// Returns true if incremental processing is enabled
1781  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1782 
1783  /// Enables the incremental processing
1784  void enableIncrementalProcessing(bool value = true) {
1785  IncrementalProcessing = value;
1786  }
1787 
1788  /// Specify the point at which code-completion will be performed.
1789  ///
1790  /// \param File the file in which code completion should occur. If
1791  /// this file is included multiple times, code-completion will
1792  /// perform completion the first time it is included. If NULL, this
1793  /// function clears out the code-completion point.
1794  ///
1795  /// \param Line the line at which code completion should occur
1796  /// (1-based).
1797  ///
1798  /// \param Column the column at which code completion should occur
1799  /// (1-based).
1800  ///
1801  /// \returns true if an error occurred, false otherwise.
1802  bool SetCodeCompletionPoint(const FileEntry *File,
1803  unsigned Line, unsigned Column);
1804 
1805  /// Determine if we are performing code completion.
1806  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1807 
1808  /// Returns the location of the code-completion point.
1809  ///
1810  /// Returns an invalid location if code-completion is not enabled or the file
1811  /// containing the code-completion point has not been lexed yet.
1812  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1813 
1814  /// Returns the start location of the file of code-completion point.
1815  ///
1816  /// Returns an invalid location if code-completion is not enabled or the file
1817  /// containing the code-completion point has not been lexed yet.
1819  return CodeCompletionFileLoc;
1820  }
1821 
1822  /// Returns true if code-completion is enabled and we have hit the
1823  /// code-completion point.
1824  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1825 
1826  /// Note that we hit the code-completion point.
1828  assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1829  CodeCompletionReached = true;
1830  // Silence any diagnostics that occur after we hit the code-completion.
1832  }
1833 
1834  /// The location of the currently-active \#pragma clang
1835  /// arc_cf_code_audited begin.
1836  ///
1837  /// Returns an invalid location if there is no such pragma active.
1838  std::pair<IdentifierInfo *, SourceLocation>
1840  return PragmaARCCFCodeAuditedInfo;
1841  }
1842 
1843  /// Set the location of the currently-active \#pragma clang
1844  /// arc_cf_code_audited begin. An invalid location ends the pragma.
1846  SourceLocation Loc) {
1847  PragmaARCCFCodeAuditedInfo = {Ident, Loc};
1848  }
1849 
1850  /// The location of the currently-active \#pragma clang
1851  /// assume_nonnull begin.
1852  ///
1853  /// Returns an invalid location if there is no such pragma active.
1855  return PragmaAssumeNonNullLoc;
1856  }
1857 
1858  /// Set the location of the currently-active \#pragma clang
1859  /// assume_nonnull begin. An invalid location ends the pragma.
1861  PragmaAssumeNonNullLoc = Loc;
1862  }
1863 
1864  /// Get the location of the recorded unterminated \#pragma clang
1865  /// assume_nonnull begin in the preamble, if one exists.
1866  ///
1867  /// Returns an invalid location if the premable did not end with
1868  /// such a pragma active or if there is no recorded preamble.
1870  return PreambleRecordedPragmaAssumeNonNullLoc;
1871  }
1872 
1873  /// Record the location of the unterminated \#pragma clang
1874  /// assume_nonnull begin in the preamble.
1876  PreambleRecordedPragmaAssumeNonNullLoc = Loc;
1877  }
1878 
1879  /// Set the directory in which the main file should be considered
1880  /// to have been found, if it is not a real file.
1881  void setMainFileDir(const DirectoryEntry *Dir) {
1882  MainFileDir = Dir;
1883  }
1884 
1885  /// Instruct the preprocessor to skip part of the main source file.
1886  ///
1887  /// \param Bytes The number of bytes in the preamble to skip.
1888  ///
1889  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1890  /// start of a line.
1891  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1892  SkipMainFilePreamble.first = Bytes;
1893  SkipMainFilePreamble.second = StartOfLine;
1894  }
1895 
1896  /// Forwarding function for diagnostics. This emits a diagnostic at
1897  /// the specified Token's location, translating the token's start
1898  /// position in the current buffer into a SourcePosition object for rendering.
1899  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1900  return Diags->Report(Loc, DiagID);
1901  }
1902 
1903  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1904  return Diags->Report(Tok.getLocation(), DiagID);
1905  }
1906 
1907  /// Return the 'spelling' of the token at the given
1908  /// location; does not go up to the spelling location or down to the
1909  /// expansion location.
1910  ///
1911  /// \param buffer A buffer which will be used only if the token requires
1912  /// "cleaning", e.g. if it contains trigraphs or escaped newlines
1913  /// \param invalid If non-null, will be set \c true if an error occurs.
1915  SmallVectorImpl<char> &buffer,
1916  bool *invalid = nullptr) const {
1917  return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1918  }
1919 
1920  /// Return the 'spelling' of the Tok token.
1921  ///
1922  /// The spelling of a token is the characters used to represent the token in
1923  /// the source file after trigraph expansion and escaped-newline folding. In
1924  /// particular, this wants to get the true, uncanonicalized, spelling of
1925  /// things like digraphs, UCNs, etc.
1926  ///
1927  /// \param Invalid If non-null, will be set \c true if an error occurs.
1928  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1929  return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1930  }
1931 
1932  /// Get the spelling of a token into a preallocated buffer, instead
1933  /// of as an std::string.
1934  ///
1935  /// The caller is required to allocate enough space for the token, which is
1936  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1937  /// actual result is returned.
1938  ///
1939  /// Note that this method may do two possible things: it may either fill in
1940  /// the buffer specified with characters, or it may *change the input pointer*
1941  /// to point to a constant buffer with the data already in it (avoiding a
1942  /// copy). The caller is not allowed to modify the returned buffer pointer
1943  /// if an internal buffer is returned.
1944  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1945  bool *Invalid = nullptr) const {
1946  return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1947  }
1948 
1949  /// Get the spelling of a token into a SmallVector.
1950  ///
1951  /// Note that the returned StringRef may not point to the
1952  /// supplied buffer if a copy can be avoided.
1953  StringRef getSpelling(const Token &Tok,
1954  SmallVectorImpl<char> &Buffer,
1955  bool *Invalid = nullptr) const;
1956 
1957  /// Relex the token at the specified location.
1958  /// \returns true if there was a failure, false on success.
1959  bool getRawToken(SourceLocation Loc, Token &Result,
1960  bool IgnoreWhiteSpace = false) {
1961  return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1962  }
1963 
1964  /// Given a Token \p Tok that is a numeric constant with length 1,
1965  /// return the character.
1966  char
1968  bool *Invalid = nullptr) const {
1969  assert(Tok.is(tok::numeric_constant) &&
1970  Tok.getLength() == 1 && "Called on unsupported token");
1971  assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1972 
1973  // If the token is carrying a literal data pointer, just use it.
1974  if (const char *D = Tok.getLiteralData())
1975  return *D;
1976 
1977  // Otherwise, fall back on getCharacterData, which is slower, but always
1978  // works.
1979  return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1980  }
1981 
1982  /// Retrieve the name of the immediate macro expansion.
1983  ///
1984  /// This routine starts from a source location, and finds the name of the
1985  /// macro responsible for its immediate expansion. It looks through any
1986  /// intervening macro argument expansions to compute this. It returns a
1987  /// StringRef that refers to the SourceManager-owned buffer of the source
1988  /// where that macro name is spelled. Thus, the result shouldn't out-live
1989  /// the SourceManager.
1991  return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1992  }
1993 
1994  /// Plop the specified string into a scratch buffer and set the
1995  /// specified token's location and length to it.
1996  ///
1997  /// If specified, the source location provides a location of the expansion
1998  /// point of the token.
1999  void CreateString(StringRef Str, Token &Tok,
2000  SourceLocation ExpansionLocStart = SourceLocation(),
2001  SourceLocation ExpansionLocEnd = SourceLocation());
2002 
2003  /// Split the first Length characters out of the token starting at TokLoc
2004  /// and return a location pointing to the split token. Re-lexing from the
2005  /// split token will return the split token rather than the original.
2006  SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
2007 
2008  /// Computes the source location just past the end of the
2009  /// token at this source location.
2010  ///
2011  /// This routine can be used to produce a source location that
2012  /// points just past the end of the token referenced by \p Loc, and
2013  /// is generally used when a diagnostic needs to point just after a
2014  /// token where it expected something different that it received. If
2015  /// the returned source location would not be meaningful (e.g., if
2016  /// it points into a macro), this routine returns an invalid
2017  /// source location.
2018  ///
2019  /// \param Offset an offset from the end of the token, where the source
2020  /// location should refer to. The default offset (0) produces a source
2021  /// location pointing just past the end of the token; an offset of 1 produces
2022  /// a source location pointing to the last character in the token, etc.
2024  return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
2025  }
2026 
2027  /// Returns true if the given MacroID location points at the first
2028  /// token of the macro expansion.
2029  ///
2030  /// \param MacroBegin If non-null and function returns true, it is set to
2031  /// begin location of the macro.
2033  SourceLocation *MacroBegin = nullptr) const {
2034  return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
2035  MacroBegin);
2036  }
2037 
2038  /// Returns true if the given MacroID location points at the last
2039  /// token of the macro expansion.
2040  ///
2041  /// \param MacroEnd If non-null and function returns true, it is set to
2042  /// end location of the macro.
2044  SourceLocation *MacroEnd = nullptr) const {
2045  return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
2046  }
2047 
2048  /// Print the token to stderr, used for debugging.
2049  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
2050  void DumpLocation(SourceLocation Loc) const;
2051  void DumpMacro(const MacroInfo &MI) const;
2052  void dumpMacroInfo(const IdentifierInfo *II);
2053 
2054  /// Given a location that specifies the start of a
2055  /// token, return a new location that specifies a character within the token.
2057  unsigned Char) const {
2058  return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
2059  }
2060 
2061  /// Increment the counters for the number of token paste operations
2062  /// performed.
2063  ///
2064  /// If fast was specified, this is a 'fast paste' case we handled.
2065  void IncrementPasteCounter(bool isFast) {
2066  if (isFast)
2067  ++NumFastTokenPaste;
2068  else
2069  ++NumTokenPaste;
2070  }
2071 
2072  void PrintStats();
2073 
2074  size_t getTotalMemory() const;
2075 
2076  /// When the macro expander pastes together a comment (/##/) in Microsoft
2077  /// mode, this method handles updating the current state, returning the
2078  /// token on the next source line.
2080 
2081  //===--------------------------------------------------------------------===//
2082  // Preprocessor callback methods. These are invoked by a lexer as various
2083  // directives and events are found.
2084 
2085  /// Given a tok::raw_identifier token, look up the
2086  /// identifier information for the token and install it into the token,
2087  /// updating the token kind accordingly.
2089 
2090 private:
2091  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
2092 
2093 public:
2094  /// Specifies the reason for poisoning an identifier.
2095  ///
2096  /// If that identifier is accessed while poisoned, then this reason will be
2097  /// used instead of the default "poisoned" diagnostic.
2098  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
2099 
2100  /// Display reason for poisoned identifier.
2102 
2104  if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
2105  if(II->isPoisoned()) {
2107  }
2108  }
2109  }
2110 
2111 private:
2112  /// Identifiers used for SEH handling in Borland. These are only
2113  /// allowed in particular circumstances
2114  // __except block
2115  IdentifierInfo *Ident__exception_code,
2116  *Ident___exception_code,
2117  *Ident_GetExceptionCode;
2118  // __except filter expression
2119  IdentifierInfo *Ident__exception_info,
2120  *Ident___exception_info,
2121  *Ident_GetExceptionInfo;
2122  // __finally
2123  IdentifierInfo *Ident__abnormal_termination,
2124  *Ident___abnormal_termination,
2125  *Ident_AbnormalTermination;
2126 
2127  const char *getCurLexerEndPos();
2128  void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
2129 
2130 public:
2131  void PoisonSEHIdentifiers(bool Poison = true); // Borland
2132 
2133  /// Callback invoked when the lexer reads an identifier and has
2134  /// filled in the tokens IdentifierInfo member.
2135  ///
2136  /// This callback potentially macro expands it or turns it into a named
2137  /// token (like 'for').
2138  ///
2139  /// \returns true if we actually computed a token, false if we need to
2140  /// lex again.
2142 
2143  /// Callback invoked when the lexer hits the end of the current file.
2144  ///
2145  /// This either returns the EOF token and returns true, or
2146  /// pops a level off the include stack and returns false, at which point the
2147  /// client should call lex again.
2148  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
2149 
2150  /// Callback invoked when the current TokenLexer hits the end of its
2151  /// token stream.
2152  bool HandleEndOfTokenLexer(Token &Result);
2153 
2154  /// Callback invoked when the lexer sees a # token at the start of a
2155  /// line.
2156  ///
2157  /// This consumes the directive, modifies the lexer/preprocessor state, and
2158  /// advances the lexer(s) so that the next token read is the correct one.
2159  void HandleDirective(Token &Result);
2160 
2161  /// Ensure that the next token is a tok::eod token.
2162  ///
2163  /// If not, emit a diagnostic and consume up until the eod.
2164  /// If \p EnableMacros is true, then we consider macros that expand to zero
2165  /// tokens as being ok.
2166  ///
2167  /// \return The location of the end of the directive (the terminating
2168  /// newline).
2169  SourceLocation CheckEndOfDirective(const char *DirType,
2170  bool EnableMacros = false);
2171 
2172  /// Read and discard all tokens remaining on the current line until
2173  /// the tok::eod token is found. Returns the range of the skipped tokens.
2175 
2176  /// Returns true if the preprocessor has seen a use of
2177  /// __DATE__ or __TIME__ in the file so far.
2178  bool SawDateOrTime() const {
2179  return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
2180  }
2181  unsigned getCounterValue() const { return CounterValue; }
2182  void setCounterValue(unsigned V) { CounterValue = V; }
2183 
2185  assert(CurrentFPEvalMethod != LangOptions::FEM_UnsetOnCommandLine &&
2186  "FPEvalMethod should be set either from command line or from the "
2187  "target info");
2188  return CurrentFPEvalMethod;
2189  }
2190 
2192  return TUFPEvalMethod;
2193  }
2194 
2196  return LastFPEvalPragmaLocation;
2197  }
2198 
2200  return LastFPEvalMethod;
2201  }
2202 
2204  LastFPEvalMethod = Val;
2205  }
2206 
2209  assert(Val != LangOptions::FEM_UnsetOnCommandLine &&
2210  "FPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2211  // This is the location of the '#pragma float_control" where the
2212  // execution state is modifed.
2213  LastFPEvalPragmaLocation = PragmaLoc;
2214  CurrentFPEvalMethod = Val;
2215  TUFPEvalMethod = Val;
2216  }
2217 
2219  assert(Val != LangOptions::FEM_UnsetOnCommandLine &&
2220  "TUPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2221  TUFPEvalMethod = Val;
2222  }
2223 
2224  /// Retrieves the module that we're currently building, if any.
2226 
2227  /// Retrieves the module whose implementation we're current compiling, if any.
2229 
2230  /// Allocate a new MacroInfo object with the provided SourceLocation.
2232 
2233  /// Turn the specified lexer token into a fully checked and spelled
2234  /// filename, e.g. as an operand of \#include.
2235  ///
2236  /// The caller is expected to provide a buffer that is large enough to hold
2237  /// the spelling of the filename, but is also expected to handle the case
2238  /// when this method decides to use a different buffer.
2239  ///
2240  /// \returns true if the input filename was in <>'s or false if it was
2241  /// in ""'s.
2242  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
2243 
2244  /// Given a "foo" or <foo> reference, look up the indicated file.
2245  ///
2246  /// Returns None on failure. \p isAngled indicates whether the file
2247  /// reference is for system \#include's or not (i.e. using <> instead of "").
2249  LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled,
2250  ConstSearchDirIterator FromDir, const FileEntry *FromFile,
2251  ConstSearchDirIterator *CurDir, SmallVectorImpl<char> *SearchPath,
2252  SmallVectorImpl<char> *RelativePath,
2253  ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped,
2254  bool *IsFrameworkFound, bool SkipCache = false,
2255  bool OpenFile = true, bool CacheFailures = true);
2256 
2257  /// Return true if we're in the top-level file, not in a \#include.
2258  bool isInPrimaryFile() const;
2259 
2260  /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
2261  /// followed by EOD. Return true if the token is not a valid on-off-switch.
2262  bool LexOnOffSwitch(tok::OnOffSwitch &Result);
2263 
2264  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
2265  bool *ShadowFlag = nullptr);
2266 
2267  void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
2268  Module *LeaveSubmodule(bool ForPragma);
2269 
2270 private:
2271  friend void TokenLexer::ExpandFunctionArguments();
2272 
2273  void PushIncludeMacroStack() {
2274  assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
2275  IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
2276  std::move(CurLexer), CurPPLexer,
2277  std::move(CurTokenLexer), CurDirLookup);
2278  CurPPLexer = nullptr;
2279  }
2280 
2281  void PopIncludeMacroStack() {
2282  CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2283  CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2284  CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2285  CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2286  CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2287  CurLexerKind = IncludeMacroStack.back().CurLexerKind;
2288  IncludeMacroStack.pop_back();
2289  }
2290 
2291  void PropagateLineStartLeadingSpaceInfo(Token &Result);
2292 
2293  /// Determine whether we need to create module macros for #defines in the
2294  /// current context.
2295  bool needModuleMacros() const;
2296 
2297  /// Update the set of active module macros and ambiguity flag for a module
2298  /// macro name.
2299  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
2300 
2301  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2302  SourceLocation Loc);
2303  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2304  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2305  bool isPublic);
2306 
2307  /// Lex and validate a macro name, which occurs after a
2308  /// \#define or \#undef.
2309  ///
2310  /// \param MacroNameTok Token that represents the name defined or undefined.
2311  /// \param IsDefineUndef Kind if preprocessor directive.
2312  /// \param ShadowFlag Points to flag that is set if macro name shadows
2313  /// a keyword.
2314  ///
2315  /// This emits a diagnostic, sets the token kind to eod,
2316  /// and discards the rest of the macro line if the macro name is invalid.
2317  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
2318  bool *ShadowFlag = nullptr);
2319 
2320  /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
2321  /// entire line) of the macro's tokens and adds them to MacroInfo, and while
2322  /// doing so performs certain validity checks including (but not limited to):
2323  /// - # (stringization) is followed by a macro parameter
2324  /// \param MacroNameTok - Token that represents the macro name
2325  /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
2326  ///
2327  /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
2328  /// returns a nullptr if an invalid sequence of tokens is encountered.
2329  MacroInfo *ReadOptionalMacroParameterListAndBody(
2330  const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
2331 
2332  /// The ( starting an argument list of a macro definition has just been read.
2333  /// Lex the rest of the parameters and the closing ), updating \p MI with
2334  /// what we learn and saving in \p LastTok the last token read.
2335  /// Return true if an error occurs parsing the arg list.
2336  bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2337 
2338  /// Provide a suggestion for a typoed directive. If there is no typo, then
2339  /// just skip suggesting.
2340  ///
2341  /// \param Tok - Token that represents the directive
2342  /// \param Directive - String reference for the directive name
2343  void SuggestTypoedDirective(const Token &Tok, StringRef Directive) const;
2344 
2345  /// We just read a \#if or related directive and decided that the
2346  /// subsequent tokens are in the \#if'd out portion of the
2347  /// file. Lex the rest of the file, until we see an \#endif. If \p
2348  /// FoundNonSkipPortion is true, then we have already emitted code for part of
2349  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
2350  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
2351  /// already seen one so a \#else directive is a duplicate. When this returns,
2352  /// the caller can lex the first valid token.
2353  void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2354  SourceLocation IfTokenLoc,
2355  bool FoundNonSkipPortion, bool FoundElse,
2356  SourceLocation ElseLoc = SourceLocation());
2357 
2358  /// Information about the result for evaluating an expression for a
2359  /// preprocessor directive.
2360  struct DirectiveEvalResult {
2361  /// Whether the expression was evaluated as true or not.
2362  bool Conditional;
2363 
2364  /// True if the expression contained identifiers that were undefined.
2365  bool IncludedUndefinedIds;
2366 
2367  /// The source range for the expression.
2368  SourceRange ExprRange;
2369  };
2370 
2371  /// Evaluate an integer constant expression that may occur after a
2372  /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2373  ///
2374  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2375  DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
2376 
2377  /// Process a '__has_include("path")' expression.
2378  ///
2379  /// Returns true if successful.
2380  bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II);
2381 
2382  /// Process '__has_include_next("path")' expression.
2383  ///
2384  /// Returns true if successful.
2385  bool EvaluateHasIncludeNext(Token &Tok, IdentifierInfo *II);
2386 
2387  /// Get the directory and file from which to start \#include_next lookup.
2388  std::pair<ConstSearchDirIterator, const FileEntry *>
2389  getIncludeNextStart(const Token &IncludeNextTok) const;
2390 
2391  /// Install the standard preprocessor pragmas:
2392  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
2393  void RegisterBuiltinPragmas();
2394 
2395  /// Register builtin macros such as __LINE__ with the identifier table.
2396  void RegisterBuiltinMacros();
2397 
2398  /// If an identifier token is read that is to be expanded as a macro, handle
2399  /// it and return the next token as 'Tok'. If we lexed a token, return true;
2400  /// otherwise the caller should lex again.
2401  bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2402 
2403  /// Cache macro expanded tokens for TokenLexers.
2404  //
2405  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2406  /// going to lex in the cache and when it finishes the tokens are removed
2407  /// from the end of the cache.
2408  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2409  ArrayRef<Token> tokens);
2410 
2411  void removeCachedMacroExpandedTokensOfLastLexer();
2412 
2413  /// Determine whether the next preprocessor token to be
2414  /// lexed is a '('. If so, consume the token and return true, if not, this
2415  /// method should have no observable side-effect on the lexed tokens.
2416  bool isNextPPTokenLParen();
2417 
2418  /// After reading "MACRO(", this method is invoked to read all of the formal
2419  /// arguments specified for the macro invocation. Returns null on error.
2420  MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2421  SourceLocation &MacroEnd);
2422 
2423  /// If an identifier token is read that is to be expanded
2424  /// as a builtin macro, handle it and return the next token as 'Tok'.
2425  void ExpandBuiltinMacro(Token &Tok);
2426 
2427  /// Read a \c _Pragma directive, slice it up, process it, then
2428  /// return the first token after the directive.
2429  /// This assumes that the \c _Pragma token has just been read into \p Tok.
2430  void Handle_Pragma(Token &Tok);
2431 
2432  /// Like Handle_Pragma except the pragma text is not enclosed within
2433  /// a string literal.
2434  void HandleMicrosoft__pragma(Token &Tok);
2435 
2436  /// Add a lexer to the top of the include stack and
2437  /// start lexing tokens from it instead of the current buffer.
2438  void EnterSourceFileWithLexer(Lexer *TheLexer, ConstSearchDirIterator Dir);
2439 
2440  /// Set the FileID for the preprocessor predefines.
2441  void setPredefinesFileID(FileID FID) {
2442  assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2443  PredefinesFileID = FID;
2444  }
2445 
2446  /// Set the FileID for the PCH through header.
2447  void setPCHThroughHeaderFileID(FileID FID);
2448 
2449  /// Returns true if we are lexing from a file and not a
2450  /// pragma or a macro.
2451  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2452  return L ? !L->isPragmaLexer() : P != nullptr;
2453  }
2454 
2455  static bool IsFileLexer(const IncludeStackInfo& I) {
2456  return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2457  }
2458 
2459  bool IsFileLexer() const {
2460  return IsFileLexer(CurLexer.get(), CurPPLexer);
2461  }
2462 
2463  //===--------------------------------------------------------------------===//
2464  // Caching stuff.
2465  void CachingLex(Token &Result);
2466 
2467  bool InCachingLexMode() const {
2468  // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2469  // that we are past EOF, not that we are in CachingLex mode.
2470  return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2471  }
2472 
2473  void EnterCachingLexMode();
2474  void EnterCachingLexModeUnchecked();
2475 
2476  void ExitCachingLexMode() {
2477  if (InCachingLexMode())
2479  }
2480 
2481  const Token &PeekAhead(unsigned N);
2482  void AnnotatePreviousCachedTokens(const Token &Tok);
2483 
2484  //===--------------------------------------------------------------------===//
2485  /// Handle*Directive - implement the various preprocessor directives. These
2486  /// should side-effect the current preprocessor object so that the next call
2487  /// to Lex() will return the appropriate token next.
2488  void HandleLineDirective();
2489  void HandleDigitDirective(Token &Tok);
2490  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2491  void HandleIdentSCCSDirective(Token &Tok);
2492  void HandleMacroPublicDirective(Token &Tok);
2493  void HandleMacroPrivateDirective();
2494 
2495  /// An additional notification that can be produced by a header inclusion or
2496  /// import to tell the parser what happened.
2497  struct ImportAction {
2498  enum ActionKind {
2499  None,
2500  ModuleBegin,
2501  ModuleImport,
2502  HeaderUnitImport,
2503  SkippedModuleImport,
2504  Failure,
2505  } Kind;
2506  Module *ModuleForHeader = nullptr;
2507 
2508  ImportAction(ActionKind AK, Module *Mod = nullptr)
2509  : Kind(AK), ModuleForHeader(Mod) {
2510  assert((AK == None || Mod || AK == Failure) &&
2511  "no module for module action");
2512  }
2513  };
2514 
2515  Optional<FileEntryRef> LookupHeaderIncludeOrImport(
2516  ConstSearchDirIterator *CurDir, StringRef &Filename,
2517  SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2518  const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
2519  bool &IsMapped, ConstSearchDirIterator LookupFrom,
2520  const FileEntry *LookupFromFile, StringRef &LookupFilename,
2521  SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2522  ModuleMap::KnownHeader &SuggestedModule, bool isAngled);
2523 
2524  // File inclusion.
2525  void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2526  ConstSearchDirIterator LookupFrom = nullptr,
2527  const FileEntry *LookupFromFile = nullptr);
2528  ImportAction
2529  HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2530  Token &FilenameTok, SourceLocation EndLoc,
2531  ConstSearchDirIterator LookupFrom = nullptr,
2532  const FileEntry *LookupFromFile = nullptr);
2533  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2534  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2535  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2536  void HandleMicrosoftImportDirective(Token &Tok);
2537 
2538 public:
2539  /// Check that the given module is available, producing a diagnostic if not.
2540  /// \return \c true if the check failed (because the module is not available).
2541  /// \c false if the module appears to be usable.
2542  static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2543  const TargetInfo &TargetInfo,
2544  DiagnosticsEngine &Diags, Module *M);
2545 
2546  // Module inclusion testing.
2547  /// Find the module that owns the source or header file that
2548  /// \p Loc points to. If the location is in a file that was included
2549  /// into a module, or is outside any module, returns nullptr.
2550  Module *getModuleForLocation(SourceLocation Loc, bool AllowTextual);
2551 
2552  /// We want to produce a diagnostic at location IncLoc concerning an
2553  /// unreachable effect at location MLoc (eg, where a desired entity was
2554  /// declared or defined). Determine whether the right way to make MLoc
2555  /// reachable is by #include, and if so, what header should be included.
2556  ///
2557  /// This is not necessarily fast, and might load unexpected module maps, so
2558  /// should only be called by code that intends to produce an error.
2559  ///
2560  /// \param IncLoc The location at which the missing effect was detected.
2561  /// \param MLoc A location within an unimported module at which the desired
2562  /// effect occurred.
2563  /// \return A file that can be #included to provide the desired effect. Null
2564  /// if no such file could be determined or if a #include is not
2565  /// appropriate (eg, if a module should be imported instead).
2566  const FileEntry *getHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2567  SourceLocation MLoc);
2568 
2569  bool isRecordingPreamble() const {
2570  return PreambleConditionalStack.isRecording();
2571  }
2572 
2573  bool hasRecordedPreamble() const {
2574  return PreambleConditionalStack.hasRecordedPreamble();
2575  }
2576 
2578  return PreambleConditionalStack.getStack();
2579  }
2580 
2582  PreambleConditionalStack.setStack(s);
2583  }
2584 
2587  PreambleConditionalStack.startReplaying();
2588  PreambleConditionalStack.setStack(s);
2589  PreambleConditionalStack.SkipInfo = SkipInfo;
2590  }
2591 
2593  return PreambleConditionalStack.SkipInfo;
2594  }
2595 
2596 private:
2597  /// After processing predefined file, initialize the conditional stack from
2598  /// the preamble.
2599  void replayPreambleConditionalStack();
2600 
2601  // Macro handling.
2602  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2603  void HandleUndefDirective();
2604 
2605  // Conditional Inclusion.
2606  void HandleIfdefDirective(Token &Result, const Token &HashToken,
2607  bool isIfndef, bool ReadAnyTokensBeforeDirective);
2608  void HandleIfDirective(Token &IfToken, const Token &HashToken,
2609  bool ReadAnyTokensBeforeDirective);
2610  void HandleEndifDirective(Token &EndifToken);
2611  void HandleElseDirective(Token &Result, const Token &HashToken);
2612  void HandleElifFamilyDirective(Token &ElifToken, const Token &HashToken,
2614 
2615  // Pragmas.
2616  void HandlePragmaDirective(PragmaIntroducer Introducer);
2617 
2618 public:
2619  void HandlePragmaOnce(Token &OnceTok);
2620  void HandlePragmaMark(Token &MarkTok);
2621  void HandlePragmaPoison();
2622  void HandlePragmaSystemHeader(Token &SysHeaderTok);
2623  void HandlePragmaDependency(Token &DependencyTok);
2624  void HandlePragmaPushMacro(Token &Tok);
2625  void HandlePragmaPopMacro(Token &Tok);
2626  void HandlePragmaIncludeAlias(Token &Tok);
2627  void HandlePragmaModuleBuild(Token &Tok);
2628  void HandlePragmaHdrstop(Token &Tok);
2630 
2631  // Return true and store the first token only if any CommentHandler
2632  // has inserted some tokens and getCommentRetentionState() is false.
2633  bool HandleComment(Token &result, SourceRange Comment);
2634 
2635  /// A macro is used, update information about macros that need unused
2636  /// warnings.
2637  void markMacroAsUsed(MacroInfo *MI);
2638 
2640  SourceLocation AnnotationLoc) {
2641  auto Annotations = AnnotationInfos.find(II);
2642  if (Annotations == AnnotationInfos.end())
2643  AnnotationInfos.insert(std::make_pair(
2644  II,
2645  MacroAnnotations::makeDeprecation(AnnotationLoc, std::move(Msg))));
2646  else
2647  Annotations->second.DeprecationInfo =
2648  MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2649  }
2650 
2652  SourceLocation AnnotationLoc) {
2653  auto Annotations = AnnotationInfos.find(II);
2654  if (Annotations == AnnotationInfos.end())
2655  AnnotationInfos.insert(
2656  std::make_pair(II, MacroAnnotations::makeRestrictExpansion(
2657  AnnotationLoc, std::move(Msg))));
2658  else
2659  Annotations->second.RestrictExpansionInfo =
2660  MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2661  }
2662 
2663  void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc) {
2664  auto Annotations = AnnotationInfos.find(II);
2665  if (Annotations == AnnotationInfos.end())
2666  AnnotationInfos.insert(
2667  std::make_pair(II, MacroAnnotations::makeFinal(AnnotationLoc)));
2668  else
2669  Annotations->second.FinalAnnotationLoc = AnnotationLoc;
2670  }
2671 
2672  const MacroAnnotations &getMacroAnnotations(const IdentifierInfo *II) const {
2673  return AnnotationInfos.find(II)->second;
2674  }
2675 
2677  if (Identifier.getIdentifierInfo()->isDeprecatedMacro())
2678  emitMacroDeprecationWarning(Identifier);
2679 
2680  if (Identifier.getIdentifierInfo()->isRestrictExpansion() &&
2681  !SourceMgr.isInMainFile(Identifier.getLocation()))
2682  emitRestrictExpansionWarning(Identifier);
2683  }
2684 
2686  const LangOptions &LangOpts,
2687  const TargetInfo &TI);
2688 
2689 private:
2690  void emitMacroDeprecationWarning(const Token &Identifier) const;
2691  void emitRestrictExpansionWarning(const Token &Identifier) const;
2692  void emitFinalMacroWarning(const Token &Identifier, bool IsUndef) const;
2693 };
2694 
2695 /// Abstract base class that describes a handler that will receive
2696 /// source ranges for each of the comments encountered in the source file.
2698 public:
2699  virtual ~CommentHandler();
2700 
2701  // The handler shall return true if it has pushed any tokens
2702  // to be read using e.g. EnterToken or EnterTokenStream.
2703  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2704 };
2705 
2706 /// Abstract base class that describes a handler that will receive
2707 /// source ranges for empty lines encountered in the source file.
2709 public:
2710  virtual ~EmptylineHandler();
2711 
2712  // The handler handles empty lines.
2713  virtual void HandleEmptyline(SourceRange Range) = 0;
2714 };
2715 
2716 /// Registry of pragma handlers added by plugins
2717 using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2718 
2719 } // namespace clang
2720 
2721 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H
clang::MacroDirective::DefInfo::getDirective
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:372
clang::Preprocessor::isParsingIfOrElifDirective
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
Definition: Preprocessor.h:1100
clang::Preprocessor::PreambleSkipInfo::ElseLoc
SourceLocation ElseLoc
Definition: Preprocessor.h:535
clang::Preprocessor::MaybeHandlePoisonedIdentifier
void MaybeHandlePoisonedIdentifier(Token &Identifier)
Definition: Preprocessor.h:2103
clang::VariadicMacroScopeGuard
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
Definition: VariadicMacroSupport.h:30
clang::Lexer::isAtEndOfMacroExpansion
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
Definition: Lexer.cpp:831
clang::Preprocessor::HandlePragmaPopMacro
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
Definition: Pragma.cpp:638
clang::IdentifierInfo::hasMacroDefinition
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
Definition: IdentifierTable.h:202
clang::Preprocessor::HandlePragmaSystemHeader
void HandlePragmaSystemHeader(Token &SysHeaderTok)
HandlePragmaSystemHeader - Implement #pragma GCC system_header.
Definition: Pragma.cpp:472
clang::TokenValue
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:90
clang::Preprocessor::setSkipMainFilePreamble
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
Definition: Preprocessor.h:1891
clang::prec::Conditional
@ Conditional
Definition: OperatorPrecedence.h:30
clang::Preprocessor::CreateString
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
Definition: Preprocessor.cpp:488
clang::Preprocessor::enableIncrementalProcessing
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
Definition: Preprocessor.h:1784
clang::Preprocessor::getCurrentFPEvalMethod
LangOptions::FPEvalMethodKind getCurrentFPEvalMethod() const
Definition: Preprocessor.h:2184
clang::Preprocessor::RemovePragmaHandler
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
Definition: Pragma.cpp:934
clang::Preprocessor::getSpelling
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
Definition: Preprocessor.h:1928
llvm
YAML serialization mapping.
Definition: Dominators.h:30
clang::HeaderSearch::getFileInfo
HeaderFileInfo & getFileInfo(const FileEntry *FE)
Return the HeaderFileInfo structure for the specified FileEntry, in preparation for updating it in so...
Definition: HeaderSearch.cpp:1315
clang::Preprocessor::AdvanceToTokenCharacter
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
Definition: Preprocessor.h:2056
clang::Preprocessor::CheckMacroName
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
Definition: PPDirectives.cpp:314
clang::Preprocessor::getPreprocessorAllocator
llvm::BumpPtrAllocator & getPreprocessorAllocator()
Definition: Preprocessor.h:1076
clang::ExternalPreprocessorSource
Abstract interface for external sources of preprocessor information.
Definition: ExternalPreprocessorSource.h:26
clang::Preprocessor::PrintStats
void PrintStats()
Definition: Preprocessor.cpp:273
clang::Preprocessor::FinalizeForModelFile
void FinalizeForModelFile()
Cleanup after model file parsing.
Definition: Preprocessor.cpp:228
clang::TokenValue::TokenValue
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:95
clang::DiagnosticBuilder
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:1266
clang::Preprocessor::setPragmaAssumeNonNullLoc
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
Definition: Preprocessor.h:1860
clang::Preprocessor::FinishLexStringLiteral
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
Definition: Preprocessor.cpp:1333
clang::Preprocessor::setCodeCompletionReached
void setCodeCompletionReached()
Note that we hit the code-completion point.
Definition: Preprocessor.h:1827
clang::ModuleMap::KnownHeader
A header that is known to reside within a given module, whether it was included or excluded.
Definition: ModuleMap.h:160
clang::Preprocessor::EnterTokenStream
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
Definition: Preprocessor.h:1548
clang::Preprocessor::LookAhead
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
Definition: Preprocessor.h:1669
clang::Preprocessor::setRecordedPreambleConditionalStack
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
Definition: Preprocessor.h:2581
clang::Preprocessor::getMacroDefinition
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:1217
clang::DeclaratorContext::File
@ File
clang::Preprocessor::SetCodeCompletionPoint
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
Definition: Preprocessor.cpp:394
clang::SourceRange
A trivial tuple used to represent a source range.
Definition: SourceLocation.h:210
string
string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} 0 ${PATH_LIB_START} PATH_HEAD) string(SUBSTRING $
Definition: CMakeLists.txt:22
clang::PreprocessorOptions
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
Definition: PreprocessorOptions.h:65
clang::Preprocessor::getModuleMacro
ModuleMacro * getModuleMacro(Module *Mod, const IdentifierInfo *II)
Definition: PPMacroExpansion.cpp:172
clang::Preprocessor::HandlePragmaMark
void HandlePragmaMark(Token &MarkTok)
Definition: Pragma.cpp:420
clang::Preprocessor::setCodeCompletionHandler
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
Definition: Preprocessor.h:1423
clang::Preprocessor::HandleEndOfTokenLexer
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
Definition: PPLexerChange.cpp:586
clang::Preprocessor::addModuleMacro
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
Definition: PPMacroExpansion.cpp:133
clang::Preprocessor::getLastMacroWithSpelling
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
Definition: Preprocessor.cpp:356
clang::Lexer::getRawToken
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
Definition: Lexer.cpp:462
clang::Preprocessor::getPPCallbacks
PPCallbacks * getPPCallbacks() const
Definition: Preprocessor.h:1162
clang::Preprocessor::setCodeCompletionTokenRange
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
Definition: Preprocessor.h:1452
clang::Preprocessor::isIncrementalProcessingEnabled
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
Definition: Preprocessor.h:1781
clang::Preprocessor::Lex
void Lex(Token &Result)
Lex the next token for this preprocessor.
Definition: Preprocessor.cpp:881
clang::Preprocessor::alreadyIncluded
bool alreadyIncluded(const FileEntry *File) const
Return true if this header has already been included.
Definition: Preprocessor.h:1357
clang::Preprocessor::HandleIdentifier
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
Definition: Preprocessor.cpp:789
clang::Preprocessor::getLocalMacroDirectiveHistory
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
Definition: PPMacroExpansion.cpp:64
ModuleMap.h
clang::Preprocessor::isAtEndOfMacroExpansion
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
Definition: Preprocessor.h:2043
clang::Preprocessor::setPreambleRecordedPragmaAssumeNonNullLoc
void setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc)
Record the location of the unterminated #pragma clang assume_nonnull begin in the preamble.
Definition: Preprocessor.h:1875
clang::Preprocessor::getCodeCompletionTokenRange
SourceRange getCodeCompletionTokenRange() const
Definition: Preprocessor.h:1456
clang::Preprocessor::macro_iterator
MacroMap::const_iterator macro_iterator
Definition: Preprocessor.h:1318
clang::Token::getLiteralData
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:219
clang::Preprocessor::emitMacroExpansionWarnings
void emitMacroExpansionWarnings(const Token &Identifier) const
Definition: Preprocessor.h:2676
clang::Lexer::getSpelling
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer,...
Definition: Lexer.cpp:404
clang::MacroDefinition
A description of the current definition of a macro.
Definition: MacroInfo.h:587
clang::Preprocessor::LexHeaderName
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
Definition: Preprocessor.cpp:997
clang::CommentHandler::HandleComment
virtual bool HandleComment(Preprocessor &PP, SourceRange Comment)=0
llvm::SmallVector
Definition: LLVM.h:38
clang::IdentifierTable::get
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
Definition: IdentifierTable.h:597
clang::Preprocessor::getMaxTokens
unsigned getMaxTokens() const
Get the max number of tokens before issuing a -Wmax-tokens warning.
Definition: Preprocessor.h:1175
clang::SourceLocation
Encodes a location in the source.
Definition: SourceLocation.h:86
clang::Preprocessor::TUKind
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
Definition: Preprocessor.h:292
clang::PPCallbacks
This interface provides a way to observe the actions of the preprocessor as it does its thing.
Definition: PPCallbacks.h:35
clang::Preprocessor::getCommentRetentionState
bool getCommentRetentionState() const
Definition: Preprocessor.h:1110
clang::driver::phases::Preprocess
@ Preprocess
Definition: Phases.h:18
clang::Preprocessor::RemoveTopOfLexerStack
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
Definition: PPLexerChange.cpp:607
clang::Preprocessor::IgnorePragmas
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
Definition: Pragma.cpp:2143
clang::MultiVersionKind::Target
@ Target
clang::PreprocessorLexer
Definition: PreprocessorLexer.h:30
clang::Preprocessor::getAffectingClangModules
const llvm::SmallSetVector< Module *, 2 > & getAffectingClangModules() const
Get the set of top-level clang modules that affected preprocessing, but were not imported.
Definition: Preprocessor.h:1345
clang::tooling::Filter
llvm::cl::opt< std::string > Filter
clang::Token::getIdentifierInfo
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:181
clang::MacroArgs
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
Filename
StringRef Filename
Definition: Format.cpp:2715
clang::Preprocessor::setExternalSource
void setExternalSource(ExternalPreprocessorSource *Source)
Definition: Preprocessor.h:1078
clang::DiagnosticsEngine
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
clang::TargetInfo
Exposes information about the current target.
Definition: TargetInfo.h:205
clang::PreprocessingRecord
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
Definition: PreprocessingRecord.h:306
clang::Preprocessor::setLastFPEvalMethod
void setLastFPEvalMethod(LangOptions::FPEvalMethodKind Val)
Definition: Preprocessor.h:2203
Module.h
clang::IdentifierInfo::isOutOfDate
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
Definition: IdentifierTable.h:415
clang::Preprocessor::CodeCompleteIncludedFile
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
Definition: Preprocessor.cpp:450
llvm::Optional
Definition: LLVM.h:40
clang::Preprocessor::macro_end
macro_iterator macro_end(bool IncludeExternalMacros=true) const
Definition: Preprocessor.cpp:339
clang::ModuleLoader::HadFatalFailure
bool HadFatalFailure
Definition: ModuleLoader.h:162
clang::MacroInfo
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
SourceManager.h
clang::Preprocessor::getCurrentLexer
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
Definition: Preprocessor.h:1142
clang::Preprocessor::appendDefMacroDirective
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
Definition: Preprocessor.h:1285
clang::Preprocessor::setReplayablePreambleConditionalStack
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, llvm::Optional< PreambleSkipInfo > SkipInfo)
Definition: Preprocessor.h:2585
clang::TokenValue::operator==
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:105
clang::Preprocessor::EnterToken
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
Definition: Preprocessor.h:1698
clang::Preprocessor::HandleMicrosoftCommentPaste
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
Definition: PPLexerChange.cpp:624
clang::DiagnosticsEngine::setSuppressAllDiagnostics
void setSuppressAllDiagnostics(bool Val)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
Definition: Diagnostic.h:692
DiagnosticIDs.h
clang::Preprocessor::getCurrentFileLexer
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
Definition: PPLexerChange.cpp:49
clang::Preprocessor::IncrementPasteCounter
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Definition: Preprocessor.h:2065
clang::Preprocessor::MacroArgs
friend class MacroArgs
Definition: Preprocessor.h:549
Identifier
StringRef Identifier
Definition: Format.cpp:2723
clang::Token
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
clang::CommentHandler
Abstract base class that describes a handler that will receive source ranges for each of the comments...
Definition: Preprocessor.h:2697
clang::Preprocessor::AddPragmaHandler
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
Definition: Pragma.cpp:903
clang::Preprocessor::ReplaceLastTokenWithAnnotation
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
Definition: Preprocessor.h:1752
End
SourceLocation End
Definition: USRLocFinder.cpp:167
TokenLexer.h
clang::Preprocessor::ReplacePreviousCachedToken
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
Definition: PPCaching.cpp:157
clang::Preprocessor::setPredefines
void setPredefines(std::string P)
Set the predefines for this Preprocessor.
Definition: Preprocessor.h:1379
clang::Preprocessor::isCodeCompletionReached
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
Definition: Preprocessor.h:1824
clang::CodeCompletionHandler
Callback handler that receives notifications when performing code completion within the preprocessor.
Definition: CodeCompletionHandler.h:25
clang::Preprocessor::getIdentifierTable
IdentifierTable & getIdentifierTable()
Definition: Preprocessor.h:1072
clang::Preprocessor::addFinalLoc
void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc)
Definition: Preprocessor.h:2663
clang::Preprocessor::PreambleSkipInfo::PreambleSkipInfo
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
Definition: Preprocessor.h:537
clang::SourceManager
This class handles loading and caching of source files into memory.
Definition: SourceManager.h:636
clang::Preprocessor::setPragmasEnabled
void setPragmasEnabled(bool Enabled)
Definition: Preprocessor.h:1112
clang::Preprocessor::processPathForFileMacro
static void processPathForFileMacro(SmallVectorImpl< char > &Path, const LangOptions &LangOpts, const TargetInfo &TI)
Definition: PPMacroExpansion.cpp:1986
clang::Preprocessor::getLangOpts
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:1065
clang::Preprocessor::getImmediateMacroName
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
Definition: Preprocessor.h:1990
clang::HeaderSearch
Encapsulates the information needed to find the file referenced by a #include or #include_next,...
Definition: HeaderSearch.h:223
clang::Preprocessor::creatingPCHWithPragmaHdrStop
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
Definition: Preprocessor.cpp:626
clang::Preprocessor::AnnotateCachedTokens
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
Definition: Preprocessor.h:1720
Offset
unsigned Offset
Definition: Format.cpp:2717
clang::Preprocessor::getPragmasEnabled
bool getPragmasEnabled() const
Definition: Preprocessor.h:1113
clang::Preprocessor::usingPCHWithThroughHeader
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
Definition: Preprocessor.cpp:621
BuiltinInfo
static const Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:21
TokenKinds.h
clang::Preprocessor::GetIncludeFilenameSpelling
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Buffer)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
Definition: PPDirectives.cpp:1756
clang::Preprocessor::getMacroDefinitionAtLoc
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
Definition: Preprocessor.h:1230
clang::Preprocessor::PreambleSkipInfo::HashTokenLoc
SourceLocation HashTokenLoc
Definition: Preprocessor.h:531
clang::Preprocessor::getCurrentModuleImplementation
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
Definition: Preprocessor.cpp:531
V
#define V(N, I)
Definition: ASTContext.h:3235
clang::Preprocessor::HandlePragmaOnce
void HandlePragmaOnce(Token &OnceTok)
HandlePragmaOnce - Handle #pragma once. OnceTok is the 'once'.
Definition: Pragma.cpp:405
clang::Preprocessor::AllocateMacroInfo
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
Definition: PPDirectives.cpp:59
clang::Preprocessor::SetPoisonReason
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
Definition: Preprocessor.cpp:747
clang::Module
Describes a module or submodule.
Definition: Module.h:98
clang::Preprocessor::TypoCorrectToken
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
Definition: Preprocessor.h:1770
clang::Preprocessor::EnterSourceFile
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
Definition: PPLexerChange.cpp:68
clang::Preprocessor::getBuildingSubmodules
ArrayRef< BuildingSubmoduleInfo > getBuildingSubmodules() const
Get the list of submodules that we're currently building.
Definition: Preprocessor.h:1310
clang::Preprocessor::getPreambleSkipInfo
llvm::Optional< PreambleSkipInfo > getPreambleSkipInfo() const
Definition: Preprocessor.h:2592
clang::Preprocessor::SetCommentRetentionState
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Definition: Preprocessor.h:1105
clang::Preprocessor::getIdentifierTable
const IdentifierTable & getIdentifierTable() const
Definition: Preprocessor.h:1073
clang::frontend::ActionKind
ActionKind
Definition: FrontendOptions.h:35
clang::Preprocessor::EndSourceFile
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
Definition: Preprocessor.cpp:699
clang::Preprocessor::ParsePragmaPushOrPopMacro
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.
Definition: Pragma.cpp:560
clang::Preprocessor::getCounterValue
unsigned getCounterValue() const
Definition: Preprocessor.h:2181
clang::Preprocessor::setLoadedMacroDirective
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
Definition: PPMacroExpansion.cpp:97
clang::DirectoryEntry
Cached information about one directory (either on disk or in the virtual file system).
Definition: DirectoryEntry.h:36
Id
int Id
Definition: ASTDiff.cpp:189
clang::Preprocessor::getSpellingOfSingleCharacterNumericConstant
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character.
Definition: Preprocessor.h:1967
hlsl::uint64_t
unsigned long uint64_t
Definition: hlsl_basic_types.h:25
clang::XRayInstrKind::None
constexpr XRayInstrMask None
Definition: XRayInstr.h:38
clang::Preprocessor::PreambleSkipInfo::FoundNonSkipPortion
bool FoundNonSkipPortion
Definition: Preprocessor.h:533
clang::Preprocessor::InitializeForModelFile
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
Definition: Preprocessor.cpp:216
clang::Preprocessor::setPreprocessToken
void setPreprocessToken(bool Preprocess)
Definition: Preprocessor.h:1191
clang::Preprocessor::addCommentHandler
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
Definition: Preprocessor.cpp:1394
clang::Token::getKind
tok::TokenKind getKind() const
Definition: Token.h:93
clang::index::SymbolKind::Module
@ Module
clang::Preprocessor::isAtStartOfMacroExpansion
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion.
Definition: Preprocessor.h:2032
clang::Preprocessor::getLastFPEvalPragmaLocation
SourceLocation getLastFPEvalPragmaLocation() const
Definition: Preprocessor.h:2195
LangOptions.h
clang::Preprocessor::isPCHThroughHeader
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
Definition: Preprocessor.cpp:610
Diagnostic.h
clang::Preprocessor::setTUFPEvalMethod
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
Definition: Preprocessor.h:2218
clang::LangOptions::FPEvalMethodKind
FPEvalMethodKind
Possible float expression evaluation method choices.
Definition: LangOptions.h:281
clang::Preprocessor::getSelectorTable
SelectorTable & getSelectorTable()
Definition: Preprocessor.h:1074
clang::Preprocessor::Diag
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
Definition: Preprocessor.h:1899
clang::Preprocessor::Diag
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
Definition: Preprocessor.h:1903
ModuleLoader.h
clang::Token::is
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:98
clang::Preprocessor::getTargetInfo
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:1066
clang::Preprocessor::createPreprocessingRecord
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
Definition: Preprocessor.cpp:1472
clang::Preprocessor::isMacroDefined
bool isMacroDefined(StringRef Id)
Definition: Preprocessor.h:1193
IdentifierTable.h
clang::Preprocessor::isBacktrackEnabled
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
Definition: Preprocessor.h:1584
clang::Preprocessor::EnterSubmodule
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
Definition: PPLexerChange.cpp:684
clang::Token::needsCleaning
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:287
clang::SelectorTable
This table allows us to fully hide how we implement multi-keyword caching.
Definition: IdentifierTable.h:904
bool
#define bool
Definition: stdbool.h:20
clang::Preprocessor::AddPragmaHandler
void AddPragmaHandler(PragmaHandler *Handler)
Definition: Preprocessor.h:1392
clang::Preprocessor::HandlePragmaPushMacro
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
Definition: Pragma.cpp:615
clang::Preprocessor::getCodeCompletionHandler
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
Definition: Preprocessor.h:1428
clang::Preprocessor::getIncludedFiles
IncludedFilesSet & getIncludedFiles()
Get the set of included files.
Definition: Preprocessor.h:1362
clang::Preprocessor::LexAfterModuleImport
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
Definition: Preprocessor.cpp:1140
clang::Preprocessor::HandleSkippedDirectiveWhileUsingPCH
void HandleSkippedDirectiveWhileUsingPCH(Token &Result, SourceLocation HashLoc)
Process directives while skipping until the through header or #pragma hdrstop is found.
Definition: PPDirectives.cpp:1111
llvm::DenseSet< const FileEntry * >
clang::Preprocessor::getMacroInfo
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
Definition: Preprocessor.h:1258
clang::PragmaIntroducer
Describes how and where the pragma was introduced.
Definition: Pragma.h:51
clang::EmptylineHandler
Abstract base class that describes a handler that will receive source ranges for empty lines encounte...
Definition: Preprocessor.h:2708
clang::CommentHandler::~CommentHandler
virtual ~CommentHandler()
clang::TokenValue::TokenValue
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:103
clang::Preprocessor::setCounterValue
void setCounterValue(unsigned V)
Definition: Preprocessor.h:2182
clang::Preprocessor::EnterMainSourceFile
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
Definition: Preprocessor.cpp:544
clang::Preprocessor::GetSuppressIncludeNotFoundError
bool GetSuppressIncludeNotFoundError()
Definition: Preprocessor.h:1119
clang::Lexer::getImmediateMacroName
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
Definition: Lexer.cpp:997
clang::EmptylineHandler::HandleEmptyline
virtual void HandleEmptyline(SourceRange Range)=0
clang::Preprocessor::getPreambleConditionalStack
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
Definition: Preprocessor.h:2577
clang::Preprocessor::getCodeCompletionFilter
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Definition: Preprocessor.h:1461
Line
const AnnotatedLine * Line
Definition: UsingDeclarationsSorter.cpp:68
clang::Preprocessor::getLastCachedTokenLocation
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
Definition: Preprocessor.h:1728
clang::Preprocessor::getRawToken
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
Definition: Preprocessor.h:1959
clang::FileEntry
Cached information about one file (either on disk or in the virtual file system).
Definition: FileEntry.h:366
clang::Preprocessor::SplitToken
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Definition: Preprocessor.cpp:508
clang::MU_Define
@ MU_Define
Definition: Preprocessor.h:117
clang::ConstSearchDirIterator
detail::SearchDirIteratorImpl< true > ConstSearchDirIterator
Definition: HeaderSearch.h:215
clang::Preprocessor::getLocalMacroDirective
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
Definition: Preprocessor.h:1247
clang::Preprocessor::LexUnexpandedNonComment
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
Definition: Preprocessor.h:1645
clang::Token::getLength
unsigned getLength() const
Definition: Token.h:129
clang::Preprocessor::getIncludedFiles
const IncludedFilesSet & getIncludedFiles() const
Definition: Preprocessor.h:1363
clang::PragmaHandler
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:65
clang::Module::isModuleMapModule
bool isModuleMapModule() const
Definition: Module.h:176
clang::DefMacroDirective
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:429
clang::MU_Undef
@ MU_Undef
Definition: Preprocessor.h:120
clang::Preprocessor::getSourceManager
SourceManager & getSourceManager() const
Definition: Preprocessor.h:1069
clang::Preprocessor::HandlePragmaModuleBuild
void HandlePragmaModuleBuild(Token &Tok)
Definition: Pragma.cpp:795
clang::LangOptions::FEM_UnsetOnCommandLine
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
Definition: LangOptions.h:295
clang::transformer::EditKind::Range
@ Range
clang::Preprocessor::HandleComment
bool HandleComment(Token &result, SourceRange Comment)
Definition: Preprocessor.cpp:1408
clang::Preprocessor::LexOnOffSwitch
bool LexOnOffSwitch(tok::OnOffSwitch &Result)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD.
Definition: Pragma.cpp:956
clang::Preprocessor::getSpelling
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
Definition: Preprocessor.h:1914
clang::Preprocessor::PoisonSEHIdentifiers
void PoisonSEHIdentifiers(bool Poison=true)
Definition: Preprocessor.cpp:751
clang::Preprocessor::getTokenCount
unsigned getTokenCount() const
Get the number of tokens processed so far.
Definition: Preprocessor.h:1172
clang::Preprocessor::SetMacroExpansionOnlyInDirectives
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
Definition: Preprocessor.h:1657
clang::Preprocessor::PreambleSkipInfo::FoundElse
bool FoundElse
Definition: Preprocessor.h:534
clang::Preprocessor::CollectPpImportSuffix
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
Definition: Preprocessor.cpp:1086
clang::Preprocessor::getAuxTargetInfo
const TargetInfo * getAuxTargetInfo() const
Definition: Preprocessor.h:1067
SourceLocation.h
clang::Preprocessor::SkipTokensWhileUsingPCH
void SkipTokensWhileUsingPCH()
Skip tokens until after the include of the through header or until after a #pragma hdrstop.
Definition: Preprocessor.cpp:639
clang::Preprocessor::getTotalMemory
size_t getTotalMemory() const
Definition: Preprocessor.cpp:326
clang::tok::TokenKind
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
clang::Preprocessor::CodeCompleteNaturalLanguage
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
Definition: Preprocessor.cpp:457
P
StringRef P
Definition: ASTMatchersInternal.cpp:563
clang::detail::SearchDirIteratorImpl< true >
clang::Preprocessor::HandlePragmaPoison
void HandlePragmaPoison()
HandlePragmaPoison - Handle #pragma GCC poison. PoisonTok is the 'poison'.
Definition: Pragma.cpp:430
clang::Preprocessor::HandleDirective
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
Definition: PPDirectives.cpp:1136
clang::Preprocessor::getBuiltinInfo
Builtin::Context & getBuiltinInfo()
Definition: Preprocessor.h:1075
clang::Preprocessor::hadModuleLoaderFatalFailure
bool hadModuleLoaderFatalFailure() const
Definition: Preprocessor.h:1089
clang::Preprocessor::getCodeCompletionFileLoc
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
Definition: Preprocessor.h:1818
clang::Preprocessor::DumpLocation
void DumpLocation(SourceLocation Loc) const
Definition: Preprocessor.cpp:260
clang::Preprocessor::setPreprocessedOutput
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
Definition: Preprocessor.h:1125
clang::Preprocessor::usingPCHWithPragmaHdrStop
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
Definition: Preprocessor.cpp:630
clang::Preprocessor::isCodeCompletionEnabled
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Definition: Preprocessor.h:1806
clang::Preprocessor::getPragmaARCCFCodeAuditedInfo
std::pair< IdentifierInfo *, SourceLocation > getPragmaARCCFCodeAuditedInfo() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
Definition: Preprocessor.h:1839
clang::Preprocessor::dumpMacroInfo
void dumpMacroInfo(const IdentifierInfo *II)
Definition: PPMacroExpansion.cpp:259
llvm::ArrayRef
Definition: LLVM.h:34
Lexer.h
Value
Value
Definition: UninitializedValues.cpp:103
clang::Preprocessor::hasRecordedPreamble
bool hasRecordedPreamble() const
Definition: Preprocessor.h:2573
clang::Preprocessor::setCodeCompletionIdentifierInfo
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
Definition: Preprocessor.h:1446
clang::Preprocessor::setEmptylineHandler
void setEmptylineHandler(EmptylineHandler *Handler)
Set empty line handler.
Definition: Preprocessor.h:1410
clang::Preprocessor::EnableBacktrackAtThisPos
void EnableBacktrackAtThisPos()
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
Definition: PPCaching.cpp:25
clang::tok::PPKeywordKind
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a '#' at the beginning of the line.
Definition: TokenKinds.h:33
HeaderSearch.h
clang::Preprocessor::getPragmaAssumeNonNullLoc
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
Definition: Preprocessor.h:1854
clang::Preprocessor::PreambleSkipInfo::IfTokenLoc
SourceLocation IfTokenLoc
Definition: Preprocessor.h:532
clang::Preprocessor::getCurrentLexerSubmodule
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
Definition: Preprocessor.h:1152
clang::syntax::NodeRole::Message
@ Message
clang::Preprocessor::appendDefMacroDirective
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
Definition: Preprocessor.h:1279
LLVM.h
clang::Preprocessor::DumpToken
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
Definition: Preprocessor.cpp:234
clang::Preprocessor::markMacroAsUsed
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
Definition: PPMacroExpansion.cpp:1978
clang::Preprocessor::setCurrentFPEvalMethod
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
Definition: Preprocessor.h:2207
State
LineState State
Definition: UnwrappedLineFormatter.cpp:1147
clang::Preprocessor::isMacroDefined
bool isMacroDefined(const IdentifierInfo *II)
Definition: Preprocessor.h:1196
clang::Preprocessor::getFileManager
FileManager & getFileManager() const
Definition: Preprocessor.h:1068
clang::VAOptDefinitionContext
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a variadic mac...
Definition: VariadicMacroSupport.h:66
clang::Preprocessor::markClangModuleAsAffecting
void markClangModuleAsAffecting(Module *M)
Mark the given clang module as affecting the current clang module or translation unit.
Definition: Preprocessor.h:1333
clang::Preprocessor::getCurrentModule
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
Definition: Preprocessor.cpp:524
clang::Preprocessor::Backtrack
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
Definition: PPCaching.cpp:40
clang::Preprocessor::isPreprocessedOutput
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
Definition: Preprocessor.h:1131
clang::Preprocessor::IsPreviousCachedToken
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
Definition: PPCaching.cpp:139
clang::ModuleLoader
Abstract interface for a module loader.
Definition: ModuleLoader.h:82
clang::IdentifierInfo
One of these records is kept for each identifier that is lexed.
Definition: IdentifierTable.h:85
clang::Token::getLocation
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:126
clang::Preprocessor::LexStringLiteral
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
Definition: Preprocessor.h:1604
clang::LangOptions
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:81
clang::ObjCPropertyAttribute::Kind
Kind
Definition: DeclObjCCommon.h:22
clang::Token::isAnnotation
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:120
clang::Preprocessor::removeCommentHandler
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Definition: Preprocessor.cpp:1401
clang::Preprocessor::getModuleImportLoc
SourceLocation getModuleImportLoc(Module *M) const
Definition: Preprocessor.h:1597
clang::Preprocessor::getPreprocessorOpts
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Definition: Preprocessor.h:1060
clang::ASTReader
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:341
clang::Preprocessor::IncludedFilesSet
llvm::DenseSet< const FileEntry * > IncludedFilesSet
Definition: Preprocessor.h:545
std
Definition: Format.h:4477
clang::Preprocessor::getModuleLoader
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
Definition: Preprocessor.h:1087
clang::TranslationUnitKind
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:914
clang::Preprocessor::EnterMacro
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
Definition: PPLexerChange.cpp:150
clang::Preprocessor::getSpelling
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
Definition: Preprocessor.h:1944
clang::Preprocessor::getCodeCompletionLoc
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
Definition: Preprocessor.h:1812
clang::Preprocessor::CheckEndOfDirective
SourceLocation CheckEndOfDirective(const char *DirType, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
Definition: PPDirectives.cpp:415
clang::Preprocessor::clearCodeCompletionHandler
void clearCodeCompletionHandler()
Clear out the code completion handler.
Definition: Preprocessor.h:1433
clang
Definition: CalledOnceCheck.h:17
clang::FileManager
Implements support for file system lookup, file system caching, and directory search management.
Definition: FileManager.h:53
clang::Preprocessor::addPPCallbacks
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:1163
clang::Preprocessor::getModuleForLocation
Module * getModuleForLocation(SourceLocation Loc, bool AllowTextual)
Find the module that owns the source or header file that Loc points to.
Definition: PPDirectives.cpp:858
clang::Lexer::isAtStartOfMacroExpansion
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion.
Definition: Lexer.cpp:809
clang::Preprocessor::overrideMaxTokens
void overrideMaxTokens(unsigned Value, SourceLocation Loc)
Definition: Preprocessor.h:1177
clang::Preprocessor::getMaxTokensOverrideLoc
SourceLocation getMaxTokensOverrideLoc() const
Definition: Preprocessor.h:1182
PPCallbacks.h
clang::Preprocessor::markIncluded
bool markIncluded(const FileEntry *File)
Mark the file as included.
Definition: Preprocessor.h:1351
clang::Preprocessor::creatingPCHWithThroughHeader
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
Definition: Preprocessor.cpp:616
clang::Preprocessor::LookUpIdentifierInfo
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Definition: Preprocessor.cpp:712
clang::Builtin::Context
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:71
clang::IdentifierInfo::hadMacroDefinition
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
Definition: IdentifierTable.h:227
clang::PragmaHandlerRegistry
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Definition: Preprocessor.h:2717
MacroInfo.h
clang::MacroDirective
Encapsulates changes to the "macros namespace" (the location where the macro name became active,...
Definition: MacroInfo.h:313
clang::Preprocessor::HandlePragmaHdrstop
void HandlePragmaHdrstop(Token &Tok)
Definition: Pragma.cpp:869
clang::Preprocessor::getDiagnostics
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:1062
clang::Preprocessor::Preprocessor
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
Definition: Preprocessor.cpp:78
clang::EmptylineHandler::~EmptylineHandler
virtual ~EmptylineHandler()
clang::Preprocessor::getLastFPEvalMethod
LangOptions::FPEvalMethodKind getLastFPEvalMethod() const
Definition: Preprocessor.h:2199
clang::Preprocessor::getNumDirectives
unsigned getNumDirectives() const
Retrieve the number of Directives that have been processed by the Preprocessor.
Definition: Preprocessor.h:1095
clang::Preprocessor::SetSuppressIncludeNotFoundError
void SetSuppressIncludeNotFoundError(bool Suppress)
Definition: Preprocessor.h:1115
clang::Preprocessor::addRestrictExpansionMsg
void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
Definition: Preprocessor.h:2651
clang::Preprocessor::DiscardUntilEndOfDirective
SourceRange DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found.
Definition: PPDirectives.cpp:82
clang::Preprocessor::getLeafModuleMacros
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
Definition: Preprocessor.h:1300
clang::tok::OnOffSwitch
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:48
clang::FileID
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Definition: SourceLocation.h:38
clang::tok::isAnnotation
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.cpp:58
clang::tok::isLiteral
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:89
clang::Preprocessor::setDiagnostics
void setDiagnostics(DiagnosticsEngine &D)
Definition: Preprocessor.h:1063
clang::Preprocessor::getPredefines
const std::string & getPredefines() const
Get the predefines for this processor.
Definition: Preprocessor.h:1374
clang::IdentifierTable
Implements an efficient mapping from strings to IdentifierInfo nodes.
Definition: IdentifierTable.h:564
clang::Preprocessor::LookupFile
Optional< FileEntryRef > LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
Definition: PPDirectives.cpp:951
s
__device__ __2f16 float bool s
Definition: __clang_hip_libdevice_declares.h:320
clang::Preprocessor::isInPrimaryFile
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
Definition: PPLexerChange.cpp:34
clang::Preprocessor::getLocForEndOfToken
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
Definition: Preprocessor.h:2023
clang::Preprocessor::makeModuleVisible
void makeModuleVisible(Module *M, SourceLocation Loc)
Definition: Preprocessor.cpp:1316
clang::Preprocessor::isCurrentLexer
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Definition: Preprocessor.h:1134
clang::Preprocessor::recomputeCurLexerKind
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Definition: Preprocessor.cpp:383
clang::Preprocessor::DumpMacro
void DumpMacro(const MacroInfo &MI) const
Definition: Preprocessor.cpp:264
clang::Preprocessor::addMacroDeprecationMsg
void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
Definition: Preprocessor.h:2639
clang::Preprocessor::HandlePragmaDependency
void HandlePragmaDependency(Token &DependencyTok)
HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
Definition: Pragma.cpp:504
clang::Preprocessor::macros
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
Definition: Preprocessor.h:1324
clang::VerifyDiagnosticConsumer::Directive
Directive - Abstract class representing a parsed verify directive.
Definition: VerifyDiagnosticConsumer.h:191
clang::Preprocessor::HandlePragmaIncludeAlias
void HandlePragmaIncludeAlias(Token &Tok)
Definition: Pragma.cpp:673
clang::Preprocessor::setPragmaARCCFCodeAuditedInfo
void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident, SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin.
Definition: Preprocessor.h:1845
clang::TU_Complete
@ TU_Complete
The translation unit is a complete translation unit.
Definition: LangOptions.h:916
clang::MU_Other
@ MU_Other
Definition: Preprocessor.h:114
clang::Preprocessor::RemovePragmaHandler
void RemovePragmaHandler(PragmaHandler *Handler)
Definition: Preprocessor.h:1402
clang::Preprocessor::LeaveSubmodule
Module * LeaveSubmodule(bool ForPragma)
Definition: PPLexerChange.cpp:761
clang::Preprocessor::getPreambleRecordedPragmaAssumeNonNullLoc
SourceLocation getPreambleRecordedPragmaAssumeNonNullLoc() const
Get the location of the recorded unterminated #pragma clang assume_nonnull begin in the preamble,...
Definition: Preprocessor.h:1869
clang::Preprocessor::setTokenWatcher
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream.
Definition: Preprocessor.h:1187
clang::Preprocessor::PreambleSkipInfo
Definition: Preprocessor.h:530
clang::Lexer::AdvanceToTokenCharacter
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Characters, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token,...
Definition: Lexer.h:399
clang::Lexer::getLocForEndOfToken
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
Definition: Lexer.cpp:787
clang::Preprocessor::checkModuleIsAvailable
static bool checkModuleIsAvailable(const LangOptions &LangOpts, const TargetInfo &TargetInfo, DiagnosticsEngine &Diags, Module *M)
Check that the given module is available, producing a diagnostic if not.
Definition: PPDirectives.cpp:1891
llvm::SmallVectorImpl
Definition: Randstruct.h:18
clang::Preprocessor::LexUnexpandedToken
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
Definition: Preprocessor.h:1632
llvm::SmallSetVector
Definition: ExternalSemaSource.h:23
clang::Preprocessor
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:129
clang::Preprocessor::isRecordingPreamble
bool isRecordingPreamble() const
Definition: Preprocessor.h:2569
SM
#define SM(sm)
Definition: Cuda.cpp:79
clang::Preprocessor::HandlePoisonedIdentifier
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
Definition: Preprocessor.cpp:765
clang::Preprocessor::appendMacroDirective
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
Definition: PPMacroExpansion.cpp:72
clang::Preprocessor::EnterTokenStream
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
Definition: Preprocessor.h:1542
clang::Preprocessor::getHeaderSearchInfo
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:1070
Token.h
clang::MacroDirective::DefInfo
Definition: MacroInfo.h:361
clang::Preprocessor::getEmptylineHandler
EmptylineHandler * getEmptylineHandler() const
Definition: Preprocessor.h:1412
clang::Preprocessor::SawDateOrTime
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
Definition: Preprocessor.h:2178
clang::Preprocessor::EnterAnnotationToken
void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind, void *AnnotationVal)
Enter an annotation token into the token stream.
Definition: PPDirectives.cpp:1804
clang::Preprocessor::getExternalSource
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:1082
clang::Preprocessor::mightHavePendingAnnotationTokens
bool mightHavePendingAnnotationTokens()
Determine whether it's possible for a future call to Lex to produce an annotation token created by a ...
Definition: Preprocessor.h:1764
clang::Preprocessor::getMacroAnnotations
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
Definition: Preprocessor.h:2672
clang::Preprocessor::~Preprocessor
~Preprocessor()
Definition: Preprocessor.cpp:163
clang::Preprocessor::LexNonComment
void LexNonComment(Token &Result)
Lex a token.
Definition: Preprocessor.h:1625
clang::Preprocessor::getPredefinesFileID
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:1155
clang::Preprocessor::getMacroInfo
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Definition: Preprocessor.h:1262
clang::MacroUse
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:112
clang::Preprocessor::RevertCachedTokens
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
Definition: Preprocessor.h:1682
clang::ModuleMacro
Represents a macro directive exported by a module.
Definition: MacroInfo.h:511
clang::Preprocessor::isMacroDefinedInLocalModule
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
Definition: Preprocessor.h:1204
clang::Preprocessor::setMainFileDir
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
Definition: Preprocessor.h:1881
clang::Preprocessor::getIdentifierInfo
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
Definition: Preprocessor.h:1383
clang::Preprocessor::macro_begin
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
Definition: Preprocessor.cpp:312
clang::Preprocessor::getHeaderToIncludeForDiagnostics
const FileEntry * getHeaderToIncludeForDiagnostics(SourceLocation IncLoc, SourceLocation MLoc)
We want to produce a diagnostic at location IncLoc concerning an unreachable effect at location MLoc ...
Definition: PPDirectives.cpp:880
clang::Preprocessor::parseSimpleIntegerLiteral
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
Definition: Preprocessor.cpp:1374
clang::Preprocessor::Initialize
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
Definition: Preprocessor.cpp:183
clang::DiagnosticsEngine::Report
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1537
clang::Preprocessor::CommitBacktrackedTokens
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
Definition: PPCaching.cpp:32
clang::Preprocessor::HandleEndOfFile
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
Definition: PPLexerChange.cpp:331
clang::Preprocessor::getTUFPEvalMethod
LangOptions::FPEvalMethodKind getTUFPEvalMethod() const
Definition: Preprocessor.h:2191
clang::Preprocessor::getPreprocessingRecord
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Definition: Preprocessor.h:1469
clang::MacroInfo::getDefinitionLoc
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:125