clang  9.0.0svn
Preprocessor.h
Go to the documentation of this file.
1 //===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines the clang::Preprocessor interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15 #define LLVM_CLANG_LEX_PREPROCESSOR_H
16 
17 #include "clang/Basic/Builtins.h"
18 #include "clang/Basic/Diagnostic.h"
20 #include "clang/Basic/LLVM.h"
22 #include "clang/Basic/Module.h"
25 #include "clang/Basic/TokenKinds.h"
26 #include "clang/Lex/Lexer.h"
27 #include "clang/Lex/MacroInfo.h"
28 #include "clang/Lex/ModuleLoader.h"
29 #include "clang/Lex/ModuleMap.h"
30 #include "clang/Lex/PPCallbacks.h"
31 #include "clang/Lex/Token.h"
32 #include "clang/Lex/TokenLexer.h"
33 #include "llvm/ADT/ArrayRef.h"
34 #include "llvm/ADT/DenseMap.h"
35 #include "llvm/ADT/FoldingSet.h"
36 #include "llvm/ADT/FunctionExtras.h"
37 #include "llvm/ADT/None.h"
38 #include "llvm/ADT/Optional.h"
39 #include "llvm/ADT/PointerUnion.h"
40 #include "llvm/ADT/STLExtras.h"
41 #include "llvm/ADT/SmallPtrSet.h"
42 #include "llvm/ADT/SmallVector.h"
43 #include "llvm/ADT/StringRef.h"
44 #include "llvm/ADT/TinyPtrVector.h"
45 #include "llvm/ADT/iterator_range.h"
46 #include "llvm/Support/Allocator.h"
47 #include "llvm/Support/Casting.h"
48 #include "llvm/Support/Registry.h"
49 #include <cassert>
50 #include <cstddef>
51 #include <cstdint>
52 #include <map>
53 #include <memory>
54 #include <string>
55 #include <utility>
56 #include <vector>
57 
58 namespace llvm {
59 
60 template<unsigned InternalLen> class SmallString;
61 
62 } // namespace llvm
63 
64 namespace clang {
65 
66 class CodeCompletionHandler;
67 class CommentHandler;
68 class DirectoryEntry;
69 class DirectoryLookup;
70 class ExternalPreprocessorSource;
71 class FileEntry;
72 class FileManager;
73 class HeaderSearch;
74 class MacroArgs;
75 class PragmaHandler;
76 class PragmaNamespace;
77 class PreprocessingRecord;
78 class PreprocessorLexer;
79 class PreprocessorOptions;
80 class ScratchBuffer;
81 class TargetInfo;
82 
83 /// Stores token information for comparing actual tokens with
84 /// predefined values. Only handles simple tokens and identifiers.
85 class TokenValue {
87  IdentifierInfo *II;
88 
89 public:
90  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
91  assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
92  assert(Kind != tok::identifier &&
93  "Identifiers should be created by TokenValue(IdentifierInfo *)");
94  assert(!tok::isLiteral(Kind) && "Literals are not supported.");
95  assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
96  }
97 
98  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
99 
100  bool operator==(const Token &Tok) const {
101  return Tok.getKind() == Kind &&
102  (!II || II == Tok.getIdentifierInfo());
103  }
104 };
105 
106 /// Context in which macro name is used.
107 enum MacroUse {
108  // other than #define or #undef
109  MU_Other = 0,
110 
111  // macro name specified in #define
113 
114  // macro name specified in #undef
116 };
117 
118 /// Engages in a tight little dance with the lexer to efficiently
119 /// preprocess tokens.
120 ///
121 /// Lexers know only about tokens within a single source file, and don't
122 /// know anything about preprocessor-level issues like the \#include stack,
123 /// token expansion, etc.
127 
128  llvm::unique_function<void(const clang::Token &)> OnToken;
129  std::shared_ptr<PreprocessorOptions> PPOpts;
130  DiagnosticsEngine *Diags;
131  LangOptions &LangOpts;
132  const TargetInfo *Target = nullptr;
133  const TargetInfo *AuxTarget = nullptr;
134  FileManager &FileMgr;
135  SourceManager &SourceMgr;
136  std::unique_ptr<ScratchBuffer> ScratchBuf;
137  HeaderSearch &HeaderInfo;
138  ModuleLoader &TheModuleLoader;
139 
140  /// External source of macros.
141  ExternalPreprocessorSource *ExternalSource;
142 
143  /// A BumpPtrAllocator object used to quickly allocate and release
144  /// objects internal to the Preprocessor.
145  llvm::BumpPtrAllocator BP;
146 
147  /// Identifiers for builtin macros and other builtins.
148  IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
149  IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
150  IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
151  IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
152  IdentifierInfo *Ident__FILE_NAME__; // __FILE_NAME__
153  IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
154  IdentifierInfo *Ident__COUNTER__; // __COUNTER__
155  IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
156  IdentifierInfo *Ident__identifier; // __identifier
157  IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
158  IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
159  IdentifierInfo *Ident__has_feature; // __has_feature
160  IdentifierInfo *Ident__has_extension; // __has_extension
161  IdentifierInfo *Ident__has_builtin; // __has_builtin
162  IdentifierInfo *Ident__has_attribute; // __has_attribute
163  IdentifierInfo *Ident__has_include; // __has_include
164  IdentifierInfo *Ident__has_include_next; // __has_include_next
165  IdentifierInfo *Ident__has_warning; // __has_warning
166  IdentifierInfo *Ident__is_identifier; // __is_identifier
167  IdentifierInfo *Ident__building_module; // __building_module
168  IdentifierInfo *Ident__MODULE__; // __MODULE__
169  IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
170  IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
171  IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
172  IdentifierInfo *Ident__is_target_arch; // __is_target_arch
173  IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
174  IdentifierInfo *Ident__is_target_os; // __is_target_os
175  IdentifierInfo *Ident__is_target_environment; // __is_target_environment
176 
177  // Weak, only valid (and set) while InMacroArgs is true.
178  Token* ArgMacro;
179 
180  SourceLocation DATELoc, TIMELoc;
181 
182  // Next __COUNTER__ value, starts at 0.
183  unsigned CounterValue = 0;
184 
185  enum {
186  /// Maximum depth of \#includes.
187  MaxAllowedIncludeStackDepth = 200
188  };
189 
190  // State that is set before the preprocessor begins.
191  bool KeepComments : 1;
192  bool KeepMacroComments : 1;
193  bool SuppressIncludeNotFoundError : 1;
194 
195  // State that changes while the preprocessor runs:
196  bool InMacroArgs : 1; // True if parsing fn macro invocation args.
197 
198  /// Whether the preprocessor owns the header search object.
199  bool OwnsHeaderSearch : 1;
200 
201  /// True if macro expansion is disabled.
202  bool DisableMacroExpansion : 1;
203 
204  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
205  /// when parsing preprocessor directives.
206  bool MacroExpansionInDirectivesOverride : 1;
207 
209 
210  /// Whether we have already loaded macros from the external source.
211  mutable bool ReadMacrosFromExternalSource : 1;
212 
213  /// True if pragmas are enabled.
214  bool PragmasEnabled : 1;
215 
216  /// True if the current build action is a preprocessing action.
217  bool PreprocessedOutput : 1;
218 
219  /// True if we are currently preprocessing a #if or #elif directive
220  bool ParsingIfOrElifDirective;
221 
222  /// True if we are pre-expanding macro arguments.
223  bool InMacroArgPreExpansion;
224 
225  /// Mapping/lookup information for all identifiers in
226  /// the program, including program keywords.
227  mutable IdentifierTable Identifiers;
228 
229  /// This table contains all the selectors in the program.
230  ///
231  /// Unlike IdentifierTable above, this table *isn't* populated by the
232  /// preprocessor. It is declared/expanded here because its role/lifetime is
233  /// conceptually similar to the IdentifierTable. In addition, the current
234  /// control flow (in clang::ParseAST()), make it convenient to put here.
235  ///
236  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
237  /// the lifetime of the preprocessor.
238  SelectorTable Selectors;
239 
240  /// Information about builtins.
242 
243  /// Tracks all of the pragmas that the client registered
244  /// with this preprocessor.
245  std::unique_ptr<PragmaNamespace> PragmaHandlers;
246 
247  /// Pragma handlers of the original source is stored here during the
248  /// parsing of a model file.
249  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
250 
251  /// Tracks all of the comment handlers that the client registered
252  /// with this preprocessor.
253  std::vector<CommentHandler *> CommentHandlers;
254 
255  /// True if we want to ignore EOF token and continue later on (thus
256  /// avoid tearing the Lexer and etc. down).
257  bool IncrementalProcessing = false;
258 
259  /// The kind of translation unit we are processing.
260  TranslationUnitKind TUKind;
261 
262  /// The code-completion handler.
263  CodeCompletionHandler *CodeComplete = nullptr;
264 
265  /// The file that we're performing code-completion for, if any.
266  const FileEntry *CodeCompletionFile = nullptr;
267 
268  /// The offset in file for the code-completion point.
269  unsigned CodeCompletionOffset = 0;
270 
271  /// The location for the code-completion point. This gets instantiated
272  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
273  SourceLocation CodeCompletionLoc;
274 
275  /// The start location for the file of the code-completion point.
276  ///
277  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
278  /// for preprocessing.
279  SourceLocation CodeCompletionFileLoc;
280 
281  /// The source location of the \c import contextual keyword we just
282  /// lexed, if any.
283  SourceLocation ModuleImportLoc;
284 
285  /// The module import path that we're currently processing.
287 
288  /// Whether the last token we lexed was an '@'.
289  bool LastTokenWasAt = false;
290 
291  /// A position within a C++20 import-seq.
292  class ImportSeq {
293  public:
294  enum State : int {
295  // Positive values represent a number of unclosed brackets.
296  AtTopLevel = 0,
297  AfterTopLevelTokenSeq = -1,
298  AfterExport = -2,
299  AfterImportSeq = -3,
300  };
301 
302  ImportSeq(State S) : S(S) {}
303 
304  /// Saw any kind of open bracket.
305  void handleOpenBracket() {
306  S = static_cast<State>(std::max<int>(S, 0) + 1);
307  }
308  /// Saw any kind of close bracket other than '}'.
309  void handleCloseBracket() {
310  S = static_cast<State>(std::max<int>(S, 1) - 1);
311  }
312  /// Saw a close brace.
313  void handleCloseBrace() {
314  handleCloseBracket();
315  if (S == AtTopLevel && !AfterHeaderName)
316  S = AfterTopLevelTokenSeq;
317  }
318  /// Saw a semicolon.
319  void handleSemi() {
320  if (atTopLevel()) {
321  S = AfterTopLevelTokenSeq;
322  AfterHeaderName = false;
323  }
324  }
325 
326  /// Saw an 'export' identifier.
327  void handleExport() {
328  if (S == AfterTopLevelTokenSeq)
329  S = AfterExport;
330  else if (S <= 0)
331  S = AtTopLevel;
332  }
333  /// Saw an 'import' identifier.
334  void handleImport() {
335  if (S == AfterTopLevelTokenSeq || S == AfterExport)
336  S = AfterImportSeq;
337  else if (S <= 0)
338  S = AtTopLevel;
339  }
340 
341  /// Saw a 'header-name' token; do not recognize any more 'import' tokens
342  /// until we reach a top-level semicolon.
343  void handleHeaderName() {
344  if (S == AfterImportSeq)
345  AfterHeaderName = true;
346  handleMisc();
347  }
348 
349  /// Saw any other token.
350  void handleMisc() {
351  if (S <= 0)
352  S = AtTopLevel;
353  }
354 
355  bool atTopLevel() { return S <= 0; }
356  bool afterImportSeq() { return S == AfterImportSeq; }
357 
358  private:
359  State S;
360  /// Whether we're in the pp-import-suffix following the header-name in a
361  /// pp-import. If so, a close-brace is not sufficient to end the
362  /// top-level-token-seq of an import-seq.
363  bool AfterHeaderName = false;
364  };
365 
366  /// Our current position within a C++20 import-seq.
367  ImportSeq ImportSeqState = ImportSeq::AfterTopLevelTokenSeq;
368 
369  /// Whether the module import expects an identifier next. Otherwise,
370  /// it expects a '.' or ';'.
371  bool ModuleImportExpectsIdentifier = false;
372 
373  /// The source location of the currently-active
374  /// \#pragma clang arc_cf_code_audited begin.
375  SourceLocation PragmaARCCFCodeAuditedLoc;
376 
377  /// The source location of the currently-active
378  /// \#pragma clang assume_nonnull begin.
379  SourceLocation PragmaAssumeNonNullLoc;
380 
381  /// True if we hit the code-completion point.
382  bool CodeCompletionReached = false;
383 
384  /// The code completion token containing the information
385  /// on the stem that is to be code completed.
386  IdentifierInfo *CodeCompletionII = nullptr;
387 
388  /// Range for the code completion token.
389  SourceRange CodeCompletionTokenRange;
390 
391  /// The directory that the main file should be considered to occupy,
392  /// if it does not correspond to a real file (as happens when building a
393  /// module).
394  const DirectoryEntry *MainFileDir = nullptr;
395 
396  /// The number of bytes that we will initially skip when entering the
397  /// main file, along with a flag that indicates whether skipping this number
398  /// of bytes will place the lexer at the start of a line.
399  ///
400  /// This is used when loading a precompiled preamble.
401  std::pair<int, bool> SkipMainFilePreamble;
402 
403  /// Whether we hit an error due to reaching max allowed include depth. Allows
404  /// to avoid hitting the same error over and over again.
405  bool HasReachedMaxIncludeDepth = false;
406 
407  /// The number of currently-active calls to Lex.
408  ///
409  /// Lex is reentrant, and asking for an (end-of-phase-4) token can often
410  /// require asking for multiple additional tokens. This counter makes it
411  /// possible for Lex to detect whether it's producing a token for the end
412  /// of phase 4 of translation or for some other situation.
413  unsigned LexLevel = 0;
414 
415 public:
420  bool FoundElse;
422 
424  bool FoundNonSkipPortion, bool FoundElse,
425  SourceLocation ElseLoc)
426  : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
427  FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
428  ElseLoc(ElseLoc) {}
429  };
430 
431 private:
432  friend class ASTReader;
433  friend class MacroArgs;
434 
435  class PreambleConditionalStackStore {
436  enum State {
437  Off = 0,
438  Recording = 1,
439  Replaying = 2,
440  };
441 
442  public:
443  PreambleConditionalStackStore() = default;
444 
445  void startRecording() { ConditionalStackState = Recording; }
446  void startReplaying() { ConditionalStackState = Replaying; }
447  bool isRecording() const { return ConditionalStackState == Recording; }
448  bool isReplaying() const { return ConditionalStackState == Replaying; }
449 
450  ArrayRef<PPConditionalInfo> getStack() const {
451  return ConditionalStack;
452  }
453 
454  void doneReplaying() {
455  ConditionalStack.clear();
456  ConditionalStackState = Off;
457  }
458 
459  void setStack(ArrayRef<PPConditionalInfo> s) {
460  if (!isRecording() && !isReplaying())
461  return;
462  ConditionalStack.clear();
463  ConditionalStack.append(s.begin(), s.end());
464  }
465 
466  bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
467 
468  bool reachedEOFWhileSkipping() const { return SkipInfo.hasValue(); }
469 
470  void clearSkipInfo() { SkipInfo.reset(); }
471 
473 
474  private:
475  SmallVector<PPConditionalInfo, 4> ConditionalStack;
476  State ConditionalStackState = Off;
477  } PreambleConditionalStack;
478 
479  /// The current top of the stack that we're lexing from if
480  /// not expanding a macro and we are lexing directly from source code.
481  ///
482  /// Only one of CurLexer, or CurTokenLexer will be non-null.
483  std::unique_ptr<Lexer> CurLexer;
484 
485  /// The current top of the stack what we're lexing from
486  /// if not expanding a macro.
487  ///
488  /// This is an alias for CurLexer.
489  PreprocessorLexer *CurPPLexer = nullptr;
490 
491  /// Used to find the current FileEntry, if CurLexer is non-null
492  /// and if applicable.
493  ///
494  /// This allows us to implement \#include_next and find directory-specific
495  /// properties.
496  const DirectoryLookup *CurDirLookup = nullptr;
497 
498  /// The current macro we are expanding, if we are expanding a macro.
499  ///
500  /// One of CurLexer and CurTokenLexer must be null.
501  std::unique_ptr<TokenLexer> CurTokenLexer;
502 
503  /// The kind of lexer we're currently working with.
504  enum CurLexerKind {
505  CLK_Lexer,
506  CLK_TokenLexer,
507  CLK_CachingLexer,
508  CLK_LexAfterModuleImport
509  } CurLexerKind = CLK_Lexer;
510 
511  /// If the current lexer is for a submodule that is being built, this
512  /// is that submodule.
513  Module *CurLexerSubmodule = nullptr;
514 
515  /// Keeps track of the stack of files currently
516  /// \#included, and macros currently being expanded from, not counting
517  /// CurLexer/CurTokenLexer.
518  struct IncludeStackInfo {
519  enum CurLexerKind CurLexerKind;
520  Module *TheSubmodule;
521  std::unique_ptr<Lexer> TheLexer;
522  PreprocessorLexer *ThePPLexer;
523  std::unique_ptr<TokenLexer> TheTokenLexer;
524  const DirectoryLookup *TheDirLookup;
525 
526  // The following constructors are completely useless copies of the default
527  // versions, only needed to pacify MSVC.
528  IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
529  std::unique_ptr<Lexer> &&TheLexer,
530  PreprocessorLexer *ThePPLexer,
531  std::unique_ptr<TokenLexer> &&TheTokenLexer,
532  const DirectoryLookup *TheDirLookup)
533  : CurLexerKind(std::move(CurLexerKind)),
534  TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
535  ThePPLexer(std::move(ThePPLexer)),
536  TheTokenLexer(std::move(TheTokenLexer)),
537  TheDirLookup(std::move(TheDirLookup)) {}
538  };
539  std::vector<IncludeStackInfo> IncludeMacroStack;
540 
541  /// Actions invoked when some preprocessor activity is
542  /// encountered (e.g. a file is \#included, etc).
543  std::unique_ptr<PPCallbacks> Callbacks;
544 
545  struct MacroExpandsInfo {
546  Token Tok;
547  MacroDefinition MD;
548  SourceRange Range;
549 
550  MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
551  : Tok(Tok), MD(MD), Range(Range) {}
552  };
553  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
554 
555  /// Information about a name that has been used to define a module macro.
556  struct ModuleMacroInfo {
557  /// The most recent macro directive for this identifier.
558  MacroDirective *MD;
559 
560  /// The active module macros for this identifier.
561  llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
562 
563  /// The generation number at which we last updated ActiveModuleMacros.
564  /// \see Preprocessor::VisibleModules.
565  unsigned ActiveModuleMacrosGeneration = 0;
566 
567  /// Whether this macro name is ambiguous.
568  bool IsAmbiguous = false;
569 
570  /// The module macros that are overridden by this macro.
571  llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
572 
573  ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
574  };
575 
576  /// The state of a macro for an identifier.
577  class MacroState {
578  mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
579 
580  ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
581  const IdentifierInfo *II) const {
582  if (II->isOutOfDate())
583  PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
584  // FIXME: Find a spare bit on IdentifierInfo and store a
585  // HasModuleMacros flag.
586  if (!II->hasMacroDefinition() ||
587  (!PP.getLangOpts().Modules &&
588  !PP.getLangOpts().ModulesLocalVisibility) ||
589  !PP.CurSubmoduleState->VisibleModules.getGeneration())
590  return nullptr;
591 
592  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
593  if (!Info) {
594  Info = new (PP.getPreprocessorAllocator())
595  ModuleMacroInfo(State.get<MacroDirective *>());
596  State = Info;
597  }
598 
599  if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
600  Info->ActiveModuleMacrosGeneration)
601  PP.updateModuleMacroInfo(II, *Info);
602  return Info;
603  }
604 
605  public:
606  MacroState() : MacroState(nullptr) {}
607  MacroState(MacroDirective *MD) : State(MD) {}
608 
609  MacroState(MacroState &&O) noexcept : State(O.State) {
610  O.State = (MacroDirective *)nullptr;
611  }
612 
613  MacroState &operator=(MacroState &&O) noexcept {
614  auto S = O.State;
615  O.State = (MacroDirective *)nullptr;
616  State = S;
617  return *this;
618  }
619 
620  ~MacroState() {
621  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
622  Info->~ModuleMacroInfo();
623  }
624 
625  MacroDirective *getLatest() const {
626  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
627  return Info->MD;
628  return State.get<MacroDirective*>();
629  }
630 
631  void setLatest(MacroDirective *MD) {
632  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
633  Info->MD = MD;
634  else
635  State = MD;
636  }
637 
638  bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
639  auto *Info = getModuleInfo(PP, II);
640  return Info ? Info->IsAmbiguous : false;
641  }
642 
644  getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
645  if (auto *Info = getModuleInfo(PP, II))
646  return Info->ActiveModuleMacros;
647  return None;
648  }
649 
650  MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
651  SourceManager &SourceMgr) const {
652  // FIXME: Incorporate module macros into the result of this.
653  if (auto *Latest = getLatest())
654  return Latest->findDirectiveAtLoc(Loc, SourceMgr);
655  return {};
656  }
657 
658  void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
659  if (auto *Info = getModuleInfo(PP, II)) {
660  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
661  Info->ActiveModuleMacros.begin(),
662  Info->ActiveModuleMacros.end());
663  Info->ActiveModuleMacros.clear();
664  Info->IsAmbiguous = false;
665  }
666  }
667 
668  ArrayRef<ModuleMacro*> getOverriddenMacros() const {
669  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
670  return Info->OverriddenMacros;
671  return None;
672  }
673 
674  void setOverriddenMacros(Preprocessor &PP,
675  ArrayRef<ModuleMacro *> Overrides) {
676  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
677  if (!Info) {
678  if (Overrides.empty())
679  return;
680  Info = new (PP.getPreprocessorAllocator())
681  ModuleMacroInfo(State.get<MacroDirective *>());
682  State = Info;
683  }
684  Info->OverriddenMacros.clear();
685  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
686  Overrides.begin(), Overrides.end());
687  Info->ActiveModuleMacrosGeneration = 0;
688  }
689  };
690 
691  /// For each IdentifierInfo that was associated with a macro, we
692  /// keep a mapping to the history of all macro definitions and #undefs in
693  /// the reverse order (the latest one is in the head of the list).
694  ///
695  /// This mapping lives within the \p CurSubmoduleState.
696  using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
697 
698  struct SubmoduleState;
699 
700  /// Information about a submodule that we're currently building.
701  struct BuildingSubmoduleInfo {
702  /// The module that we are building.
703  Module *M;
704 
705  /// The location at which the module was included.
706  SourceLocation ImportLoc;
707 
708  /// Whether we entered this submodule via a pragma.
709  bool IsPragma;
710 
711  /// The previous SubmoduleState.
712  SubmoduleState *OuterSubmoduleState;
713 
714  /// The number of pending module macro names when we started building this.
715  unsigned OuterPendingModuleMacroNames;
716 
717  BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
718  SubmoduleState *OuterSubmoduleState,
719  unsigned OuterPendingModuleMacroNames)
720  : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
721  OuterSubmoduleState(OuterSubmoduleState),
722  OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
723  };
724  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
725 
726  /// Information about a submodule's preprocessor state.
727  struct SubmoduleState {
728  /// The macros for the submodule.
729  MacroMap Macros;
730 
731  /// The set of modules that are visible within the submodule.
732  VisibleModuleSet VisibleModules;
733 
734  // FIXME: CounterValue?
735  // FIXME: PragmaPushMacroInfo?
736  };
737  std::map<Module *, SubmoduleState> Submodules;
738 
739  /// The preprocessor state for preprocessing outside of any submodule.
740  SubmoduleState NullSubmoduleState;
741 
742  /// The current submodule state. Will be \p NullSubmoduleState if we're not
743  /// in a submodule.
744  SubmoduleState *CurSubmoduleState;
745 
746  /// The set of known macros exported from modules.
747  llvm::FoldingSet<ModuleMacro> ModuleMacros;
748 
749  /// The names of potential module macros that we've not yet processed.
750  llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
751 
752  /// The list of module macros, for each identifier, that are not overridden by
753  /// any other module macro.
754  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
755  LeafModuleMacros;
756 
757  /// Macros that we want to warn because they are not used at the end
758  /// of the translation unit.
759  ///
760  /// We store just their SourceLocations instead of
761  /// something like MacroInfo*. The benefit of this is that when we are
762  /// deserializing from PCH, we don't need to deserialize identifier & macros
763  /// just so that we can report that they are unused, we just warn using
764  /// the SourceLocations of this set (that will be filled by the ASTReader).
765  /// We are using SmallPtrSet instead of a vector for faster removal.
766  using WarnUnusedMacroLocsTy = llvm::SmallPtrSet<SourceLocation, 32>;
767  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
768 
769  /// A "freelist" of MacroArg objects that can be
770  /// reused for quick allocation.
771  MacroArgs *MacroArgCache = nullptr;
772 
773  /// For each IdentifierInfo used in a \#pragma push_macro directive,
774  /// we keep a MacroInfo stack used to restore the previous macro value.
775  llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
776  PragmaPushMacroInfo;
777 
778  // Various statistics we track for performance analysis.
779  unsigned NumDirectives = 0;
780  unsigned NumDefined = 0;
781  unsigned NumUndefined = 0;
782  unsigned NumPragma = 0;
783  unsigned NumIf = 0;
784  unsigned NumElse = 0;
785  unsigned NumEndif = 0;
786  unsigned NumEnteredSourceFiles = 0;
787  unsigned MaxIncludeStackDepth = 0;
788  unsigned NumMacroExpanded = 0;
789  unsigned NumFnMacroExpanded = 0;
790  unsigned NumBuiltinMacroExpanded = 0;
791  unsigned NumFastMacroExpanded = 0;
792  unsigned NumTokenPaste = 0;
793  unsigned NumFastTokenPaste = 0;
794  unsigned NumSkipped = 0;
795 
796  /// The predefined macros that preprocessor should use from the
797  /// command line etc.
798  std::string Predefines;
799 
800  /// The file ID for the preprocessor predefines.
801  FileID PredefinesFileID;
802 
803  /// The file ID for the PCH through header.
804  FileID PCHThroughHeaderFileID;
805 
806  /// Whether tokens are being skipped until a #pragma hdrstop is seen.
807  bool SkippingUntilPragmaHdrStop = false;
808 
809  /// Whether tokens are being skipped until the through header is seen.
810  bool SkippingUntilPCHThroughHeader = false;
811 
812  /// \{
813  /// Cache of macro expanders to reduce malloc traffic.
814  enum { TokenLexerCacheSize = 8 };
815  unsigned NumCachedTokenLexers;
816  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
817  /// \}
818 
819  /// Keeps macro expanded tokens for TokenLexers.
820  //
821  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
822  /// going to lex in the cache and when it finishes the tokens are removed
823  /// from the end of the cache.
824  SmallVector<Token, 16> MacroExpandedTokens;
825  std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
826 
827  /// A record of the macro definitions and expansions that
828  /// occurred during preprocessing.
829  ///
830  /// This is an optional side structure that can be enabled with
831  /// \c createPreprocessingRecord() prior to preprocessing.
832  PreprocessingRecord *Record = nullptr;
833 
834  /// Cached tokens state.
836 
837  /// Cached tokens are stored here when we do backtracking or
838  /// lookahead. They are "lexed" by the CachingLex() method.
840 
841  /// The position of the cached token that CachingLex() should
842  /// "lex" next.
843  ///
844  /// If it points beyond the CachedTokens vector, it means that a normal
845  /// Lex() should be invoked.
846  CachedTokensTy::size_type CachedLexPos = 0;
847 
848  /// Stack of backtrack positions, allowing nested backtracks.
849  ///
850  /// The EnableBacktrackAtThisPos() method pushes a position to
851  /// indicate where CachedLexPos should be set when the BackTrack() method is
852  /// invoked (at which point the last position is popped).
853  std::vector<CachedTokensTy::size_type> BacktrackPositions;
854 
855  struct MacroInfoChain {
856  MacroInfo MI;
857  MacroInfoChain *Next;
858  };
859 
860  /// MacroInfos are managed as a chain for easy disposal. This is the head
861  /// of that list.
862  MacroInfoChain *MIChainHead = nullptr;
863 
864  void updateOutOfDateIdentifier(IdentifierInfo &II) const;
865 
866 public:
867  Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
869  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
870  IdentifierInfoLookup *IILookup = nullptr,
871  bool OwnsHeaderSearch = false,
873 
874  ~Preprocessor();
875 
876  /// Initialize the preprocessor using information about the target.
877  ///
878  /// \param Target is owned by the caller and must remain valid for the
879  /// lifetime of the preprocessor.
880  /// \param AuxTarget is owned by the caller and must remain valid for
881  /// the lifetime of the preprocessor.
882  void Initialize(const TargetInfo &Target,
883  const TargetInfo *AuxTarget = nullptr);
884 
885  /// Initialize the preprocessor to parse a model file
886  ///
887  /// To parse model files the preprocessor of the original source is reused to
888  /// preserver the identifier table. However to avoid some duplicate
889  /// information in the preprocessor some cleanup is needed before it is used
890  /// to parse model files. This method does that cleanup.
891  void InitializeForModelFile();
892 
893  /// Cleanup after model file parsing
894  void FinalizeForModelFile();
895 
896  /// Retrieve the preprocessor options used to initialize this
897  /// preprocessor.
898  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
899 
900  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
901  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
902 
903  const LangOptions &getLangOpts() const { return LangOpts; }
904  const TargetInfo &getTargetInfo() const { return *Target; }
905  const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
906  FileManager &getFileManager() const { return FileMgr; }
907  SourceManager &getSourceManager() const { return SourceMgr; }
908  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
909 
910  IdentifierTable &getIdentifierTable() { return Identifiers; }
911  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
912  SelectorTable &getSelectorTable() { return Selectors; }
914  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
915 
917  ExternalSource = Source;
918  }
919 
921  return ExternalSource;
922  }
923 
924  /// Retrieve the module loader associated with this preprocessor.
925  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
926 
928  return TheModuleLoader.HadFatalFailure;
929  }
930 
931  /// True if we are currently preprocessing a #if or #elif directive
933  return ParsingIfOrElifDirective;
934  }
935 
936  /// Control whether the preprocessor retains comments in output.
937  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
938  this->KeepComments = KeepComments | KeepMacroComments;
939  this->KeepMacroComments = KeepMacroComments;
940  }
941 
942  bool getCommentRetentionState() const { return KeepComments; }
943 
944  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
945  bool getPragmasEnabled() const { return PragmasEnabled; }
946 
947  void SetSuppressIncludeNotFoundError(bool Suppress) {
948  SuppressIncludeNotFoundError = Suppress;
949  }
950 
952  return SuppressIncludeNotFoundError;
953  }
954 
955  /// Sets whether the preprocessor is responsible for producing output or if
956  /// it is producing tokens to be consumed by Parse and Sema.
957  void setPreprocessedOutput(bool IsPreprocessedOutput) {
958  PreprocessedOutput = IsPreprocessedOutput;
959  }
960 
961  /// Returns true if the preprocessor is responsible for generating output,
962  /// false if it is producing tokens to be consumed by Parse and Sema.
963  bool isPreprocessedOutput() const { return PreprocessedOutput; }
964 
965  /// Return true if we are lexing directly from the specified lexer.
966  bool isCurrentLexer(const PreprocessorLexer *L) const {
967  return CurPPLexer == L;
968  }
969 
970  /// Return the current lexer being lexed from.
971  ///
972  /// Note that this ignores any potentially active macro expansions and _Pragma
973  /// expansions going on at the time.
974  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
975 
976  /// Return the current file lexer being lexed from.
977  ///
978  /// Note that this ignores any potentially active macro expansions and _Pragma
979  /// expansions going on at the time.
980  PreprocessorLexer *getCurrentFileLexer() const;
981 
982  /// Return the submodule owning the file being lexed. This may not be
983  /// the current module if we have changed modules since entering the file.
984  Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
985 
986  /// Returns the FileID for the preprocessor predefines.
987  FileID getPredefinesFileID() const { return PredefinesFileID; }
988 
989  /// \{
990  /// Accessors for preprocessor callbacks.
991  ///
992  /// Note that this class takes ownership of any PPCallbacks object given to
993  /// it.
994  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
995  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
996  if (Callbacks)
997  C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
998  std::move(Callbacks));
999  Callbacks = std::move(C);
1000  }
1001  /// \}
1002 
1003  /// Register a function that would be called on each token in the final
1004  /// expanded token stream.
1005  /// This also reports annotation tokens produced by the parser.
1006  void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
1007  OnToken = std::move(F);
1008  }
1009 
1010  bool isMacroDefined(StringRef Id) {
1011  return isMacroDefined(&Identifiers.get(Id));
1012  }
1013  bool isMacroDefined(const IdentifierInfo *II) {
1014  return II->hasMacroDefinition() &&
1015  (!getLangOpts().Modules || (bool)getMacroDefinition(II));
1016  }
1017 
1018  /// Determine whether II is defined as a macro within the module M,
1019  /// if that is a module that we've already preprocessed. Does not check for
1020  /// macros imported into M.
1022  if (!II->hasMacroDefinition())
1023  return false;
1024  auto I = Submodules.find(M);
1025  if (I == Submodules.end())
1026  return false;
1027  auto J = I->second.Macros.find(II);
1028  if (J == I->second.Macros.end())
1029  return false;
1030  auto *MD = J->second.getLatest();
1031  return MD && MD->isDefined();
1032  }
1033 
1035  if (!II->hasMacroDefinition())
1036  return {};
1037 
1038  MacroState &S = CurSubmoduleState->Macros[II];
1039  auto *MD = S.getLatest();
1040  while (MD && isa<VisibilityMacroDirective>(MD))
1041  MD = MD->getPrevious();
1042  return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
1043  S.getActiveModuleMacros(*this, II),
1044  S.isAmbiguous(*this, II));
1045  }
1046 
1048  SourceLocation Loc) {
1049  if (!II->hadMacroDefinition())
1050  return {};
1051 
1052  MacroState &S = CurSubmoduleState->Macros[II];
1054  if (auto *MD = S.getLatest())
1055  DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
1056  // FIXME: Compute the set of active module macros at the specified location.
1057  return MacroDefinition(DI.getDirective(),
1058  S.getActiveModuleMacros(*this, II),
1059  S.isAmbiguous(*this, II));
1060  }
1061 
1062  /// Given an identifier, return its latest non-imported MacroDirective
1063  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
1065  if (!II->hasMacroDefinition())
1066  return nullptr;
1067 
1068  auto *MD = getLocalMacroDirectiveHistory(II);
1069  if (!MD || MD->getDefinition().isUndefined())
1070  return nullptr;
1071 
1072  return MD;
1073  }
1074 
1075  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
1076  return const_cast<Preprocessor*>(this)->getMacroInfo(II);
1077  }
1078 
1080  if (!II->hasMacroDefinition())
1081  return nullptr;
1082  if (auto MD = getMacroDefinition(II))
1083  return MD.getMacroInfo();
1084  return nullptr;
1085  }
1086 
1087  /// Given an identifier, return the latest non-imported macro
1088  /// directive for that identifier.
1089  ///
1090  /// One can iterate over all previous macro directives from the most recent
1091  /// one.
1092  MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const;
1093 
1094  /// Add a directive to the macro directive history for this identifier.
1095  void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
1097  SourceLocation Loc) {
1098  DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1099  appendMacroDirective(II, MD);
1100  return MD;
1101  }
1103  MacroInfo *MI) {
1104  return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1105  }
1106 
1107  /// Set a MacroDirective that was loaded from a PCH file.
1108  void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED,
1109  MacroDirective *MD);
1110 
1111  /// Register an exported macro for a module and identifier.
1112  ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro,
1113  ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1114  ModuleMacro *getModuleMacro(Module *Mod, IdentifierInfo *II);
1115 
1116  /// Get the list of leaf (non-overridden) module macros for a name.
1118  if (II->isOutOfDate())
1119  updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1120  auto I = LeafModuleMacros.find(II);
1121  if (I != LeafModuleMacros.end())
1122  return I->second;
1123  return None;
1124  }
1125 
1126  /// \{
1127  /// Iterators for the macro history table. Currently defined macros have
1128  /// IdentifierInfo::hasMacroDefinition() set and an empty
1129  /// MacroInfo::getUndefLoc() at the head of the list.
1130  using macro_iterator = MacroMap::const_iterator;
1131 
1132  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1133  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1134 
1135  llvm::iterator_range<macro_iterator>
1136  macros(bool IncludeExternalMacros = true) const {
1137  macro_iterator begin = macro_begin(IncludeExternalMacros);
1138  macro_iterator end = macro_end(IncludeExternalMacros);
1139  return llvm::make_range(begin, end);
1140  }
1141 
1142  /// \}
1143 
1144  /// Return the name of the macro defined before \p Loc that has
1145  /// spelling \p Tokens. If there are multiple macros with same spelling,
1146  /// return the last one defined.
1147  StringRef getLastMacroWithSpelling(SourceLocation Loc,
1148  ArrayRef<TokenValue> Tokens) const;
1149 
1150  const std::string &getPredefines() const { return Predefines; }
1151 
1152  /// Set the predefines for this Preprocessor.
1153  ///
1154  /// These predefines are automatically injected when parsing the main file.
1155  void setPredefines(const char *P) { Predefines = P; }
1156  void setPredefines(StringRef P) { Predefines = P; }
1157 
1158  /// Return information about the specified preprocessor
1159  /// identifier token.
1160  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1161  return &Identifiers.get(Name);
1162  }
1163 
1164  /// Add the specified pragma handler to this preprocessor.
1165  ///
1166  /// If \p Namespace is non-null, then it is a token required to exist on the
1167  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1168  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1170  AddPragmaHandler(StringRef(), Handler);
1171  }
1172 
1173  /// Remove the specific pragma handler from this preprocessor.
1174  ///
1175  /// If \p Namespace is non-null, then it should be the namespace that
1176  /// \p Handler was added to. It is an error to remove a handler that
1177  /// has not been registered.
1178  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1180  RemovePragmaHandler(StringRef(), Handler);
1181  }
1182 
1183  /// Install empty handlers for all pragmas (making them ignored).
1184  void IgnorePragmas();
1185 
1186  /// Add the specified comment handler to the preprocessor.
1187  void addCommentHandler(CommentHandler *Handler);
1188 
1189  /// Remove the specified comment handler.
1190  ///
1191  /// It is an error to remove a handler that has not been registered.
1192  void removeCommentHandler(CommentHandler *Handler);
1193 
1194  /// Set the code completion handler to the given object.
1196  CodeComplete = &Handler;
1197  }
1198 
1199  /// Retrieve the current code-completion handler.
1201  return CodeComplete;
1202  }
1203 
1204  /// Clear out the code completion handler.
1206  CodeComplete = nullptr;
1207  }
1208 
1209  /// Hook used by the lexer to invoke the "included file" code
1210  /// completion point.
1211  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1212 
1213  /// Hook used by the lexer to invoke the "natural language" code
1214  /// completion point.
1215  void CodeCompleteNaturalLanguage();
1216 
1217  /// Set the code completion token for filtering purposes.
1219  CodeCompletionII = Filter;
1220  }
1221 
1222  /// Set the code completion token range for detecting replacement range later
1223  /// on.
1225  const SourceLocation End) {
1226  CodeCompletionTokenRange = {Start, End};
1227  }
1229  return CodeCompletionTokenRange;
1230  }
1231 
1232  /// Get the code completion token for filtering purposes.
1234  if (CodeCompletionII)
1235  return CodeCompletionII->getName();
1236  return {};
1237  }
1238 
1239  /// Retrieve the preprocessing record, or NULL if there is no
1240  /// preprocessing record.
1241  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1242 
1243  /// Create a new preprocessing record, which will keep track of
1244  /// all macro expansions, macro definitions, etc.
1245  void createPreprocessingRecord();
1246 
1247  /// Returns true if the FileEntry is the PCH through header.
1248  bool isPCHThroughHeader(const FileEntry *FE);
1249 
1250  /// True if creating a PCH with a through header.
1251  bool creatingPCHWithThroughHeader();
1252 
1253  /// True if using a PCH with a through header.
1254  bool usingPCHWithThroughHeader();
1255 
1256  /// True if creating a PCH with a #pragma hdrstop.
1257  bool creatingPCHWithPragmaHdrStop();
1258 
1259  /// True if using a PCH with a #pragma hdrstop.
1260  bool usingPCHWithPragmaHdrStop();
1261 
1262  /// Skip tokens until after the #include of the through header or
1263  /// until after a #pragma hdrstop.
1264  void SkipTokensWhileUsingPCH();
1265 
1266  /// Process directives while skipping until the through header or
1267  /// #pragma hdrstop is found.
1268  void HandleSkippedDirectiveWhileUsingPCH(Token &Result,
1269  SourceLocation HashLoc);
1270 
1271  /// Enter the specified FileID as the main source file,
1272  /// which implicitly adds the builtin defines etc.
1273  void EnterMainSourceFile();
1274 
1275  /// Inform the preprocessor callbacks that processing is complete.
1276  void EndSourceFile();
1277 
1278  /// Add a source file to the top of the include stack and
1279  /// start lexing tokens from it instead of the current buffer.
1280  ///
1281  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1282  bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir,
1283  SourceLocation Loc);
1284 
1285  /// Add a Macro to the top of the include stack and start lexing
1286  /// tokens from it instead of the current buffer.
1287  ///
1288  /// \param Args specifies the tokens input to a function-like macro.
1289  /// \param ILEnd specifies the location of the ')' for a function-like macro
1290  /// or the identifier for an object-like macro.
1291  void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1292  MacroArgs *Args);
1293 
1294 private:
1295  /// Add a "macro" context to the top of the include stack,
1296  /// which will cause the lexer to start returning the specified tokens.
1297  ///
1298  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1299  /// will not be subject to further macro expansion. Otherwise, these tokens
1300  /// will be re-macro-expanded when/if expansion is enabled.
1301  ///
1302  /// If \p OwnsTokens is false, this method assumes that the specified stream
1303  /// of tokens has a permanent owner somewhere, so they do not need to be
1304  /// copied. If it is true, it assumes the array of tokens is allocated with
1305  /// \c new[] and the Preprocessor will delete[] it.
1306  ///
1307  /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
1308  /// set, see the flag documentation for details.
1309  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1310  bool DisableMacroExpansion, bool OwnsTokens,
1311  bool IsReinject);
1312 
1313 public:
1314  void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1315  bool DisableMacroExpansion, bool IsReinject) {
1316  EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true,
1317  IsReinject);
1318  }
1319 
1320  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
1321  bool IsReinject) {
1322  EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false,
1323  IsReinject);
1324  }
1325 
1326  /// Pop the current lexer/macro exp off the top of the lexer stack.
1327  ///
1328  /// This should only be used in situations where the current state of the
1329  /// top-of-stack lexer is known.
1330  void RemoveTopOfLexerStack();
1331 
1332  /// From the point that this method is called, and until
1333  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1334  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1335  /// make the Preprocessor re-lex the same tokens.
1336  ///
1337  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1338  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1339  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1340  ///
1341  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1342  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1343  /// tokens will continue indefinitely.
1344  ///
1345  void EnableBacktrackAtThisPos();
1346 
1347  /// Disable the last EnableBacktrackAtThisPos call.
1348  void CommitBacktrackedTokens();
1349 
1350  /// Make Preprocessor re-lex the tokens that were lexed since
1351  /// EnableBacktrackAtThisPos() was previously called.
1352  void Backtrack();
1353 
1354  /// True if EnableBacktrackAtThisPos() was called and
1355  /// caching of tokens is on.
1356  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1357 
1358  /// Lex the next token for this preprocessor.
1359  void Lex(Token &Result);
1360 
1361  /// Lex a token, forming a header-name token if possible.
1362  bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
1363 
1364  bool LexAfterModuleImport(Token &Result);
1365  void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks);
1366 
1367  void makeModuleVisible(Module *M, SourceLocation Loc);
1368 
1370  return CurSubmoduleState->VisibleModules.getImportLoc(M);
1371  }
1372 
1373  /// Lex a string literal, which may be the concatenation of multiple
1374  /// string literals and may even come from macro expansion.
1375  /// \returns true on success, false if a error diagnostic has been generated.
1376  bool LexStringLiteral(Token &Result, std::string &String,
1377  const char *DiagnosticTag, bool AllowMacroExpansion) {
1378  if (AllowMacroExpansion)
1379  Lex(Result);
1380  else
1381  LexUnexpandedToken(Result);
1382  return FinishLexStringLiteral(Result, String, DiagnosticTag,
1383  AllowMacroExpansion);
1384  }
1385 
1386  /// Complete the lexing of a string literal where the first token has
1387  /// already been lexed (see LexStringLiteral).
1388  bool FinishLexStringLiteral(Token &Result, std::string &String,
1389  const char *DiagnosticTag,
1390  bool AllowMacroExpansion);
1391 
1392  /// Lex a token. If it's a comment, keep lexing until we get
1393  /// something not a comment.
1394  ///
1395  /// This is useful in -E -C mode where comments would foul up preprocessor
1396  /// directive handling.
1397  void LexNonComment(Token &Result) {
1398  do
1399  Lex(Result);
1400  while (Result.getKind() == tok::comment);
1401  }
1402 
1403  /// Just like Lex, but disables macro expansion of identifier tokens.
1404  void LexUnexpandedToken(Token &Result) {
1405  // Disable macro expansion.
1406  bool OldVal = DisableMacroExpansion;
1407  DisableMacroExpansion = true;
1408  // Lex the token.
1409  Lex(Result);
1410 
1411  // Reenable it.
1412  DisableMacroExpansion = OldVal;
1413  }
1414 
1415  /// Like LexNonComment, but this disables macro expansion of
1416  /// identifier tokens.
1418  do
1419  LexUnexpandedToken(Result);
1420  while (Result.getKind() == tok::comment);
1421  }
1422 
1423  /// Parses a simple integer literal to get its numeric value. Floating
1424  /// point literals and user defined literals are rejected. Used primarily to
1425  /// handle pragmas that accept integer arguments.
1426  bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1427 
1428  /// Disables macro expansion everywhere except for preprocessor directives.
1430  DisableMacroExpansion = true;
1431  MacroExpansionInDirectivesOverride = true;
1432  }
1433 
1434  /// Peeks ahead N tokens and returns that token without consuming any
1435  /// tokens.
1436  ///
1437  /// LookAhead(0) returns the next token that would be returned by Lex(),
1438  /// LookAhead(1) returns the token after it, etc. This returns normal
1439  /// tokens after phase 5. As such, it is equivalent to using
1440  /// 'Lex', not 'LexUnexpandedToken'.
1441  const Token &LookAhead(unsigned N) {
1442  assert(LexLevel == 0 && "cannot use lookahead while lexing");
1443  if (CachedLexPos + N < CachedTokens.size())
1444  return CachedTokens[CachedLexPos+N];
1445  else
1446  return PeekAhead(N+1);
1447  }
1448 
1449  /// When backtracking is enabled and tokens are cached,
1450  /// this allows to revert a specific number of tokens.
1451  ///
1452  /// Note that the number of tokens being reverted should be up to the last
1453  /// backtrack position, not more.
1454  void RevertCachedTokens(unsigned N) {
1455  assert(isBacktrackEnabled() &&
1456  "Should only be called when tokens are cached for backtracking");
1457  assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1458  && "Should revert tokens up to the last backtrack position, not more");
1459  assert(signed(CachedLexPos) - signed(N) >= 0 &&
1460  "Corrupted backtrack positions ?");
1461  CachedLexPos -= N;
1462  }
1463 
1464  /// Enters a token in the token stream to be lexed next.
1465  ///
1466  /// If BackTrack() is called afterwards, the token will remain at the
1467  /// insertion point.
1468  /// If \p IsReinject is true, resulting token will have Token::IsReinjected
1469  /// flag set. See the flag documentation for details.
1470  void EnterToken(const Token &Tok, bool IsReinject) {
1471  if (LexLevel) {
1472  // It's not correct in general to enter caching lex mode while in the
1473  // middle of a nested lexing action.
1474  auto TokCopy = llvm::make_unique<Token[]>(1);
1475  TokCopy[0] = Tok;
1476  EnterTokenStream(std::move(TokCopy), 1, true, IsReinject);
1477  } else {
1478  EnterCachingLexMode();
1479  assert(IsReinject && "new tokens in the middle of cached stream");
1480  CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1481  }
1482  }
1483 
1484  /// We notify the Preprocessor that if it is caching tokens (because
1485  /// backtrack is enabled) it should replace the most recent cached tokens
1486  /// with the given annotation token. This function has no effect if
1487  /// backtracking is not enabled.
1488  ///
1489  /// Note that the use of this function is just for optimization, so that the
1490  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1491  /// invoked.
1492  void AnnotateCachedTokens(const Token &Tok) {
1493  assert(Tok.isAnnotation() && "Expected annotation token");
1494  if (CachedLexPos != 0 && isBacktrackEnabled())
1495  AnnotatePreviousCachedTokens(Tok);
1496  }
1497 
1498  /// Get the location of the last cached token, suitable for setting the end
1499  /// location of an annotation token.
1501  assert(CachedLexPos != 0);
1502  return CachedTokens[CachedLexPos-1].getLastLoc();
1503  }
1504 
1505  /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1506  /// CachedTokens.
1507  bool IsPreviousCachedToken(const Token &Tok) const;
1508 
1509  /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1510  /// in \p NewToks.
1511  ///
1512  /// Useful when a token needs to be split in smaller ones and CachedTokens
1513  /// most recent token must to be updated to reflect that.
1514  void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1515 
1516  /// Replace the last token with an annotation token.
1517  ///
1518  /// Like AnnotateCachedTokens(), this routine replaces an
1519  /// already-parsed (and resolved) token with an annotation
1520  /// token. However, this routine only replaces the last token with
1521  /// the annotation token; it does not affect any other cached
1522  /// tokens. This function has no effect if backtracking is not
1523  /// enabled.
1525  assert(Tok.isAnnotation() && "Expected annotation token");
1526  if (CachedLexPos != 0 && isBacktrackEnabled())
1527  CachedTokens[CachedLexPos-1] = Tok;
1528  }
1529 
1530  /// Enter an annotation token into the token stream.
1531  void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1532  void *AnnotationVal);
1533 
1534  /// Update the current token to represent the provided
1535  /// identifier, in order to cache an action performed by typo correction.
1536  void TypoCorrectToken(const Token &Tok) {
1537  assert(Tok.getIdentifierInfo() && "Expected identifier token");
1538  if (CachedLexPos != 0 && isBacktrackEnabled())
1539  CachedTokens[CachedLexPos-1] = Tok;
1540  }
1541 
1542  /// Recompute the current lexer kind based on the CurLexer/
1543  /// CurTokenLexer pointers.
1544  void recomputeCurLexerKind();
1545 
1546  /// Returns true if incremental processing is enabled
1547  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1548 
1549  /// Enables the incremental processing
1550  void enableIncrementalProcessing(bool value = true) {
1551  IncrementalProcessing = value;
1552  }
1553 
1554  /// Specify the point at which code-completion will be performed.
1555  ///
1556  /// \param File the file in which code completion should occur. If
1557  /// this file is included multiple times, code-completion will
1558  /// perform completion the first time it is included. If NULL, this
1559  /// function clears out the code-completion point.
1560  ///
1561  /// \param Line the line at which code completion should occur
1562  /// (1-based).
1563  ///
1564  /// \param Column the column at which code completion should occur
1565  /// (1-based).
1566  ///
1567  /// \returns true if an error occurred, false otherwise.
1568  bool SetCodeCompletionPoint(const FileEntry *File,
1569  unsigned Line, unsigned Column);
1570 
1571  /// Determine if we are performing code completion.
1572  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1573 
1574  /// Returns the location of the code-completion point.
1575  ///
1576  /// Returns an invalid location if code-completion is not enabled or the file
1577  /// containing the code-completion point has not been lexed yet.
1578  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1579 
1580  /// Returns the start location of the file of code-completion point.
1581  ///
1582  /// Returns an invalid location if code-completion is not enabled or the file
1583  /// containing the code-completion point has not been lexed yet.
1585  return CodeCompletionFileLoc;
1586  }
1587 
1588  /// Returns true if code-completion is enabled and we have hit the
1589  /// code-completion point.
1590  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1591 
1592  /// Note that we hit the code-completion point.
1594  assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1595  CodeCompletionReached = true;
1596  // Silence any diagnostics that occur after we hit the code-completion.
1597  getDiagnostics().setSuppressAllDiagnostics(true);
1598  }
1599 
1600  /// The location of the currently-active \#pragma clang
1601  /// arc_cf_code_audited begin.
1602  ///
1603  /// Returns an invalid location if there is no such pragma active.
1605  return PragmaARCCFCodeAuditedLoc;
1606  }
1607 
1608  /// Set the location of the currently-active \#pragma clang
1609  /// arc_cf_code_audited begin. An invalid location ends the pragma.
1611  PragmaARCCFCodeAuditedLoc = Loc;
1612  }
1613 
1614  /// The location of the currently-active \#pragma clang
1615  /// assume_nonnull begin.
1616  ///
1617  /// Returns an invalid location if there is no such pragma active.
1619  return PragmaAssumeNonNullLoc;
1620  }
1621 
1622  /// Set the location of the currently-active \#pragma clang
1623  /// assume_nonnull begin. An invalid location ends the pragma.
1625  PragmaAssumeNonNullLoc = Loc;
1626  }
1627 
1628  /// Set the directory in which the main file should be considered
1629  /// to have been found, if it is not a real file.
1630  void setMainFileDir(const DirectoryEntry *Dir) {
1631  MainFileDir = Dir;
1632  }
1633 
1634  /// Instruct the preprocessor to skip part of the main source file.
1635  ///
1636  /// \param Bytes The number of bytes in the preamble to skip.
1637  ///
1638  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1639  /// start of a line.
1640  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1641  SkipMainFilePreamble.first = Bytes;
1642  SkipMainFilePreamble.second = StartOfLine;
1643  }
1644 
1645  /// Forwarding function for diagnostics. This emits a diagnostic at
1646  /// the specified Token's location, translating the token's start
1647  /// position in the current buffer into a SourcePosition object for rendering.
1648  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1649  return Diags->Report(Loc, DiagID);
1650  }
1651 
1652  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1653  return Diags->Report(Tok.getLocation(), DiagID);
1654  }
1655 
1656  /// Return the 'spelling' of the token at the given
1657  /// location; does not go up to the spelling location or down to the
1658  /// expansion location.
1659  ///
1660  /// \param buffer A buffer which will be used only if the token requires
1661  /// "cleaning", e.g. if it contains trigraphs or escaped newlines
1662  /// \param invalid If non-null, will be set \c true if an error occurs.
1664  SmallVectorImpl<char> &buffer,
1665  bool *invalid = nullptr) const {
1666  return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1667  }
1668 
1669  /// Return the 'spelling' of the Tok token.
1670  ///
1671  /// The spelling of a token is the characters used to represent the token in
1672  /// the source file after trigraph expansion and escaped-newline folding. In
1673  /// particular, this wants to get the true, uncanonicalized, spelling of
1674  /// things like digraphs, UCNs, etc.
1675  ///
1676  /// \param Invalid If non-null, will be set \c true if an error occurs.
1677  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1678  return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1679  }
1680 
1681  /// Get the spelling of a token into a preallocated buffer, instead
1682  /// of as an std::string.
1683  ///
1684  /// The caller is required to allocate enough space for the token, which is
1685  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1686  /// actual result is returned.
1687  ///
1688  /// Note that this method may do two possible things: it may either fill in
1689  /// the buffer specified with characters, or it may *change the input pointer*
1690  /// to point to a constant buffer with the data already in it (avoiding a
1691  /// copy). The caller is not allowed to modify the returned buffer pointer
1692  /// if an internal buffer is returned.
1693  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1694  bool *Invalid = nullptr) const {
1695  return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1696  }
1697 
1698  /// Get the spelling of a token into a SmallVector.
1699  ///
1700  /// Note that the returned StringRef may not point to the
1701  /// supplied buffer if a copy can be avoided.
1702  StringRef getSpelling(const Token &Tok,
1703  SmallVectorImpl<char> &Buffer,
1704  bool *Invalid = nullptr) const;
1705 
1706  /// Relex the token at the specified location.
1707  /// \returns true if there was a failure, false on success.
1708  bool getRawToken(SourceLocation Loc, Token &Result,
1709  bool IgnoreWhiteSpace = false) {
1710  return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1711  }
1712 
1713  /// Given a Token \p Tok that is a numeric constant with length 1,
1714  /// return the character.
1715  char
1717  bool *Invalid = nullptr) const {
1718  assert(Tok.is(tok::numeric_constant) &&
1719  Tok.getLength() == 1 && "Called on unsupported token");
1720  assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1721 
1722  // If the token is carrying a literal data pointer, just use it.
1723  if (const char *D = Tok.getLiteralData())
1724  return *D;
1725 
1726  // Otherwise, fall back on getCharacterData, which is slower, but always
1727  // works.
1728  return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1729  }
1730 
1731  /// Retrieve the name of the immediate macro expansion.
1732  ///
1733  /// This routine starts from a source location, and finds the name of the
1734  /// macro responsible for its immediate expansion. It looks through any
1735  /// intervening macro argument expansions to compute this. It returns a
1736  /// StringRef that refers to the SourceManager-owned buffer of the source
1737  /// where that macro name is spelled. Thus, the result shouldn't out-live
1738  /// the SourceManager.
1740  return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1741  }
1742 
1743  /// Plop the specified string into a scratch buffer and set the
1744  /// specified token's location and length to it.
1745  ///
1746  /// If specified, the source location provides a location of the expansion
1747  /// point of the token.
1748  void CreateString(StringRef Str, Token &Tok,
1749  SourceLocation ExpansionLocStart = SourceLocation(),
1750  SourceLocation ExpansionLocEnd = SourceLocation());
1751 
1752  /// Split the first Length characters out of the token starting at TokLoc
1753  /// and return a location pointing to the split token. Re-lexing from the
1754  /// split token will return the split token rather than the original.
1755  SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
1756 
1757  /// Computes the source location just past the end of the
1758  /// token at this source location.
1759  ///
1760  /// This routine can be used to produce a source location that
1761  /// points just past the end of the token referenced by \p Loc, and
1762  /// is generally used when a diagnostic needs to point just after a
1763  /// token where it expected something different that it received. If
1764  /// the returned source location would not be meaningful (e.g., if
1765  /// it points into a macro), this routine returns an invalid
1766  /// source location.
1767  ///
1768  /// \param Offset an offset from the end of the token, where the source
1769  /// location should refer to. The default offset (0) produces a source
1770  /// location pointing just past the end of the token; an offset of 1 produces
1771  /// a source location pointing to the last character in the token, etc.
1773  return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1774  }
1775 
1776  /// Returns true if the given MacroID location points at the first
1777  /// token of the macro expansion.
1778  ///
1779  /// \param MacroBegin If non-null and function returns true, it is set to
1780  /// begin location of the macro.
1782  SourceLocation *MacroBegin = nullptr) const {
1783  return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1784  MacroBegin);
1785  }
1786 
1787  /// Returns true if the given MacroID location points at the last
1788  /// token of the macro expansion.
1789  ///
1790  /// \param MacroEnd If non-null and function returns true, it is set to
1791  /// end location of the macro.
1793  SourceLocation *MacroEnd = nullptr) const {
1794  return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1795  }
1796 
1797  /// Print the token to stderr, used for debugging.
1798  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1799  void DumpLocation(SourceLocation Loc) const;
1800  void DumpMacro(const MacroInfo &MI) const;
1801  void dumpMacroInfo(const IdentifierInfo *II);
1802 
1803  /// Given a location that specifies the start of a
1804  /// token, return a new location that specifies a character within the token.
1806  unsigned Char) const {
1807  return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1808  }
1809 
1810  /// Increment the counters for the number of token paste operations
1811  /// performed.
1812  ///
1813  /// If fast was specified, this is a 'fast paste' case we handled.
1814  void IncrementPasteCounter(bool isFast) {
1815  if (isFast)
1816  ++NumFastTokenPaste;
1817  else
1818  ++NumTokenPaste;
1819  }
1820 
1821  void PrintStats();
1822 
1823  size_t getTotalMemory() const;
1824 
1825  /// When the macro expander pastes together a comment (/##/) in Microsoft
1826  /// mode, this method handles updating the current state, returning the
1827  /// token on the next source line.
1828  void HandleMicrosoftCommentPaste(Token &Tok);
1829 
1830  //===--------------------------------------------------------------------===//
1831  // Preprocessor callback methods. These are invoked by a lexer as various
1832  // directives and events are found.
1833 
1834  /// Given a tok::raw_identifier token, look up the
1835  /// identifier information for the token and install it into the token,
1836  /// updating the token kind accordingly.
1837  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1838 
1839 private:
1840  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1841 
1842 public:
1843  /// Specifies the reason for poisoning an identifier.
1844  ///
1845  /// If that identifier is accessed while poisoned, then this reason will be
1846  /// used instead of the default "poisoned" diagnostic.
1847  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1848 
1849  /// Display reason for poisoned identifier.
1850  void HandlePoisonedIdentifier(Token & Identifier);
1851 
1853  if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1854  if(II->isPoisoned()) {
1855  HandlePoisonedIdentifier(Identifier);
1856  }
1857  }
1858  }
1859 
1860 private:
1861  /// Identifiers used for SEH handling in Borland. These are only
1862  /// allowed in particular circumstances
1863  // __except block
1864  IdentifierInfo *Ident__exception_code,
1865  *Ident___exception_code,
1866  *Ident_GetExceptionCode;
1867  // __except filter expression
1868  IdentifierInfo *Ident__exception_info,
1869  *Ident___exception_info,
1870  *Ident_GetExceptionInfo;
1871  // __finally
1872  IdentifierInfo *Ident__abnormal_termination,
1873  *Ident___abnormal_termination,
1874  *Ident_AbnormalTermination;
1875 
1876  const char *getCurLexerEndPos();
1877  void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
1878 
1879 public:
1880  void PoisonSEHIdentifiers(bool Poison = true); // Borland
1881 
1882  /// Callback invoked when the lexer reads an identifier and has
1883  /// filled in the tokens IdentifierInfo member.
1884  ///
1885  /// This callback potentially macro expands it or turns it into a named
1886  /// token (like 'for').
1887  ///
1888  /// \returns true if we actually computed a token, false if we need to
1889  /// lex again.
1890  bool HandleIdentifier(Token &Identifier);
1891 
1892  /// Callback invoked when the lexer hits the end of the current file.
1893  ///
1894  /// This either returns the EOF token and returns true, or
1895  /// pops a level off the include stack and returns false, at which point the
1896  /// client should call lex again.
1897  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1898 
1899  /// Callback invoked when the current TokenLexer hits the end of its
1900  /// token stream.
1901  bool HandleEndOfTokenLexer(Token &Result);
1902 
1903  /// Callback invoked when the lexer sees a # token at the start of a
1904  /// line.
1905  ///
1906  /// This consumes the directive, modifies the lexer/preprocessor state, and
1907  /// advances the lexer(s) so that the next token read is the correct one.
1908  void HandleDirective(Token &Result);
1909 
1910  /// Ensure that the next token is a tok::eod token.
1911  ///
1912  /// If not, emit a diagnostic and consume up until the eod.
1913  /// If \p EnableMacros is true, then we consider macros that expand to zero
1914  /// tokens as being ok.
1915  ///
1916  /// \return The location of the end of the directive (the terminating
1917  /// newline).
1918  SourceLocation CheckEndOfDirective(const char *DirType,
1919  bool EnableMacros = false);
1920 
1921  /// Read and discard all tokens remaining on the current line until
1922  /// the tok::eod token is found. Returns the range of the skipped tokens.
1923  SourceRange DiscardUntilEndOfDirective();
1924 
1925  /// Returns true if the preprocessor has seen a use of
1926  /// __DATE__ or __TIME__ in the file so far.
1927  bool SawDateOrTime() const {
1928  return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
1929  }
1930  unsigned getCounterValue() const { return CounterValue; }
1931  void setCounterValue(unsigned V) { CounterValue = V; }
1932 
1933  /// Retrieves the module that we're currently building, if any.
1934  Module *getCurrentModule();
1935 
1936  /// Allocate a new MacroInfo object with the provided SourceLocation.
1937  MacroInfo *AllocateMacroInfo(SourceLocation L);
1938 
1939  /// Turn the specified lexer token into a fully checked and spelled
1940  /// filename, e.g. as an operand of \#include.
1941  ///
1942  /// The caller is expected to provide a buffer that is large enough to hold
1943  /// the spelling of the filename, but is also expected to handle the case
1944  /// when this method decides to use a different buffer.
1945  ///
1946  /// \returns true if the input filename was in <>'s or false if it was
1947  /// in ""'s.
1948  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
1949 
1950  /// Given a "foo" or <foo> reference, look up the indicated file.
1951  ///
1952  /// Returns null on failure. \p isAngled indicates whether the file
1953  /// reference is for system \#include's or not (i.e. using <> instead of "").
1954  const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename,
1955  bool isAngled, const DirectoryLookup *FromDir,
1956  const FileEntry *FromFile,
1957  const DirectoryLookup *&CurDir,
1958  SmallVectorImpl<char> *SearchPath,
1959  SmallVectorImpl<char> *RelativePath,
1960  ModuleMap::KnownHeader *SuggestedModule,
1961  bool *IsMapped, bool *IsFrameworkFound,
1962  bool SkipCache = false);
1963 
1964  /// Get the DirectoryLookup structure used to find the current
1965  /// FileEntry, if CurLexer is non-null and if applicable.
1966  ///
1967  /// This allows us to implement \#include_next and find directory-specific
1968  /// properties.
1969  const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1970 
1971  /// Return true if we're in the top-level file, not in a \#include.
1972  bool isInPrimaryFile() const;
1973 
1974  /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1975  /// followed by EOD. Return true if the token is not a valid on-off-switch.
1976  bool LexOnOffSwitch(tok::OnOffSwitch &Result);
1977 
1978  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
1979  bool *ShadowFlag = nullptr);
1980 
1981  void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
1982  Module *LeaveSubmodule(bool ForPragma);
1983 
1984 private:
1985  friend void TokenLexer::ExpandFunctionArguments();
1986 
1987  void PushIncludeMacroStack() {
1988  assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
1989  IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
1990  std::move(CurLexer), CurPPLexer,
1991  std::move(CurTokenLexer), CurDirLookup);
1992  CurPPLexer = nullptr;
1993  }
1994 
1995  void PopIncludeMacroStack() {
1996  CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1997  CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1998  CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1999  CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2000  CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2001  CurLexerKind = IncludeMacroStack.back().CurLexerKind;
2002  IncludeMacroStack.pop_back();
2003  }
2004 
2005  void PropagateLineStartLeadingSpaceInfo(Token &Result);
2006 
2007  /// Determine whether we need to create module macros for #defines in the
2008  /// current context.
2009  bool needModuleMacros() const;
2010 
2011  /// Update the set of active module macros and ambiguity flag for a module
2012  /// macro name.
2013  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
2014 
2015  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2016  SourceLocation Loc);
2017  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2018  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2019  bool isPublic);
2020 
2021  /// Lex and validate a macro name, which occurs after a
2022  /// \#define or \#undef.
2023  ///
2024  /// \param MacroNameTok Token that represents the name defined or undefined.
2025  /// \param IsDefineUndef Kind if preprocessor directive.
2026  /// \param ShadowFlag Points to flag that is set if macro name shadows
2027  /// a keyword.
2028  ///
2029  /// This emits a diagnostic, sets the token kind to eod,
2030  /// and discards the rest of the macro line if the macro name is invalid.
2031  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
2032  bool *ShadowFlag = nullptr);
2033 
2034  /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
2035  /// entire line) of the macro's tokens and adds them to MacroInfo, and while
2036  /// doing so performs certain validity checks including (but not limited to):
2037  /// - # (stringization) is followed by a macro parameter
2038  /// \param MacroNameTok - Token that represents the macro name
2039  /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
2040  ///
2041  /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
2042  /// returns a nullptr if an invalid sequence of tokens is encountered.
2043  MacroInfo *ReadOptionalMacroParameterListAndBody(
2044  const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
2045 
2046  /// The ( starting an argument list of a macro definition has just been read.
2047  /// Lex the rest of the parameters and the closing ), updating \p MI with
2048  /// what we learn and saving in \p LastTok the last token read.
2049  /// Return true if an error occurs parsing the arg list.
2050  bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2051 
2052  /// We just read a \#if or related directive and decided that the
2053  /// subsequent tokens are in the \#if'd out portion of the
2054  /// file. Lex the rest of the file, until we see an \#endif. If \p
2055  /// FoundNonSkipPortion is true, then we have already emitted code for part of
2056  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
2057  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
2058  /// already seen one so a \#else directive is a duplicate. When this returns,
2059  /// the caller can lex the first valid token.
2060  void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2061  SourceLocation IfTokenLoc,
2062  bool FoundNonSkipPortion, bool FoundElse,
2063  SourceLocation ElseLoc = SourceLocation());
2064 
2065  /// Information about the result for evaluating an expression for a
2066  /// preprocessor directive.
2067  struct DirectiveEvalResult {
2068  /// Whether the expression was evaluated as true or not.
2069  bool Conditional;
2070 
2071  /// True if the expression contained identifiers that were undefined.
2072  bool IncludedUndefinedIds;
2073 
2074  /// The source range for the expression.
2075  SourceRange ExprRange;
2076  };
2077 
2078  /// Evaluate an integer constant expression that may occur after a
2079  /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2080  ///
2081  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2082  DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
2083 
2084  /// Install the standard preprocessor pragmas:
2085  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
2086  void RegisterBuiltinPragmas();
2087 
2088  /// Register builtin macros such as __LINE__ with the identifier table.
2089  void RegisterBuiltinMacros();
2090 
2091  /// If an identifier token is read that is to be expanded as a macro, handle
2092  /// it and return the next token as 'Tok'. If we lexed a token, return true;
2093  /// otherwise the caller should lex again.
2094  bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2095 
2096  /// Cache macro expanded tokens for TokenLexers.
2097  //
2098  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2099  /// going to lex in the cache and when it finishes the tokens are removed
2100  /// from the end of the cache.
2101  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2102  ArrayRef<Token> tokens);
2103 
2104  void removeCachedMacroExpandedTokensOfLastLexer();
2105 
2106  /// Determine whether the next preprocessor token to be
2107  /// lexed is a '('. If so, consume the token and return true, if not, this
2108  /// method should have no observable side-effect on the lexed tokens.
2109  bool isNextPPTokenLParen();
2110 
2111  /// After reading "MACRO(", this method is invoked to read all of the formal
2112  /// arguments specified for the macro invocation. Returns null on error.
2113  MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2114  SourceLocation &MacroEnd);
2115 
2116  /// If an identifier token is read that is to be expanded
2117  /// as a builtin macro, handle it and return the next token as 'Tok'.
2118  void ExpandBuiltinMacro(Token &Tok);
2119 
2120  /// Read a \c _Pragma directive, slice it up, process it, then
2121  /// return the first token after the directive.
2122  /// This assumes that the \c _Pragma token has just been read into \p Tok.
2123  void Handle_Pragma(Token &Tok);
2124 
2125  /// Like Handle_Pragma except the pragma text is not enclosed within
2126  /// a string literal.
2127  void HandleMicrosoft__pragma(Token &Tok);
2128 
2129  /// Add a lexer to the top of the include stack and
2130  /// start lexing tokens from it instead of the current buffer.
2131  void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
2132 
2133  /// Set the FileID for the preprocessor predefines.
2134  void setPredefinesFileID(FileID FID) {
2135  assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2136  PredefinesFileID = FID;
2137  }
2138 
2139  /// Set the FileID for the PCH through header.
2140  void setPCHThroughHeaderFileID(FileID FID);
2141 
2142  /// Returns true if we are lexing from a file and not a
2143  /// pragma or a macro.
2144  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2145  return L ? !L->isPragmaLexer() : P != nullptr;
2146  }
2147 
2148  static bool IsFileLexer(const IncludeStackInfo& I) {
2149  return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2150  }
2151 
2152  bool IsFileLexer() const {
2153  return IsFileLexer(CurLexer.get(), CurPPLexer);
2154  }
2155 
2156  //===--------------------------------------------------------------------===//
2157  // Caching stuff.
2158  void CachingLex(Token &Result);
2159 
2160  bool InCachingLexMode() const {
2161  // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2162  // that we are past EOF, not that we are in CachingLex mode.
2163  return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2164  }
2165 
2166  void EnterCachingLexMode();
2167  void EnterCachingLexModeUnchecked();
2168 
2169  void ExitCachingLexMode() {
2170  if (InCachingLexMode())
2171  RemoveTopOfLexerStack();
2172  }
2173 
2174  const Token &PeekAhead(unsigned N);
2175  void AnnotatePreviousCachedTokens(const Token &Tok);
2176 
2177  //===--------------------------------------------------------------------===//
2178  /// Handle*Directive - implement the various preprocessor directives. These
2179  /// should side-effect the current preprocessor object so that the next call
2180  /// to Lex() will return the appropriate token next.
2181  void HandleLineDirective();
2182  void HandleDigitDirective(Token &Tok);
2183  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2184  void HandleIdentSCCSDirective(Token &Tok);
2185  void HandleMacroPublicDirective(Token &Tok);
2186  void HandleMacroPrivateDirective();
2187 
2188  /// An additional notification that can be produced by a header inclusion or
2189  /// import to tell the parser what happened.
2190  struct ImportAction {
2191  enum ActionKind {
2192  None,
2193  ModuleBegin,
2194  ModuleImport,
2195  SkippedModuleImport,
2196  } Kind;
2197  Module *ModuleForHeader = nullptr;
2198 
2199  ImportAction(ActionKind AK, Module *Mod = nullptr)
2200  : Kind(AK), ModuleForHeader(Mod) {
2201  assert((AK == None || Mod) && "no module for module action");
2202  }
2203  };
2204 
2205  // File inclusion.
2206  void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2207  const DirectoryLookup *LookupFrom = nullptr,
2208  const FileEntry *LookupFromFile = nullptr);
2209  ImportAction
2210  HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2211  Token &FilenameTok, SourceLocation EndLoc,
2212  const DirectoryLookup *LookupFrom = nullptr,
2213  const FileEntry *LookupFromFile = nullptr);
2214  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2215  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2216  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2217  void HandleMicrosoftImportDirective(Token &Tok);
2218 
2219 public:
2220  /// Check that the given module is available, producing a diagnostic if not.
2221  /// \return \c true if the check failed (because the module is not available).
2222  /// \c false if the module appears to be usable.
2223  static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2224  const TargetInfo &TargetInfo,
2225  DiagnosticsEngine &Diags, Module *M);
2226 
2227  // Module inclusion testing.
2228  /// Find the module that owns the source or header file that
2229  /// \p Loc points to. If the location is in a file that was included
2230  /// into a module, or is outside any module, returns nullptr.
2231  Module *getModuleForLocation(SourceLocation Loc);
2232 
2233  /// We want to produce a diagnostic at location IncLoc concerning a
2234  /// missing module import.
2235  ///
2236  /// \param IncLoc The location at which the missing import was detected.
2237  /// \param M The desired module.
2238  /// \param MLoc A location within the desired module at which some desired
2239  /// effect occurred (eg, where a desired entity was declared).
2240  ///
2241  /// \return A file that can be #included to import a module containing MLoc.
2242  /// Null if no such file could be determined or if a #include is not
2243  /// appropriate.
2244  const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2245  Module *M,
2246  SourceLocation MLoc);
2247 
2248  bool isRecordingPreamble() const {
2249  return PreambleConditionalStack.isRecording();
2250  }
2251 
2252  bool hasRecordedPreamble() const {
2253  return PreambleConditionalStack.hasRecordedPreamble();
2254  }
2255 
2257  return PreambleConditionalStack.getStack();
2258  }
2259 
2261  PreambleConditionalStack.setStack(s);
2262  }
2263 
2266  PreambleConditionalStack.startReplaying();
2267  PreambleConditionalStack.setStack(s);
2268  PreambleConditionalStack.SkipInfo = SkipInfo;
2269  }
2270 
2272  return PreambleConditionalStack.SkipInfo;
2273  }
2274 
2275 private:
2276  /// After processing predefined file, initialize the conditional stack from
2277  /// the preamble.
2278  void replayPreambleConditionalStack();
2279 
2280  // Macro handling.
2281  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2282  void HandleUndefDirective();
2283 
2284  // Conditional Inclusion.
2285  void HandleIfdefDirective(Token &Result, const Token &HashToken,
2286  bool isIfndef, bool ReadAnyTokensBeforeDirective);
2287  void HandleIfDirective(Token &IfToken, const Token &HashToken,
2288  bool ReadAnyTokensBeforeDirective);
2289  void HandleEndifDirective(Token &EndifToken);
2290  void HandleElseDirective(Token &Result, const Token &HashToken);
2291  void HandleElifDirective(Token &ElifToken, const Token &HashToken);
2292 
2293  // Pragmas.
2294  void HandlePragmaDirective(PragmaIntroducer Introducer);
2295 
2296 public:
2297  void HandlePragmaOnce(Token &OnceTok);
2298  void HandlePragmaMark();
2299  void HandlePragmaPoison();
2300  void HandlePragmaSystemHeader(Token &SysHeaderTok);
2301  void HandlePragmaDependency(Token &DependencyTok);
2302  void HandlePragmaPushMacro(Token &Tok);
2303  void HandlePragmaPopMacro(Token &Tok);
2304  void HandlePragmaIncludeAlias(Token &Tok);
2305  void HandlePragmaModuleBuild(Token &Tok);
2306  void HandlePragmaHdrstop(Token &Tok);
2307  IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2308 
2309  // Return true and store the first token only if any CommentHandler
2310  // has inserted some tokens and getCommentRetentionState() is false.
2311  bool HandleComment(Token &result, SourceRange Comment);
2312 
2313  /// A macro is used, update information about macros that need unused
2314  /// warnings.
2315  void markMacroAsUsed(MacroInfo *MI);
2316 };
2317 
2318 /// Abstract base class that describes a handler that will receive
2319 /// source ranges for each of the comments encountered in the source file.
2321 public:
2322  virtual ~CommentHandler();
2323 
2324  // The handler shall return true if it has pushed any tokens
2325  // to be read using e.g. EnterToken or EnterTokenStream.
2326  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2327 };
2328 
2329 /// Registry of pragma handlers added by plugins
2330 using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2331 
2332 } // namespace clang
2333 
2334 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
Definition: Preprocessor.h:423
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
A set of visible modules.
Definition: Module.h:596
StringRef Identifier
Definition: Format.cpp:1716
llvm::BumpPtrAllocator & getPreprocessorAllocator()
Definition: Preprocessor.h:914
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:76
SelectorTable & getSelectorTable()
Definition: Preprocessor.h:912
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:67
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:116
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the &#39;spelling&#39; of the Tok token.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:29
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:97
Defines the SourceManager interface.
static const Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:20
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
FileManager & getFileManager() const
Definition: Preprocessor.h:906
Defines the clang::Module class, which describes a module in the source code.
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
SourceLocation getPragmaARCCFCodeAuditedLoc() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
StringRef P
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:564
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1296
A directive for an undefined macro.
Definition: MacroInfo.h:429
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool hadModuleLoaderFatalFailure() const
Definition: Preprocessor.h:927
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:100
Builtin::Context & getBuiltinInfo()
Definition: Preprocessor.h:913
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:35
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:120
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Definition: MacroInfo.h:328
tok::TokenKind getKind() const
Definition: Token.h:92
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:90
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
One of these records is kept for each identifier that is lexed.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
Represents a macro directive exported by a module.
Definition: MacroInfo.h:488
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:406
This table allows us to fully hide how we implement multi-keyword caching.
LineState State
Definition: Format.h:2261
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:904
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Definition: Preprocessor.h:937
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:49
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:903
Describes a module or submodule.
Definition: Module.h:64
bool isPragmaLexer() const
isPragmaLexer - Returns true if this Lexer is being used to lex a pragma.
Definition: Lexer.h:183
A directive for setting the module visibility of a macro.
Definition: MacroInfo.h:444
bool isInvalid() const
void SetSuppressIncludeNotFoundError(bool Suppress)
Definition: Preprocessor.h:947
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:107
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
Definition: Preprocessor.h:984
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:908
const FormatToken & Tok
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void LexNonComment(Token &Result)
Lex a token.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:148
bool isMacroDefined(const IdentifierInfo *II)
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we&#39;ve alread...
Defines the Diagnostic-related interfaces.
const TargetInfo * getAuxTargetInfo() const
Definition: Preprocessor.h:905
TokenLexer - This implements a lexer that returns tokens from a macro body or token stream instead of...
Definition: TokenLexer.h:30
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream...
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
Definition: Preprocessor.h:957
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:158
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:1044
void clearCodeCompletionHandler()
Clear out the code completion handler.
StringRef Filename
Definition: Format.cpp:1709
Provides lookups to, and iteration over, IdentiferInfo objects.
unsigned Offset
Definition: Format.cpp:1711
Exposes information about the current target.
Definition: TargetInfo.h:161
const IdentifierTable & getIdentifierTable() const
Definition: Preprocessor.h:911
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Abstract interface for external sources of preprocessor information.
Defines the clang::LangOptions interface.
unsigned getCounterValue() const
SourceLocation End
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:349
int Id
Definition: ASTDiff.cpp:190
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
const AnnotatedLine * Line
bool isDefined() const
Definition: MacroInfo.h:384
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Definition: Preprocessor.h:966
Implements an efficient mapping from strings to IdentifierInfo nodes.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
#define V(N, I)
Definition: ASTContext.h:2912
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
A class for tracking whether we&#39;re inside a VA_OPT during a traversal of the tokens of a variadic mac...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:126
#define bool
Definition: stdbool.h:15
DefInfo getDefinition()
Traverses the macro directives history and returns the next macro definition directive along with inf...
Definition: MacroInfo.cpp:178
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:85
llvm::Optional< PreambleSkipInfo > getPreambleSkipInfo() const
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
bool getCommentRetentionState() const
Definition: Preprocessor.h:942
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:98
PPCallbacks * getPPCallbacks() const
Definition: Preprocessor.h:994
const SourceManager & SM
Definition: Format.cpp:1570
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
bool isRecordingPreamble() const
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
SourceManager & getSourceManager() const
Definition: Preprocessor.h:907
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define&#39;d and not #undef&#39;...
const DefInfo findDirectiveAtLoc(SourceLocation L, const SourceManager &SM) const
Find macro definition active in the specified source location.
Definition: MacroInfo.cpp:202
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:920
llvm::cl::opt< std::string > Filter
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:290
Kind
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
void setExternalSource(ExternalPreprocessorSource *Source)
Definition: Preprocessor.h:916
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.h:93
void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:179
IdentifierTable & getIdentifierTable()
Definition: Preprocessor.h:910
void setPragmasEnabled(bool Enabled)
Definition: Preprocessor.h:944
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:85
bool isPoisoned() const
Return true if this token has been poisoned.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:24
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
Definition: Preprocessor.h:974
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
const MacroInfo * getMacroInfo() const
Definition: MacroInfo.h:390
StringRef getName() const
Return the actual identifier string.
bool isMacroDefined(StringRef Id)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
Dataflow Directional Tag Classes.
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
Definition: DeclSpec.h:1152
SourceRange getCodeCompletionTokenRange() const
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:354
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:65
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Abstract interface for a module loader.
Definition: ModuleLoader.h:73
unsigned getLength() const
Definition: Token.h:129
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:47
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:217
bool getPragmasEnabled() const
Definition: Preprocessor.h:945
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:123
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
Definition: Preprocessor.h:925
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Definition: Preprocessor.h:898
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:45
Defines the PPCallbacks interface.
Defines the clang::TokenKind enum and support functions.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
SourceLocation getModuleImportLoc(Module *M) const
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:987
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:900
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:362
bool hasRecordedPreamble() const
const std::string & getPredefines() const
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:285
void setPredefines(StringRef P)
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
The translation unit is a complete translation unit.
Definition: LangOptions.h:364
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
MacroMap::const_iterator macro_iterator
Abstract base class that describes a handler that will receive source ranges for each of the comments...
result[0]
Definition: emmintrin.h:120
Describes how and where the pragma was introduced.
Definition: Pragma.h:51
void setCounterValue(unsigned V)
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
Definition: Preprocessor.h:932
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
Definition: Preprocessor.h:951
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, llvm::Optional< PreambleSkipInfo > SkipInfo)
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Callback handler that receives notifications when performing code completion within the preprocessor...
A header that is known to reside within a given module, whether it was included or excluded...
Definition: ModuleMap.h:150
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
Definition: Preprocessor.h:963
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:995
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:124
void setDiagnostics(DiagnosticsEngine &D)
Definition: Preprocessor.h:901