clang 20.0.0git
Preprocessor.h
Go to the documentation of this file.
1//===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Defines the clang::Preprocessor interface.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15#define LLVM_CLANG_LEX_PREPROCESSOR_H
16
20#include "clang/Basic/LLVM.h"
22#include "clang/Basic/Module.h"
27#include "clang/Lex/Lexer.h"
28#include "clang/Lex/MacroInfo.h"
30#include "clang/Lex/ModuleMap.h"
33#include "clang/Lex/Token.h"
35#include "llvm/ADT/APSInt.h"
36#include "llvm/ADT/ArrayRef.h"
37#include "llvm/ADT/DenseMap.h"
38#include "llvm/ADT/FoldingSet.h"
39#include "llvm/ADT/FunctionExtras.h"
40#include "llvm/ADT/PointerUnion.h"
41#include "llvm/ADT/STLExtras.h"
42#include "llvm/ADT/SmallPtrSet.h"
43#include "llvm/ADT/SmallVector.h"
44#include "llvm/ADT/StringRef.h"
45#include "llvm/ADT/TinyPtrVector.h"
46#include "llvm/ADT/iterator_range.h"
47#include "llvm/Support/Allocator.h"
48#include "llvm/Support/Casting.h"
49#include "llvm/Support/Registry.h"
50#include <cassert>
51#include <cstddef>
52#include <cstdint>
53#include <map>
54#include <memory>
55#include <optional>
56#include <string>
57#include <utility>
58#include <vector>
59
60namespace llvm {
61
62template<unsigned InternalLen> class SmallString;
63
64} // namespace llvm
65
66namespace clang {
67
68class CodeCompletionHandler;
69class CommentHandler;
70class DirectoryEntry;
71class EmptylineHandler;
72class ExternalPreprocessorSource;
73class FileEntry;
74class FileManager;
75class HeaderSearch;
76class MacroArgs;
77class PragmaHandler;
78class PragmaNamespace;
79class PreprocessingRecord;
80class PreprocessorLexer;
81class PreprocessorOptions;
82class ScratchBuffer;
83class TargetInfo;
84
85namespace Builtin {
86class Context;
87}
88
89/// Stores token information for comparing actual tokens with
90/// predefined values. Only handles simple tokens and identifiers.
92 tok::TokenKind Kind;
94
95public:
96 TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
97 assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
98 assert(Kind != tok::identifier &&
99 "Identifiers should be created by TokenValue(IdentifierInfo *)");
100 assert(!tok::isLiteral(Kind) && "Literals are not supported.");
101 assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
102 }
103
104 TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
105
106 bool operator==(const Token &Tok) const {
107 return Tok.getKind() == Kind &&
108 (!II || II == Tok.getIdentifierInfo());
109 }
110};
111
112/// Context in which macro name is used.
114 // other than #define or #undef
116
117 // macro name specified in #define
119
120 // macro name specified in #undef
121 MU_Undef = 2
123
124enum class EmbedResult {
125 Invalid = -1, // Parsing error occurred.
126 NotFound = 0, // Corresponds to __STDC_EMBED_NOT_FOUND__
127 Found = 1, // Corresponds to __STDC_EMBED_FOUND__
128 Empty = 2, // Corresponds to __STDC_EMBED_EMPTY__
129};
130
131/// Engages in a tight little dance with the lexer to efficiently
132/// preprocess tokens.
133///
134/// Lexers know only about tokens within a single source file, and don't
135/// know anything about preprocessor-level issues like the \#include stack,
136/// token expansion, etc.
140
141 llvm::unique_function<void(const clang::Token &)> OnToken;
142 std::shared_ptr<PreprocessorOptions> PPOpts;
143 DiagnosticsEngine *Diags;
144 const LangOptions &LangOpts;
145 const TargetInfo *Target = nullptr;
146 const TargetInfo *AuxTarget = nullptr;
147 FileManager &FileMgr;
148 SourceManager &SourceMgr;
149 std::unique_ptr<ScratchBuffer> ScratchBuf;
150 HeaderSearch &HeaderInfo;
151 ModuleLoader &TheModuleLoader;
152
153 /// External source of macros.
155
156 /// A BumpPtrAllocator object used to quickly allocate and release
157 /// objects internal to the Preprocessor.
158 llvm::BumpPtrAllocator BP;
159
160 /// Identifiers for builtin macros and other builtins.
161 IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
162 IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
163 IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
164 IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
165 IdentifierInfo *Ident__FILE_NAME__; // __FILE_NAME__
166 IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
167 IdentifierInfo *Ident__COUNTER__; // __COUNTER__
168 IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
169 IdentifierInfo *Ident__identifier; // __identifier
170 IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
171 IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
172 IdentifierInfo *Ident__has_feature; // __has_feature
173 IdentifierInfo *Ident__has_extension; // __has_extension
174 IdentifierInfo *Ident__has_builtin; // __has_builtin
175 IdentifierInfo *Ident__has_constexpr_builtin; // __has_constexpr_builtin
176 IdentifierInfo *Ident__has_attribute; // __has_attribute
177 IdentifierInfo *Ident__has_embed; // __has_embed
178 IdentifierInfo *Ident__has_include; // __has_include
179 IdentifierInfo *Ident__has_include_next; // __has_include_next
180 IdentifierInfo *Ident__has_warning; // __has_warning
181 IdentifierInfo *Ident__is_identifier; // __is_identifier
182 IdentifierInfo *Ident__building_module; // __building_module
183 IdentifierInfo *Ident__MODULE__; // __MODULE__
184 IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
185 IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
186 IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
187 IdentifierInfo *Ident__is_target_arch; // __is_target_arch
188 IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
189 IdentifierInfo *Ident__is_target_os; // __is_target_os
190 IdentifierInfo *Ident__is_target_environment; // __is_target_environment
191 IdentifierInfo *Ident__is_target_variant_os;
192 IdentifierInfo *Ident__is_target_variant_environment;
193 IdentifierInfo *Ident__FLT_EVAL_METHOD__; // __FLT_EVAL_METHOD
194
195 // Weak, only valid (and set) while InMacroArgs is true.
196 Token* ArgMacro;
197
198 SourceLocation DATELoc, TIMELoc;
199
200 // FEM_UnsetOnCommandLine means that an explicit evaluation method was
201 // not specified on the command line. The target is queried to set the
202 // default evaluation method.
203 LangOptions::FPEvalMethodKind CurrentFPEvalMethod =
205
206 // The most recent pragma location where the floating point evaluation
207 // method was modified. This is used to determine whether the
208 // 'pragma clang fp eval_method' was used whithin the current scope.
209 SourceLocation LastFPEvalPragmaLocation;
210
211 LangOptions::FPEvalMethodKind TUFPEvalMethod =
213
214 // Next __COUNTER__ value, starts at 0.
215 unsigned CounterValue = 0;
216
217 enum {
218 /// Maximum depth of \#includes.
219 MaxAllowedIncludeStackDepth = 200
220 };
221
222 // State that is set before the preprocessor begins.
223 bool KeepComments : 1;
224 bool KeepMacroComments : 1;
225 bool SuppressIncludeNotFoundError : 1;
226
227 // State that changes while the preprocessor runs:
228 bool InMacroArgs : 1; // True if parsing fn macro invocation args.
229
230 /// Whether the preprocessor owns the header search object.
231 bool OwnsHeaderSearch : 1;
232
233 /// True if macro expansion is disabled.
234 bool DisableMacroExpansion : 1;
235
236 /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
237 /// when parsing preprocessor directives.
238 bool MacroExpansionInDirectivesOverride : 1;
239
240 class ResetMacroExpansionHelper;
241
242 /// Whether we have already loaded macros from the external source.
243 mutable bool ReadMacrosFromExternalSource : 1;
244
245 /// True if pragmas are enabled.
246 bool PragmasEnabled : 1;
247
248 /// True if the current build action is a preprocessing action.
249 bool PreprocessedOutput : 1;
250
251 /// True if we are currently preprocessing a #if or #elif directive
252 bool ParsingIfOrElifDirective;
253
254 /// True if we are pre-expanding macro arguments.
255 bool InMacroArgPreExpansion;
256
257 /// Mapping/lookup information for all identifiers in
258 /// the program, including program keywords.
259 mutable IdentifierTable Identifiers;
260
261 /// This table contains all the selectors in the program.
262 ///
263 /// Unlike IdentifierTable above, this table *isn't* populated by the
264 /// preprocessor. It is declared/expanded here because its role/lifetime is
265 /// conceptually similar to the IdentifierTable. In addition, the current
266 /// control flow (in clang::ParseAST()), make it convenient to put here.
267 ///
268 /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
269 /// the lifetime of the preprocessor.
270 SelectorTable Selectors;
271
272 /// Information about builtins.
273 std::unique_ptr<Builtin::Context> BuiltinInfo;
274
275 /// Tracks all of the pragmas that the client registered
276 /// with this preprocessor.
277 std::unique_ptr<PragmaNamespace> PragmaHandlers;
278
279 /// Pragma handlers of the original source is stored here during the
280 /// parsing of a model file.
281 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
282
283 /// Tracks all of the comment handlers that the client registered
284 /// with this preprocessor.
285 std::vector<CommentHandler *> CommentHandlers;
286
287 /// Empty line handler.
288 EmptylineHandler *Emptyline = nullptr;
289
290 /// True to avoid tearing down the lexer etc on EOF
291 bool IncrementalProcessing = false;
292
293public:
294 /// The kind of translation unit we are processing.
296
297 /// Returns a pointer into the given file's buffer that's guaranteed
298 /// to be between tokens. The returned pointer is always before \p Start.
299 /// The maximum distance betweenthe returned pointer and \p Start is
300 /// limited by a constant value, but also an implementation detail.
301 /// If no such check point exists, \c nullptr is returned.
302 const char *getCheckPoint(FileID FID, const char *Start) const;
303
304private:
305 /// The code-completion handler.
306 CodeCompletionHandler *CodeComplete = nullptr;
307
308 /// The file that we're performing code-completion for, if any.
309 const FileEntry *CodeCompletionFile = nullptr;
310
311 /// The offset in file for the code-completion point.
312 unsigned CodeCompletionOffset = 0;
313
314 /// The location for the code-completion point. This gets instantiated
315 /// when the CodeCompletionFile gets \#include'ed for preprocessing.
316 SourceLocation CodeCompletionLoc;
317
318 /// The start location for the file of the code-completion point.
319 ///
320 /// This gets instantiated when the CodeCompletionFile gets \#include'ed
321 /// for preprocessing.
322 SourceLocation CodeCompletionFileLoc;
323
324 /// The source location of the \c import contextual keyword we just
325 /// lexed, if any.
326 SourceLocation ModuleImportLoc;
327
328 /// The import path for named module that we're currently processing.
330
331 llvm::DenseMap<FileID, SmallVector<const char *>> CheckPoints;
332 unsigned CheckPointCounter = 0;
333
334 /// Whether the import is an `@import` or a standard c++ modules import.
335 bool IsAtImport = false;
336
337 /// Whether the last token we lexed was an '@'.
338 bool LastTokenWasAt = false;
339
340 /// A position within a C++20 import-seq.
341 class StdCXXImportSeq {
342 public:
343 enum State : int {
344 // Positive values represent a number of unclosed brackets.
345 AtTopLevel = 0,
346 AfterTopLevelTokenSeq = -1,
347 AfterExport = -2,
348 AfterImportSeq = -3,
349 };
350
351 StdCXXImportSeq(State S) : S(S) {}
352
353 /// Saw any kind of open bracket.
354 void handleOpenBracket() {
355 S = static_cast<State>(std::max<int>(S, 0) + 1);
356 }
357 /// Saw any kind of close bracket other than '}'.
358 void handleCloseBracket() {
359 S = static_cast<State>(std::max<int>(S, 1) - 1);
360 }
361 /// Saw a close brace.
362 void handleCloseBrace() {
363 handleCloseBracket();
364 if (S == AtTopLevel && !AfterHeaderName)
365 S = AfterTopLevelTokenSeq;
366 }
367 /// Saw a semicolon.
368 void handleSemi() {
369 if (atTopLevel()) {
370 S = AfterTopLevelTokenSeq;
371 AfterHeaderName = false;
372 }
373 }
374
375 /// Saw an 'export' identifier.
376 void handleExport() {
377 if (S == AfterTopLevelTokenSeq)
378 S = AfterExport;
379 else if (S <= 0)
380 S = AtTopLevel;
381 }
382 /// Saw an 'import' identifier.
383 void handleImport() {
384 if (S == AfterTopLevelTokenSeq || S == AfterExport)
385 S = AfterImportSeq;
386 else if (S <= 0)
387 S = AtTopLevel;
388 }
389
390 /// Saw a 'header-name' token; do not recognize any more 'import' tokens
391 /// until we reach a top-level semicolon.
392 void handleHeaderName() {
393 if (S == AfterImportSeq)
394 AfterHeaderName = true;
395 handleMisc();
396 }
397
398 /// Saw any other token.
399 void handleMisc() {
400 if (S <= 0)
401 S = AtTopLevel;
402 }
403
404 bool atTopLevel() { return S <= 0; }
405 bool afterImportSeq() { return S == AfterImportSeq; }
406 bool afterTopLevelSeq() { return S == AfterTopLevelTokenSeq; }
407
408 private:
409 State S;
410 /// Whether we're in the pp-import-suffix following the header-name in a
411 /// pp-import. If so, a close-brace is not sufficient to end the
412 /// top-level-token-seq of an import-seq.
413 bool AfterHeaderName = false;
414 };
415
416 /// Our current position within a C++20 import-seq.
417 StdCXXImportSeq StdCXXImportSeqState = StdCXXImportSeq::AfterTopLevelTokenSeq;
418
419 /// Track whether we are in a Global Module Fragment
420 class TrackGMF {
421 public:
422 enum GMFState : int {
423 GMFActive = 1,
424 MaybeGMF = 0,
425 BeforeGMFIntroducer = -1,
426 GMFAbsentOrEnded = -2,
427 };
428
429 TrackGMF(GMFState S) : S(S) {}
430
431 /// Saw a semicolon.
432 void handleSemi() {
433 // If it is immediately after the first instance of the module keyword,
434 // then that introduces the GMF.
435 if (S == MaybeGMF)
436 S = GMFActive;
437 }
438
439 /// Saw an 'export' identifier.
440 void handleExport() {
441 // The presence of an 'export' keyword always ends or excludes a GMF.
442 S = GMFAbsentOrEnded;
443 }
444
445 /// Saw an 'import' identifier.
446 void handleImport(bool AfterTopLevelTokenSeq) {
447 // If we see this before any 'module' kw, then we have no GMF.
448 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
449 S = GMFAbsentOrEnded;
450 }
451
452 /// Saw a 'module' identifier.
453 void handleModule(bool AfterTopLevelTokenSeq) {
454 // This was the first module identifier and not preceded by any token
455 // that would exclude a GMF. It could begin a GMF, but only if directly
456 // followed by a semicolon.
457 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
458 S = MaybeGMF;
459 else
460 S = GMFAbsentOrEnded;
461 }
462
463 /// Saw any other token.
464 void handleMisc() {
465 // We saw something other than ; after the 'module' kw, so not a GMF.
466 if (S == MaybeGMF)
467 S = GMFAbsentOrEnded;
468 }
469
470 bool inGMF() { return S == GMFActive; }
471
472 private:
473 /// Track the transitions into and out of a Global Module Fragment,
474 /// if one is present.
475 GMFState S;
476 };
477
478 TrackGMF TrackGMFState = TrackGMF::BeforeGMFIntroducer;
479
480 /// Track the status of the c++20 module decl.
481 ///
482 /// module-declaration:
483 /// 'export'[opt] 'module' module-name module-partition[opt]
484 /// attribute-specifier-seq[opt] ';'
485 ///
486 /// module-name:
487 /// module-name-qualifier[opt] identifier
488 ///
489 /// module-partition:
490 /// ':' module-name-qualifier[opt] identifier
491 ///
492 /// module-name-qualifier:
493 /// identifier '.'
494 /// module-name-qualifier identifier '.'
495 ///
496 /// Transition state:
497 ///
498 /// NotAModuleDecl --- export ---> FoundExport
499 /// NotAModuleDecl --- module ---> ImplementationCandidate
500 /// FoundExport --- module ---> InterfaceCandidate
501 /// ImplementationCandidate --- Identifier ---> ImplementationCandidate
502 /// ImplementationCandidate --- period ---> ImplementationCandidate
503 /// ImplementationCandidate --- colon ---> ImplementationCandidate
504 /// InterfaceCandidate --- Identifier ---> InterfaceCandidate
505 /// InterfaceCandidate --- period ---> InterfaceCandidate
506 /// InterfaceCandidate --- colon ---> InterfaceCandidate
507 /// ImplementationCandidate --- Semi ---> NamedModuleImplementation
508 /// NamedModuleInterface --- Semi ---> NamedModuleInterface
509 /// NamedModuleImplementation --- Anything ---> NamedModuleImplementation
510 /// NamedModuleInterface --- Anything ---> NamedModuleInterface
511 ///
512 /// FIXME: We haven't handle attribute-specifier-seq here. It may not be bad
513 /// soon since we don't support any module attributes yet.
514 class ModuleDeclSeq {
515 enum ModuleDeclState : int {
516 NotAModuleDecl,
517 FoundExport,
518 InterfaceCandidate,
519 ImplementationCandidate,
520 NamedModuleInterface,
521 NamedModuleImplementation,
522 };
523
524 public:
525 ModuleDeclSeq() = default;
526
527 void handleExport() {
528 if (State == NotAModuleDecl)
529 State = FoundExport;
530 else if (!isNamedModule())
531 reset();
532 }
533
534 void handleModule() {
535 if (State == FoundExport)
536 State = InterfaceCandidate;
537 else if (State == NotAModuleDecl)
538 State = ImplementationCandidate;
539 else if (!isNamedModule())
540 reset();
541 }
542
543 void handleIdentifier(IdentifierInfo *Identifier) {
544 if (isModuleCandidate() && Identifier)
545 Name += Identifier->getName().str();
546 else if (!isNamedModule())
547 reset();
548 }
549
550 void handleColon() {
551 if (isModuleCandidate())
552 Name += ":";
553 else if (!isNamedModule())
554 reset();
555 }
556
557 void handlePeriod() {
558 if (isModuleCandidate())
559 Name += ".";
560 else if (!isNamedModule())
561 reset();
562 }
563
564 void handleSemi() {
565 if (!Name.empty() && isModuleCandidate()) {
566 if (State == InterfaceCandidate)
567 State = NamedModuleInterface;
568 else if (State == ImplementationCandidate)
569 State = NamedModuleImplementation;
570 else
571 llvm_unreachable("Unimaged ModuleDeclState.");
572 } else if (!isNamedModule())
573 reset();
574 }
575
576 void handleMisc() {
577 if (!isNamedModule())
578 reset();
579 }
580
581 bool isModuleCandidate() const {
582 return State == InterfaceCandidate || State == ImplementationCandidate;
583 }
584
585 bool isNamedModule() const {
586 return State == NamedModuleInterface ||
587 State == NamedModuleImplementation;
588 }
589
590 bool isNamedInterface() const { return State == NamedModuleInterface; }
591
592 bool isImplementationUnit() const {
593 return State == NamedModuleImplementation && !getName().contains(':');
594 }
595
596 StringRef getName() const {
597 assert(isNamedModule() && "Can't get name from a non named module");
598 return Name;
599 }
600
601 StringRef getPrimaryName() const {
602 assert(isNamedModule() && "Can't get name from a non named module");
603 return getName().split(':').first;
604 }
605
606 void reset() {
607 Name.clear();
608 State = NotAModuleDecl;
609 }
610
611 private:
612 ModuleDeclState State = NotAModuleDecl;
613 std::string Name;
614 };
615
616 ModuleDeclSeq ModuleDeclState;
617
618 /// Whether the module import expects an identifier next. Otherwise,
619 /// it expects a '.' or ';'.
620 bool ModuleImportExpectsIdentifier = false;
621
622 /// The identifier and source location of the currently-active
623 /// \#pragma clang arc_cf_code_audited begin.
624 std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
625
626 /// The source location of the currently-active
627 /// \#pragma clang assume_nonnull begin.
628 SourceLocation PragmaAssumeNonNullLoc;
629
630 /// Set only for preambles which end with an active
631 /// \#pragma clang assume_nonnull begin.
632 ///
633 /// When the preamble is loaded into the main file,
634 /// `PragmaAssumeNonNullLoc` will be set to this to
635 /// replay the unterminated assume_nonnull.
636 SourceLocation PreambleRecordedPragmaAssumeNonNullLoc;
637
638 /// True if we hit the code-completion point.
639 bool CodeCompletionReached = false;
640
641 /// The code completion token containing the information
642 /// on the stem that is to be code completed.
643 IdentifierInfo *CodeCompletionII = nullptr;
644
645 /// Range for the code completion token.
646 SourceRange CodeCompletionTokenRange;
647
648 /// The directory that the main file should be considered to occupy,
649 /// if it does not correspond to a real file (as happens when building a
650 /// module).
651 OptionalDirectoryEntryRef MainFileDir;
652
653 /// The number of bytes that we will initially skip when entering the
654 /// main file, along with a flag that indicates whether skipping this number
655 /// of bytes will place the lexer at the start of a line.
656 ///
657 /// This is used when loading a precompiled preamble.
658 std::pair<int, bool> SkipMainFilePreamble;
659
660 /// Whether we hit an error due to reaching max allowed include depth. Allows
661 /// to avoid hitting the same error over and over again.
662 bool HasReachedMaxIncludeDepth = false;
663
664 /// The number of currently-active calls to Lex.
665 ///
666 /// Lex is reentrant, and asking for an (end-of-phase-4) token can often
667 /// require asking for multiple additional tokens. This counter makes it
668 /// possible for Lex to detect whether it's producing a token for the end
669 /// of phase 4 of translation or for some other situation.
670 unsigned LexLevel = 0;
671
672 /// The number of (LexLevel 0) preprocessor tokens.
673 unsigned TokenCount = 0;
674
675 /// Preprocess every token regardless of LexLevel.
676 bool PreprocessToken = false;
677
678 /// The maximum number of (LexLevel 0) tokens before issuing a -Wmax-tokens
679 /// warning, or zero for unlimited.
680 unsigned MaxTokens = 0;
681 SourceLocation MaxTokensOverrideLoc;
682
683public:
690
696 ElseLoc(ElseLoc) {}
697 };
698
700
701private:
702 friend class ASTReader;
703 friend class MacroArgs;
704
705 class PreambleConditionalStackStore {
706 enum State {
707 Off = 0,
708 Recording = 1,
709 Replaying = 2,
710 };
711
712 public:
713 PreambleConditionalStackStore() = default;
714
715 void startRecording() { ConditionalStackState = Recording; }
716 void startReplaying() { ConditionalStackState = Replaying; }
717 bool isRecording() const { return ConditionalStackState == Recording; }
718 bool isReplaying() const { return ConditionalStackState == Replaying; }
719
720 ArrayRef<PPConditionalInfo> getStack() const {
721 return ConditionalStack;
722 }
723
724 void doneReplaying() {
725 ConditionalStack.clear();
726 ConditionalStackState = Off;
727 }
728
729 void setStack(ArrayRef<PPConditionalInfo> s) {
730 if (!isRecording() && !isReplaying())
731 return;
732 ConditionalStack.clear();
733 ConditionalStack.append(s.begin(), s.end());
734 }
735
736 bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
737
738 bool reachedEOFWhileSkipping() const { return SkipInfo.has_value(); }
739
740 void clearSkipInfo() { SkipInfo.reset(); }
741
742 std::optional<PreambleSkipInfo> SkipInfo;
743
744 private:
745 SmallVector<PPConditionalInfo, 4> ConditionalStack;
746 State ConditionalStackState = Off;
747 } PreambleConditionalStack;
748
749 /// The current top of the stack that we're lexing from if
750 /// not expanding a macro and we are lexing directly from source code.
751 ///
752 /// Only one of CurLexer, or CurTokenLexer will be non-null.
753 std::unique_ptr<Lexer> CurLexer;
754
755 /// The current top of the stack that we're lexing from
756 /// if not expanding a macro.
757 ///
758 /// This is an alias for CurLexer.
759 PreprocessorLexer *CurPPLexer = nullptr;
760
761 /// Used to find the current FileEntry, if CurLexer is non-null
762 /// and if applicable.
763 ///
764 /// This allows us to implement \#include_next and find directory-specific
765 /// properties.
766 ConstSearchDirIterator CurDirLookup = nullptr;
767
768 /// The current macro we are expanding, if we are expanding a macro.
769 ///
770 /// One of CurLexer and CurTokenLexer must be null.
771 std::unique_ptr<TokenLexer> CurTokenLexer;
772
773 /// The kind of lexer we're currently working with.
774 typedef bool (*LexerCallback)(Preprocessor &, Token &);
775 LexerCallback CurLexerCallback = &CLK_Lexer;
776
777 /// If the current lexer is for a submodule that is being built, this
778 /// is that submodule.
779 Module *CurLexerSubmodule = nullptr;
780
781 /// Keeps track of the stack of files currently
782 /// \#included, and macros currently being expanded from, not counting
783 /// CurLexer/CurTokenLexer.
784 struct IncludeStackInfo {
785 LexerCallback CurLexerCallback;
786 Module *TheSubmodule;
787 std::unique_ptr<Lexer> TheLexer;
788 PreprocessorLexer *ThePPLexer;
789 std::unique_ptr<TokenLexer> TheTokenLexer;
790 ConstSearchDirIterator TheDirLookup;
791
792 // The following constructors are completely useless copies of the default
793 // versions, only needed to pacify MSVC.
794 IncludeStackInfo(LexerCallback CurLexerCallback, Module *TheSubmodule,
795 std::unique_ptr<Lexer> &&TheLexer,
796 PreprocessorLexer *ThePPLexer,
797 std::unique_ptr<TokenLexer> &&TheTokenLexer,
798 ConstSearchDirIterator TheDirLookup)
799 : CurLexerCallback(std::move(CurLexerCallback)),
800 TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
801 ThePPLexer(std::move(ThePPLexer)),
802 TheTokenLexer(std::move(TheTokenLexer)),
803 TheDirLookup(std::move(TheDirLookup)) {}
804 };
805 std::vector<IncludeStackInfo> IncludeMacroStack;
806
807 /// Actions invoked when some preprocessor activity is
808 /// encountered (e.g. a file is \#included, etc).
809 std::unique_ptr<PPCallbacks> Callbacks;
810
811 struct MacroExpandsInfo {
812 Token Tok;
813 MacroDefinition MD;
814 SourceRange Range;
815
816 MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
817 : Tok(Tok), MD(MD), Range(Range) {}
818 };
819 SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
820
821 /// Information about a name that has been used to define a module macro.
822 struct ModuleMacroInfo {
823 /// The most recent macro directive for this identifier.
824 MacroDirective *MD;
825
826 /// The active module macros for this identifier.
827 llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
828
829 /// The generation number at which we last updated ActiveModuleMacros.
830 /// \see Preprocessor::VisibleModules.
831 unsigned ActiveModuleMacrosGeneration = 0;
832
833 /// Whether this macro name is ambiguous.
834 bool IsAmbiguous = false;
835
836 /// The module macros that are overridden by this macro.
837 llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
838
839 ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
840 };
841
842 /// The state of a macro for an identifier.
843 class MacroState {
844 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
845
846 ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
847 const IdentifierInfo *II) const {
848 if (II->isOutOfDate())
849 PP.updateOutOfDateIdentifier(*II);
850 // FIXME: Find a spare bit on IdentifierInfo and store a
851 // HasModuleMacros flag.
852 if (!II->hasMacroDefinition() ||
853 (!PP.getLangOpts().Modules &&
854 !PP.getLangOpts().ModulesLocalVisibility) ||
855 !PP.CurSubmoduleState->VisibleModules.getGeneration())
856 return nullptr;
857
858 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
859 if (!Info) {
860 Info = new (PP.getPreprocessorAllocator())
861 ModuleMacroInfo(State.get<MacroDirective *>());
862 State = Info;
863 }
864
865 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
866 Info->ActiveModuleMacrosGeneration)
867 PP.updateModuleMacroInfo(II, *Info);
868 return Info;
869 }
870
871 public:
872 MacroState() : MacroState(nullptr) {}
873 MacroState(MacroDirective *MD) : State(MD) {}
874
875 MacroState(MacroState &&O) noexcept : State(O.State) {
876 O.State = (MacroDirective *)nullptr;
877 }
878
879 MacroState &operator=(MacroState &&O) noexcept {
880 auto S = O.State;
881 O.State = (MacroDirective *)nullptr;
882 State = S;
883 return *this;
884 }
885
886 ~MacroState() {
887 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
888 Info->~ModuleMacroInfo();
889 }
890
891 MacroDirective *getLatest() const {
892 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
893 return Info->MD;
894 return State.get<MacroDirective*>();
895 }
896
897 void setLatest(MacroDirective *MD) {
898 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
899 Info->MD = MD;
900 else
901 State = MD;
902 }
903
904 bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
905 auto *Info = getModuleInfo(PP, II);
906 return Info ? Info->IsAmbiguous : false;
907 }
908
909 ArrayRef<ModuleMacro *>
910 getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
911 if (auto *Info = getModuleInfo(PP, II))
912 return Info->ActiveModuleMacros;
913 return std::nullopt;
914 }
915
916 MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
917 SourceManager &SourceMgr) const {
918 // FIXME: Incorporate module macros into the result of this.
919 if (auto *Latest = getLatest())
920 return Latest->findDirectiveAtLoc(Loc, SourceMgr);
921 return {};
922 }
923
924 void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
925 if (auto *Info = getModuleInfo(PP, II)) {
926 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
927 Info->ActiveModuleMacros.begin(),
928 Info->ActiveModuleMacros.end());
929 Info->ActiveModuleMacros.clear();
930 Info->IsAmbiguous = false;
931 }
932 }
933
934 ArrayRef<ModuleMacro*> getOverriddenMacros() const {
935 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
936 return Info->OverriddenMacros;
937 return std::nullopt;
938 }
939
940 void setOverriddenMacros(Preprocessor &PP,
941 ArrayRef<ModuleMacro *> Overrides) {
942 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
943 if (!Info) {
944 if (Overrides.empty())
945 return;
946 Info = new (PP.getPreprocessorAllocator())
947 ModuleMacroInfo(State.get<MacroDirective *>());
948 State = Info;
949 }
950 Info->OverriddenMacros.clear();
951 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
952 Overrides.begin(), Overrides.end());
953 Info->ActiveModuleMacrosGeneration = 0;
954 }
955 };
956
957 /// For each IdentifierInfo that was associated with a macro, we
958 /// keep a mapping to the history of all macro definitions and #undefs in
959 /// the reverse order (the latest one is in the head of the list).
960 ///
961 /// This mapping lives within the \p CurSubmoduleState.
962 using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
963
964 struct SubmoduleState;
965
966 /// Information about a submodule that we're currently building.
967 struct BuildingSubmoduleInfo {
968 /// The module that we are building.
969 Module *M;
970
971 /// The location at which the module was included.
972 SourceLocation ImportLoc;
973
974 /// Whether we entered this submodule via a pragma.
975 bool IsPragma;
976
977 /// The previous SubmoduleState.
978 SubmoduleState *OuterSubmoduleState;
979
980 /// The number of pending module macro names when we started building this.
981 unsigned OuterPendingModuleMacroNames;
982
983 BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
984 SubmoduleState *OuterSubmoduleState,
985 unsigned OuterPendingModuleMacroNames)
986 : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
987 OuterSubmoduleState(OuterSubmoduleState),
988 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
989 };
990 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
991
992 /// Information about a submodule's preprocessor state.
993 struct SubmoduleState {
994 /// The macros for the submodule.
995 MacroMap Macros;
996
997 /// The set of modules that are visible within the submodule.
998 VisibleModuleSet VisibleModules;
999
1000 // FIXME: CounterValue?
1001 // FIXME: PragmaPushMacroInfo?
1002 };
1003 std::map<Module *, SubmoduleState> Submodules;
1004
1005 /// The preprocessor state for preprocessing outside of any submodule.
1006 SubmoduleState NullSubmoduleState;
1007
1008 /// The current submodule state. Will be \p NullSubmoduleState if we're not
1009 /// in a submodule.
1010 SubmoduleState *CurSubmoduleState;
1011
1012 /// The files that have been included.
1013 IncludedFilesSet IncludedFiles;
1014
1015 /// The set of top-level modules that affected preprocessing, but were not
1016 /// imported.
1017 llvm::SmallSetVector<Module *, 2> AffectingClangModules;
1018
1019 /// The set of known macros exported from modules.
1020 llvm::FoldingSet<ModuleMacro> ModuleMacros;
1021
1022 /// The names of potential module macros that we've not yet processed.
1023 llvm::SmallVector<IdentifierInfo *, 32> PendingModuleMacroNames;
1024
1025 /// The list of module macros, for each identifier, that are not overridden by
1026 /// any other module macro.
1027 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
1028 LeafModuleMacros;
1029
1030 /// Macros that we want to warn because they are not used at the end
1031 /// of the translation unit.
1032 ///
1033 /// We store just their SourceLocations instead of
1034 /// something like MacroInfo*. The benefit of this is that when we are
1035 /// deserializing from PCH, we don't need to deserialize identifier & macros
1036 /// just so that we can report that they are unused, we just warn using
1037 /// the SourceLocations of this set (that will be filled by the ASTReader).
1038 using WarnUnusedMacroLocsTy = llvm::SmallDenseSet<SourceLocation, 32>;
1039 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
1040
1041 /// This is a pair of an optional message and source location used for pragmas
1042 /// that annotate macros like pragma clang restrict_expansion and pragma clang
1043 /// deprecated. This pair stores the optional message and the location of the
1044 /// annotation pragma for use producing diagnostics and notes.
1045 using MsgLocationPair = std::pair<std::string, SourceLocation>;
1046
1047 struct MacroAnnotationInfo {
1048 SourceLocation Location;
1049 std::string Message;
1050 };
1051
1052 struct MacroAnnotations {
1053 std::optional<MacroAnnotationInfo> DeprecationInfo;
1054 std::optional<MacroAnnotationInfo> RestrictExpansionInfo;
1055 std::optional<SourceLocation> FinalAnnotationLoc;
1056
1057 static MacroAnnotations makeDeprecation(SourceLocation Loc,
1058 std::string Msg) {
1059 return MacroAnnotations{MacroAnnotationInfo{Loc, std::move(Msg)},
1060 std::nullopt, std::nullopt};
1061 }
1062
1063 static MacroAnnotations makeRestrictExpansion(SourceLocation Loc,
1064 std::string Msg) {
1065 return MacroAnnotations{
1066 std::nullopt, MacroAnnotationInfo{Loc, std::move(Msg)}, std::nullopt};
1067 }
1068
1069 static MacroAnnotations makeFinal(SourceLocation Loc) {
1070 return MacroAnnotations{std::nullopt, std::nullopt, Loc};
1071 }
1072 };
1073
1074 /// Warning information for macro annotations.
1075 llvm::DenseMap<const IdentifierInfo *, MacroAnnotations> AnnotationInfos;
1076
1077 /// A "freelist" of MacroArg objects that can be
1078 /// reused for quick allocation.
1079 MacroArgs *MacroArgCache = nullptr;
1080
1081 /// For each IdentifierInfo used in a \#pragma push_macro directive,
1082 /// we keep a MacroInfo stack used to restore the previous macro value.
1083 llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
1084 PragmaPushMacroInfo;
1085
1086 // Various statistics we track for performance analysis.
1087 unsigned NumDirectives = 0;
1088 unsigned NumDefined = 0;
1089 unsigned NumUndefined = 0;
1090 unsigned NumPragma = 0;
1091 unsigned NumIf = 0;
1092 unsigned NumElse = 0;
1093 unsigned NumEndif = 0;
1094 unsigned NumEnteredSourceFiles = 0;
1095 unsigned MaxIncludeStackDepth = 0;
1096 unsigned NumMacroExpanded = 0;
1097 unsigned NumFnMacroExpanded = 0;
1098 unsigned NumBuiltinMacroExpanded = 0;
1099 unsigned NumFastMacroExpanded = 0;
1100 unsigned NumTokenPaste = 0;
1101 unsigned NumFastTokenPaste = 0;
1102 unsigned NumSkipped = 0;
1103
1104 /// The predefined macros that preprocessor should use from the
1105 /// command line etc.
1106 std::string Predefines;
1107
1108 /// The file ID for the preprocessor predefines.
1109 FileID PredefinesFileID;
1110
1111 /// The file ID for the PCH through header.
1112 FileID PCHThroughHeaderFileID;
1113
1114 /// Whether tokens are being skipped until a #pragma hdrstop is seen.
1115 bool SkippingUntilPragmaHdrStop = false;
1116
1117 /// Whether tokens are being skipped until the through header is seen.
1118 bool SkippingUntilPCHThroughHeader = false;
1119
1120 /// \{
1121 /// Cache of macro expanders to reduce malloc traffic.
1122 enum { TokenLexerCacheSize = 8 };
1123 unsigned NumCachedTokenLexers;
1124 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
1125 /// \}
1126
1127 /// Keeps macro expanded tokens for TokenLexers.
1128 //
1129 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
1130 /// going to lex in the cache and when it finishes the tokens are removed
1131 /// from the end of the cache.
1132 SmallVector<Token, 16> MacroExpandedTokens;
1133 std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
1134
1135 /// A record of the macro definitions and expansions that
1136 /// occurred during preprocessing.
1137 ///
1138 /// This is an optional side structure that can be enabled with
1139 /// \c createPreprocessingRecord() prior to preprocessing.
1140 PreprocessingRecord *Record = nullptr;
1141
1142 /// Cached tokens state.
1143 using CachedTokensTy = SmallVector<Token, 1>;
1144
1145 /// Cached tokens are stored here when we do backtracking or
1146 /// lookahead. They are "lexed" by the CachingLex() method.
1147 CachedTokensTy CachedTokens;
1148
1149 /// The position of the cached token that CachingLex() should
1150 /// "lex" next.
1151 ///
1152 /// If it points beyond the CachedTokens vector, it means that a normal
1153 /// Lex() should be invoked.
1154 CachedTokensTy::size_type CachedLexPos = 0;
1155
1156 /// Stack of backtrack positions, allowing nested backtracks.
1157 ///
1158 /// The EnableBacktrackAtThisPos() method pushes a position to
1159 /// indicate where CachedLexPos should be set when the BackTrack() method is
1160 /// invoked (at which point the last position is popped).
1161 std::vector<CachedTokensTy::size_type> BacktrackPositions;
1162
1163 /// Stack of cached tokens/initial number of cached tokens pairs, allowing
1164 /// nested unannotated backtracks.
1165 std::vector<std::pair<CachedTokensTy, CachedTokensTy::size_type>>
1166 UnannotatedBacktrackTokens;
1167
1168 /// True if \p Preprocessor::SkipExcludedConditionalBlock() is running.
1169 /// This is used to guard against calling this function recursively.
1170 ///
1171 /// See comments at the use-site for more context about why it is needed.
1172 bool SkippingExcludedConditionalBlock = false;
1173
1174 /// Keeps track of skipped range mappings that were recorded while skipping
1175 /// excluded conditional directives. It maps the source buffer pointer at
1176 /// the beginning of a skipped block, to the number of bytes that should be
1177 /// skipped.
1178 llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges;
1179
1180 void updateOutOfDateIdentifier(const IdentifierInfo &II) const;
1181
1182public:
1183 Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
1184 DiagnosticsEngine &diags, const LangOptions &LangOpts,
1185 SourceManager &SM, HeaderSearch &Headers,
1186 ModuleLoader &TheModuleLoader,
1187 IdentifierInfoLookup *IILookup = nullptr,
1188 bool OwnsHeaderSearch = false,
1190
1191 ~Preprocessor();
1192
1193 /// Initialize the preprocessor using information about the target.
1194 ///
1195 /// \param Target is owned by the caller and must remain valid for the
1196 /// lifetime of the preprocessor.
1197 /// \param AuxTarget is owned by the caller and must remain valid for
1198 /// the lifetime of the preprocessor.
1199 void Initialize(const TargetInfo &Target,
1200 const TargetInfo *AuxTarget = nullptr);
1201
1202 /// Initialize the preprocessor to parse a model file
1203 ///
1204 /// To parse model files the preprocessor of the original source is reused to
1205 /// preserver the identifier table. However to avoid some duplicate
1206 /// information in the preprocessor some cleanup is needed before it is used
1207 /// to parse model files. This method does that cleanup.
1209
1210 /// Cleanup after model file parsing
1211 void FinalizeForModelFile();
1212
1213 /// Retrieve the preprocessor options used to initialize this
1214 /// preprocessor.
1215 PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
1216
1217 DiagnosticsEngine &getDiagnostics() const { return *Diags; }
1219
1220 const LangOptions &getLangOpts() const { return LangOpts; }
1221 const TargetInfo &getTargetInfo() const { return *Target; }
1222 const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
1223 FileManager &getFileManager() const { return FileMgr; }
1224 SourceManager &getSourceManager() const { return SourceMgr; }
1225 HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
1226
1227 IdentifierTable &getIdentifierTable() { return Identifiers; }
1228 const IdentifierTable &getIdentifierTable() const { return Identifiers; }
1229 SelectorTable &getSelectorTable() { return Selectors; }
1230 Builtin::Context &getBuiltinInfo() { return *BuiltinInfo; }
1231 llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
1232
1234 ExternalSource = Source;
1235 }
1236
1238 return ExternalSource;
1239 }
1240
1241 /// Retrieve the module loader associated with this preprocessor.
1242 ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
1243
1245 return TheModuleLoader.HadFatalFailure;
1246 }
1247
1248 /// Retrieve the number of Directives that have been processed by the
1249 /// Preprocessor.
1250 unsigned getNumDirectives() const {
1251 return NumDirectives;
1252 }
1253
1254 /// True if we are currently preprocessing a #if or #elif directive
1256 return ParsingIfOrElifDirective;
1257 }
1258
1259 /// Control whether the preprocessor retains comments in output.
1260 void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
1261 this->KeepComments = KeepComments | KeepMacroComments;
1262 this->KeepMacroComments = KeepMacroComments;
1263 }
1264
1265 bool getCommentRetentionState() const { return KeepComments; }
1266
1267 void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
1268 bool getPragmasEnabled() const { return PragmasEnabled; }
1269
1271 SuppressIncludeNotFoundError = Suppress;
1272 }
1273
1275 return SuppressIncludeNotFoundError;
1276 }
1277
1278 /// Sets whether the preprocessor is responsible for producing output or if
1279 /// it is producing tokens to be consumed by Parse and Sema.
1280 void setPreprocessedOutput(bool IsPreprocessedOutput) {
1281 PreprocessedOutput = IsPreprocessedOutput;
1282 }
1283
1284 /// Returns true if the preprocessor is responsible for generating output,
1285 /// false if it is producing tokens to be consumed by Parse and Sema.
1286 bool isPreprocessedOutput() const { return PreprocessedOutput; }
1287
1288 /// Return true if we are lexing directly from the specified lexer.
1289 bool isCurrentLexer(const PreprocessorLexer *L) const {
1290 return CurPPLexer == L;
1291 }
1292
1293 /// Return the current lexer being lexed from.
1294 ///
1295 /// Note that this ignores any potentially active macro expansions and _Pragma
1296 /// expansions going on at the time.
1297 PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
1298
1299 /// Return the current file lexer being lexed from.
1300 ///
1301 /// Note that this ignores any potentially active macro expansions and _Pragma
1302 /// expansions going on at the time.
1304
1305 /// Return the submodule owning the file being lexed. This may not be
1306 /// the current module if we have changed modules since entering the file.
1307 Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
1308
1309 /// Returns the FileID for the preprocessor predefines.
1310 FileID getPredefinesFileID() const { return PredefinesFileID; }
1311
1312 /// \{
1313 /// Accessors for preprocessor callbacks.
1314 ///
1315 /// Note that this class takes ownership of any PPCallbacks object given to
1316 /// it.
1317 PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
1318 void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
1319 if (Callbacks)
1320 C = std::make_unique<PPChainedCallbacks>(std::move(C),
1321 std::move(Callbacks));
1322 Callbacks = std::move(C);
1323 }
1324 /// \}
1325
1326 /// Get the number of tokens processed so far.
1327 unsigned getTokenCount() const { return TokenCount; }
1328
1329 /// Get the max number of tokens before issuing a -Wmax-tokens warning.
1330 unsigned getMaxTokens() const { return MaxTokens; }
1331
1333 MaxTokens = Value;
1334 MaxTokensOverrideLoc = Loc;
1335 };
1336
1337 SourceLocation getMaxTokensOverrideLoc() const { return MaxTokensOverrideLoc; }
1338
1339 /// Register a function that would be called on each token in the final
1340 /// expanded token stream.
1341 /// This also reports annotation tokens produced by the parser.
1342 void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
1343 OnToken = std::move(F);
1344 }
1345
1346 void setPreprocessToken(bool Preprocess) { PreprocessToken = Preprocess; }
1347
1348 bool isMacroDefined(StringRef Id) {
1349 return isMacroDefined(&Identifiers.get(Id));
1350 }
1352 return II->hasMacroDefinition() &&
1353 (!getLangOpts().Modules || (bool)getMacroDefinition(II));
1354 }
1355
1356 /// Determine whether II is defined as a macro within the module M,
1357 /// if that is a module that we've already preprocessed. Does not check for
1358 /// macros imported into M.
1360 if (!II->hasMacroDefinition())
1361 return false;
1362 auto I = Submodules.find(M);
1363 if (I == Submodules.end())
1364 return false;
1365 auto J = I->second.Macros.find(II);
1366 if (J == I->second.Macros.end())
1367 return false;
1368 auto *MD = J->second.getLatest();
1369 return MD && MD->isDefined();
1370 }
1371
1373 if (!II->hasMacroDefinition())
1374 return {};
1375
1376 MacroState &S = CurSubmoduleState->Macros[II];
1377 auto *MD = S.getLatest();
1378 while (isa_and_nonnull<VisibilityMacroDirective>(MD))
1379 MD = MD->getPrevious();
1380 return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
1381 S.getActiveModuleMacros(*this, II),
1382 S.isAmbiguous(*this, II));
1383 }
1384
1387 if (!II->hadMacroDefinition())
1388 return {};
1389
1390 MacroState &S = CurSubmoduleState->Macros[II];
1392 if (auto *MD = S.getLatest())
1393 DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
1394 // FIXME: Compute the set of active module macros at the specified location.
1395 return MacroDefinition(DI.getDirective(),
1396 S.getActiveModuleMacros(*this, II),
1397 S.isAmbiguous(*this, II));
1398 }
1399
1400 /// Given an identifier, return its latest non-imported MacroDirective
1401 /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
1403 if (!II->hasMacroDefinition())
1404 return nullptr;
1405
1406 auto *MD = getLocalMacroDirectiveHistory(II);
1407 if (!MD || MD->getDefinition().isUndefined())
1408 return nullptr;
1409
1410 return MD;
1411 }
1412
1413 const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
1414 return const_cast<Preprocessor*>(this)->getMacroInfo(II);
1415 }
1416
1418 if (!II->hasMacroDefinition())
1419 return nullptr;
1420 if (auto MD = getMacroDefinition(II))
1421 return MD.getMacroInfo();
1422 return nullptr;
1423 }
1424
1425 /// Given an identifier, return the latest non-imported macro
1426 /// directive for that identifier.
1427 ///
1428 /// One can iterate over all previous macro directives from the most recent
1429 /// one.
1431
1432 /// Add a directive to the macro directive history for this identifier.
1436 DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1437 appendMacroDirective(II, MD);
1438 return MD;
1439 }
1441 MacroInfo *MI) {
1442 return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1443 }
1444
1445 /// Set a MacroDirective that was loaded from a PCH file.
1447 MacroDirective *MD);
1448
1449 /// Register an exported macro for a module and identifier.
1451 MacroInfo *Macro,
1452 ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1454
1455 /// Get the list of leaf (non-overridden) module macros for a name.
1457 if (II->isOutOfDate())
1458 updateOutOfDateIdentifier(*II);
1459 auto I = LeafModuleMacros.find(II);
1460 if (I != LeafModuleMacros.end())
1461 return I->second;
1462 return std::nullopt;
1463 }
1464
1465 /// Get the list of submodules that we're currently building.
1467 return BuildingSubmoduleStack;
1468 }
1469
1470 /// \{
1471 /// Iterators for the macro history table. Currently defined macros have
1472 /// IdentifierInfo::hasMacroDefinition() set and an empty
1473 /// MacroInfo::getUndefLoc() at the head of the list.
1474 using macro_iterator = MacroMap::const_iterator;
1475
1476 macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1477 macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1478
1479 llvm::iterator_range<macro_iterator>
1480 macros(bool IncludeExternalMacros = true) const {
1481 macro_iterator begin = macro_begin(IncludeExternalMacros);
1482 macro_iterator end = macro_end(IncludeExternalMacros);
1483 return llvm::make_range(begin, end);
1484 }
1485
1486 /// \}
1487
1488 /// Mark the given clang module as affecting the current clang module or translation unit.
1490 assert(M->isModuleMapModule());
1491 if (!BuildingSubmoduleStack.empty()) {
1492 if (M != BuildingSubmoduleStack.back().M)
1493 BuildingSubmoduleStack.back().M->AffectingClangModules.insert(M);
1494 } else {
1495 AffectingClangModules.insert(M);
1496 }
1497 }
1498
1499 /// Get the set of top-level clang modules that affected preprocessing, but were not
1500 /// imported.
1502 return AffectingClangModules;
1503 }
1504
1505 /// Mark the file as included.
1506 /// Returns true if this is the first time the file was included.
1508 HeaderInfo.getFileInfo(File);
1509 return IncludedFiles.insert(File).second;
1510 }
1511
1512 /// Return true if this header has already been included.
1514 HeaderInfo.getFileInfo(File);
1515 return IncludedFiles.count(File);
1516 }
1517
1518 /// Get the set of included files.
1519 IncludedFilesSet &getIncludedFiles() { return IncludedFiles; }
1520 const IncludedFilesSet &getIncludedFiles() const { return IncludedFiles; }
1521
1522 /// Return the name of the macro defined before \p Loc that has
1523 /// spelling \p Tokens. If there are multiple macros with same spelling,
1524 /// return the last one defined.
1526 ArrayRef<TokenValue> Tokens) const;
1527
1528 /// Get the predefines for this processor.
1529 /// Used by some third-party tools to inspect and add predefines (see
1530 /// https://github.com/llvm/llvm-project/issues/57483).
1531 const std::string &getPredefines() const { return Predefines; }
1532
1533 /// Set the predefines for this Preprocessor.
1534 ///
1535 /// These predefines are automatically injected when parsing the main file.
1536 void setPredefines(std::string P) { Predefines = std::move(P); }
1537
1538 /// Return information about the specified preprocessor
1539 /// identifier token.
1540 IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1541 return &Identifiers.get(Name);
1542 }
1543
1544 /// Add the specified pragma handler to this preprocessor.
1545 ///
1546 /// If \p Namespace is non-null, then it is a token required to exist on the
1547 /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1548 void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1550 AddPragmaHandler(StringRef(), Handler);
1551 }
1552
1553 /// Remove the specific pragma handler from this preprocessor.
1554 ///
1555 /// If \p Namespace is non-null, then it should be the namespace that
1556 /// \p Handler was added to. It is an error to remove a handler that
1557 /// has not been registered.
1558 void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1560 RemovePragmaHandler(StringRef(), Handler);
1561 }
1562
1563 /// Install empty handlers for all pragmas (making them ignored).
1564 void IgnorePragmas();
1565
1566 /// Set empty line handler.
1567 void setEmptylineHandler(EmptylineHandler *Handler) { Emptyline = Handler; }
1568
1569 EmptylineHandler *getEmptylineHandler() const { return Emptyline; }
1570
1571 /// Add the specified comment handler to the preprocessor.
1572 void addCommentHandler(CommentHandler *Handler);
1573
1574 /// Remove the specified comment handler.
1575 ///
1576 /// It is an error to remove a handler that has not been registered.
1577 void removeCommentHandler(CommentHandler *Handler);
1578
1579 /// Set the code completion handler to the given object.
1581 CodeComplete = &Handler;
1582 }
1583
1584 /// Retrieve the current code-completion handler.
1586 return CodeComplete;
1587 }
1588
1589 /// Clear out the code completion handler.
1591 CodeComplete = nullptr;
1592 }
1593
1594 /// Hook used by the lexer to invoke the "included file" code
1595 /// completion point.
1596 void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1597
1598 /// Hook used by the lexer to invoke the "natural language" code
1599 /// completion point.
1601
1602 /// Set the code completion token for filtering purposes.
1604 CodeCompletionII = Filter;
1605 }
1606
1607 /// Set the code completion token range for detecting replacement range later
1608 /// on.
1610 const SourceLocation End) {
1611 CodeCompletionTokenRange = {Start, End};
1612 }
1614 return CodeCompletionTokenRange;
1615 }
1616
1617 /// Get the code completion token for filtering purposes.
1619 if (CodeCompletionII)
1620 return CodeCompletionII->getName();
1621 return {};
1622 }
1623
1624 /// Retrieve the preprocessing record, or NULL if there is no
1625 /// preprocessing record.
1627
1628 /// Create a new preprocessing record, which will keep track of
1629 /// all macro expansions, macro definitions, etc.
1631
1632 /// Returns true if the FileEntry is the PCH through header.
1633 bool isPCHThroughHeader(const FileEntry *FE);
1634
1635 /// True if creating a PCH with a through header.
1637
1638 /// True if using a PCH with a through header.
1640
1641 /// True if creating a PCH with a #pragma hdrstop.
1643
1644 /// True if using a PCH with a #pragma hdrstop.
1646
1647 /// Skip tokens until after the #include of the through header or
1648 /// until after a #pragma hdrstop.
1650
1651 /// Process directives while skipping until the through header or
1652 /// #pragma hdrstop is found.
1654 SourceLocation HashLoc);
1655
1656 /// Enter the specified FileID as the main source file,
1657 /// which implicitly adds the builtin defines etc.
1658 void EnterMainSourceFile();
1659
1660 /// Inform the preprocessor callbacks that processing is complete.
1661 void EndSourceFile();
1662
1663 /// Add a source file to the top of the include stack and
1664 /// start lexing tokens from it instead of the current buffer.
1665 ///
1666 /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1668 SourceLocation Loc, bool IsFirstIncludeOfFile = true);
1669
1670 /// Add a Macro to the top of the include stack and start lexing
1671 /// tokens from it instead of the current buffer.
1672 ///
1673 /// \param Args specifies the tokens input to a function-like macro.
1674 /// \param ILEnd specifies the location of the ')' for a function-like macro
1675 /// or the identifier for an object-like macro.
1676 void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1677 MacroArgs *Args);
1678
1679private:
1680 /// Add a "macro" context to the top of the include stack,
1681 /// which will cause the lexer to start returning the specified tokens.
1682 ///
1683 /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1684 /// will not be subject to further macro expansion. Otherwise, these tokens
1685 /// will be re-macro-expanded when/if expansion is enabled.
1686 ///
1687 /// If \p OwnsTokens is false, this method assumes that the specified stream
1688 /// of tokens has a permanent owner somewhere, so they do not need to be
1689 /// copied. If it is true, it assumes the array of tokens is allocated with
1690 /// \c new[] and the Preprocessor will delete[] it.
1691 ///
1692 /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
1693 /// set, see the flag documentation for details.
1694 void EnterTokenStream(const Token *Toks, unsigned NumToks,
1695 bool DisableMacroExpansion, bool OwnsTokens,
1696 bool IsReinject);
1697
1698public:
1699 void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1700 bool DisableMacroExpansion, bool IsReinject) {
1701 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true,
1702 IsReinject);
1703 }
1704
1705 void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
1706 bool IsReinject) {
1707 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false,
1708 IsReinject);
1709 }
1710
1711 /// Pop the current lexer/macro exp off the top of the lexer stack.
1712 ///
1713 /// This should only be used in situations where the current state of the
1714 /// top-of-stack lexer is known.
1715 void RemoveTopOfLexerStack();
1716
1717 /// From the point that this method is called, and until
1718 /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1719 /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1720 /// make the Preprocessor re-lex the same tokens.
1721 ///
1722 /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1723 /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1724 /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1725 ///
1726 /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1727 /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1728 /// tokens will continue indefinitely.
1729 ///
1730 /// \param Unannotated Whether token annotations are reverted upon calling
1731 /// Backtrack().
1732 void EnableBacktrackAtThisPos(bool Unannotated = false);
1733
1734private:
1735 std::pair<CachedTokensTy::size_type, bool> LastBacktrackPos();
1736
1737 CachedTokensTy PopUnannotatedBacktrackTokens();
1738
1739public:
1740 /// Disable the last EnableBacktrackAtThisPos call.
1742
1743 /// Make Preprocessor re-lex the tokens that were lexed since
1744 /// EnableBacktrackAtThisPos() was previously called.
1745 void Backtrack();
1746
1747 /// True if EnableBacktrackAtThisPos() was called and
1748 /// caching of tokens is on.
1749 bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1750
1751 /// True if EnableBacktrackAtThisPos() was called and
1752 /// caching of unannotated tokens is on.
1754 return !UnannotatedBacktrackTokens.empty();
1755 }
1756
1757 /// Lex the next token for this preprocessor.
1758 void Lex(Token &Result);
1759
1760 /// Lex all tokens for this preprocessor until (and excluding) end of file.
1761 void LexTokensUntilEOF(std::vector<Token> *Tokens = nullptr);
1762
1763 /// Lex a token, forming a header-name token if possible.
1764 bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
1765
1766 /// Lex the parameters for an #embed directive, returns nullopt on error.
1767 std::optional<LexEmbedParametersResult> LexEmbedParameters(Token &Current,
1768 bool ForHasEmbed);
1769
1772
1774
1776 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1777 }
1778
1779 /// Lex a string literal, which may be the concatenation of multiple
1780 /// string literals and may even come from macro expansion.
1781 /// \returns true on success, false if a error diagnostic has been generated.
1782 bool LexStringLiteral(Token &Result, std::string &String,
1783 const char *DiagnosticTag, bool AllowMacroExpansion) {
1784 if (AllowMacroExpansion)
1785 Lex(Result);
1786 else
1788 return FinishLexStringLiteral(Result, String, DiagnosticTag,
1789 AllowMacroExpansion);
1790 }
1791
1792 /// Complete the lexing of a string literal where the first token has
1793 /// already been lexed (see LexStringLiteral).
1794 bool FinishLexStringLiteral(Token &Result, std::string &String,
1795 const char *DiagnosticTag,
1796 bool AllowMacroExpansion);
1797
1798 /// Lex a token. If it's a comment, keep lexing until we get
1799 /// something not a comment.
1800 ///
1801 /// This is useful in -E -C mode where comments would foul up preprocessor
1802 /// directive handling.
1804 do
1805 Lex(Result);
1806 while (Result.getKind() == tok::comment);
1807 }
1808
1809 /// Just like Lex, but disables macro expansion of identifier tokens.
1811 // Disable macro expansion.
1812 bool OldVal = DisableMacroExpansion;
1813 DisableMacroExpansion = true;
1814 // Lex the token.
1815 Lex(Result);
1816
1817 // Reenable it.
1818 DisableMacroExpansion = OldVal;
1819 }
1820
1821 /// Like LexNonComment, but this disables macro expansion of
1822 /// identifier tokens.
1824 do
1826 while (Result.getKind() == tok::comment);
1827 }
1828
1829 /// Parses a simple integer literal to get its numeric value. Floating
1830 /// point literals and user defined literals are rejected. Used primarily to
1831 /// handle pragmas that accept integer arguments.
1832 bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1833
1834 /// Disables macro expansion everywhere except for preprocessor directives.
1836 DisableMacroExpansion = true;
1837 MacroExpansionInDirectivesOverride = true;
1838 }
1839
1840 /// Peeks ahead N tokens and returns that token without consuming any
1841 /// tokens.
1842 ///
1843 /// LookAhead(0) returns the next token that would be returned by Lex(),
1844 /// LookAhead(1) returns the token after it, etc. This returns normal
1845 /// tokens after phase 5. As such, it is equivalent to using
1846 /// 'Lex', not 'LexUnexpandedToken'.
1847 const Token &LookAhead(unsigned N) {
1848 assert(LexLevel == 0 && "cannot use lookahead while lexing");
1849 if (CachedLexPos + N < CachedTokens.size())
1850 return CachedTokens[CachedLexPos+N];
1851 else
1852 return PeekAhead(N+1);
1853 }
1854
1855 /// When backtracking is enabled and tokens are cached,
1856 /// this allows to revert a specific number of tokens.
1857 ///
1858 /// Note that the number of tokens being reverted should be up to the last
1859 /// backtrack position, not more.
1860 void RevertCachedTokens(unsigned N) {
1861 assert(isBacktrackEnabled() &&
1862 "Should only be called when tokens are cached for backtracking");
1863 assert(signed(CachedLexPos) - signed(N) >=
1864 signed(LastBacktrackPos().first) &&
1865 "Should revert tokens up to the last backtrack position, not more");
1866 assert(signed(CachedLexPos) - signed(N) >= 0 &&
1867 "Corrupted backtrack positions ?");
1868 CachedLexPos -= N;
1869 }
1870
1871 /// Enters a token in the token stream to be lexed next.
1872 ///
1873 /// If BackTrack() is called afterwards, the token will remain at the
1874 /// insertion point.
1875 /// If \p IsReinject is true, resulting token will have Token::IsReinjected
1876 /// flag set. See the flag documentation for details.
1877 void EnterToken(const Token &Tok, bool IsReinject) {
1878 if (LexLevel) {
1879 // It's not correct in general to enter caching lex mode while in the
1880 // middle of a nested lexing action.
1881 auto TokCopy = std::make_unique<Token[]>(1);
1882 TokCopy[0] = Tok;
1883 EnterTokenStream(std::move(TokCopy), 1, true, IsReinject);
1884 } else {
1885 EnterCachingLexMode();
1886 assert(IsReinject && "new tokens in the middle of cached stream");
1887 CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1888 }
1889 }
1890
1891 /// We notify the Preprocessor that if it is caching tokens (because
1892 /// backtrack is enabled) it should replace the most recent cached tokens
1893 /// with the given annotation token. This function has no effect if
1894 /// backtracking is not enabled.
1895 ///
1896 /// Note that the use of this function is just for optimization, so that the
1897 /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1898 /// invoked.
1899 void AnnotateCachedTokens(const Token &Tok) {
1900 assert(Tok.isAnnotation() && "Expected annotation token");
1901 if (CachedLexPos != 0 && isBacktrackEnabled())
1902 AnnotatePreviousCachedTokens(Tok);
1903 }
1904
1905 /// Get the location of the last cached token, suitable for setting the end
1906 /// location of an annotation token.
1908 assert(CachedLexPos != 0);
1909 return CachedTokens[CachedLexPos-1].getLastLoc();
1910 }
1911
1912 /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1913 /// CachedTokens.
1914 bool IsPreviousCachedToken(const Token &Tok) const;
1915
1916 /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1917 /// in \p NewToks.
1918 ///
1919 /// Useful when a token needs to be split in smaller ones and CachedTokens
1920 /// most recent token must to be updated to reflect that.
1922
1923 /// Replace the last token with an annotation token.
1924 ///
1925 /// Like AnnotateCachedTokens(), this routine replaces an
1926 /// already-parsed (and resolved) token with an annotation
1927 /// token. However, this routine only replaces the last token with
1928 /// the annotation token; it does not affect any other cached
1929 /// tokens. This function has no effect if backtracking is not
1930 /// enabled.
1932 assert(Tok.isAnnotation() && "Expected annotation token");
1933 if (CachedLexPos != 0 && isBacktrackEnabled())
1934 CachedTokens[CachedLexPos-1] = Tok;
1935 }
1936
1937 /// Enter an annotation token into the token stream.
1939 void *AnnotationVal);
1940
1941 /// Determine whether it's possible for a future call to Lex to produce an
1942 /// annotation token created by a previous call to EnterAnnotationToken.
1944 return CurLexerCallback != CLK_Lexer;
1945 }
1946
1947 /// Update the current token to represent the provided
1948 /// identifier, in order to cache an action performed by typo correction.
1949 void TypoCorrectToken(const Token &Tok) {
1950 assert(Tok.getIdentifierInfo() && "Expected identifier token");
1951 if (CachedLexPos != 0 && isBacktrackEnabled())
1952 CachedTokens[CachedLexPos-1] = Tok;
1953 }
1954
1955 /// Recompute the current lexer kind based on the CurLexer/
1956 /// CurTokenLexer pointers.
1957 void recomputeCurLexerKind();
1958
1959 /// Returns true if incremental processing is enabled
1960 bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1961
1962 /// Enables the incremental processing
1963 void enableIncrementalProcessing(bool value = true) {
1964 IncrementalProcessing = value;
1965 }
1966
1967 /// Specify the point at which code-completion will be performed.
1968 ///
1969 /// \param File the file in which code completion should occur. If
1970 /// this file is included multiple times, code-completion will
1971 /// perform completion the first time it is included. If NULL, this
1972 /// function clears out the code-completion point.
1973 ///
1974 /// \param Line the line at which code completion should occur
1975 /// (1-based).
1976 ///
1977 /// \param Column the column at which code completion should occur
1978 /// (1-based).
1979 ///
1980 /// \returns true if an error occurred, false otherwise.
1982 unsigned Column);
1983
1984 /// Determine if we are performing code completion.
1985 bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1986
1987 /// Returns the location of the code-completion point.
1988 ///
1989 /// Returns an invalid location if code-completion is not enabled or the file
1990 /// containing the code-completion point has not been lexed yet.
1991 SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1992
1993 /// Returns the start location of the file of code-completion point.
1994 ///
1995 /// Returns an invalid location if code-completion is not enabled or the file
1996 /// containing the code-completion point has not been lexed yet.
1998 return CodeCompletionFileLoc;
1999 }
2000
2001 /// Returns true if code-completion is enabled and we have hit the
2002 /// code-completion point.
2003 bool isCodeCompletionReached() const { return CodeCompletionReached; }
2004
2005 /// Note that we hit the code-completion point.
2007 assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
2008 CodeCompletionReached = true;
2009 // Silence any diagnostics that occur after we hit the code-completion.
2011 }
2012
2013 /// The location of the currently-active \#pragma clang
2014 /// arc_cf_code_audited begin.
2015 ///
2016 /// Returns an invalid location if there is no such pragma active.
2017 std::pair<IdentifierInfo *, SourceLocation>
2019 return PragmaARCCFCodeAuditedInfo;
2020 }
2021
2022 /// Set the location of the currently-active \#pragma clang
2023 /// arc_cf_code_audited begin. An invalid location ends the pragma.
2026 PragmaARCCFCodeAuditedInfo = {Ident, Loc};
2027 }
2028
2029 /// The location of the currently-active \#pragma clang
2030 /// assume_nonnull begin.
2031 ///
2032 /// Returns an invalid location if there is no such pragma active.
2034 return PragmaAssumeNonNullLoc;
2035 }
2036
2037 /// Set the location of the currently-active \#pragma clang
2038 /// assume_nonnull begin. An invalid location ends the pragma.
2040 PragmaAssumeNonNullLoc = Loc;
2041 }
2042
2043 /// Get the location of the recorded unterminated \#pragma clang
2044 /// assume_nonnull begin in the preamble, if one exists.
2045 ///
2046 /// Returns an invalid location if the premable did not end with
2047 /// such a pragma active or if there is no recorded preamble.
2049 return PreambleRecordedPragmaAssumeNonNullLoc;
2050 }
2051
2052 /// Record the location of the unterminated \#pragma clang
2053 /// assume_nonnull begin in the preamble.
2055 PreambleRecordedPragmaAssumeNonNullLoc = Loc;
2056 }
2057
2058 /// Set the directory in which the main file should be considered
2059 /// to have been found, if it is not a real file.
2060 void setMainFileDir(DirectoryEntryRef Dir) { MainFileDir = Dir; }
2061
2062 /// Instruct the preprocessor to skip part of the main source file.
2063 ///
2064 /// \param Bytes The number of bytes in the preamble to skip.
2065 ///
2066 /// \param StartOfLine Whether skipping these bytes puts the lexer at the
2067 /// start of a line.
2068 void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
2069 SkipMainFilePreamble.first = Bytes;
2070 SkipMainFilePreamble.second = StartOfLine;
2071 }
2072
2073 /// Forwarding function for diagnostics. This emits a diagnostic at
2074 /// the specified Token's location, translating the token's start
2075 /// position in the current buffer into a SourcePosition object for rendering.
2076 DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
2077 return Diags->Report(Loc, DiagID);
2078 }
2079
2080 DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
2081 return Diags->Report(Tok.getLocation(), DiagID);
2082 }
2083
2084 /// Return the 'spelling' of the token at the given
2085 /// location; does not go up to the spelling location or down to the
2086 /// expansion location.
2087 ///
2088 /// \param buffer A buffer which will be used only if the token requires
2089 /// "cleaning", e.g. if it contains trigraphs or escaped newlines
2090 /// \param invalid If non-null, will be set \c true if an error occurs.
2092 SmallVectorImpl<char> &buffer,
2093 bool *invalid = nullptr) const {
2094 return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
2095 }
2096
2097 /// Return the 'spelling' of the Tok token.
2098 ///
2099 /// The spelling of a token is the characters used to represent the token in
2100 /// the source file after trigraph expansion and escaped-newline folding. In
2101 /// particular, this wants to get the true, uncanonicalized, spelling of
2102 /// things like digraphs, UCNs, etc.
2103 ///
2104 /// \param Invalid If non-null, will be set \c true if an error occurs.
2105 std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
2106 return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
2107 }
2108
2109 /// Get the spelling of a token into a preallocated buffer, instead
2110 /// of as an std::string.
2111 ///
2112 /// The caller is required to allocate enough space for the token, which is
2113 /// guaranteed to be at least Tok.getLength() bytes long. The length of the
2114 /// actual result is returned.
2115 ///
2116 /// Note that this method may do two possible things: it may either fill in
2117 /// the buffer specified with characters, or it may *change the input pointer*
2118 /// to point to a constant buffer with the data already in it (avoiding a
2119 /// copy). The caller is not allowed to modify the returned buffer pointer
2120 /// if an internal buffer is returned.
2121 unsigned getSpelling(const Token &Tok, const char *&Buffer,
2122 bool *Invalid = nullptr) const {
2123 return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
2124 }
2125
2126 /// Get the spelling of a token into a SmallVector.
2127 ///
2128 /// Note that the returned StringRef may not point to the
2129 /// supplied buffer if a copy can be avoided.
2130 StringRef getSpelling(const Token &Tok,
2131 SmallVectorImpl<char> &Buffer,
2132 bool *Invalid = nullptr) const;
2133
2134 /// Relex the token at the specified location.
2135 /// \returns true if there was a failure, false on success.
2137 bool IgnoreWhiteSpace = false) {
2138 return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
2139 }
2140
2141 /// Given a Token \p Tok that is a numeric constant with length 1,
2142 /// return the value of constant as an unsigned 8-bit integer.
2143 uint8_t
2145 bool *Invalid = nullptr) const {
2146 assert((Tok.is(tok::numeric_constant) || Tok.is(tok::binary_data)) &&
2147 Tok.getLength() == 1 && "Called on unsupported token");
2148 assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
2149
2150 // If the token is carrying a literal data pointer, just use it.
2151 if (const char *D = Tok.getLiteralData())
2152 return (Tok.getKind() == tok::binary_data) ? *D : *D - '0';
2153
2154 assert(Tok.is(tok::numeric_constant) && "binary data with no data");
2155 // Otherwise, fall back on getCharacterData, which is slower, but always
2156 // works.
2157 return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid) - '0';
2158 }
2159
2160 /// Retrieve the name of the immediate macro expansion.
2161 ///
2162 /// This routine starts from a source location, and finds the name of the
2163 /// macro responsible for its immediate expansion. It looks through any
2164 /// intervening macro argument expansions to compute this. It returns a
2165 /// StringRef that refers to the SourceManager-owned buffer of the source
2166 /// where that macro name is spelled. Thus, the result shouldn't out-live
2167 /// the SourceManager.
2169 return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
2170 }
2171
2172 /// Plop the specified string into a scratch buffer and set the
2173 /// specified token's location and length to it.
2174 ///
2175 /// If specified, the source location provides a location of the expansion
2176 /// point of the token.
2177 void CreateString(StringRef Str, Token &Tok,
2178 SourceLocation ExpansionLocStart = SourceLocation(),
2179 SourceLocation ExpansionLocEnd = SourceLocation());
2180
2181 /// Split the first Length characters out of the token starting at TokLoc
2182 /// and return a location pointing to the split token. Re-lexing from the
2183 /// split token will return the split token rather than the original.
2184 SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
2185
2186 /// Computes the source location just past the end of the
2187 /// token at this source location.
2188 ///
2189 /// This routine can be used to produce a source location that
2190 /// points just past the end of the token referenced by \p Loc, and
2191 /// is generally used when a diagnostic needs to point just after a
2192 /// token where it expected something different that it received. If
2193 /// the returned source location would not be meaningful (e.g., if
2194 /// it points into a macro), this routine returns an invalid
2195 /// source location.
2196 ///
2197 /// \param Offset an offset from the end of the token, where the source
2198 /// location should refer to. The default offset (0) produces a source
2199 /// location pointing just past the end of the token; an offset of 1 produces
2200 /// a source location pointing to the last character in the token, etc.
2202 return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
2203 }
2204
2205 /// Returns true if the given MacroID location points at the first
2206 /// token of the macro expansion.
2207 ///
2208 /// \param MacroBegin If non-null and function returns true, it is set to
2209 /// begin location of the macro.
2211 SourceLocation *MacroBegin = nullptr) const {
2212 return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
2213 MacroBegin);
2214 }
2215
2216 /// Returns true if the given MacroID location points at the last
2217 /// token of the macro expansion.
2218 ///
2219 /// \param MacroEnd If non-null and function returns true, it is set to
2220 /// end location of the macro.
2222 SourceLocation *MacroEnd = nullptr) const {
2223 return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
2224 }
2225
2226 /// Print the token to stderr, used for debugging.
2227 void DumpToken(const Token &Tok, bool DumpFlags = false) const;
2228 void DumpLocation(SourceLocation Loc) const;
2229 void DumpMacro(const MacroInfo &MI) const;
2230 void dumpMacroInfo(const IdentifierInfo *II);
2231
2232 /// Given a location that specifies the start of a
2233 /// token, return a new location that specifies a character within the token.
2235 unsigned Char) const {
2236 return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
2237 }
2238
2239 /// Increment the counters for the number of token paste operations
2240 /// performed.
2241 ///
2242 /// If fast was specified, this is a 'fast paste' case we handled.
2243 void IncrementPasteCounter(bool isFast) {
2244 if (isFast)
2245 ++NumFastTokenPaste;
2246 else
2247 ++NumTokenPaste;
2248 }
2249
2250 void PrintStats();
2251
2252 size_t getTotalMemory() const;
2253
2254 /// When the macro expander pastes together a comment (/##/) in Microsoft
2255 /// mode, this method handles updating the current state, returning the
2256 /// token on the next source line.
2258
2259 //===--------------------------------------------------------------------===//
2260 // Preprocessor callback methods. These are invoked by a lexer as various
2261 // directives and events are found.
2262
2263 /// Given a tok::raw_identifier token, look up the
2264 /// identifier information for the token and install it into the token,
2265 /// updating the token kind accordingly.
2267
2268private:
2269 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
2270
2271public:
2272 /// Specifies the reason for poisoning an identifier.
2273 ///
2274 /// If that identifier is accessed while poisoned, then this reason will be
2275 /// used instead of the default "poisoned" diagnostic.
2276 void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
2277
2278 /// Display reason for poisoned identifier.
2280
2282 if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
2283 if(II->isPoisoned()) {
2285 }
2286 }
2287 }
2288
2289private:
2290 /// Identifiers used for SEH handling in Borland. These are only
2291 /// allowed in particular circumstances
2292 // __except block
2293 IdentifierInfo *Ident__exception_code,
2294 *Ident___exception_code,
2295 *Ident_GetExceptionCode;
2296 // __except filter expression
2297 IdentifierInfo *Ident__exception_info,
2298 *Ident___exception_info,
2299 *Ident_GetExceptionInfo;
2300 // __finally
2301 IdentifierInfo *Ident__abnormal_termination,
2302 *Ident___abnormal_termination,
2303 *Ident_AbnormalTermination;
2304
2305 const char *getCurLexerEndPos();
2306 void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
2307
2308public:
2309 void PoisonSEHIdentifiers(bool Poison = true); // Borland
2310
2311 /// Callback invoked when the lexer reads an identifier and has
2312 /// filled in the tokens IdentifierInfo member.
2313 ///
2314 /// This callback potentially macro expands it or turns it into a named
2315 /// token (like 'for').
2316 ///
2317 /// \returns true if we actually computed a token, false if we need to
2318 /// lex again.
2320
2321 /// Callback invoked when the lexer hits the end of the current file.
2322 ///
2323 /// This either returns the EOF token and returns true, or
2324 /// pops a level off the include stack and returns false, at which point the
2325 /// client should call lex again.
2326 bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
2327
2328 /// Callback invoked when the current TokenLexer hits the end of its
2329 /// token stream.
2331
2332 /// Callback invoked when the lexer sees a # token at the start of a
2333 /// line.
2334 ///
2335 /// This consumes the directive, modifies the lexer/preprocessor state, and
2336 /// advances the lexer(s) so that the next token read is the correct one.
2338
2339 /// Ensure that the next token is a tok::eod token.
2340 ///
2341 /// If not, emit a diagnostic and consume up until the eod.
2342 /// If \p EnableMacros is true, then we consider macros that expand to zero
2343 /// tokens as being ok.
2344 ///
2345 /// \return The location of the end of the directive (the terminating
2346 /// newline).
2347 SourceLocation CheckEndOfDirective(const char *DirType,
2348 bool EnableMacros = false);
2349
2350 /// Read and discard all tokens remaining on the current line until
2351 /// the tok::eod token is found. Returns the range of the skipped tokens.
2353 Token Tmp;
2354 return DiscardUntilEndOfDirective(Tmp);
2355 }
2356
2357 /// Same as above except retains the token that was found.
2359
2360 /// Returns true if the preprocessor has seen a use of
2361 /// __DATE__ or __TIME__ in the file so far.
2362 bool SawDateOrTime() const {
2363 return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
2364 }
2365 unsigned getCounterValue() const { return CounterValue; }
2366 void setCounterValue(unsigned V) { CounterValue = V; }
2367
2369 assert(CurrentFPEvalMethod != LangOptions::FEM_UnsetOnCommandLine &&
2370 "FPEvalMethod should be set either from command line or from the "
2371 "target info");
2372 return CurrentFPEvalMethod;
2373 }
2374
2376 return TUFPEvalMethod;
2377 }
2378
2380 return LastFPEvalPragmaLocation;
2381 }
2382
2386 "FPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2387 // This is the location of the '#pragma float_control" where the
2388 // execution state is modifed.
2389 LastFPEvalPragmaLocation = PragmaLoc;
2390 CurrentFPEvalMethod = Val;
2391 TUFPEvalMethod = Val;
2392 }
2393
2396 "TUPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2397 TUFPEvalMethod = Val;
2398 }
2399
2400 /// Retrieves the module that we're currently building, if any.
2402
2403 /// Retrieves the module whose implementation we're current compiling, if any.
2405
2406 /// If we are preprocessing a named module.
2407 bool isInNamedModule() const { return ModuleDeclState.isNamedModule(); }
2408
2409 /// If we are proprocessing a named interface unit.
2410 /// Note that a module implementation partition is not considered as an
2411 /// named interface unit here although it is importable
2412 /// to ease the parsing.
2414 return ModuleDeclState.isNamedInterface();
2415 }
2416
2417 /// Get the named module name we're preprocessing.
2418 /// Requires we're preprocessing a named module.
2419 StringRef getNamedModuleName() const { return ModuleDeclState.getName(); }
2420
2421 /// If we are implementing an implementation module unit.
2422 /// Note that the module implementation partition is not considered as an
2423 /// implementation unit.
2425 return ModuleDeclState.isImplementationUnit();
2426 }
2427
2428 /// If we're importing a standard C++20 Named Modules.
2430 // NamedModuleImportPath will be non-empty only if we're importing
2431 // Standard C++ named modules.
2432 return !NamedModuleImportPath.empty() && getLangOpts().CPlusPlusModules &&
2433 !IsAtImport;
2434 }
2435
2436 /// Allocate a new MacroInfo object with the provided SourceLocation.
2438
2439 /// Turn the specified lexer token into a fully checked and spelled
2440 /// filename, e.g. as an operand of \#include.
2441 ///
2442 /// The caller is expected to provide a buffer that is large enough to hold
2443 /// the spelling of the filename, but is also expected to handle the case
2444 /// when this method decides to use a different buffer.
2445 ///
2446 /// \returns true if the input filename was in <>'s or false if it was
2447 /// in ""'s.
2448 bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
2449
2450 /// Given a "foo" or <foo> reference, look up the indicated file.
2451 ///
2452 /// Returns std::nullopt on failure. \p isAngled indicates whether the file
2453 /// reference is for system \#include's or not (i.e. using <> instead of "").
2455 LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled,
2456 ConstSearchDirIterator FromDir, const FileEntry *FromFile,
2457 ConstSearchDirIterator *CurDir, SmallVectorImpl<char> *SearchPath,
2458 SmallVectorImpl<char> *RelativePath,
2459 ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped,
2460 bool *IsFrameworkFound, bool SkipCache = false,
2461 bool OpenFile = true, bool CacheFailures = true);
2462
2463 /// Given a "Filename" or <Filename> reference, look up the indicated embed
2464 /// resource. \p isAngled indicates whether the file reference is for
2465 /// system \#include's or not (i.e. using <> instead of ""). If \p OpenFile
2466 /// is true, the file looked up is opened for reading, otherwise it only
2467 /// validates that the file exists. Quoted filenames are looked up relative
2468 /// to \p LookupFromFile if it is nonnull.
2469 ///
2470 /// Returns std::nullopt on failure.
2472 LookupEmbedFile(StringRef Filename, bool isAngled, bool OpenFile,
2473 const FileEntry *LookupFromFile = nullptr);
2474
2475 /// Return true if we're in the top-level file, not in a \#include.
2476 bool isInPrimaryFile() const;
2477
2478 /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
2479 /// followed by EOD. Return true if the token is not a valid on-off-switch.
2481
2482 bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
2483 bool *ShadowFlag = nullptr);
2484
2485 void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
2486 Module *LeaveSubmodule(bool ForPragma);
2487
2488private:
2490
2491 void PushIncludeMacroStack() {
2492 assert(CurLexerCallback != CLK_CachingLexer &&
2493 "cannot push a caching lexer");
2494 IncludeMacroStack.emplace_back(CurLexerCallback, CurLexerSubmodule,
2495 std::move(CurLexer), CurPPLexer,
2496 std::move(CurTokenLexer), CurDirLookup);
2497 CurPPLexer = nullptr;
2498 }
2499
2500 void PopIncludeMacroStack() {
2501 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2502 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2503 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2504 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2505 CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2506 CurLexerCallback = IncludeMacroStack.back().CurLexerCallback;
2507 IncludeMacroStack.pop_back();
2508 }
2509
2510 void PropagateLineStartLeadingSpaceInfo(Token &Result);
2511
2512 /// Determine whether we need to create module macros for #defines in the
2513 /// current context.
2514 bool needModuleMacros() const;
2515
2516 /// Update the set of active module macros and ambiguity flag for a module
2517 /// macro name.
2518 void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
2519
2520 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2521 SourceLocation Loc);
2522 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2523 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2524 bool isPublic);
2525
2526 /// Lex and validate a macro name, which occurs after a
2527 /// \#define or \#undef.
2528 ///
2529 /// \param MacroNameTok Token that represents the name defined or undefined.
2530 /// \param IsDefineUndef Kind if preprocessor directive.
2531 /// \param ShadowFlag Points to flag that is set if macro name shadows
2532 /// a keyword.
2533 ///
2534 /// This emits a diagnostic, sets the token kind to eod,
2535 /// and discards the rest of the macro line if the macro name is invalid.
2536 void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
2537 bool *ShadowFlag = nullptr);
2538
2539 /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
2540 /// entire line) of the macro's tokens and adds them to MacroInfo, and while
2541 /// doing so performs certain validity checks including (but not limited to):
2542 /// - # (stringization) is followed by a macro parameter
2543 /// \param MacroNameTok - Token that represents the macro name
2544 /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
2545 ///
2546 /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
2547 /// returns a nullptr if an invalid sequence of tokens is encountered.
2548 MacroInfo *ReadOptionalMacroParameterListAndBody(
2549 const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
2550
2551 /// The ( starting an argument list of a macro definition has just been read.
2552 /// Lex the rest of the parameters and the closing ), updating \p MI with
2553 /// what we learn and saving in \p LastTok the last token read.
2554 /// Return true if an error occurs parsing the arg list.
2555 bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2556
2557 /// Provide a suggestion for a typoed directive. If there is no typo, then
2558 /// just skip suggesting.
2559 ///
2560 /// \param Tok - Token that represents the directive
2561 /// \param Directive - String reference for the directive name
2562 void SuggestTypoedDirective(const Token &Tok, StringRef Directive) const;
2563
2564 /// We just read a \#if or related directive and decided that the
2565 /// subsequent tokens are in the \#if'd out portion of the
2566 /// file. Lex the rest of the file, until we see an \#endif. If \p
2567 /// FoundNonSkipPortion is true, then we have already emitted code for part of
2568 /// this \#if directive, so \#else/\#elif blocks should never be entered. If
2569 /// \p FoundElse is false, then \#else directives are ok, if not, then we have
2570 /// already seen one so a \#else directive is a duplicate. When this returns,
2571 /// the caller can lex the first valid token.
2572 void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2573 SourceLocation IfTokenLoc,
2574 bool FoundNonSkipPortion, bool FoundElse,
2575 SourceLocation ElseLoc = SourceLocation());
2576
2577 /// Information about the result for evaluating an expression for a
2578 /// preprocessor directive.
2579 struct DirectiveEvalResult {
2580 /// The integral value of the expression.
2581 std::optional<llvm::APSInt> Value;
2582
2583 /// Whether the expression was evaluated as true or not.
2584 bool Conditional;
2585
2586 /// True if the expression contained identifiers that were undefined.
2587 bool IncludedUndefinedIds;
2588
2589 /// The source range for the expression.
2590 SourceRange ExprRange;
2591 };
2592
2593 /// Evaluate an integer constant expression that may occur after a
2594 /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2595 ///
2596 /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2597 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro,
2598 bool CheckForEoD = true);
2599
2600 /// Evaluate an integer constant expression that may occur after a
2601 /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2602 ///
2603 /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2604 /// \p EvaluatedDefined will contain the result of whether "defined" appeared
2605 /// in the evaluated expression or not.
2606 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro,
2607 Token &Tok,
2608 bool &EvaluatedDefined,
2609 bool CheckForEoD = true);
2610
2611 /// Process a '__has_embed("path" [, ...])' expression.
2612 ///
2613 /// Returns predefined `__STDC_EMBED_*` macro values if
2614 /// successful.
2615 EmbedResult EvaluateHasEmbed(Token &Tok, IdentifierInfo *II);
2616
2617 /// Process a '__has_include("path")' expression.
2618 ///
2619 /// Returns true if successful.
2620 bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II);
2621
2622 /// Process '__has_include_next("path")' expression.
2623 ///
2624 /// Returns true if successful.
2625 bool EvaluateHasIncludeNext(Token &Tok, IdentifierInfo *II);
2626
2627 /// Get the directory and file from which to start \#include_next lookup.
2628 std::pair<ConstSearchDirIterator, const FileEntry *>
2629 getIncludeNextStart(const Token &IncludeNextTok) const;
2630
2631 /// Install the standard preprocessor pragmas:
2632 /// \#pragma GCC poison/system_header/dependency and \#pragma once.
2633 void RegisterBuiltinPragmas();
2634
2635 /// Register builtin macros such as __LINE__ with the identifier table.
2636 void RegisterBuiltinMacros();
2637
2638 /// If an identifier token is read that is to be expanded as a macro, handle
2639 /// it and return the next token as 'Tok'. If we lexed a token, return true;
2640 /// otherwise the caller should lex again.
2641 bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2642
2643 /// Cache macro expanded tokens for TokenLexers.
2644 //
2645 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2646 /// going to lex in the cache and when it finishes the tokens are removed
2647 /// from the end of the cache.
2648 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2649 ArrayRef<Token> tokens);
2650
2651 void removeCachedMacroExpandedTokensOfLastLexer();
2652
2653 /// Determine whether the next preprocessor token to be
2654 /// lexed is a '('. If so, consume the token and return true, if not, this
2655 /// method should have no observable side-effect on the lexed tokens.
2656 bool isNextPPTokenLParen();
2657
2658 /// After reading "MACRO(", this method is invoked to read all of the formal
2659 /// arguments specified for the macro invocation. Returns null on error.
2660 MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2661 SourceLocation &MacroEnd);
2662
2663 /// If an identifier token is read that is to be expanded
2664 /// as a builtin macro, handle it and return the next token as 'Tok'.
2665 void ExpandBuiltinMacro(Token &Tok);
2666
2667 /// Read a \c _Pragma directive, slice it up, process it, then
2668 /// return the first token after the directive.
2669 /// This assumes that the \c _Pragma token has just been read into \p Tok.
2670 void Handle_Pragma(Token &Tok);
2671
2672 /// Like Handle_Pragma except the pragma text is not enclosed within
2673 /// a string literal.
2674 void HandleMicrosoft__pragma(Token &Tok);
2675
2676 /// Add a lexer to the top of the include stack and
2677 /// start lexing tokens from it instead of the current buffer.
2678 void EnterSourceFileWithLexer(Lexer *TheLexer, ConstSearchDirIterator Dir);
2679
2680 /// Set the FileID for the preprocessor predefines.
2681 void setPredefinesFileID(FileID FID) {
2682 assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2683 PredefinesFileID = FID;
2684 }
2685
2686 /// Set the FileID for the PCH through header.
2687 void setPCHThroughHeaderFileID(FileID FID);
2688
2689 /// Returns true if we are lexing from a file and not a
2690 /// pragma or a macro.
2691 static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2692 return L ? !L->isPragmaLexer() : P != nullptr;
2693 }
2694
2695 static bool IsFileLexer(const IncludeStackInfo& I) {
2696 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2697 }
2698
2699 bool IsFileLexer() const {
2700 return IsFileLexer(CurLexer.get(), CurPPLexer);
2701 }
2702
2703 //===--------------------------------------------------------------------===//
2704 // Caching stuff.
2705 void CachingLex(Token &Result);
2706
2707 bool InCachingLexMode() const {
2708 // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2709 // that we are past EOF, not that we are in CachingLex mode.
2710 return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2711 }
2712
2713 void EnterCachingLexMode();
2714 void EnterCachingLexModeUnchecked();
2715
2716 void ExitCachingLexMode() {
2717 if (InCachingLexMode())
2719 }
2720
2721 const Token &PeekAhead(unsigned N);
2722 void AnnotatePreviousCachedTokens(const Token &Tok);
2723
2724 //===--------------------------------------------------------------------===//
2725 /// Handle*Directive - implement the various preprocessor directives. These
2726 /// should side-effect the current preprocessor object so that the next call
2727 /// to Lex() will return the appropriate token next.
2728 void HandleLineDirective();
2729 void HandleDigitDirective(Token &Tok);
2730 void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2731 void HandleIdentSCCSDirective(Token &Tok);
2732 void HandleMacroPublicDirective(Token &Tok);
2733 void HandleMacroPrivateDirective();
2734
2735 /// An additional notification that can be produced by a header inclusion or
2736 /// import to tell the parser what happened.
2737 struct ImportAction {
2738 enum ActionKind {
2739 None,
2740 ModuleBegin,
2741 ModuleImport,
2742 HeaderUnitImport,
2743 SkippedModuleImport,
2744 Failure,
2745 } Kind;
2746 Module *ModuleForHeader = nullptr;
2747
2748 ImportAction(ActionKind AK, Module *Mod = nullptr)
2749 : Kind(AK), ModuleForHeader(Mod) {
2750 assert((AK == None || Mod || AK == Failure) &&
2751 "no module for module action");
2752 }
2753 };
2754
2755 OptionalFileEntryRef LookupHeaderIncludeOrImport(
2756 ConstSearchDirIterator *CurDir, StringRef &Filename,
2757 SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2758 const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
2759 bool &IsMapped, ConstSearchDirIterator LookupFrom,
2760 const FileEntry *LookupFromFile, StringRef &LookupFilename,
2761 SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2762 ModuleMap::KnownHeader &SuggestedModule, bool isAngled);
2763 // Binary data inclusion
2764 void HandleEmbedDirective(SourceLocation HashLoc, Token &Tok,
2765 const FileEntry *LookupFromFile = nullptr);
2766 void HandleEmbedDirectiveImpl(SourceLocation HashLoc,
2767 const LexEmbedParametersResult &Params,
2768 StringRef BinaryContents);
2769
2770 // File inclusion.
2771 void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2772 ConstSearchDirIterator LookupFrom = nullptr,
2773 const FileEntry *LookupFromFile = nullptr);
2774 ImportAction
2775 HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2776 Token &FilenameTok, SourceLocation EndLoc,
2777 ConstSearchDirIterator LookupFrom = nullptr,
2778 const FileEntry *LookupFromFile = nullptr);
2779 void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2780 void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2781 void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2782 void HandleMicrosoftImportDirective(Token &Tok);
2783
2784public:
2785 /// Check that the given module is available, producing a diagnostic if not.
2786 /// \return \c true if the check failed (because the module is not available).
2787 /// \c false if the module appears to be usable.
2788 static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2789 const TargetInfo &TargetInfo,
2790 const Module &M, DiagnosticsEngine &Diags);
2791
2792 // Module inclusion testing.
2793 /// Find the module that owns the source or header file that
2794 /// \p Loc points to. If the location is in a file that was included
2795 /// into a module, or is outside any module, returns nullptr.
2796 Module *getModuleForLocation(SourceLocation Loc, bool AllowTextual);
2797
2798 /// We want to produce a diagnostic at location IncLoc concerning an
2799 /// unreachable effect at location MLoc (eg, where a desired entity was
2800 /// declared or defined). Determine whether the right way to make MLoc
2801 /// reachable is by #include, and if so, what header should be included.
2802 ///
2803 /// This is not necessarily fast, and might load unexpected module maps, so
2804 /// should only be called by code that intends to produce an error.
2805 ///
2806 /// \param IncLoc The location at which the missing effect was detected.
2807 /// \param MLoc A location within an unimported module at which the desired
2808 /// effect occurred.
2809 /// \return A file that can be #included to provide the desired effect. Null
2810 /// if no such file could be determined or if a #include is not
2811 /// appropriate (eg, if a module should be imported instead).
2813 SourceLocation MLoc);
2814
2815 bool isRecordingPreamble() const {
2816 return PreambleConditionalStack.isRecording();
2817 }
2818
2819 bool hasRecordedPreamble() const {
2820 return PreambleConditionalStack.hasRecordedPreamble();
2821 }
2822
2824 return PreambleConditionalStack.getStack();
2825 }
2826
2828 PreambleConditionalStack.setStack(s);
2829 }
2830
2832 ArrayRef<PPConditionalInfo> s, std::optional<PreambleSkipInfo> SkipInfo) {
2833 PreambleConditionalStack.startReplaying();
2834 PreambleConditionalStack.setStack(s);
2835 PreambleConditionalStack.SkipInfo = SkipInfo;
2836 }
2837
2838 std::optional<PreambleSkipInfo> getPreambleSkipInfo() const {
2839 return PreambleConditionalStack.SkipInfo;
2840 }
2841
2842private:
2843 /// After processing predefined file, initialize the conditional stack from
2844 /// the preamble.
2845 void replayPreambleConditionalStack();
2846
2847 // Macro handling.
2848 void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2849 void HandleUndefDirective();
2850
2851 // Conditional Inclusion.
2852 void HandleIfdefDirective(Token &Result, const Token &HashToken,
2853 bool isIfndef, bool ReadAnyTokensBeforeDirective);
2854 void HandleIfDirective(Token &IfToken, const Token &HashToken,
2855 bool ReadAnyTokensBeforeDirective);
2856 void HandleEndifDirective(Token &EndifToken);
2857 void HandleElseDirective(Token &Result, const Token &HashToken);
2858 void HandleElifFamilyDirective(Token &ElifToken, const Token &HashToken,
2860
2861 // Pragmas.
2862 void HandlePragmaDirective(PragmaIntroducer Introducer);
2863
2864public:
2865 void HandlePragmaOnce(Token &OnceTok);
2866 void HandlePragmaMark(Token &MarkTok);
2867 void HandlePragmaPoison();
2868 void HandlePragmaSystemHeader(Token &SysHeaderTok);
2869 void HandlePragmaDependency(Token &DependencyTok);
2870 void HandlePragmaPushMacro(Token &Tok);
2871 void HandlePragmaPopMacro(Token &Tok);
2873 void HandlePragmaModuleBuild(Token &Tok);
2874 void HandlePragmaHdrstop(Token &Tok);
2876
2877 // Return true and store the first token only if any CommentHandler
2878 // has inserted some tokens and getCommentRetentionState() is false.
2879 bool HandleComment(Token &result, SourceRange Comment);
2880
2881 /// A macro is used, update information about macros that need unused
2882 /// warnings.
2883 void markMacroAsUsed(MacroInfo *MI);
2884
2885 void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg,
2886 SourceLocation AnnotationLoc) {
2887 auto Annotations = AnnotationInfos.find(II);
2888 if (Annotations == AnnotationInfos.end())
2889 AnnotationInfos.insert(std::make_pair(
2890 II,
2891 MacroAnnotations::makeDeprecation(AnnotationLoc, std::move(Msg))));
2892 else
2893 Annotations->second.DeprecationInfo =
2894 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2895 }
2896
2897 void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg,
2898 SourceLocation AnnotationLoc) {
2899 auto Annotations = AnnotationInfos.find(II);
2900 if (Annotations == AnnotationInfos.end())
2901 AnnotationInfos.insert(
2902 std::make_pair(II, MacroAnnotations::makeRestrictExpansion(
2903 AnnotationLoc, std::move(Msg))));
2904 else
2905 Annotations->second.RestrictExpansionInfo =
2906 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2907 }
2908
2909 void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc) {
2910 auto Annotations = AnnotationInfos.find(II);
2911 if (Annotations == AnnotationInfos.end())
2912 AnnotationInfos.insert(
2913 std::make_pair(II, MacroAnnotations::makeFinal(AnnotationLoc)));
2914 else
2915 Annotations->second.FinalAnnotationLoc = AnnotationLoc;
2916 }
2917
2918 const MacroAnnotations &getMacroAnnotations(const IdentifierInfo *II) const {
2919 return AnnotationInfos.find(II)->second;
2920 }
2921
2923 bool IsIfnDef = false) const {
2924 IdentifierInfo *Info = Identifier.getIdentifierInfo();
2925 if (Info->isDeprecatedMacro())
2926 emitMacroDeprecationWarning(Identifier);
2927
2928 if (Info->isRestrictExpansion() &&
2929 !SourceMgr.isInMainFile(Identifier.getLocation()))
2930 emitRestrictExpansionWarning(Identifier);
2931
2932 if (!IsIfnDef) {
2933 if (Info->getName() == "INFINITY" && getLangOpts().NoHonorInfs)
2934 emitRestrictInfNaNWarning(Identifier, 0);
2935 if (Info->getName() == "NAN" && getLangOpts().NoHonorNaNs)
2936 emitRestrictInfNaNWarning(Identifier, 1);
2937 }
2938 }
2939
2941 const LangOptions &LangOpts,
2942 const TargetInfo &TI);
2943
2945 const PresumedLoc &PLoc,
2946 const LangOptions &LangOpts,
2947 const TargetInfo &TI);
2948
2949private:
2950 void emitMacroDeprecationWarning(const Token &Identifier) const;
2951 void emitRestrictExpansionWarning(const Token &Identifier) const;
2952 void emitFinalMacroWarning(const Token &Identifier, bool IsUndef) const;
2953 void emitRestrictInfNaNWarning(const Token &Identifier,
2954 unsigned DiagSelection) const;
2955
2956 /// This boolean state keeps track if the current scanned token (by this PP)
2957 /// is in an "-Wunsafe-buffer-usage" opt-out region. Assuming PP scans a
2958 /// translation unit in a linear order.
2959 bool InSafeBufferOptOutRegion = false;
2960
2961 /// Hold the start location of the current "-Wunsafe-buffer-usage" opt-out
2962 /// region if PP is currently in such a region. Hold undefined value
2963 /// otherwise.
2964 SourceLocation CurrentSafeBufferOptOutStart; // It is used to report the start location of an never-closed region.
2965
2966 using SafeBufferOptOutRegionsTy =
2968 // An ordered sequence of "-Wunsafe-buffer-usage" opt-out regions in this
2969 // translation unit. Each region is represented by a pair of start and
2970 // end locations.
2971 SafeBufferOptOutRegionsTy SafeBufferOptOutMap;
2972
2973 // The "-Wunsafe-buffer-usage" opt-out regions in loaded ASTs. We use the
2974 // following structure to manage them by their ASTs.
2975 struct {
2976 // A map from unique IDs to region maps of loaded ASTs. The ID identifies a
2977 // loaded AST. See `SourceManager::getUniqueLoadedASTID`.
2978 llvm::DenseMap<FileID, SafeBufferOptOutRegionsTy> LoadedRegions;
2979
2980 // Returns a reference to the safe buffer opt-out regions of the loaded
2981 // AST where `Loc` belongs to. (Construct if absent)
2983 findAndConsLoadedOptOutMap(SourceLocation Loc, SourceManager &SrcMgr) {
2985 }
2986
2987 // Returns a reference to the safe buffer opt-out regions of the loaded
2988 // AST where `Loc` belongs to. (This const function returns nullptr if
2989 // absent.)
2990 const SafeBufferOptOutRegionsTy *
2991 lookupLoadedOptOutMap(SourceLocation Loc,
2992 const SourceManager &SrcMgr) const {
2993 FileID FID = SrcMgr.getUniqueLoadedASTFileID(Loc);
2994 auto Iter = LoadedRegions.find(FID);
2995
2996 if (Iter == LoadedRegions.end())
2997 return nullptr;
2998 return &Iter->getSecond();
2999 }
3000 } LoadedSafeBufferOptOutMap;
3001
3002public:
3003 /// \return true iff the given `Loc` is in a "-Wunsafe-buffer-usage" opt-out
3004 /// region. This `Loc` must be a source location that has been pre-processed.
3005 bool isSafeBufferOptOut(const SourceManager&SourceMgr, const SourceLocation &Loc) const;
3006
3007 /// Alter the state of whether this PP currently is in a
3008 /// "-Wunsafe-buffer-usage" opt-out region.
3009 ///
3010 /// \param isEnter true if this PP is entering a region; otherwise, this PP
3011 /// is exiting a region
3012 /// \param Loc the location of the entry or exit of a
3013 /// region
3014 /// \return true iff it is INVALID to enter or exit a region, i.e.,
3015 /// attempt to enter a region before exiting a previous region, or exiting a
3016 /// region that PP is not currently in.
3017 bool enterOrExitSafeBufferOptOutRegion(bool isEnter,
3018 const SourceLocation &Loc);
3019
3020 /// \return true iff this PP is currently in a "-Wunsafe-buffer-usage"
3021 /// opt-out region
3023
3024 /// \param StartLoc output argument. It will be set to the start location of
3025 /// the current "-Wunsafe-buffer-usage" opt-out region iff this function
3026 /// returns true.
3027 /// \return true iff this PP is currently in a "-Wunsafe-buffer-usage"
3028 /// opt-out region
3029 bool isPPInSafeBufferOptOutRegion(SourceLocation &StartLoc);
3030
3031 /// \return a sequence of SourceLocations representing ordered opt-out regions
3032 /// specified by
3033 /// `\#pragma clang unsafe_buffer_usage begin/end`s of this translation unit.
3034 SmallVector<SourceLocation, 64> serializeSafeBufferOptOutMap() const;
3035
3036 /// \param SrcLocSeqs a sequence of SourceLocations deserialized from a
3037 /// record of code `PP_UNSAFE_BUFFER_USAGE`.
3038 /// \return true iff the `Preprocessor` has been updated; false `Preprocessor`
3039 /// is same as itself before the call.
3041 const SmallVectorImpl<SourceLocation> &SrcLocSeqs);
3042
3043private:
3044 /// Helper functions to forward lexing to the actual lexer. They all share the
3045 /// same signature.
3046 static bool CLK_Lexer(Preprocessor &P, Token &Result) {
3047 return P.CurLexer->Lex(Result);
3048 }
3049 static bool CLK_TokenLexer(Preprocessor &P, Token &Result) {
3050 return P.CurTokenLexer->Lex(Result);
3051 }
3052 static bool CLK_CachingLexer(Preprocessor &P, Token &Result) {
3053 P.CachingLex(Result);
3054 return true;
3055 }
3056 static bool CLK_DependencyDirectivesLexer(Preprocessor &P, Token &Result) {
3057 return P.CurLexer->LexDependencyDirectiveToken(Result);
3058 }
3059 static bool CLK_LexAfterModuleImport(Preprocessor &P, Token &Result) {
3060 return P.LexAfterModuleImport(Result);
3061 }
3062};
3063
3064/// Abstract base class that describes a handler that will receive
3065/// source ranges for each of the comments encountered in the source file.
3067public:
3069
3070 // The handler shall return true if it has pushed any tokens
3071 // to be read using e.g. EnterToken or EnterTokenStream.
3072 virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
3073};
3074
3075/// Abstract base class that describes a handler that will receive
3076/// source ranges for empty lines encountered in the source file.
3078public:
3080
3081 // The handler handles empty lines.
3083};
3084
3085/// Helper class to shuttle information about #embed directives from the
3086/// preprocessor to the parser through an annotation token.
3088 StringRef BinaryData;
3089};
3090
3091/// Registry of pragma handlers added by plugins
3092using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
3093
3094} // namespace clang
3095
3096#endif // LLVM_CLANG_LEX_PREPROCESSOR_H
#define V(N, I)
Definition: ASTContext.h:3341
StringRef P
#define SM(sm)
Definition: Cuda.cpp:83
Defines the Diagnostic-related interfaces.
static constexpr Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:32
const Decl * D
IndirectLocalPath & Path
enum clang::sema::@1658::IndirectLocalPathEntry::EntryKind Kind
Defines the Diagnostic IDs-related interfaces.
StringRef Filename
Definition: Format.cpp:3001
StringRef Identifier
Definition: Format.cpp:3009
unsigned Iter
Definition: HTMLLogger.cpp:154
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
llvm::MachO::Target Target
Definition: MachO.h:51
llvm::MachO::Record Record
Definition: MachO.h:31
Defines the clang::MacroInfo and clang::MacroDirective classes.
Defines the clang::Module class, which describes a module in the source code.
Defines the PPCallbacks interface.
uint32_t Id
Definition: SemaARM.cpp:1143
SourceRange Range
Definition: SemaObjC.cpp:757
SourceLocation Loc
Definition: SemaObjC.cpp:758
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
Defines the clang::TokenKind enum and support functions.
__device__ __2f16 float __ockl_bool s
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:378
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:85
Callback handler that receives notifications when performing code completion within the preprocessor.
Abstract base class that describes a handler that will receive source ranges for each of the comments...
virtual bool HandleComment(Preprocessor &PP, SourceRange Comment)=0
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:432
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:1271
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1547
void setSuppressAllDiagnostics(bool Val)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
Definition: Diagnostic.h:697
A reference to a DirectoryEntry that includes the name of the directory as it was accessed by the Fil...
Abstract base class that describes a handler that will receive source ranges for empty lines encounte...
virtual void HandleEmptyline(SourceRange Range)=0
Abstract interface for external sources of preprocessor information.
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
Definition: FileEntry.h:57
Cached information about one file (either on disk or in the virtual file system).
Definition: FileEntry.h:300
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Implements support for file system lookup, file system caching, and directory search management.
Definition: FileManager.h:53
Encapsulates the information needed to find the file referenced by a #include or #include_next,...
Definition: HeaderSearch.h:249
HeaderFileInfo & getFileInfo(FileEntryRef FE)
Return the HeaderFileInfo structure for the specified FileEntry, in preparation for updating it in so...
One of these records is kept for each identifier that is lexed.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
bool isDeprecatedMacro() const
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
StringRef getName() const
Return the actual identifier string.
bool isRestrictExpansion() const
Implements an efficient mapping from strings to IdentifierInfo nodes.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
FPEvalMethodKind
Possible float expression evaluation method choices.
Definition: LangOptions.h:288
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
Definition: LangOptions.h:302
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:461
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
Definition: Lexer.cpp:1060
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion.
Definition: Lexer.cpp:872
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Characters, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token,...
Definition: Lexer.h:399
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
Definition: Lexer.cpp:894
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer,...
Definition: Lexer.cpp:452
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
Definition: Lexer.cpp:510
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
Definition: Lexer.cpp:850
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
A description of the current definition of a macro.
Definition: MacroInfo.h:590
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:375
Encapsulates changes to the "macros namespace" (the location where the macro name became active,...
Definition: MacroInfo.h:313
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:125
Abstract interface for a module loader.
Definition: ModuleLoader.h:82
Represents a macro directive exported by a module.
Definition: MacroInfo.h:514
A header that is known to reside within a given module, whether it was included or excluded.
Definition: ModuleMap.h:159
Describes a module or submodule.
Definition: Module.h:105
bool isModuleMapModule() const
Definition: Module.h:212
This interface provides a way to observe the actions of the preprocessor as it does its thing.
Definition: PPCallbacks.h:36
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:65
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:137
SourceLocation getLastFPEvalPragmaLocation() const
bool isMacroDefined(const IdentifierInfo *II)
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
bool markIncluded(FileEntryRef File)
Mark the file as included.
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
Definition: Pragma.cpp:627
void FinalizeForModelFile()
Cleanup after model file parsing.
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void HandlePragmaPoison()
HandlePragmaPoison - Handle #pragma GCC poison. PoisonTok is the 'poison'.
Definition: Pragma.cpp:442
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void dumpMacroInfo(const IdentifierInfo *II)
std::pair< IdentifierInfo *, SourceLocation > getPragmaARCCFCodeAuditedInfo() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaSystemHeader(Token &SysHeaderTok)
HandlePragmaSystemHeader - Implement #pragma GCC system_header.
Definition: Pragma.cpp:484
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
void markClangModuleAsAffecting(Module *M)
Mark the given clang module as affecting the current clang module or translation unit.
void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident, SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaModuleBuild(Token &Tok)
Definition: Pragma.cpp:807
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
OptionalFileEntryRef LookupEmbedFile(StringRef Filename, bool isAngled, bool OpenFile, const FileEntry *LookupFromFile=nullptr)
Given a "Filename" or <Filename> reference, look up the indicated embed resource.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
Definition: Pragma.cpp:2196
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
PPCallbacks * getPPCallbacks() const
bool isInNamedInterfaceUnit() const
If we are proprocessing a named interface unit.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
void setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc)
Record the location of the unterminated #pragma clang assume_nonnull begin in the preamble.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
ArrayRef< BuildingSubmoduleInfo > getBuildingSubmodules() const
Get the list of submodules that we're currently building.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
SourceRange getCodeCompletionTokenRange() const
SourceLocation getModuleImportLoc(Module *M) const
void overrideMaxTokens(unsigned Value, SourceLocation Loc)
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
bool isRecordingPreamble() const
void HandleSkippedDirectiveWhileUsingPCH(Token &Result, SourceLocation HashLoc)
Process directives while skipping until the through header or #pragma hdrstop is found.
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
bool GetSuppressIncludeNotFoundError()
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
LangOptions::FPEvalMethodKind getCurrentFPEvalMethod() const
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
bool isSafeBufferOptOut(const SourceManager &SourceMgr, const SourceLocation &Loc) const
void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
const char * getCheckPoint(FileID FID, const char *Start) const
Returns a pointer into the given file's buffer that's guaranteed to be between tokens.
void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc)
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
Definition: PPCaching.cpp:171
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
const TargetInfo * getAuxTargetInfo() const
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
Definition: PPCaching.cpp:56
friend class MacroArgs
Definition: Preprocessor.h:703
void DumpMacro(const MacroInfo &MI) const
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
void setDiagnostics(DiagnosticsEngine &D)
IncludedFilesSet & getIncludedFiles()
Get the set of included files.
friend void TokenLexer::ExpandFunctionArguments()
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool SetCodeCompletionPoint(FileEntryRef File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
StringRef getNamedModuleName() const
Get the named module name we're preprocessing.
void makeModuleVisible(Module *M, SourceLocation Loc)
unsigned getCounterValue() const
bool mightHavePendingAnnotationTokens()
Determine whether it's possible for a future call to Lex to produce an annotation token created by a ...
bool isInImportingCXXNamedModules() const
If we're importing a standard C++20 Named Modules.
void Lex(Token &Result)
Lex the next token for this preprocessor.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
Definition: Preprocessor.h:295
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
unsigned getNumDirectives() const
Retrieve the number of Directives that have been processed by the Preprocessor.
bool isInImplementationUnit() const
If we are implementing an implementation module unit.
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
void LexNonComment(Token &Result)
Lex a token.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
SourceRange DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found.
bool LexOnOffSwitch(tok::OnOffSwitch &Result)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD.
Definition: Pragma.cpp:968
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
void setMainFileDir(DirectoryEntryRef Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
const IdentifierTable & getIdentifierTable() const
void HandlePragmaDependency(Token &DependencyTok)
HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
Definition: Pragma.cpp:516
SourceLocation CheckEndOfDirective(const char *DirType, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
Definition: PPCaching.cpp:66
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
bool enterOrExitSafeBufferOptOutRegion(bool isEnter, const SourceLocation &Loc)
Alter the state of whether this PP currently is in a "-Wunsafe-buffer-usage" opt-out region.
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, std::optional< PreambleSkipInfo > SkipInfo)
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
uint8_t getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the value of constant as an unsign...
macro_iterator macro_end(bool IncludeExternalMacros=true) const
SourceManager & getSourceManager() const
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
std::optional< PreambleSkipInfo > getPreambleSkipInfo() const
void setPreprocessToken(bool Preprocess)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
void HandlePragmaOnce(Token &OnceTok)
HandlePragmaOnce - Handle #pragma once. OnceTok is the 'once'.
Definition: Pragma.cpp:417
EmptylineHandler * getEmptylineHandler() const
bool getCommentRetentionState() const
bool isMacroDefined(StringRef Id)
static bool checkModuleIsAvailable(const LangOptions &LangOpts, const TargetInfo &TargetInfo, const Module &M, DiagnosticsEngine &Diags)
Check that the given module is available, producing a diagnostic if not.
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool hasRecordedPreamble() const
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
MacroMap::const_iterator macro_iterator
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
bool isUnannotatedBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of unannotated tokens is on.
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
void RemovePragmaHandler(PragmaHandler *Handler)
bool isPPInSafeBufferOptOutRegion()
unsigned getTokenCount() const
Get the number of tokens processed so far.
unsigned getMaxTokens() const
Get the max number of tokens before issuing a -Wmax-tokens warning.
SourceLocation getMaxTokensOverrideLoc() const
bool hadModuleLoaderFatalFailure() const
static void processPathToFileName(SmallVectorImpl< char > &FileName, const PresumedLoc &PLoc, const LangOptions &LangOpts, const TargetInfo &TI)
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
const TargetInfo & getTargetInfo() const
FileManager & getFileManager() const
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void DumpLocation(SourceLocation Loc) const
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
bool alreadyIncluded(FileEntryRef File) const
Return true if this header has already been included.
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
Definition: Pragma.cpp:915
llvm::BumpPtrAllocator & getPreprocessorAllocator()
ModuleMacro * getModuleMacro(Module *Mod, const IdentifierInfo *II)
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleComment(Token &result, SourceRange Comment)
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Buffer)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
HeaderSearch & getHeaderSearchInfo() const
void emitMacroExpansionWarnings(const Token &Identifier, bool IsIfnDef=false) const
bool setDeserializedSafeBufferOptOutMap(const SmallVectorImpl< SourceLocation > &SrcLocSeqs)
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
Definition: Pragma.cpp:650
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
ExternalPreprocessorSource * getExternalSource() const
Module * LeaveSubmodule(bool ForPragma)
const std::string & getPredefines() const
Get the predefines for this processor.
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
SmallVector< SourceLocation, 64 > serializeSafeBufferOptOutMap() const
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind, void *AnnotationVal)
Enter an annotation token into the token stream.
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, const LangOptions &LangOpts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
MacroInfo * getMacroInfo(const IdentifierInfo *II)
void setPredefines(std::string P)
Set the predefines for this Preprocessor.
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
IdentifierTable & getIdentifierTable()
Builtin::Context & getBuiltinInfo()
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
Definition: PPCaching.cpp:189
LangOptions::FPEvalMethodKind getTUFPEvalMethod() const
const LangOptions & getLangOpts() const
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void SetSuppressIncludeNotFoundError(bool Suppress)
static void processPathForFileMacro(SmallVectorImpl< char > &Path, const LangOptions &LangOpts, const TargetInfo &TI)
llvm::DenseMap< FileID, SafeBufferOptOutRegionsTy > LoadedRegions
bool isInNamedModule() const
If we are preprocessing a named module.
void EnableBacktrackAtThisPos(bool Unannotated=false)
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
Definition: PPCaching.cpp:34
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
void PoisonSEHIdentifiers(bool Poison=true)
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion.
size_t getTotalMemory() const
void setExternalSource(ExternalPreprocessorSource *Source)
void clearCodeCompletionHandler()
Clear out the code completion handler.
void AddPragmaHandler(PragmaHandler *Handler)
OptionalFileEntryRef getHeaderToIncludeForDiagnostics(SourceLocation IncLoc, SourceLocation MLoc)
We want to produce a diagnostic at location IncLoc concerning an unreachable effect at location MLoc ...
void setCounterValue(unsigned V)
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.
Definition: Pragma.cpp:572
void LexTokensUntilEOF(std::vector< Token > *Tokens=nullptr)
Lex all tokens for this preprocessor until (and excluding) end of file.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
void setPragmasEnabled(bool Enabled)
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void HandlePragmaMark(Token &MarkTok)
Definition: Pragma.cpp:432
bool getPragmasEnabled() const
void HandlePragmaHdrstop(Token &Tok)
Definition: Pragma.cpp:881
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
void setEmptylineHandler(EmptylineHandler *Handler)
Set empty line handler.
DiagnosticsEngine & getDiagnostics() const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
llvm::DenseSet< const FileEntry * > IncludedFilesSet
Definition: Preprocessor.h:699
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SelectorTable & getSelectorTable()
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
Definition: Pragma.cpp:946
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
const llvm::SmallSetVector< Module *, 2 > & getAffectingClangModules() const
Get the set of top-level clang modules that affected preprocessing, but were not imported.
std::optional< LexEmbedParametersResult > LexEmbedParameters(Token &Current, bool ForHasEmbed)
Lex the parameters for an #embed directive, returns nullopt on error.
const IncludedFilesSet & getIncludedFiles() const
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void HandlePragmaIncludeAlias(Token &Tok)
Definition: Pragma.cpp:685
Module * getModuleForLocation(SourceLocation Loc, bool AllowTextual)
Find the module that owns the source or header file that Loc points to.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
SourceLocation getPreambleRecordedPragmaAssumeNonNullLoc() const
Get the location of the recorded unterminated #pragma clang assume_nonnull begin in the preamble,...
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop.
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
Represents an unpacked "presumed" location which can be presented to the user.
This table allows us to fully hide how we implement multi-keyword caching.
Encodes a location in the source.
This class handles loading and caching of source files into memory.
FileID getUniqueLoadedASTFileID(SourceLocation Loc) const
A trivial tuple used to represent a source range.
Exposes information about the current target.
Definition: TargetInfo.h:218
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:91
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:104
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:96
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:106
Token - This structure provides full information about a lexed token.
Definition: Token.h:36
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:187
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:132
unsigned getLength() const
Definition: Token.h:135
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:99
tok::TokenKind getKind() const
Definition: Token.h:94
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:121
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:295
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:225
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a variadic mac...
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
Directive - Abstract class representing a parsed verify directive.
constexpr XRayInstrMask None
Definition: XRayInstr.h:38
StringRef getName(const HeaderType T)
Definition: HeaderFile.h:38
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:56
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:97
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a '#' at the beginning of the line.
Definition: TokenKinds.h:33
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.cpp:58
@ HeaderSearch
Remove unused header search paths including header maps.
The JSON file list parser is used to communicate input to InstallAPI.
CustomizableOptional< DirectoryEntryRef > OptionalDirectoryEntryRef
detail::SearchDirIteratorImpl< true > ConstSearchDirIterator
Definition: HeaderSearch.h:241
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:113
@ MU_Undef
Definition: Preprocessor.h:121
@ MU_Other
Definition: Preprocessor.h:115
@ MU_Define
Definition: Preprocessor.h:118
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
@ Module
Module linkage, which indicates that the entity can be referred to from other translation units withi...
@ Result
The result type of a method or function.
CustomizableOptional< FileEntryRef > OptionalFileEntryRef
Definition: FileEntry.h:202
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:1037
@ TU_Complete
The translation unit is a complete translation unit.
Definition: LangOptions.h:1039
Diagnostic wrappers for TextAPI types for error reporting.
Definition: Dominators.h:30
#define bool
Definition: stdbool.h:24
Helper class to shuttle information about #embed directives from the preprocessor to the parser throu...
Describes how and where the pragma was introduced.
Definition: Pragma.h:51
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
Definition: Preprocessor.h:691