clang 19.0.0git
Preprocessor.h
Go to the documentation of this file.
1//===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Defines the clang::Preprocessor interface.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15#define LLVM_CLANG_LEX_PREPROCESSOR_H
16
20#include "clang/Basic/LLVM.h"
22#include "clang/Basic/Module.h"
27#include "clang/Lex/Lexer.h"
28#include "clang/Lex/MacroInfo.h"
30#include "clang/Lex/ModuleMap.h"
33#include "clang/Lex/Token.h"
35#include "llvm/ADT/APSInt.h"
36#include "llvm/ADT/ArrayRef.h"
37#include "llvm/ADT/DenseMap.h"
38#include "llvm/ADT/FoldingSet.h"
39#include "llvm/ADT/FunctionExtras.h"
40#include "llvm/ADT/PointerUnion.h"
41#include "llvm/ADT/STLExtras.h"
42#include "llvm/ADT/SmallPtrSet.h"
43#include "llvm/ADT/SmallVector.h"
44#include "llvm/ADT/StringRef.h"
45#include "llvm/ADT/TinyPtrVector.h"
46#include "llvm/ADT/iterator_range.h"
47#include "llvm/Support/Allocator.h"
48#include "llvm/Support/Casting.h"
49#include "llvm/Support/Registry.h"
50#include <cassert>
51#include <cstddef>
52#include <cstdint>
53#include <map>
54#include <memory>
55#include <optional>
56#include <string>
57#include <utility>
58#include <vector>
59
60namespace llvm {
61
62template<unsigned InternalLen> class SmallString;
63
64} // namespace llvm
65
66namespace clang {
67
68class CodeCompletionHandler;
69class CommentHandler;
70class DirectoryEntry;
71class EmptylineHandler;
72class ExternalPreprocessorSource;
73class FileEntry;
74class FileManager;
75class HeaderSearch;
76class MacroArgs;
77class PragmaHandler;
78class PragmaNamespace;
79class PreprocessingRecord;
80class PreprocessorLexer;
81class PreprocessorOptions;
82class ScratchBuffer;
83class TargetInfo;
84
85namespace Builtin {
86class Context;
87}
88
89/// Stores token information for comparing actual tokens with
90/// predefined values. Only handles simple tokens and identifiers.
92 tok::TokenKind Kind;
94
95public:
96 TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
97 assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
98 assert(Kind != tok::identifier &&
99 "Identifiers should be created by TokenValue(IdentifierInfo *)");
100 assert(!tok::isLiteral(Kind) && "Literals are not supported.");
101 assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
102 }
103
104 TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
105
106 bool operator==(const Token &Tok) const {
107 return Tok.getKind() == Kind &&
108 (!II || II == Tok.getIdentifierInfo());
109 }
110};
111
112/// Context in which macro name is used.
114 // other than #define or #undef
116
117 // macro name specified in #define
119
120 // macro name specified in #undef
121 MU_Undef = 2
123
124enum class EmbedResult {
125 Invalid = -1, // Parsing error occurred.
126 NotFound = 0, // Corresponds to __STDC_EMBED_NOT_FOUND__
127 Found = 1, // Corresponds to __STDC_EMBED_FOUND__
128 Empty = 2, // Corresponds to __STDC_EMBED_EMPTY__
129};
130
131/// Engages in a tight little dance with the lexer to efficiently
132/// preprocess tokens.
133///
134/// Lexers know only about tokens within a single source file, and don't
135/// know anything about preprocessor-level issues like the \#include stack,
136/// token expansion, etc.
140
141 llvm::unique_function<void(const clang::Token &)> OnToken;
142 std::shared_ptr<PreprocessorOptions> PPOpts;
143 DiagnosticsEngine *Diags;
144 const LangOptions &LangOpts;
145 const TargetInfo *Target = nullptr;
146 const TargetInfo *AuxTarget = nullptr;
147 FileManager &FileMgr;
148 SourceManager &SourceMgr;
149 std::unique_ptr<ScratchBuffer> ScratchBuf;
150 HeaderSearch &HeaderInfo;
151 ModuleLoader &TheModuleLoader;
152
153 /// External source of macros.
155
156 /// A BumpPtrAllocator object used to quickly allocate and release
157 /// objects internal to the Preprocessor.
158 llvm::BumpPtrAllocator BP;
159
160 /// Identifiers for builtin macros and other builtins.
161 IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
162 IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
163 IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
164 IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
165 IdentifierInfo *Ident__FILE_NAME__; // __FILE_NAME__
166 IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
167 IdentifierInfo *Ident__COUNTER__; // __COUNTER__
168 IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
169 IdentifierInfo *Ident__identifier; // __identifier
170 IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
171 IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
172 IdentifierInfo *Ident__has_feature; // __has_feature
173 IdentifierInfo *Ident__has_extension; // __has_extension
174 IdentifierInfo *Ident__has_builtin; // __has_builtin
175 IdentifierInfo *Ident__has_constexpr_builtin; // __has_constexpr_builtin
176 IdentifierInfo *Ident__has_attribute; // __has_attribute
177 IdentifierInfo *Ident__has_embed; // __has_embed
178 IdentifierInfo *Ident__has_include; // __has_include
179 IdentifierInfo *Ident__has_include_next; // __has_include_next
180 IdentifierInfo *Ident__has_warning; // __has_warning
181 IdentifierInfo *Ident__is_identifier; // __is_identifier
182 IdentifierInfo *Ident__building_module; // __building_module
183 IdentifierInfo *Ident__MODULE__; // __MODULE__
184 IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
185 IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
186 IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
187 IdentifierInfo *Ident__is_target_arch; // __is_target_arch
188 IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
189 IdentifierInfo *Ident__is_target_os; // __is_target_os
190 IdentifierInfo *Ident__is_target_environment; // __is_target_environment
191 IdentifierInfo *Ident__is_target_variant_os;
192 IdentifierInfo *Ident__is_target_variant_environment;
193 IdentifierInfo *Ident__FLT_EVAL_METHOD__; // __FLT_EVAL_METHOD
194
195 // Weak, only valid (and set) while InMacroArgs is true.
196 Token* ArgMacro;
197
198 SourceLocation DATELoc, TIMELoc;
199
200 // FEM_UnsetOnCommandLine means that an explicit evaluation method was
201 // not specified on the command line. The target is queried to set the
202 // default evaluation method.
203 LangOptions::FPEvalMethodKind CurrentFPEvalMethod =
205
206 // The most recent pragma location where the floating point evaluation
207 // method was modified. This is used to determine whether the
208 // 'pragma clang fp eval_method' was used whithin the current scope.
209 SourceLocation LastFPEvalPragmaLocation;
210
211 LangOptions::FPEvalMethodKind TUFPEvalMethod =
213
214 // Next __COUNTER__ value, starts at 0.
215 unsigned CounterValue = 0;
216
217 enum {
218 /// Maximum depth of \#includes.
219 MaxAllowedIncludeStackDepth = 200
220 };
221
222 // State that is set before the preprocessor begins.
223 bool KeepComments : 1;
224 bool KeepMacroComments : 1;
225 bool SuppressIncludeNotFoundError : 1;
226
227 // State that changes while the preprocessor runs:
228 bool InMacroArgs : 1; // True if parsing fn macro invocation args.
229
230 /// Whether the preprocessor owns the header search object.
231 bool OwnsHeaderSearch : 1;
232
233 /// True if macro expansion is disabled.
234 bool DisableMacroExpansion : 1;
235
236 /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
237 /// when parsing preprocessor directives.
238 bool MacroExpansionInDirectivesOverride : 1;
239
240 class ResetMacroExpansionHelper;
241
242 /// Whether we have already loaded macros from the external source.
243 mutable bool ReadMacrosFromExternalSource : 1;
244
245 /// True if pragmas are enabled.
246 bool PragmasEnabled : 1;
247
248 /// True if the current build action is a preprocessing action.
249 bool PreprocessedOutput : 1;
250
251 /// True if we are currently preprocessing a #if or #elif directive
252 bool ParsingIfOrElifDirective;
253
254 /// True if we are pre-expanding macro arguments.
255 bool InMacroArgPreExpansion;
256
257 /// Mapping/lookup information for all identifiers in
258 /// the program, including program keywords.
259 mutable IdentifierTable Identifiers;
260
261 /// This table contains all the selectors in the program.
262 ///
263 /// Unlike IdentifierTable above, this table *isn't* populated by the
264 /// preprocessor. It is declared/expanded here because its role/lifetime is
265 /// conceptually similar to the IdentifierTable. In addition, the current
266 /// control flow (in clang::ParseAST()), make it convenient to put here.
267 ///
268 /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
269 /// the lifetime of the preprocessor.
270 SelectorTable Selectors;
271
272 /// Information about builtins.
273 std::unique_ptr<Builtin::Context> BuiltinInfo;
274
275 /// Tracks all of the pragmas that the client registered
276 /// with this preprocessor.
277 std::unique_ptr<PragmaNamespace> PragmaHandlers;
278
279 /// Pragma handlers of the original source is stored here during the
280 /// parsing of a model file.
281 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
282
283 /// Tracks all of the comment handlers that the client registered
284 /// with this preprocessor.
285 std::vector<CommentHandler *> CommentHandlers;
286
287 /// Empty line handler.
288 EmptylineHandler *Emptyline = nullptr;
289
290 /// True to avoid tearing down the lexer etc on EOF
291 bool IncrementalProcessing = false;
292
293public:
294 /// The kind of translation unit we are processing.
296
297 /// Returns a pointer into the given file's buffer that's guaranteed
298 /// to be between tokens. The returned pointer is always before \p Start.
299 /// The maximum distance betweenthe returned pointer and \p Start is
300 /// limited by a constant value, but also an implementation detail.
301 /// If no such check point exists, \c nullptr is returned.
302 const char *getCheckPoint(FileID FID, const char *Start) const;
303
304private:
305 /// The code-completion handler.
306 CodeCompletionHandler *CodeComplete = nullptr;
307
308 /// The file that we're performing code-completion for, if any.
309 const FileEntry *CodeCompletionFile = nullptr;
310
311 /// The offset in file for the code-completion point.
312 unsigned CodeCompletionOffset = 0;
313
314 /// The location for the code-completion point. This gets instantiated
315 /// when the CodeCompletionFile gets \#include'ed for preprocessing.
316 SourceLocation CodeCompletionLoc;
317
318 /// The start location for the file of the code-completion point.
319 ///
320 /// This gets instantiated when the CodeCompletionFile gets \#include'ed
321 /// for preprocessing.
322 SourceLocation CodeCompletionFileLoc;
323
324 /// The source location of the \c import contextual keyword we just
325 /// lexed, if any.
326 SourceLocation ModuleImportLoc;
327
328 /// The import path for named module that we're currently processing.
330
331 llvm::DenseMap<FileID, SmallVector<const char *>> CheckPoints;
332 unsigned CheckPointCounter = 0;
333
334 /// Whether the import is an `@import` or a standard c++ modules import.
335 bool IsAtImport = false;
336
337 /// Whether the last token we lexed was an '@'.
338 bool LastTokenWasAt = false;
339
340 /// A position within a C++20 import-seq.
341 class StdCXXImportSeq {
342 public:
343 enum State : int {
344 // Positive values represent a number of unclosed brackets.
345 AtTopLevel = 0,
346 AfterTopLevelTokenSeq = -1,
347 AfterExport = -2,
348 AfterImportSeq = -3,
349 };
350
351 StdCXXImportSeq(State S) : S(S) {}
352
353 /// Saw any kind of open bracket.
354 void handleOpenBracket() {
355 S = static_cast<State>(std::max<int>(S, 0) + 1);
356 }
357 /// Saw any kind of close bracket other than '}'.
358 void handleCloseBracket() {
359 S = static_cast<State>(std::max<int>(S, 1) - 1);
360 }
361 /// Saw a close brace.
362 void handleCloseBrace() {
363 handleCloseBracket();
364 if (S == AtTopLevel && !AfterHeaderName)
365 S = AfterTopLevelTokenSeq;
366 }
367 /// Saw a semicolon.
368 void handleSemi() {
369 if (atTopLevel()) {
370 S = AfterTopLevelTokenSeq;
371 AfterHeaderName = false;
372 }
373 }
374
375 /// Saw an 'export' identifier.
376 void handleExport() {
377 if (S == AfterTopLevelTokenSeq)
378 S = AfterExport;
379 else if (S <= 0)
380 S = AtTopLevel;
381 }
382 /// Saw an 'import' identifier.
383 void handleImport() {
384 if (S == AfterTopLevelTokenSeq || S == AfterExport)
385 S = AfterImportSeq;
386 else if (S <= 0)
387 S = AtTopLevel;
388 }
389
390 /// Saw a 'header-name' token; do not recognize any more 'import' tokens
391 /// until we reach a top-level semicolon.
392 void handleHeaderName() {
393 if (S == AfterImportSeq)
394 AfterHeaderName = true;
395 handleMisc();
396 }
397
398 /// Saw any other token.
399 void handleMisc() {
400 if (S <= 0)
401 S = AtTopLevel;
402 }
403
404 bool atTopLevel() { return S <= 0; }
405 bool afterImportSeq() { return S == AfterImportSeq; }
406 bool afterTopLevelSeq() { return S == AfterTopLevelTokenSeq; }
407
408 private:
409 State S;
410 /// Whether we're in the pp-import-suffix following the header-name in a
411 /// pp-import. If so, a close-brace is not sufficient to end the
412 /// top-level-token-seq of an import-seq.
413 bool AfterHeaderName = false;
414 };
415
416 /// Our current position within a C++20 import-seq.
417 StdCXXImportSeq StdCXXImportSeqState = StdCXXImportSeq::AfterTopLevelTokenSeq;
418
419 /// Track whether we are in a Global Module Fragment
420 class TrackGMF {
421 public:
422 enum GMFState : int {
423 GMFActive = 1,
424 MaybeGMF = 0,
425 BeforeGMFIntroducer = -1,
426 GMFAbsentOrEnded = -2,
427 };
428
429 TrackGMF(GMFState S) : S(S) {}
430
431 /// Saw a semicolon.
432 void handleSemi() {
433 // If it is immediately after the first instance of the module keyword,
434 // then that introduces the GMF.
435 if (S == MaybeGMF)
436 S = GMFActive;
437 }
438
439 /// Saw an 'export' identifier.
440 void handleExport() {
441 // The presence of an 'export' keyword always ends or excludes a GMF.
442 S = GMFAbsentOrEnded;
443 }
444
445 /// Saw an 'import' identifier.
446 void handleImport(bool AfterTopLevelTokenSeq) {
447 // If we see this before any 'module' kw, then we have no GMF.
448 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
449 S = GMFAbsentOrEnded;
450 }
451
452 /// Saw a 'module' identifier.
453 void handleModule(bool AfterTopLevelTokenSeq) {
454 // This was the first module identifier and not preceded by any token
455 // that would exclude a GMF. It could begin a GMF, but only if directly
456 // followed by a semicolon.
457 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
458 S = MaybeGMF;
459 else
460 S = GMFAbsentOrEnded;
461 }
462
463 /// Saw any other token.
464 void handleMisc() {
465 // We saw something other than ; after the 'module' kw, so not a GMF.
466 if (S == MaybeGMF)
467 S = GMFAbsentOrEnded;
468 }
469
470 bool inGMF() { return S == GMFActive; }
471
472 private:
473 /// Track the transitions into and out of a Global Module Fragment,
474 /// if one is present.
475 GMFState S;
476 };
477
478 TrackGMF TrackGMFState = TrackGMF::BeforeGMFIntroducer;
479
480 /// Track the status of the c++20 module decl.
481 ///
482 /// module-declaration:
483 /// 'export'[opt] 'module' module-name module-partition[opt]
484 /// attribute-specifier-seq[opt] ';'
485 ///
486 /// module-name:
487 /// module-name-qualifier[opt] identifier
488 ///
489 /// module-partition:
490 /// ':' module-name-qualifier[opt] identifier
491 ///
492 /// module-name-qualifier:
493 /// identifier '.'
494 /// module-name-qualifier identifier '.'
495 ///
496 /// Transition state:
497 ///
498 /// NotAModuleDecl --- export ---> FoundExport
499 /// NotAModuleDecl --- module ---> ImplementationCandidate
500 /// FoundExport --- module ---> InterfaceCandidate
501 /// ImplementationCandidate --- Identifier ---> ImplementationCandidate
502 /// ImplementationCandidate --- period ---> ImplementationCandidate
503 /// ImplementationCandidate --- colon ---> ImplementationCandidate
504 /// InterfaceCandidate --- Identifier ---> InterfaceCandidate
505 /// InterfaceCandidate --- period ---> InterfaceCandidate
506 /// InterfaceCandidate --- colon ---> InterfaceCandidate
507 /// ImplementationCandidate --- Semi ---> NamedModuleImplementation
508 /// NamedModuleInterface --- Semi ---> NamedModuleInterface
509 /// NamedModuleImplementation --- Anything ---> NamedModuleImplementation
510 /// NamedModuleInterface --- Anything ---> NamedModuleInterface
511 ///
512 /// FIXME: We haven't handle attribute-specifier-seq here. It may not be bad
513 /// soon since we don't support any module attributes yet.
514 class ModuleDeclSeq {
515 enum ModuleDeclState : int {
516 NotAModuleDecl,
517 FoundExport,
518 InterfaceCandidate,
519 ImplementationCandidate,
520 NamedModuleInterface,
521 NamedModuleImplementation,
522 };
523
524 public:
525 ModuleDeclSeq() = default;
526
527 void handleExport() {
528 if (State == NotAModuleDecl)
529 State = FoundExport;
530 else if (!isNamedModule())
531 reset();
532 }
533
534 void handleModule() {
535 if (State == FoundExport)
536 State = InterfaceCandidate;
537 else if (State == NotAModuleDecl)
538 State = ImplementationCandidate;
539 else if (!isNamedModule())
540 reset();
541 }
542
543 void handleIdentifier(IdentifierInfo *Identifier) {
544 if (isModuleCandidate() && Identifier)
545 Name += Identifier->getName().str();
546 else if (!isNamedModule())
547 reset();
548 }
549
550 void handleColon() {
551 if (isModuleCandidate())
552 Name += ":";
553 else if (!isNamedModule())
554 reset();
555 }
556
557 void handlePeriod() {
558 if (isModuleCandidate())
559 Name += ".";
560 else if (!isNamedModule())
561 reset();
562 }
563
564 void handleSemi() {
565 if (!Name.empty() && isModuleCandidate()) {
566 if (State == InterfaceCandidate)
567 State = NamedModuleInterface;
568 else if (State == ImplementationCandidate)
569 State = NamedModuleImplementation;
570 else
571 llvm_unreachable("Unimaged ModuleDeclState.");
572 } else if (!isNamedModule())
573 reset();
574 }
575
576 void handleMisc() {
577 if (!isNamedModule())
578 reset();
579 }
580
581 bool isModuleCandidate() const {
582 return State == InterfaceCandidate || State == ImplementationCandidate;
583 }
584
585 bool isNamedModule() const {
586 return State == NamedModuleInterface ||
587 State == NamedModuleImplementation;
588 }
589
590 bool isNamedInterface() const { return State == NamedModuleInterface; }
591
592 bool isImplementationUnit() const {
593 return State == NamedModuleImplementation && !getName().contains(':');
594 }
595
596 StringRef getName() const {
597 assert(isNamedModule() && "Can't get name from a non named module");
598 return Name;
599 }
600
601 StringRef getPrimaryName() const {
602 assert(isNamedModule() && "Can't get name from a non named module");
603 return getName().split(':').first;
604 }
605
606 void reset() {
607 Name.clear();
608 State = NotAModuleDecl;
609 }
610
611 private:
612 ModuleDeclState State = NotAModuleDecl;
613 std::string Name;
614 };
615
616 ModuleDeclSeq ModuleDeclState;
617
618 /// Whether the module import expects an identifier next. Otherwise,
619 /// it expects a '.' or ';'.
620 bool ModuleImportExpectsIdentifier = false;
621
622 /// The identifier and source location of the currently-active
623 /// \#pragma clang arc_cf_code_audited begin.
624 std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
625
626 /// The source location of the currently-active
627 /// \#pragma clang assume_nonnull begin.
628 SourceLocation PragmaAssumeNonNullLoc;
629
630 /// Set only for preambles which end with an active
631 /// \#pragma clang assume_nonnull begin.
632 ///
633 /// When the preamble is loaded into the main file,
634 /// `PragmaAssumeNonNullLoc` will be set to this to
635 /// replay the unterminated assume_nonnull.
636 SourceLocation PreambleRecordedPragmaAssumeNonNullLoc;
637
638 /// True if we hit the code-completion point.
639 bool CodeCompletionReached = false;
640
641 /// The code completion token containing the information
642 /// on the stem that is to be code completed.
643 IdentifierInfo *CodeCompletionII = nullptr;
644
645 /// Range for the code completion token.
646 SourceRange CodeCompletionTokenRange;
647
648 /// The directory that the main file should be considered to occupy,
649 /// if it does not correspond to a real file (as happens when building a
650 /// module).
651 OptionalDirectoryEntryRef MainFileDir;
652
653 /// The number of bytes that we will initially skip when entering the
654 /// main file, along with a flag that indicates whether skipping this number
655 /// of bytes will place the lexer at the start of a line.
656 ///
657 /// This is used when loading a precompiled preamble.
658 std::pair<int, bool> SkipMainFilePreamble;
659
660 /// Whether we hit an error due to reaching max allowed include depth. Allows
661 /// to avoid hitting the same error over and over again.
662 bool HasReachedMaxIncludeDepth = false;
663
664 /// The number of currently-active calls to Lex.
665 ///
666 /// Lex is reentrant, and asking for an (end-of-phase-4) token can often
667 /// require asking for multiple additional tokens. This counter makes it
668 /// possible for Lex to detect whether it's producing a token for the end
669 /// of phase 4 of translation or for some other situation.
670 unsigned LexLevel = 0;
671
672 /// The number of (LexLevel 0) preprocessor tokens.
673 unsigned TokenCount = 0;
674
675 /// Preprocess every token regardless of LexLevel.
676 bool PreprocessToken = false;
677
678 /// The maximum number of (LexLevel 0) tokens before issuing a -Wmax-tokens
679 /// warning, or zero for unlimited.
680 unsigned MaxTokens = 0;
681 SourceLocation MaxTokensOverrideLoc;
682
683public:
690
696 ElseLoc(ElseLoc) {}
697 };
698
700
701private:
702 friend class ASTReader;
703 friend class MacroArgs;
704
705 class PreambleConditionalStackStore {
706 enum State {
707 Off = 0,
708 Recording = 1,
709 Replaying = 2,
710 };
711
712 public:
713 PreambleConditionalStackStore() = default;
714
715 void startRecording() { ConditionalStackState = Recording; }
716 void startReplaying() { ConditionalStackState = Replaying; }
717 bool isRecording() const { return ConditionalStackState == Recording; }
718 bool isReplaying() const { return ConditionalStackState == Replaying; }
719
720 ArrayRef<PPConditionalInfo> getStack() const {
721 return ConditionalStack;
722 }
723
724 void doneReplaying() {
725 ConditionalStack.clear();
726 ConditionalStackState = Off;
727 }
728
729 void setStack(ArrayRef<PPConditionalInfo> s) {
730 if (!isRecording() && !isReplaying())
731 return;
732 ConditionalStack.clear();
733 ConditionalStack.append(s.begin(), s.end());
734 }
735
736 bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
737
738 bool reachedEOFWhileSkipping() const { return SkipInfo.has_value(); }
739
740 void clearSkipInfo() { SkipInfo.reset(); }
741
742 std::optional<PreambleSkipInfo> SkipInfo;
743
744 private:
745 SmallVector<PPConditionalInfo, 4> ConditionalStack;
746 State ConditionalStackState = Off;
747 } PreambleConditionalStack;
748
749 /// The current top of the stack that we're lexing from if
750 /// not expanding a macro and we are lexing directly from source code.
751 ///
752 /// Only one of CurLexer, or CurTokenLexer will be non-null.
753 std::unique_ptr<Lexer> CurLexer;
754
755 /// The current top of the stack that we're lexing from
756 /// if not expanding a macro.
757 ///
758 /// This is an alias for CurLexer.
759 PreprocessorLexer *CurPPLexer = nullptr;
760
761 /// Used to find the current FileEntry, if CurLexer is non-null
762 /// and if applicable.
763 ///
764 /// This allows us to implement \#include_next and find directory-specific
765 /// properties.
766 ConstSearchDirIterator CurDirLookup = nullptr;
767
768 /// The current macro we are expanding, if we are expanding a macro.
769 ///
770 /// One of CurLexer and CurTokenLexer must be null.
771 std::unique_ptr<TokenLexer> CurTokenLexer;
772
773 /// The kind of lexer we're currently working with.
774 typedef bool (*LexerCallback)(Preprocessor &, Token &);
775 LexerCallback CurLexerCallback = &CLK_Lexer;
776
777 /// If the current lexer is for a submodule that is being built, this
778 /// is that submodule.
779 Module *CurLexerSubmodule = nullptr;
780
781 /// Keeps track of the stack of files currently
782 /// \#included, and macros currently being expanded from, not counting
783 /// CurLexer/CurTokenLexer.
784 struct IncludeStackInfo {
785 LexerCallback CurLexerCallback;
786 Module *TheSubmodule;
787 std::unique_ptr<Lexer> TheLexer;
788 PreprocessorLexer *ThePPLexer;
789 std::unique_ptr<TokenLexer> TheTokenLexer;
790 ConstSearchDirIterator TheDirLookup;
791
792 // The following constructors are completely useless copies of the default
793 // versions, only needed to pacify MSVC.
794 IncludeStackInfo(LexerCallback CurLexerCallback, Module *TheSubmodule,
795 std::unique_ptr<Lexer> &&TheLexer,
796 PreprocessorLexer *ThePPLexer,
797 std::unique_ptr<TokenLexer> &&TheTokenLexer,
798 ConstSearchDirIterator TheDirLookup)
799 : CurLexerCallback(std::move(CurLexerCallback)),
800 TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
801 ThePPLexer(std::move(ThePPLexer)),
802 TheTokenLexer(std::move(TheTokenLexer)),
803 TheDirLookup(std::move(TheDirLookup)) {}
804 };
805 std::vector<IncludeStackInfo> IncludeMacroStack;
806
807 /// Actions invoked when some preprocessor activity is
808 /// encountered (e.g. a file is \#included, etc).
809 std::unique_ptr<PPCallbacks> Callbacks;
810
811 struct MacroExpandsInfo {
812 Token Tok;
813 MacroDefinition MD;
814 SourceRange Range;
815
816 MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
817 : Tok(Tok), MD(MD), Range(Range) {}
818 };
819 SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
820
821 /// Information about a name that has been used to define a module macro.
822 struct ModuleMacroInfo {
823 /// The most recent macro directive for this identifier.
824 MacroDirective *MD;
825
826 /// The active module macros for this identifier.
827 llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
828
829 /// The generation number at which we last updated ActiveModuleMacros.
830 /// \see Preprocessor::VisibleModules.
831 unsigned ActiveModuleMacrosGeneration = 0;
832
833 /// Whether this macro name is ambiguous.
834 bool IsAmbiguous = false;
835
836 /// The module macros that are overridden by this macro.
837 llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
838
839 ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
840 };
841
842 /// The state of a macro for an identifier.
843 class MacroState {
844 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
845
846 ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
847 const IdentifierInfo *II) const {
848 if (II->isOutOfDate())
849 PP.updateOutOfDateIdentifier(*II);
850 // FIXME: Find a spare bit on IdentifierInfo and store a
851 // HasModuleMacros flag.
852 if (!II->hasMacroDefinition() ||
853 (!PP.getLangOpts().Modules &&
854 !PP.getLangOpts().ModulesLocalVisibility) ||
855 !PP.CurSubmoduleState->VisibleModules.getGeneration())
856 return nullptr;
857
858 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
859 if (!Info) {
860 Info = new (PP.getPreprocessorAllocator())
861 ModuleMacroInfo(State.get<MacroDirective *>());
862 State = Info;
863 }
864
865 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
866 Info->ActiveModuleMacrosGeneration)
867 PP.updateModuleMacroInfo(II, *Info);
868 return Info;
869 }
870
871 public:
872 MacroState() : MacroState(nullptr) {}
873 MacroState(MacroDirective *MD) : State(MD) {}
874
875 MacroState(MacroState &&O) noexcept : State(O.State) {
876 O.State = (MacroDirective *)nullptr;
877 }
878
879 MacroState &operator=(MacroState &&O) noexcept {
880 auto S = O.State;
881 O.State = (MacroDirective *)nullptr;
882 State = S;
883 return *this;
884 }
885
886 ~MacroState() {
887 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
888 Info->~ModuleMacroInfo();
889 }
890
891 MacroDirective *getLatest() const {
892 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
893 return Info->MD;
894 return State.get<MacroDirective*>();
895 }
896
897 void setLatest(MacroDirective *MD) {
898 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
899 Info->MD = MD;
900 else
901 State = MD;
902 }
903
904 bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
905 auto *Info = getModuleInfo(PP, II);
906 return Info ? Info->IsAmbiguous : false;
907 }
908
909 ArrayRef<ModuleMacro *>
910 getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
911 if (auto *Info = getModuleInfo(PP, II))
912 return Info->ActiveModuleMacros;
913 return std::nullopt;
914 }
915
916 MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
917 SourceManager &SourceMgr) const {
918 // FIXME: Incorporate module macros into the result of this.
919 if (auto *Latest = getLatest())
920 return Latest->findDirectiveAtLoc(Loc, SourceMgr);
921 return {};
922 }
923
924 void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
925 if (auto *Info = getModuleInfo(PP, II)) {
926 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
927 Info->ActiveModuleMacros.begin(),
928 Info->ActiveModuleMacros.end());
929 Info->ActiveModuleMacros.clear();
930 Info->IsAmbiguous = false;
931 }
932 }
933
934 ArrayRef<ModuleMacro*> getOverriddenMacros() const {
935 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
936 return Info->OverriddenMacros;
937 return std::nullopt;
938 }
939
940 void setOverriddenMacros(Preprocessor &PP,
941 ArrayRef<ModuleMacro *> Overrides) {
942 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
943 if (!Info) {
944 if (Overrides.empty())
945 return;
946 Info = new (PP.getPreprocessorAllocator())
947 ModuleMacroInfo(State.get<MacroDirective *>());
948 State = Info;
949 }
950 Info->OverriddenMacros.clear();
951 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
952 Overrides.begin(), Overrides.end());
953 Info->ActiveModuleMacrosGeneration = 0;
954 }
955 };
956
957 /// For each IdentifierInfo that was associated with a macro, we
958 /// keep a mapping to the history of all macro definitions and #undefs in
959 /// the reverse order (the latest one is in the head of the list).
960 ///
961 /// This mapping lives within the \p CurSubmoduleState.
962 using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
963
964 struct SubmoduleState;
965
966 /// Information about a submodule that we're currently building.
967 struct BuildingSubmoduleInfo {
968 /// The module that we are building.
969 Module *M;
970
971 /// The location at which the module was included.
972 SourceLocation ImportLoc;
973
974 /// Whether we entered this submodule via a pragma.
975 bool IsPragma;
976
977 /// The previous SubmoduleState.
978 SubmoduleState *OuterSubmoduleState;
979
980 /// The number of pending module macro names when we started building this.
981 unsigned OuterPendingModuleMacroNames;
982
983 BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
984 SubmoduleState *OuterSubmoduleState,
985 unsigned OuterPendingModuleMacroNames)
986 : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
987 OuterSubmoduleState(OuterSubmoduleState),
988 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
989 };
990 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
991
992 /// Information about a submodule's preprocessor state.
993 struct SubmoduleState {
994 /// The macros for the submodule.
995 MacroMap Macros;
996
997 /// The set of modules that are visible within the submodule.
998 VisibleModuleSet VisibleModules;
999
1000 // FIXME: CounterValue?
1001 // FIXME: PragmaPushMacroInfo?
1002 };
1003 std::map<Module *, SubmoduleState> Submodules;
1004
1005 /// The preprocessor state for preprocessing outside of any submodule.
1006 SubmoduleState NullSubmoduleState;
1007
1008 /// The current submodule state. Will be \p NullSubmoduleState if we're not
1009 /// in a submodule.
1010 SubmoduleState *CurSubmoduleState;
1011
1012 /// The files that have been included.
1013 IncludedFilesSet IncludedFiles;
1014
1015 /// The set of top-level modules that affected preprocessing, but were not
1016 /// imported.
1017 llvm::SmallSetVector<Module *, 2> AffectingClangModules;
1018
1019 /// The set of known macros exported from modules.
1020 llvm::FoldingSet<ModuleMacro> ModuleMacros;
1021
1022 /// The names of potential module macros that we've not yet processed.
1023 llvm::SmallVector<IdentifierInfo *, 32> PendingModuleMacroNames;
1024
1025 /// The list of module macros, for each identifier, that are not overridden by
1026 /// any other module macro.
1027 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
1028 LeafModuleMacros;
1029
1030 /// Macros that we want to warn because they are not used at the end
1031 /// of the translation unit.
1032 ///
1033 /// We store just their SourceLocations instead of
1034 /// something like MacroInfo*. The benefit of this is that when we are
1035 /// deserializing from PCH, we don't need to deserialize identifier & macros
1036 /// just so that we can report that they are unused, we just warn using
1037 /// the SourceLocations of this set (that will be filled by the ASTReader).
1038 using WarnUnusedMacroLocsTy = llvm::SmallDenseSet<SourceLocation, 32>;
1039 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
1040
1041 /// This is a pair of an optional message and source location used for pragmas
1042 /// that annotate macros like pragma clang restrict_expansion and pragma clang
1043 /// deprecated. This pair stores the optional message and the location of the
1044 /// annotation pragma for use producing diagnostics and notes.
1045 using MsgLocationPair = std::pair<std::string, SourceLocation>;
1046
1047 struct MacroAnnotationInfo {
1048 SourceLocation Location;
1049 std::string Message;
1050 };
1051
1052 struct MacroAnnotations {
1053 std::optional<MacroAnnotationInfo> DeprecationInfo;
1054 std::optional<MacroAnnotationInfo> RestrictExpansionInfo;
1055 std::optional<SourceLocation> FinalAnnotationLoc;
1056
1057 static MacroAnnotations makeDeprecation(SourceLocation Loc,
1058 std::string Msg) {
1059 return MacroAnnotations{MacroAnnotationInfo{Loc, std::move(Msg)},
1060 std::nullopt, std::nullopt};
1061 }
1062
1063 static MacroAnnotations makeRestrictExpansion(SourceLocation Loc,
1064 std::string Msg) {
1065 return MacroAnnotations{
1066 std::nullopt, MacroAnnotationInfo{Loc, std::move(Msg)}, std::nullopt};
1067 }
1068
1069 static MacroAnnotations makeFinal(SourceLocation Loc) {
1070 return MacroAnnotations{std::nullopt, std::nullopt, Loc};
1071 }
1072 };
1073
1074 /// Warning information for macro annotations.
1075 llvm::DenseMap<const IdentifierInfo *, MacroAnnotations> AnnotationInfos;
1076
1077 /// A "freelist" of MacroArg objects that can be
1078 /// reused for quick allocation.
1079 MacroArgs *MacroArgCache = nullptr;
1080
1081 /// For each IdentifierInfo used in a \#pragma push_macro directive,
1082 /// we keep a MacroInfo stack used to restore the previous macro value.
1083 llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
1084 PragmaPushMacroInfo;
1085
1086 // Various statistics we track for performance analysis.
1087 unsigned NumDirectives = 0;
1088 unsigned NumDefined = 0;
1089 unsigned NumUndefined = 0;
1090 unsigned NumPragma = 0;
1091 unsigned NumIf = 0;
1092 unsigned NumElse = 0;
1093 unsigned NumEndif = 0;
1094 unsigned NumEnteredSourceFiles = 0;
1095 unsigned MaxIncludeStackDepth = 0;
1096 unsigned NumMacroExpanded = 0;
1097 unsigned NumFnMacroExpanded = 0;
1098 unsigned NumBuiltinMacroExpanded = 0;
1099 unsigned NumFastMacroExpanded = 0;
1100 unsigned NumTokenPaste = 0;
1101 unsigned NumFastTokenPaste = 0;
1102 unsigned NumSkipped = 0;
1103
1104 /// The predefined macros that preprocessor should use from the
1105 /// command line etc.
1106 std::string Predefines;
1107
1108 /// The file ID for the preprocessor predefines.
1109 FileID PredefinesFileID;
1110
1111 /// The file ID for the PCH through header.
1112 FileID PCHThroughHeaderFileID;
1113
1114 /// Whether tokens are being skipped until a #pragma hdrstop is seen.
1115 bool SkippingUntilPragmaHdrStop = false;
1116
1117 /// Whether tokens are being skipped until the through header is seen.
1118 bool SkippingUntilPCHThroughHeader = false;
1119
1120 /// \{
1121 /// Cache of macro expanders to reduce malloc traffic.
1122 enum { TokenLexerCacheSize = 8 };
1123 unsigned NumCachedTokenLexers;
1124 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
1125 /// \}
1126
1127 /// Keeps macro expanded tokens for TokenLexers.
1128 //
1129 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
1130 /// going to lex in the cache and when it finishes the tokens are removed
1131 /// from the end of the cache.
1132 SmallVector<Token, 16> MacroExpandedTokens;
1133 std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
1134
1135 /// A record of the macro definitions and expansions that
1136 /// occurred during preprocessing.
1137 ///
1138 /// This is an optional side structure that can be enabled with
1139 /// \c createPreprocessingRecord() prior to preprocessing.
1140 PreprocessingRecord *Record = nullptr;
1141
1142 /// Cached tokens state.
1143 using CachedTokensTy = SmallVector<Token, 1>;
1144
1145 /// Cached tokens are stored here when we do backtracking or
1146 /// lookahead. They are "lexed" by the CachingLex() method.
1147 CachedTokensTy CachedTokens;
1148
1149 /// The position of the cached token that CachingLex() should
1150 /// "lex" next.
1151 ///
1152 /// If it points beyond the CachedTokens vector, it means that a normal
1153 /// Lex() should be invoked.
1154 CachedTokensTy::size_type CachedLexPos = 0;
1155
1156 /// Stack of backtrack positions, allowing nested backtracks.
1157 ///
1158 /// The EnableBacktrackAtThisPos() method pushes a position to
1159 /// indicate where CachedLexPos should be set when the BackTrack() method is
1160 /// invoked (at which point the last position is popped).
1161 std::vector<CachedTokensTy::size_type> BacktrackPositions;
1162
1163 /// True if \p Preprocessor::SkipExcludedConditionalBlock() is running.
1164 /// This is used to guard against calling this function recursively.
1165 ///
1166 /// See comments at the use-site for more context about why it is needed.
1167 bool SkippingExcludedConditionalBlock = false;
1168
1169 /// Keeps track of skipped range mappings that were recorded while skipping
1170 /// excluded conditional directives. It maps the source buffer pointer at
1171 /// the beginning of a skipped block, to the number of bytes that should be
1172 /// skipped.
1173 llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges;
1174
1175 void updateOutOfDateIdentifier(const IdentifierInfo &II) const;
1176
1177public:
1178 Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
1179 DiagnosticsEngine &diags, const LangOptions &LangOpts,
1180 SourceManager &SM, HeaderSearch &Headers,
1181 ModuleLoader &TheModuleLoader,
1182 IdentifierInfoLookup *IILookup = nullptr,
1183 bool OwnsHeaderSearch = false,
1185
1186 ~Preprocessor();
1187
1188 /// Initialize the preprocessor using information about the target.
1189 ///
1190 /// \param Target is owned by the caller and must remain valid for the
1191 /// lifetime of the preprocessor.
1192 /// \param AuxTarget is owned by the caller and must remain valid for
1193 /// the lifetime of the preprocessor.
1194 void Initialize(const TargetInfo &Target,
1195 const TargetInfo *AuxTarget = nullptr);
1196
1197 /// Initialize the preprocessor to parse a model file
1198 ///
1199 /// To parse model files the preprocessor of the original source is reused to
1200 /// preserver the identifier table. However to avoid some duplicate
1201 /// information in the preprocessor some cleanup is needed before it is used
1202 /// to parse model files. This method does that cleanup.
1204
1205 /// Cleanup after model file parsing
1206 void FinalizeForModelFile();
1207
1208 /// Retrieve the preprocessor options used to initialize this
1209 /// preprocessor.
1210 PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
1211
1212 DiagnosticsEngine &getDiagnostics() const { return *Diags; }
1213 void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
1214
1215 const LangOptions &getLangOpts() const { return LangOpts; }
1216 const TargetInfo &getTargetInfo() const { return *Target; }
1217 const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
1218 FileManager &getFileManager() const { return FileMgr; }
1219 SourceManager &getSourceManager() const { return SourceMgr; }
1220 HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
1221
1222 IdentifierTable &getIdentifierTable() { return Identifiers; }
1223 const IdentifierTable &getIdentifierTable() const { return Identifiers; }
1224 SelectorTable &getSelectorTable() { return Selectors; }
1225 Builtin::Context &getBuiltinInfo() { return *BuiltinInfo; }
1226 llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
1227
1229 ExternalSource = Source;
1230 }
1231
1233 return ExternalSource;
1234 }
1235
1236 /// Retrieve the module loader associated with this preprocessor.
1237 ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
1238
1240 return TheModuleLoader.HadFatalFailure;
1241 }
1242
1243 /// Retrieve the number of Directives that have been processed by the
1244 /// Preprocessor.
1245 unsigned getNumDirectives() const {
1246 return NumDirectives;
1247 }
1248
1249 /// True if we are currently preprocessing a #if or #elif directive
1251 return ParsingIfOrElifDirective;
1252 }
1253
1254 /// Control whether the preprocessor retains comments in output.
1255 void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
1256 this->KeepComments = KeepComments | KeepMacroComments;
1257 this->KeepMacroComments = KeepMacroComments;
1258 }
1259
1260 bool getCommentRetentionState() const { return KeepComments; }
1261
1262 void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
1263 bool getPragmasEnabled() const { return PragmasEnabled; }
1264
1266 SuppressIncludeNotFoundError = Suppress;
1267 }
1268
1270 return SuppressIncludeNotFoundError;
1271 }
1272
1273 /// Sets whether the preprocessor is responsible for producing output or if
1274 /// it is producing tokens to be consumed by Parse and Sema.
1275 void setPreprocessedOutput(bool IsPreprocessedOutput) {
1276 PreprocessedOutput = IsPreprocessedOutput;
1277 }
1278
1279 /// Returns true if the preprocessor is responsible for generating output,
1280 /// false if it is producing tokens to be consumed by Parse and Sema.
1281 bool isPreprocessedOutput() const { return PreprocessedOutput; }
1282
1283 /// Return true if we are lexing directly from the specified lexer.
1284 bool isCurrentLexer(const PreprocessorLexer *L) const {
1285 return CurPPLexer == L;
1286 }
1287
1288 /// Return the current lexer being lexed from.
1289 ///
1290 /// Note that this ignores any potentially active macro expansions and _Pragma
1291 /// expansions going on at the time.
1292 PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
1293
1294 /// Return the current file lexer being lexed from.
1295 ///
1296 /// Note that this ignores any potentially active macro expansions and _Pragma
1297 /// expansions going on at the time.
1299
1300 /// Return the submodule owning the file being lexed. This may not be
1301 /// the current module if we have changed modules since entering the file.
1302 Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
1303
1304 /// Returns the FileID for the preprocessor predefines.
1305 FileID getPredefinesFileID() const { return PredefinesFileID; }
1306
1307 /// \{
1308 /// Accessors for preprocessor callbacks.
1309 ///
1310 /// Note that this class takes ownership of any PPCallbacks object given to
1311 /// it.
1312 PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
1313 void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
1314 if (Callbacks)
1315 C = std::make_unique<PPChainedCallbacks>(std::move(C),
1316 std::move(Callbacks));
1317 Callbacks = std::move(C);
1318 }
1319 /// \}
1320
1321 /// Get the number of tokens processed so far.
1322 unsigned getTokenCount() const { return TokenCount; }
1323
1324 /// Get the max number of tokens before issuing a -Wmax-tokens warning.
1325 unsigned getMaxTokens() const { return MaxTokens; }
1326
1328 MaxTokens = Value;
1329 MaxTokensOverrideLoc = Loc;
1330 };
1331
1332 SourceLocation getMaxTokensOverrideLoc() const { return MaxTokensOverrideLoc; }
1333
1334 /// Register a function that would be called on each token in the final
1335 /// expanded token stream.
1336 /// This also reports annotation tokens produced by the parser.
1337 void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
1338 OnToken = std::move(F);
1339 }
1340
1341 void setPreprocessToken(bool Preprocess) { PreprocessToken = Preprocess; }
1342
1343 bool isMacroDefined(StringRef Id) {
1344 return isMacroDefined(&Identifiers.get(Id));
1345 }
1347 return II->hasMacroDefinition() &&
1348 (!getLangOpts().Modules || (bool)getMacroDefinition(II));
1349 }
1350
1351 /// Determine whether II is defined as a macro within the module M,
1352 /// if that is a module that we've already preprocessed. Does not check for
1353 /// macros imported into M.
1355 if (!II->hasMacroDefinition())
1356 return false;
1357 auto I = Submodules.find(M);
1358 if (I == Submodules.end())
1359 return false;
1360 auto J = I->second.Macros.find(II);
1361 if (J == I->second.Macros.end())
1362 return false;
1363 auto *MD = J->second.getLatest();
1364 return MD && MD->isDefined();
1365 }
1366
1368 if (!II->hasMacroDefinition())
1369 return {};
1370
1371 MacroState &S = CurSubmoduleState->Macros[II];
1372 auto *MD = S.getLatest();
1373 while (isa_and_nonnull<VisibilityMacroDirective>(MD))
1374 MD = MD->getPrevious();
1375 return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
1376 S.getActiveModuleMacros(*this, II),
1377 S.isAmbiguous(*this, II));
1378 }
1379
1382 if (!II->hadMacroDefinition())
1383 return {};
1384
1385 MacroState &S = CurSubmoduleState->Macros[II];
1387 if (auto *MD = S.getLatest())
1388 DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
1389 // FIXME: Compute the set of active module macros at the specified location.
1390 return MacroDefinition(DI.getDirective(),
1391 S.getActiveModuleMacros(*this, II),
1392 S.isAmbiguous(*this, II));
1393 }
1394
1395 /// Given an identifier, return its latest non-imported MacroDirective
1396 /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
1398 if (!II->hasMacroDefinition())
1399 return nullptr;
1400
1401 auto *MD = getLocalMacroDirectiveHistory(II);
1402 if (!MD || MD->getDefinition().isUndefined())
1403 return nullptr;
1404
1405 return MD;
1406 }
1407
1408 const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
1409 return const_cast<Preprocessor*>(this)->getMacroInfo(II);
1410 }
1411
1413 if (!II->hasMacroDefinition())
1414 return nullptr;
1415 if (auto MD = getMacroDefinition(II))
1416 return MD.getMacroInfo();
1417 return nullptr;
1418 }
1419
1420 /// Given an identifier, return the latest non-imported macro
1421 /// directive for that identifier.
1422 ///
1423 /// One can iterate over all previous macro directives from the most recent
1424 /// one.
1426
1427 /// Add a directive to the macro directive history for this identifier.
1431 DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1432 appendMacroDirective(II, MD);
1433 return MD;
1434 }
1436 MacroInfo *MI) {
1437 return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1438 }
1439
1440 /// Set a MacroDirective that was loaded from a PCH file.
1442 MacroDirective *MD);
1443
1444 /// Register an exported macro for a module and identifier.
1446 MacroInfo *Macro,
1447 ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1449
1450 /// Get the list of leaf (non-overridden) module macros for a name.
1452 if (II->isOutOfDate())
1453 updateOutOfDateIdentifier(*II);
1454 auto I = LeafModuleMacros.find(II);
1455 if (I != LeafModuleMacros.end())
1456 return I->second;
1457 return std::nullopt;
1458 }
1459
1460 /// Get the list of submodules that we're currently building.
1462 return BuildingSubmoduleStack;
1463 }
1464
1465 /// \{
1466 /// Iterators for the macro history table. Currently defined macros have
1467 /// IdentifierInfo::hasMacroDefinition() set and an empty
1468 /// MacroInfo::getUndefLoc() at the head of the list.
1469 using macro_iterator = MacroMap::const_iterator;
1470
1471 macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1472 macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1473
1474 llvm::iterator_range<macro_iterator>
1475 macros(bool IncludeExternalMacros = true) const {
1476 macro_iterator begin = macro_begin(IncludeExternalMacros);
1477 macro_iterator end = macro_end(IncludeExternalMacros);
1478 return llvm::make_range(begin, end);
1479 }
1480
1481 /// \}
1482
1483 /// Mark the given clang module as affecting the current clang module or translation unit.
1485 assert(M->isModuleMapModule());
1486 if (!BuildingSubmoduleStack.empty()) {
1487 if (M != BuildingSubmoduleStack.back().M)
1488 BuildingSubmoduleStack.back().M->AffectingClangModules.insert(M);
1489 } else {
1490 AffectingClangModules.insert(M);
1491 }
1492 }
1493
1494 /// Get the set of top-level clang modules that affected preprocessing, but were not
1495 /// imported.
1497 return AffectingClangModules;
1498 }
1499
1500 /// Mark the file as included.
1501 /// Returns true if this is the first time the file was included.
1503 HeaderInfo.getFileInfo(File);
1504 return IncludedFiles.insert(File).second;
1505 }
1506
1507 /// Return true if this header has already been included.
1509 HeaderInfo.getFileInfo(File);
1510 return IncludedFiles.count(File);
1511 }
1512
1513 /// Get the set of included files.
1514 IncludedFilesSet &getIncludedFiles() { return IncludedFiles; }
1515 const IncludedFilesSet &getIncludedFiles() const { return IncludedFiles; }
1516
1517 /// Return the name of the macro defined before \p Loc that has
1518 /// spelling \p Tokens. If there are multiple macros with same spelling,
1519 /// return the last one defined.
1521 ArrayRef<TokenValue> Tokens) const;
1522
1523 /// Get the predefines for this processor.
1524 /// Used by some third-party tools to inspect and add predefines (see
1525 /// https://github.com/llvm/llvm-project/issues/57483).
1526 const std::string &getPredefines() const { return Predefines; }
1527
1528 /// Set the predefines for this Preprocessor.
1529 ///
1530 /// These predefines are automatically injected when parsing the main file.
1531 void setPredefines(std::string P) { Predefines = std::move(P); }
1532
1533 /// Return information about the specified preprocessor
1534 /// identifier token.
1535 IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1536 return &Identifiers.get(Name);
1537 }
1538
1539 /// Add the specified pragma handler to this preprocessor.
1540 ///
1541 /// If \p Namespace is non-null, then it is a token required to exist on the
1542 /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1543 void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1545 AddPragmaHandler(StringRef(), Handler);
1546 }
1547
1548 /// Remove the specific pragma handler from this preprocessor.
1549 ///
1550 /// If \p Namespace is non-null, then it should be the namespace that
1551 /// \p Handler was added to. It is an error to remove a handler that
1552 /// has not been registered.
1553 void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1555 RemovePragmaHandler(StringRef(), Handler);
1556 }
1557
1558 /// Install empty handlers for all pragmas (making them ignored).
1559 void IgnorePragmas();
1560
1561 /// Set empty line handler.
1562 void setEmptylineHandler(EmptylineHandler *Handler) { Emptyline = Handler; }
1563
1564 EmptylineHandler *getEmptylineHandler() const { return Emptyline; }
1565
1566 /// Add the specified comment handler to the preprocessor.
1567 void addCommentHandler(CommentHandler *Handler);
1568
1569 /// Remove the specified comment handler.
1570 ///
1571 /// It is an error to remove a handler that has not been registered.
1572 void removeCommentHandler(CommentHandler *Handler);
1573
1574 /// Set the code completion handler to the given object.
1576 CodeComplete = &Handler;
1577 }
1578
1579 /// Retrieve the current code-completion handler.
1581 return CodeComplete;
1582 }
1583
1584 /// Clear out the code completion handler.
1586 CodeComplete = nullptr;
1587 }
1588
1589 /// Hook used by the lexer to invoke the "included file" code
1590 /// completion point.
1591 void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1592
1593 /// Hook used by the lexer to invoke the "natural language" code
1594 /// completion point.
1596
1597 /// Set the code completion token for filtering purposes.
1599 CodeCompletionII = Filter;
1600 }
1601
1602 /// Set the code completion token range for detecting replacement range later
1603 /// on.
1605 const SourceLocation End) {
1606 CodeCompletionTokenRange = {Start, End};
1607 }
1609 return CodeCompletionTokenRange;
1610 }
1611
1612 /// Get the code completion token for filtering purposes.
1614 if (CodeCompletionII)
1615 return CodeCompletionII->getName();
1616 return {};
1617 }
1618
1619 /// Retrieve the preprocessing record, or NULL if there is no
1620 /// preprocessing record.
1622
1623 /// Create a new preprocessing record, which will keep track of
1624 /// all macro expansions, macro definitions, etc.
1626
1627 /// Returns true if the FileEntry is the PCH through header.
1628 bool isPCHThroughHeader(const FileEntry *FE);
1629
1630 /// True if creating a PCH with a through header.
1632
1633 /// True if using a PCH with a through header.
1635
1636 /// True if creating a PCH with a #pragma hdrstop.
1638
1639 /// True if using a PCH with a #pragma hdrstop.
1641
1642 /// Skip tokens until after the #include of the through header or
1643 /// until after a #pragma hdrstop.
1645
1646 /// Process directives while skipping until the through header or
1647 /// #pragma hdrstop is found.
1649 SourceLocation HashLoc);
1650
1651 /// Enter the specified FileID as the main source file,
1652 /// which implicitly adds the builtin defines etc.
1653 void EnterMainSourceFile();
1654
1655 /// Inform the preprocessor callbacks that processing is complete.
1656 void EndSourceFile();
1657
1658 /// Add a source file to the top of the include stack and
1659 /// start lexing tokens from it instead of the current buffer.
1660 ///
1661 /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1663 SourceLocation Loc, bool IsFirstIncludeOfFile = true);
1664
1665 /// Add a Macro to the top of the include stack and start lexing
1666 /// tokens from it instead of the current buffer.
1667 ///
1668 /// \param Args specifies the tokens input to a function-like macro.
1669 /// \param ILEnd specifies the location of the ')' for a function-like macro
1670 /// or the identifier for an object-like macro.
1671 void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1672 MacroArgs *Args);
1673
1674private:
1675 /// Add a "macro" context to the top of the include stack,
1676 /// which will cause the lexer to start returning the specified tokens.
1677 ///
1678 /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1679 /// will not be subject to further macro expansion. Otherwise, these tokens
1680 /// will be re-macro-expanded when/if expansion is enabled.
1681 ///
1682 /// If \p OwnsTokens is false, this method assumes that the specified stream
1683 /// of tokens has a permanent owner somewhere, so they do not need to be
1684 /// copied. If it is true, it assumes the array of tokens is allocated with
1685 /// \c new[] and the Preprocessor will delete[] it.
1686 ///
1687 /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
1688 /// set, see the flag documentation for details.
1689 void EnterTokenStream(const Token *Toks, unsigned NumToks,
1690 bool DisableMacroExpansion, bool OwnsTokens,
1691 bool IsReinject);
1692
1693public:
1694 void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1695 bool DisableMacroExpansion, bool IsReinject) {
1696 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true,
1697 IsReinject);
1698 }
1699
1700 void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
1701 bool IsReinject) {
1702 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false,
1703 IsReinject);
1704 }
1705
1706 /// Pop the current lexer/macro exp off the top of the lexer stack.
1707 ///
1708 /// This should only be used in situations where the current state of the
1709 /// top-of-stack lexer is known.
1710 void RemoveTopOfLexerStack();
1711
1712 /// From the point that this method is called, and until
1713 /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1714 /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1715 /// make the Preprocessor re-lex the same tokens.
1716 ///
1717 /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1718 /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1719 /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1720 ///
1721 /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1722 /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1723 /// tokens will continue indefinitely.
1724 ///
1726
1727 /// Disable the last EnableBacktrackAtThisPos call.
1729
1730 /// Make Preprocessor re-lex the tokens that were lexed since
1731 /// EnableBacktrackAtThisPos() was previously called.
1732 void Backtrack();
1733
1734 /// True if EnableBacktrackAtThisPos() was called and
1735 /// caching of tokens is on.
1736 bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1737
1738 /// Lex the next token for this preprocessor.
1739 void Lex(Token &Result);
1740
1741 /// Lex all tokens for this preprocessor until (and excluding) end of file.
1742 void LexTokensUntilEOF(std::vector<Token> *Tokens = nullptr);
1743
1744 /// Lex a token, forming a header-name token if possible.
1745 bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
1746
1747 /// Lex the parameters for an #embed directive, returns nullopt on error.
1748 std::optional<LexEmbedParametersResult> LexEmbedParameters(Token &Current,
1749 bool ForHasEmbed);
1750
1753
1755
1757 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1758 }
1759
1760 /// Lex a string literal, which may be the concatenation of multiple
1761 /// string literals and may even come from macro expansion.
1762 /// \returns true on success, false if a error diagnostic has been generated.
1763 bool LexStringLiteral(Token &Result, std::string &String,
1764 const char *DiagnosticTag, bool AllowMacroExpansion) {
1765 if (AllowMacroExpansion)
1766 Lex(Result);
1767 else
1769 return FinishLexStringLiteral(Result, String, DiagnosticTag,
1770 AllowMacroExpansion);
1771 }
1772
1773 /// Complete the lexing of a string literal where the first token has
1774 /// already been lexed (see LexStringLiteral).
1775 bool FinishLexStringLiteral(Token &Result, std::string &String,
1776 const char *DiagnosticTag,
1777 bool AllowMacroExpansion);
1778
1779 /// Lex a token. If it's a comment, keep lexing until we get
1780 /// something not a comment.
1781 ///
1782 /// This is useful in -E -C mode where comments would foul up preprocessor
1783 /// directive handling.
1785 do
1786 Lex(Result);
1787 while (Result.getKind() == tok::comment);
1788 }
1789
1790 /// Just like Lex, but disables macro expansion of identifier tokens.
1792 // Disable macro expansion.
1793 bool OldVal = DisableMacroExpansion;
1794 DisableMacroExpansion = true;
1795 // Lex the token.
1796 Lex(Result);
1797
1798 // Reenable it.
1799 DisableMacroExpansion = OldVal;
1800 }
1801
1802 /// Like LexNonComment, but this disables macro expansion of
1803 /// identifier tokens.
1805 do
1807 while (Result.getKind() == tok::comment);
1808 }
1809
1810 /// Parses a simple integer literal to get its numeric value. Floating
1811 /// point literals and user defined literals are rejected. Used primarily to
1812 /// handle pragmas that accept integer arguments.
1813 bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1814
1815 /// Disables macro expansion everywhere except for preprocessor directives.
1817 DisableMacroExpansion = true;
1818 MacroExpansionInDirectivesOverride = true;
1819 }
1820
1821 /// Peeks ahead N tokens and returns that token without consuming any
1822 /// tokens.
1823 ///
1824 /// LookAhead(0) returns the next token that would be returned by Lex(),
1825 /// LookAhead(1) returns the token after it, etc. This returns normal
1826 /// tokens after phase 5. As such, it is equivalent to using
1827 /// 'Lex', not 'LexUnexpandedToken'.
1828 const Token &LookAhead(unsigned N) {
1829 assert(LexLevel == 0 && "cannot use lookahead while lexing");
1830 if (CachedLexPos + N < CachedTokens.size())
1831 return CachedTokens[CachedLexPos+N];
1832 else
1833 return PeekAhead(N+1);
1834 }
1835
1836 /// When backtracking is enabled and tokens are cached,
1837 /// this allows to revert a specific number of tokens.
1838 ///
1839 /// Note that the number of tokens being reverted should be up to the last
1840 /// backtrack position, not more.
1841 void RevertCachedTokens(unsigned N) {
1842 assert(isBacktrackEnabled() &&
1843 "Should only be called when tokens are cached for backtracking");
1844 assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1845 && "Should revert tokens up to the last backtrack position, not more");
1846 assert(signed(CachedLexPos) - signed(N) >= 0 &&
1847 "Corrupted backtrack positions ?");
1848 CachedLexPos -= N;
1849 }
1850
1851 /// Enters a token in the token stream to be lexed next.
1852 ///
1853 /// If BackTrack() is called afterwards, the token will remain at the
1854 /// insertion point.
1855 /// If \p IsReinject is true, resulting token will have Token::IsReinjected
1856 /// flag set. See the flag documentation for details.
1857 void EnterToken(const Token &Tok, bool IsReinject) {
1858 if (LexLevel) {
1859 // It's not correct in general to enter caching lex mode while in the
1860 // middle of a nested lexing action.
1861 auto TokCopy = std::make_unique<Token[]>(1);
1862 TokCopy[0] = Tok;
1863 EnterTokenStream(std::move(TokCopy), 1, true, IsReinject);
1864 } else {
1865 EnterCachingLexMode();
1866 assert(IsReinject && "new tokens in the middle of cached stream");
1867 CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1868 }
1869 }
1870
1871 /// We notify the Preprocessor that if it is caching tokens (because
1872 /// backtrack is enabled) it should replace the most recent cached tokens
1873 /// with the given annotation token. This function has no effect if
1874 /// backtracking is not enabled.
1875 ///
1876 /// Note that the use of this function is just for optimization, so that the
1877 /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1878 /// invoked.
1879 void AnnotateCachedTokens(const Token &Tok) {
1880 assert(Tok.isAnnotation() && "Expected annotation token");
1881 if (CachedLexPos != 0 && isBacktrackEnabled())
1882 AnnotatePreviousCachedTokens(Tok);
1883 }
1884
1885 /// Get the location of the last cached token, suitable for setting the end
1886 /// location of an annotation token.
1888 assert(CachedLexPos != 0);
1889 return CachedTokens[CachedLexPos-1].getLastLoc();
1890 }
1891
1892 /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1893 /// CachedTokens.
1894 bool IsPreviousCachedToken(const Token &Tok) const;
1895
1896 /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1897 /// in \p NewToks.
1898 ///
1899 /// Useful when a token needs to be split in smaller ones and CachedTokens
1900 /// most recent token must to be updated to reflect that.
1902
1903 /// Replace the last token with an annotation token.
1904 ///
1905 /// Like AnnotateCachedTokens(), this routine replaces an
1906 /// already-parsed (and resolved) token with an annotation
1907 /// token. However, this routine only replaces the last token with
1908 /// the annotation token; it does not affect any other cached
1909 /// tokens. This function has no effect if backtracking is not
1910 /// enabled.
1912 assert(Tok.isAnnotation() && "Expected annotation token");
1913 if (CachedLexPos != 0 && isBacktrackEnabled())
1914 CachedTokens[CachedLexPos-1] = Tok;
1915 }
1916
1917 /// Enter an annotation token into the token stream.
1919 void *AnnotationVal);
1920
1921 /// Determine whether it's possible for a future call to Lex to produce an
1922 /// annotation token created by a previous call to EnterAnnotationToken.
1924 return CurLexerCallback != CLK_Lexer;
1925 }
1926
1927 /// Update the current token to represent the provided
1928 /// identifier, in order to cache an action performed by typo correction.
1929 void TypoCorrectToken(const Token &Tok) {
1930 assert(Tok.getIdentifierInfo() && "Expected identifier token");
1931 if (CachedLexPos != 0 && isBacktrackEnabled())
1932 CachedTokens[CachedLexPos-1] = Tok;
1933 }
1934
1935 /// Recompute the current lexer kind based on the CurLexer/
1936 /// CurTokenLexer pointers.
1937 void recomputeCurLexerKind();
1938
1939 /// Returns true if incremental processing is enabled
1940 bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1941
1942 /// Enables the incremental processing
1943 void enableIncrementalProcessing(bool value = true) {
1944 IncrementalProcessing = value;
1945 }
1946
1947 /// Specify the point at which code-completion will be performed.
1948 ///
1949 /// \param File the file in which code completion should occur. If
1950 /// this file is included multiple times, code-completion will
1951 /// perform completion the first time it is included. If NULL, this
1952 /// function clears out the code-completion point.
1953 ///
1954 /// \param Line the line at which code completion should occur
1955 /// (1-based).
1956 ///
1957 /// \param Column the column at which code completion should occur
1958 /// (1-based).
1959 ///
1960 /// \returns true if an error occurred, false otherwise.
1962 unsigned Column);
1963
1964 /// Determine if we are performing code completion.
1965 bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1966
1967 /// Returns the location of the code-completion point.
1968 ///
1969 /// Returns an invalid location if code-completion is not enabled or the file
1970 /// containing the code-completion point has not been lexed yet.
1971 SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1972
1973 /// Returns the start location of the file of code-completion point.
1974 ///
1975 /// Returns an invalid location if code-completion is not enabled or the file
1976 /// containing the code-completion point has not been lexed yet.
1978 return CodeCompletionFileLoc;
1979 }
1980
1981 /// Returns true if code-completion is enabled and we have hit the
1982 /// code-completion point.
1983 bool isCodeCompletionReached() const { return CodeCompletionReached; }
1984
1985 /// Note that we hit the code-completion point.
1987 assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1988 CodeCompletionReached = true;
1989 // Silence any diagnostics that occur after we hit the code-completion.
1991 }
1992
1993 /// The location of the currently-active \#pragma clang
1994 /// arc_cf_code_audited begin.
1995 ///
1996 /// Returns an invalid location if there is no such pragma active.
1997 std::pair<IdentifierInfo *, SourceLocation>
1999 return PragmaARCCFCodeAuditedInfo;
2000 }
2001
2002 /// Set the location of the currently-active \#pragma clang
2003 /// arc_cf_code_audited begin. An invalid location ends the pragma.
2006 PragmaARCCFCodeAuditedInfo = {Ident, Loc};
2007 }
2008
2009 /// The location of the currently-active \#pragma clang
2010 /// assume_nonnull begin.
2011 ///
2012 /// Returns an invalid location if there is no such pragma active.
2014 return PragmaAssumeNonNullLoc;
2015 }
2016
2017 /// Set the location of the currently-active \#pragma clang
2018 /// assume_nonnull begin. An invalid location ends the pragma.
2020 PragmaAssumeNonNullLoc = Loc;
2021 }
2022
2023 /// Get the location of the recorded unterminated \#pragma clang
2024 /// assume_nonnull begin in the preamble, if one exists.
2025 ///
2026 /// Returns an invalid location if the premable did not end with
2027 /// such a pragma active or if there is no recorded preamble.
2029 return PreambleRecordedPragmaAssumeNonNullLoc;
2030 }
2031
2032 /// Record the location of the unterminated \#pragma clang
2033 /// assume_nonnull begin in the preamble.
2035 PreambleRecordedPragmaAssumeNonNullLoc = Loc;
2036 }
2037
2038 /// Set the directory in which the main file should be considered
2039 /// to have been found, if it is not a real file.
2040 void setMainFileDir(DirectoryEntryRef Dir) { MainFileDir = Dir; }
2041
2042 /// Instruct the preprocessor to skip part of the main source file.
2043 ///
2044 /// \param Bytes The number of bytes in the preamble to skip.
2045 ///
2046 /// \param StartOfLine Whether skipping these bytes puts the lexer at the
2047 /// start of a line.
2048 void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
2049 SkipMainFilePreamble.first = Bytes;
2050 SkipMainFilePreamble.second = StartOfLine;
2051 }
2052
2053 /// Forwarding function for diagnostics. This emits a diagnostic at
2054 /// the specified Token's location, translating the token's start
2055 /// position in the current buffer into a SourcePosition object for rendering.
2056 DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
2057 return Diags->Report(Loc, DiagID);
2058 }
2059
2060 DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
2061 return Diags->Report(Tok.getLocation(), DiagID);
2062 }
2063
2064 /// Return the 'spelling' of the token at the given
2065 /// location; does not go up to the spelling location or down to the
2066 /// expansion location.
2067 ///
2068 /// \param buffer A buffer which will be used only if the token requires
2069 /// "cleaning", e.g. if it contains trigraphs or escaped newlines
2070 /// \param invalid If non-null, will be set \c true if an error occurs.
2072 SmallVectorImpl<char> &buffer,
2073 bool *invalid = nullptr) const {
2074 return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
2075 }
2076
2077 /// Return the 'spelling' of the Tok token.
2078 ///
2079 /// The spelling of a token is the characters used to represent the token in
2080 /// the source file after trigraph expansion and escaped-newline folding. In
2081 /// particular, this wants to get the true, uncanonicalized, spelling of
2082 /// things like digraphs, UCNs, etc.
2083 ///
2084 /// \param Invalid If non-null, will be set \c true if an error occurs.
2085 std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
2086 return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
2087 }
2088
2089 /// Get the spelling of a token into a preallocated buffer, instead
2090 /// of as an std::string.
2091 ///
2092 /// The caller is required to allocate enough space for the token, which is
2093 /// guaranteed to be at least Tok.getLength() bytes long. The length of the
2094 /// actual result is returned.
2095 ///
2096 /// Note that this method may do two possible things: it may either fill in
2097 /// the buffer specified with characters, or it may *change the input pointer*
2098 /// to point to a constant buffer with the data already in it (avoiding a
2099 /// copy). The caller is not allowed to modify the returned buffer pointer
2100 /// if an internal buffer is returned.
2101 unsigned getSpelling(const Token &Tok, const char *&Buffer,
2102 bool *Invalid = nullptr) const {
2103 return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
2104 }
2105
2106 /// Get the spelling of a token into a SmallVector.
2107 ///
2108 /// Note that the returned StringRef may not point to the
2109 /// supplied buffer if a copy can be avoided.
2110 StringRef getSpelling(const Token &Tok,
2111 SmallVectorImpl<char> &Buffer,
2112 bool *Invalid = nullptr) const;
2113
2114 /// Relex the token at the specified location.
2115 /// \returns true if there was a failure, false on success.
2117 bool IgnoreWhiteSpace = false) {
2118 return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
2119 }
2120
2121 /// Given a Token \p Tok that is a numeric constant with length 1,
2122 /// return the character.
2123 char
2125 bool *Invalid = nullptr) const {
2126 assert(Tok.is(tok::numeric_constant) &&
2127 Tok.getLength() == 1 && "Called on unsupported token");
2128 assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
2129
2130 // If the token is carrying a literal data pointer, just use it.
2131 if (const char *D = Tok.getLiteralData())
2132 return *D;
2133
2134 // Otherwise, fall back on getCharacterData, which is slower, but always
2135 // works.
2136 return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
2137 }
2138
2139 /// Retrieve the name of the immediate macro expansion.
2140 ///
2141 /// This routine starts from a source location, and finds the name of the
2142 /// macro responsible for its immediate expansion. It looks through any
2143 /// intervening macro argument expansions to compute this. It returns a
2144 /// StringRef that refers to the SourceManager-owned buffer of the source
2145 /// where that macro name is spelled. Thus, the result shouldn't out-live
2146 /// the SourceManager.
2148 return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
2149 }
2150
2151 /// Plop the specified string into a scratch buffer and set the
2152 /// specified token's location and length to it.
2153 ///
2154 /// If specified, the source location provides a location of the expansion
2155 /// point of the token.
2156 void CreateString(StringRef Str, Token &Tok,
2157 SourceLocation ExpansionLocStart = SourceLocation(),
2158 SourceLocation ExpansionLocEnd = SourceLocation());
2159
2160 /// Split the first Length characters out of the token starting at TokLoc
2161 /// and return a location pointing to the split token. Re-lexing from the
2162 /// split token will return the split token rather than the original.
2163 SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
2164
2165 /// Computes the source location just past the end of the
2166 /// token at this source location.
2167 ///
2168 /// This routine can be used to produce a source location that
2169 /// points just past the end of the token referenced by \p Loc, and
2170 /// is generally used when a diagnostic needs to point just after a
2171 /// token where it expected something different that it received. If
2172 /// the returned source location would not be meaningful (e.g., if
2173 /// it points into a macro), this routine returns an invalid
2174 /// source location.
2175 ///
2176 /// \param Offset an offset from the end of the token, where the source
2177 /// location should refer to. The default offset (0) produces a source
2178 /// location pointing just past the end of the token; an offset of 1 produces
2179 /// a source location pointing to the last character in the token, etc.
2181 return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
2182 }
2183
2184 /// Returns true if the given MacroID location points at the first
2185 /// token of the macro expansion.
2186 ///
2187 /// \param MacroBegin If non-null and function returns true, it is set to
2188 /// begin location of the macro.
2190 SourceLocation *MacroBegin = nullptr) const {
2191 return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
2192 MacroBegin);
2193 }
2194
2195 /// Returns true if the given MacroID location points at the last
2196 /// token of the macro expansion.
2197 ///
2198 /// \param MacroEnd If non-null and function returns true, it is set to
2199 /// end location of the macro.
2201 SourceLocation *MacroEnd = nullptr) const {
2202 return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
2203 }
2204
2205 /// Print the token to stderr, used for debugging.
2206 void DumpToken(const Token &Tok, bool DumpFlags = false) const;
2207 void DumpLocation(SourceLocation Loc) const;
2208 void DumpMacro(const MacroInfo &MI) const;
2209 void dumpMacroInfo(const IdentifierInfo *II);
2210
2211 /// Given a location that specifies the start of a
2212 /// token, return a new location that specifies a character within the token.
2214 unsigned Char) const {
2215 return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
2216 }
2217
2218 /// Increment the counters for the number of token paste operations
2219 /// performed.
2220 ///
2221 /// If fast was specified, this is a 'fast paste' case we handled.
2222 void IncrementPasteCounter(bool isFast) {
2223 if (isFast)
2224 ++NumFastTokenPaste;
2225 else
2226 ++NumTokenPaste;
2227 }
2228
2229 void PrintStats();
2230
2231 size_t getTotalMemory() const;
2232
2233 /// When the macro expander pastes together a comment (/##/) in Microsoft
2234 /// mode, this method handles updating the current state, returning the
2235 /// token on the next source line.
2237
2238 //===--------------------------------------------------------------------===//
2239 // Preprocessor callback methods. These are invoked by a lexer as various
2240 // directives and events are found.
2241
2242 /// Given a tok::raw_identifier token, look up the
2243 /// identifier information for the token and install it into the token,
2244 /// updating the token kind accordingly.
2246
2247private:
2248 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
2249
2250public:
2251 /// Specifies the reason for poisoning an identifier.
2252 ///
2253 /// If that identifier is accessed while poisoned, then this reason will be
2254 /// used instead of the default "poisoned" diagnostic.
2255 void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
2256
2257 /// Display reason for poisoned identifier.
2259
2261 if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
2262 if(II->isPoisoned()) {
2264 }
2265 }
2266 }
2267
2268private:
2269 /// Identifiers used for SEH handling in Borland. These are only
2270 /// allowed in particular circumstances
2271 // __except block
2272 IdentifierInfo *Ident__exception_code,
2273 *Ident___exception_code,
2274 *Ident_GetExceptionCode;
2275 // __except filter expression
2276 IdentifierInfo *Ident__exception_info,
2277 *Ident___exception_info,
2278 *Ident_GetExceptionInfo;
2279 // __finally
2280 IdentifierInfo *Ident__abnormal_termination,
2281 *Ident___abnormal_termination,
2282 *Ident_AbnormalTermination;
2283
2284 const char *getCurLexerEndPos();
2285 void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
2286
2287public:
2288 void PoisonSEHIdentifiers(bool Poison = true); // Borland
2289
2290 /// Callback invoked when the lexer reads an identifier and has
2291 /// filled in the tokens IdentifierInfo member.
2292 ///
2293 /// This callback potentially macro expands it or turns it into a named
2294 /// token (like 'for').
2295 ///
2296 /// \returns true if we actually computed a token, false if we need to
2297 /// lex again.
2299
2300 /// Callback invoked when the lexer hits the end of the current file.
2301 ///
2302 /// This either returns the EOF token and returns true, or
2303 /// pops a level off the include stack and returns false, at which point the
2304 /// client should call lex again.
2305 bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
2306
2307 /// Callback invoked when the current TokenLexer hits the end of its
2308 /// token stream.
2310
2311 /// Callback invoked when the lexer sees a # token at the start of a
2312 /// line.
2313 ///
2314 /// This consumes the directive, modifies the lexer/preprocessor state, and
2315 /// advances the lexer(s) so that the next token read is the correct one.
2317
2318 /// Ensure that the next token is a tok::eod token.
2319 ///
2320 /// If not, emit a diagnostic and consume up until the eod.
2321 /// If \p EnableMacros is true, then we consider macros that expand to zero
2322 /// tokens as being ok.
2323 ///
2324 /// \return The location of the end of the directive (the terminating
2325 /// newline).
2326 SourceLocation CheckEndOfDirective(const char *DirType,
2327 bool EnableMacros = false);
2328
2329 /// Read and discard all tokens remaining on the current line until
2330 /// the tok::eod token is found. Returns the range of the skipped tokens.
2332 Token Tmp;
2333 return DiscardUntilEndOfDirective(Tmp);
2334 }
2335
2336 /// Same as above except retains the token that was found.
2338
2339 /// Returns true if the preprocessor has seen a use of
2340 /// __DATE__ or __TIME__ in the file so far.
2341 bool SawDateOrTime() const {
2342 return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
2343 }
2344 unsigned getCounterValue() const { return CounterValue; }
2345 void setCounterValue(unsigned V) { CounterValue = V; }
2346
2348 assert(CurrentFPEvalMethod != LangOptions::FEM_UnsetOnCommandLine &&
2349 "FPEvalMethod should be set either from command line or from the "
2350 "target info");
2351 return CurrentFPEvalMethod;
2352 }
2353
2355 return TUFPEvalMethod;
2356 }
2357
2359 return LastFPEvalPragmaLocation;
2360 }
2361
2365 "FPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2366 // This is the location of the '#pragma float_control" where the
2367 // execution state is modifed.
2368 LastFPEvalPragmaLocation = PragmaLoc;
2369 CurrentFPEvalMethod = Val;
2370 TUFPEvalMethod = Val;
2371 }
2372
2375 "TUPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2376 TUFPEvalMethod = Val;
2377 }
2378
2379 /// Retrieves the module that we're currently building, if any.
2381
2382 /// Retrieves the module whose implementation we're current compiling, if any.
2384
2385 /// If we are preprocessing a named module.
2386 bool isInNamedModule() const { return ModuleDeclState.isNamedModule(); }
2387
2388 /// If we are proprocessing a named interface unit.
2389 /// Note that a module implementation partition is not considered as an
2390 /// named interface unit here although it is importable
2391 /// to ease the parsing.
2393 return ModuleDeclState.isNamedInterface();
2394 }
2395
2396 /// Get the named module name we're preprocessing.
2397 /// Requires we're preprocessing a named module.
2398 StringRef getNamedModuleName() const { return ModuleDeclState.getName(); }
2399
2400 /// If we are implementing an implementation module unit.
2401 /// Note that the module implementation partition is not considered as an
2402 /// implementation unit.
2404 return ModuleDeclState.isImplementationUnit();
2405 }
2406
2407 /// If we're importing a standard C++20 Named Modules.
2409 // NamedModuleImportPath will be non-empty only if we're importing
2410 // Standard C++ named modules.
2411 return !NamedModuleImportPath.empty() && getLangOpts().CPlusPlusModules &&
2412 !IsAtImport;
2413 }
2414
2415 /// Allocate a new MacroInfo object with the provided SourceLocation.
2417
2418 /// Turn the specified lexer token into a fully checked and spelled
2419 /// filename, e.g. as an operand of \#include.
2420 ///
2421 /// The caller is expected to provide a buffer that is large enough to hold
2422 /// the spelling of the filename, but is also expected to handle the case
2423 /// when this method decides to use a different buffer.
2424 ///
2425 /// \returns true if the input filename was in <>'s or false if it was
2426 /// in ""'s.
2427 bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
2428
2429 /// Given a "foo" or <foo> reference, look up the indicated file.
2430 ///
2431 /// Returns std::nullopt on failure. \p isAngled indicates whether the file
2432 /// reference is for system \#include's or not (i.e. using <> instead of "").
2434 LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled,
2435 ConstSearchDirIterator FromDir, const FileEntry *FromFile,
2436 ConstSearchDirIterator *CurDir, SmallVectorImpl<char> *SearchPath,
2437 SmallVectorImpl<char> *RelativePath,
2438 ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped,
2439 bool *IsFrameworkFound, bool SkipCache = false,
2440 bool OpenFile = true, bool CacheFailures = true);
2441
2442 /// Given a "Filename" or <Filename> reference, look up the indicated embed
2443 /// resource. \p isAngled indicates whether the file reference is for
2444 /// system \#include's or not (i.e. using <> instead of ""). If \p OpenFile
2445 /// is true, the file looked up is opened for reading, otherwise it only
2446 /// validates that the file exists. Quoted filenames are looked up relative
2447 /// to \p LookupFromFile if it is nonnull.
2448 ///
2449 /// Returns std::nullopt on failure.
2451 LookupEmbedFile(StringRef Filename, bool isAngled, bool OpenFile,
2452 const FileEntry *LookupFromFile = nullptr);
2453
2454 /// Return true if we're in the top-level file, not in a \#include.
2455 bool isInPrimaryFile() const;
2456
2457 /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
2458 /// followed by EOD. Return true if the token is not a valid on-off-switch.
2460
2461 bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
2462 bool *ShadowFlag = nullptr);
2463
2464 void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
2465 Module *LeaveSubmodule(bool ForPragma);
2466
2467private:
2469
2470 void PushIncludeMacroStack() {
2471 assert(CurLexerCallback != CLK_CachingLexer &&
2472 "cannot push a caching lexer");
2473 IncludeMacroStack.emplace_back(CurLexerCallback, CurLexerSubmodule,
2474 std::move(CurLexer), CurPPLexer,
2475 std::move(CurTokenLexer), CurDirLookup);
2476 CurPPLexer = nullptr;
2477 }
2478
2479 void PopIncludeMacroStack() {
2480 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2481 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2482 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2483 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2484 CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2485 CurLexerCallback = IncludeMacroStack.back().CurLexerCallback;
2486 IncludeMacroStack.pop_back();
2487 }
2488
2489 void PropagateLineStartLeadingSpaceInfo(Token &Result);
2490
2491 /// Determine whether we need to create module macros for #defines in the
2492 /// current context.
2493 bool needModuleMacros() const;
2494
2495 /// Update the set of active module macros and ambiguity flag for a module
2496 /// macro name.
2497 void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
2498
2499 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2500 SourceLocation Loc);
2501 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2502 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2503 bool isPublic);
2504
2505 /// Lex and validate a macro name, which occurs after a
2506 /// \#define or \#undef.
2507 ///
2508 /// \param MacroNameTok Token that represents the name defined or undefined.
2509 /// \param IsDefineUndef Kind if preprocessor directive.
2510 /// \param ShadowFlag Points to flag that is set if macro name shadows
2511 /// a keyword.
2512 ///
2513 /// This emits a diagnostic, sets the token kind to eod,
2514 /// and discards the rest of the macro line if the macro name is invalid.
2515 void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
2516 bool *ShadowFlag = nullptr);
2517
2518 /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
2519 /// entire line) of the macro's tokens and adds them to MacroInfo, and while
2520 /// doing so performs certain validity checks including (but not limited to):
2521 /// - # (stringization) is followed by a macro parameter
2522 /// \param MacroNameTok - Token that represents the macro name
2523 /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
2524 ///
2525 /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
2526 /// returns a nullptr if an invalid sequence of tokens is encountered.
2527 MacroInfo *ReadOptionalMacroParameterListAndBody(
2528 const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
2529
2530 /// The ( starting an argument list of a macro definition has just been read.
2531 /// Lex the rest of the parameters and the closing ), updating \p MI with
2532 /// what we learn and saving in \p LastTok the last token read.
2533 /// Return true if an error occurs parsing the arg list.
2534 bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2535
2536 /// Provide a suggestion for a typoed directive. If there is no typo, then
2537 /// just skip suggesting.
2538 ///
2539 /// \param Tok - Token that represents the directive
2540 /// \param Directive - String reference for the directive name
2541 void SuggestTypoedDirective(const Token &Tok, StringRef Directive) const;
2542
2543 /// We just read a \#if or related directive and decided that the
2544 /// subsequent tokens are in the \#if'd out portion of the
2545 /// file. Lex the rest of the file, until we see an \#endif. If \p
2546 /// FoundNonSkipPortion is true, then we have already emitted code for part of
2547 /// this \#if directive, so \#else/\#elif blocks should never be entered. If
2548 /// \p FoundElse is false, then \#else directives are ok, if not, then we have
2549 /// already seen one so a \#else directive is a duplicate. When this returns,
2550 /// the caller can lex the first valid token.
2551 void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2552 SourceLocation IfTokenLoc,
2553 bool FoundNonSkipPortion, bool FoundElse,
2554 SourceLocation ElseLoc = SourceLocation());
2555
2556 /// Information about the result for evaluating an expression for a
2557 /// preprocessor directive.
2558 struct DirectiveEvalResult {
2559 /// The integral value of the expression.
2560 std::optional<llvm::APSInt> Value;
2561
2562 /// Whether the expression was evaluated as true or not.
2563 bool Conditional;
2564
2565 /// True if the expression contained identifiers that were undefined.
2566 bool IncludedUndefinedIds;
2567
2568 /// The source range for the expression.
2569 SourceRange ExprRange;
2570 };
2571
2572 /// Evaluate an integer constant expression that may occur after a
2573 /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2574 ///
2575 /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2576 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro,
2577 bool CheckForEoD = true);
2578
2579 /// Evaluate an integer constant expression that may occur after a
2580 /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2581 ///
2582 /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2583 /// \p EvaluatedDefined will contain the result of whether "defined" appeared
2584 /// in the evaluated expression or not.
2585 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro,
2586 Token &Tok,
2587 bool &EvaluatedDefined,
2588 bool CheckForEoD = true);
2589
2590 /// Process a '__has_embed("path" [, ...])' expression.
2591 ///
2592 /// Returns predefined `__STDC_EMBED_*` macro values if
2593 /// successful.
2594 EmbedResult EvaluateHasEmbed(Token &Tok, IdentifierInfo *II);
2595
2596 /// Process a '__has_include("path")' expression.
2597 ///
2598 /// Returns true if successful.
2599 bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II);
2600
2601 /// Process '__has_include_next("path")' expression.
2602 ///
2603 /// Returns true if successful.
2604 bool EvaluateHasIncludeNext(Token &Tok, IdentifierInfo *II);
2605
2606 /// Get the directory and file from which to start \#include_next lookup.
2607 std::pair<ConstSearchDirIterator, const FileEntry *>
2608 getIncludeNextStart(const Token &IncludeNextTok) const;
2609
2610 /// Install the standard preprocessor pragmas:
2611 /// \#pragma GCC poison/system_header/dependency and \#pragma once.
2612 void RegisterBuiltinPragmas();
2613
2614 /// Register builtin macros such as __LINE__ with the identifier table.
2615 void RegisterBuiltinMacros();
2616
2617 /// If an identifier token is read that is to be expanded as a macro, handle
2618 /// it and return the next token as 'Tok'. If we lexed a token, return true;
2619 /// otherwise the caller should lex again.
2620 bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2621
2622 /// Cache macro expanded tokens for TokenLexers.
2623 //
2624 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2625 /// going to lex in the cache and when it finishes the tokens are removed
2626 /// from the end of the cache.
2627 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2628 ArrayRef<Token> tokens);
2629
2630 void removeCachedMacroExpandedTokensOfLastLexer();
2631
2632 /// Determine whether the next preprocessor token to be
2633 /// lexed is a '('. If so, consume the token and return true, if not, this
2634 /// method should have no observable side-effect on the lexed tokens.
2635 bool isNextPPTokenLParen();
2636
2637 /// After reading "MACRO(", this method is invoked to read all of the formal
2638 /// arguments specified for the macro invocation. Returns null on error.
2639 MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2640 SourceLocation &MacroEnd);
2641
2642 /// If an identifier token is read that is to be expanded
2643 /// as a builtin macro, handle it and return the next token as 'Tok'.
2644 void ExpandBuiltinMacro(Token &Tok);
2645
2646 /// Read a \c _Pragma directive, slice it up, process it, then
2647 /// return the first token after the directive.
2648 /// This assumes that the \c _Pragma token has just been read into \p Tok.
2649 void Handle_Pragma(Token &Tok);
2650
2651 /// Like Handle_Pragma except the pragma text is not enclosed within
2652 /// a string literal.
2653 void HandleMicrosoft__pragma(Token &Tok);
2654
2655 /// Add a lexer to the top of the include stack and
2656 /// start lexing tokens from it instead of the current buffer.
2657 void EnterSourceFileWithLexer(Lexer *TheLexer, ConstSearchDirIterator Dir);
2658
2659 /// Set the FileID for the preprocessor predefines.
2660 void setPredefinesFileID(FileID FID) {
2661 assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2662 PredefinesFileID = FID;
2663 }
2664
2665 /// Set the FileID for the PCH through header.
2666 void setPCHThroughHeaderFileID(FileID FID);
2667
2668 /// Returns true if we are lexing from a file and not a
2669 /// pragma or a macro.
2670 static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2671 return L ? !L->isPragmaLexer() : P != nullptr;
2672 }
2673
2674 static bool IsFileLexer(const IncludeStackInfo& I) {
2675 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2676 }
2677
2678 bool IsFileLexer() const {
2679 return IsFileLexer(CurLexer.get(), CurPPLexer);
2680 }
2681
2682 //===--------------------------------------------------------------------===//
2683 // Caching stuff.
2684 void CachingLex(Token &Result);
2685
2686 bool InCachingLexMode() const {
2687 // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2688 // that we are past EOF, not that we are in CachingLex mode.
2689 return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2690 }
2691
2692 void EnterCachingLexMode();
2693 void EnterCachingLexModeUnchecked();
2694
2695 void ExitCachingLexMode() {
2696 if (InCachingLexMode())
2698 }
2699
2700 const Token &PeekAhead(unsigned N);
2701 void AnnotatePreviousCachedTokens(const Token &Tok);
2702
2703 //===--------------------------------------------------------------------===//
2704 /// Handle*Directive - implement the various preprocessor directives. These
2705 /// should side-effect the current preprocessor object so that the next call
2706 /// to Lex() will return the appropriate token next.
2707 void HandleLineDirective();
2708 void HandleDigitDirective(Token &Tok);
2709 void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2710 void HandleIdentSCCSDirective(Token &Tok);
2711 void HandleMacroPublicDirective(Token &Tok);
2712 void HandleMacroPrivateDirective();
2713
2714 /// An additional notification that can be produced by a header inclusion or
2715 /// import to tell the parser what happened.
2716 struct ImportAction {
2717 enum ActionKind {
2718 None,
2719 ModuleBegin,
2720 ModuleImport,
2721 HeaderUnitImport,
2722 SkippedModuleImport,
2723 Failure,
2724 } Kind;
2725 Module *ModuleForHeader = nullptr;
2726
2727 ImportAction(ActionKind AK, Module *Mod = nullptr)
2728 : Kind(AK), ModuleForHeader(Mod) {
2729 assert((AK == None || Mod || AK == Failure) &&
2730 "no module for module action");
2731 }
2732 };
2733
2734 OptionalFileEntryRef LookupHeaderIncludeOrImport(
2735 ConstSearchDirIterator *CurDir, StringRef &Filename,
2736 SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2737 const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
2738 bool &IsMapped, ConstSearchDirIterator LookupFrom,
2739 const FileEntry *LookupFromFile, StringRef &LookupFilename,
2740 SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2741 ModuleMap::KnownHeader &SuggestedModule, bool isAngled);
2742 // Binary data inclusion
2743 void HandleEmbedDirective(SourceLocation HashLoc, Token &Tok,
2744 const FileEntry *LookupFromFile = nullptr);
2745 void HandleEmbedDirectiveImpl(SourceLocation HashLoc,
2746 const LexEmbedParametersResult &Params,
2747 StringRef BinaryContents);
2748
2749 // File inclusion.
2750 void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2751 ConstSearchDirIterator LookupFrom = nullptr,
2752 const FileEntry *LookupFromFile = nullptr);
2753 ImportAction
2754 HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2755 Token &FilenameTok, SourceLocation EndLoc,
2756 ConstSearchDirIterator LookupFrom = nullptr,
2757 const FileEntry *LookupFromFile = nullptr);
2758 void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2759 void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2760 void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2761 void HandleMicrosoftImportDirective(Token &Tok);
2762
2763public:
2764 /// Check that the given module is available, producing a diagnostic if not.
2765 /// \return \c true if the check failed (because the module is not available).
2766 /// \c false if the module appears to be usable.
2767 static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2768 const TargetInfo &TargetInfo,
2769 const Module &M, DiagnosticsEngine &Diags);
2770
2771 // Module inclusion testing.
2772 /// Find the module that owns the source or header file that
2773 /// \p Loc points to. If the location is in a file that was included
2774 /// into a module, or is outside any module, returns nullptr.
2775 Module *getModuleForLocation(SourceLocation Loc, bool AllowTextual);
2776
2777 /// We want to produce a diagnostic at location IncLoc concerning an
2778 /// unreachable effect at location MLoc (eg, where a desired entity was
2779 /// declared or defined). Determine whether the right way to make MLoc
2780 /// reachable is by #include, and if so, what header should be included.
2781 ///
2782 /// This is not necessarily fast, and might load unexpected module maps, so
2783 /// should only be called by code that intends to produce an error.
2784 ///
2785 /// \param IncLoc The location at which the missing effect was detected.
2786 /// \param MLoc A location within an unimported module at which the desired
2787 /// effect occurred.
2788 /// \return A file that can be #included to provide the desired effect. Null
2789 /// if no such file could be determined or if a #include is not
2790 /// appropriate (eg, if a module should be imported instead).
2792 SourceLocation MLoc);
2793
2794 bool isRecordingPreamble() const {
2795 return PreambleConditionalStack.isRecording();
2796 }
2797
2798 bool hasRecordedPreamble() const {
2799 return PreambleConditionalStack.hasRecordedPreamble();
2800 }
2801
2803 return PreambleConditionalStack.getStack();
2804 }
2805
2807 PreambleConditionalStack.setStack(s);
2808 }
2809
2811 ArrayRef<PPConditionalInfo> s, std::optional<PreambleSkipInfo> SkipInfo) {
2812 PreambleConditionalStack.startReplaying();
2813 PreambleConditionalStack.setStack(s);
2814 PreambleConditionalStack.SkipInfo = SkipInfo;
2815 }
2816
2817 std::optional<PreambleSkipInfo> getPreambleSkipInfo() const {
2818 return PreambleConditionalStack.SkipInfo;
2819 }
2820
2821private:
2822 /// After processing predefined file, initialize the conditional stack from
2823 /// the preamble.
2824 void replayPreambleConditionalStack();
2825
2826 // Macro handling.
2827 void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2828 void HandleUndefDirective();
2829
2830 // Conditional Inclusion.
2831 void HandleIfdefDirective(Token &Result, const Token &HashToken,
2832 bool isIfndef, bool ReadAnyTokensBeforeDirective);
2833 void HandleIfDirective(Token &IfToken, const Token &HashToken,
2834 bool ReadAnyTokensBeforeDirective);
2835 void HandleEndifDirective(Token &EndifToken);
2836 void HandleElseDirective(Token &Result, const Token &HashToken);
2837 void HandleElifFamilyDirective(Token &ElifToken, const Token &HashToken,
2838 tok::PPKeywordKind Kind);
2839
2840 // Pragmas.
2841 void HandlePragmaDirective(PragmaIntroducer Introducer);
2842
2843public:
2844 void HandlePragmaOnce(Token &OnceTok);
2845 void HandlePragmaMark(Token &MarkTok);
2846 void HandlePragmaPoison();
2847 void HandlePragmaSystemHeader(Token &SysHeaderTok);
2848 void HandlePragmaDependency(Token &DependencyTok);
2849 void HandlePragmaPushMacro(Token &Tok);
2850 void HandlePragmaPopMacro(Token &Tok);
2852 void HandlePragmaModuleBuild(Token &Tok);
2853 void HandlePragmaHdrstop(Token &Tok);
2855
2856 // Return true and store the first token only if any CommentHandler
2857 // has inserted some tokens and getCommentRetentionState() is false.
2858 bool HandleComment(Token &result, SourceRange Comment);
2859
2860 /// A macro is used, update information about macros that need unused
2861 /// warnings.
2862 void markMacroAsUsed(MacroInfo *MI);
2863
2864 void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg,
2865 SourceLocation AnnotationLoc) {
2866 auto Annotations = AnnotationInfos.find(II);
2867 if (Annotations == AnnotationInfos.end())
2868 AnnotationInfos.insert(std::make_pair(
2869 II,
2870 MacroAnnotations::makeDeprecation(AnnotationLoc, std::move(Msg))));
2871 else
2872 Annotations->second.DeprecationInfo =
2873 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2874 }
2875
2876 void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg,
2877 SourceLocation AnnotationLoc) {
2878 auto Annotations = AnnotationInfos.find(II);
2879 if (Annotations == AnnotationInfos.end())
2880 AnnotationInfos.insert(
2881 std::make_pair(II, MacroAnnotations::makeRestrictExpansion(
2882 AnnotationLoc, std::move(Msg))));
2883 else
2884 Annotations->second.RestrictExpansionInfo =
2885 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2886 }
2887
2888 void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc) {
2889 auto Annotations = AnnotationInfos.find(II);
2890 if (Annotations == AnnotationInfos.end())
2891 AnnotationInfos.insert(
2892 std::make_pair(II, MacroAnnotations::makeFinal(AnnotationLoc)));
2893 else
2894 Annotations->second.FinalAnnotationLoc = AnnotationLoc;
2895 }
2896
2897 const MacroAnnotations &getMacroAnnotations(const IdentifierInfo *II) const {
2898 return AnnotationInfos.find(II)->second;
2899 }
2900
2902 bool IsIfnDef = false) const {
2903 IdentifierInfo *Info = Identifier.getIdentifierInfo();
2904 if (Info->isDeprecatedMacro())
2905 emitMacroDeprecationWarning(Identifier);
2906
2907 if (Info->isRestrictExpansion() &&
2908 !SourceMgr.isInMainFile(Identifier.getLocation()))
2909 emitRestrictExpansionWarning(Identifier);
2910
2911 if (!IsIfnDef) {
2912 if (Info->getName() == "INFINITY" && getLangOpts().NoHonorInfs)
2913 emitRestrictInfNaNWarning(Identifier, 0);
2914 if (Info->getName() == "NAN" && getLangOpts().NoHonorNaNs)
2915 emitRestrictInfNaNWarning(Identifier, 1);
2916 }
2917 }
2918
2920 const LangOptions &LangOpts,
2921 const TargetInfo &TI);
2922
2924 const PresumedLoc &PLoc,
2925 const LangOptions &LangOpts,
2926 const TargetInfo &TI);
2927
2928private:
2929 void emitMacroDeprecationWarning(const Token &Identifier) const;
2930 void emitRestrictExpansionWarning(const Token &Identifier) const;
2931 void emitFinalMacroWarning(const Token &Identifier, bool IsUndef) const;
2932 void emitRestrictInfNaNWarning(const Token &Identifier,
2933 unsigned DiagSelection) const;
2934
2935 /// This boolean state keeps track if the current scanned token (by this PP)
2936 /// is in an "-Wunsafe-buffer-usage" opt-out region. Assuming PP scans a
2937 /// translation unit in a linear order.
2938 bool InSafeBufferOptOutRegion = false;
2939
2940 /// Hold the start location of the current "-Wunsafe-buffer-usage" opt-out
2941 /// region if PP is currently in such a region. Hold undefined value
2942 /// otherwise.
2943 SourceLocation CurrentSafeBufferOptOutStart; // It is used to report the start location of an never-closed region.
2944
2945 using SafeBufferOptOutRegionsTy =
2947 // An ordered sequence of "-Wunsafe-buffer-usage" opt-out regions in this
2948 // translation unit. Each region is represented by a pair of start and
2949 // end locations.
2950 SafeBufferOptOutRegionsTy SafeBufferOptOutMap;
2951
2952 // The "-Wunsafe-buffer-usage" opt-out regions in loaded ASTs. We use the
2953 // following structure to manage them by their ASTs.
2954 struct {
2955 // A map from unique IDs to region maps of loaded ASTs. The ID identifies a
2956 // loaded AST. See `SourceManager::getUniqueLoadedASTID`.
2957 llvm::DenseMap<FileID, SafeBufferOptOutRegionsTy> LoadedRegions;
2958
2959 // Returns a reference to the safe buffer opt-out regions of the loaded
2960 // AST where `Loc` belongs to. (Construct if absent)
2962 findAndConsLoadedOptOutMap(SourceLocation Loc, SourceManager &SrcMgr) {
2964 }
2965
2966 // Returns a reference to the safe buffer opt-out regions of the loaded
2967 // AST where `Loc` belongs to. (This const function returns nullptr if
2968 // absent.)
2969 const SafeBufferOptOutRegionsTy *
2970 lookupLoadedOptOutMap(SourceLocation Loc,
2971 const SourceManager &SrcMgr) const {
2972 FileID FID = SrcMgr.getUniqueLoadedASTFileID(Loc);
2973 auto Iter = LoadedRegions.find(FID);
2974
2975 if (Iter == LoadedRegions.end())
2976 return nullptr;
2977 return &Iter->getSecond();
2978 }
2979 } LoadedSafeBufferOptOutMap;
2980
2981public:
2982 /// \return true iff the given `Loc` is in a "-Wunsafe-buffer-usage" opt-out
2983 /// region. This `Loc` must be a source location that has been pre-processed.
2984 bool isSafeBufferOptOut(const SourceManager&SourceMgr, const SourceLocation &Loc) const;
2985
2986 /// Alter the state of whether this PP currently is in a
2987 /// "-Wunsafe-buffer-usage" opt-out region.
2988 ///
2989 /// \param isEnter true if this PP is entering a region; otherwise, this PP
2990 /// is exiting a region
2991 /// \param Loc the location of the entry or exit of a
2992 /// region
2993 /// \return true iff it is INVALID to enter or exit a region, i.e.,
2994 /// attempt to enter a region before exiting a previous region, or exiting a
2995 /// region that PP is not currently in.
2996 bool enterOrExitSafeBufferOptOutRegion(bool isEnter,
2997 const SourceLocation &Loc);
2998
2999 /// \return true iff this PP is currently in a "-Wunsafe-buffer-usage"
3000 /// opt-out region
3002
3003 /// \param StartLoc output argument. It will be set to the start location of
3004 /// the current "-Wunsafe-buffer-usage" opt-out region iff this function
3005 /// returns true.
3006 /// \return true iff this PP is currently in a "-Wunsafe-buffer-usage"
3007 /// opt-out region
3008 bool isPPInSafeBufferOptOutRegion(SourceLocation &StartLoc);
3009
3010 /// \return a sequence of SourceLocations representing ordered opt-out regions
3011 /// specified by
3012 /// `\#pragma clang unsafe_buffer_usage begin/end`s of this translation unit.
3013 SmallVector<SourceLocation, 64> serializeSafeBufferOptOutMap() const;
3014
3015 /// \param SrcLocSeqs a sequence of SourceLocations deserialized from a
3016 /// record of code `PP_UNSAFE_BUFFER_USAGE`.
3017 /// \return true iff the `Preprocessor` has been updated; false `Preprocessor`
3018 /// is same as itself before the call.
3020 const SmallVectorImpl<SourceLocation> &SrcLocSeqs);
3021
3022private:
3023 /// Helper functions to forward lexing to the actual lexer. They all share the
3024 /// same signature.
3025 static bool CLK_Lexer(Preprocessor &P, Token &Result) {
3026 return P.CurLexer->Lex(Result);
3027 }
3028 static bool CLK_TokenLexer(Preprocessor &P, Token &Result) {
3029 return P.CurTokenLexer->Lex(Result);
3030 }
3031 static bool CLK_CachingLexer(Preprocessor &P, Token &Result) {
3032 P.CachingLex(Result);
3033 return true;
3034 }
3035 static bool CLK_DependencyDirectivesLexer(Preprocessor &P, Token &Result) {
3036 return P.CurLexer->LexDependencyDirectiveToken(Result);
3037 }
3038 static bool CLK_LexAfterModuleImport(Preprocessor &P, Token &Result) {
3039 return P.LexAfterModuleImport(Result);
3040 }
3041};
3042
3043/// Abstract base class that describes a handler that will receive
3044/// source ranges for each of the comments encountered in the source file.
3046public:
3048
3049 // The handler shall return true if it has pushed any tokens
3050 // to be read using e.g. EnterToken or EnterTokenStream.
3051 virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
3052};
3053
3054/// Abstract base class that describes a handler that will receive
3055/// source ranges for empty lines encountered in the source file.
3057public:
3059
3060 // The handler handles empty lines.
3062};
3063
3064/// Helper class to shuttle information about #embed directives from the
3065/// preprocessor to the parser through an annotation token.
3067 StringRef BinaryData;
3068};
3069
3070/// Registry of pragma handlers added by plugins
3071using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
3072
3073} // namespace clang
3074
3075#endif // LLVM_CLANG_LEX_PREPROCESSOR_H
#define V(N, I)
Definition: ASTContext.h:3300
StringRef P
#define SM(sm)
Definition: Cuda.cpp:84
Defines the Diagnostic-related interfaces.
static constexpr Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:32
Defines the Diagnostic IDs-related interfaces.
StringRef Filename
Definition: Format.cpp:2976
StringRef Identifier
Definition: Format.cpp:2984
unsigned Iter
Definition: HTMLLogger.cpp:154
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
llvm::MachO::Target Target
Definition: MachO.h:51
llvm::MachO::Record Record
Definition: MachO.h:31
Defines the clang::MacroInfo and clang::MacroDirective classes.
Defines the clang::Module class, which describes a module in the source code.
Defines the PPCallbacks interface.
uint32_t Id
Definition: SemaARM.cpp:1140
SourceRange Range
Definition: SemaObjC.cpp:757
SourceLocation Loc
Definition: SemaObjC.cpp:758
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
Defines the clang::TokenKind enum and support functions.
__device__ __2f16 float __ockl_bool s
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:366
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:85
Callback handler that receives notifications when performing code completion within the preprocessor.
Abstract base class that describes a handler that will receive source ranges for each of the comments...
virtual bool HandleComment(Preprocessor &PP, SourceRange Comment)=0
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:432
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:1271
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1547
void setSuppressAllDiagnostics(bool Val)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
Definition: Diagnostic.h:697
A reference to a DirectoryEntry that includes the name of the directory as it was accessed by the Fil...
Abstract base class that describes a handler that will receive source ranges for empty lines encounte...
virtual void HandleEmptyline(SourceRange Range)=0
Abstract interface for external sources of preprocessor information.
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
Definition: FileEntry.h:57
Cached information about one file (either on disk or in the virtual file system).
Definition: FileEntry.h:300
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Implements support for file system lookup, file system caching, and directory search management.
Definition: FileManager.h:53
Encapsulates the information needed to find the file referenced by a #include or #include_next,...
Definition: HeaderSearch.h:249
HeaderFileInfo & getFileInfo(FileEntryRef FE)
Return the HeaderFileInfo structure for the specified FileEntry, in preparation for updating it in so...
One of these records is kept for each identifier that is lexed.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
bool isDeprecatedMacro() const
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
StringRef getName() const
Return the actual identifier string.
bool isRestrictExpansion() const
Implements an efficient mapping from strings to IdentifierInfo nodes.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
FPEvalMethodKind
Possible float expression evaluation method choices.
Definition: LangOptions.h:288
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
Definition: LangOptions.h:302
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:461
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
Definition: Lexer.cpp:1060
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion.
Definition: Lexer.cpp:872
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Characters, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token,...
Definition: Lexer.h:399
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
Definition: Lexer.cpp:894
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer,...
Definition: Lexer.cpp:452
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
Definition: Lexer.cpp:510
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
Definition: Lexer.cpp:850
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
A description of the current definition of a macro.
Definition: MacroInfo.h:590
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:375
Encapsulates changes to the "macros namespace" (the location where the macro name became active,...
Definition: MacroInfo.h:313
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:125
Abstract interface for a module loader.
Definition: ModuleLoader.h:82
Represents a macro directive exported by a module.
Definition: MacroInfo.h:514
A header that is known to reside within a given module, whether it was included or excluded.
Definition: ModuleMap.h:159
Describes a module or submodule.
Definition: Module.h:105
bool isModuleMapModule() const
Definition: Module.h:212
This interface provides a way to observe the actions of the preprocessor as it does its thing.
Definition: PPCallbacks.h:36
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:65
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:137
SourceLocation getLastFPEvalPragmaLocation() const
bool isMacroDefined(const IdentifierInfo *II)
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
bool markIncluded(FileEntryRef File)
Mark the file as included.
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
Definition: Pragma.cpp:627
void FinalizeForModelFile()
Cleanup after model file parsing.
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void HandlePragmaPoison()
HandlePragmaPoison - Handle #pragma GCC poison. PoisonTok is the 'poison'.
Definition: Pragma.cpp:442
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void dumpMacroInfo(const IdentifierInfo *II)
std::pair< IdentifierInfo *, SourceLocation > getPragmaARCCFCodeAuditedInfo() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaSystemHeader(Token &SysHeaderTok)
HandlePragmaSystemHeader - Implement #pragma GCC system_header.
Definition: Pragma.cpp:484
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
void markClangModuleAsAffecting(Module *M)
Mark the given clang module as affecting the current clang module or translation unit.
void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident, SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaModuleBuild(Token &Tok)
Definition: Pragma.cpp:807
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
OptionalFileEntryRef LookupEmbedFile(StringRef Filename, bool isAngled, bool OpenFile, const FileEntry *LookupFromFile=nullptr)
Given a "Filename" or <Filename> reference, look up the indicated embed resource.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
Definition: Pragma.cpp:2196
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
PPCallbacks * getPPCallbacks() const
bool isInNamedInterfaceUnit() const
If we are proprocessing a named interface unit.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
void setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc)
Record the location of the unterminated #pragma clang assume_nonnull begin in the preamble.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
ArrayRef< BuildingSubmoduleInfo > getBuildingSubmodules() const
Get the list of submodules that we're currently building.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
SourceRange getCodeCompletionTokenRange() const
SourceLocation getModuleImportLoc(Module *M) const
void overrideMaxTokens(unsigned Value, SourceLocation Loc)
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
bool isRecordingPreamble() const
void HandleSkippedDirectiveWhileUsingPCH(Token &Result, SourceLocation HashLoc)
Process directives while skipping until the through header or #pragma hdrstop is found.
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
bool GetSuppressIncludeNotFoundError()
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
LangOptions::FPEvalMethodKind getCurrentFPEvalMethod() const
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
bool isSafeBufferOptOut(const SourceManager &SourceMgr, const SourceLocation &Loc) const
void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
const char * getCheckPoint(FileID FID, const char *Start) const
Returns a pointer into the given file's buffer that's guaranteed to be between tokens.
void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc)
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
Definition: PPCaching.cpp:139
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
const TargetInfo * getAuxTargetInfo() const
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
Definition: PPCaching.cpp:32
friend class MacroArgs
Definition: Preprocessor.h:703
void DumpMacro(const MacroInfo &MI) const
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
void setDiagnostics(DiagnosticsEngine &D)
IncludedFilesSet & getIncludedFiles()
Get the set of included files.
friend void TokenLexer::ExpandFunctionArguments()
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool SetCodeCompletionPoint(FileEntryRef File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
StringRef getNamedModuleName() const
Get the named module name we're preprocessing.
void makeModuleVisible(Module *M, SourceLocation Loc)
unsigned getCounterValue() const
bool mightHavePendingAnnotationTokens()
Determine whether it's possible for a future call to Lex to produce an annotation token created by a ...
bool isInImportingCXXNamedModules() const
If we're importing a standard C++20 Named Modules.
void Lex(Token &Result)
Lex the next token for this preprocessor.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
Definition: Preprocessor.h:295
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
unsigned getNumDirectives() const
Retrieve the number of Directives that have been processed by the Preprocessor.
bool isInImplementationUnit() const
If we are implementing an implementation module unit.
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
void LexNonComment(Token &Result)
Lex a token.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
SourceRange DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found.
bool LexOnOffSwitch(tok::OnOffSwitch &Result)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD.
Definition: Pragma.cpp:968
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
void setMainFileDir(DirectoryEntryRef Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
const IdentifierTable & getIdentifierTable() const
void HandlePragmaDependency(Token &DependencyTok)
HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
Definition: Pragma.cpp:516
SourceLocation CheckEndOfDirective(const char *DirType, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
Definition: PPCaching.cpp:40
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
bool enterOrExitSafeBufferOptOutRegion(bool isEnter, const SourceLocation &Loc)
Alter the state of whether this PP currently is in a "-Wunsafe-buffer-usage" opt-out region.
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, std::optional< PreambleSkipInfo > SkipInfo)
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
macro_iterator macro_end(bool IncludeExternalMacros=true) const
SourceManager & getSourceManager() const
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
std::optional< PreambleSkipInfo > getPreambleSkipInfo() const
void setPreprocessToken(bool Preprocess)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
void HandlePragmaOnce(Token &OnceTok)
HandlePragmaOnce - Handle #pragma once. OnceTok is the 'once'.
Definition: Pragma.cpp:417
EmptylineHandler * getEmptylineHandler() const
bool getCommentRetentionState() const
bool isMacroDefined(StringRef Id)
static bool checkModuleIsAvailable(const LangOptions &LangOpts, const TargetInfo &TargetInfo, const Module &M, DiagnosticsEngine &Diags)
Check that the given module is available, producing a diagnostic if not.
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool hasRecordedPreamble() const
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
MacroMap::const_iterator macro_iterator
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
void RemovePragmaHandler(PragmaHandler *Handler)
bool isPPInSafeBufferOptOutRegion()
unsigned getTokenCount() const
Get the number of tokens processed so far.
unsigned getMaxTokens() const
Get the max number of tokens before issuing a -Wmax-tokens warning.
SourceLocation getMaxTokensOverrideLoc() const
bool hadModuleLoaderFatalFailure() const
static void processPathToFileName(SmallVectorImpl< char > &FileName, const PresumedLoc &PLoc, const LangOptions &LangOpts, const TargetInfo &TI)
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
const TargetInfo & getTargetInfo() const
FileManager & getFileManager() const
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void DumpLocation(SourceLocation Loc) const
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
bool alreadyIncluded(FileEntryRef File) const
Return true if this header has already been included.
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character.
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
Definition: Pragma.cpp:915
llvm::BumpPtrAllocator & getPreprocessorAllocator()
ModuleMacro * getModuleMacro(Module *Mod, const IdentifierInfo *II)
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleComment(Token &result, SourceRange Comment)
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Buffer)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
HeaderSearch & getHeaderSearchInfo() const
void emitMacroExpansionWarnings(const Token &Identifier, bool IsIfnDef=false) const
bool setDeserializedSafeBufferOptOutMap(const SmallVectorImpl< SourceLocation > &SrcLocSeqs)
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
Definition: Pragma.cpp:650
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
ExternalPreprocessorSource * getExternalSource() const
Module * LeaveSubmodule(bool ForPragma)
const std::string & getPredefines() const
Get the predefines for this processor.
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
SmallVector< SourceLocation, 64 > serializeSafeBufferOptOutMap() const
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind, void *AnnotationVal)
Enter an annotation token into the token stream.
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, const LangOptions &LangOpts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
MacroInfo * getMacroInfo(const IdentifierInfo *II)
void setPredefines(std::string P)
Set the predefines for this Preprocessor.
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
IdentifierTable & getIdentifierTable()
Builtin::Context & getBuiltinInfo()
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
Definition: PPCaching.cpp:157
LangOptions::FPEvalMethodKind getTUFPEvalMethod() const
const LangOptions & getLangOpts() const
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void SetSuppressIncludeNotFoundError(bool Suppress)
static void processPathForFileMacro(SmallVectorImpl< char > &Path, const LangOptions &LangOpts, const TargetInfo &TI)
llvm::DenseMap< FileID, SafeBufferOptOutRegionsTy > LoadedRegions
bool isInNamedModule() const
If we are preprocessing a named module.
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
void PoisonSEHIdentifiers(bool Poison=true)
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion.
size_t getTotalMemory() const
void setExternalSource(ExternalPreprocessorSource *Source)
void clearCodeCompletionHandler()
Clear out the code completion handler.
void AddPragmaHandler(PragmaHandler *Handler)
OptionalFileEntryRef getHeaderToIncludeForDiagnostics(SourceLocation IncLoc, SourceLocation MLoc)
We want to produce a diagnostic at location IncLoc concerning an unreachable effect at location MLoc ...
void setCounterValue(unsigned V)
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.
Definition: Pragma.cpp:572
void LexTokensUntilEOF(std::vector< Token > *Tokens=nullptr)
Lex all tokens for this preprocessor until (and excluding) end of file.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
void setPragmasEnabled(bool Enabled)
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void HandlePragmaMark(Token &MarkTok)
Definition: Pragma.cpp:432
bool getPragmasEnabled() const
void HandlePragmaHdrstop(Token &Tok)
Definition: Pragma.cpp:881
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
void setEmptylineHandler(EmptylineHandler *Handler)
Set empty line handler.
DiagnosticsEngine & getDiagnostics() const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
llvm::DenseSet< const FileEntry * > IncludedFilesSet
Definition: Preprocessor.h:699
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SelectorTable & getSelectorTable()
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
Definition: Pragma.cpp:946
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
const llvm::SmallSetVector< Module *, 2 > & getAffectingClangModules() const
Get the set of top-level clang modules that affected preprocessing, but were not imported.
std::optional< LexEmbedParametersResult > LexEmbedParameters(Token &Current, bool ForHasEmbed)
Lex the parameters for an #embed directive, returns nullopt on error.
const IncludedFilesSet & getIncludedFiles() const
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void HandlePragmaIncludeAlias(Token &Tok)
Definition: Pragma.cpp:685
Module * getModuleForLocation(SourceLocation Loc, bool AllowTextual)
Find the module that owns the source or header file that Loc points to.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
SourceLocation getPreambleRecordedPragmaAssumeNonNullLoc() const
Get the location of the recorded unterminated #pragma clang assume_nonnull begin in the preamble,...
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
void EnableBacktrackAtThisPos()
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
Definition: PPCaching.cpp:25
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop.
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
Represents an unpacked "presumed" location which can be presented to the user.
This table allows us to fully hide how we implement multi-keyword caching.
Encodes a location in the source.
This class handles loading and caching of source files into memory.
FileID getUniqueLoadedASTFileID(SourceLocation Loc) const
A trivial tuple used to represent a source range.
Exposes information about the current target.
Definition: TargetInfo.h:218
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:91
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:104
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:96
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:106
Token - This structure provides full information about a lexed token.
Definition: Token.h:36
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:187
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:132
unsigned getLength() const
Definition: Token.h:135
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:99
tok::TokenKind getKind() const
Definition: Token.h:94
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:121
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:295
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:225
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a variadic mac...
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
Directive - Abstract class representing a parsed verify directive.
constexpr XRayInstrMask None
Definition: XRayInstr.h:38
StringRef getName(const HeaderType T)
Definition: HeaderFile.h:38
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:56
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:97
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a '#' at the beginning of the line.
Definition: TokenKinds.h:33
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.cpp:58
@ HeaderSearch
Remove unused header search paths including header maps.
The JSON file list parser is used to communicate input to InstallAPI.
CustomizableOptional< DirectoryEntryRef > OptionalDirectoryEntryRef
detail::SearchDirIteratorImpl< true > ConstSearchDirIterator
Definition: HeaderSearch.h:241
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:113
@ MU_Undef
Definition: Preprocessor.h:121
@ MU_Other
Definition: Preprocessor.h:115
@ MU_Define
Definition: Preprocessor.h:118
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
@ Module
Module linkage, which indicates that the entity can be referred to from other translation units withi...
@ Result
The result type of a method or function.
CustomizableOptional< FileEntryRef > OptionalFileEntryRef
Definition: FileEntry.h:202
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:1033
@ TU_Complete
The translation unit is a complete translation unit.
Definition: LangOptions.h:1035
Diagnostic wrappers for TextAPI types for error reporting.
Definition: Dominators.h:30
#define bool
Definition: stdbool.h:24
Helper class to shuttle information about #embed directives from the preprocessor to the parser throu...
Describes how and where the pragma was introduced.
Definition: Pragma.h:51
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
Definition: Preprocessor.h:691