14#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15#define LLVM_CLANG_LEX_PREPROCESSOR_H
35#include "llvm/ADT/APSInt.h"
36#include "llvm/ADT/ArrayRef.h"
37#include "llvm/ADT/DenseMap.h"
38#include "llvm/ADT/FoldingSet.h"
39#include "llvm/ADT/FunctionExtras.h"
40#include "llvm/ADT/PointerUnion.h"
41#include "llvm/ADT/STLExtras.h"
42#include "llvm/ADT/SmallPtrSet.h"
43#include "llvm/ADT/SmallVector.h"
44#include "llvm/ADT/StringRef.h"
45#include "llvm/ADT/TinyPtrVector.h"
46#include "llvm/ADT/iterator_range.h"
47#include "llvm/Support/Allocator.h"
48#include "llvm/Support/Casting.h"
49#include "llvm/Support/Registry.h"
62template<
unsigned InternalLen>
class SmallString;
68class CodeCompletionHandler;
71class EmptylineHandler;
72class ExternalPreprocessorSource;
79class PreprocessingRecord;
80class PreprocessorLexer;
81class PreprocessorOptions;
97 assert(Kind != tok::raw_identifier &&
"Raw identifiers are not supported.");
98 assert(Kind != tok::identifier &&
99 "Identifiers should be created by TokenValue(IdentifierInfo *)");
107 return Tok.
getKind() == Kind &&
141 llvm::unique_function<void(
const clang::Token &)> OnToken;
142 std::shared_ptr<PreprocessorOptions> PPOpts;
149 std::unique_ptr<ScratchBuffer> ScratchBuf;
158 llvm::BumpPtrAllocator BP;
215 unsigned CounterValue = 0;
219 MaxAllowedIncludeStackDepth = 200
223 bool KeepComments : 1;
224 bool KeepMacroComments : 1;
225 bool SuppressIncludeNotFoundError : 1;
228 bool InMacroArgs : 1;
231 bool OwnsHeaderSearch : 1;
234 bool DisableMacroExpansion : 1;
238 bool MacroExpansionInDirectivesOverride : 1;
240 class ResetMacroExpansionHelper;
243 mutable bool ReadMacrosFromExternalSource : 1;
246 bool PragmasEnabled : 1;
249 bool PreprocessedOutput : 1;
252 bool ParsingIfOrElifDirective;
255 bool InMacroArgPreExpansion;
277 std::unique_ptr<PragmaNamespace> PragmaHandlers;
281 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
285 std::vector<CommentHandler *> CommentHandlers;
291 bool IncrementalProcessing =
false;
309 const FileEntry *CodeCompletionFile =
nullptr;
312 unsigned CodeCompletionOffset = 0;
331 llvm::DenseMap<FileID, SmallVector<const char *>> CheckPoints;
332 unsigned CheckPointCounter = 0;
335 bool IsAtImport =
false;
338 bool LastTokenWasAt =
false;
341 class StdCXXImportSeq {
346 AfterTopLevelTokenSeq = -1,
351 StdCXXImportSeq(State S) : S(S) {}
354 void handleOpenBracket() {
355 S =
static_cast<State
>(std::max<int>(S, 0) + 1);
358 void handleCloseBracket() {
359 S =
static_cast<State
>(std::max<int>(S, 1) - 1);
362 void handleCloseBrace() {
363 handleCloseBracket();
364 if (S == AtTopLevel && !AfterHeaderName)
365 S = AfterTopLevelTokenSeq;
370 S = AfterTopLevelTokenSeq;
371 AfterHeaderName =
false;
376 void handleExport() {
377 if (S == AfterTopLevelTokenSeq)
383 void handleImport() {
384 if (S == AfterTopLevelTokenSeq || S == AfterExport)
392 void handleHeaderName() {
393 if (S == AfterImportSeq)
394 AfterHeaderName =
true;
404 bool atTopLevel() {
return S <= 0; }
405 bool afterImportSeq() {
return S == AfterImportSeq; }
406 bool afterTopLevelSeq() {
return S == AfterTopLevelTokenSeq; }
413 bool AfterHeaderName =
false;
417 StdCXXImportSeq StdCXXImportSeqState = StdCXXImportSeq::AfterTopLevelTokenSeq;
422 enum GMFState :
int {
425 BeforeGMFIntroducer = -1,
426 GMFAbsentOrEnded = -2,
429 TrackGMF(GMFState S) : S(S) {}
440 void handleExport() {
442 S = GMFAbsentOrEnded;
446 void handleImport(
bool AfterTopLevelTokenSeq) {
448 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
449 S = GMFAbsentOrEnded;
453 void handleModule(
bool AfterTopLevelTokenSeq) {
457 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
460 S = GMFAbsentOrEnded;
467 S = GMFAbsentOrEnded;
470 bool inGMF() {
return S == GMFActive; }
478 TrackGMF TrackGMFState = TrackGMF::BeforeGMFIntroducer;
514 class ModuleDeclSeq {
515 enum ModuleDeclState :
int {
519 ImplementationCandidate,
520 NamedModuleInterface,
521 NamedModuleImplementation,
525 ModuleDeclSeq() =
default;
527 void handleExport() {
528 if (State == NotAModuleDecl)
530 else if (!isNamedModule())
534 void handleModule() {
535 if (State == FoundExport)
536 State = InterfaceCandidate;
537 else if (State == NotAModuleDecl)
538 State = ImplementationCandidate;
539 else if (!isNamedModule())
543 void handleIdentifier(IdentifierInfo *
Identifier) {
546 else if (!isNamedModule())
551 if (isModuleCandidate())
553 else if (!isNamedModule())
557 void handlePeriod() {
558 if (isModuleCandidate())
560 else if (!isNamedModule())
565 if (!Name.empty() && isModuleCandidate()) {
566 if (State == InterfaceCandidate)
567 State = NamedModuleInterface;
568 else if (State == ImplementationCandidate)
569 State = NamedModuleImplementation;
571 llvm_unreachable(
"Unimaged ModuleDeclState.");
572 }
else if (!isNamedModule())
577 if (!isNamedModule())
581 bool isModuleCandidate()
const {
582 return State == InterfaceCandidate || State == ImplementationCandidate;
585 bool isNamedModule()
const {
586 return State == NamedModuleInterface ||
587 State == NamedModuleImplementation;
590 bool isNamedInterface()
const {
return State == NamedModuleInterface; }
592 bool isImplementationUnit()
const {
593 return State == NamedModuleImplementation && !
getName().contains(
':');
597 assert(isNamedModule() &&
"Can't get name from a non named module");
601 StringRef getPrimaryName()
const {
602 assert(isNamedModule() &&
"Can't get name from a non named module");
603 return getName().split(
':').first;
608 State = NotAModuleDecl;
612 ModuleDeclState State = NotAModuleDecl;
616 ModuleDeclSeq ModuleDeclState;
620 bool ModuleImportExpectsIdentifier =
false;
624 std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
628 SourceLocation PragmaAssumeNonNullLoc;
636 SourceLocation PreambleRecordedPragmaAssumeNonNullLoc;
639 bool CodeCompletionReached =
false;
643 IdentifierInfo *CodeCompletionII =
nullptr;
646 SourceRange CodeCompletionTokenRange;
658 std::pair<int, bool> SkipMainFilePreamble;
662 bool HasReachedMaxIncludeDepth =
false;
670 unsigned LexLevel = 0;
673 unsigned TokenCount = 0;
676 bool PreprocessToken =
false;
680 unsigned MaxTokens = 0;
681 SourceLocation MaxTokensOverrideLoc;
705 class PreambleConditionalStackStore {
713 PreambleConditionalStackStore() =
default;
715 void startRecording() { ConditionalStackState = Recording; }
716 void startReplaying() { ConditionalStackState = Replaying; }
717 bool isRecording()
const {
return ConditionalStackState == Recording; }
718 bool isReplaying()
const {
return ConditionalStackState == Replaying; }
720 ArrayRef<PPConditionalInfo> getStack()
const {
721 return ConditionalStack;
724 void doneReplaying() {
725 ConditionalStack.clear();
726 ConditionalStackState = Off;
729 void setStack(ArrayRef<PPConditionalInfo>
s) {
730 if (!isRecording() && !isReplaying())
732 ConditionalStack.clear();
733 ConditionalStack.append(
s.begin(),
s.end());
736 bool hasRecordedPreamble()
const {
return !ConditionalStack.empty(); }
738 bool reachedEOFWhileSkipping()
const {
return SkipInfo.has_value(); }
740 void clearSkipInfo() { SkipInfo.reset(); }
742 std::optional<PreambleSkipInfo> SkipInfo;
745 SmallVector<PPConditionalInfo, 4> ConditionalStack;
746 State ConditionalStackState = Off;
747 } PreambleConditionalStack;
753 std::unique_ptr<Lexer> CurLexer;
759 PreprocessorLexer *CurPPLexer =
nullptr;
771 std::unique_ptr<TokenLexer> CurTokenLexer;
775 LexerCallback CurLexerCallback = &CLK_Lexer;
779 Module *CurLexerSubmodule =
nullptr;
784 struct IncludeStackInfo {
785 LexerCallback CurLexerCallback;
787 std::unique_ptr<Lexer> TheLexer;
788 PreprocessorLexer *ThePPLexer;
789 std::unique_ptr<TokenLexer> TheTokenLexer;
794 IncludeStackInfo(LexerCallback CurLexerCallback, Module *TheSubmodule,
795 std::unique_ptr<Lexer> &&TheLexer,
796 PreprocessorLexer *ThePPLexer,
797 std::unique_ptr<TokenLexer> &&TheTokenLexer,
799 : CurLexerCallback(
std::move(CurLexerCallback)),
800 TheSubmodule(
std::move(TheSubmodule)), TheLexer(
std::move(TheLexer)),
801 ThePPLexer(
std::move(ThePPLexer)),
802 TheTokenLexer(
std::move(TheTokenLexer)),
803 TheDirLookup(
std::move(TheDirLookup)) {}
805 std::vector<IncludeStackInfo> IncludeMacroStack;
809 std::unique_ptr<PPCallbacks> Callbacks;
811 struct MacroExpandsInfo {
816 MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
819 SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
822 struct ModuleMacroInfo {
827 llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
831 unsigned ActiveModuleMacrosGeneration = 0;
834 bool IsAmbiguous =
false;
837 llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
839 ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
844 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
846 ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
847 const IdentifierInfo *II)
const {
848 if (II->isOutOfDate())
849 PP.updateOutOfDateIdentifier(*II);
852 if (!II->hasMacroDefinition() ||
853 (!PP.getLangOpts().Modules &&
854 !PP.getLangOpts().ModulesLocalVisibility) ||
855 !PP.CurSubmoduleState->VisibleModules.getGeneration())
858 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
860 Info =
new (PP.getPreprocessorAllocator())
861 ModuleMacroInfo(State.get<MacroDirective *>());
865 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
866 Info->ActiveModuleMacrosGeneration)
867 PP.updateModuleMacroInfo(II, *Info);
872 MacroState() : MacroState(nullptr) {}
873 MacroState(MacroDirective *MD) : State(MD) {}
875 MacroState(MacroState &&O) noexcept : State(O.State) {
876 O.State = (MacroDirective *)
nullptr;
879 MacroState &operator=(MacroState &&O)
noexcept {
881 O.State = (MacroDirective *)
nullptr;
887 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
888 Info->~ModuleMacroInfo();
891 MacroDirective *getLatest()
const {
892 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
894 return State.get<MacroDirective*>();
897 void setLatest(MacroDirective *MD) {
898 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
904 bool isAmbiguous(Preprocessor &PP,
const IdentifierInfo *II)
const {
905 auto *Info = getModuleInfo(PP, II);
906 return Info ? Info->IsAmbiguous :
false;
909 ArrayRef<ModuleMacro *>
910 getActiveModuleMacros(Preprocessor &PP,
const IdentifierInfo *II)
const {
911 if (
auto *Info = getModuleInfo(PP, II))
912 return Info->ActiveModuleMacros;
916 MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation
Loc,
917 SourceManager &SourceMgr)
const {
919 if (
auto *Latest = getLatest())
920 return Latest->findDirectiveAtLoc(
Loc, SourceMgr);
924 void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
925 if (
auto *Info = getModuleInfo(PP, II)) {
926 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
927 Info->ActiveModuleMacros.begin(),
928 Info->ActiveModuleMacros.end());
929 Info->ActiveModuleMacros.clear();
930 Info->IsAmbiguous =
false;
934 ArrayRef<ModuleMacro*> getOverriddenMacros()
const {
935 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
936 return Info->OverriddenMacros;
940 void setOverriddenMacros(Preprocessor &PP,
941 ArrayRef<ModuleMacro *> Overrides) {
942 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
944 if (Overrides.empty())
946 Info =
new (PP.getPreprocessorAllocator())
947 ModuleMacroInfo(State.get<MacroDirective *>());
950 Info->OverriddenMacros.clear();
951 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
952 Overrides.begin(), Overrides.end());
953 Info->ActiveModuleMacrosGeneration = 0;
962 using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
964 struct SubmoduleState;
967 struct BuildingSubmoduleInfo {
972 SourceLocation ImportLoc;
978 SubmoduleState *OuterSubmoduleState;
981 unsigned OuterPendingModuleMacroNames;
983 BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc,
bool IsPragma,
984 SubmoduleState *OuterSubmoduleState,
985 unsigned OuterPendingModuleMacroNames)
986 : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
987 OuterSubmoduleState(OuterSubmoduleState),
988 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
990 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
993 struct SubmoduleState {
998 VisibleModuleSet VisibleModules;
1003 std::map<Module *, SubmoduleState> Submodules;
1006 SubmoduleState NullSubmoduleState;
1010 SubmoduleState *CurSubmoduleState;
1020 llvm::FoldingSet<ModuleMacro> ModuleMacros;
1027 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
1038 using WarnUnusedMacroLocsTy = llvm::SmallDenseSet<SourceLocation, 32>;
1039 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
1045 using MsgLocationPair = std::pair<std::string, SourceLocation>;
1047 struct MacroAnnotationInfo {
1048 SourceLocation Location;
1049 std::string Message;
1052 struct MacroAnnotations {
1053 std::optional<MacroAnnotationInfo> DeprecationInfo;
1054 std::optional<MacroAnnotationInfo> RestrictExpansionInfo;
1055 std::optional<SourceLocation> FinalAnnotationLoc;
1057 static MacroAnnotations makeDeprecation(SourceLocation
Loc,
1059 return MacroAnnotations{MacroAnnotationInfo{
Loc, std::move(Msg)},
1060 std::nullopt, std::nullopt};
1063 static MacroAnnotations makeRestrictExpansion(SourceLocation
Loc,
1065 return MacroAnnotations{
1066 std::nullopt, MacroAnnotationInfo{
Loc, std::move(Msg)}, std::nullopt};
1069 static MacroAnnotations makeFinal(SourceLocation
Loc) {
1070 return MacroAnnotations{std::nullopt, std::nullopt,
Loc};
1075 llvm::DenseMap<const IdentifierInfo *, MacroAnnotations> AnnotationInfos;
1083 llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
1084 PragmaPushMacroInfo;
1087 unsigned NumDirectives = 0;
1088 unsigned NumDefined = 0;
1089 unsigned NumUndefined = 0;
1090 unsigned NumPragma = 0;
1092 unsigned NumElse = 0;
1093 unsigned NumEndif = 0;
1094 unsigned NumEnteredSourceFiles = 0;
1095 unsigned MaxIncludeStackDepth = 0;
1096 unsigned NumMacroExpanded = 0;
1097 unsigned NumFnMacroExpanded = 0;
1098 unsigned NumBuiltinMacroExpanded = 0;
1099 unsigned NumFastMacroExpanded = 0;
1100 unsigned NumTokenPaste = 0;
1101 unsigned NumFastTokenPaste = 0;
1102 unsigned NumSkipped = 0;
1106 std::string Predefines;
1109 FileID PredefinesFileID;
1112 FileID PCHThroughHeaderFileID;
1115 bool SkippingUntilPragmaHdrStop =
false;
1118 bool SkippingUntilPCHThroughHeader =
false;
1122 enum { TokenLexerCacheSize = 8 };
1123 unsigned NumCachedTokenLexers;
1124 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
1132 SmallVector<Token, 16> MacroExpandedTokens;
1133 std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
1140 PreprocessingRecord *
Record =
nullptr;
1143 using CachedTokensTy = SmallVector<Token, 1>;
1147 CachedTokensTy CachedTokens;
1154 CachedTokensTy::size_type CachedLexPos = 0;
1161 std::vector<CachedTokensTy::size_type> BacktrackPositions;
1165 std::vector<std::pair<CachedTokensTy, CachedTokensTy::size_type>>
1166 UnannotatedBacktrackTokens;
1172 bool SkippingExcludedConditionalBlock =
false;
1178 llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges;
1180 void updateOutOfDateIdentifier(
const IdentifierInfo &II)
const;
1183 Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
1184 DiagnosticsEngine &diags,
const LangOptions &LangOpts,
1185 SourceManager &
SM, HeaderSearch &Headers,
1186 ModuleLoader &TheModuleLoader,
1187 IdentifierInfoLookup *IILookup =
nullptr,
1188 bool OwnsHeaderSearch =
false,
1200 const TargetInfo *AuxTarget =
nullptr);
1251 return NumDirectives;
1256 return ParsingIfOrElifDirective;
1261 this->KeepComments = KeepComments | KeepMacroComments;
1262 this->KeepMacroComments = KeepMacroComments;
1271 SuppressIncludeNotFoundError = Suppress;
1275 return SuppressIncludeNotFoundError;
1281 PreprocessedOutput = IsPreprocessedOutput;
1290 return CurPPLexer == L;
1320 C = std::make_unique<PPChainedCallbacks>(std::move(
C),
1321 std::move(Callbacks));
1322 Callbacks = std::move(
C);
1334 MaxTokensOverrideLoc =
Loc;
1343 OnToken = std::move(F);
1362 auto I = Submodules.find(M);
1363 if (I == Submodules.end())
1365 auto J = I->second.Macros.find(II);
1366 if (J == I->second.Macros.end())
1368 auto *MD = J->second.getLatest();
1369 return MD && MD->isDefined();
1376 MacroState &S = CurSubmoduleState->Macros[II];
1377 auto *MD = S.getLatest();
1378 while (isa_and_nonnull<VisibilityMacroDirective>(MD))
1379 MD = MD->getPrevious();
1381 S.getActiveModuleMacros(*
this, II),
1382 S.isAmbiguous(*
this, II));
1390 MacroState &S = CurSubmoduleState->Macros[II];
1392 if (
auto *MD = S.getLatest())
1396 S.getActiveModuleMacros(*
this, II),
1397 S.isAmbiguous(*
this, II));
1407 if (!MD || MD->getDefinition().isUndefined())
1421 return MD.getMacroInfo();
1458 updateOutOfDateIdentifier(*II);
1459 auto I = LeafModuleMacros.find(II);
1460 if (I != LeafModuleMacros.end())
1462 return std::nullopt;
1467 return BuildingSubmoduleStack;
1479 llvm::iterator_range<macro_iterator>
1480 macros(
bool IncludeExternalMacros =
true)
const {
1483 return llvm::make_range(begin, end);
1491 if (!BuildingSubmoduleStack.empty()) {
1492 if (M != BuildingSubmoduleStack.back().M)
1493 BuildingSubmoduleStack.back().M->AffectingClangModules.insert(M);
1495 AffectingClangModules.insert(M);
1502 return AffectingClangModules;
1509 return IncludedFiles.insert(
File).second;
1515 return IncludedFiles.count(
File);
1541 return &Identifiers.
get(Name);
1581 CodeComplete = &Handler;
1586 return CodeComplete;
1591 CodeComplete =
nullptr;
1604 CodeCompletionII = Filter;
1611 CodeCompletionTokenRange = {Start, End};
1614 return CodeCompletionTokenRange;
1619 if (CodeCompletionII)
1620 return CodeCompletionII->getName();
1694 void EnterTokenStream(
const Token *Toks,
unsigned NumToks,
1695 bool DisableMacroExpansion,
bool OwnsTokens,
1700 bool DisableMacroExpansion,
bool IsReinject) {
1701 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion,
true,
1707 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion,
false,
1735 std::pair<CachedTokensTy::size_type, bool> LastBacktrackPos();
1737 CachedTokensTy PopUnannotatedBacktrackTokens();
1754 return !UnannotatedBacktrackTokens.empty();
1776 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1783 const char *DiagnosticTag,
bool AllowMacroExpansion) {
1784 if (AllowMacroExpansion)
1789 AllowMacroExpansion);
1795 const char *DiagnosticTag,
1796 bool AllowMacroExpansion);
1806 while (
Result.getKind() == tok::comment);
1812 bool OldVal = DisableMacroExpansion;
1813 DisableMacroExpansion =
true;
1818 DisableMacroExpansion = OldVal;
1826 while (
Result.getKind() == tok::comment);
1836 DisableMacroExpansion =
true;
1837 MacroExpansionInDirectivesOverride =
true;
1848 assert(LexLevel == 0 &&
"cannot use lookahead while lexing");
1852 return PeekAhead(N+1);
1862 "Should only be called when tokens are cached for backtracking");
1863 assert(
signed(CachedLexPos) -
signed(N) >=
1864 signed(LastBacktrackPos().first) &&
1865 "Should revert tokens up to the last backtrack position, not more");
1866 assert(
signed(CachedLexPos) -
signed(N) >= 0 &&
1867 "Corrupted backtrack positions ?");
1881 auto TokCopy = std::make_unique<Token[]>(1);
1883 EnterTokenStream(std::move(TokCopy), 1,
true, IsReinject);
1885 EnterCachingLexMode();
1886 assert(IsReinject &&
"new tokens in the middle of cached stream");
1900 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1902 AnnotatePreviousCachedTokens(Tok);
1908 assert(CachedLexPos != 0);
1932 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1939 void *AnnotationVal);
1944 return CurLexerCallback != CLK_Lexer;
1964 IncrementalProcessing = value;
1998 return CodeCompletionFileLoc;
2008 CodeCompletionReached =
true;
2017 std::pair<IdentifierInfo *, SourceLocation>
2019 return PragmaARCCFCodeAuditedInfo;
2026 PragmaARCCFCodeAuditedInfo = {Ident,
Loc};
2034 return PragmaAssumeNonNullLoc;
2040 PragmaAssumeNonNullLoc =
Loc;
2049 return PreambleRecordedPragmaAssumeNonNullLoc;
2055 PreambleRecordedPragmaAssumeNonNullLoc =
Loc;
2069 SkipMainFilePreamble.first = Bytes;
2070 SkipMainFilePreamble.second = StartOfLine;
2093 bool *invalid =
nullptr)
const {
2122 bool *
Invalid =
nullptr)
const {
2132 bool *
Invalid =
nullptr)
const;
2137 bool IgnoreWhiteSpace =
false) {
2145 bool *
Invalid =
nullptr)
const {
2146 assert((Tok.
is(tok::numeric_constant) || Tok.
is(tok::binary_data)) &&
2147 Tok.
getLength() == 1 &&
"Called on unsupported token");
2148 assert(!Tok.
needsCleaning() &&
"Token can't need cleaning with length 1");
2152 return (Tok.
getKind() == tok::binary_data) ? *
D : *
D -
'0';
2154 assert(Tok.
is(tok::numeric_constant) &&
"binary data with no data");
2235 unsigned Char)
const {
2245 ++NumFastTokenPaste;
2269 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
2283 if(II->isPoisoned()) {
2294 *Ident___exception_code,
2295 *Ident_GetExceptionCode;
2298 *Ident___exception_info,
2299 *Ident_GetExceptionInfo;
2302 *Ident___abnormal_termination,
2303 *Ident_AbnormalTermination;
2305 const char *getCurLexerEndPos();
2306 void diagnoseMissingHeaderInUmbrellaDir(
const Module &Mod);
2348 bool EnableMacros =
false);
2370 "FPEvalMethod should be set either from command line or from the "
2372 return CurrentFPEvalMethod;
2376 return TUFPEvalMethod;
2380 return LastFPEvalPragmaLocation;
2386 "FPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2389 LastFPEvalPragmaLocation = PragmaLoc;
2390 CurrentFPEvalMethod = Val;
2391 TUFPEvalMethod = Val;
2396 "TUPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2397 TUFPEvalMethod = Val;
2414 return ModuleDeclState.isNamedInterface();
2425 return ModuleDeclState.isImplementationUnit();
2432 return !NamedModuleImportPath.empty() &&
getLangOpts().CPlusPlusModules &&
2460 bool *IsFrameworkFound,
bool SkipCache =
false,
2461 bool OpenFile =
true,
bool CacheFailures =
true);
2473 const FileEntry *LookupFromFile =
nullptr);
2483 bool *ShadowFlag =
nullptr);
2491 void PushIncludeMacroStack() {
2492 assert(CurLexerCallback != CLK_CachingLexer &&
2493 "cannot push a caching lexer");
2494 IncludeMacroStack.emplace_back(CurLexerCallback, CurLexerSubmodule,
2495 std::move(CurLexer), CurPPLexer,
2496 std::move(CurTokenLexer), CurDirLookup);
2497 CurPPLexer =
nullptr;
2500 void PopIncludeMacroStack() {
2501 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2502 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2503 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2504 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2505 CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2506 CurLexerCallback = IncludeMacroStack.back().CurLexerCallback;
2507 IncludeMacroStack.pop_back();
2510 void PropagateLineStartLeadingSpaceInfo(Token &
Result);
2514 bool needModuleMacros()
const;
2518 void updateModuleMacroInfo(
const IdentifierInfo *II, ModuleMacroInfo &Info);
2520 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2521 SourceLocation
Loc);
2522 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2523 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation
Loc,
2537 bool *ShadowFlag =
nullptr);
2548 MacroInfo *ReadOptionalMacroParameterListAndBody(
2549 const Token &MacroNameTok,
bool ImmediatelyAfterHeaderGuard);
2555 bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2562 void SuggestTypoedDirective(
const Token &Tok, StringRef
Directive)
const;
2572 void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2573 SourceLocation IfTokenLoc,
2574 bool FoundNonSkipPortion,
bool FoundElse,
2575 SourceLocation ElseLoc = SourceLocation());
2579 struct DirectiveEvalResult {
2581 std::optional<llvm::APSInt>
Value;
2587 bool IncludedUndefinedIds;
2590 SourceRange ExprRange;
2597 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro,
2598 bool CheckForEoD =
true);
2606 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro,
2608 bool &EvaluatedDefined,
2609 bool CheckForEoD =
true);
2615 EmbedResult EvaluateHasEmbed(Token &Tok, IdentifierInfo *II);
2620 bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II);
2625 bool EvaluateHasIncludeNext(Token &Tok, IdentifierInfo *II);
2628 std::pair<ConstSearchDirIterator, const FileEntry *>
2629 getIncludeNextStart(
const Token &IncludeNextTok)
const;
2633 void RegisterBuiltinPragmas();
2636 void RegisterBuiltinMacros();
2641 bool HandleMacroExpandedIdentifier(Token &
Identifier,
const MacroDefinition &MD);
2648 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2649 ArrayRef<Token> tokens);
2651 void removeCachedMacroExpandedTokensOfLastLexer();
2656 bool isNextPPTokenLParen();
2660 MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2661 SourceLocation &MacroEnd);
2665 void ExpandBuiltinMacro(Token &Tok);
2670 void Handle_Pragma(Token &Tok);
2674 void HandleMicrosoft__pragma(Token &Tok);
2681 void setPredefinesFileID(FileID FID) {
2682 assert(PredefinesFileID.isInvalid() &&
"PredefinesFileID already set!");
2683 PredefinesFileID = FID;
2687 void setPCHThroughHeaderFileID(FileID FID);
2691 static bool IsFileLexer(
const Lexer* L,
const PreprocessorLexer*
P) {
2692 return L ? !L->isPragmaLexer() :
P !=
nullptr;
2695 static bool IsFileLexer(
const IncludeStackInfo& I) {
2696 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2699 bool IsFileLexer()
const {
2700 return IsFileLexer(CurLexer.get(), CurPPLexer);
2705 void CachingLex(Token &
Result);
2707 bool InCachingLexMode()
const {
2710 return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2713 void EnterCachingLexMode();
2714 void EnterCachingLexModeUnchecked();
2716 void ExitCachingLexMode() {
2717 if (InCachingLexMode())
2721 const Token &PeekAhead(
unsigned N);
2722 void AnnotatePreviousCachedTokens(
const Token &Tok);
2728 void HandleLineDirective();
2729 void HandleDigitDirective(Token &Tok);
2730 void HandleUserDiagnosticDirective(Token &Tok,
bool isWarning);
2731 void HandleIdentSCCSDirective(Token &Tok);
2732 void HandleMacroPublicDirective(Token &Tok);
2733 void HandleMacroPrivateDirective();
2737 struct ImportAction {
2743 SkippedModuleImport,
2746 Module *ModuleForHeader =
nullptr;
2748 ImportAction(ActionKind AK, Module *Mod =
nullptr)
2749 :
Kind(AK), ModuleForHeader(Mod) {
2750 assert((AK == None || Mod || AK == Failure) &&
2751 "no module for module action");
2757 SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2758 const Token &FilenameTok,
bool &IsFrameworkFound,
bool IsImportDecl,
2760 const FileEntry *LookupFromFile, StringRef &LookupFilename,
2761 SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2762 ModuleMap::KnownHeader &SuggestedModule,
bool isAngled);
2764 void HandleEmbedDirective(SourceLocation HashLoc, Token &Tok,
2765 const FileEntry *LookupFromFile =
nullptr);
2766 void HandleEmbedDirectiveImpl(SourceLocation HashLoc,
2767 const LexEmbedParametersResult &Params,
2768 StringRef BinaryContents);
2771 void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2773 const FileEntry *LookupFromFile =
nullptr);
2775 HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2776 Token &FilenameTok, SourceLocation EndLoc,
2778 const FileEntry *LookupFromFile =
nullptr);
2779 void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2780 void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2781 void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2782 void HandleMicrosoftImportDirective(Token &Tok);
2789 const TargetInfo &TargetInfo,
2790 const Module &M, DiagnosticsEngine &Diags);
2813 SourceLocation MLoc);
2816 return PreambleConditionalStack.isRecording();
2820 return PreambleConditionalStack.hasRecordedPreamble();
2824 return PreambleConditionalStack.getStack();
2828 PreambleConditionalStack.setStack(
s);
2833 PreambleConditionalStack.startReplaying();
2834 PreambleConditionalStack.setStack(
s);
2835 PreambleConditionalStack.SkipInfo = SkipInfo;
2839 return PreambleConditionalStack.SkipInfo;
2845 void replayPreambleConditionalStack();
2848 void HandleDefineDirective(
Token &Tok,
bool ImmediatelyAfterHeaderGuard);
2849 void HandleUndefDirective();
2853 bool isIfndef,
bool ReadAnyTokensBeforeDirective);
2854 void HandleIfDirective(
Token &IfToken,
const Token &HashToken,
2855 bool ReadAnyTokensBeforeDirective);
2856 void HandleEndifDirective(
Token &EndifToken);
2858 void HandleElifFamilyDirective(
Token &ElifToken,
const Token &HashToken,
2887 auto Annotations = AnnotationInfos.find(II);
2888 if (Annotations == AnnotationInfos.end())
2889 AnnotationInfos.insert(std::make_pair(
2891 MacroAnnotations::makeDeprecation(AnnotationLoc, std::move(Msg))));
2893 Annotations->second.DeprecationInfo =
2894 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2899 auto Annotations = AnnotationInfos.find(II);
2900 if (Annotations == AnnotationInfos.end())
2901 AnnotationInfos.insert(
2902 std::make_pair(II, MacroAnnotations::makeRestrictExpansion(
2903 AnnotationLoc, std::move(Msg))));
2905 Annotations->second.RestrictExpansionInfo =
2906 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2910 auto Annotations = AnnotationInfos.find(II);
2911 if (Annotations == AnnotationInfos.end())
2912 AnnotationInfos.insert(
2913 std::make_pair(II, MacroAnnotations::makeFinal(AnnotationLoc)));
2915 Annotations->second.FinalAnnotationLoc = AnnotationLoc;
2919 return AnnotationInfos.find(II)->second;
2923 bool IsIfnDef =
false)
const {
2929 !SourceMgr.isInMainFile(
Identifier.getLocation()))
2952 void emitFinalMacroWarning(
const Token &
Identifier,
bool IsUndef)
const;
2954 unsigned DiagSelection)
const;
2959 bool InSafeBufferOptOutRegion =
false;
2966 using SafeBufferOptOutRegionsTy =
2971 SafeBufferOptOutRegionsTy SafeBufferOptOutMap;
2990 const SafeBufferOptOutRegionsTy *
2998 return &
Iter->getSecond();
3000 } LoadedSafeBufferOptOutMap;
3018 const SourceLocation &
Loc);
3041 const SmallVectorImpl<SourceLocation> &SrcLocSeqs);
3046 static bool CLK_Lexer(Preprocessor &
P, Token &
Result) {
3047 return P.CurLexer->Lex(
Result);
3049 static bool CLK_TokenLexer(Preprocessor &
P, Token &
Result) {
3050 return P.CurTokenLexer->Lex(
Result);
3052 static bool CLK_CachingLexer(Preprocessor &
P, Token &
Result) {
3056 static bool CLK_DependencyDirectivesLexer(Preprocessor &
P, Token &
Result) {
3057 return P.CurLexer->LexDependencyDirectiveToken(
Result);
3059 static bool CLK_LexAfterModuleImport(Preprocessor &
P, Token &
Result) {
3060 return P.LexAfterModuleImport(
Result);
Defines the Diagnostic-related interfaces.
static constexpr Builtin::Info BuiltinInfo[]
enum clang::sema::@1658::IndirectLocalPathEntry::EntryKind Kind
Defines the Diagnostic IDs-related interfaces.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
llvm::MachO::Target Target
llvm::MachO::Record Record
Defines the clang::MacroInfo and clang::MacroDirective classes.
Defines the clang::Module class, which describes a module in the source code.
Defines the PPCallbacks interface.
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
Defines the clang::TokenKind enum and support functions.
__device__ __2f16 float __ockl_bool s
Reads an AST files chain containing the contents of a translation unit.
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Callback handler that receives notifications when performing code completion within the preprocessor.
A directive for a defined macro or a macro imported from a module.
A little helper class used to produce diagnostics.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
void setSuppressAllDiagnostics(bool Val)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
A reference to a DirectoryEntry that includes the name of the directory as it was accessed by the Fil...
Abstract base class that describes a handler that will receive source ranges for empty lines encounte...
virtual void HandleEmptyline(SourceRange Range)=0
virtual ~EmptylineHandler()
Abstract interface for external sources of preprocessor information.
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
Cached information about one file (either on disk or in the virtual file system).
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Implements support for file system lookup, file system caching, and directory search management.
One of these records is kept for each identifier that is lexed.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
bool isDeprecatedMacro() const
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
StringRef getName() const
Return the actual identifier string.
bool isRestrictExpansion() const
Implements an efficient mapping from strings to IdentifierInfo nodes.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
FPEvalMethodKind
Possible float expression evaluation method choices.
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion.
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Characters, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token,...
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer,...
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
A description of the current definition of a macro.
const DefMacroDirective * getDirective() const
Encapsulates changes to the "macros namespace" (the location where the macro name became active,...
Encapsulates the data about a macro definition (e.g.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Abstract interface for a module loader.
Represents a macro directive exported by a module.
Describes a module or submodule.
bool isModuleMapModule() const
This interface provides a way to observe the actions of the preprocessor as it does its thing.
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
SourceLocation getLastFPEvalPragmaLocation() const
bool isMacroDefined(const IdentifierInfo *II)
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
bool markIncluded(FileEntryRef File)
Mark the file as included.
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
void FinalizeForModelFile()
Cleanup after model file parsing.
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void HandlePragmaPoison()
HandlePragmaPoison - Handle #pragma GCC poison. PoisonTok is the 'poison'.
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void dumpMacroInfo(const IdentifierInfo *II)
std::pair< IdentifierInfo *, SourceLocation > getPragmaARCCFCodeAuditedInfo() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaSystemHeader(Token &SysHeaderTok)
HandlePragmaSystemHeader - Implement #pragma GCC system_header.
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
void markClangModuleAsAffecting(Module *M)
Mark the given clang module as affecting the current clang module or translation unit.
void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident, SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaModuleBuild(Token &Tok)
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
OptionalFileEntryRef LookupEmbedFile(StringRef Filename, bool isAngled, bool OpenFile, const FileEntry *LookupFromFile=nullptr)
Given a "Filename" or <Filename> reference, look up the indicated embed resource.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
PPCallbacks * getPPCallbacks() const
bool isInNamedInterfaceUnit() const
If we are proprocessing a named interface unit.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
void setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc)
Record the location of the unterminated #pragma clang assume_nonnull begin in the preamble.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
ArrayRef< BuildingSubmoduleInfo > getBuildingSubmodules() const
Get the list of submodules that we're currently building.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
SourceRange getCodeCompletionTokenRange() const
SourceLocation getModuleImportLoc(Module *M) const
void overrideMaxTokens(unsigned Value, SourceLocation Loc)
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
bool isRecordingPreamble() const
void HandleSkippedDirectiveWhileUsingPCH(Token &Result, SourceLocation HashLoc)
Process directives while skipping until the through header or #pragma hdrstop is found.
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
bool GetSuppressIncludeNotFoundError()
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
LangOptions::FPEvalMethodKind getCurrentFPEvalMethod() const
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
bool isSafeBufferOptOut(const SourceManager &SourceMgr, const SourceLocation &Loc) const
void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
const char * getCheckPoint(FileID FID, const char *Start) const
Returns a pointer into the given file's buffer that's guaranteed to be between tokens.
void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc)
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
const TargetInfo * getAuxTargetInfo() const
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
void DumpMacro(const MacroInfo &MI) const
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
void setDiagnostics(DiagnosticsEngine &D)
IncludedFilesSet & getIncludedFiles()
Get the set of included files.
friend void TokenLexer::ExpandFunctionArguments()
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool SetCodeCompletionPoint(FileEntryRef File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
StringRef getNamedModuleName() const
Get the named module name we're preprocessing.
void makeModuleVisible(Module *M, SourceLocation Loc)
unsigned getCounterValue() const
bool mightHavePendingAnnotationTokens()
Determine whether it's possible for a future call to Lex to produce an annotation token created by a ...
bool isInImportingCXXNamedModules() const
If we're importing a standard C++20 Named Modules.
void Lex(Token &Result)
Lex the next token for this preprocessor.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
unsigned getNumDirectives() const
Retrieve the number of Directives that have been processed by the Preprocessor.
bool isInImplementationUnit() const
If we are implementing an implementation module unit.
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
void LexNonComment(Token &Result)
Lex a token.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
SourceRange DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found.
bool LexOnOffSwitch(tok::OnOffSwitch &Result)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD.
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
void setMainFileDir(DirectoryEntryRef Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
const IdentifierTable & getIdentifierTable() const
void HandlePragmaDependency(Token &DependencyTok)
HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
SourceLocation CheckEndOfDirective(const char *DirType, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
bool enterOrExitSafeBufferOptOutRegion(bool isEnter, const SourceLocation &Loc)
Alter the state of whether this PP currently is in a "-Wunsafe-buffer-usage" opt-out region.
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, std::optional< PreambleSkipInfo > SkipInfo)
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
uint8_t getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the value of constant as an unsign...
macro_iterator macro_end(bool IncludeExternalMacros=true) const
SourceManager & getSourceManager() const
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
std::optional< PreambleSkipInfo > getPreambleSkipInfo() const
void setPreprocessToken(bool Preprocess)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
void HandlePragmaOnce(Token &OnceTok)
HandlePragmaOnce - Handle #pragma once. OnceTok is the 'once'.
EmptylineHandler * getEmptylineHandler() const
bool getCommentRetentionState() const
bool isMacroDefined(StringRef Id)
static bool checkModuleIsAvailable(const LangOptions &LangOpts, const TargetInfo &TargetInfo, const Module &M, DiagnosticsEngine &Diags)
Check that the given module is available, producing a diagnostic if not.
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool hasRecordedPreamble() const
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
MacroMap::const_iterator macro_iterator
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
bool isUnannotatedBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of unannotated tokens is on.
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
void RemovePragmaHandler(PragmaHandler *Handler)
bool isPPInSafeBufferOptOutRegion()
unsigned getTokenCount() const
Get the number of tokens processed so far.
unsigned getMaxTokens() const
Get the max number of tokens before issuing a -Wmax-tokens warning.
SourceLocation getMaxTokensOverrideLoc() const
bool hadModuleLoaderFatalFailure() const
static void processPathToFileName(SmallVectorImpl< char > &FileName, const PresumedLoc &PLoc, const LangOptions &LangOpts, const TargetInfo &TI)
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
const TargetInfo & getTargetInfo() const
FileManager & getFileManager() const
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void DumpLocation(SourceLocation Loc) const
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
bool alreadyIncluded(FileEntryRef File) const
Return true if this header has already been included.
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
llvm::BumpPtrAllocator & getPreprocessorAllocator()
ModuleMacro * getModuleMacro(Module *Mod, const IdentifierInfo *II)
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleComment(Token &result, SourceRange Comment)
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Buffer)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
HeaderSearch & getHeaderSearchInfo() const
void emitMacroExpansionWarnings(const Token &Identifier, bool IsIfnDef=false) const
bool setDeserializedSafeBufferOptOutMap(const SmallVectorImpl< SourceLocation > &SrcLocSeqs)
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
ExternalPreprocessorSource * getExternalSource() const
Module * LeaveSubmodule(bool ForPragma)
const std::string & getPredefines() const
Get the predefines for this processor.
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
SmallVector< SourceLocation, 64 > serializeSafeBufferOptOutMap() const
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind, void *AnnotationVal)
Enter an annotation token into the token stream.
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, const LangOptions &LangOpts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
MacroInfo * getMacroInfo(const IdentifierInfo *II)
void setPredefines(std::string P)
Set the predefines for this Preprocessor.
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
IdentifierTable & getIdentifierTable()
Builtin::Context & getBuiltinInfo()
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
LangOptions::FPEvalMethodKind getTUFPEvalMethod() const
const LangOptions & getLangOpts() const
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void SetSuppressIncludeNotFoundError(bool Suppress)
static void processPathForFileMacro(SmallVectorImpl< char > &Path, const LangOptions &LangOpts, const TargetInfo &TI)
llvm::DenseMap< FileID, SafeBufferOptOutRegionsTy > LoadedRegions
bool isInNamedModule() const
If we are preprocessing a named module.
void EnableBacktrackAtThisPos(bool Unannotated=false)
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
void PoisonSEHIdentifiers(bool Poison=true)
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion.
size_t getTotalMemory() const
void setExternalSource(ExternalPreprocessorSource *Source)
void clearCodeCompletionHandler()
Clear out the code completion handler.
void AddPragmaHandler(PragmaHandler *Handler)
OptionalFileEntryRef getHeaderToIncludeForDiagnostics(SourceLocation IncLoc, SourceLocation MLoc)
We want to produce a diagnostic at location IncLoc concerning an unreachable effect at location MLoc ...
void setCounterValue(unsigned V)
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.
void LexTokensUntilEOF(std::vector< Token > *Tokens=nullptr)
Lex all tokens for this preprocessor until (and excluding) end of file.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
void setPragmasEnabled(bool Enabled)
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void HandlePragmaMark(Token &MarkTok)
bool getPragmasEnabled() const
void HandlePragmaHdrstop(Token &Tok)
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
void setEmptylineHandler(EmptylineHandler *Handler)
Set empty line handler.
DiagnosticsEngine & getDiagnostics() const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
llvm::DenseSet< const FileEntry * > IncludedFilesSet
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SelectorTable & getSelectorTable()
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
const llvm::SmallSetVector< Module *, 2 > & getAffectingClangModules() const
Get the set of top-level clang modules that affected preprocessing, but were not imported.
std::optional< LexEmbedParametersResult > LexEmbedParameters(Token &Current, bool ForHasEmbed)
Lex the parameters for an #embed directive, returns nullopt on error.
const IncludedFilesSet & getIncludedFiles() const
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void HandlePragmaIncludeAlias(Token &Tok)
Module * getModuleForLocation(SourceLocation Loc, bool AllowTextual)
Find the module that owns the source or header file that Loc points to.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
SourceLocation getPreambleRecordedPragmaAssumeNonNullLoc() const
Get the location of the recorded unterminated #pragma clang assume_nonnull begin in the preamble,...
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop.
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
Represents an unpacked "presumed" location which can be presented to the user.
This table allows us to fully hide how we implement multi-keyword caching.
Encodes a location in the source.
This class handles loading and caching of source files into memory.
FileID getUniqueLoadedASTFileID(SourceLocation Loc) const
A trivial tuple used to represent a source range.
Exposes information about the current target.
Stores token information for comparing actual tokens with predefined values.
TokenValue(IdentifierInfo *II)
TokenValue(tok::TokenKind Kind)
bool operator==(const Token &Tok) const
Token - This structure provides full information about a lexed token.
IdentifierInfo * getIdentifierInfo() const
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
unsigned getLength() const
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
tok::TokenKind getKind() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a variadic mac...
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
Directive - Abstract class representing a parsed verify directive.
constexpr XRayInstrMask None
StringRef getName(const HeaderType T)
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a '#' at the beginning of the line.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
The JSON file list parser is used to communicate input to InstallAPI.
CustomizableOptional< DirectoryEntryRef > OptionalDirectoryEntryRef
detail::SearchDirIteratorImpl< true > ConstSearchDirIterator
MacroUse
Context in which macro name is used.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
@ Module
Module linkage, which indicates that the entity can be referred to from other translation units withi...
@ Result
The result type of a method or function.
CustomizableOptional< FileEntryRef > OptionalFileEntryRef
TranslationUnitKind
Describes the kind of translation unit being processed.
@ TU_Complete
The translation unit is a complete translation unit.
Diagnostic wrappers for TextAPI types for error reporting.
Helper class to shuttle information about #embed directives from the preprocessor to the parser throu...
Describes how and where the pragma was introduced.
SourceLocation IfTokenLoc
SourceLocation HashTokenLoc
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)