14#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15#define LLVM_CLANG_LEX_PREPROCESSOR_H
34#include "llvm/ADT/ArrayRef.h"
35#include "llvm/ADT/DenseMap.h"
36#include "llvm/ADT/FoldingSet.h"
37#include "llvm/ADT/FunctionExtras.h"
38#include "llvm/ADT/PointerUnion.h"
39#include "llvm/ADT/STLExtras.h"
40#include "llvm/ADT/SmallPtrSet.h"
41#include "llvm/ADT/SmallVector.h"
42#include "llvm/ADT/StringRef.h"
43#include "llvm/ADT/TinyPtrVector.h"
44#include "llvm/ADT/iterator_range.h"
45#include "llvm/Support/Allocator.h"
46#include "llvm/Support/Casting.h"
47#include "llvm/Support/Registry.h"
60template<
unsigned InternalLen>
class SmallString;
66class CodeCompletionHandler;
69class EmptylineHandler;
70class ExternalPreprocessorSource;
77class PreprocessingRecord;
78class PreprocessorLexer;
79class PreprocessorOptions;
95 assert(Kind != tok::raw_identifier &&
"Raw identifiers are not supported.");
96 assert(Kind != tok::identifier &&
97 "Identifiers should be created by TokenValue(IdentifierInfo *)");
105 return Tok.
getKind() == Kind &&
132 llvm::unique_function<void(
const clang::Token &)> OnToken;
133 std::shared_ptr<PreprocessorOptions> PPOpts;
140 std::unique_ptr<ScratchBuffer> ScratchBuf;
149 llvm::BumpPtrAllocator BP;
205 unsigned CounterValue = 0;
209 MaxAllowedIncludeStackDepth = 200
213 bool KeepComments : 1;
214 bool KeepMacroComments : 1;
215 bool SuppressIncludeNotFoundError : 1;
218 bool InMacroArgs : 1;
221 bool OwnsHeaderSearch : 1;
224 bool DisableMacroExpansion : 1;
228 bool MacroExpansionInDirectivesOverride : 1;
230 class ResetMacroExpansionHelper;
233 mutable bool ReadMacrosFromExternalSource : 1;
236 bool PragmasEnabled : 1;
239 bool PreprocessedOutput : 1;
242 bool ParsingIfOrElifDirective;
245 bool InMacroArgPreExpansion;
267 std::unique_ptr<PragmaNamespace> PragmaHandlers;
271 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
275 std::vector<CommentHandler *> CommentHandlers;
289 const FileEntry *CodeCompletionFile =
nullptr;
292 unsigned CodeCompletionOffset = 0;
312 bool IsAtImport =
false;
315 bool LastTokenWasAt =
false;
318 class StdCXXImportSeq {
323 AfterTopLevelTokenSeq = -1,
328 StdCXXImportSeq(State S) : S(S) {}
331 void handleOpenBracket() {
332 S =
static_cast<State
>(std::max<int>(S, 0) + 1);
335 void handleCloseBracket() {
336 S =
static_cast<State
>(std::max<int>(S, 1) - 1);
339 void handleCloseBrace() {
340 handleCloseBracket();
341 if (S == AtTopLevel && !AfterHeaderName)
342 S = AfterTopLevelTokenSeq;
347 S = AfterTopLevelTokenSeq;
348 AfterHeaderName =
false;
353 void handleExport() {
354 if (S == AfterTopLevelTokenSeq)
360 void handleImport() {
361 if (S == AfterTopLevelTokenSeq || S == AfterExport)
369 void handleHeaderName() {
370 if (S == AfterImportSeq)
371 AfterHeaderName =
true;
381 bool atTopLevel() {
return S <= 0; }
382 bool afterImportSeq() {
return S == AfterImportSeq; }
383 bool afterTopLevelSeq() {
return S == AfterTopLevelTokenSeq; }
390 bool AfterHeaderName =
false;
394 StdCXXImportSeq StdCXXImportSeqState = StdCXXImportSeq::AfterTopLevelTokenSeq;
399 enum GMFState :
int {
402 BeforeGMFIntroducer = -1,
403 GMFAbsentOrEnded = -2,
406 TrackGMF(GMFState S) : S(S) {}
417 void handleExport() {
419 S = GMFAbsentOrEnded;
423 void handleImport(
bool AfterTopLevelTokenSeq) {
425 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
426 S = GMFAbsentOrEnded;
430 void handleModule(
bool AfterTopLevelTokenSeq) {
434 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
437 S = GMFAbsentOrEnded;
444 S = GMFAbsentOrEnded;
447 bool inGMF() {
return S == GMFActive; }
455 TrackGMF TrackGMFState = TrackGMF::BeforeGMFIntroducer;
491 class ModuleDeclSeq {
492 enum ModuleDeclState :
int {
496 ImplementationCandidate,
497 NamedModuleInterface,
498 NamedModuleImplementation,
502 ModuleDeclSeq() : State(NotAModuleDecl) {}
504 void handleExport() {
505 if (State == NotAModuleDecl)
507 else if (!isNamedModule())
511 void handleModule() {
512 if (State == FoundExport)
513 State = InterfaceCandidate;
514 else if (State == NotAModuleDecl)
515 State = ImplementationCandidate;
516 else if (!isNamedModule())
520 void handleIdentifier(IdentifierInfo *
Identifier) {
523 else if (!isNamedModule())
528 if (isModuleCandidate())
530 else if (!isNamedModule())
534 void handlePeriod() {
535 if (isModuleCandidate())
537 else if (!isNamedModule())
542 if (!Name.empty() && isModuleCandidate()) {
543 if (State == InterfaceCandidate)
544 State = NamedModuleInterface;
545 else if (State == ImplementationCandidate)
546 State = NamedModuleImplementation;
548 llvm_unreachable(
"Unimaged ModuleDeclState.");
549 }
else if (!isNamedModule())
554 if (!isNamedModule())
558 bool isModuleCandidate()
const {
559 return State == InterfaceCandidate || State == ImplementationCandidate;
562 bool isNamedModule()
const {
563 return State == NamedModuleInterface ||
564 State == NamedModuleImplementation;
567 bool isNamedInterface()
const {
return State == NamedModuleInterface; }
569 bool isImplementationUnit()
const {
570 return State == NamedModuleImplementation && !getName().contains(
':');
573 StringRef getName()
const {
574 assert(isNamedModule() &&
"Can't get name from a non named module");
578 StringRef getPrimaryName()
const {
579 assert(isNamedModule() &&
"Can't get name from a non named module");
580 return getName().split(
':').first;
585 State = NotAModuleDecl;
589 ModuleDeclState State;
593 ModuleDeclSeq ModuleDeclState;
597 bool ModuleImportExpectsIdentifier =
false;
601 std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
605 SourceLocation PragmaAssumeNonNullLoc;
613 SourceLocation PreambleRecordedPragmaAssumeNonNullLoc;
616 bool CodeCompletionReached =
false;
620 IdentifierInfo *CodeCompletionII =
nullptr;
623 SourceRange CodeCompletionTokenRange;
628 const DirectoryEntry *MainFileDir =
nullptr;
635 std::pair<int, bool> SkipMainFilePreamble;
639 bool HasReachedMaxIncludeDepth =
false;
647 unsigned LexLevel = 0;
650 unsigned TokenCount = 0;
653 bool PreprocessToken =
false;
657 unsigned MaxTokens = 0;
658 SourceLocation MaxTokensOverrideLoc;
682 class PreambleConditionalStackStore {
690 PreambleConditionalStackStore() =
default;
692 void startRecording() { ConditionalStackState = Recording; }
693 void startReplaying() { ConditionalStackState = Replaying; }
694 bool isRecording()
const {
return ConditionalStackState == Recording; }
695 bool isReplaying()
const {
return ConditionalStackState == Replaying; }
697 ArrayRef<PPConditionalInfo> getStack()
const {
698 return ConditionalStack;
701 void doneReplaying() {
702 ConditionalStack.clear();
703 ConditionalStackState = Off;
706 void setStack(ArrayRef<PPConditionalInfo>
s) {
707 if (!isRecording() && !isReplaying())
709 ConditionalStack.clear();
710 ConditionalStack.append(
s.begin(),
s.end());
713 bool hasRecordedPreamble()
const {
return !ConditionalStack.empty(); }
715 bool reachedEOFWhileSkipping()
const {
return SkipInfo.has_value(); }
717 void clearSkipInfo() { SkipInfo.reset(); }
719 std::optional<PreambleSkipInfo> SkipInfo;
722 SmallVector<PPConditionalInfo, 4> ConditionalStack;
723 State ConditionalStackState = Off;
724 } PreambleConditionalStack;
730 std::unique_ptr<Lexer> CurLexer;
736 PreprocessorLexer *CurPPLexer =
nullptr;
748 std::unique_ptr<TokenLexer> CurTokenLexer;
755 CLK_DependencyDirectivesLexer,
756 CLK_LexAfterModuleImport
757 } CurLexerKind = CLK_Lexer;
761 Module *CurLexerSubmodule =
nullptr;
766 struct IncludeStackInfo {
767 enum CurLexerKind CurLexerKind;
769 std::unique_ptr<Lexer> TheLexer;
770 PreprocessorLexer *ThePPLexer;
771 std::unique_ptr<TokenLexer> TheTokenLexer;
776 IncludeStackInfo(
enum CurLexerKind CurLexerKind, Module *TheSubmodule,
777 std::unique_ptr<Lexer> &&TheLexer,
778 PreprocessorLexer *ThePPLexer,
779 std::unique_ptr<TokenLexer> &&TheTokenLexer,
781 : CurLexerKind(
std::move(CurLexerKind)),
782 TheSubmodule(
std::move(TheSubmodule)), TheLexer(
std::move(TheLexer)),
783 ThePPLexer(
std::move(ThePPLexer)),
784 TheTokenLexer(
std::move(TheTokenLexer)),
785 TheDirLookup(
std::move(TheDirLookup)) {}
787 std::vector<IncludeStackInfo> IncludeMacroStack;
791 std::unique_ptr<PPCallbacks> Callbacks;
793 struct MacroExpandsInfo {
798 MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
801 SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
804 struct ModuleMacroInfo {
809 llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
813 unsigned ActiveModuleMacrosGeneration = 0;
816 bool IsAmbiguous =
false;
819 llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
821 ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
826 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
828 ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
829 const IdentifierInfo *II)
const {
830 if (II->isOutOfDate())
831 PP.updateOutOfDateIdentifier(
const_cast<IdentifierInfo&
>(*II));
834 if (!II->hasMacroDefinition() ||
835 (!PP.getLangOpts().Modules &&
836 !PP.getLangOpts().ModulesLocalVisibility) ||
837 !PP.CurSubmoduleState->VisibleModules.getGeneration())
840 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
842 Info =
new (PP.getPreprocessorAllocator())
843 ModuleMacroInfo(State.get<MacroDirective *>());
847 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
848 Info->ActiveModuleMacrosGeneration)
849 PP.updateModuleMacroInfo(II, *Info);
854 MacroState() : MacroState(nullptr) {}
855 MacroState(MacroDirective *MD) : State(MD) {}
857 MacroState(MacroState &&O) noexcept : State(O.State) {
858 O.State = (MacroDirective *)
nullptr;
861 MacroState &operator=(MacroState &&O)
noexcept {
863 O.State = (MacroDirective *)
nullptr;
869 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
870 Info->~ModuleMacroInfo();
873 MacroDirective *getLatest()
const {
874 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
876 return State.get<MacroDirective*>();
879 void setLatest(MacroDirective *MD) {
880 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
886 bool isAmbiguous(Preprocessor &PP,
const IdentifierInfo *II)
const {
887 auto *Info = getModuleInfo(PP, II);
888 return Info ? Info->IsAmbiguous :
false;
891 ArrayRef<ModuleMacro *>
892 getActiveModuleMacros(Preprocessor &PP,
const IdentifierInfo *II)
const {
893 if (
auto *Info = getModuleInfo(PP, II))
894 return Info->ActiveModuleMacros;
898 MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
899 SourceManager &SourceMgr)
const {
901 if (
auto *Latest = getLatest())
902 return Latest->findDirectiveAtLoc(Loc, SourceMgr);
906 void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
907 if (
auto *Info = getModuleInfo(PP, II)) {
908 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
909 Info->ActiveModuleMacros.begin(),
910 Info->ActiveModuleMacros.end());
911 Info->ActiveModuleMacros.clear();
912 Info->IsAmbiguous =
false;
916 ArrayRef<ModuleMacro*> getOverriddenMacros()
const {
917 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
918 return Info->OverriddenMacros;
922 void setOverriddenMacros(Preprocessor &PP,
923 ArrayRef<ModuleMacro *> Overrides) {
924 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
926 if (Overrides.empty())
928 Info =
new (PP.getPreprocessorAllocator())
929 ModuleMacroInfo(State.get<MacroDirective *>());
932 Info->OverriddenMacros.clear();
933 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
934 Overrides.begin(), Overrides.end());
935 Info->ActiveModuleMacrosGeneration = 0;
944 using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
946 struct SubmoduleState;
949 struct BuildingSubmoduleInfo {
954 SourceLocation ImportLoc;
960 SubmoduleState *OuterSubmoduleState;
963 unsigned OuterPendingModuleMacroNames;
965 BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc,
bool IsPragma,
966 SubmoduleState *OuterSubmoduleState,
967 unsigned OuterPendingModuleMacroNames)
968 : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
969 OuterSubmoduleState(OuterSubmoduleState),
970 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
972 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
975 struct SubmoduleState {
980 VisibleModuleSet VisibleModules;
985 std::map<Module *, SubmoduleState> Submodules;
988 SubmoduleState NullSubmoduleState;
992 SubmoduleState *CurSubmoduleState;
1002 llvm::FoldingSet<ModuleMacro> ModuleMacros;
1009 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
1020 using WarnUnusedMacroLocsTy = llvm::SmallDenseSet<SourceLocation, 32>;
1021 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
1027 using MsgLocationPair = std::pair<std::string, SourceLocation>;
1029 struct MacroAnnotationInfo {
1030 SourceLocation Location;
1034 struct MacroAnnotations {
1035 std::optional<MacroAnnotationInfo> DeprecationInfo;
1036 std::optional<MacroAnnotationInfo> RestrictExpansionInfo;
1037 std::optional<SourceLocation> FinalAnnotationLoc;
1039 static MacroAnnotations makeDeprecation(SourceLocation Loc,
1041 return MacroAnnotations{MacroAnnotationInfo{Loc, std::move(Msg)},
1042 std::nullopt, std::nullopt};
1045 static MacroAnnotations makeRestrictExpansion(SourceLocation Loc,
1047 return MacroAnnotations{
1048 std::nullopt, MacroAnnotationInfo{Loc, std::move(Msg)}, std::nullopt};
1051 static MacroAnnotations makeFinal(SourceLocation Loc) {
1052 return MacroAnnotations{std::nullopt, std::nullopt, Loc};
1057 llvm::DenseMap<const IdentifierInfo *, MacroAnnotations> AnnotationInfos;
1065 llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
1066 PragmaPushMacroInfo;
1069 unsigned NumDirectives = 0;
1070 unsigned NumDefined = 0;
1071 unsigned NumUndefined = 0;
1072 unsigned NumPragma = 0;
1074 unsigned NumElse = 0;
1075 unsigned NumEndif = 0;
1076 unsigned NumEnteredSourceFiles = 0;
1077 unsigned MaxIncludeStackDepth = 0;
1078 unsigned NumMacroExpanded = 0;
1079 unsigned NumFnMacroExpanded = 0;
1080 unsigned NumBuiltinMacroExpanded = 0;
1081 unsigned NumFastMacroExpanded = 0;
1082 unsigned NumTokenPaste = 0;
1083 unsigned NumFastTokenPaste = 0;
1084 unsigned NumSkipped = 0;
1088 std::string Predefines;
1091 FileID PredefinesFileID;
1094 FileID PCHThroughHeaderFileID;
1097 bool SkippingUntilPragmaHdrStop =
false;
1100 bool SkippingUntilPCHThroughHeader =
false;
1104 enum { TokenLexerCacheSize = 8 };
1105 unsigned NumCachedTokenLexers;
1106 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
1114 SmallVector<Token, 16> MacroExpandedTokens;
1115 std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
1122 PreprocessingRecord *Record =
nullptr;
1125 using CachedTokensTy = SmallVector<Token, 1>;
1129 CachedTokensTy CachedTokens;
1136 CachedTokensTy::size_type CachedLexPos = 0;
1143 std::vector<CachedTokensTy::size_type> BacktrackPositions;
1149 bool SkippingExcludedConditionalBlock =
false;
1155 llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges;
1157 void updateOutOfDateIdentifier(IdentifierInfo &II)
const;
1160 Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
1161 DiagnosticsEngine &diags, LangOptions &opts, SourceManager &
SM,
1162 HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
1163 IdentifierInfoLookup *IILookup =
nullptr,
1164 bool OwnsHeaderSearch =
false,
1176 const TargetInfo *AuxTarget =
nullptr);
1210 ExternalSource = Source;
1214 return ExternalSource;
1227 return NumDirectives;
1232 return ParsingIfOrElifDirective;
1237 this->KeepComments = KeepComments | KeepMacroComments;
1238 this->KeepMacroComments = KeepMacroComments;
1247 SuppressIncludeNotFoundError = Suppress;
1251 return SuppressIncludeNotFoundError;
1257 PreprocessedOutput = IsPreprocessedOutput;
1266 return CurPPLexer == L;
1296 C = std::make_unique<PPChainedCallbacks>(std::move(
C),
1297 std::move(Callbacks));
1298 Callbacks = std::move(
C);
1310 MaxTokensOverrideLoc = Loc;
1319 OnToken = std::move(F);
1338 auto I = Submodules.find(M);
1339 if (I == Submodules.end())
1341 auto J = I->second.Macros.find(II);
1342 if (J == I->second.Macros.end())
1344 auto *MD = J->second.getLatest();
1345 return MD && MD->isDefined();
1352 MacroState &S = CurSubmoduleState->Macros[II];
1353 auto *MD = S.getLatest();
1354 while (MD && isa<VisibilityMacroDirective>(MD))
1355 MD = MD->getPrevious();
1357 S.getActiveModuleMacros(*
this, II),
1358 S.isAmbiguous(*
this, II));
1366 MacroState &S = CurSubmoduleState->Macros[II];
1368 if (
auto *MD = S.getLatest())
1372 S.getActiveModuleMacros(*
this, II),
1373 S.isAmbiguous(*
this, II));
1383 if (!MD || MD->getDefinition().isUndefined())
1397 return MD.getMacroInfo();
1434 auto I = LeafModuleMacros.find(II);
1435 if (I != LeafModuleMacros.end())
1437 return std::nullopt;
1442 return BuildingSubmoduleStack;
1454 llvm::iterator_range<macro_iterator>
1455 macros(
bool IncludeExternalMacros =
true)
const {
1458 return llvm::make_range(begin, end);
1466 if (!BuildingSubmoduleStack.empty()) {
1467 if (M != BuildingSubmoduleStack.back().M)
1468 BuildingSubmoduleStack.back().M->AffectingClangModules.insert(M);
1470 AffectingClangModules.insert(M);
1477 return AffectingClangModules;
1484 return IncludedFiles.insert(
File).second;
1489 return IncludedFiles.count(
File);
1515 return &Identifiers.
get(Name);
1555 CodeComplete = &Handler;
1560 return CodeComplete;
1565 CodeComplete =
nullptr;
1578 CodeCompletionII = Filter;
1585 CodeCompletionTokenRange = {Start, End};
1588 return CodeCompletionTokenRange;
1593 if (CodeCompletionII)
1594 return CodeCompletionII->getName();
1668 void EnterTokenStream(
const Token *Toks,
unsigned NumToks,
1669 bool DisableMacroExpansion,
bool OwnsTokens,
1674 bool DisableMacroExpansion,
bool IsReinject) {
1675 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion,
true,
1681 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion,
false,
1729 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1736 const char *DiagnosticTag,
bool AllowMacroExpansion) {
1737 if (AllowMacroExpansion)
1742 AllowMacroExpansion);
1748 const char *DiagnosticTag,
1749 bool AllowMacroExpansion);
1759 while (
Result.getKind() == tok::comment);
1765 bool OldVal = DisableMacroExpansion;
1766 DisableMacroExpansion =
true;
1771 DisableMacroExpansion = OldVal;
1779 while (
Result.getKind() == tok::comment);
1789 DisableMacroExpansion =
true;
1790 MacroExpansionInDirectivesOverride =
true;
1801 assert(LexLevel == 0 &&
"cannot use lookahead while lexing");
1805 return PeekAhead(N+1);
1815 "Should only be called when tokens are cached for backtracking");
1816 assert(
signed(CachedLexPos) -
signed(N) >=
signed(BacktrackPositions.back())
1817 &&
"Should revert tokens up to the last backtrack position, not more");
1818 assert(
signed(CachedLexPos) -
signed(N) >= 0 &&
1819 "Corrupted backtrack positions ?");
1833 auto TokCopy = std::make_unique<Token[]>(1);
1835 EnterTokenStream(std::move(TokCopy), 1,
true, IsReinject);
1837 EnterCachingLexMode();
1838 assert(IsReinject &&
"new tokens in the middle of cached stream");
1852 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1854 AnnotatePreviousCachedTokens(Tok);
1860 assert(CachedLexPos != 0);
1884 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1891 void *AnnotationVal);
1896 return CurLexerKind != CLK_Lexer;
1937 unsigned Line,
unsigned Column);
1953 return CodeCompletionFileLoc;
1963 CodeCompletionReached =
true;
1972 std::pair<IdentifierInfo *, SourceLocation>
1974 return PragmaARCCFCodeAuditedInfo;
1981 PragmaARCCFCodeAuditedInfo = {Ident, Loc};
1989 return PragmaAssumeNonNullLoc;
1995 PragmaAssumeNonNullLoc = Loc;
2004 return PreambleRecordedPragmaAssumeNonNullLoc;
2010 PreambleRecordedPragmaAssumeNonNullLoc = Loc;
2026 SkipMainFilePreamble.first = Bytes;
2027 SkipMainFilePreamble.second = StartOfLine;
2034 return Diags->
Report(Loc, DiagID);
2050 bool *invalid =
nullptr)
const {
2079 bool *
Invalid =
nullptr)
const {
2089 bool *
Invalid =
nullptr)
const;
2094 bool IgnoreWhiteSpace =
false) {
2102 bool *
Invalid =
nullptr)
const {
2103 assert(Tok.
is(tok::numeric_constant) &&
2104 Tok.
getLength() == 1 &&
"Called on unsupported token");
2105 assert(!Tok.
needsCleaning() &&
"Token can't need cleaning with length 1");
2191 unsigned Char)
const {
2201 ++NumFastTokenPaste;
2225 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
2239 if(II->isPoisoned()) {
2250 *Ident___exception_code,
2251 *Ident_GetExceptionCode;
2254 *Ident___exception_info,
2255 *Ident_GetExceptionInfo;
2258 *Ident___abnormal_termination,
2259 *Ident_AbnormalTermination;
2261 const char *getCurLexerEndPos();
2262 void diagnoseMissingHeaderInUmbrellaDir(
const Module &Mod);
2304 bool EnableMacros =
false);
2320 "FPEvalMethod should be set either from command line or from the "
2322 return CurrentFPEvalMethod;
2326 return TUFPEvalMethod;
2330 return LastFPEvalPragmaLocation;
2336 "FPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2339 LastFPEvalPragmaLocation = PragmaLoc;
2340 CurrentFPEvalMethod = Val;
2341 TUFPEvalMethod = Val;
2346 "TUPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2347 TUFPEvalMethod = Val;
2364 return ModuleDeclState.isNamedInterface();
2375 return ModuleDeclState.isImplementationUnit();
2382 return !NamedModuleImportPath.empty() &&
getLangOpts().CPlusPlusModules &&
2410 bool *IsFrameworkFound,
bool SkipCache =
false,
2411 bool OpenFile =
true,
bool CacheFailures =
true);
2421 bool *ShadowFlag =
nullptr);
2429 void PushIncludeMacroStack() {
2430 assert(CurLexerKind != CLK_CachingLexer &&
"cannot push a caching lexer");
2431 IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
2432 std::move(CurLexer), CurPPLexer,
2433 std::move(CurTokenLexer), CurDirLookup);
2434 CurPPLexer =
nullptr;
2437 void PopIncludeMacroStack() {
2438 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2439 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2440 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2441 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2442 CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2443 CurLexerKind = IncludeMacroStack.back().CurLexerKind;
2444 IncludeMacroStack.pop_back();
2447 void PropagateLineStartLeadingSpaceInfo(Token &
Result);
2451 bool needModuleMacros()
const;
2455 void updateModuleMacroInfo(
const IdentifierInfo *II, ModuleMacroInfo &Info);
2457 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2458 SourceLocation Loc);
2459 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2460 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2474 bool *ShadowFlag =
nullptr);
2485 MacroInfo *ReadOptionalMacroParameterListAndBody(
2486 const Token &MacroNameTok,
bool ImmediatelyAfterHeaderGuard);
2492 bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2499 void SuggestTypoedDirective(
const Token &Tok, StringRef
Directive)
const;
2509 void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2510 SourceLocation IfTokenLoc,
2511 bool FoundNonSkipPortion,
bool FoundElse,
2512 SourceLocation ElseLoc = SourceLocation());
2516 struct DirectiveEvalResult {
2521 bool IncludedUndefinedIds;
2524 SourceRange ExprRange;
2531 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
2536 bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II);
2541 bool EvaluateHasIncludeNext(Token &Tok, IdentifierInfo *II);
2544 std::pair<ConstSearchDirIterator, const FileEntry *>
2545 getIncludeNextStart(
const Token &IncludeNextTok)
const;
2549 void RegisterBuiltinPragmas();
2552 void RegisterBuiltinMacros();
2557 bool HandleMacroExpandedIdentifier(Token &
Identifier,
const MacroDefinition &MD);
2564 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2565 ArrayRef<Token> tokens);
2567 void removeCachedMacroExpandedTokensOfLastLexer();
2572 bool isNextPPTokenLParen();
2576 MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2577 SourceLocation &MacroEnd);
2581 void ExpandBuiltinMacro(Token &Tok);
2586 void Handle_Pragma(Token &Tok);
2590 void HandleMicrosoft__pragma(Token &Tok);
2597 void setPredefinesFileID(FileID FID) {
2598 assert(PredefinesFileID.isInvalid() &&
"PredefinesFileID already set!");
2599 PredefinesFileID = FID;
2603 void setPCHThroughHeaderFileID(FileID FID);
2607 static bool IsFileLexer(
const Lexer* L,
const PreprocessorLexer*
P) {
2608 return L ? !L->isPragmaLexer() :
P !=
nullptr;
2611 static bool IsFileLexer(
const IncludeStackInfo& I) {
2612 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2615 bool IsFileLexer()
const {
2616 return IsFileLexer(CurLexer.get(), CurPPLexer);
2621 void CachingLex(Token &
Result);
2623 bool InCachingLexMode()
const {
2626 return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2629 void EnterCachingLexMode();
2630 void EnterCachingLexModeUnchecked();
2632 void ExitCachingLexMode() {
2633 if (InCachingLexMode())
2637 const Token &PeekAhead(
unsigned N);
2638 void AnnotatePreviousCachedTokens(
const Token &Tok);
2644 void HandleLineDirective();
2645 void HandleDigitDirective(Token &Tok);
2646 void HandleUserDiagnosticDirective(Token &Tok,
bool isWarning);
2647 void HandleIdentSCCSDirective(Token &Tok);
2648 void HandleMacroPublicDirective(Token &Tok);
2649 void HandleMacroPrivateDirective();
2653 struct ImportAction {
2659 SkippedModuleImport,
2662 Module *ModuleForHeader =
nullptr;
2664 ImportAction(ActionKind AK, Module *Mod =
nullptr)
2665 :
Kind(AK), ModuleForHeader(Mod) {
2666 assert((AK == None || Mod || AK == Failure) &&
2667 "no module for module action");
2673 SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2674 const Token &FilenameTok,
bool &IsFrameworkFound,
bool IsImportDecl,
2676 const FileEntry *LookupFromFile, StringRef &LookupFilename,
2677 SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2678 ModuleMap::KnownHeader &SuggestedModule,
bool isAngled);
2681 void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2683 const FileEntry *LookupFromFile =
nullptr);
2685 HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2686 Token &FilenameTok, SourceLocation EndLoc,
2688 const FileEntry *LookupFromFile =
nullptr);
2689 void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2690 void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2691 void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2692 void HandleMicrosoftImportDirective(Token &Tok);
2699 const TargetInfo &TargetInfo,
2700 DiagnosticsEngine &Diags, Module *M);
2723 SourceLocation MLoc);
2726 return PreambleConditionalStack.isRecording();
2730 return PreambleConditionalStack.hasRecordedPreamble();
2734 return PreambleConditionalStack.getStack();
2738 PreambleConditionalStack.setStack(
s);
2743 PreambleConditionalStack.startReplaying();
2744 PreambleConditionalStack.setStack(
s);
2745 PreambleConditionalStack.SkipInfo = SkipInfo;
2749 return PreambleConditionalStack.SkipInfo;
2755 void replayPreambleConditionalStack();
2758 void HandleDefineDirective(
Token &Tok,
bool ImmediatelyAfterHeaderGuard);
2759 void HandleUndefDirective();
2763 bool isIfndef,
bool ReadAnyTokensBeforeDirective);
2764 void HandleIfDirective(
Token &IfToken,
const Token &HashToken,
2765 bool ReadAnyTokensBeforeDirective);
2766 void HandleEndifDirective(
Token &EndifToken);
2768 void HandleElifFamilyDirective(
Token &ElifToken,
const Token &HashToken,
2797 auto Annotations = AnnotationInfos.find(II);
2798 if (Annotations == AnnotationInfos.end())
2799 AnnotationInfos.insert(std::make_pair(
2801 MacroAnnotations::makeDeprecation(AnnotationLoc, std::move(Msg))));
2803 Annotations->second.DeprecationInfo =
2804 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2809 auto Annotations = AnnotationInfos.find(II);
2810 if (Annotations == AnnotationInfos.end())
2811 AnnotationInfos.insert(
2812 std::make_pair(II, MacroAnnotations::makeRestrictExpansion(
2813 AnnotationLoc, std::move(Msg))));
2815 Annotations->second.RestrictExpansionInfo =
2816 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2820 auto Annotations = AnnotationInfos.find(II);
2821 if (Annotations == AnnotationInfos.end())
2822 AnnotationInfos.insert(
2823 std::make_pair(II, MacroAnnotations::makeFinal(AnnotationLoc)));
2825 Annotations->second.FinalAnnotationLoc = AnnotationLoc;
2829 return AnnotationInfos.find(II)->second;
2833 if (
Identifier.getIdentifierInfo()->isDeprecatedMacro())
2836 if (
Identifier.getIdentifierInfo()->isRestrictExpansion() &&
2837 !SourceMgr.isInMainFile(
Identifier.getLocation()))
2853 void emitFinalMacroWarning(
const Token &
Identifier,
bool IsUndef)
const;
2858 bool InSafeBufferOptOutRegion =
false;
Defines the Diagnostic-related interfaces.
static constexpr Builtin::Info BuiltinInfo[]
Defines the Diagnostic IDs-related interfaces.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
Defines the clang::MacroInfo and clang::MacroDirective classes.
Defines the clang::Module class, which describes a module in the source code.
Defines the PPCallbacks interface.
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
Defines the clang::TokenKind enum and support functions.
__device__ __2f16 float bool s
Reads an AST files chain containing the contents of a translation unit.
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Callback handler that receives notifications when performing code completion within the preprocessor.
A directive for a defined macro or a macro imported from a module.
A little helper class used to produce diagnostics.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
void setSuppressAllDiagnostics(bool Val)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
Cached information about one directory (either on disk or in the virtual file system).
Abstract base class that describes a handler that will receive source ranges for empty lines encounte...
virtual void HandleEmptyline(SourceRange Range)=0
virtual ~EmptylineHandler()
Abstract interface for external sources of preprocessor information.
Cached information about one file (either on disk or in the virtual file system).
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Implements support for file system lookup, file system caching, and directory search management.
One of these records is kept for each identifier that is lexed.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
Implements an efficient mapping from strings to IdentifierInfo nodes.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
FPEvalMethodKind
Possible float expression evaluation method choices.
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion.
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Characters, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token,...
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer,...
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
A description of the current definition of a macro.
const DefMacroDirective * getDirective() const
Encapsulates changes to the "macros namespace" (the location where the macro name became active,...
Encapsulates the data about a macro definition (e.g.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Abstract interface for a module loader.
Represents a macro directive exported by a module.
Describes a module or submodule.
bool isModuleMapModule() const
This interface provides a way to observe the actions of the preprocessor as it does its thing.
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
SourceLocation getLastFPEvalPragmaLocation() const
bool isMacroDefined(const IdentifierInfo *II)
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
void FinalizeForModelFile()
Cleanup after model file parsing.
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void HandlePragmaPoison()
HandlePragmaPoison - Handle #pragma GCC poison. PoisonTok is the 'poison'.
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void dumpMacroInfo(const IdentifierInfo *II)
std::pair< IdentifierInfo *, SourceLocation > getPragmaARCCFCodeAuditedInfo() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaSystemHeader(Token &SysHeaderTok)
HandlePragmaSystemHeader - Implement #pragma GCC system_header.
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
void markClangModuleAsAffecting(Module *M)
Mark the given clang module as affecting the current clang module or translation unit.
void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident, SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaModuleBuild(Token &Tok)
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
PPCallbacks * getPPCallbacks() const
bool isInNamedInterfaceUnit() const
If we are proprocessing a named interface unit.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
void setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc)
Record the location of the unterminated #pragma clang assume_nonnull begin in the preamble.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
ArrayRef< BuildingSubmoduleInfo > getBuildingSubmodules() const
Get the list of submodules that we're currently building.
const FileEntry * getHeaderToIncludeForDiagnostics(SourceLocation IncLoc, SourceLocation MLoc)
We want to produce a diagnostic at location IncLoc concerning an unreachable effect at location MLoc ...
bool markIncluded(const FileEntry *File)
Mark the file as included.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
SourceRange getCodeCompletionTokenRange() const
SourceLocation getModuleImportLoc(Module *M) const
void overrideMaxTokens(unsigned Value, SourceLocation Loc)
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
bool isRecordingPreamble() const
void HandleSkippedDirectiveWhileUsingPCH(Token &Result, SourceLocation HashLoc)
Process directives while skipping until the through header or #pragma hdrstop is found.
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
bool GetSuppressIncludeNotFoundError()
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
LangOptions::FPEvalMethodKind getCurrentFPEvalMethod() const
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
bool isSafeBufferOptOut(const SourceManager &SourceMgr, const SourceLocation &Loc) const
void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc)
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
const TargetInfo * getAuxTargetInfo() const
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
void DumpMacro(const MacroInfo &MI) const
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
void setDiagnostics(DiagnosticsEngine &D)
IncludedFilesSet & getIncludedFiles()
Get the set of included files.
friend void TokenLexer::ExpandFunctionArguments()
void setCodeCompletionReached()
Note that we hit the code-completion point.
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
StringRef getNamedModuleName() const
Get the named module name we're preprocessing.
void makeModuleVisible(Module *M, SourceLocation Loc)
unsigned getCounterValue() const
bool mightHavePendingAnnotationTokens()
Determine whether it's possible for a future call to Lex to produce an annotation token created by a ...
bool isInImportingCXXNamedModules() const
If we're importing a standard C++20 Named Modules.
void Lex(Token &Result)
Lex the next token for this preprocessor.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
unsigned getNumDirectives() const
Retrieve the number of Directives that have been processed by the Preprocessor.
bool isInImplementationUnit() const
If we are implementing an implementation module unit.
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
void LexNonComment(Token &Result)
Lex a token.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
bool LexOnOffSwitch(tok::OnOffSwitch &Result)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD.
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
const IdentifierTable & getIdentifierTable() const
void HandlePragmaDependency(Token &DependencyTok)
HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
SourceLocation CheckEndOfDirective(const char *DirType, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
bool enterOrExitSafeBufferOptOutRegion(bool isEnter, const SourceLocation &Loc)
Alter the state of whether this PP currently is in a "-Wunsafe-buffer-usage" opt-out region.
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, std::optional< PreambleSkipInfo > SkipInfo)
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
macro_iterator macro_end(bool IncludeExternalMacros=true) const
SourceManager & getSourceManager() const
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
std::optional< PreambleSkipInfo > getPreambleSkipInfo() const
void setPreprocessToken(bool Preprocess)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
void HandlePragmaOnce(Token &OnceTok)
HandlePragmaOnce - Handle #pragma once. OnceTok is the 'once'.
EmptylineHandler * getEmptylineHandler() const
bool getCommentRetentionState() const
bool isMacroDefined(StringRef Id)
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool hasRecordedPreamble() const
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
MacroMap::const_iterator macro_iterator
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
void RemovePragmaHandler(PragmaHandler *Handler)
bool isPPInSafeBufferOptOutRegion()
unsigned getTokenCount() const
Get the number of tokens processed so far.
unsigned getMaxTokens() const
Get the max number of tokens before issuing a -Wmax-tokens warning.
SourceLocation getMaxTokensOverrideLoc() const
void emitMacroExpansionWarnings(const Token &Identifier) const
bool hadModuleLoaderFatalFailure() const
static void processPathToFileName(SmallVectorImpl< char > &FileName, const PresumedLoc &PLoc, const LangOptions &LangOpts, const TargetInfo &TI)
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
const TargetInfo & getTargetInfo() const
FileManager & getFileManager() const
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void DumpLocation(SourceLocation Loc) const
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
static bool checkModuleIsAvailable(const LangOptions &LangOpts, const TargetInfo &TargetInfo, DiagnosticsEngine &Diags, Module *M)
Check that the given module is available, producing a diagnostic if not.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character.
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
llvm::BumpPtrAllocator & getPreprocessorAllocator()
ModuleMacro * getModuleMacro(Module *Mod, const IdentifierInfo *II)
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleComment(Token &result, SourceRange Comment)
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Buffer)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
HeaderSearch & getHeaderSearchInfo() const
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
ExternalPreprocessorSource * getExternalSource() const
Module * LeaveSubmodule(bool ForPragma)
SourceRange DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found.
const std::string & getPredefines() const
Get the predefines for this processor.
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind, void *AnnotationVal)
Enter an annotation token into the token stream.
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream.
bool alreadyIncluded(const FileEntry *File) const
Return true if this header has already been included.
MacroInfo * getMacroInfo(const IdentifierInfo *II)
void setPredefines(std::string P)
Set the predefines for this Preprocessor.
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
IdentifierTable & getIdentifierTable()
Builtin::Context & getBuiltinInfo()
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
LangOptions::FPEvalMethodKind getTUFPEvalMethod() const
const LangOptions & getLangOpts() const
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void SetSuppressIncludeNotFoundError(bool Suppress)
static void processPathForFileMacro(SmallVectorImpl< char > &Path, const LangOptions &LangOpts, const TargetInfo &TI)
bool isInNamedModule() const
If we are preprocessing a named module.
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
void PoisonSEHIdentifiers(bool Poison=true)
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion.
size_t getTotalMemory() const
void setExternalSource(ExternalPreprocessorSource *Source)
void clearCodeCompletionHandler()
Clear out the code completion handler.
void AddPragmaHandler(PragmaHandler *Handler)
void setCounterValue(unsigned V)
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
void setPragmasEnabled(bool Enabled)
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
void HandlePragmaMark(Token &MarkTok)
bool getPragmasEnabled() const
void HandlePragmaHdrstop(Token &Tok)
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
void setEmptylineHandler(EmptylineHandler *Handler)
Set empty line handler.
DiagnosticsEngine & getDiagnostics() const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
llvm::DenseSet< const FileEntry * > IncludedFilesSet
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SelectorTable & getSelectorTable()
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
const llvm::SmallSetVector< Module *, 2 > & getAffectingClangModules() const
Get the set of top-level clang modules that affected preprocessing, but were not imported.
const IncludedFilesSet & getIncludedFiles() const
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void HandlePragmaIncludeAlias(Token &Tok)
Module * getModuleForLocation(SourceLocation Loc, bool AllowTextual)
Find the module that owns the source or header file that Loc points to.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
SourceLocation getPreambleRecordedPragmaAssumeNonNullLoc() const
Get the location of the recorded unterminated #pragma clang assume_nonnull begin in the preamble,...
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
void EnableBacktrackAtThisPos()
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop.
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
Represents an unpacked "presumed" location which can be presented to the user.
This table allows us to fully hide how we implement multi-keyword caching.
Encodes a location in the source.
This class handles loading and caching of source files into memory.
A trivial tuple used to represent a source range.
Exposes information about the current target.
Stores token information for comparing actual tokens with predefined values.
TokenValue(IdentifierInfo *II)
TokenValue(tok::TokenKind Kind)
bool operator==(const Token &Tok) const
Token - This structure provides full information about a lexed token.
IdentifierInfo * getIdentifierInfo() const
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
unsigned getLength() const
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
tok::TokenKind getKind() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a variadic mac...
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
Directive - Abstract class representing a parsed verify directive.
constexpr XRayInstrMask None
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a '#' at the beginning of the line.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
detail::SearchDirIteratorImpl< true > ConstSearchDirIterator
MacroUse
Context in which macro name is used.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
@ C
Languages that the frontend can parse and compile.
@ Result
The result type of a method or function.
CustomizableOptional< FileEntryRef > OptionalFileEntryRef
TranslationUnitKind
Describes the kind of translation unit being processed.
@ TU_Complete
The translation unit is a complete translation unit.
YAML serialization mapping.
Describes how and where the pragma was introduced.
SourceLocation IfTokenLoc
SourceLocation HashTokenLoc
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)