14#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15#define LLVM_CLANG_LEX_PREPROCESSOR_H
34#include "llvm/ADT/ArrayRef.h"
35#include "llvm/ADT/DenseMap.h"
36#include "llvm/ADT/FoldingSet.h"
37#include "llvm/ADT/FunctionExtras.h"
38#include "llvm/ADT/PointerUnion.h"
39#include "llvm/ADT/STLExtras.h"
40#include "llvm/ADT/SmallPtrSet.h"
41#include "llvm/ADT/SmallVector.h"
42#include "llvm/ADT/StringRef.h"
43#include "llvm/ADT/TinyPtrVector.h"
44#include "llvm/ADT/iterator_range.h"
45#include "llvm/Support/Allocator.h"
46#include "llvm/Support/Casting.h"
47#include "llvm/Support/Registry.h"
60template<
unsigned InternalLen>
class SmallString;
66class CodeCompletionHandler;
69class EmptylineHandler;
70class ExternalPreprocessorSource;
77class PreprocessingRecord;
78class PreprocessorLexer;
79class PreprocessorOptions;
95 assert(Kind != tok::raw_identifier &&
"Raw identifiers are not supported.");
96 assert(Kind != tok::identifier &&
97 "Identifiers should be created by TokenValue(IdentifierInfo *)");
105 return Tok.
getKind() == Kind &&
132 llvm::unique_function<void(
const clang::Token &)> OnToken;
133 std::shared_ptr<PreprocessorOptions> PPOpts;
140 std::unique_ptr<ScratchBuffer> ScratchBuf;
149 llvm::BumpPtrAllocator BP;
205 unsigned CounterValue = 0;
209 MaxAllowedIncludeStackDepth = 200
213 bool KeepComments : 1;
214 bool KeepMacroComments : 1;
215 bool SuppressIncludeNotFoundError : 1;
218 bool InMacroArgs : 1;
221 bool OwnsHeaderSearch : 1;
224 bool DisableMacroExpansion : 1;
228 bool MacroExpansionInDirectivesOverride : 1;
230 class ResetMacroExpansionHelper;
233 mutable bool ReadMacrosFromExternalSource : 1;
236 bool PragmasEnabled : 1;
239 bool PreprocessedOutput : 1;
242 bool ParsingIfOrElifDirective;
245 bool InMacroArgPreExpansion;
267 std::unique_ptr<PragmaNamespace> PragmaHandlers;
271 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
275 std::vector<CommentHandler *> CommentHandlers;
281 bool IncrementalProcessing =
false;
292 const FileEntry *CodeCompletionFile =
nullptr;
295 unsigned CodeCompletionOffset = 0;
315 bool IsAtImport =
false;
318 bool LastTokenWasAt =
false;
321 class StdCXXImportSeq {
326 AfterTopLevelTokenSeq = -1,
331 StdCXXImportSeq(State S) : S(S) {}
334 void handleOpenBracket() {
335 S =
static_cast<State
>(std::max<int>(S, 0) + 1);
338 void handleCloseBracket() {
339 S =
static_cast<State
>(std::max<int>(S, 1) - 1);
342 void handleCloseBrace() {
343 handleCloseBracket();
344 if (S == AtTopLevel && !AfterHeaderName)
345 S = AfterTopLevelTokenSeq;
350 S = AfterTopLevelTokenSeq;
351 AfterHeaderName =
false;
356 void handleExport() {
357 if (S == AfterTopLevelTokenSeq)
363 void handleImport() {
364 if (S == AfterTopLevelTokenSeq || S == AfterExport)
372 void handleHeaderName() {
373 if (S == AfterImportSeq)
374 AfterHeaderName =
true;
384 bool atTopLevel() {
return S <= 0; }
385 bool afterImportSeq() {
return S == AfterImportSeq; }
386 bool afterTopLevelSeq() {
return S == AfterTopLevelTokenSeq; }
393 bool AfterHeaderName =
false;
397 StdCXXImportSeq StdCXXImportSeqState = StdCXXImportSeq::AfterTopLevelTokenSeq;
402 enum GMFState :
int {
405 BeforeGMFIntroducer = -1,
406 GMFAbsentOrEnded = -2,
409 TrackGMF(GMFState S) : S(S) {}
420 void handleExport() {
422 S = GMFAbsentOrEnded;
426 void handleImport(
bool AfterTopLevelTokenSeq) {
428 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
429 S = GMFAbsentOrEnded;
433 void handleModule(
bool AfterTopLevelTokenSeq) {
437 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
440 S = GMFAbsentOrEnded;
447 S = GMFAbsentOrEnded;
450 bool inGMF() {
return S == GMFActive; }
458 TrackGMF TrackGMFState = TrackGMF::BeforeGMFIntroducer;
494 class ModuleDeclSeq {
495 enum ModuleDeclState :
int {
499 ImplementationCandidate,
500 NamedModuleInterface,
501 NamedModuleImplementation,
505 ModuleDeclSeq() =
default;
507 void handleExport() {
508 if (State == NotAModuleDecl)
510 else if (!isNamedModule())
514 void handleModule() {
515 if (State == FoundExport)
516 State = InterfaceCandidate;
517 else if (State == NotAModuleDecl)
518 State = ImplementationCandidate;
519 else if (!isNamedModule())
523 void handleIdentifier(IdentifierInfo *
Identifier) {
526 else if (!isNamedModule())
531 if (isModuleCandidate())
533 else if (!isNamedModule())
537 void handlePeriod() {
538 if (isModuleCandidate())
540 else if (!isNamedModule())
545 if (!Name.empty() && isModuleCandidate()) {
546 if (State == InterfaceCandidate)
547 State = NamedModuleInterface;
548 else if (State == ImplementationCandidate)
549 State = NamedModuleImplementation;
551 llvm_unreachable(
"Unimaged ModuleDeclState.");
552 }
else if (!isNamedModule())
557 if (!isNamedModule())
561 bool isModuleCandidate()
const {
562 return State == InterfaceCandidate || State == ImplementationCandidate;
565 bool isNamedModule()
const {
566 return State == NamedModuleInterface ||
567 State == NamedModuleImplementation;
570 bool isNamedInterface()
const {
return State == NamedModuleInterface; }
572 bool isImplementationUnit()
const {
573 return State == NamedModuleImplementation && !getName().contains(
':');
576 StringRef getName()
const {
577 assert(isNamedModule() &&
"Can't get name from a non named module");
581 StringRef getPrimaryName()
const {
582 assert(isNamedModule() &&
"Can't get name from a non named module");
583 return getName().split(
':').first;
588 State = NotAModuleDecl;
592 ModuleDeclState State = NotAModuleDecl;
596 ModuleDeclSeq ModuleDeclState;
600 bool ModuleImportExpectsIdentifier =
false;
604 std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
608 SourceLocation PragmaAssumeNonNullLoc;
616 SourceLocation PreambleRecordedPragmaAssumeNonNullLoc;
619 bool CodeCompletionReached =
false;
623 IdentifierInfo *CodeCompletionII =
nullptr;
626 SourceRange CodeCompletionTokenRange;
638 std::pair<int, bool> SkipMainFilePreamble;
642 bool HasReachedMaxIncludeDepth =
false;
650 unsigned LexLevel = 0;
653 unsigned TokenCount = 0;
656 bool PreprocessToken =
false;
660 unsigned MaxTokens = 0;
661 SourceLocation MaxTokensOverrideLoc;
685 class PreambleConditionalStackStore {
693 PreambleConditionalStackStore() =
default;
695 void startRecording() { ConditionalStackState = Recording; }
696 void startReplaying() { ConditionalStackState = Replaying; }
697 bool isRecording()
const {
return ConditionalStackState == Recording; }
698 bool isReplaying()
const {
return ConditionalStackState == Replaying; }
700 ArrayRef<PPConditionalInfo> getStack()
const {
701 return ConditionalStack;
704 void doneReplaying() {
705 ConditionalStack.clear();
706 ConditionalStackState = Off;
709 void setStack(ArrayRef<PPConditionalInfo>
s) {
710 if (!isRecording() && !isReplaying())
712 ConditionalStack.clear();
713 ConditionalStack.append(
s.begin(),
s.end());
716 bool hasRecordedPreamble()
const {
return !ConditionalStack.empty(); }
718 bool reachedEOFWhileSkipping()
const {
return SkipInfo.has_value(); }
720 void clearSkipInfo() { SkipInfo.reset(); }
722 std::optional<PreambleSkipInfo> SkipInfo;
725 SmallVector<PPConditionalInfo, 4> ConditionalStack;
726 State ConditionalStackState = Off;
727 } PreambleConditionalStack;
733 std::unique_ptr<Lexer> CurLexer;
739 PreprocessorLexer *CurPPLexer =
nullptr;
751 std::unique_ptr<TokenLexer> CurTokenLexer;
758 CLK_DependencyDirectivesLexer,
759 CLK_LexAfterModuleImport
760 } CurLexerKind = CLK_Lexer;
764 Module *CurLexerSubmodule =
nullptr;
769 struct IncludeStackInfo {
770 enum CurLexerKind CurLexerKind;
772 std::unique_ptr<Lexer> TheLexer;
773 PreprocessorLexer *ThePPLexer;
774 std::unique_ptr<TokenLexer> TheTokenLexer;
779 IncludeStackInfo(
enum CurLexerKind CurLexerKind, Module *TheSubmodule,
780 std::unique_ptr<Lexer> &&TheLexer,
781 PreprocessorLexer *ThePPLexer,
782 std::unique_ptr<TokenLexer> &&TheTokenLexer,
784 : CurLexerKind(
std::move(CurLexerKind)),
785 TheSubmodule(
std::move(TheSubmodule)), TheLexer(
std::move(TheLexer)),
786 ThePPLexer(
std::move(ThePPLexer)),
787 TheTokenLexer(
std::move(TheTokenLexer)),
788 TheDirLookup(
std::move(TheDirLookup)) {}
790 std::vector<IncludeStackInfo> IncludeMacroStack;
794 std::unique_ptr<PPCallbacks> Callbacks;
796 struct MacroExpandsInfo {
801 MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
804 SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
807 struct ModuleMacroInfo {
812 llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
816 unsigned ActiveModuleMacrosGeneration = 0;
819 bool IsAmbiguous =
false;
822 llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
824 ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
829 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
831 ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
832 const IdentifierInfo *II)
const {
833 if (II->isOutOfDate())
834 PP.updateOutOfDateIdentifier(
const_cast<IdentifierInfo&
>(*II));
837 if (!II->hasMacroDefinition() ||
838 (!PP.getLangOpts().Modules &&
839 !PP.getLangOpts().ModulesLocalVisibility) ||
840 !PP.CurSubmoduleState->VisibleModules.getGeneration())
843 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
845 Info =
new (PP.getPreprocessorAllocator())
846 ModuleMacroInfo(State.get<MacroDirective *>());
850 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
851 Info->ActiveModuleMacrosGeneration)
852 PP.updateModuleMacroInfo(II, *Info);
857 MacroState() : MacroState(nullptr) {}
858 MacroState(MacroDirective *MD) : State(MD) {}
860 MacroState(MacroState &&O) noexcept : State(O.State) {
861 O.State = (MacroDirective *)
nullptr;
864 MacroState &operator=(MacroState &&O)
noexcept {
866 O.State = (MacroDirective *)
nullptr;
872 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
873 Info->~ModuleMacroInfo();
876 MacroDirective *getLatest()
const {
877 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
879 return State.get<MacroDirective*>();
882 void setLatest(MacroDirective *MD) {
883 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
889 bool isAmbiguous(Preprocessor &PP,
const IdentifierInfo *II)
const {
890 auto *Info = getModuleInfo(PP, II);
891 return Info ? Info->IsAmbiguous :
false;
894 ArrayRef<ModuleMacro *>
895 getActiveModuleMacros(Preprocessor &PP,
const IdentifierInfo *II)
const {
896 if (
auto *Info = getModuleInfo(PP, II))
897 return Info->ActiveModuleMacros;
901 MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
902 SourceManager &SourceMgr)
const {
904 if (
auto *Latest = getLatest())
905 return Latest->findDirectiveAtLoc(Loc, SourceMgr);
909 void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
910 if (
auto *Info = getModuleInfo(PP, II)) {
911 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
912 Info->ActiveModuleMacros.begin(),
913 Info->ActiveModuleMacros.end());
914 Info->ActiveModuleMacros.clear();
915 Info->IsAmbiguous =
false;
919 ArrayRef<ModuleMacro*> getOverriddenMacros()
const {
920 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
921 return Info->OverriddenMacros;
925 void setOverriddenMacros(Preprocessor &PP,
926 ArrayRef<ModuleMacro *> Overrides) {
927 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
929 if (Overrides.empty())
931 Info =
new (PP.getPreprocessorAllocator())
932 ModuleMacroInfo(State.get<MacroDirective *>());
935 Info->OverriddenMacros.clear();
936 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
937 Overrides.begin(), Overrides.end());
938 Info->ActiveModuleMacrosGeneration = 0;
947 using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
949 struct SubmoduleState;
952 struct BuildingSubmoduleInfo {
957 SourceLocation ImportLoc;
963 SubmoduleState *OuterSubmoduleState;
966 unsigned OuterPendingModuleMacroNames;
968 BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc,
bool IsPragma,
969 SubmoduleState *OuterSubmoduleState,
970 unsigned OuterPendingModuleMacroNames)
971 : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
972 OuterSubmoduleState(OuterSubmoduleState),
973 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
975 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
978 struct SubmoduleState {
983 VisibleModuleSet VisibleModules;
988 std::map<Module *, SubmoduleState> Submodules;
991 SubmoduleState NullSubmoduleState;
995 SubmoduleState *CurSubmoduleState;
1005 llvm::FoldingSet<ModuleMacro> ModuleMacros;
1012 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
1023 using WarnUnusedMacroLocsTy = llvm::SmallDenseSet<SourceLocation, 32>;
1024 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
1030 using MsgLocationPair = std::pair<std::string, SourceLocation>;
1032 struct MacroAnnotationInfo {
1033 SourceLocation Location;
1034 std::string Message;
1037 struct MacroAnnotations {
1038 std::optional<MacroAnnotationInfo> DeprecationInfo;
1039 std::optional<MacroAnnotationInfo> RestrictExpansionInfo;
1040 std::optional<SourceLocation> FinalAnnotationLoc;
1042 static MacroAnnotations makeDeprecation(SourceLocation Loc,
1044 return MacroAnnotations{MacroAnnotationInfo{Loc, std::move(Msg)},
1045 std::nullopt, std::nullopt};
1048 static MacroAnnotations makeRestrictExpansion(SourceLocation Loc,
1050 return MacroAnnotations{
1051 std::nullopt, MacroAnnotationInfo{Loc, std::move(Msg)}, std::nullopt};
1054 static MacroAnnotations makeFinal(SourceLocation Loc) {
1055 return MacroAnnotations{std::nullopt, std::nullopt, Loc};
1060 llvm::DenseMap<const IdentifierInfo *, MacroAnnotations> AnnotationInfos;
1068 llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
1069 PragmaPushMacroInfo;
1072 unsigned NumDirectives = 0;
1073 unsigned NumDefined = 0;
1074 unsigned NumUndefined = 0;
1075 unsigned NumPragma = 0;
1077 unsigned NumElse = 0;
1078 unsigned NumEndif = 0;
1079 unsigned NumEnteredSourceFiles = 0;
1080 unsigned MaxIncludeStackDepth = 0;
1081 unsigned NumMacroExpanded = 0;
1082 unsigned NumFnMacroExpanded = 0;
1083 unsigned NumBuiltinMacroExpanded = 0;
1084 unsigned NumFastMacroExpanded = 0;
1085 unsigned NumTokenPaste = 0;
1086 unsigned NumFastTokenPaste = 0;
1087 unsigned NumSkipped = 0;
1091 std::string Predefines;
1094 FileID PredefinesFileID;
1097 FileID PCHThroughHeaderFileID;
1100 bool SkippingUntilPragmaHdrStop =
false;
1103 bool SkippingUntilPCHThroughHeader =
false;
1107 enum { TokenLexerCacheSize = 8 };
1108 unsigned NumCachedTokenLexers;
1109 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
1117 SmallVector<Token, 16> MacroExpandedTokens;
1118 std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
1125 PreprocessingRecord *Record =
nullptr;
1128 using CachedTokensTy = SmallVector<Token, 1>;
1132 CachedTokensTy CachedTokens;
1139 CachedTokensTy::size_type CachedLexPos = 0;
1146 std::vector<CachedTokensTy::size_type> BacktrackPositions;
1152 bool SkippingExcludedConditionalBlock =
false;
1158 llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges;
1160 void updateOutOfDateIdentifier(IdentifierInfo &II)
const;
1163 Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
1164 DiagnosticsEngine &diags,
const LangOptions &LangOpts,
1165 SourceManager &
SM, HeaderSearch &Headers,
1166 ModuleLoader &TheModuleLoader,
1167 IdentifierInfoLookup *IILookup =
nullptr,
1168 bool OwnsHeaderSearch =
false,
1180 const TargetInfo *AuxTarget =
nullptr);
1231 return NumDirectives;
1236 return ParsingIfOrElifDirective;
1241 this->KeepComments = KeepComments | KeepMacroComments;
1242 this->KeepMacroComments = KeepMacroComments;
1251 SuppressIncludeNotFoundError = Suppress;
1255 return SuppressIncludeNotFoundError;
1261 PreprocessedOutput = IsPreprocessedOutput;
1270 return CurPPLexer == L;
1300 C = std::make_unique<PPChainedCallbacks>(std::move(
C),
1301 std::move(Callbacks));
1302 Callbacks = std::move(
C);
1314 MaxTokensOverrideLoc = Loc;
1323 OnToken = std::move(F);
1342 auto I = Submodules.find(M);
1343 if (I == Submodules.end())
1345 auto J = I->second.Macros.find(II);
1346 if (J == I->second.Macros.end())
1348 auto *MD = J->second.getLatest();
1349 return MD && MD->isDefined();
1356 MacroState &S = CurSubmoduleState->Macros[II];
1357 auto *MD = S.getLatest();
1358 while (MD && isa<VisibilityMacroDirective>(MD))
1359 MD = MD->getPrevious();
1361 S.getActiveModuleMacros(*
this, II),
1362 S.isAmbiguous(*
this, II));
1370 MacroState &S = CurSubmoduleState->Macros[II];
1372 if (
auto *MD = S.getLatest())
1376 S.getActiveModuleMacros(*
this, II),
1377 S.isAmbiguous(*
this, II));
1387 if (!MD || MD->getDefinition().isUndefined())
1401 return MD.getMacroInfo();
1438 auto I = LeafModuleMacros.find(II);
1439 if (I != LeafModuleMacros.end())
1441 return std::nullopt;
1446 return BuildingSubmoduleStack;
1458 llvm::iterator_range<macro_iterator>
1459 macros(
bool IncludeExternalMacros =
true)
const {
1462 return llvm::make_range(begin, end);
1470 if (!BuildingSubmoduleStack.empty()) {
1471 if (M != BuildingSubmoduleStack.back().M)
1472 BuildingSubmoduleStack.back().M->AffectingClangModules.insert(M);
1474 AffectingClangModules.insert(M);
1481 return AffectingClangModules;
1488 return IncludedFiles.insert(
File).second;
1494 return IncludedFiles.count(
File);
1520 return &Identifiers.
get(Name);
1560 CodeComplete = &Handler;
1565 return CodeComplete;
1570 CodeComplete =
nullptr;
1583 CodeCompletionII = Filter;
1590 CodeCompletionTokenRange = {Start, End};
1593 return CodeCompletionTokenRange;
1598 if (CodeCompletionII)
1599 return CodeCompletionII->getName();
1673 void EnterTokenStream(
const Token *Toks,
unsigned NumToks,
1674 bool DisableMacroExpansion,
bool OwnsTokens,
1679 bool DisableMacroExpansion,
bool IsReinject) {
1680 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion,
true,
1686 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion,
false,
1734 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1741 const char *DiagnosticTag,
bool AllowMacroExpansion) {
1742 if (AllowMacroExpansion)
1747 AllowMacroExpansion);
1753 const char *DiagnosticTag,
1754 bool AllowMacroExpansion);
1764 while (
Result.getKind() == tok::comment);
1770 bool OldVal = DisableMacroExpansion;
1771 DisableMacroExpansion =
true;
1776 DisableMacroExpansion = OldVal;
1784 while (
Result.getKind() == tok::comment);
1794 DisableMacroExpansion =
true;
1795 MacroExpansionInDirectivesOverride =
true;
1806 assert(LexLevel == 0 &&
"cannot use lookahead while lexing");
1810 return PeekAhead(N+1);
1820 "Should only be called when tokens are cached for backtracking");
1821 assert(
signed(CachedLexPos) -
signed(N) >=
signed(BacktrackPositions.back())
1822 &&
"Should revert tokens up to the last backtrack position, not more");
1823 assert(
signed(CachedLexPos) -
signed(N) >= 0 &&
1824 "Corrupted backtrack positions ?");
1838 auto TokCopy = std::make_unique<Token[]>(1);
1840 EnterTokenStream(std::move(TokCopy), 1,
true, IsReinject);
1842 EnterCachingLexMode();
1843 assert(IsReinject &&
"new tokens in the middle of cached stream");
1857 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1859 AnnotatePreviousCachedTokens(Tok);
1865 assert(CachedLexPos != 0);
1889 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1896 void *AnnotationVal);
1901 return CurLexerKind != CLK_Lexer;
1921 IncrementalProcessing = value;
1955 return CodeCompletionFileLoc;
1965 CodeCompletionReached =
true;
1974 std::pair<IdentifierInfo *, SourceLocation>
1976 return PragmaARCCFCodeAuditedInfo;
1983 PragmaARCCFCodeAuditedInfo = {Ident, Loc};
1991 return PragmaAssumeNonNullLoc;
1997 PragmaAssumeNonNullLoc = Loc;
2006 return PreambleRecordedPragmaAssumeNonNullLoc;
2012 PreambleRecordedPragmaAssumeNonNullLoc = Loc;
2026 SkipMainFilePreamble.first = Bytes;
2027 SkipMainFilePreamble.second = StartOfLine;
2034 return Diags->
Report(Loc, DiagID);
2050 bool *invalid =
nullptr)
const {
2079 bool *
Invalid =
nullptr)
const {
2089 bool *
Invalid =
nullptr)
const;
2094 bool IgnoreWhiteSpace =
false) {
2102 bool *
Invalid =
nullptr)
const {
2103 assert(Tok.
is(tok::numeric_constant) &&
2104 Tok.
getLength() == 1 &&
"Called on unsupported token");
2105 assert(!Tok.
needsCleaning() &&
"Token can't need cleaning with length 1");
2191 unsigned Char)
const {
2201 ++NumFastTokenPaste;
2225 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
2239 if(II->isPoisoned()) {
2250 *Ident___exception_code,
2251 *Ident_GetExceptionCode;
2254 *Ident___exception_info,
2255 *Ident_GetExceptionInfo;
2258 *Ident___abnormal_termination,
2259 *Ident_AbnormalTermination;
2261 const char *getCurLexerEndPos();
2262 void diagnoseMissingHeaderInUmbrellaDir(
const Module &Mod);
2304 bool EnableMacros =
false);
2320 "FPEvalMethod should be set either from command line or from the "
2322 return CurrentFPEvalMethod;
2326 return TUFPEvalMethod;
2330 return LastFPEvalPragmaLocation;
2336 "FPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2339 LastFPEvalPragmaLocation = PragmaLoc;
2340 CurrentFPEvalMethod = Val;
2341 TUFPEvalMethod = Val;
2346 "TUPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2347 TUFPEvalMethod = Val;
2364 return ModuleDeclState.isNamedInterface();
2375 return ModuleDeclState.isImplementationUnit();
2382 return !NamedModuleImportPath.empty() &&
getLangOpts().CPlusPlusModules &&
2410 bool *IsFrameworkFound,
bool SkipCache =
false,
2411 bool OpenFile =
true,
bool CacheFailures =
true);
2421 bool *ShadowFlag =
nullptr);
2429 void PushIncludeMacroStack() {
2430 assert(CurLexerKind != CLK_CachingLexer &&
"cannot push a caching lexer");
2431 IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
2432 std::move(CurLexer), CurPPLexer,
2433 std::move(CurTokenLexer), CurDirLookup);
2434 CurPPLexer =
nullptr;
2437 void PopIncludeMacroStack() {
2438 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2439 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2440 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2441 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2442 CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2443 CurLexerKind = IncludeMacroStack.back().CurLexerKind;
2444 IncludeMacroStack.pop_back();
2447 void PropagateLineStartLeadingSpaceInfo(Token &
Result);
2451 bool needModuleMacros()
const;
2455 void updateModuleMacroInfo(
const IdentifierInfo *II, ModuleMacroInfo &Info);
2457 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2458 SourceLocation Loc);
2459 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2460 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2474 bool *ShadowFlag =
nullptr);
2485 MacroInfo *ReadOptionalMacroParameterListAndBody(
2486 const Token &MacroNameTok,
bool ImmediatelyAfterHeaderGuard);
2492 bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2499 void SuggestTypoedDirective(
const Token &Tok, StringRef
Directive)
const;
2509 void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2510 SourceLocation IfTokenLoc,
2511 bool FoundNonSkipPortion,
bool FoundElse,
2512 SourceLocation ElseLoc = SourceLocation());
2516 struct DirectiveEvalResult {
2521 bool IncludedUndefinedIds;
2524 SourceRange ExprRange;
2531 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
2536 bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II);
2541 bool EvaluateHasIncludeNext(Token &Tok, IdentifierInfo *II);
2544 std::pair<ConstSearchDirIterator, const FileEntry *>
2545 getIncludeNextStart(
const Token &IncludeNextTok)
const;
2549 void RegisterBuiltinPragmas();
2552 void RegisterBuiltinMacros();
2557 bool HandleMacroExpandedIdentifier(Token &
Identifier,
const MacroDefinition &MD);
2564 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2565 ArrayRef<Token> tokens);
2567 void removeCachedMacroExpandedTokensOfLastLexer();
2572 bool isNextPPTokenLParen();
2576 MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2577 SourceLocation &MacroEnd);
2581 void ExpandBuiltinMacro(Token &Tok);
2586 void Handle_Pragma(Token &Tok);
2590 void HandleMicrosoft__pragma(Token &Tok);
2597 void setPredefinesFileID(FileID FID) {
2598 assert(PredefinesFileID.isInvalid() &&
"PredefinesFileID already set!");
2599 PredefinesFileID = FID;
2603 void setPCHThroughHeaderFileID(FileID FID);
2607 static bool IsFileLexer(
const Lexer* L,
const PreprocessorLexer*
P) {
2608 return L ? !L->isPragmaLexer() :
P !=
nullptr;
2611 static bool IsFileLexer(
const IncludeStackInfo& I) {
2612 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2615 bool IsFileLexer()
const {
2616 return IsFileLexer(CurLexer.get(), CurPPLexer);
2621 void CachingLex(Token &
Result);
2623 bool InCachingLexMode()
const {
2626 return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2629 void EnterCachingLexMode();
2630 void EnterCachingLexModeUnchecked();
2632 void ExitCachingLexMode() {
2633 if (InCachingLexMode())
2637 const Token &PeekAhead(
unsigned N);
2638 void AnnotatePreviousCachedTokens(
const Token &Tok);
2644 void HandleLineDirective();
2645 void HandleDigitDirective(Token &Tok);
2646 void HandleUserDiagnosticDirective(Token &Tok,
bool isWarning);
2647 void HandleIdentSCCSDirective(Token &Tok);
2648 void HandleMacroPublicDirective(Token &Tok);
2649 void HandleMacroPrivateDirective();
2653 struct ImportAction {
2659 SkippedModuleImport,
2662 Module *ModuleForHeader =
nullptr;
2664 ImportAction(ActionKind AK, Module *Mod =
nullptr)
2665 :
Kind(AK), ModuleForHeader(Mod) {
2666 assert((AK == None || Mod || AK == Failure) &&
2667 "no module for module action");
2673 SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2674 const Token &FilenameTok,
bool &IsFrameworkFound,
bool IsImportDecl,
2676 const FileEntry *LookupFromFile, StringRef &LookupFilename,
2677 SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2678 ModuleMap::KnownHeader &SuggestedModule,
bool isAngled);
2681 void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2683 const FileEntry *LookupFromFile =
nullptr);
2685 HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2686 Token &FilenameTok, SourceLocation EndLoc,
2688 const FileEntry *LookupFromFile =
nullptr);
2689 void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2690 void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2691 void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2692 void HandleMicrosoftImportDirective(Token &Tok);
2699 const TargetInfo &TargetInfo,
2700 DiagnosticsEngine &Diags, Module *M);
2723 SourceLocation MLoc);
2726 return PreambleConditionalStack.isRecording();
2730 return PreambleConditionalStack.hasRecordedPreamble();
2734 return PreambleConditionalStack.getStack();
2738 PreambleConditionalStack.setStack(
s);
2743 PreambleConditionalStack.startReplaying();
2744 PreambleConditionalStack.setStack(
s);
2745 PreambleConditionalStack.SkipInfo = SkipInfo;
2749 return PreambleConditionalStack.SkipInfo;
2755 void replayPreambleConditionalStack();
2758 void HandleDefineDirective(
Token &Tok,
bool ImmediatelyAfterHeaderGuard);
2759 void HandleUndefDirective();
2763 bool isIfndef,
bool ReadAnyTokensBeforeDirective);
2764 void HandleIfDirective(
Token &IfToken,
const Token &HashToken,
2765 bool ReadAnyTokensBeforeDirective);
2766 void HandleEndifDirective(
Token &EndifToken);
2768 void HandleElifFamilyDirective(
Token &ElifToken,
const Token &HashToken,
2797 auto Annotations = AnnotationInfos.find(II);
2798 if (Annotations == AnnotationInfos.end())
2799 AnnotationInfos.insert(std::make_pair(
2801 MacroAnnotations::makeDeprecation(AnnotationLoc, std::move(Msg))));
2803 Annotations->second.DeprecationInfo =
2804 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2809 auto Annotations = AnnotationInfos.find(II);
2810 if (Annotations == AnnotationInfos.end())
2811 AnnotationInfos.insert(
2812 std::make_pair(II, MacroAnnotations::makeRestrictExpansion(
2813 AnnotationLoc, std::move(Msg))));
2815 Annotations->second.RestrictExpansionInfo =
2816 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2820 auto Annotations = AnnotationInfos.find(II);
2821 if (Annotations == AnnotationInfos.end())
2822 AnnotationInfos.insert(
2823 std::make_pair(II, MacroAnnotations::makeFinal(AnnotationLoc)));
2825 Annotations->second.FinalAnnotationLoc = AnnotationLoc;
2829 return AnnotationInfos.find(II)->second;
2833 if (
Identifier.getIdentifierInfo()->isDeprecatedMacro())
2836 if (
Identifier.getIdentifierInfo()->isRestrictExpansion() &&
2837 !SourceMgr.isInMainFile(
Identifier.getLocation()))
2853 void emitFinalMacroWarning(
const Token &
Identifier,
bool IsUndef)
const;
2858 bool InSafeBufferOptOutRegion =
false;
Defines the Diagnostic-related interfaces.
static constexpr Builtin::Info BuiltinInfo[]
Defines the Diagnostic IDs-related interfaces.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
Defines the clang::MacroInfo and clang::MacroDirective classes.
Defines the clang::Module class, which describes a module in the source code.
Defines the PPCallbacks interface.
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
Defines the clang::TokenKind enum and support functions.
__device__ __2f16 float __ockl_bool s
Reads an AST files chain containing the contents of a translation unit.
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Callback handler that receives notifications when performing code completion within the preprocessor.
A directive for a defined macro or a macro imported from a module.
A little helper class used to produce diagnostics.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
void setSuppressAllDiagnostics(bool Val)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
A reference to a DirectoryEntry that includes the name of the directory as it was accessed by the Fil...
Abstract base class that describes a handler that will receive source ranges for empty lines encounte...
virtual void HandleEmptyline(SourceRange Range)=0
virtual ~EmptylineHandler()
Abstract interface for external sources of preprocessor information.
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
Cached information about one file (either on disk or in the virtual file system).
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Implements support for file system lookup, file system caching, and directory search management.
One of these records is kept for each identifier that is lexed.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
Implements an efficient mapping from strings to IdentifierInfo nodes.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
FPEvalMethodKind
Possible float expression evaluation method choices.
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion.
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Characters, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token,...
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer,...
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
A description of the current definition of a macro.
const DefMacroDirective * getDirective() const
Encapsulates changes to the "macros namespace" (the location where the macro name became active,...
Encapsulates the data about a macro definition (e.g.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Abstract interface for a module loader.
Represents a macro directive exported by a module.
Describes a module or submodule.
bool isModuleMapModule() const
This interface provides a way to observe the actions of the preprocessor as it does its thing.
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
SourceLocation getLastFPEvalPragmaLocation() const
bool isMacroDefined(const IdentifierInfo *II)
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
bool markIncluded(FileEntryRef File)
Mark the file as included.
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
void FinalizeForModelFile()
Cleanup after model file parsing.
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void HandlePragmaPoison()
HandlePragmaPoison - Handle #pragma GCC poison. PoisonTok is the 'poison'.
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void dumpMacroInfo(const IdentifierInfo *II)
std::pair< IdentifierInfo *, SourceLocation > getPragmaARCCFCodeAuditedInfo() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaSystemHeader(Token &SysHeaderTok)
HandlePragmaSystemHeader - Implement #pragma GCC system_header.
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
void markClangModuleAsAffecting(Module *M)
Mark the given clang module as affecting the current clang module or translation unit.
void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident, SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaModuleBuild(Token &Tok)
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
PPCallbacks * getPPCallbacks() const
bool isInNamedInterfaceUnit() const
If we are proprocessing a named interface unit.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
void setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc)
Record the location of the unterminated #pragma clang assume_nonnull begin in the preamble.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
ArrayRef< BuildingSubmoduleInfo > getBuildingSubmodules() const
Get the list of submodules that we're currently building.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
SourceRange getCodeCompletionTokenRange() const
SourceLocation getModuleImportLoc(Module *M) const
void overrideMaxTokens(unsigned Value, SourceLocation Loc)
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
bool isRecordingPreamble() const
void HandleSkippedDirectiveWhileUsingPCH(Token &Result, SourceLocation HashLoc)
Process directives while skipping until the through header or #pragma hdrstop is found.
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
bool GetSuppressIncludeNotFoundError()
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
LangOptions::FPEvalMethodKind getCurrentFPEvalMethod() const
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
bool isSafeBufferOptOut(const SourceManager &SourceMgr, const SourceLocation &Loc) const
void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc)
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
const TargetInfo * getAuxTargetInfo() const
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
void DumpMacro(const MacroInfo &MI) const
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
void setDiagnostics(DiagnosticsEngine &D)
IncludedFilesSet & getIncludedFiles()
Get the set of included files.
friend void TokenLexer::ExpandFunctionArguments()
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool SetCodeCompletionPoint(FileEntryRef File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
StringRef getNamedModuleName() const
Get the named module name we're preprocessing.
void makeModuleVisible(Module *M, SourceLocation Loc)
unsigned getCounterValue() const
bool mightHavePendingAnnotationTokens()
Determine whether it's possible for a future call to Lex to produce an annotation token created by a ...
bool isInImportingCXXNamedModules() const
If we're importing a standard C++20 Named Modules.
void Lex(Token &Result)
Lex the next token for this preprocessor.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
unsigned getNumDirectives() const
Retrieve the number of Directives that have been processed by the Preprocessor.
bool isInImplementationUnit() const
If we are implementing an implementation module unit.
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
void LexNonComment(Token &Result)
Lex a token.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
bool LexOnOffSwitch(tok::OnOffSwitch &Result)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD.
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
void setMainFileDir(DirectoryEntryRef Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
const IdentifierTable & getIdentifierTable() const
void HandlePragmaDependency(Token &DependencyTok)
HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
SourceLocation CheckEndOfDirective(const char *DirType, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
bool enterOrExitSafeBufferOptOutRegion(bool isEnter, const SourceLocation &Loc)
Alter the state of whether this PP currently is in a "-Wunsafe-buffer-usage" opt-out region.
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, std::optional< PreambleSkipInfo > SkipInfo)
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
macro_iterator macro_end(bool IncludeExternalMacros=true) const
SourceManager & getSourceManager() const
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
std::optional< PreambleSkipInfo > getPreambleSkipInfo() const
void setPreprocessToken(bool Preprocess)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
void HandlePragmaOnce(Token &OnceTok)
HandlePragmaOnce - Handle #pragma once. OnceTok is the 'once'.
EmptylineHandler * getEmptylineHandler() const
bool getCommentRetentionState() const
bool isMacroDefined(StringRef Id)
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool hasRecordedPreamble() const
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
MacroMap::const_iterator macro_iterator
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
void RemovePragmaHandler(PragmaHandler *Handler)
bool isPPInSafeBufferOptOutRegion()
unsigned getTokenCount() const
Get the number of tokens processed so far.
unsigned getMaxTokens() const
Get the max number of tokens before issuing a -Wmax-tokens warning.
SourceLocation getMaxTokensOverrideLoc() const
void emitMacroExpansionWarnings(const Token &Identifier) const
bool hadModuleLoaderFatalFailure() const
static void processPathToFileName(SmallVectorImpl< char > &FileName, const PresumedLoc &PLoc, const LangOptions &LangOpts, const TargetInfo &TI)
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
const TargetInfo & getTargetInfo() const
FileManager & getFileManager() const
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void DumpLocation(SourceLocation Loc) const
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
bool alreadyIncluded(FileEntryRef File) const
Return true if this header has already been included.
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
static bool checkModuleIsAvailable(const LangOptions &LangOpts, const TargetInfo &TargetInfo, DiagnosticsEngine &Diags, Module *M)
Check that the given module is available, producing a diagnostic if not.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character.
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
llvm::BumpPtrAllocator & getPreprocessorAllocator()
ModuleMacro * getModuleMacro(Module *Mod, const IdentifierInfo *II)
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleComment(Token &result, SourceRange Comment)
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Buffer)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
HeaderSearch & getHeaderSearchInfo() const
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
ExternalPreprocessorSource * getExternalSource() const
Module * LeaveSubmodule(bool ForPragma)
SourceRange DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found.
const std::string & getPredefines() const
Get the predefines for this processor.
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind, void *AnnotationVal)
Enter an annotation token into the token stream.
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, const LangOptions &LangOpts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
MacroInfo * getMacroInfo(const IdentifierInfo *II)
void setPredefines(std::string P)
Set the predefines for this Preprocessor.
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
IdentifierTable & getIdentifierTable()
Builtin::Context & getBuiltinInfo()
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
LangOptions::FPEvalMethodKind getTUFPEvalMethod() const
const LangOptions & getLangOpts() const
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void SetSuppressIncludeNotFoundError(bool Suppress)
static void processPathForFileMacro(SmallVectorImpl< char > &Path, const LangOptions &LangOpts, const TargetInfo &TI)
bool isInNamedModule() const
If we are preprocessing a named module.
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
void PoisonSEHIdentifiers(bool Poison=true)
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion.
size_t getTotalMemory() const
void setExternalSource(ExternalPreprocessorSource *Source)
void clearCodeCompletionHandler()
Clear out the code completion handler.
void AddPragmaHandler(PragmaHandler *Handler)
OptionalFileEntryRef getHeaderToIncludeForDiagnostics(SourceLocation IncLoc, SourceLocation MLoc)
We want to produce a diagnostic at location IncLoc concerning an unreachable effect at location MLoc ...
void setCounterValue(unsigned V)
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
void setPragmasEnabled(bool Enabled)
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void HandlePragmaMark(Token &MarkTok)
bool getPragmasEnabled() const
void HandlePragmaHdrstop(Token &Tok)
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
void setEmptylineHandler(EmptylineHandler *Handler)
Set empty line handler.
DiagnosticsEngine & getDiagnostics() const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
llvm::DenseSet< const FileEntry * > IncludedFilesSet
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SelectorTable & getSelectorTable()
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
const llvm::SmallSetVector< Module *, 2 > & getAffectingClangModules() const
Get the set of top-level clang modules that affected preprocessing, but were not imported.
const IncludedFilesSet & getIncludedFiles() const
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void HandlePragmaIncludeAlias(Token &Tok)
Module * getModuleForLocation(SourceLocation Loc, bool AllowTextual)
Find the module that owns the source or header file that Loc points to.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
SourceLocation getPreambleRecordedPragmaAssumeNonNullLoc() const
Get the location of the recorded unterminated #pragma clang assume_nonnull begin in the preamble,...
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
void EnableBacktrackAtThisPos()
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop.
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
Represents an unpacked "presumed" location which can be presented to the user.
This table allows us to fully hide how we implement multi-keyword caching.
Encodes a location in the source.
This class handles loading and caching of source files into memory.
A trivial tuple used to represent a source range.
Exposes information about the current target.
Stores token information for comparing actual tokens with predefined values.
TokenValue(IdentifierInfo *II)
TokenValue(tok::TokenKind Kind)
bool operator==(const Token &Tok) const
Token - This structure provides full information about a lexed token.
IdentifierInfo * getIdentifierInfo() const
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
unsigned getLength() const
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
tok::TokenKind getKind() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a variadic mac...
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
Directive - Abstract class representing a parsed verify directive.
constexpr XRayInstrMask None
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a '#' at the beginning of the line.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
CustomizableOptional< DirectoryEntryRef > OptionalDirectoryEntryRef
detail::SearchDirIteratorImpl< true > ConstSearchDirIterator
MacroUse
Context in which macro name is used.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
@ C
Languages that the frontend can parse and compile.
@ Result
The result type of a method or function.
CustomizableOptional< FileEntryRef > OptionalFileEntryRef
TranslationUnitKind
Describes the kind of translation unit being processed.
@ TU_Complete
The translation unit is a complete translation unit.
YAML serialization mapping.
Describes how and where the pragma was introduced.
SourceLocation IfTokenLoc
SourceLocation HashTokenLoc
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)