14#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15#define LLVM_CLANG_LEX_PREPROCESSOR_H
34#include "llvm/ADT/ArrayRef.h"
35#include "llvm/ADT/DenseMap.h"
36#include "llvm/ADT/FoldingSet.h"
37#include "llvm/ADT/FunctionExtras.h"
38#include "llvm/ADT/PointerUnion.h"
39#include "llvm/ADT/STLExtras.h"
40#include "llvm/ADT/SmallPtrSet.h"
41#include "llvm/ADT/SmallVector.h"
42#include "llvm/ADT/StringRef.h"
43#include "llvm/ADT/TinyPtrVector.h"
44#include "llvm/ADT/iterator_range.h"
45#include "llvm/Support/Allocator.h"
46#include "llvm/Support/Casting.h"
47#include "llvm/Support/Registry.h"
60template<
unsigned InternalLen>
class SmallString;
66class CodeCompletionHandler;
69class EmptylineHandler;
70class ExternalPreprocessorSource;
77class PreprocessingRecord;
78class PreprocessorLexer;
79class PreprocessorOptions;
95 assert(Kind != tok::raw_identifier &&
"Raw identifiers are not supported.");
96 assert(Kind != tok::identifier &&
97 "Identifiers should be created by TokenValue(IdentifierInfo *)");
105 return Tok.
getKind() == Kind &&
132 llvm::unique_function<void(
const clang::Token &)> OnToken;
133 std::shared_ptr<PreprocessorOptions> PPOpts;
140 std::unique_ptr<ScratchBuffer> ScratchBuf;
149 llvm::BumpPtrAllocator BP;
205 unsigned CounterValue = 0;
209 MaxAllowedIncludeStackDepth = 200
213 bool KeepComments : 1;
214 bool KeepMacroComments : 1;
215 bool SuppressIncludeNotFoundError : 1;
218 bool InMacroArgs : 1;
221 bool OwnsHeaderSearch : 1;
224 bool DisableMacroExpansion : 1;
228 bool MacroExpansionInDirectivesOverride : 1;
230 class ResetMacroExpansionHelper;
233 mutable bool ReadMacrosFromExternalSource : 1;
236 bool PragmasEnabled : 1;
239 bool PreprocessedOutput : 1;
242 bool ParsingIfOrElifDirective;
245 bool InMacroArgPreExpansion;
267 std::unique_ptr<PragmaNamespace> PragmaHandlers;
271 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
275 std::vector<CommentHandler *> CommentHandlers;
281 bool IncrementalProcessing =
false;
299 const FileEntry *CodeCompletionFile =
nullptr;
302 unsigned CodeCompletionOffset = 0;
321 llvm::DenseMap<FileID, SmallVector<const char *>> CheckPoints;
322 unsigned CheckPointCounter = 0;
325 bool IsAtImport =
false;
328 bool LastTokenWasAt =
false;
331 class StdCXXImportSeq {
336 AfterTopLevelTokenSeq = -1,
341 StdCXXImportSeq(State S) : S(S) {}
344 void handleOpenBracket() {
345 S =
static_cast<State
>(std::max<int>(S, 0) + 1);
348 void handleCloseBracket() {
349 S =
static_cast<State
>(std::max<int>(S, 1) - 1);
352 void handleCloseBrace() {
353 handleCloseBracket();
354 if (S == AtTopLevel && !AfterHeaderName)
355 S = AfterTopLevelTokenSeq;
360 S = AfterTopLevelTokenSeq;
361 AfterHeaderName =
false;
366 void handleExport() {
367 if (S == AfterTopLevelTokenSeq)
373 void handleImport() {
374 if (S == AfterTopLevelTokenSeq || S == AfterExport)
382 void handleHeaderName() {
383 if (S == AfterImportSeq)
384 AfterHeaderName =
true;
394 bool atTopLevel() {
return S <= 0; }
395 bool afterImportSeq() {
return S == AfterImportSeq; }
396 bool afterTopLevelSeq() {
return S == AfterTopLevelTokenSeq; }
403 bool AfterHeaderName =
false;
407 StdCXXImportSeq StdCXXImportSeqState = StdCXXImportSeq::AfterTopLevelTokenSeq;
412 enum GMFState :
int {
415 BeforeGMFIntroducer = -1,
416 GMFAbsentOrEnded = -2,
419 TrackGMF(GMFState S) : S(S) {}
430 void handleExport() {
432 S = GMFAbsentOrEnded;
436 void handleImport(
bool AfterTopLevelTokenSeq) {
438 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
439 S = GMFAbsentOrEnded;
443 void handleModule(
bool AfterTopLevelTokenSeq) {
447 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
450 S = GMFAbsentOrEnded;
457 S = GMFAbsentOrEnded;
460 bool inGMF() {
return S == GMFActive; }
468 TrackGMF TrackGMFState = TrackGMF::BeforeGMFIntroducer;
504 class ModuleDeclSeq {
505 enum ModuleDeclState :
int {
509 ImplementationCandidate,
510 NamedModuleInterface,
511 NamedModuleImplementation,
515 ModuleDeclSeq() =
default;
517 void handleExport() {
518 if (State == NotAModuleDecl)
520 else if (!isNamedModule())
524 void handleModule() {
525 if (State == FoundExport)
526 State = InterfaceCandidate;
527 else if (State == NotAModuleDecl)
528 State = ImplementationCandidate;
529 else if (!isNamedModule())
533 void handleIdentifier(IdentifierInfo *
Identifier) {
536 else if (!isNamedModule())
541 if (isModuleCandidate())
543 else if (!isNamedModule())
547 void handlePeriod() {
548 if (isModuleCandidate())
550 else if (!isNamedModule())
555 if (!Name.empty() && isModuleCandidate()) {
556 if (State == InterfaceCandidate)
557 State = NamedModuleInterface;
558 else if (State == ImplementationCandidate)
559 State = NamedModuleImplementation;
561 llvm_unreachable(
"Unimaged ModuleDeclState.");
562 }
else if (!isNamedModule())
567 if (!isNamedModule())
571 bool isModuleCandidate()
const {
572 return State == InterfaceCandidate || State == ImplementationCandidate;
575 bool isNamedModule()
const {
576 return State == NamedModuleInterface ||
577 State == NamedModuleImplementation;
580 bool isNamedInterface()
const {
return State == NamedModuleInterface; }
582 bool isImplementationUnit()
const {
583 return State == NamedModuleImplementation && !
getName().contains(
':');
587 assert(isNamedModule() &&
"Can't get name from a non named module");
591 StringRef getPrimaryName()
const {
592 assert(isNamedModule() &&
"Can't get name from a non named module");
593 return getName().split(
':').first;
598 State = NotAModuleDecl;
602 ModuleDeclState State = NotAModuleDecl;
606 ModuleDeclSeq ModuleDeclState;
610 bool ModuleImportExpectsIdentifier =
false;
614 std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
618 SourceLocation PragmaAssumeNonNullLoc;
626 SourceLocation PreambleRecordedPragmaAssumeNonNullLoc;
629 bool CodeCompletionReached =
false;
633 IdentifierInfo *CodeCompletionII =
nullptr;
636 SourceRange CodeCompletionTokenRange;
648 std::pair<int, bool> SkipMainFilePreamble;
652 bool HasReachedMaxIncludeDepth =
false;
660 unsigned LexLevel = 0;
663 unsigned TokenCount = 0;
666 bool PreprocessToken =
false;
670 unsigned MaxTokens = 0;
671 SourceLocation MaxTokensOverrideLoc;
695 class PreambleConditionalStackStore {
703 PreambleConditionalStackStore() =
default;
705 void startRecording() { ConditionalStackState = Recording; }
706 void startReplaying() { ConditionalStackState = Replaying; }
707 bool isRecording()
const {
return ConditionalStackState == Recording; }
708 bool isReplaying()
const {
return ConditionalStackState == Replaying; }
710 ArrayRef<PPConditionalInfo> getStack()
const {
711 return ConditionalStack;
714 void doneReplaying() {
715 ConditionalStack.clear();
716 ConditionalStackState = Off;
719 void setStack(ArrayRef<PPConditionalInfo>
s) {
720 if (!isRecording() && !isReplaying())
722 ConditionalStack.clear();
723 ConditionalStack.append(
s.begin(),
s.end());
726 bool hasRecordedPreamble()
const {
return !ConditionalStack.empty(); }
728 bool reachedEOFWhileSkipping()
const {
return SkipInfo.has_value(); }
730 void clearSkipInfo() { SkipInfo.reset(); }
732 std::optional<PreambleSkipInfo> SkipInfo;
735 SmallVector<PPConditionalInfo, 4> ConditionalStack;
736 State ConditionalStackState = Off;
737 } PreambleConditionalStack;
743 std::unique_ptr<Lexer> CurLexer;
749 PreprocessorLexer *CurPPLexer =
nullptr;
761 std::unique_ptr<TokenLexer> CurTokenLexer;
765 LexerCallback CurLexerCallback = &CLK_Lexer;
769 Module *CurLexerSubmodule =
nullptr;
774 struct IncludeStackInfo {
775 LexerCallback CurLexerCallback;
777 std::unique_ptr<Lexer> TheLexer;
778 PreprocessorLexer *ThePPLexer;
779 std::unique_ptr<TokenLexer> TheTokenLexer;
784 IncludeStackInfo(LexerCallback CurLexerCallback, Module *TheSubmodule,
785 std::unique_ptr<Lexer> &&TheLexer,
786 PreprocessorLexer *ThePPLexer,
787 std::unique_ptr<TokenLexer> &&TheTokenLexer,
789 : CurLexerCallback(
std::move(CurLexerCallback)),
790 TheSubmodule(
std::move(TheSubmodule)), TheLexer(
std::move(TheLexer)),
791 ThePPLexer(
std::move(ThePPLexer)),
792 TheTokenLexer(
std::move(TheTokenLexer)),
793 TheDirLookup(
std::move(TheDirLookup)) {}
795 std::vector<IncludeStackInfo> IncludeMacroStack;
799 std::unique_ptr<PPCallbacks> Callbacks;
801 struct MacroExpandsInfo {
806 MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
809 SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
812 struct ModuleMacroInfo {
817 llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
821 unsigned ActiveModuleMacrosGeneration = 0;
824 bool IsAmbiguous =
false;
827 llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
829 ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
834 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
836 ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
837 const IdentifierInfo *II)
const {
838 if (II->isOutOfDate())
839 PP.updateOutOfDateIdentifier(*II);
842 if (!II->hasMacroDefinition() ||
843 (!PP.getLangOpts().Modules &&
844 !PP.getLangOpts().ModulesLocalVisibility) ||
845 !PP.CurSubmoduleState->VisibleModules.getGeneration())
848 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
850 Info =
new (PP.getPreprocessorAllocator())
851 ModuleMacroInfo(State.get<MacroDirective *>());
855 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
856 Info->ActiveModuleMacrosGeneration)
857 PP.updateModuleMacroInfo(II, *Info);
862 MacroState() : MacroState(nullptr) {}
863 MacroState(MacroDirective *MD) : State(MD) {}
865 MacroState(MacroState &&O) noexcept : State(O.State) {
866 O.State = (MacroDirective *)
nullptr;
869 MacroState &operator=(MacroState &&O)
noexcept {
871 O.State = (MacroDirective *)
nullptr;
877 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
878 Info->~ModuleMacroInfo();
881 MacroDirective *getLatest()
const {
882 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
884 return State.get<MacroDirective*>();
887 void setLatest(MacroDirective *MD) {
888 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
894 bool isAmbiguous(Preprocessor &PP,
const IdentifierInfo *II)
const {
895 auto *Info = getModuleInfo(PP, II);
896 return Info ? Info->IsAmbiguous :
false;
899 ArrayRef<ModuleMacro *>
900 getActiveModuleMacros(Preprocessor &PP,
const IdentifierInfo *II)
const {
901 if (
auto *Info = getModuleInfo(PP, II))
902 return Info->ActiveModuleMacros;
906 MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
907 SourceManager &SourceMgr)
const {
909 if (
auto *Latest = getLatest())
910 return Latest->findDirectiveAtLoc(Loc, SourceMgr);
914 void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
915 if (
auto *Info = getModuleInfo(PP, II)) {
916 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
917 Info->ActiveModuleMacros.begin(),
918 Info->ActiveModuleMacros.end());
919 Info->ActiveModuleMacros.clear();
920 Info->IsAmbiguous =
false;
924 ArrayRef<ModuleMacro*> getOverriddenMacros()
const {
925 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
926 return Info->OverriddenMacros;
930 void setOverriddenMacros(Preprocessor &PP,
931 ArrayRef<ModuleMacro *> Overrides) {
932 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
934 if (Overrides.empty())
936 Info =
new (PP.getPreprocessorAllocator())
937 ModuleMacroInfo(State.get<MacroDirective *>());
940 Info->OverriddenMacros.clear();
941 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
942 Overrides.begin(), Overrides.end());
943 Info->ActiveModuleMacrosGeneration = 0;
952 using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
954 struct SubmoduleState;
957 struct BuildingSubmoduleInfo {
962 SourceLocation ImportLoc;
968 SubmoduleState *OuterSubmoduleState;
971 unsigned OuterPendingModuleMacroNames;
973 BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc,
bool IsPragma,
974 SubmoduleState *OuterSubmoduleState,
975 unsigned OuterPendingModuleMacroNames)
976 : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
977 OuterSubmoduleState(OuterSubmoduleState),
978 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
980 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
983 struct SubmoduleState {
988 VisibleModuleSet VisibleModules;
993 std::map<Module *, SubmoduleState> Submodules;
996 SubmoduleState NullSubmoduleState;
1000 SubmoduleState *CurSubmoduleState;
1010 llvm::FoldingSet<ModuleMacro> ModuleMacros;
1017 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
1028 using WarnUnusedMacroLocsTy = llvm::SmallDenseSet<SourceLocation, 32>;
1029 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
1035 using MsgLocationPair = std::pair<std::string, SourceLocation>;
1037 struct MacroAnnotationInfo {
1038 SourceLocation Location;
1039 std::string Message;
1042 struct MacroAnnotations {
1043 std::optional<MacroAnnotationInfo> DeprecationInfo;
1044 std::optional<MacroAnnotationInfo> RestrictExpansionInfo;
1045 std::optional<SourceLocation> FinalAnnotationLoc;
1047 static MacroAnnotations makeDeprecation(SourceLocation Loc,
1049 return MacroAnnotations{MacroAnnotationInfo{Loc, std::move(Msg)},
1050 std::nullopt, std::nullopt};
1053 static MacroAnnotations makeRestrictExpansion(SourceLocation Loc,
1055 return MacroAnnotations{
1056 std::nullopt, MacroAnnotationInfo{Loc, std::move(Msg)}, std::nullopt};
1059 static MacroAnnotations makeFinal(SourceLocation Loc) {
1060 return MacroAnnotations{std::nullopt, std::nullopt, Loc};
1065 llvm::DenseMap<const IdentifierInfo *, MacroAnnotations> AnnotationInfos;
1073 llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
1074 PragmaPushMacroInfo;
1077 unsigned NumDirectives = 0;
1078 unsigned NumDefined = 0;
1079 unsigned NumUndefined = 0;
1080 unsigned NumPragma = 0;
1082 unsigned NumElse = 0;
1083 unsigned NumEndif = 0;
1084 unsigned NumEnteredSourceFiles = 0;
1085 unsigned MaxIncludeStackDepth = 0;
1086 unsigned NumMacroExpanded = 0;
1087 unsigned NumFnMacroExpanded = 0;
1088 unsigned NumBuiltinMacroExpanded = 0;
1089 unsigned NumFastMacroExpanded = 0;
1090 unsigned NumTokenPaste = 0;
1091 unsigned NumFastTokenPaste = 0;
1092 unsigned NumSkipped = 0;
1096 std::string Predefines;
1099 FileID PredefinesFileID;
1102 FileID PCHThroughHeaderFileID;
1105 bool SkippingUntilPragmaHdrStop =
false;
1108 bool SkippingUntilPCHThroughHeader =
false;
1112 enum { TokenLexerCacheSize = 8 };
1113 unsigned NumCachedTokenLexers;
1114 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
1122 SmallVector<Token, 16> MacroExpandedTokens;
1123 std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
1130 PreprocessingRecord *
Record =
nullptr;
1133 using CachedTokensTy = SmallVector<Token, 1>;
1137 CachedTokensTy CachedTokens;
1144 CachedTokensTy::size_type CachedLexPos = 0;
1151 std::vector<CachedTokensTy::size_type> BacktrackPositions;
1157 bool SkippingExcludedConditionalBlock =
false;
1163 llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges;
1165 void updateOutOfDateIdentifier(
const IdentifierInfo &II)
const;
1168 Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
1169 DiagnosticsEngine &diags,
const LangOptions &LangOpts,
1170 SourceManager &
SM, HeaderSearch &Headers,
1171 ModuleLoader &TheModuleLoader,
1172 IdentifierInfoLookup *IILookup =
nullptr,
1173 bool OwnsHeaderSearch =
false,
1185 const TargetInfo *AuxTarget =
nullptr);
1236 return NumDirectives;
1241 return ParsingIfOrElifDirective;
1246 this->KeepComments = KeepComments | KeepMacroComments;
1247 this->KeepMacroComments = KeepMacroComments;
1256 SuppressIncludeNotFoundError = Suppress;
1260 return SuppressIncludeNotFoundError;
1266 PreprocessedOutput = IsPreprocessedOutput;
1275 return CurPPLexer == L;
1305 C = std::make_unique<PPChainedCallbacks>(std::move(
C),
1306 std::move(Callbacks));
1307 Callbacks = std::move(
C);
1319 MaxTokensOverrideLoc = Loc;
1328 OnToken = std::move(F);
1347 auto I = Submodules.find(M);
1348 if (I == Submodules.end())
1350 auto J = I->second.Macros.find(II);
1351 if (J == I->second.Macros.end())
1353 auto *MD = J->second.getLatest();
1354 return MD && MD->isDefined();
1361 MacroState &S = CurSubmoduleState->Macros[II];
1362 auto *MD = S.getLatest();
1363 while (MD && isa<VisibilityMacroDirective>(MD))
1364 MD = MD->getPrevious();
1366 S.getActiveModuleMacros(*
this, II),
1367 S.isAmbiguous(*
this, II));
1375 MacroState &S = CurSubmoduleState->Macros[II];
1377 if (
auto *MD = S.getLatest())
1381 S.getActiveModuleMacros(*
this, II),
1382 S.isAmbiguous(*
this, II));
1392 if (!MD || MD->getDefinition().isUndefined())
1406 return MD.getMacroInfo();
1443 updateOutOfDateIdentifier(*II);
1444 auto I = LeafModuleMacros.find(II);
1445 if (I != LeafModuleMacros.end())
1447 return std::nullopt;
1452 return BuildingSubmoduleStack;
1464 llvm::iterator_range<macro_iterator>
1465 macros(
bool IncludeExternalMacros =
true)
const {
1468 return llvm::make_range(begin, end);
1476 if (!BuildingSubmoduleStack.empty()) {
1477 if (M != BuildingSubmoduleStack.back().M)
1478 BuildingSubmoduleStack.back().M->AffectingClangModules.insert(M);
1480 AffectingClangModules.insert(M);
1487 return AffectingClangModules;
1494 return IncludedFiles.insert(
File).second;
1500 return IncludedFiles.count(
File);
1526 return &Identifiers.
get(Name);
1566 CodeComplete = &Handler;
1571 return CodeComplete;
1576 CodeComplete =
nullptr;
1589 CodeCompletionII = Filter;
1596 CodeCompletionTokenRange = {Start, End};
1599 return CodeCompletionTokenRange;
1604 if (CodeCompletionII)
1605 return CodeCompletionII->getName();
1679 void EnterTokenStream(
const Token *Toks,
unsigned NumToks,
1680 bool DisableMacroExpansion,
bool OwnsTokens,
1685 bool DisableMacroExpansion,
bool IsReinject) {
1686 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion,
true,
1692 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion,
false,
1743 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1750 const char *DiagnosticTag,
bool AllowMacroExpansion) {
1751 if (AllowMacroExpansion)
1756 AllowMacroExpansion);
1762 const char *DiagnosticTag,
1763 bool AllowMacroExpansion);
1773 while (
Result.getKind() == tok::comment);
1779 bool OldVal = DisableMacroExpansion;
1780 DisableMacroExpansion =
true;
1785 DisableMacroExpansion = OldVal;
1793 while (
Result.getKind() == tok::comment);
1803 DisableMacroExpansion =
true;
1804 MacroExpansionInDirectivesOverride =
true;
1815 assert(LexLevel == 0 &&
"cannot use lookahead while lexing");
1819 return PeekAhead(N+1);
1829 "Should only be called when tokens are cached for backtracking");
1830 assert(
signed(CachedLexPos) -
signed(N) >=
signed(BacktrackPositions.back())
1831 &&
"Should revert tokens up to the last backtrack position, not more");
1832 assert(
signed(CachedLexPos) -
signed(N) >= 0 &&
1833 "Corrupted backtrack positions ?");
1847 auto TokCopy = std::make_unique<Token[]>(1);
1849 EnterTokenStream(std::move(TokCopy), 1,
true, IsReinject);
1851 EnterCachingLexMode();
1852 assert(IsReinject &&
"new tokens in the middle of cached stream");
1866 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1868 AnnotatePreviousCachedTokens(Tok);
1874 assert(CachedLexPos != 0);
1898 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1905 void *AnnotationVal);
1910 return CurLexerCallback != CLK_Lexer;
1930 IncrementalProcessing = value;
1964 return CodeCompletionFileLoc;
1974 CodeCompletionReached =
true;
1983 std::pair<IdentifierInfo *, SourceLocation>
1985 return PragmaARCCFCodeAuditedInfo;
1992 PragmaARCCFCodeAuditedInfo = {Ident, Loc};
2000 return PragmaAssumeNonNullLoc;
2006 PragmaAssumeNonNullLoc = Loc;
2015 return PreambleRecordedPragmaAssumeNonNullLoc;
2021 PreambleRecordedPragmaAssumeNonNullLoc = Loc;
2035 SkipMainFilePreamble.first = Bytes;
2036 SkipMainFilePreamble.second = StartOfLine;
2043 return Diags->
Report(Loc, DiagID);
2059 bool *invalid =
nullptr)
const {
2088 bool *
Invalid =
nullptr)
const {
2098 bool *
Invalid =
nullptr)
const;
2103 bool IgnoreWhiteSpace =
false) {
2111 bool *
Invalid =
nullptr)
const {
2112 assert(Tok.
is(tok::numeric_constant) &&
2113 Tok.
getLength() == 1 &&
"Called on unsupported token");
2114 assert(!Tok.
needsCleaning() &&
"Token can't need cleaning with length 1");
2200 unsigned Char)
const {
2210 ++NumFastTokenPaste;
2234 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
2248 if(II->isPoisoned()) {
2259 *Ident___exception_code,
2260 *Ident_GetExceptionCode;
2263 *Ident___exception_info,
2264 *Ident_GetExceptionInfo;
2267 *Ident___abnormal_termination,
2268 *Ident_AbnormalTermination;
2270 const char *getCurLexerEndPos();
2271 void diagnoseMissingHeaderInUmbrellaDir(
const Module &Mod);
2313 bool EnableMacros =
false);
2329 "FPEvalMethod should be set either from command line or from the "
2331 return CurrentFPEvalMethod;
2335 return TUFPEvalMethod;
2339 return LastFPEvalPragmaLocation;
2345 "FPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2348 LastFPEvalPragmaLocation = PragmaLoc;
2349 CurrentFPEvalMethod = Val;
2350 TUFPEvalMethod = Val;
2355 "TUPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2356 TUFPEvalMethod = Val;
2373 return ModuleDeclState.isNamedInterface();
2384 return ModuleDeclState.isImplementationUnit();
2391 return !NamedModuleImportPath.empty() &&
getLangOpts().CPlusPlusModules &&
2419 bool *IsFrameworkFound,
bool SkipCache =
false,
2420 bool OpenFile =
true,
bool CacheFailures =
true);
2430 bool *ShadowFlag =
nullptr);
2438 void PushIncludeMacroStack() {
2439 assert(CurLexerCallback != CLK_CachingLexer &&
2440 "cannot push a caching lexer");
2441 IncludeMacroStack.emplace_back(CurLexerCallback, CurLexerSubmodule,
2442 std::move(CurLexer), CurPPLexer,
2443 std::move(CurTokenLexer), CurDirLookup);
2444 CurPPLexer =
nullptr;
2447 void PopIncludeMacroStack() {
2448 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2449 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2450 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2451 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2452 CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2453 CurLexerCallback = IncludeMacroStack.back().CurLexerCallback;
2454 IncludeMacroStack.pop_back();
2457 void PropagateLineStartLeadingSpaceInfo(Token &
Result);
2461 bool needModuleMacros()
const;
2465 void updateModuleMacroInfo(
const IdentifierInfo *II, ModuleMacroInfo &Info);
2467 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2468 SourceLocation Loc);
2469 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2470 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2484 bool *ShadowFlag =
nullptr);
2495 MacroInfo *ReadOptionalMacroParameterListAndBody(
2496 const Token &MacroNameTok,
bool ImmediatelyAfterHeaderGuard);
2502 bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2509 void SuggestTypoedDirective(
const Token &Tok, StringRef
Directive)
const;
2519 void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2520 SourceLocation IfTokenLoc,
2521 bool FoundNonSkipPortion,
bool FoundElse,
2522 SourceLocation ElseLoc = SourceLocation());
2526 struct DirectiveEvalResult {
2531 bool IncludedUndefinedIds;
2534 SourceRange ExprRange;
2541 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
2546 bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II);
2551 bool EvaluateHasIncludeNext(Token &Tok, IdentifierInfo *II);
2554 std::pair<ConstSearchDirIterator, const FileEntry *>
2555 getIncludeNextStart(
const Token &IncludeNextTok)
const;
2559 void RegisterBuiltinPragmas();
2562 void RegisterBuiltinMacros();
2567 bool HandleMacroExpandedIdentifier(Token &
Identifier,
const MacroDefinition &MD);
2574 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2575 ArrayRef<Token> tokens);
2577 void removeCachedMacroExpandedTokensOfLastLexer();
2582 bool isNextPPTokenLParen();
2586 MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2587 SourceLocation &MacroEnd);
2591 void ExpandBuiltinMacro(Token &Tok);
2596 void Handle_Pragma(Token &Tok);
2600 void HandleMicrosoft__pragma(Token &Tok);
2607 void setPredefinesFileID(FileID FID) {
2608 assert(PredefinesFileID.isInvalid() &&
"PredefinesFileID already set!");
2609 PredefinesFileID = FID;
2613 void setPCHThroughHeaderFileID(FileID FID);
2617 static bool IsFileLexer(
const Lexer* L,
const PreprocessorLexer*
P) {
2618 return L ? !L->isPragmaLexer() :
P !=
nullptr;
2621 static bool IsFileLexer(
const IncludeStackInfo& I) {
2622 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2625 bool IsFileLexer()
const {
2626 return IsFileLexer(CurLexer.get(), CurPPLexer);
2631 void CachingLex(Token &
Result);
2633 bool InCachingLexMode()
const {
2636 return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2639 void EnterCachingLexMode();
2640 void EnterCachingLexModeUnchecked();
2642 void ExitCachingLexMode() {
2643 if (InCachingLexMode())
2647 const Token &PeekAhead(
unsigned N);
2648 void AnnotatePreviousCachedTokens(
const Token &Tok);
2654 void HandleLineDirective();
2655 void HandleDigitDirective(Token &Tok);
2656 void HandleUserDiagnosticDirective(Token &Tok,
bool isWarning);
2657 void HandleIdentSCCSDirective(Token &Tok);
2658 void HandleMacroPublicDirective(Token &Tok);
2659 void HandleMacroPrivateDirective();
2663 struct ImportAction {
2669 SkippedModuleImport,
2672 Module *ModuleForHeader =
nullptr;
2674 ImportAction(ActionKind AK, Module *Mod =
nullptr)
2675 :
Kind(AK), ModuleForHeader(Mod) {
2676 assert((AK == None || Mod || AK == Failure) &&
2677 "no module for module action");
2683 SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2684 const Token &FilenameTok,
bool &IsFrameworkFound,
bool IsImportDecl,
2686 const FileEntry *LookupFromFile, StringRef &LookupFilename,
2687 SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2688 ModuleMap::KnownHeader &SuggestedModule,
bool isAngled);
2691 void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2693 const FileEntry *LookupFromFile =
nullptr);
2695 HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2696 Token &FilenameTok, SourceLocation EndLoc,
2698 const FileEntry *LookupFromFile =
nullptr);
2699 void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2700 void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2701 void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2702 void HandleMicrosoftImportDirective(Token &Tok);
2709 const TargetInfo &TargetInfo,
2710 const Module &M, DiagnosticsEngine &Diags);
2733 SourceLocation MLoc);
2736 return PreambleConditionalStack.isRecording();
2740 return PreambleConditionalStack.hasRecordedPreamble();
2744 return PreambleConditionalStack.getStack();
2748 PreambleConditionalStack.setStack(
s);
2753 PreambleConditionalStack.startReplaying();
2754 PreambleConditionalStack.setStack(
s);
2755 PreambleConditionalStack.SkipInfo = SkipInfo;
2759 return PreambleConditionalStack.SkipInfo;
2765 void replayPreambleConditionalStack();
2768 void HandleDefineDirective(
Token &Tok,
bool ImmediatelyAfterHeaderGuard);
2769 void HandleUndefDirective();
2773 bool isIfndef,
bool ReadAnyTokensBeforeDirective);
2774 void HandleIfDirective(
Token &IfToken,
const Token &HashToken,
2775 bool ReadAnyTokensBeforeDirective);
2776 void HandleEndifDirective(
Token &EndifToken);
2778 void HandleElifFamilyDirective(
Token &ElifToken,
const Token &HashToken,
2807 auto Annotations = AnnotationInfos.find(II);
2808 if (Annotations == AnnotationInfos.end())
2809 AnnotationInfos.insert(std::make_pair(
2811 MacroAnnotations::makeDeprecation(AnnotationLoc, std::move(Msg))));
2813 Annotations->second.DeprecationInfo =
2814 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2819 auto Annotations = AnnotationInfos.find(II);
2820 if (Annotations == AnnotationInfos.end())
2821 AnnotationInfos.insert(
2822 std::make_pair(II, MacroAnnotations::makeRestrictExpansion(
2823 AnnotationLoc, std::move(Msg))));
2825 Annotations->second.RestrictExpansionInfo =
2826 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2830 auto Annotations = AnnotationInfos.find(II);
2831 if (Annotations == AnnotationInfos.end())
2832 AnnotationInfos.insert(
2833 std::make_pair(II, MacroAnnotations::makeFinal(AnnotationLoc)));
2835 Annotations->second.FinalAnnotationLoc = AnnotationLoc;
2839 return AnnotationInfos.find(II)->second;
2843 bool IsIfnDef =
false)
const {
2849 !SourceMgr.isInMainFile(
Identifier.getLocation()))
2872 void emitFinalMacroWarning(
const Token &
Identifier,
bool IsUndef)
const;
2874 unsigned DiagSelection)
const;
2879 bool InSafeBufferOptOutRegion =
false;
2925 return P.CurLexer->Lex(
Result);
2927 static bool CLK_TokenLexer(Preprocessor &
P, Token &
Result) {
2928 return P.CurTokenLexer->Lex(
Result);
2930 static bool CLK_CachingLexer(Preprocessor &
P, Token &
Result) {
2934 static bool CLK_DependencyDirectivesLexer(Preprocessor &
P, Token &
Result) {
2935 return P.CurLexer->LexDependencyDirectiveToken(
Result);
2937 static bool CLK_LexAfterModuleImport(Preprocessor &
P, Token &
Result) {
2938 return P.LexAfterModuleImport(
Result);
Defines the Diagnostic-related interfaces.
static constexpr Builtin::Info BuiltinInfo[]
Defines the Diagnostic IDs-related interfaces.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
llvm::MachO::Target Target
llvm::MachO::Record Record
Defines the clang::MacroInfo and clang::MacroDirective classes.
Defines the clang::Module class, which describes a module in the source code.
Defines the PPCallbacks interface.
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
Defines the clang::TokenKind enum and support functions.
__device__ __2f16 float __ockl_bool s
Reads an AST files chain containing the contents of a translation unit.
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Callback handler that receives notifications when performing code completion within the preprocessor.
A directive for a defined macro or a macro imported from a module.
A little helper class used to produce diagnostics.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
void setSuppressAllDiagnostics(bool Val)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
A reference to a DirectoryEntry that includes the name of the directory as it was accessed by the Fil...
Abstract base class that describes a handler that will receive source ranges for empty lines encounte...
virtual void HandleEmptyline(SourceRange Range)=0
virtual ~EmptylineHandler()
Abstract interface for external sources of preprocessor information.
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
Cached information about one file (either on disk or in the virtual file system).
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Implements support for file system lookup, file system caching, and directory search management.
One of these records is kept for each identifier that is lexed.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
bool isDeprecatedMacro() const
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
StringRef getName() const
Return the actual identifier string.
bool isRestrictExpansion() const
Implements an efficient mapping from strings to IdentifierInfo nodes.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
FPEvalMethodKind
Possible float expression evaluation method choices.
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion.
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Characters, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token,...
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer,...
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
A description of the current definition of a macro.
const DefMacroDirective * getDirective() const
Encapsulates changes to the "macros namespace" (the location where the macro name became active,...
Encapsulates the data about a macro definition (e.g.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Abstract interface for a module loader.
Represents a macro directive exported by a module.
Describes a module or submodule.
bool isModuleMapModule() const
This interface provides a way to observe the actions of the preprocessor as it does its thing.
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
SourceLocation getLastFPEvalPragmaLocation() const
bool isMacroDefined(const IdentifierInfo *II)
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
bool markIncluded(FileEntryRef File)
Mark the file as included.
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
void FinalizeForModelFile()
Cleanup after model file parsing.
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void HandlePragmaPoison()
HandlePragmaPoison - Handle #pragma GCC poison. PoisonTok is the 'poison'.
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void dumpMacroInfo(const IdentifierInfo *II)
std::pair< IdentifierInfo *, SourceLocation > getPragmaARCCFCodeAuditedInfo() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaSystemHeader(Token &SysHeaderTok)
HandlePragmaSystemHeader - Implement #pragma GCC system_header.
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
void markClangModuleAsAffecting(Module *M)
Mark the given clang module as affecting the current clang module or translation unit.
void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident, SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaModuleBuild(Token &Tok)
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
PPCallbacks * getPPCallbacks() const
bool isInNamedInterfaceUnit() const
If we are proprocessing a named interface unit.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
void setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc)
Record the location of the unterminated #pragma clang assume_nonnull begin in the preamble.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
ArrayRef< BuildingSubmoduleInfo > getBuildingSubmodules() const
Get the list of submodules that we're currently building.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
SourceRange getCodeCompletionTokenRange() const
SourceLocation getModuleImportLoc(Module *M) const
void overrideMaxTokens(unsigned Value, SourceLocation Loc)
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
bool isRecordingPreamble() const
void HandleSkippedDirectiveWhileUsingPCH(Token &Result, SourceLocation HashLoc)
Process directives while skipping until the through header or #pragma hdrstop is found.
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
bool GetSuppressIncludeNotFoundError()
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
LangOptions::FPEvalMethodKind getCurrentFPEvalMethod() const
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
bool isSafeBufferOptOut(const SourceManager &SourceMgr, const SourceLocation &Loc) const
void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
const char * getCheckPoint(FileID FID, const char *Start) const
Returns a pointer into the given file's buffer that's guaranteed to be between tokens.
void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc)
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
const TargetInfo * getAuxTargetInfo() const
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
void DumpMacro(const MacroInfo &MI) const
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
void setDiagnostics(DiagnosticsEngine &D)
IncludedFilesSet & getIncludedFiles()
Get the set of included files.
friend void TokenLexer::ExpandFunctionArguments()
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool SetCodeCompletionPoint(FileEntryRef File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
StringRef getNamedModuleName() const
Get the named module name we're preprocessing.
void makeModuleVisible(Module *M, SourceLocation Loc)
unsigned getCounterValue() const
bool mightHavePendingAnnotationTokens()
Determine whether it's possible for a future call to Lex to produce an annotation token created by a ...
bool isInImportingCXXNamedModules() const
If we're importing a standard C++20 Named Modules.
void Lex(Token &Result)
Lex the next token for this preprocessor.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
unsigned getNumDirectives() const
Retrieve the number of Directives that have been processed by the Preprocessor.
bool isInImplementationUnit() const
If we are implementing an implementation module unit.
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
void LexNonComment(Token &Result)
Lex a token.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
bool LexOnOffSwitch(tok::OnOffSwitch &Result)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD.
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
void setMainFileDir(DirectoryEntryRef Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
const IdentifierTable & getIdentifierTable() const
void HandlePragmaDependency(Token &DependencyTok)
HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
SourceLocation CheckEndOfDirective(const char *DirType, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
bool enterOrExitSafeBufferOptOutRegion(bool isEnter, const SourceLocation &Loc)
Alter the state of whether this PP currently is in a "-Wunsafe-buffer-usage" opt-out region.
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, std::optional< PreambleSkipInfo > SkipInfo)
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
ModuleMacro * addModuleMacro(Module *Mod, const IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
macro_iterator macro_end(bool IncludeExternalMacros=true) const
SourceManager & getSourceManager() const
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
std::optional< PreambleSkipInfo > getPreambleSkipInfo() const
void setPreprocessToken(bool Preprocess)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
void HandlePragmaOnce(Token &OnceTok)
HandlePragmaOnce - Handle #pragma once. OnceTok is the 'once'.
EmptylineHandler * getEmptylineHandler() const
bool getCommentRetentionState() const
bool isMacroDefined(StringRef Id)
static bool checkModuleIsAvailable(const LangOptions &LangOpts, const TargetInfo &TargetInfo, const Module &M, DiagnosticsEngine &Diags)
Check that the given module is available, producing a diagnostic if not.
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool hasRecordedPreamble() const
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
MacroMap::const_iterator macro_iterator
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
void RemovePragmaHandler(PragmaHandler *Handler)
bool isPPInSafeBufferOptOutRegion()
unsigned getTokenCount() const
Get the number of tokens processed so far.
unsigned getMaxTokens() const
Get the max number of tokens before issuing a -Wmax-tokens warning.
SourceLocation getMaxTokensOverrideLoc() const
bool hadModuleLoaderFatalFailure() const
static void processPathToFileName(SmallVectorImpl< char > &FileName, const PresumedLoc &PLoc, const LangOptions &LangOpts, const TargetInfo &TI)
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
const TargetInfo & getTargetInfo() const
FileManager & getFileManager() const
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void DumpLocation(SourceLocation Loc) const
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
bool alreadyIncluded(FileEntryRef File) const
Return true if this header has already been included.
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character.
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
llvm::BumpPtrAllocator & getPreprocessorAllocator()
ModuleMacro * getModuleMacro(Module *Mod, const IdentifierInfo *II)
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleComment(Token &result, SourceRange Comment)
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Buffer)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
HeaderSearch & getHeaderSearchInfo() const
void emitMacroExpansionWarnings(const Token &Identifier, bool IsIfnDef=false) const
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
ExternalPreprocessorSource * getExternalSource() const
Module * LeaveSubmodule(bool ForPragma)
SourceRange DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found.
const std::string & getPredefines() const
Get the predefines for this processor.
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind, void *AnnotationVal)
Enter an annotation token into the token stream.
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, const LangOptions &LangOpts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
MacroInfo * getMacroInfo(const IdentifierInfo *II)
void setPredefines(std::string P)
Set the predefines for this Preprocessor.
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
IdentifierTable & getIdentifierTable()
Builtin::Context & getBuiltinInfo()
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
LangOptions::FPEvalMethodKind getTUFPEvalMethod() const
const LangOptions & getLangOpts() const
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void SetSuppressIncludeNotFoundError(bool Suppress)
static void processPathForFileMacro(SmallVectorImpl< char > &Path, const LangOptions &LangOpts, const TargetInfo &TI)
bool isInNamedModule() const
If we are preprocessing a named module.
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
void PoisonSEHIdentifiers(bool Poison=true)
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion.
size_t getTotalMemory() const
void setExternalSource(ExternalPreprocessorSource *Source)
void clearCodeCompletionHandler()
Clear out the code completion handler.
void AddPragmaHandler(PragmaHandler *Handler)
OptionalFileEntryRef getHeaderToIncludeForDiagnostics(SourceLocation IncLoc, SourceLocation MLoc)
We want to produce a diagnostic at location IncLoc concerning an unreachable effect at location MLoc ...
void setCounterValue(unsigned V)
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.
void LexTokensUntilEOF(std::vector< Token > *Tokens=nullptr)
Lex all tokens for this preprocessor until (and excluding) end of file.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
void setPragmasEnabled(bool Enabled)
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void HandlePragmaMark(Token &MarkTok)
bool getPragmasEnabled() const
void HandlePragmaHdrstop(Token &Tok)
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
void setEmptylineHandler(EmptylineHandler *Handler)
Set empty line handler.
DiagnosticsEngine & getDiagnostics() const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
llvm::DenseSet< const FileEntry * > IncludedFilesSet
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SelectorTable & getSelectorTable()
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
const llvm::SmallSetVector< Module *, 2 > & getAffectingClangModules() const
Get the set of top-level clang modules that affected preprocessing, but were not imported.
const IncludedFilesSet & getIncludedFiles() const
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void HandlePragmaIncludeAlias(Token &Tok)
Module * getModuleForLocation(SourceLocation Loc, bool AllowTextual)
Find the module that owns the source or header file that Loc points to.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
SourceLocation getPreambleRecordedPragmaAssumeNonNullLoc() const
Get the location of the recorded unterminated #pragma clang assume_nonnull begin in the preamble,...
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
void EnableBacktrackAtThisPos()
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop.
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
Represents an unpacked "presumed" location which can be presented to the user.
This table allows us to fully hide how we implement multi-keyword caching.
Encodes a location in the source.
This class handles loading and caching of source files into memory.
A trivial tuple used to represent a source range.
Exposes information about the current target.
Stores token information for comparing actual tokens with predefined values.
TokenValue(IdentifierInfo *II)
TokenValue(tok::TokenKind Kind)
bool operator==(const Token &Tok) const
Token - This structure provides full information about a lexed token.
IdentifierInfo * getIdentifierInfo() const
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
unsigned getLength() const
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
tok::TokenKind getKind() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a variadic mac...
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
Directive - Abstract class representing a parsed verify directive.
constexpr XRayInstrMask None
StringRef getName(const HeaderType T)
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a '#' at the beginning of the line.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
The JSON file list parser is used to communicate input to InstallAPI.
CustomizableOptional< DirectoryEntryRef > OptionalDirectoryEntryRef
detail::SearchDirIteratorImpl< true > ConstSearchDirIterator
MacroUse
Context in which macro name is used.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
@ Module
Module linkage, which indicates that the entity can be referred to from other translation units withi...
@ Result
The result type of a method or function.
CustomizableOptional< FileEntryRef > OptionalFileEntryRef
TranslationUnitKind
Describes the kind of translation unit being processed.
@ TU_Complete
The translation unit is a complete translation unit.
Diagnostic wrappers for TextAPI types for error reporting.
Describes how and where the pragma was introduced.
SourceLocation IfTokenLoc
SourceLocation HashTokenLoc
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)