14#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15#define LLVM_CLANG_LEX_PREPROCESSOR_H
34#include "llvm/ADT/ArrayRef.h"
35#include "llvm/ADT/DenseMap.h"
36#include "llvm/ADT/FoldingSet.h"
37#include "llvm/ADT/FunctionExtras.h"
38#include "llvm/ADT/PointerUnion.h"
39#include "llvm/ADT/STLExtras.h"
40#include "llvm/ADT/SmallPtrSet.h"
41#include "llvm/ADT/SmallVector.h"
42#include "llvm/ADT/StringRef.h"
43#include "llvm/ADT/TinyPtrVector.h"
44#include "llvm/ADT/iterator_range.h"
45#include "llvm/Support/Allocator.h"
46#include "llvm/Support/Casting.h"
47#include "llvm/Support/Registry.h"
60template<
unsigned InternalLen>
class SmallString;
66class CodeCompletionHandler;
69class EmptylineHandler;
70class ExternalPreprocessorSource;
77class PreprocessingRecord;
78class PreprocessorLexer;
79class PreprocessorOptions;
95 assert(Kind != tok::raw_identifier &&
"Raw identifiers are not supported.");
96 assert(Kind != tok::identifier &&
97 "Identifiers should be created by TokenValue(IdentifierInfo *)");
105 return Tok.
getKind() == Kind &&
132 llvm::unique_function<void(
const clang::Token &)> OnToken;
133 std::shared_ptr<PreprocessorOptions> PPOpts;
140 std::unique_ptr<ScratchBuffer> ScratchBuf;
149 llvm::BumpPtrAllocator BP;
205 unsigned CounterValue = 0;
209 MaxAllowedIncludeStackDepth = 200
213 bool KeepComments : 1;
214 bool KeepMacroComments : 1;
215 bool SuppressIncludeNotFoundError : 1;
218 bool InMacroArgs : 1;
221 bool OwnsHeaderSearch : 1;
224 bool DisableMacroExpansion : 1;
228 bool MacroExpansionInDirectivesOverride : 1;
230 class ResetMacroExpansionHelper;
233 mutable bool ReadMacrosFromExternalSource : 1;
236 bool PragmasEnabled : 1;
239 bool PreprocessedOutput : 1;
242 bool ParsingIfOrElifDirective;
245 bool InMacroArgPreExpansion;
267 std::unique_ptr<PragmaNamespace> PragmaHandlers;
271 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
275 std::vector<CommentHandler *> CommentHandlers;
281 bool IncrementalProcessing =
false;
299 const FileEntry *CodeCompletionFile =
nullptr;
302 unsigned CodeCompletionOffset = 0;
321 llvm::DenseMap<FileID, SmallVector<const char *>> CheckPoints;
322 unsigned CheckPointCounter = 0;
325 bool IsAtImport =
false;
328 bool LastTokenWasAt =
false;
331 class StdCXXImportSeq {
336 AfterTopLevelTokenSeq = -1,
341 StdCXXImportSeq(State S) : S(S) {}
344 void handleOpenBracket() {
345 S =
static_cast<State
>(std::max<int>(S, 0) + 1);
348 void handleCloseBracket() {
349 S =
static_cast<State
>(std::max<int>(S, 1) - 1);
352 void handleCloseBrace() {
353 handleCloseBracket();
354 if (S == AtTopLevel && !AfterHeaderName)
355 S = AfterTopLevelTokenSeq;
360 S = AfterTopLevelTokenSeq;
361 AfterHeaderName =
false;
366 void handleExport() {
367 if (S == AfterTopLevelTokenSeq)
373 void handleImport() {
374 if (S == AfterTopLevelTokenSeq || S == AfterExport)
382 void handleHeaderName() {
383 if (S == AfterImportSeq)
384 AfterHeaderName =
true;
394 bool atTopLevel() {
return S <= 0; }
395 bool afterImportSeq() {
return S == AfterImportSeq; }
396 bool afterTopLevelSeq() {
return S == AfterTopLevelTokenSeq; }
403 bool AfterHeaderName =
false;
407 StdCXXImportSeq StdCXXImportSeqState = StdCXXImportSeq::AfterTopLevelTokenSeq;
412 enum GMFState :
int {
415 BeforeGMFIntroducer = -1,
416 GMFAbsentOrEnded = -2,
419 TrackGMF(GMFState S) : S(S) {}
430 void handleExport() {
432 S = GMFAbsentOrEnded;
436 void handleImport(
bool AfterTopLevelTokenSeq) {
438 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
439 S = GMFAbsentOrEnded;
443 void handleModule(
bool AfterTopLevelTokenSeq) {
447 if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
450 S = GMFAbsentOrEnded;
457 S = GMFAbsentOrEnded;
460 bool inGMF() {
return S == GMFActive; }
468 TrackGMF TrackGMFState = TrackGMF::BeforeGMFIntroducer;
504 class ModuleDeclSeq {
505 enum ModuleDeclState :
int {
509 ImplementationCandidate,
510 NamedModuleInterface,
511 NamedModuleImplementation,
515 ModuleDeclSeq() =
default;
517 void handleExport() {
518 if (State == NotAModuleDecl)
520 else if (!isNamedModule())
524 void handleModule() {
525 if (State == FoundExport)
526 State = InterfaceCandidate;
527 else if (State == NotAModuleDecl)
528 State = ImplementationCandidate;
529 else if (!isNamedModule())
533 void handleIdentifier(IdentifierInfo *
Identifier) {
536 else if (!isNamedModule())
541 if (isModuleCandidate())
543 else if (!isNamedModule())
547 void handlePeriod() {
548 if (isModuleCandidate())
550 else if (!isNamedModule())
555 if (!Name.empty() && isModuleCandidate()) {
556 if (State == InterfaceCandidate)
557 State = NamedModuleInterface;
558 else if (State == ImplementationCandidate)
559 State = NamedModuleImplementation;
561 llvm_unreachable(
"Unimaged ModuleDeclState.");
562 }
else if (!isNamedModule())
567 if (!isNamedModule())
571 bool isModuleCandidate()
const {
572 return State == InterfaceCandidate || State == ImplementationCandidate;
575 bool isNamedModule()
const {
576 return State == NamedModuleInterface ||
577 State == NamedModuleImplementation;
580 bool isNamedInterface()
const {
return State == NamedModuleInterface; }
582 bool isImplementationUnit()
const {
583 return State == NamedModuleImplementation && !
getName().contains(
':');
587 assert(isNamedModule() &&
"Can't get name from a non named module");
591 StringRef getPrimaryName()
const {
592 assert(isNamedModule() &&
"Can't get name from a non named module");
593 return getName().split(
':').first;
598 State = NotAModuleDecl;
602 ModuleDeclState State = NotAModuleDecl;
606 ModuleDeclSeq ModuleDeclState;
610 bool ModuleImportExpectsIdentifier =
false;
614 std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
618 SourceLocation PragmaAssumeNonNullLoc;
626 SourceLocation PreambleRecordedPragmaAssumeNonNullLoc;
629 bool CodeCompletionReached =
false;
633 IdentifierInfo *CodeCompletionII =
nullptr;
636 SourceRange CodeCompletionTokenRange;
648 std::pair<int, bool> SkipMainFilePreamble;
652 bool HasReachedMaxIncludeDepth =
false;
660 unsigned LexLevel = 0;
663 unsigned TokenCount = 0;
666 bool PreprocessToken =
false;
670 unsigned MaxTokens = 0;
671 SourceLocation MaxTokensOverrideLoc;
695 class PreambleConditionalStackStore {
703 PreambleConditionalStackStore() =
default;
705 void startRecording() { ConditionalStackState = Recording; }
706 void startReplaying() { ConditionalStackState = Replaying; }
707 bool isRecording()
const {
return ConditionalStackState == Recording; }
708 bool isReplaying()
const {
return ConditionalStackState == Replaying; }
710 ArrayRef<PPConditionalInfo> getStack()
const {
711 return ConditionalStack;
714 void doneReplaying() {
715 ConditionalStack.clear();
716 ConditionalStackState = Off;
719 void setStack(ArrayRef<PPConditionalInfo>
s) {
720 if (!isRecording() && !isReplaying())
722 ConditionalStack.clear();
723 ConditionalStack.append(
s.begin(),
s.end());
726 bool hasRecordedPreamble()
const {
return !ConditionalStack.empty(); }
728 bool reachedEOFWhileSkipping()
const {
return SkipInfo.has_value(); }
730 void clearSkipInfo() { SkipInfo.reset(); }
732 std::optional<PreambleSkipInfo> SkipInfo;
735 SmallVector<PPConditionalInfo, 4> ConditionalStack;
736 State ConditionalStackState = Off;
737 } PreambleConditionalStack;
743 std::unique_ptr<Lexer> CurLexer;
749 PreprocessorLexer *CurPPLexer =
nullptr;
761 std::unique_ptr<TokenLexer> CurTokenLexer;
765 LexerCallback CurLexerCallback = &CLK_Lexer;
769 Module *CurLexerSubmodule =
nullptr;
774 struct IncludeStackInfo {
775 LexerCallback CurLexerCallback;
777 std::unique_ptr<Lexer> TheLexer;
778 PreprocessorLexer *ThePPLexer;
779 std::unique_ptr<TokenLexer> TheTokenLexer;
784 IncludeStackInfo(LexerCallback CurLexerCallback, Module *TheSubmodule,
785 std::unique_ptr<Lexer> &&TheLexer,
786 PreprocessorLexer *ThePPLexer,
787 std::unique_ptr<TokenLexer> &&TheTokenLexer,
789 : CurLexerCallback(
std::move(CurLexerCallback)),
790 TheSubmodule(
std::move(TheSubmodule)), TheLexer(
std::move(TheLexer)),
791 ThePPLexer(
std::move(ThePPLexer)),
792 TheTokenLexer(
std::move(TheTokenLexer)),
793 TheDirLookup(
std::move(TheDirLookup)) {}
795 std::vector<IncludeStackInfo> IncludeMacroStack;
799 std::unique_ptr<PPCallbacks> Callbacks;
801 struct MacroExpandsInfo {
806 MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
809 SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
812 struct ModuleMacroInfo {
817 llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
821 unsigned ActiveModuleMacrosGeneration = 0;
824 bool IsAmbiguous =
false;
827 llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
829 ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
834 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
836 ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
837 const IdentifierInfo *II)
const {
838 if (II->isOutOfDate())
839 PP.updateOutOfDateIdentifier(
const_cast<IdentifierInfo&
>(*II));
842 if (!II->hasMacroDefinition() ||
843 (!PP.getLangOpts().Modules &&
844 !PP.getLangOpts().ModulesLocalVisibility) ||
845 !PP.CurSubmoduleState->VisibleModules.getGeneration())
848 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
850 Info =
new (PP.getPreprocessorAllocator())
851 ModuleMacroInfo(State.get<MacroDirective *>());
855 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
856 Info->ActiveModuleMacrosGeneration)
857 PP.updateModuleMacroInfo(II, *Info);
862 MacroState() : MacroState(nullptr) {}
863 MacroState(MacroDirective *MD) : State(MD) {}
865 MacroState(MacroState &&O) noexcept : State(O.State) {
866 O.State = (MacroDirective *)
nullptr;
869 MacroState &operator=(MacroState &&O)
noexcept {
871 O.State = (MacroDirective *)
nullptr;
877 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
878 Info->~ModuleMacroInfo();
881 MacroDirective *getLatest()
const {
882 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
884 return State.get<MacroDirective*>();
887 void setLatest(MacroDirective *MD) {
888 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
894 bool isAmbiguous(Preprocessor &PP,
const IdentifierInfo *II)
const {
895 auto *Info = getModuleInfo(PP, II);
896 return Info ? Info->IsAmbiguous :
false;
899 ArrayRef<ModuleMacro *>
900 getActiveModuleMacros(Preprocessor &PP,
const IdentifierInfo *II)
const {
901 if (
auto *Info = getModuleInfo(PP, II))
902 return Info->ActiveModuleMacros;
906 MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
907 SourceManager &SourceMgr)
const {
909 if (
auto *Latest = getLatest())
910 return Latest->findDirectiveAtLoc(Loc, SourceMgr);
914 void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
915 if (
auto *Info = getModuleInfo(PP, II)) {
916 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
917 Info->ActiveModuleMacros.begin(),
918 Info->ActiveModuleMacros.end());
919 Info->ActiveModuleMacros.clear();
920 Info->IsAmbiguous =
false;
924 ArrayRef<ModuleMacro*> getOverriddenMacros()
const {
925 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
926 return Info->OverriddenMacros;
930 void setOverriddenMacros(Preprocessor &PP,
931 ArrayRef<ModuleMacro *> Overrides) {
932 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
934 if (Overrides.empty())
936 Info =
new (PP.getPreprocessorAllocator())
937 ModuleMacroInfo(State.get<MacroDirective *>());
940 Info->OverriddenMacros.clear();
941 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
942 Overrides.begin(), Overrides.end());
943 Info->ActiveModuleMacrosGeneration = 0;
952 using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
954 struct SubmoduleState;
957 struct BuildingSubmoduleInfo {
962 SourceLocation ImportLoc;
968 SubmoduleState *OuterSubmoduleState;
971 unsigned OuterPendingModuleMacroNames;
973 BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc,
bool IsPragma,
974 SubmoduleState *OuterSubmoduleState,
975 unsigned OuterPendingModuleMacroNames)
976 : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
977 OuterSubmoduleState(OuterSubmoduleState),
978 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
980 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
983 struct SubmoduleState {
988 VisibleModuleSet VisibleModules;
993 std::map<Module *, SubmoduleState> Submodules;
996 SubmoduleState NullSubmoduleState;
1000 SubmoduleState *CurSubmoduleState;
1010 llvm::FoldingSet<ModuleMacro> ModuleMacros;
1017 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
1028 using WarnUnusedMacroLocsTy = llvm::SmallDenseSet<SourceLocation, 32>;
1029 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
1035 using MsgLocationPair = std::pair<std::string, SourceLocation>;
1037 struct MacroAnnotationInfo {
1038 SourceLocation Location;
1039 std::string Message;
1042 struct MacroAnnotations {
1043 std::optional<MacroAnnotationInfo> DeprecationInfo;
1044 std::optional<MacroAnnotationInfo> RestrictExpansionInfo;
1045 std::optional<SourceLocation> FinalAnnotationLoc;
1047 static MacroAnnotations makeDeprecation(SourceLocation Loc,
1049 return MacroAnnotations{MacroAnnotationInfo{Loc, std::move(Msg)},
1050 std::nullopt, std::nullopt};
1053 static MacroAnnotations makeRestrictExpansion(SourceLocation Loc,
1055 return MacroAnnotations{
1056 std::nullopt, MacroAnnotationInfo{Loc, std::move(Msg)}, std::nullopt};
1059 static MacroAnnotations makeFinal(SourceLocation Loc) {
1060 return MacroAnnotations{std::nullopt, std::nullopt, Loc};
1065 llvm::DenseMap<const IdentifierInfo *, MacroAnnotations> AnnotationInfos;
1073 llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
1074 PragmaPushMacroInfo;
1077 unsigned NumDirectives = 0;
1078 unsigned NumDefined = 0;
1079 unsigned NumUndefined = 0;
1080 unsigned NumPragma = 0;
1082 unsigned NumElse = 0;
1083 unsigned NumEndif = 0;
1084 unsigned NumEnteredSourceFiles = 0;
1085 unsigned MaxIncludeStackDepth = 0;
1086 unsigned NumMacroExpanded = 0;
1087 unsigned NumFnMacroExpanded = 0;
1088 unsigned NumBuiltinMacroExpanded = 0;
1089 unsigned NumFastMacroExpanded = 0;
1090 unsigned NumTokenPaste = 0;
1091 unsigned NumFastTokenPaste = 0;
1092 unsigned NumSkipped = 0;
1096 std::string Predefines;
1099 FileID PredefinesFileID;
1102 FileID PCHThroughHeaderFileID;
1105 bool SkippingUntilPragmaHdrStop =
false;
1108 bool SkippingUntilPCHThroughHeader =
false;
1112 enum { TokenLexerCacheSize = 8 };
1113 unsigned NumCachedTokenLexers;
1114 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
1122 SmallVector<Token, 16> MacroExpandedTokens;
1123 std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
1130 PreprocessingRecord *
Record =
nullptr;
1133 using CachedTokensTy = SmallVector<Token, 1>;
1137 CachedTokensTy CachedTokens;
1144 CachedTokensTy::size_type CachedLexPos = 0;
1151 std::vector<CachedTokensTy::size_type> BacktrackPositions;
1157 bool SkippingExcludedConditionalBlock =
false;
1163 llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges;
1165 void updateOutOfDateIdentifier(IdentifierInfo &II)
const;
1168 Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
1169 DiagnosticsEngine &diags,
const LangOptions &LangOpts,
1170 SourceManager &
SM, HeaderSearch &Headers,
1171 ModuleLoader &TheModuleLoader,
1172 IdentifierInfoLookup *IILookup =
nullptr,
1173 bool OwnsHeaderSearch =
false,
1185 const TargetInfo *AuxTarget =
nullptr);
1236 return NumDirectives;
1241 return ParsingIfOrElifDirective;
1246 this->KeepComments = KeepComments | KeepMacroComments;
1247 this->KeepMacroComments = KeepMacroComments;
1256 SuppressIncludeNotFoundError = Suppress;
1260 return SuppressIncludeNotFoundError;
1266 PreprocessedOutput = IsPreprocessedOutput;
1275 return CurPPLexer == L;
1305 C = std::make_unique<PPChainedCallbacks>(std::move(
C),
1306 std::move(Callbacks));
1307 Callbacks = std::move(
C);
1319 MaxTokensOverrideLoc = Loc;
1328 OnToken = std::move(F);
1347 auto I = Submodules.find(M);
1348 if (I == Submodules.end())
1350 auto J = I->second.Macros.find(II);
1351 if (J == I->second.Macros.end())
1353 auto *MD = J->second.getLatest();
1354 return MD && MD->isDefined();
1361 MacroState &S = CurSubmoduleState->Macros[II];
1362 auto *MD = S.getLatest();
1363 while (MD && isa<VisibilityMacroDirective>(MD))
1364 MD = MD->getPrevious();
1366 S.getActiveModuleMacros(*
this, II),
1367 S.isAmbiguous(*
this, II));
1375 MacroState &S = CurSubmoduleState->Macros[II];
1377 if (
auto *MD = S.getLatest())
1381 S.getActiveModuleMacros(*
this, II),
1382 S.isAmbiguous(*
this, II));
1392 if (!MD || MD->getDefinition().isUndefined())
1406 return MD.getMacroInfo();
1443 auto I = LeafModuleMacros.find(II);
1444 if (I != LeafModuleMacros.end())
1446 return std::nullopt;
1451 return BuildingSubmoduleStack;
1463 llvm::iterator_range<macro_iterator>
1464 macros(
bool IncludeExternalMacros =
true)
const {
1467 return llvm::make_range(begin, end);
1475 if (!BuildingSubmoduleStack.empty()) {
1476 if (M != BuildingSubmoduleStack.back().M)
1477 BuildingSubmoduleStack.back().M->AffectingClangModules.insert(M);
1479 AffectingClangModules.insert(M);
1486 return AffectingClangModules;
1493 return IncludedFiles.insert(
File).second;
1499 return IncludedFiles.count(
File);
1525 return &Identifiers.
get(Name);
1565 CodeComplete = &Handler;
1570 return CodeComplete;
1575 CodeComplete =
nullptr;
1588 CodeCompletionII = Filter;
1595 CodeCompletionTokenRange = {Start, End};
1598 return CodeCompletionTokenRange;
1603 if (CodeCompletionII)
1604 return CodeCompletionII->getName();
1678 void EnterTokenStream(
const Token *Toks,
unsigned NumToks,
1679 bool DisableMacroExpansion,
bool OwnsTokens,
1684 bool DisableMacroExpansion,
bool IsReinject) {
1685 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion,
true,
1691 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion,
false,
1742 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1749 const char *DiagnosticTag,
bool AllowMacroExpansion) {
1750 if (AllowMacroExpansion)
1755 AllowMacroExpansion);
1761 const char *DiagnosticTag,
1762 bool AllowMacroExpansion);
1772 while (
Result.getKind() == tok::comment);
1778 bool OldVal = DisableMacroExpansion;
1779 DisableMacroExpansion =
true;
1784 DisableMacroExpansion = OldVal;
1792 while (
Result.getKind() == tok::comment);
1802 DisableMacroExpansion =
true;
1803 MacroExpansionInDirectivesOverride =
true;
1814 assert(LexLevel == 0 &&
"cannot use lookahead while lexing");
1818 return PeekAhead(N+1);
1828 "Should only be called when tokens are cached for backtracking");
1829 assert(
signed(CachedLexPos) -
signed(N) >=
signed(BacktrackPositions.back())
1830 &&
"Should revert tokens up to the last backtrack position, not more");
1831 assert(
signed(CachedLexPos) -
signed(N) >= 0 &&
1832 "Corrupted backtrack positions ?");
1846 auto TokCopy = std::make_unique<Token[]>(1);
1848 EnterTokenStream(std::move(TokCopy), 1,
true, IsReinject);
1850 EnterCachingLexMode();
1851 assert(IsReinject &&
"new tokens in the middle of cached stream");
1865 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1867 AnnotatePreviousCachedTokens(Tok);
1873 assert(CachedLexPos != 0);
1897 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1904 void *AnnotationVal);
1909 return CurLexerCallback != CLK_Lexer;
1929 IncrementalProcessing = value;
1963 return CodeCompletionFileLoc;
1973 CodeCompletionReached =
true;
1982 std::pair<IdentifierInfo *, SourceLocation>
1984 return PragmaARCCFCodeAuditedInfo;
1991 PragmaARCCFCodeAuditedInfo = {Ident, Loc};
1999 return PragmaAssumeNonNullLoc;
2005 PragmaAssumeNonNullLoc = Loc;
2014 return PreambleRecordedPragmaAssumeNonNullLoc;
2020 PreambleRecordedPragmaAssumeNonNullLoc = Loc;
2034 SkipMainFilePreamble.first = Bytes;
2035 SkipMainFilePreamble.second = StartOfLine;
2042 return Diags->
Report(Loc, DiagID);
2058 bool *invalid =
nullptr)
const {
2087 bool *
Invalid =
nullptr)
const {
2097 bool *
Invalid =
nullptr)
const;
2102 bool IgnoreWhiteSpace =
false) {
2110 bool *
Invalid =
nullptr)
const {
2111 assert(Tok.
is(tok::numeric_constant) &&
2112 Tok.
getLength() == 1 &&
"Called on unsupported token");
2113 assert(!Tok.
needsCleaning() &&
"Token can't need cleaning with length 1");
2199 unsigned Char)
const {
2209 ++NumFastTokenPaste;
2233 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
2247 if(II->isPoisoned()) {
2258 *Ident___exception_code,
2259 *Ident_GetExceptionCode;
2262 *Ident___exception_info,
2263 *Ident_GetExceptionInfo;
2266 *Ident___abnormal_termination,
2267 *Ident_AbnormalTermination;
2269 const char *getCurLexerEndPos();
2270 void diagnoseMissingHeaderInUmbrellaDir(
const Module &Mod);
2312 bool EnableMacros =
false);
2328 "FPEvalMethod should be set either from command line or from the "
2330 return CurrentFPEvalMethod;
2334 return TUFPEvalMethod;
2338 return LastFPEvalPragmaLocation;
2344 "FPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2347 LastFPEvalPragmaLocation = PragmaLoc;
2348 CurrentFPEvalMethod = Val;
2349 TUFPEvalMethod = Val;
2354 "TUPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2355 TUFPEvalMethod = Val;
2372 return ModuleDeclState.isNamedInterface();
2383 return ModuleDeclState.isImplementationUnit();
2390 return !NamedModuleImportPath.empty() &&
getLangOpts().CPlusPlusModules &&
2418 bool *IsFrameworkFound,
bool SkipCache =
false,
2419 bool OpenFile =
true,
bool CacheFailures =
true);
2429 bool *ShadowFlag =
nullptr);
2437 void PushIncludeMacroStack() {
2438 assert(CurLexerCallback != CLK_CachingLexer &&
2439 "cannot push a caching lexer");
2440 IncludeMacroStack.emplace_back(CurLexerCallback, CurLexerSubmodule,
2441 std::move(CurLexer), CurPPLexer,
2442 std::move(CurTokenLexer), CurDirLookup);
2443 CurPPLexer =
nullptr;
2446 void PopIncludeMacroStack() {
2447 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2448 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2449 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2450 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2451 CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2452 CurLexerCallback = IncludeMacroStack.back().CurLexerCallback;
2453 IncludeMacroStack.pop_back();
2456 void PropagateLineStartLeadingSpaceInfo(Token &
Result);
2460 bool needModuleMacros()
const;
2464 void updateModuleMacroInfo(
const IdentifierInfo *II, ModuleMacroInfo &Info);
2466 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2467 SourceLocation Loc);
2468 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2469 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2483 bool *ShadowFlag =
nullptr);
2494 MacroInfo *ReadOptionalMacroParameterListAndBody(
2495 const Token &MacroNameTok,
bool ImmediatelyAfterHeaderGuard);
2501 bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2508 void SuggestTypoedDirective(
const Token &Tok, StringRef
Directive)
const;
2518 void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2519 SourceLocation IfTokenLoc,
2520 bool FoundNonSkipPortion,
bool FoundElse,
2521 SourceLocation ElseLoc = SourceLocation());
2525 struct DirectiveEvalResult {
2530 bool IncludedUndefinedIds;
2533 SourceRange ExprRange;
2540 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
2545 bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II);
2550 bool EvaluateHasIncludeNext(Token &Tok, IdentifierInfo *II);
2553 std::pair<ConstSearchDirIterator, const FileEntry *>
2554 getIncludeNextStart(
const Token &IncludeNextTok)
const;
2558 void RegisterBuiltinPragmas();
2561 void RegisterBuiltinMacros();
2566 bool HandleMacroExpandedIdentifier(Token &
Identifier,
const MacroDefinition &MD);
2573 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2574 ArrayRef<Token> tokens);
2576 void removeCachedMacroExpandedTokensOfLastLexer();
2581 bool isNextPPTokenLParen();
2585 MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2586 SourceLocation &MacroEnd);
2590 void ExpandBuiltinMacro(Token &Tok);
2595 void Handle_Pragma(Token &Tok);
2599 void HandleMicrosoft__pragma(Token &Tok);
2606 void setPredefinesFileID(FileID FID) {
2607 assert(PredefinesFileID.isInvalid() &&
"PredefinesFileID already set!");
2608 PredefinesFileID = FID;
2612 void setPCHThroughHeaderFileID(FileID FID);
2616 static bool IsFileLexer(
const Lexer* L,
const PreprocessorLexer*
P) {
2617 return L ? !L->isPragmaLexer() :
P !=
nullptr;
2620 static bool IsFileLexer(
const IncludeStackInfo& I) {
2621 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2624 bool IsFileLexer()
const {
2625 return IsFileLexer(CurLexer.get(), CurPPLexer);
2630 void CachingLex(Token &
Result);
2632 bool InCachingLexMode()
const {
2635 return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2638 void EnterCachingLexMode();
2639 void EnterCachingLexModeUnchecked();
2641 void ExitCachingLexMode() {
2642 if (InCachingLexMode())
2646 const Token &PeekAhead(
unsigned N);
2647 void AnnotatePreviousCachedTokens(
const Token &Tok);
2653 void HandleLineDirective();
2654 void HandleDigitDirective(Token &Tok);
2655 void HandleUserDiagnosticDirective(Token &Tok,
bool isWarning);
2656 void HandleIdentSCCSDirective(Token &Tok);
2657 void HandleMacroPublicDirective(Token &Tok);
2658 void HandleMacroPrivateDirective();
2662 struct ImportAction {
2668 SkippedModuleImport,
2671 Module *ModuleForHeader =
nullptr;
2673 ImportAction(ActionKind AK, Module *Mod =
nullptr)
2674 :
Kind(AK), ModuleForHeader(Mod) {
2675 assert((AK == None || Mod || AK == Failure) &&
2676 "no module for module action");
2682 SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2683 const Token &FilenameTok,
bool &IsFrameworkFound,
bool IsImportDecl,
2685 const FileEntry *LookupFromFile, StringRef &LookupFilename,
2686 SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2687 ModuleMap::KnownHeader &SuggestedModule,
bool isAngled);
2690 void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2692 const FileEntry *LookupFromFile =
nullptr);
2694 HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2695 Token &FilenameTok, SourceLocation EndLoc,
2697 const FileEntry *LookupFromFile =
nullptr);
2698 void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2699 void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2700 void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2701 void HandleMicrosoftImportDirective(Token &Tok);
2708 const TargetInfo &TargetInfo,
2709 const Module &M, DiagnosticsEngine &Diags);
2732 SourceLocation MLoc);
2735 return PreambleConditionalStack.isRecording();
2739 return PreambleConditionalStack.hasRecordedPreamble();
2743 return PreambleConditionalStack.getStack();
2747 PreambleConditionalStack.setStack(
s);
2752 PreambleConditionalStack.startReplaying();
2753 PreambleConditionalStack.setStack(
s);
2754 PreambleConditionalStack.SkipInfo = SkipInfo;
2758 return PreambleConditionalStack.SkipInfo;
2764 void replayPreambleConditionalStack();
2767 void HandleDefineDirective(
Token &Tok,
bool ImmediatelyAfterHeaderGuard);
2768 void HandleUndefDirective();
2772 bool isIfndef,
bool ReadAnyTokensBeforeDirective);
2773 void HandleIfDirective(
Token &IfToken,
const Token &HashToken,
2774 bool ReadAnyTokensBeforeDirective);
2775 void HandleEndifDirective(
Token &EndifToken);
2777 void HandleElifFamilyDirective(
Token &ElifToken,
const Token &HashToken,
2806 auto Annotations = AnnotationInfos.find(II);
2807 if (Annotations == AnnotationInfos.end())
2808 AnnotationInfos.insert(std::make_pair(
2810 MacroAnnotations::makeDeprecation(AnnotationLoc, std::move(Msg))));
2812 Annotations->second.DeprecationInfo =
2813 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2818 auto Annotations = AnnotationInfos.find(II);
2819 if (Annotations == AnnotationInfos.end())
2820 AnnotationInfos.insert(
2821 std::make_pair(II, MacroAnnotations::makeRestrictExpansion(
2822 AnnotationLoc, std::move(Msg))));
2824 Annotations->second.RestrictExpansionInfo =
2825 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2829 auto Annotations = AnnotationInfos.find(II);
2830 if (Annotations == AnnotationInfos.end())
2831 AnnotationInfos.insert(
2832 std::make_pair(II, MacroAnnotations::makeFinal(AnnotationLoc)));
2834 Annotations->second.FinalAnnotationLoc = AnnotationLoc;
2838 return AnnotationInfos.find(II)->second;
2842 bool IsIfnDef =
false)
const {
2848 !SourceMgr.isInMainFile(
Identifier.getLocation()))
2871 void emitFinalMacroWarning(
const Token &
Identifier,
bool IsUndef)
const;
2873 unsigned DiagSelection)
const;
2878 bool InSafeBufferOptOutRegion =
false;
2924 return P.CurLexer->Lex(
Result);
2926 static bool CLK_TokenLexer(Preprocessor &
P, Token &
Result) {
2927 return P.CurTokenLexer->Lex(
Result);
2929 static bool CLK_CachingLexer(Preprocessor &
P, Token &
Result) {
2933 static bool CLK_DependencyDirectivesLexer(Preprocessor &
P, Token &
Result) {
2934 return P.CurLexer->LexDependencyDirectiveToken(
Result);
2936 static bool CLK_LexAfterModuleImport(Preprocessor &
P, Token &
Result) {
2937 return P.LexAfterModuleImport(
Result);
Defines the Diagnostic-related interfaces.
static constexpr Builtin::Info BuiltinInfo[]
Defines the Diagnostic IDs-related interfaces.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
llvm::MachO::Target Target
llvm::MachO::Record Record
Defines the clang::MacroInfo and clang::MacroDirective classes.
Defines the clang::Module class, which describes a module in the source code.
Defines the PPCallbacks interface.
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
Defines the clang::TokenKind enum and support functions.
__device__ __2f16 float __ockl_bool s
Reads an AST files chain containing the contents of a translation unit.
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Callback handler that receives notifications when performing code completion within the preprocessor.
A directive for a defined macro or a macro imported from a module.
A little helper class used to produce diagnostics.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
void setSuppressAllDiagnostics(bool Val)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
A reference to a DirectoryEntry that includes the name of the directory as it was accessed by the Fil...
Abstract base class that describes a handler that will receive source ranges for empty lines encounte...
virtual void HandleEmptyline(SourceRange Range)=0
virtual ~EmptylineHandler()
Abstract interface for external sources of preprocessor information.
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
Cached information about one file (either on disk or in the virtual file system).
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Implements support for file system lookup, file system caching, and directory search management.
One of these records is kept for each identifier that is lexed.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
bool isDeprecatedMacro() const
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
StringRef getName() const
Return the actual identifier string.
bool isRestrictExpansion() const
Implements an efficient mapping from strings to IdentifierInfo nodes.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
FPEvalMethodKind
Possible float expression evaluation method choices.
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion.
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Characters, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token,...
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer,...
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
A description of the current definition of a macro.
const DefMacroDirective * getDirective() const
Encapsulates changes to the "macros namespace" (the location where the macro name became active,...
Encapsulates the data about a macro definition (e.g.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Abstract interface for a module loader.
Represents a macro directive exported by a module.
Describes a module or submodule.
bool isModuleMapModule() const
This interface provides a way to observe the actions of the preprocessor as it does its thing.
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
SourceLocation getLastFPEvalPragmaLocation() const
bool isMacroDefined(const IdentifierInfo *II)
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
bool markIncluded(FileEntryRef File)
Mark the file as included.
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
void FinalizeForModelFile()
Cleanup after model file parsing.
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void HandlePragmaPoison()
HandlePragmaPoison - Handle #pragma GCC poison. PoisonTok is the 'poison'.
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void dumpMacroInfo(const IdentifierInfo *II)
std::pair< IdentifierInfo *, SourceLocation > getPragmaARCCFCodeAuditedInfo() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaSystemHeader(Token &SysHeaderTok)
HandlePragmaSystemHeader - Implement #pragma GCC system_header.
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
void markClangModuleAsAffecting(Module *M)
Mark the given clang module as affecting the current clang module or translation unit.
void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident, SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin.
void HandlePragmaModuleBuild(Token &Tok)
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
PPCallbacks * getPPCallbacks() const
bool isInNamedInterfaceUnit() const
If we are proprocessing a named interface unit.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
void setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc)
Record the location of the unterminated #pragma clang assume_nonnull begin in the preamble.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
ArrayRef< BuildingSubmoduleInfo > getBuildingSubmodules() const
Get the list of submodules that we're currently building.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
SourceRange getCodeCompletionTokenRange() const
SourceLocation getModuleImportLoc(Module *M) const
void overrideMaxTokens(unsigned Value, SourceLocation Loc)
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
bool isRecordingPreamble() const
void HandleSkippedDirectiveWhileUsingPCH(Token &Result, SourceLocation HashLoc)
Process directives while skipping until the through header or #pragma hdrstop is found.
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
bool GetSuppressIncludeNotFoundError()
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
LangOptions::FPEvalMethodKind getCurrentFPEvalMethod() const
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
bool isSafeBufferOptOut(const SourceManager &SourceMgr, const SourceLocation &Loc) const
void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
const char * getCheckPoint(FileID FID, const char *Start) const
Returns a pointer into the given file's buffer that's guaranteed to be between tokens.
void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg, SourceLocation AnnotationLoc)
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc)
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
const TargetInfo * getAuxTargetInfo() const
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
void DumpMacro(const MacroInfo &MI) const
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
void setDiagnostics(DiagnosticsEngine &D)
IncludedFilesSet & getIncludedFiles()
Get the set of included files.
friend void TokenLexer::ExpandFunctionArguments()
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool SetCodeCompletionPoint(FileEntryRef File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
StringRef getNamedModuleName() const
Get the named module name we're preprocessing.
void makeModuleVisible(Module *M, SourceLocation Loc)
unsigned getCounterValue() const
bool mightHavePendingAnnotationTokens()
Determine whether it's possible for a future call to Lex to produce an annotation token created by a ...
bool isInImportingCXXNamedModules() const
If we're importing a standard C++20 Named Modules.
void Lex(Token &Result)
Lex the next token for this preprocessor.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
unsigned getNumDirectives() const
Retrieve the number of Directives that have been processed by the Preprocessor.
bool isInImplementationUnit() const
If we are implementing an implementation module unit.
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
void LexNonComment(Token &Result)
Lex a token.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
bool LexOnOffSwitch(tok::OnOffSwitch &Result)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD.
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
void setMainFileDir(DirectoryEntryRef Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
const IdentifierTable & getIdentifierTable() const
void HandlePragmaDependency(Token &DependencyTok)
HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
SourceLocation CheckEndOfDirective(const char *DirType, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
bool enterOrExitSafeBufferOptOutRegion(bool isEnter, const SourceLocation &Loc)
Alter the state of whether this PP currently is in a "-Wunsafe-buffer-usage" opt-out region.
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, std::optional< PreambleSkipInfo > SkipInfo)
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
macro_iterator macro_end(bool IncludeExternalMacros=true) const
SourceManager & getSourceManager() const
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
std::optional< PreambleSkipInfo > getPreambleSkipInfo() const
void setPreprocessToken(bool Preprocess)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
void HandlePragmaOnce(Token &OnceTok)
HandlePragmaOnce - Handle #pragma once. OnceTok is the 'once'.
EmptylineHandler * getEmptylineHandler() const
bool getCommentRetentionState() const
bool isMacroDefined(StringRef Id)
static bool checkModuleIsAvailable(const LangOptions &LangOpts, const TargetInfo &TargetInfo, const Module &M, DiagnosticsEngine &Diags)
Check that the given module is available, producing a diagnostic if not.
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool hasRecordedPreamble() const
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
MacroMap::const_iterator macro_iterator
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
void RemovePragmaHandler(PragmaHandler *Handler)
bool isPPInSafeBufferOptOutRegion()
unsigned getTokenCount() const
Get the number of tokens processed so far.
unsigned getMaxTokens() const
Get the max number of tokens before issuing a -Wmax-tokens warning.
SourceLocation getMaxTokensOverrideLoc() const
bool hadModuleLoaderFatalFailure() const
static void processPathToFileName(SmallVectorImpl< char > &FileName, const PresumedLoc &PLoc, const LangOptions &LangOpts, const TargetInfo &TI)
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
const TargetInfo & getTargetInfo() const
FileManager & getFileManager() const
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void DumpLocation(SourceLocation Loc) const
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
bool alreadyIncluded(FileEntryRef File) const
Return true if this header has already been included.
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character.
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
llvm::BumpPtrAllocator & getPreprocessorAllocator()
ModuleMacro * getModuleMacro(Module *Mod, const IdentifierInfo *II)
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleComment(Token &result, SourceRange Comment)
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Buffer)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
HeaderSearch & getHeaderSearchInfo() const
void emitMacroExpansionWarnings(const Token &Identifier, bool IsIfnDef=false) const
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
ExternalPreprocessorSource * getExternalSource() const
Module * LeaveSubmodule(bool ForPragma)
SourceRange DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found.
const std::string & getPredefines() const
Get the predefines for this processor.
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind, void *AnnotationVal)
Enter an annotation token into the token stream.
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, const LangOptions &LangOpts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
MacroInfo * getMacroInfo(const IdentifierInfo *II)
void setPredefines(std::string P)
Set the predefines for this Preprocessor.
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
IdentifierTable & getIdentifierTable()
Builtin::Context & getBuiltinInfo()
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
LangOptions::FPEvalMethodKind getTUFPEvalMethod() const
const LangOptions & getLangOpts() const
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void SetSuppressIncludeNotFoundError(bool Suppress)
static void processPathForFileMacro(SmallVectorImpl< char > &Path, const LangOptions &LangOpts, const TargetInfo &TI)
bool isInNamedModule() const
If we are preprocessing a named module.
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
void PoisonSEHIdentifiers(bool Poison=true)
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion.
size_t getTotalMemory() const
void setExternalSource(ExternalPreprocessorSource *Source)
void clearCodeCompletionHandler()
Clear out the code completion handler.
void AddPragmaHandler(PragmaHandler *Handler)
OptionalFileEntryRef getHeaderToIncludeForDiagnostics(SourceLocation IncLoc, SourceLocation MLoc)
We want to produce a diagnostic at location IncLoc concerning an unreachable effect at location MLoc ...
void setCounterValue(unsigned V)
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.
void LexTokensUntilEOF(std::vector< Token > *Tokens=nullptr)
Lex all tokens for this preprocessor until (and excluding) end of file.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
void setPragmasEnabled(bool Enabled)
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void HandlePragmaMark(Token &MarkTok)
bool getPragmasEnabled() const
void HandlePragmaHdrstop(Token &Tok)
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
void setEmptylineHandler(EmptylineHandler *Handler)
Set empty line handler.
DiagnosticsEngine & getDiagnostics() const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
llvm::DenseSet< const FileEntry * > IncludedFilesSet
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SelectorTable & getSelectorTable()
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
const llvm::SmallSetVector< Module *, 2 > & getAffectingClangModules() const
Get the set of top-level clang modules that affected preprocessing, but were not imported.
const IncludedFilesSet & getIncludedFiles() const
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void HandlePragmaIncludeAlias(Token &Tok)
Module * getModuleForLocation(SourceLocation Loc, bool AllowTextual)
Find the module that owns the source or header file that Loc points to.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
SourceLocation getPreambleRecordedPragmaAssumeNonNullLoc() const
Get the location of the recorded unterminated #pragma clang assume_nonnull begin in the preamble,...
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
void EnableBacktrackAtThisPos()
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop.
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
Represents an unpacked "presumed" location which can be presented to the user.
This table allows us to fully hide how we implement multi-keyword caching.
Encodes a location in the source.
This class handles loading and caching of source files into memory.
A trivial tuple used to represent a source range.
Exposes information about the current target.
Stores token information for comparing actual tokens with predefined values.
TokenValue(IdentifierInfo *II)
TokenValue(tok::TokenKind Kind)
bool operator==(const Token &Tok) const
Token - This structure provides full information about a lexed token.
IdentifierInfo * getIdentifierInfo() const
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
unsigned getLength() const
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
tok::TokenKind getKind() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a variadic mac...
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
Directive - Abstract class representing a parsed verify directive.
constexpr XRayInstrMask None
StringRef getName(const HeaderType T)
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a '#' at the beginning of the line.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
The JSON file list parser is used to communicate input to InstallAPI.
CustomizableOptional< DirectoryEntryRef > OptionalDirectoryEntryRef
detail::SearchDirIteratorImpl< true > ConstSearchDirIterator
MacroUse
Context in which macro name is used.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
@ Module
Module linkage, which indicates that the entity can be referred to from other translation units withi...
@ Result
The result type of a method or function.
CustomizableOptional< FileEntryRef > OptionalFileEntryRef
TranslationUnitKind
Describes the kind of translation unit being processed.
@ TU_Complete
The translation unit is a complete translation unit.
YAML serialization mapping.
Describes how and where the pragma was introduced.
SourceLocation IfTokenLoc
SourceLocation HashTokenLoc
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)