54#include "llvm/ADT/APInt.h"
55#include "llvm/ADT/ArrayRef.h"
56#include "llvm/ADT/DenseMap.h"
57#include "llvm/ADT/STLExtras.h"
58#include "llvm/ADT/SmallString.h"
59#include "llvm/ADT/SmallVector.h"
60#include "llvm/ADT/StringRef.h"
61#include "llvm/Support/Capacity.h"
62#include "llvm/Support/ErrorHandling.h"
63#include "llvm/Support/MemoryBuffer.h"
64#include "llvm/Support/raw_ostream.h"
85 : PPOpts(
std::move(PPOpts)), Diags(&diags), LangOpts(opts),
86 FileMgr(Headers.getFileMgr()), SourceMgr(
SM),
87 ScratchBuf(new
ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
92 Identifiers(IILookup), PragmaHandlers(new
PragmaNamespace(StringRef())),
93 TUKind(TUKind), SkipMainFilePreamble(0,
true),
94 CurSubmoduleState(&NullSubmoduleState) {
95 OwnsHeaderSearch = OwnsHeaders;
99 KeepMacroComments =
false;
100 SuppressIncludeNotFoundError =
false;
103 DisableMacroExpansion =
false;
104 MacroExpansionInDirectivesOverride =
false;
107 InMacroArgPreExpansion =
false;
108 NumCachedTokenLexers = 0;
109 PragmasEnabled =
true;
110 ParsingIfOrElifDirective =
false;
111 PreprocessedOutput =
false;
114 ReadMacrosFromExternalSource =
false;
116 BuiltinInfo = std::make_unique<Builtin::Context>();
126 RegisterBuiltinPragmas();
129 RegisterBuiltinMacros();
131 if(LangOpts.Borland) {
142 Ident__exception_info = Ident__exception_code =
nullptr;
143 Ident__abnormal_termination = Ident___exception_info =
nullptr;
144 Ident___exception_code = Ident___abnormal_termination =
nullptr;
145 Ident_GetExceptionInfo = Ident_GetExceptionCode =
nullptr;
146 Ident_AbnormalTermination =
nullptr;
151 IncrementalProcessing = LangOpts.IncrementalExtensions;
155 SkippingUntilPragmaHdrStop =
true;
158 if (!this->PPOpts->PCHThroughHeader.empty() &&
159 !this->PPOpts->ImplicitPCHInclude.empty())
160 SkippingUntilPCHThroughHeader =
true;
162 if (this->PPOpts->GeneratePreamble)
163 PreambleConditionalStack.startRecording();
165 MaxTokens = LangOpts.MaxTokens;
169 assert(BacktrackPositions.empty() &&
"EnableBacktrack/Backtrack imbalance!");
171 IncludeMacroStack.clear();
176 std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers,
nullptr);
177 CurTokenLexer.reset();
180 for (
MacroArgs *ArgList = MacroArgCache; ArgList;)
181 ArgList = ArgList->deallocate();
184 if (OwnsHeaderSearch)
190 assert((!this->Target || this->Target == &
Target) &&
191 "Invalid override of target information");
194 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
195 "Invalid override of aux target information.");
196 this->AuxTarget = AuxTarget;
199 BuiltinInfo->InitializeTarget(
Target, AuxTarget);
217 NumEnteredSourceFiles = 0;
220 PragmaHandlersBackup = std::move(PragmaHandlers);
221 PragmaHandlers = std::make_unique<PragmaNamespace>(StringRef());
222 RegisterBuiltinPragmas();
225 PredefinesFileID =
FileID();
229 NumEnteredSourceFiles = 1;
231 PragmaHandlers = std::move(PragmaHandlersBackup);
240 if (!DumpFlags)
return;
242 llvm::errs() <<
"\t";
244 llvm::errs() <<
" [StartOfLine]";
246 llvm::errs() <<
" [LeadingSpace]";
248 llvm::errs() <<
" [ExpandDisabled]";
251 llvm::errs() <<
" [UnClean='" << StringRef(Start, Tok.
getLength())
255 llvm::errs() <<
"\tLoc=<";
261 Loc.
print(llvm::errs(), SourceMgr);
265 llvm::errs() <<
"MACRO: ";
266 for (
unsigned i = 0, e = MI.
getNumTokens(); i != e; ++i) {
270 llvm::errs() <<
"\n";
274 llvm::errs() <<
"\n*** Preprocessor Stats:\n";
275 llvm::errs() << NumDirectives <<
" directives found:\n";
276 llvm::errs() <<
" " << NumDefined <<
" #define.\n";
277 llvm::errs() <<
" " << NumUndefined <<
" #undef.\n";
278 llvm::errs() <<
" #include/#include_next/#import:\n";
279 llvm::errs() <<
" " << NumEnteredSourceFiles <<
" source files entered.\n";
280 llvm::errs() <<
" " << MaxIncludeStackDepth <<
" max include stack depth\n";
281 llvm::errs() <<
" " << NumIf <<
" #if/#ifndef/#ifdef.\n";
282 llvm::errs() <<
" " << NumElse <<
" #else/#elif/#elifdef/#elifndef.\n";
283 llvm::errs() <<
" " << NumEndif <<
" #endif.\n";
284 llvm::errs() <<
" " << NumPragma <<
" #pragma.\n";
285 llvm::errs() << NumSkipped <<
" #if/#ifndef#ifdef regions skipped\n";
287 llvm::errs() << NumMacroExpanded <<
"/" << NumFnMacroExpanded <<
"/"
288 << NumBuiltinMacroExpanded <<
" obj/fn/builtin macros expanded, "
289 << NumFastMacroExpanded <<
" on the fast path.\n";
290 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
291 <<
" token paste (##) operations performed, "
292 << NumFastTokenPaste <<
" on the fast path.\n";
294 llvm::errs() <<
"\nPreprocessor Memory: " <<
getTotalMemory() <<
"B total";
296 llvm::errs() <<
"\n BumpPtr: " << BP.getTotalMemory();
297 llvm::errs() <<
"\n Macro Expanded Tokens: "
298 << llvm::capacity_in_bytes(MacroExpandedTokens);
299 llvm::errs() <<
"\n Predefines Buffer: " << Predefines.capacity();
301 llvm::errs() <<
"\n Macros: "
302 << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
303 llvm::errs() <<
"\n #pragma push_macro Info: "
304 << llvm::capacity_in_bytes(PragmaPushMacroInfo);
305 llvm::errs() <<
"\n Poison Reasons: "
306 << llvm::capacity_in_bytes(PoisonReasons);
307 llvm::errs() <<
"\n Comment Handlers: "
308 << llvm::capacity_in_bytes(CommentHandlers) <<
"\n";
314 !ReadMacrosFromExternalSource) {
315 ReadMacrosFromExternalSource =
true;
321 CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
323 return CurSubmoduleState->Macros.begin();
327 return BP.getTotalMemory()
328 + llvm::capacity_in_bytes(MacroExpandedTokens)
329 + Predefines.capacity()
332 + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
333 + llvm::capacity_in_bytes(PragmaPushMacroInfo)
334 + llvm::capacity_in_bytes(PoisonReasons)
335 + llvm::capacity_in_bytes(CommentHandlers);
341 !ReadMacrosFromExternalSource) {
342 ReadMacrosFromExternalSource =
true;
346 return CurSubmoduleState->Macros.end();
353 std::equal(Tokens.begin(), Tokens.end(), MI->
tokens_begin());
360 StringRef BestSpelling;
364 Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
376 BestLocation = Location;
377 BestSpelling = I->first->getName();
385 CurLexerCallback = CurLexer->isDependencyDirectivesLexer()
386 ? CLK_DependencyDirectivesLexer
388 else if (CurTokenLexer)
389 CurLexerCallback = CLK_TokenLexer;
391 CurLexerCallback = CLK_CachingLexer;
395 unsigned CompleteLine,
396 unsigned CompleteColumn) {
397 assert(CompleteLine && CompleteColumn &&
"Starts from 1:1");
398 assert(!CodeCompletionFile &&
"Already set");
401 std::optional<llvm::MemoryBufferRef> Buffer =
407 const char *Position = Buffer->getBufferStart();
409 for (; *Position; ++Position) {
410 if (*Position !=
'\r' && *Position !=
'\n')
414 if ((Position[1] ==
'\r' || Position[1] ==
'\n') &&
415 Position[0] != Position[1])
422 Position += CompleteColumn - 1;
426 if (SkipMainFilePreamble.first &&
428 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
429 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
432 if (Position > Buffer->getBufferEnd())
433 Position = Buffer->getBufferEnd();
435 CodeCompletionFile =
File;
436 CodeCompletionOffset = Position - Buffer->getBufferStart();
438 auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
439 Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
440 char *NewBuf = NewBuffer->getBufferStart();
441 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
443 std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
469 if (Tok.
isNot(tok::raw_identifier) && !Tok.
hasUCN()) {
472 return II->getName();
479 const char *Ptr = Buffer.data();
481 return StringRef(Ptr, Len);
493 SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
495 if (ExpansionLocStart.
isValid())
497 ExpansionLocEnd, Str.size());
501 if (Tok.
is(tok::raw_identifier))
510 std::pair<FileID, unsigned> LocInfo =
SM.getDecomposedLoc(SpellingLoc);
512 StringRef Buffer =
SM.getBufferData(LocInfo.first, &
Invalid);
519 ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
531 if (!
getLangOpts().isCompilingModuleImplementation())
547 assert(NumEnteredSourceFiles == 0 &&
"Cannot reenter the main file!");
558 if (SkipMainFilePreamble.first > 0)
559 CurLexer->SetByteOffset(SkipMainFilePreamble.first,
560 SkipMainFilePreamble.second);
569 std::unique_ptr<llvm::MemoryBuffer> SB =
570 llvm::MemoryBuffer::getMemBufferCopy(Predefines,
"<built-in>");
571 assert(SB &&
"Cannot create predefined source buffer");
573 assert(FID.
isValid() &&
"Could not create FileID for predefines?");
574 setPredefinesFileID(FID);
579 if (!PPOpts->PCHThroughHeader.empty()) {
584 false,
nullptr,
nullptr,
585 nullptr,
nullptr,
nullptr,
590 << PPOpts->PCHThroughHeader;
593 setPCHThroughHeaderFileID(
603void Preprocessor::setPCHThroughHeaderFileID(
FileID FID) {
604 assert(PCHThroughHeaderFileID.
isInvalid() &&
605 "PCHThroughHeaderFileID already set!");
606 PCHThroughHeaderFileID = FID;
610 assert(PCHThroughHeaderFileID.
isValid() &&
611 "Invalid PCH through header FileID");
617 PCHThroughHeaderFileID.
isValid();
622 PCHThroughHeaderFileID.
isValid();
639 bool ReachedMainFileEOF =
false;
640 bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
641 bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
646 CurLexerCallback(*
this, Tok);
647 if (Tok.
is(tok::eof) && !InPredefines) {
648 ReachedMainFileEOF =
true;
651 if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
653 if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
656 if (ReachedMainFileEOF) {
657 if (UsingPCHThroughHeader)
659 << PPOpts->PCHThroughHeader << 1;
660 else if (!PPOpts->PCHWithHdrStopCreate)
665void Preprocessor::replayPreambleConditionalStack() {
667 if (PreambleConditionalStack.isReplaying()) {
669 "CurPPLexer is null when calling replayPreambleConditionalStack.");
671 PreambleConditionalStack.doneReplaying();
672 if (PreambleConditionalStack.reachedEOFWhileSkipping())
673 SkipExcludedConditionalBlock(
674 PreambleConditionalStack.SkipInfo->HashTokenLoc,
675 PreambleConditionalStack.SkipInfo->IfTokenLoc,
676 PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
677 PreambleConditionalStack.SkipInfo->FoundElse,
678 PreambleConditionalStack.SkipInfo->ElseLoc);
685 Callbacks->EndOfMainFile();
696 assert(!
Identifier.getRawIdentifier().empty() &&
"No raw identifier data!");
731 PoisonReasons[II] = DiagID;
735 assert(Ident__exception_code && Ident__exception_info);
736 assert(Ident___exception_code && Ident___exception_info);
750 "Can't handle identifiers without identifier info!");
751 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
752 PoisonReasons.find(
Identifier.getIdentifierInfo());
753 if(it == PoisonReasons.end())
759void Preprocessor::updateOutOfDateIdentifier(
IdentifierInfo &II)
const {
774 "Can't handle identifiers without identifier info!");
784 bool CurrentIsPoisoned =
false;
785 const bool IsSpecialVariadicMacro =
786 &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
787 if (IsSpecialVariadicMacro)
790 updateOutOfDateIdentifier(II);
793 if (IsSpecialVariadicMacro)
805 const auto *MI = MD.getMacroInfo();
806 assert(MI &&
"macro definition with no macro info?");
807 if (!DisableMacroExpansion) {
808 if (!
Identifier.isExpandDisabled() && MI->isEnabled()) {
811 if (!MI->isFunctionLike() || isNextPPTokenLParen())
812 return HandleMacroExpandedIdentifier(
Identifier, MD);
818 if (MI->isObjectLike() || isNextPPTokenLParen())
853 !InMacroArgs && !DisableMacroExpansion &&
855 CurLexerCallback != CLK_CachingLexer) {
857 NamedModuleImportPath.clear();
859 ModuleImportExpectsIdentifier =
true;
860 CurLexerCallback = CLK_LexAfterModuleImport;
869 while (!CurLexerCallback(*
this,
Result))
875 if (
Result.is(tok::code_completion) &&
Result.getIdentifierInfo()) {
881 Result.setIdentifierInfo(
nullptr);
889 if (
getLangOpts().CPlusPlusModules && LexLevel == 1 &&
891 switch (
Result.getKind()) {
892 case tok::l_paren:
case tok::l_square:
case tok::l_brace:
893 StdCXXImportSeqState.handleOpenBracket();
895 case tok::r_paren:
case tok::r_square:
896 StdCXXImportSeqState.handleCloseBracket();
899 StdCXXImportSeqState.handleCloseBrace();
903 case tok::annot_module_include:
905 TrackGMFState.handleSemi();
906 StdCXXImportSeqState.handleSemi();
907 ModuleDeclState.handleSemi();
909 case tok::header_name:
910 case tok::annot_header_unit:
911 StdCXXImportSeqState.handleHeaderName();
914 TrackGMFState.handleExport();
915 StdCXXImportSeqState.handleExport();
916 ModuleDeclState.handleExport();
919 ModuleDeclState.handleColon();
922 ModuleDeclState.handlePeriod();
924 case tok::identifier:
927 if (StdCXXImportSeqState.atTopLevel()) {
928 if (
Result.getIdentifierInfo()->isModulesImport()) {
929 TrackGMFState.handleImport(StdCXXImportSeqState.afterTopLevelSeq());
930 StdCXXImportSeqState.handleImport();
931 if (StdCXXImportSeqState.afterImportSeq()) {
932 ModuleImportLoc =
Result.getLocation();
933 NamedModuleImportPath.clear();
935 ModuleImportExpectsIdentifier =
true;
936 CurLexerCallback = CLK_LexAfterModuleImport;
940 TrackGMFState.handleModule(StdCXXImportSeqState.afterTopLevelSeq());
941 ModuleDeclState.handleModule();
945 ModuleDeclState.handleIdentifier(
Result.getIdentifierInfo());
946 if (ModuleDeclState.isModuleCandidate())
950 TrackGMFState.handleMisc();
951 StdCXXImportSeqState.handleMisc();
952 ModuleDeclState.handleMisc();
957 LastTokenWasAt =
Result.is(tok::at);
960 if ((LexLevel == 0 || PreprocessToken) &&
973 if (Tok.
isOneOf(tok::unknown, tok::eof, tok::eod,
974 tok::annot_repl_input_end))
976 if (Tokens !=
nullptr)
977 Tokens->push_back(Tok);
1003 if (FilenameTok.
is(tok::less) && AllowMacroExpansion) {
1010 FilenameBuffer.push_back(
'<');
1016 while (FilenameTok.
isNot(tok::greater)) {
1018 if (FilenameTok.
isOneOf(tok::eod, tok::eof)) {
1020 Diag(Start, diag::note_matching) << tok::less;
1027 if (FilenameTok.
is(tok::code_completion)) {
1036 FilenameBuffer.push_back(
' ');
1040 size_t PreAppendSize = FilenameBuffer.size();
1041 FilenameBuffer.resize(PreAppendSize + FilenameTok.
getLength());
1043 const char *BufPtr = &FilenameBuffer[PreAppendSize];
1044 unsigned ActualLen =
getSpelling(FilenameTok, BufPtr);
1047 if (BufPtr != &FilenameBuffer[PreAppendSize])
1048 memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
1051 if (FilenameTok.
getLength() != ActualLen)
1052 FilenameBuffer.resize(PreAppendSize + ActualLen);
1056 FilenameTok.
setKind(tok::header_name);
1061 }
else if (FilenameTok.
is(tok::string_literal) && AllowMacroExpansion) {
1072 StringRef Str =
getSpelling(FilenameTok, FilenameBuffer);
1073 if (Str.size() >= 2 && Str.front() ==
'"' && Str.back() ==
'"')
1074 FilenameTok.
setKind(tok::header_name);
1085 unsigned BracketDepth = 0;
1087 Toks.emplace_back();
1090 switch (Toks.back().getKind()) {
1091 case tok::l_paren:
case tok::l_square:
case tok::l_brace:
1095 case tok::r_paren:
case tok::r_square:
case tok::r_brace:
1096 if (BracketDepth == 0)
1102 if (BracketDepth == 0)
1145 if (NamedModuleImportPath.empty() &&
getLangOpts().CPlusPlusModules) {
1149 if (
Result.is(tok::colon) && ModuleDeclState.isNamedModule()) {
1150 std::string Name = ModuleDeclState.getPrimaryName().str();
1152 NamedModuleImportPath.push_back(
1154 CurLexerCallback = CLK_LexAfterModuleImport;
1164 auto ToksCopy = std::make_unique<Token[]>(Toks.size());
1165 std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
1166 EnterTokenStream(std::move(ToksCopy), Toks.size(),
1170 bool ImportingHeader =
Result.is(tok::header_name);
1173 if (ImportingHeader) {
1177 Suffix.push_back(
Result);
1182 if (Suffix.back().isNot(tok::semi)) {
1184 EnterTokens(Suffix);
1193 Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1198 ImportTok.
setKind(tok::kw_import);
1203 auto Action = HandleHeaderIncludeOrImport(
1205 switch (Action.Kind) {
1206 case ImportAction::None:
1209 case ImportAction::ModuleBegin:
1211 Suffix.emplace_back();
1212 Suffix.back().startToken();
1213 Suffix.back().setKind(tok::annot_module_begin);
1214 Suffix.back().setLocation(SemiLoc);
1215 Suffix.back().setAnnotationEndLoc(SemiLoc);
1216 Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1219 case ImportAction::ModuleImport:
1220 case ImportAction::HeaderUnitImport:
1221 case ImportAction::SkippedModuleImport:
1224 Suffix[0].setKind(tok::annot_header_unit);
1225 Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1226 Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1229 case ImportAction::Failure:
1231 "This should be an early exit only to a fatal error");
1232 Result.setKind(tok::eof);
1233 CurLexer->cutOffLexing();
1234 EnterTokens(Suffix);
1238 EnterTokens(Suffix);
1248 if (ModuleImportExpectsIdentifier &&
Result.getKind() == tok::identifier) {
1251 NamedModuleImportPath.push_back(
1252 std::make_pair(
Result.getIdentifierInfo(),
Result.getLocation()));
1253 ModuleImportExpectsIdentifier =
false;
1254 CurLexerCallback = CLK_LexAfterModuleImport;
1261 if (!ModuleImportExpectsIdentifier &&
Result.getKind() == tok::period) {
1262 ModuleImportExpectsIdentifier =
true;
1263 CurLexerCallback = CLK_LexAfterModuleImport;
1268 if (NamedModuleImportPath.empty() ||
Result.is(tok::eof))
1274 if (
Result.isNot(tok::semi)) {
1275 Suffix.push_back(
Result);
1277 if (Suffix.back().isNot(tok::semi)) {
1279 EnterTokens(Suffix);
1282 SemiLoc = Suffix.back().getLocation();
1289 std::string FlatModuleName;
1291 for (
auto &Piece : NamedModuleImportPath) {
1293 if (!FlatModuleName.empty() && FlatModuleName.back() !=
':')
1294 FlatModuleName +=
".";
1295 FlatModuleName += Piece.first->getName();
1298 NamedModuleImportPath.clear();
1299 NamedModuleImportPath.push_back(
1303 Module *Imported =
nullptr;
1306 Imported = TheModuleLoader.
loadModule(ModuleImportLoc,
1307 NamedModuleImportPath,
1315 Callbacks->moduleImport(ModuleImportLoc, NamedModuleImportPath, Imported);
1317 if (!Suffix.empty()) {
1318 EnterTokens(Suffix);
1325 CurSubmoduleState->VisibleModules.setVisible(
1330 Diag(ModuleImportLoc, diag::warn_module_conflict)
1331 << Path[0]->getFullModuleName()
1332 << Conflict->getFullModuleName()
1337 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1338 BuildingSubmoduleStack.back().M->Imports.insert(M);
1342 const char *DiagnosticTag,
1343 bool AllowMacroExpansion) {
1345 if (
Result.isNot(tok::string_literal)) {
1346 Diag(
Result, diag::err_expected_string_literal)
1347 << 0 << DiagnosticTag;
1354 StrToks.push_back(
Result);
1356 if (
Result.hasUDSuffix())
1359 if (AllowMacroExpansion)
1363 }
while (
Result.is(tok::string_literal));
1367 assert(Literal.isOrdinary() &&
"Didn't allow wide strings in");
1369 if (Literal.hadError)
1372 if (Literal.Pascal) {
1373 Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1374 << 0 << DiagnosticTag;
1378 String = std::string(Literal.GetString());
1383 assert(Tok.
is(tok::numeric_constant));
1385 bool NumberInvalid =
false;
1386 StringRef Spelling =
getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1392 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1394 llvm::APInt APVal(64, 0);
1395 if (Literal.GetIntegerValue(APVal))
1398 Value = APVal.getLimitedValue();
1403 assert(Handler &&
"NULL comment handler");
1404 assert(!llvm::is_contained(CommentHandlers, Handler) &&
1405 "Comment handler already registered");
1406 CommentHandlers.push_back(Handler);
1410 std::vector<CommentHandler *>::iterator Pos =
1411 llvm::find(CommentHandlers, Handler);
1412 assert(Pos != CommentHandlers.end() &&
"Comment handler not registered");
1413 CommentHandlers.erase(Pos);
1417 bool AnyPendingTokens =
false;
1418 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1419 HEnd = CommentHandlers.end();
1421 if ((*H)->HandleComment(*
this, Comment))
1422 AnyPendingTokens =
true;
1430void Preprocessor::emitMacroDeprecationWarning(
const Token &
Identifier)
const {
1431 const MacroAnnotations &A =
1433 assert(A.DeprecationInfo &&
1434 "Macro deprecation warning without recorded annotation!");
1435 const MacroAnnotationInfo &Info = *A.DeprecationInfo;
1436 if (Info.Message.empty())
1441 <<
Identifier.getIdentifierInfo() << 1 << Info.Message;
1442 Diag(Info.Location, diag::note_pp_macro_annotation) << 0;
1445void Preprocessor::emitRestrictExpansionWarning(
const Token &
Identifier)
const {
1446 const MacroAnnotations &A =
1448 assert(A.RestrictExpansionInfo &&
1449 "Macro restricted expansion warning without recorded annotation!");
1450 const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo;
1451 if (Info.Message.empty())
1456 <<
Identifier.getIdentifierInfo() << 1 << Info.Message;
1457 Diag(Info.Location, diag::note_pp_macro_annotation) << 1;
1461 bool IsUndef)
const {
1462 const MacroAnnotations &A =
1464 assert(A.FinalAnnotationLoc &&
1465 "Final macro warning without recorded annotation!");
1468 <<
Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1);
1469 Diag(*A.FinalAnnotationLoc, diag::note_pp_macro_annotation) << 2;
1475 auto FirstRegionEndingAfterLoc = llvm::partition_point(
1476 SafeBufferOptOutMap,
1478 &Loc](
const std::pair<SourceLocation, SourceLocation> &Region) {
1482 if (FirstRegionEndingAfterLoc != SafeBufferOptOutMap.end()) {
1489 if (!SafeBufferOptOutMap.empty() &&
1490 SafeBufferOptOutMap.back().first == SafeBufferOptOutMap.back().second)
1501 InSafeBufferOptOutRegion =
true;
1502 CurrentSafeBufferOptOutStart = Loc;
1506 if (!SafeBufferOptOutMap.empty()) {
1507 [[maybe_unused]]
auto *PrevRegion = &SafeBufferOptOutMap.back();
1508 assert(PrevRegion->first != PrevRegion->second &&
1509 "Shall not begin a safe buffer opt-out region before closing the "
1515 SafeBufferOptOutMap.emplace_back(Loc, Loc);
1519 InSafeBufferOptOutRegion =
false;
1523 assert(!SafeBufferOptOutMap.empty() &&
1524 "Misordered safe buffer opt-out regions");
1525 auto *CurrRegion = &SafeBufferOptOutMap.back();
1526 assert(CurrRegion->first == CurrRegion->second &&
1527 "Set end location to a closed safe buffer opt-out region");
1528 CurrRegion->second = Loc;
1534 return InSafeBufferOptOutRegion;
1537 StartLoc = CurrentSafeBufferOptOutStart;
1538 return InSafeBufferOptOutRegion;
Defines enum values for all the target-independent builtin functions.
Defines the clang::FileManager interface and associated types.
Defines the FileSystemStatCache interface.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
Defines the clang::MacroInfo and clang::MacroDirective classes.
Defines the clang::Module class, which describes a module in the source code.
Defines the PreprocessorLexer interface.
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
Defines the clang::Preprocessor interface.
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an #include directive.
virtual ~CodeCompletionHandler()
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
Concrete class used by the front-end to report problems and issues.
virtual ~EmptylineHandler()
virtual ~ExternalPreprocessorSource()
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
Cached information about one file (either on disk or in the virtual file system).
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Provides lookups to, and iteration over, IdentiferInfo objects.
One of these records is kept for each identifier that is lexed.
bool isModulesImport() const
Determine whether this is the contextual keyword import.
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
bool isPoisoned() const
Return true if this token has been poisoned.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
void setIsFutureCompatKeyword(bool Val)
StringRef getName() const
Return the actual identifier string.
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
bool isExtensionToken() const
get/setExtension - Initialize information about whether or not this language token is an extension.
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
A description of the current definition of a macro.
MacroInfo * getMacroInfo()
SourceLocation getLocation() const
Encapsulates the data about a macro definition (e.g.
const_tokens_iterator tokens_begin() const
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
const Token & getReplacementToken(unsigned Tok) const
bool isObjectLike() const
Abstract interface for a module loader.
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
Represents a macro directive exported by a module.
Describes a module or submodule.
@ Hidden
All of the names in this module are hidden.
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber,...
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
void LexIncludeFilename(Token &FilenameTok)
Lex a token, producing a header-name token if possible.
bool markIncluded(FileEntryRef File)
Mark the file as included.
void FinalizeForModelFile()
Cleanup after model file parsing.
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
bool isSafeBufferOptOut(const SourceManager &SourceMgr, const SourceLocation &Loc) const
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
void DumpMacro(const MacroInfo &MI) const
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool SetCodeCompletionPoint(FileEntryRef File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void makeModuleVisible(Module *M, SourceLocation Loc)
bool isInImportingCXXNamedModules() const
If we're importing a standard C++20 Named Modules.
void Lex(Token &Result)
Lex the next token for this preprocessor.
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
bool enterOrExitSafeBufferOptOutRegion(bool isEnter, const SourceLocation &Loc)
Alter the state of whether this PP currently is in a "-Wunsafe-buffer-usage" opt-out region.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
macro_iterator macro_end(bool IncludeExternalMacros=true) const
SourceManager & getSourceManager() const
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
bool getCommentRetentionState() const
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
MacroMap::const_iterator macro_iterator
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
bool isPPInSafeBufferOptOutRegion()
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
const TargetInfo & getTargetInfo() const
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void DumpLocation(SourceLocation Loc) const
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleComment(Token &result, SourceRange Comment)
HeaderSearch & getHeaderSearchInfo() const
ExternalPreprocessorSource * getExternalSource() const
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, const LangOptions &LangOpts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
IdentifierTable & getIdentifierTable()
const LangOptions & getLangOpts() const
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void PoisonSEHIdentifiers(bool Poison=true)
size_t getTotalMemory() const
void LexTokensUntilEOF(std::vector< Token > *Tokens=nullptr)
Lex all tokens for this preprocessor until (and excluding) end of file.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
DiagnosticsEngine & getDiagnostics() const
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop.
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens.
Encodes a location in the source.
bool isValid() const
Return true if this is a valid SourceLocation object.
void print(raw_ostream &OS, const SourceManager &SM) const
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
This class handles loading and caching of source files into memory.
OptionalFileEntryRef getFileEntryRefForID(FileID FID) const
Returns the FileEntryRef for the provided FileID.
FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
FileID getMainFileID() const
Returns the FileID of the main source file.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer.
void overrideFileContents(FileEntryRef SourceFile, const llvm::MemoryBufferRef &Buffer)
Override the contents of the given source file by providing an already-allocated buffer.
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
SourceLocation createExpansionLoc(SourceLocation SpellingLoc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned Length, bool ExpansionIsTokenRange=true, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Creates an expansion SLocEntry for a macro use.
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
std::optional< llvm::MemoryBufferRef > getMemoryBufferForFileOrNone(FileEntryRef File)
Retrieve the memory buffer associated with the given file.
A trivial tuple used to represent a source range.
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
Exposes information about the current target.
Token - This structure provides full information about a lexed token.
IdentifierInfo * getIdentifierInfo() const
void setLiteralData(const char *Ptr)
bool hasUCN() const
Returns true if this token contains a universal character name.
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
unsigned getLength() const
void setLength(unsigned Len)
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6....
void setKind(tok::TokenKind K)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
tok::TokenKind getKind() const
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
void setLocation(SourceLocation L)
bool hasLeadingEmptyMacro() const
Return true if this token has an empty macro before it.
void setRawIdentifierData(const char *Ptr)
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
bool isNot(tok::TokenKind K) const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
void startToken()
Reset all flags to cleared.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
void setIdentifierInfo(IdentifierInfo *II)
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Defines the clang::TargetInfo interface.
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
@ Result
The result type of a method or function.
TranslationUnitKind
Describes the kind of translation unit being processed.
@ TU_Prefix
The translation unit is a prefix to a translation unit, and is not complete.