54#include "llvm/ADT/APInt.h"
55#include "llvm/ADT/ArrayRef.h"
56#include "llvm/ADT/DenseMap.h"
57#include "llvm/ADT/STLExtras.h"
58#include "llvm/ADT/SmallString.h"
59#include "llvm/ADT/SmallVector.h"
60#include "llvm/ADT/StringRef.h"
61#include "llvm/Support/Capacity.h"
62#include "llvm/Support/ErrorHandling.h"
63#include "llvm/Support/MemoryBuffer.h"
64#include "llvm/Support/raw_ostream.h"
88 : PPOpts(
std::move(PPOpts)), Diags(&diags), LangOpts(opts),
89 FileMgr(Headers.getFileMgr()), SourceMgr(
SM),
90 ScratchBuf(new
ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
95 Identifiers(IILookup), PragmaHandlers(new
PragmaNamespace(StringRef())),
96 TUKind(TUKind), SkipMainFilePreamble(0,
true),
97 CurSubmoduleState(&NullSubmoduleState) {
98 OwnsHeaderSearch = OwnsHeaders;
101 KeepComments =
false;
102 KeepMacroComments =
false;
103 SuppressIncludeNotFoundError =
false;
106 DisableMacroExpansion =
false;
107 MacroExpansionInDirectivesOverride =
false;
110 InMacroArgPreExpansion =
false;
111 NumCachedTokenLexers = 0;
112 PragmasEnabled =
true;
113 ParsingIfOrElifDirective =
false;
114 PreprocessedOutput =
false;
117 ReadMacrosFromExternalSource =
false;
119 BuiltinInfo = std::make_unique<Builtin::Context>();
129 RegisterBuiltinPragmas();
132 RegisterBuiltinMacros();
134 if(LangOpts.Borland) {
145 Ident__exception_info = Ident__exception_code =
nullptr;
146 Ident__abnormal_termination = Ident___exception_info =
nullptr;
147 Ident___exception_code = Ident___abnormal_termination =
nullptr;
148 Ident_GetExceptionInfo = Ident_GetExceptionCode =
nullptr;
149 Ident_AbnormalTermination =
nullptr;
154 IncrementalProcessing = LangOpts.IncrementalExtensions;
158 SkippingUntilPragmaHdrStop =
true;
161 if (!this->PPOpts->PCHThroughHeader.empty() &&
162 !this->PPOpts->ImplicitPCHInclude.empty())
163 SkippingUntilPCHThroughHeader =
true;
165 if (this->PPOpts->GeneratePreamble)
166 PreambleConditionalStack.startRecording();
168 MaxTokens = LangOpts.MaxTokens;
172 assert(BacktrackPositions.empty() &&
"EnableBacktrack/Backtrack imbalance!");
174 IncludeMacroStack.clear();
179 std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers,
nullptr);
180 CurTokenLexer.reset();
183 for (
MacroArgs *ArgList = MacroArgCache; ArgList;)
184 ArgList = ArgList->deallocate();
187 if (OwnsHeaderSearch)
193 assert((!this->Target || this->Target == &
Target) &&
194 "Invalid override of target information");
197 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
198 "Invalid override of aux target information.");
199 this->AuxTarget = AuxTarget;
202 BuiltinInfo->InitializeTarget(
Target, AuxTarget);
220 NumEnteredSourceFiles = 0;
223 PragmaHandlersBackup = std::move(PragmaHandlers);
224 PragmaHandlers = std::make_unique<PragmaNamespace>(StringRef());
225 RegisterBuiltinPragmas();
228 PredefinesFileID =
FileID();
232 NumEnteredSourceFiles = 1;
234 PragmaHandlers = std::move(PragmaHandlersBackup);
243 if (!DumpFlags)
return;
245 llvm::errs() <<
"\t";
247 llvm::errs() <<
" [StartOfLine]";
249 llvm::errs() <<
" [LeadingSpace]";
251 llvm::errs() <<
" [ExpandDisabled]";
254 llvm::errs() <<
" [UnClean='" << StringRef(Start, Tok.
getLength())
258 llvm::errs() <<
"\tLoc=<";
264 Loc.
print(llvm::errs(), SourceMgr);
268 llvm::errs() <<
"MACRO: ";
269 for (
unsigned i = 0, e = MI.
getNumTokens(); i != e; ++i) {
273 llvm::errs() <<
"\n";
277 llvm::errs() <<
"\n*** Preprocessor Stats:\n";
278 llvm::errs() << NumDirectives <<
" directives found:\n";
279 llvm::errs() <<
" " << NumDefined <<
" #define.\n";
280 llvm::errs() <<
" " << NumUndefined <<
" #undef.\n";
281 llvm::errs() <<
" #include/#include_next/#import:\n";
282 llvm::errs() <<
" " << NumEnteredSourceFiles <<
" source files entered.\n";
283 llvm::errs() <<
" " << MaxIncludeStackDepth <<
" max include stack depth\n";
284 llvm::errs() <<
" " << NumIf <<
" #if/#ifndef/#ifdef.\n";
285 llvm::errs() <<
" " << NumElse <<
" #else/#elif/#elifdef/#elifndef.\n";
286 llvm::errs() <<
" " << NumEndif <<
" #endif.\n";
287 llvm::errs() <<
" " << NumPragma <<
" #pragma.\n";
288 llvm::errs() << NumSkipped <<
" #if/#ifndef#ifdef regions skipped\n";
290 llvm::errs() << NumMacroExpanded <<
"/" << NumFnMacroExpanded <<
"/"
291 << NumBuiltinMacroExpanded <<
" obj/fn/builtin macros expanded, "
292 << NumFastMacroExpanded <<
" on the fast path.\n";
293 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
294 <<
" token paste (##) operations performed, "
295 << NumFastTokenPaste <<
" on the fast path.\n";
297 llvm::errs() <<
"\nPreprocessor Memory: " <<
getTotalMemory() <<
"B total";
299 llvm::errs() <<
"\n BumpPtr: " << BP.getTotalMemory();
300 llvm::errs() <<
"\n Macro Expanded Tokens: "
301 << llvm::capacity_in_bytes(MacroExpandedTokens);
302 llvm::errs() <<
"\n Predefines Buffer: " << Predefines.capacity();
304 llvm::errs() <<
"\n Macros: "
305 << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
306 llvm::errs() <<
"\n #pragma push_macro Info: "
307 << llvm::capacity_in_bytes(PragmaPushMacroInfo);
308 llvm::errs() <<
"\n Poison Reasons: "
309 << llvm::capacity_in_bytes(PoisonReasons);
310 llvm::errs() <<
"\n Comment Handlers: "
311 << llvm::capacity_in_bytes(CommentHandlers) <<
"\n";
317 !ReadMacrosFromExternalSource) {
318 ReadMacrosFromExternalSource =
true;
324 CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
326 return CurSubmoduleState->Macros.begin();
330 return BP.getTotalMemory()
331 + llvm::capacity_in_bytes(MacroExpandedTokens)
332 + Predefines.capacity()
335 + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
336 + llvm::capacity_in_bytes(PragmaPushMacroInfo)
337 + llvm::capacity_in_bytes(PoisonReasons)
338 + llvm::capacity_in_bytes(CommentHandlers);
344 !ReadMacrosFromExternalSource) {
345 ReadMacrosFromExternalSource =
true;
349 return CurSubmoduleState->Macros.end();
356 std::equal(Tokens.begin(), Tokens.end(), MI->
tokens_begin());
363 StringRef BestSpelling;
367 Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
379 BestLocation = Location;
380 BestSpelling = I->first->getName();
388 CurLexerCallback = CurLexer->isDependencyDirectivesLexer()
389 ? CLK_DependencyDirectivesLexer
391 else if (CurTokenLexer)
392 CurLexerCallback = CLK_TokenLexer;
394 CurLexerCallback = CLK_CachingLexer;
398 unsigned CompleteLine,
399 unsigned CompleteColumn) {
400 assert(CompleteLine && CompleteColumn &&
"Starts from 1:1");
401 assert(!CodeCompletionFile &&
"Already set");
404 std::optional<llvm::MemoryBufferRef> Buffer =
410 const char *Position = Buffer->getBufferStart();
412 for (; *Position; ++Position) {
413 if (*Position !=
'\r' && *Position !=
'\n')
417 if ((Position[1] ==
'\r' || Position[1] ==
'\n') &&
418 Position[0] != Position[1])
425 Position += CompleteColumn - 1;
429 if (SkipMainFilePreamble.first &&
431 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
432 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
435 if (Position > Buffer->getBufferEnd())
436 Position = Buffer->getBufferEnd();
438 CodeCompletionFile =
File;
439 CodeCompletionOffset = Position - Buffer->getBufferStart();
441 auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
442 Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
443 char *NewBuf = NewBuffer->getBufferStart();
444 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
446 std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
472 if (Tok.
isNot(tok::raw_identifier) && !Tok.
hasUCN()) {
475 return II->getName();
482 const char *Ptr = Buffer.data();
484 return StringRef(Ptr, Len);
496 SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
498 if (ExpansionLocStart.
isValid())
500 ExpansionLocEnd, Str.size());
504 if (Tok.
is(tok::raw_identifier))
513 std::pair<FileID, unsigned> LocInfo =
SM.getDecomposedLoc(SpellingLoc);
515 StringRef Buffer =
SM.getBufferData(LocInfo.first, &
Invalid);
522 ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
534 if (!
getLangOpts().isCompilingModuleImplementation())
550 assert(NumEnteredSourceFiles == 0 &&
"Cannot reenter the main file!");
561 if (SkipMainFilePreamble.first > 0)
562 CurLexer->SetByteOffset(SkipMainFilePreamble.first,
563 SkipMainFilePreamble.second);
572 std::unique_ptr<llvm::MemoryBuffer> SB =
573 llvm::MemoryBuffer::getMemBufferCopy(Predefines,
"<built-in>");
574 assert(SB &&
"Cannot create predefined source buffer");
576 assert(FID.
isValid() &&
"Could not create FileID for predefines?");
577 setPredefinesFileID(FID);
582 if (!PPOpts->PCHThroughHeader.empty()) {
587 false,
nullptr,
nullptr,
588 nullptr,
nullptr,
nullptr,
593 << PPOpts->PCHThroughHeader;
596 setPCHThroughHeaderFileID(
606void Preprocessor::setPCHThroughHeaderFileID(
FileID FID) {
607 assert(PCHThroughHeaderFileID.
isInvalid() &&
608 "PCHThroughHeaderFileID already set!");
609 PCHThroughHeaderFileID = FID;
613 assert(PCHThroughHeaderFileID.
isValid() &&
614 "Invalid PCH through header FileID");
620 PCHThroughHeaderFileID.
isValid();
625 PCHThroughHeaderFileID.
isValid();
642 bool ReachedMainFileEOF =
false;
643 bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
644 bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
649 CurLexerCallback(*
this, Tok);
650 if (Tok.
is(tok::eof) && !InPredefines) {
651 ReachedMainFileEOF =
true;
654 if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
656 if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
659 if (ReachedMainFileEOF) {
660 if (UsingPCHThroughHeader)
662 << PPOpts->PCHThroughHeader << 1;
663 else if (!PPOpts->PCHWithHdrStopCreate)
668void Preprocessor::replayPreambleConditionalStack() {
670 if (PreambleConditionalStack.isReplaying()) {
672 "CurPPLexer is null when calling replayPreambleConditionalStack.");
674 PreambleConditionalStack.doneReplaying();
675 if (PreambleConditionalStack.reachedEOFWhileSkipping())
676 SkipExcludedConditionalBlock(
677 PreambleConditionalStack.SkipInfo->HashTokenLoc,
678 PreambleConditionalStack.SkipInfo->IfTokenLoc,
679 PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
680 PreambleConditionalStack.SkipInfo->FoundElse,
681 PreambleConditionalStack.SkipInfo->ElseLoc);
688 Callbacks->EndOfMainFile();
699 assert(!
Identifier.getRawIdentifier().empty() &&
"No raw identifier data!");
734 PoisonReasons[II] = DiagID;
738 assert(Ident__exception_code && Ident__exception_info);
739 assert(Ident___exception_code && Ident___exception_info);
753 "Can't handle identifiers without identifier info!");
754 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
755 PoisonReasons.find(
Identifier.getIdentifierInfo());
756 if(it == PoisonReasons.end())
762void Preprocessor::updateOutOfDateIdentifier(
IdentifierInfo &II)
const {
777 "Can't handle identifiers without identifier info!");
787 bool CurrentIsPoisoned =
false;
788 const bool IsSpecialVariadicMacro =
789 &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
790 if (IsSpecialVariadicMacro)
793 updateOutOfDateIdentifier(II);
796 if (IsSpecialVariadicMacro)
808 const auto *MI = MD.getMacroInfo();
809 assert(MI &&
"macro definition with no macro info?");
810 if (!DisableMacroExpansion) {
811 if (!
Identifier.isExpandDisabled() && MI->isEnabled()) {
814 if (!MI->isFunctionLike() || isNextPPTokenLParen())
815 return HandleMacroExpandedIdentifier(
Identifier, MD);
821 if (MI->isObjectLike() || isNextPPTokenLParen())
856 !InMacroArgs && !DisableMacroExpansion &&
858 CurLexerCallback != CLK_CachingLexer) {
860 NamedModuleImportPath.clear();
862 ModuleImportExpectsIdentifier =
true;
863 CurLexerCallback = CLK_LexAfterModuleImport;
872 while (!CurLexerCallback(*
this,
Result))
878 if (
Result.is(tok::code_completion) &&
Result.getIdentifierInfo()) {
884 Result.setIdentifierInfo(
nullptr);
892 if (
getLangOpts().CPlusPlusModules && LexLevel == 1 &&
894 switch (
Result.getKind()) {
895 case tok::l_paren:
case tok::l_square:
case tok::l_brace:
896 StdCXXImportSeqState.handleOpenBracket();
898 case tok::r_paren:
case tok::r_square:
899 StdCXXImportSeqState.handleCloseBracket();
902 StdCXXImportSeqState.handleCloseBrace();
906 case tok::annot_module_include:
908 TrackGMFState.handleSemi();
909 StdCXXImportSeqState.handleSemi();
910 ModuleDeclState.handleSemi();
912 case tok::header_name:
913 case tok::annot_header_unit:
914 StdCXXImportSeqState.handleHeaderName();
917 TrackGMFState.handleExport();
918 StdCXXImportSeqState.handleExport();
919 ModuleDeclState.handleExport();
922 ModuleDeclState.handleColon();
925 ModuleDeclState.handlePeriod();
927 case tok::identifier:
930 if (StdCXXImportSeqState.atTopLevel()) {
931 if (
Result.getIdentifierInfo()->isModulesImport()) {
932 TrackGMFState.handleImport(StdCXXImportSeqState.afterTopLevelSeq());
933 StdCXXImportSeqState.handleImport();
934 if (StdCXXImportSeqState.afterImportSeq()) {
935 ModuleImportLoc =
Result.getLocation();
936 NamedModuleImportPath.clear();
938 ModuleImportExpectsIdentifier =
true;
939 CurLexerCallback = CLK_LexAfterModuleImport;
943 TrackGMFState.handleModule(StdCXXImportSeqState.afterTopLevelSeq());
944 ModuleDeclState.handleModule();
948 ModuleDeclState.handleIdentifier(
Result.getIdentifierInfo());
949 if (ModuleDeclState.isModuleCandidate())
953 TrackGMFState.handleMisc();
954 StdCXXImportSeqState.handleMisc();
955 ModuleDeclState.handleMisc();
961 CheckPoints[CurLexer->getFileID()].push_back(CurLexer->BufferPtr);
962 CheckPointCounter = 0;
965 LastTokenWasAt =
Result.is(tok::at);
968 if ((LexLevel == 0 || PreprocessToken) &&
981 if (Tok.
isOneOf(tok::unknown, tok::eof, tok::eod,
982 tok::annot_repl_input_end))
984 if (Tokens !=
nullptr)
985 Tokens->push_back(Tok);
1011 if (FilenameTok.
is(tok::less) && AllowMacroExpansion) {
1018 FilenameBuffer.push_back(
'<');
1024 while (FilenameTok.
isNot(tok::greater)) {
1026 if (FilenameTok.
isOneOf(tok::eod, tok::eof)) {
1028 Diag(Start, diag::note_matching) << tok::less;
1035 if (FilenameTok.
is(tok::code_completion)) {
1044 FilenameBuffer.push_back(
' ');
1048 size_t PreAppendSize = FilenameBuffer.size();
1049 FilenameBuffer.resize(PreAppendSize + FilenameTok.
getLength());
1051 const char *BufPtr = &FilenameBuffer[PreAppendSize];
1052 unsigned ActualLen =
getSpelling(FilenameTok, BufPtr);
1055 if (BufPtr != &FilenameBuffer[PreAppendSize])
1056 memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
1059 if (FilenameTok.
getLength() != ActualLen)
1060 FilenameBuffer.resize(PreAppendSize + ActualLen);
1064 FilenameTok.
setKind(tok::header_name);
1069 }
else if (FilenameTok.
is(tok::string_literal) && AllowMacroExpansion) {
1080 StringRef Str =
getSpelling(FilenameTok, FilenameBuffer);
1081 if (Str.size() >= 2 && Str.front() ==
'"' && Str.back() ==
'"')
1082 FilenameTok.
setKind(tok::header_name);
1093 unsigned BracketDepth = 0;
1095 Toks.emplace_back();
1098 switch (Toks.back().getKind()) {
1099 case tok::l_paren:
case tok::l_square:
case tok::l_brace:
1103 case tok::r_paren:
case tok::r_square:
case tok::r_brace:
1104 if (BracketDepth == 0)
1110 if (BracketDepth == 0)
1153 if (NamedModuleImportPath.empty() &&
getLangOpts().CPlusPlusModules) {
1157 if (
Result.is(tok::colon) && ModuleDeclState.isNamedModule()) {
1158 std::string Name = ModuleDeclState.getPrimaryName().str();
1160 NamedModuleImportPath.push_back(
1162 CurLexerCallback = CLK_LexAfterModuleImport;
1172 auto ToksCopy = std::make_unique<Token[]>(Toks.size());
1173 std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
1174 EnterTokenStream(std::move(ToksCopy), Toks.size(),
1178 bool ImportingHeader =
Result.is(tok::header_name);
1181 if (ImportingHeader) {
1185 Suffix.push_back(
Result);
1190 if (Suffix.back().isNot(tok::semi)) {
1192 EnterTokens(Suffix);
1201 Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1206 ImportTok.
setKind(tok::kw_import);
1211 auto Action = HandleHeaderIncludeOrImport(
1213 switch (Action.Kind) {
1214 case ImportAction::None:
1217 case ImportAction::ModuleBegin:
1219 Suffix.emplace_back();
1220 Suffix.back().startToken();
1221 Suffix.back().setKind(tok::annot_module_begin);
1222 Suffix.back().setLocation(SemiLoc);
1223 Suffix.back().setAnnotationEndLoc(SemiLoc);
1224 Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1227 case ImportAction::ModuleImport:
1228 case ImportAction::HeaderUnitImport:
1229 case ImportAction::SkippedModuleImport:
1232 Suffix[0].setKind(tok::annot_header_unit);
1233 Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1234 Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1237 case ImportAction::Failure:
1239 "This should be an early exit only to a fatal error");
1240 Result.setKind(tok::eof);
1241 CurLexer->cutOffLexing();
1242 EnterTokens(Suffix);
1246 EnterTokens(Suffix);
1256 if (ModuleImportExpectsIdentifier &&
Result.getKind() == tok::identifier) {
1259 NamedModuleImportPath.push_back(
1260 std::make_pair(
Result.getIdentifierInfo(),
Result.getLocation()));
1261 ModuleImportExpectsIdentifier =
false;
1262 CurLexerCallback = CLK_LexAfterModuleImport;
1269 if (!ModuleImportExpectsIdentifier &&
Result.getKind() == tok::period) {
1270 ModuleImportExpectsIdentifier =
true;
1271 CurLexerCallback = CLK_LexAfterModuleImport;
1276 if (NamedModuleImportPath.empty() ||
Result.is(tok::eof))
1282 if (
Result.isNot(tok::semi)) {
1283 Suffix.push_back(
Result);
1285 if (Suffix.back().isNot(tok::semi)) {
1287 EnterTokens(Suffix);
1290 SemiLoc = Suffix.back().getLocation();
1297 std::string FlatModuleName;
1299 for (
auto &Piece : NamedModuleImportPath) {
1301 if (!FlatModuleName.empty() && FlatModuleName.back() !=
':')
1302 FlatModuleName +=
".";
1303 FlatModuleName += Piece.first->getName();
1306 NamedModuleImportPath.clear();
1307 NamedModuleImportPath.push_back(
1311 Module *Imported =
nullptr;
1314 Imported = TheModuleLoader.
loadModule(ModuleImportLoc,
1315 NamedModuleImportPath,
1323 Callbacks->moduleImport(ModuleImportLoc, NamedModuleImportPath, Imported);
1325 if (!Suffix.empty()) {
1326 EnterTokens(Suffix);
1333 CurSubmoduleState->VisibleModules.setVisible(
1338 Diag(ModuleImportLoc, diag::warn_module_conflict)
1339 << Path[0]->getFullModuleName()
1340 << Conflict->getFullModuleName()
1345 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1346 BuildingSubmoduleStack.back().M->Imports.insert(M);
1350 const char *DiagnosticTag,
1351 bool AllowMacroExpansion) {
1353 if (
Result.isNot(tok::string_literal)) {
1354 Diag(
Result, diag::err_expected_string_literal)
1355 << 0 << DiagnosticTag;
1362 StrToks.push_back(
Result);
1364 if (
Result.hasUDSuffix())
1367 if (AllowMacroExpansion)
1371 }
while (
Result.is(tok::string_literal));
1375 assert(Literal.isOrdinary() &&
"Didn't allow wide strings in");
1377 if (Literal.hadError)
1380 if (Literal.Pascal) {
1381 Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1382 << 0 << DiagnosticTag;
1386 String = std::string(Literal.GetString());
1391 assert(Tok.
is(tok::numeric_constant));
1393 bool NumberInvalid =
false;
1394 StringRef Spelling =
getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1400 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1402 llvm::APInt APVal(64, 0);
1403 if (Literal.GetIntegerValue(APVal))
1406 Value = APVal.getLimitedValue();
1411 assert(Handler &&
"NULL comment handler");
1412 assert(!llvm::is_contained(CommentHandlers, Handler) &&
1413 "Comment handler already registered");
1414 CommentHandlers.push_back(Handler);
1418 std::vector<CommentHandler *>::iterator Pos =
1419 llvm::find(CommentHandlers, Handler);
1420 assert(Pos != CommentHandlers.end() &&
"Comment handler not registered");
1421 CommentHandlers.erase(Pos);
1425 bool AnyPendingTokens =
false;
1426 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1427 HEnd = CommentHandlers.end();
1429 if ((*H)->HandleComment(*
this, Comment))
1430 AnyPendingTokens =
true;
1438void Preprocessor::emitMacroDeprecationWarning(
const Token &
Identifier)
const {
1439 const MacroAnnotations &A =
1441 assert(A.DeprecationInfo &&
1442 "Macro deprecation warning without recorded annotation!");
1443 const MacroAnnotationInfo &Info = *A.DeprecationInfo;
1444 if (Info.Message.empty())
1449 <<
Identifier.getIdentifierInfo() << 1 << Info.Message;
1450 Diag(Info.Location, diag::note_pp_macro_annotation) << 0;
1453void Preprocessor::emitRestrictExpansionWarning(
const Token &
Identifier)
const {
1454 const MacroAnnotations &A =
1456 assert(A.RestrictExpansionInfo &&
1457 "Macro restricted expansion warning without recorded annotation!");
1458 const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo;
1459 if (Info.Message.empty())
1464 <<
Identifier.getIdentifierInfo() << 1 << Info.Message;
1465 Diag(Info.Location, diag::note_pp_macro_annotation) << 1;
1469 unsigned DiagSelection)
const {
1470 Diag(
Identifier, diag::warn_fp_nan_inf_when_disabled) << DiagSelection << 1;
1474 bool IsUndef)
const {
1475 const MacroAnnotations &A =
1477 assert(A.FinalAnnotationLoc &&
1478 "Final macro warning without recorded annotation!");
1481 <<
Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1);
1482 Diag(*A.FinalAnnotationLoc, diag::note_pp_macro_annotation) << 2;
1488 auto FirstRegionEndingAfterLoc = llvm::partition_point(
1489 SafeBufferOptOutMap,
1491 &Loc](
const std::pair<SourceLocation, SourceLocation> &Region) {
1495 if (FirstRegionEndingAfterLoc != SafeBufferOptOutMap.end()) {
1502 if (!SafeBufferOptOutMap.empty() &&
1503 SafeBufferOptOutMap.back().first == SafeBufferOptOutMap.back().second)
1514 InSafeBufferOptOutRegion =
true;
1515 CurrentSafeBufferOptOutStart = Loc;
1519 if (!SafeBufferOptOutMap.empty()) {
1520 [[maybe_unused]]
auto *PrevRegion = &SafeBufferOptOutMap.back();
1521 assert(PrevRegion->first != PrevRegion->second &&
1522 "Shall not begin a safe buffer opt-out region before closing the "
1528 SafeBufferOptOutMap.emplace_back(Loc, Loc);
1532 InSafeBufferOptOutRegion =
false;
1536 assert(!SafeBufferOptOutMap.empty() &&
1537 "Misordered safe buffer opt-out regions");
1538 auto *CurrRegion = &SafeBufferOptOutMap.back();
1539 assert(CurrRegion->first == CurrRegion->second &&
1540 "Set end location to a closed safe buffer opt-out region");
1541 CurrRegion->second = Loc;
1547 return InSafeBufferOptOutRegion;
1550 StartLoc = CurrentSafeBufferOptOutStart;
1551 return InSafeBufferOptOutRegion;
1571 if (
auto It = CheckPoints.find(FID); It != CheckPoints.end()) {
1573 const char *
Last =
nullptr;
1575 for (
const char *
P : FileCheckPoints) {
Defines enum values for all the target-independent builtin functions.
Defines the clang::FileManager interface and associated types.
Defines the FileSystemStatCache interface.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
llvm::MachO::Target Target
llvm::MachO::Record Record
Defines the clang::MacroInfo and clang::MacroDirective classes.
Defines the clang::Module class, which describes a module in the source code.
Defines the PreprocessorLexer interface.
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
static constexpr unsigned CheckPointStepSize
Minimum distance between two check points, in tokens.
Defines the clang::Preprocessor interface.
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an #include directive.
virtual ~CodeCompletionHandler()
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
Concrete class used by the front-end to report problems and issues.
virtual ~EmptylineHandler()
virtual ~ExternalPreprocessorSource()
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
Cached information about one file (either on disk or in the virtual file system).
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Provides lookups to, and iteration over, IdentiferInfo objects.
One of these records is kept for each identifier that is lexed.
bool isModulesImport() const
Determine whether this is the contextual keyword import.
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
bool isPoisoned() const
Return true if this token has been poisoned.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
void setIsFutureCompatKeyword(bool Val)
StringRef getName() const
Return the actual identifier string.
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
bool isExtensionToken() const
get/setExtension - Initialize information about whether or not this language token is an extension.
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
A description of the current definition of a macro.
MacroInfo * getMacroInfo()
SourceLocation getLocation() const
Encapsulates the data about a macro definition (e.g.
const_tokens_iterator tokens_begin() const
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
const Token & getReplacementToken(unsigned Tok) const
bool isObjectLike() const
Abstract interface for a module loader.
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
Represents a macro directive exported by a module.
Describes a module or submodule.
@ Hidden
All of the names in this module are hidden.
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber,...
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
void LexIncludeFilename(Token &FilenameTok)
Lex a token, producing a header-name token if possible.
bool markIncluded(FileEntryRef File)
Mark the file as included.
void FinalizeForModelFile()
Cleanup after model file parsing.
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
bool isSafeBufferOptOut(const SourceManager &SourceMgr, const SourceLocation &Loc) const
const char * getCheckPoint(FileID FID, const char *Start) const
Returns a pointer into the given file's buffer that's guaranteed to be between tokens.
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
void DumpMacro(const MacroInfo &MI) const
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool SetCodeCompletionPoint(FileEntryRef File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void makeModuleVisible(Module *M, SourceLocation Loc)
bool isInImportingCXXNamedModules() const
If we're importing a standard C++20 Named Modules.
void Lex(Token &Result)
Lex the next token for this preprocessor.
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
bool enterOrExitSafeBufferOptOutRegion(bool isEnter, const SourceLocation &Loc)
Alter the state of whether this PP currently is in a "-Wunsafe-buffer-usage" opt-out region.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
macro_iterator macro_end(bool IncludeExternalMacros=true) const
SourceManager & getSourceManager() const
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
bool getCommentRetentionState() const
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
MacroMap::const_iterator macro_iterator
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
bool isPPInSafeBufferOptOutRegion()
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
const TargetInfo & getTargetInfo() const
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void DumpLocation(SourceLocation Loc) const
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleComment(Token &result, SourceRange Comment)
HeaderSearch & getHeaderSearchInfo() const
ExternalPreprocessorSource * getExternalSource() const
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, const LangOptions &LangOpts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
IdentifierTable & getIdentifierTable()
const LangOptions & getLangOpts() const
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void PoisonSEHIdentifiers(bool Poison=true)
size_t getTotalMemory() const
void LexTokensUntilEOF(std::vector< Token > *Tokens=nullptr)
Lex all tokens for this preprocessor until (and excluding) end of file.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
DiagnosticsEngine & getDiagnostics() const
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop.
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens.
Encodes a location in the source.
bool isValid() const
Return true if this is a valid SourceLocation object.
void print(raw_ostream &OS, const SourceManager &SM) const
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
This class handles loading and caching of source files into memory.
OptionalFileEntryRef getFileEntryRefForID(FileID FID) const
Returns the FileEntryRef for the provided FileID.
FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
FileID getMainFileID() const
Returns the FileID of the main source file.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer.
void overrideFileContents(FileEntryRef SourceFile, const llvm::MemoryBufferRef &Buffer)
Override the contents of the given source file by providing an already-allocated buffer.
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
SourceLocation createExpansionLoc(SourceLocation SpellingLoc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned Length, bool ExpansionIsTokenRange=true, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Creates an expansion SLocEntry for a macro use.
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
std::optional< llvm::MemoryBufferRef > getMemoryBufferForFileOrNone(FileEntryRef File)
Retrieve the memory buffer associated with the given file.
A trivial tuple used to represent a source range.
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
Exposes information about the current target.
Token - This structure provides full information about a lexed token.
IdentifierInfo * getIdentifierInfo() const
void setLiteralData(const char *Ptr)
bool hasUCN() const
Returns true if this token contains a universal character name.
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
unsigned getLength() const
void setLength(unsigned Len)
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6....
void setKind(tok::TokenKind K)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
tok::TokenKind getKind() const
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
void setLocation(SourceLocation L)
bool hasLeadingEmptyMacro() const
Return true if this token has an empty macro before it.
void setRawIdentifierData(const char *Ptr)
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
bool isNot(tok::TokenKind K) const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
void startToken()
Reset all flags to cleared.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
void setIdentifierInfo(IdentifierInfo *II)
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Defines the clang::TargetInfo interface.
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
The JSON file list parser is used to communicate input to InstallAPI.
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
@ Result
The result type of a method or function.
TranslationUnitKind
Describes the kind of translation unit being processed.
@ TU_Prefix
The translation unit is a prefix to a translation unit, and is not complete.