54#include "llvm/ADT/APInt.h"
55#include "llvm/ADT/ArrayRef.h"
56#include "llvm/ADT/DenseMap.h"
57#include "llvm/ADT/STLExtras.h"
58#include "llvm/ADT/SmallString.h"
59#include "llvm/ADT/SmallVector.h"
60#include "llvm/ADT/StringRef.h"
61#include "llvm/Support/Capacity.h"
62#include "llvm/Support/ErrorHandling.h"
63#include "llvm/Support/MemoryBuffer.h"
64#include "llvm/Support/raw_ostream.h"
85 : PPOpts(
std::move(PPOpts)), Diags(&diags), LangOpts(opts),
86 FileMgr(Headers.getFileMgr()), SourceMgr(
SM),
87 ScratchBuf(new
ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
88 TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
92 Identifiers(IILookup), PragmaHandlers(new
PragmaNamespace(StringRef())),
93 TUKind(TUKind), SkipMainFilePreamble(0,
true),
94 CurSubmoduleState(&NullSubmoduleState) {
95 OwnsHeaderSearch = OwnsHeaders;
99 KeepMacroComments =
false;
100 SuppressIncludeNotFoundError =
false;
103 DisableMacroExpansion =
false;
104 MacroExpansionInDirectivesOverride =
false;
107 InMacroArgPreExpansion =
false;
108 NumCachedTokenLexers = 0;
109 PragmasEnabled =
true;
110 ParsingIfOrElifDirective =
false;
111 PreprocessedOutput =
false;
114 ReadMacrosFromExternalSource =
false;
116 BuiltinInfo = std::make_unique<Builtin::Context>();
126 RegisterBuiltinPragmas();
129 RegisterBuiltinMacros();
131 if(LangOpts.Borland) {
142 Ident__exception_info = Ident__exception_code =
nullptr;
143 Ident__abnormal_termination = Ident___exception_info =
nullptr;
144 Ident___exception_code = Ident___abnormal_termination =
nullptr;
145 Ident_GetExceptionInfo = Ident_GetExceptionCode =
nullptr;
146 Ident_AbnormalTermination =
nullptr;
151 SkippingUntilPragmaHdrStop =
true;
154 if (!this->PPOpts->PCHThroughHeader.empty() &&
155 !this->PPOpts->ImplicitPCHInclude.empty())
156 SkippingUntilPCHThroughHeader =
true;
158 if (this->PPOpts->GeneratePreamble)
159 PreambleConditionalStack.startRecording();
161 MaxTokens = LangOpts.MaxTokens;
165 assert(BacktrackPositions.empty() &&
"EnableBacktrack/Backtrack imbalance!");
167 IncludeMacroStack.clear();
172 std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers,
nullptr);
173 CurTokenLexer.reset();
176 for (
MacroArgs *ArgList = MacroArgCache; ArgList;)
177 ArgList = ArgList->deallocate();
180 if (OwnsHeaderSearch)
186 assert((!this->Target || this->Target == &
Target) &&
187 "Invalid override of target information");
190 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
191 "Invalid override of aux target information.");
192 this->AuxTarget = AuxTarget;
195 BuiltinInfo->InitializeTarget(
Target, AuxTarget);
213 NumEnteredSourceFiles = 0;
216 PragmaHandlersBackup = std::move(PragmaHandlers);
217 PragmaHandlers = std::make_unique<PragmaNamespace>(StringRef());
218 RegisterBuiltinPragmas();
221 PredefinesFileID =
FileID();
225 NumEnteredSourceFiles = 1;
227 PragmaHandlers = std::move(PragmaHandlersBackup);
236 if (!DumpFlags)
return;
238 llvm::errs() <<
"\t";
240 llvm::errs() <<
" [StartOfLine]";
242 llvm::errs() <<
" [LeadingSpace]";
244 llvm::errs() <<
" [ExpandDisabled]";
247 llvm::errs() <<
" [UnClean='" << StringRef(Start, Tok.
getLength())
251 llvm::errs() <<
"\tLoc=<";
257 Loc.
print(llvm::errs(), SourceMgr);
261 llvm::errs() <<
"MACRO: ";
262 for (
unsigned i = 0, e = MI.
getNumTokens(); i != e; ++i) {
266 llvm::errs() <<
"\n";
270 llvm::errs() <<
"\n*** Preprocessor Stats:\n";
271 llvm::errs() << NumDirectives <<
" directives found:\n";
272 llvm::errs() <<
" " << NumDefined <<
" #define.\n";
273 llvm::errs() <<
" " << NumUndefined <<
" #undef.\n";
274 llvm::errs() <<
" #include/#include_next/#import:\n";
275 llvm::errs() <<
" " << NumEnteredSourceFiles <<
" source files entered.\n";
276 llvm::errs() <<
" " << MaxIncludeStackDepth <<
" max include stack depth\n";
277 llvm::errs() <<
" " << NumIf <<
" #if/#ifndef/#ifdef.\n";
278 llvm::errs() <<
" " << NumElse <<
" #else/#elif/#elifdef/#elifndef.\n";
279 llvm::errs() <<
" " << NumEndif <<
" #endif.\n";
280 llvm::errs() <<
" " << NumPragma <<
" #pragma.\n";
281 llvm::errs() << NumSkipped <<
" #if/#ifndef#ifdef regions skipped\n";
283 llvm::errs() << NumMacroExpanded <<
"/" << NumFnMacroExpanded <<
"/"
284 << NumBuiltinMacroExpanded <<
" obj/fn/builtin macros expanded, "
285 << NumFastMacroExpanded <<
" on the fast path.\n";
286 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
287 <<
" token paste (##) operations performed, "
288 << NumFastTokenPaste <<
" on the fast path.\n";
290 llvm::errs() <<
"\nPreprocessor Memory: " <<
getTotalMemory() <<
"B total";
292 llvm::errs() <<
"\n BumpPtr: " << BP.getTotalMemory();
293 llvm::errs() <<
"\n Macro Expanded Tokens: "
294 << llvm::capacity_in_bytes(MacroExpandedTokens);
295 llvm::errs() <<
"\n Predefines Buffer: " << Predefines.capacity();
297 llvm::errs() <<
"\n Macros: "
298 << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
299 llvm::errs() <<
"\n #pragma push_macro Info: "
300 << llvm::capacity_in_bytes(PragmaPushMacroInfo);
301 llvm::errs() <<
"\n Poison Reasons: "
302 << llvm::capacity_in_bytes(PoisonReasons);
303 llvm::errs() <<
"\n Comment Handlers: "
304 << llvm::capacity_in_bytes(CommentHandlers) <<
"\n";
309 if (IncludeExternalMacros && ExternalSource &&
310 !ReadMacrosFromExternalSource) {
311 ReadMacrosFromExternalSource =
true;
317 CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
319 return CurSubmoduleState->Macros.begin();
323 return BP.getTotalMemory()
324 + llvm::capacity_in_bytes(MacroExpandedTokens)
325 + Predefines.capacity()
328 + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
329 + llvm::capacity_in_bytes(PragmaPushMacroInfo)
330 + llvm::capacity_in_bytes(PoisonReasons)
331 + llvm::capacity_in_bytes(CommentHandlers);
336 if (IncludeExternalMacros && ExternalSource &&
337 !ReadMacrosFromExternalSource) {
338 ReadMacrosFromExternalSource =
true;
342 return CurSubmoduleState->Macros.end();
349 std::equal(Tokens.begin(), Tokens.end(), MI->
tokens_begin());
356 StringRef BestSpelling;
360 Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
372 BestLocation = Location;
373 BestSpelling = I->first->getName();
381 CurLexerKind = CurLexer->isDependencyDirectivesLexer()
382 ? CLK_DependencyDirectivesLexer
384 else if (CurTokenLexer)
385 CurLexerKind = CLK_TokenLexer;
387 CurLexerKind = CLK_CachingLexer;
391 unsigned CompleteLine,
392 unsigned CompleteColumn) {
394 assert(CompleteLine && CompleteColumn &&
"Starts from 1:1");
395 assert(!CodeCompletionFile &&
"Already set");
398 std::optional<llvm::MemoryBufferRef> Buffer =
404 const char *Position = Buffer->getBufferStart();
405 for (
unsigned Line = 1; Line < CompleteLine; ++Line) {
406 for (; *Position; ++Position) {
407 if (*Position !=
'\r' && *Position !=
'\n')
411 if ((Position[1] ==
'\r' || Position[1] ==
'\n') &&
412 Position[0] != Position[1])
419 Position += CompleteColumn - 1;
423 if (SkipMainFilePreamble.first &&
425 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
426 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
429 if (Position > Buffer->getBufferEnd())
430 Position = Buffer->getBufferEnd();
432 CodeCompletionFile =
File;
433 CodeCompletionOffset = Position - Buffer->getBufferStart();
435 auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
436 Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
437 char *NewBuf = NewBuffer->getBufferStart();
438 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
440 std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
466 if (Tok.
isNot(tok::raw_identifier) && !Tok.
hasUCN()) {
469 return II->getName();
476 const char *Ptr = Buffer.data();
478 return StringRef(Ptr, Len);
490 SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
492 if (ExpansionLocStart.
isValid())
494 ExpansionLocEnd, Str.size());
498 if (Tok.
is(tok::raw_identifier))
507 std::pair<FileID, unsigned> LocInfo =
SM.getDecomposedLoc(SpellingLoc);
509 StringRef Buffer =
SM.getBufferData(LocInfo.first, &
Invalid);
516 ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
528 if (!
getLangOpts().isCompilingModuleImplementation())
544 assert(NumEnteredSourceFiles == 0 &&
"Cannot reenter the main file!");
555 if (SkipMainFilePreamble.first > 0)
556 CurLexer->SetByteOffset(SkipMainFilePreamble.first,
557 SkipMainFilePreamble.second);
566 std::unique_ptr<llvm::MemoryBuffer> SB =
567 llvm::MemoryBuffer::getMemBufferCopy(Predefines,
"<built-in>");
568 assert(SB &&
"Cannot create predefined source buffer");
570 assert(FID.
isValid() &&
"Could not create FileID for predefines?");
571 setPredefinesFileID(FID);
576 if (!PPOpts->PCHThroughHeader.empty()) {
581 false,
nullptr,
nullptr,
582 nullptr,
nullptr,
nullptr,
587 << PPOpts->PCHThroughHeader;
590 setPCHThroughHeaderFileID(
600void Preprocessor::setPCHThroughHeaderFileID(
FileID FID) {
601 assert(PCHThroughHeaderFileID.
isInvalid() &&
602 "PCHThroughHeaderFileID already set!");
603 PCHThroughHeaderFileID = FID;
607 assert(PCHThroughHeaderFileID.
isValid() &&
608 "Invalid PCH through header FileID");
614 PCHThroughHeaderFileID.
isValid();
619 PCHThroughHeaderFileID.
isValid();
636 bool ReachedMainFileEOF =
false;
637 bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
638 bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
643 switch (CurLexerKind) {
648 CurTokenLexer->Lex(Tok);
650 case CLK_CachingLexer:
653 case CLK_DependencyDirectivesLexer:
654 CurLexer->LexDependencyDirectiveToken(Tok);
656 case CLK_LexAfterModuleImport:
660 if (Tok.
is(tok::eof) && !InPredefines) {
661 ReachedMainFileEOF =
true;
664 if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
666 if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
669 if (ReachedMainFileEOF) {
670 if (UsingPCHThroughHeader)
672 << PPOpts->PCHThroughHeader << 1;
673 else if (!PPOpts->PCHWithHdrStopCreate)
678void Preprocessor::replayPreambleConditionalStack() {
680 if (PreambleConditionalStack.isReplaying()) {
682 "CurPPLexer is null when calling replayPreambleConditionalStack.");
684 PreambleConditionalStack.doneReplaying();
685 if (PreambleConditionalStack.reachedEOFWhileSkipping())
686 SkipExcludedConditionalBlock(
687 PreambleConditionalStack.SkipInfo->HashTokenLoc,
688 PreambleConditionalStack.SkipInfo->IfTokenLoc,
689 PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
690 PreambleConditionalStack.SkipInfo->FoundElse,
691 PreambleConditionalStack.SkipInfo->ElseLoc);
698 Callbacks->EndOfMainFile();
709 assert(!
Identifier.getRawIdentifier().empty() &&
"No raw identifier data!");
744 PoisonReasons[II] = DiagID;
748 assert(Ident__exception_code && Ident__exception_info);
749 assert(Ident___exception_code && Ident___exception_info);
763 "Can't handle identifiers without identifier info!");
764 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
765 PoisonReasons.find(
Identifier.getIdentifierInfo());
766 if(it == PoisonReasons.end())
772void Preprocessor::updateOutOfDateIdentifier(
IdentifierInfo &II)
const {
787 "Can't handle identifiers without identifier info!");
797 bool CurrentIsPoisoned =
false;
798 const bool IsSpecialVariadicMacro =
799 &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
800 if (IsSpecialVariadicMacro)
803 updateOutOfDateIdentifier(II);
806 if (IsSpecialVariadicMacro)
818 auto *MI = MD.getMacroInfo();
819 assert(MI &&
"macro definition with no macro info?");
820 if (!DisableMacroExpansion) {
821 if (!
Identifier.isExpandDisabled() && MI->isEnabled()) {
824 if (!MI->isFunctionLike() || isNextPPTokenLParen())
825 return HandleMacroExpandedIdentifier(
Identifier, MD);
831 if (MI->isObjectLike() || isNextPPTokenLParen())
866 !InMacroArgs && !DisableMacroExpansion &&
868 CurLexerKind != CLK_CachingLexer) {
870 NamedModuleImportPath.clear();
872 ModuleImportExpectsIdentifier =
true;
873 CurLexerKind = CLK_LexAfterModuleImport;
884 switch (CurLexerKind) {
886 ReturnedToken = CurLexer->Lex(
Result);
889 ReturnedToken = CurTokenLexer->Lex(
Result);
891 case CLK_CachingLexer:
893 ReturnedToken =
true;
895 case CLK_DependencyDirectivesLexer:
896 ReturnedToken = CurLexer->LexDependencyDirectiveToken(
Result);
898 case CLK_LexAfterModuleImport:
902 }
while (!ReturnedToken);
907 if (
Result.is(tok::code_completion) &&
Result.getIdentifierInfo()) {
913 Result.setIdentifierInfo(
nullptr);
921 if (
getLangOpts().CPlusPlusModules && LexLevel == 1 &&
923 switch (
Result.getKind()) {
924 case tok::l_paren:
case tok::l_square:
case tok::l_brace:
925 StdCXXImportSeqState.handleOpenBracket();
927 case tok::r_paren:
case tok::r_square:
928 StdCXXImportSeqState.handleCloseBracket();
931 StdCXXImportSeqState.handleCloseBrace();
935 case tok::annot_module_include:
937 TrackGMFState.handleSemi();
938 StdCXXImportSeqState.handleSemi();
939 ModuleDeclState.handleSemi();
941 case tok::header_name:
942 case tok::annot_header_unit:
943 StdCXXImportSeqState.handleHeaderName();
946 TrackGMFState.handleExport();
947 StdCXXImportSeqState.handleExport();
948 ModuleDeclState.handleExport();
951 ModuleDeclState.handleColon();
954 ModuleDeclState.handlePeriod();
956 case tok::identifier:
957 if (
Result.getIdentifierInfo()->isModulesImport()) {
958 TrackGMFState.handleImport(StdCXXImportSeqState.afterTopLevelSeq());
959 StdCXXImportSeqState.handleImport();
960 if (StdCXXImportSeqState.afterImportSeq()) {
961 ModuleImportLoc =
Result.getLocation();
962 NamedModuleImportPath.clear();
964 ModuleImportExpectsIdentifier =
true;
965 CurLexerKind = CLK_LexAfterModuleImport;
969 TrackGMFState.handleModule(StdCXXImportSeqState.afterTopLevelSeq());
970 ModuleDeclState.handleModule();
973 ModuleDeclState.handleIdentifier(
Result.getIdentifierInfo());
974 if (ModuleDeclState.isModuleCandidate())
979 TrackGMFState.handleMisc();
980 StdCXXImportSeqState.handleMisc();
981 ModuleDeclState.handleMisc();
986 LastTokenWasAt =
Result.is(tok::at);
989 if ((LexLevel == 0 || PreprocessToken) &&
1020 if (FilenameTok.
is(tok::less) && AllowMacroExpansion) {
1027 FilenameBuffer.push_back(
'<');
1033 while (FilenameTok.
isNot(tok::greater)) {
1035 if (FilenameTok.
isOneOf(tok::eod, tok::eof)) {
1037 Diag(Start, diag::note_matching) << tok::less;
1044 if (FilenameTok.
is(tok::code_completion)) {
1053 FilenameBuffer.push_back(
' ');
1057 size_t PreAppendSize = FilenameBuffer.size();
1058 FilenameBuffer.resize(PreAppendSize + FilenameTok.
getLength());
1060 const char *BufPtr = &FilenameBuffer[PreAppendSize];
1061 unsigned ActualLen =
getSpelling(FilenameTok, BufPtr);
1064 if (BufPtr != &FilenameBuffer[PreAppendSize])
1065 memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
1068 if (FilenameTok.
getLength() != ActualLen)
1069 FilenameBuffer.resize(PreAppendSize + ActualLen);
1073 FilenameTok.
setKind(tok::header_name);
1078 }
else if (FilenameTok.
is(tok::string_literal) && AllowMacroExpansion) {
1089 StringRef Str =
getSpelling(FilenameTok, FilenameBuffer);
1090 if (Str.size() >= 2 && Str.front() ==
'"' && Str.back() ==
'"')
1091 FilenameTok.
setKind(tok::header_name);
1102 unsigned BracketDepth = 0;
1104 Toks.emplace_back();
1107 switch (Toks.back().getKind()) {
1108 case tok::l_paren:
case tok::l_square:
case tok::l_brace:
1112 case tok::r_paren:
case tok::r_square:
case tok::r_brace:
1113 if (BracketDepth == 0)
1119 if (BracketDepth == 0)
1162 if (NamedModuleImportPath.empty() &&
getLangOpts().CPlusPlusModules) {
1166 if (
Result.is(tok::colon) && ModuleDeclState.isNamedModule()) {
1167 std::string Name = ModuleDeclState.getPrimaryName().str();
1169 NamedModuleImportPath.push_back(
1171 CurLexerKind = CLK_LexAfterModuleImport;
1181 auto ToksCopy = std::make_unique<Token[]>(Toks.size());
1182 std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
1183 EnterTokenStream(std::move(ToksCopy), Toks.size(),
1187 bool ImportingHeader =
Result.is(tok::header_name);
1190 if (ImportingHeader) {
1194 Suffix.push_back(
Result);
1199 if (Suffix.back().isNot(tok::semi)) {
1201 EnterTokens(Suffix);
1210 Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1215 ImportTok.
setKind(tok::kw_import);
1220 auto Action = HandleHeaderIncludeOrImport(
1222 switch (Action.Kind) {
1223 case ImportAction::None:
1226 case ImportAction::ModuleBegin:
1228 Suffix.emplace_back();
1229 Suffix.back().startToken();
1230 Suffix.back().setKind(tok::annot_module_begin);
1231 Suffix.back().setLocation(SemiLoc);
1232 Suffix.back().setAnnotationEndLoc(SemiLoc);
1233 Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1236 case ImportAction::ModuleImport:
1237 case ImportAction::HeaderUnitImport:
1238 case ImportAction::SkippedModuleImport:
1241 Suffix[0].setKind(tok::annot_header_unit);
1242 Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1243 Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1246 case ImportAction::Failure:
1248 "This should be an early exit only to a fatal error");
1249 Result.setKind(tok::eof);
1250 CurLexer->cutOffLexing();
1251 EnterTokens(Suffix);
1255 EnterTokens(Suffix);
1265 if (ModuleImportExpectsIdentifier &&
Result.getKind() == tok::identifier) {
1268 NamedModuleImportPath.push_back(
1269 std::make_pair(
Result.getIdentifierInfo(),
Result.getLocation()));
1270 ModuleImportExpectsIdentifier =
false;
1271 CurLexerKind = CLK_LexAfterModuleImport;
1278 if (!ModuleImportExpectsIdentifier &&
Result.getKind() == tok::period) {
1279 ModuleImportExpectsIdentifier =
true;
1280 CurLexerKind = CLK_LexAfterModuleImport;
1285 if (NamedModuleImportPath.empty() ||
Result.is(tok::eof))
1291 if (
Result.isNot(tok::semi)) {
1292 Suffix.push_back(
Result);
1294 if (Suffix.back().isNot(tok::semi)) {
1296 EnterTokens(Suffix);
1299 SemiLoc = Suffix.back().getLocation();
1306 std::string FlatModuleName;
1308 for (
auto &Piece : NamedModuleImportPath) {
1310 if (!FlatModuleName.empty() && FlatModuleName.back() !=
':')
1311 FlatModuleName +=
".";
1312 FlatModuleName += Piece.first->getName();
1315 NamedModuleImportPath.clear();
1316 NamedModuleImportPath.push_back(
1320 Module *Imported =
nullptr;
1323 Imported = TheModuleLoader.
loadModule(ModuleImportLoc,
1324 NamedModuleImportPath,
1332 Callbacks->moduleImport(ModuleImportLoc, NamedModuleImportPath, Imported);
1334 if (!Suffix.empty()) {
1335 EnterTokens(Suffix);
1342 CurSubmoduleState->VisibleModules.setVisible(
1347 Diag(ModuleImportLoc, diag::warn_module_conflict)
1348 << Path[0]->getFullModuleName()
1349 << Conflict->getFullModuleName()
1354 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1355 BuildingSubmoduleStack.back().M->Imports.insert(M);
1359 const char *DiagnosticTag,
1360 bool AllowMacroExpansion) {
1362 if (
Result.isNot(tok::string_literal)) {
1363 Diag(
Result, diag::err_expected_string_literal)
1364 << 0 << DiagnosticTag;
1371 StrToks.push_back(
Result);
1373 if (
Result.hasUDSuffix())
1376 if (AllowMacroExpansion)
1380 }
while (
Result.is(tok::string_literal));
1384 assert(Literal.isOrdinary() &&
"Didn't allow wide strings in");
1386 if (Literal.hadError)
1389 if (Literal.Pascal) {
1390 Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1391 << 0 << DiagnosticTag;
1395 String = std::string(Literal.GetString());
1400 assert(Tok.
is(tok::numeric_constant));
1402 bool NumberInvalid =
false;
1403 StringRef Spelling =
getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1409 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1411 llvm::APInt APVal(64, 0);
1412 if (Literal.GetIntegerValue(APVal))
1415 Value = APVal.getLimitedValue();
1420 assert(Handler &&
"NULL comment handler");
1421 assert(!llvm::is_contained(CommentHandlers, Handler) &&
1422 "Comment handler already registered");
1423 CommentHandlers.push_back(Handler);
1427 std::vector<CommentHandler *>::iterator Pos =
1428 llvm::find(CommentHandlers, Handler);
1429 assert(Pos != CommentHandlers.end() &&
"Comment handler not registered");
1430 CommentHandlers.erase(Pos);
1434 bool AnyPendingTokens =
false;
1435 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1436 HEnd = CommentHandlers.end();
1438 if ((*H)->HandleComment(*
this, Comment))
1439 AnyPendingTokens =
true;
1447void Preprocessor::emitMacroDeprecationWarning(
const Token &
Identifier)
const {
1448 const MacroAnnotations &A =
1450 assert(A.DeprecationInfo &&
1451 "Macro deprecation warning without recorded annotation!");
1452 const MacroAnnotationInfo &Info = *A.DeprecationInfo;
1453 if (Info.Message.empty())
1458 <<
Identifier.getIdentifierInfo() << 1 << Info.Message;
1459 Diag(Info.Location, diag::note_pp_macro_annotation) << 0;
1462void Preprocessor::emitRestrictExpansionWarning(
const Token &
Identifier)
const {
1463 const MacroAnnotations &A =
1465 assert(A.RestrictExpansionInfo &&
1466 "Macro restricted expansion warning without recorded annotation!");
1467 const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo;
1468 if (Info.Message.empty())
1473 <<
Identifier.getIdentifierInfo() << 1 << Info.Message;
1474 Diag(Info.Location, diag::note_pp_macro_annotation) << 1;
1478 bool IsUndef)
const {
1479 const MacroAnnotations &A =
1481 assert(A.FinalAnnotationLoc &&
1482 "Final macro warning without recorded annotation!");
1485 <<
Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1);
1486 Diag(*A.FinalAnnotationLoc, diag::note_pp_macro_annotation) << 2;
1492 auto FirstRegionEndingAfterLoc = llvm::partition_point(
1493 SafeBufferOptOutMap,
1495 &Loc](
const std::pair<SourceLocation, SourceLocation> &Region) {
1499 if (FirstRegionEndingAfterLoc != SafeBufferOptOutMap.end()) {
1506 if (!SafeBufferOptOutMap.empty() &&
1507 SafeBufferOptOutMap.back().first == SafeBufferOptOutMap.back().second)
1518 InSafeBufferOptOutRegion =
true;
1519 CurrentSafeBufferOptOutStart = Loc;
1523 if (!SafeBufferOptOutMap.empty()) {
1524 [[maybe_unused]]
auto *PrevRegion = &SafeBufferOptOutMap.back();
1525 assert(PrevRegion->first != PrevRegion->second &&
1526 "Shall not begin a safe buffer opt-out region before closing the "
1532 SafeBufferOptOutMap.emplace_back(Loc, Loc);
1536 InSafeBufferOptOutRegion =
false;
1540 assert(!SafeBufferOptOutMap.empty() &&
1541 "Misordered safe buffer opt-out regions");
1542 auto *CurrRegion = &SafeBufferOptOutMap.back();
1543 assert(CurrRegion->first == CurrRegion->second &&
1544 "Set end location to a closed safe buffer opt-out region");
1545 CurrRegion->second = Loc;
1551 return InSafeBufferOptOutRegion;
1554 StartLoc = CurrentSafeBufferOptOutStart;
1555 return InSafeBufferOptOutRegion;
Defines enum values for all the target-independent builtin functions.
Defines the clang::FileManager interface and associated types.
Defines the FileSystemStatCache interface.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
Defines the clang::MacroInfo and clang::MacroDirective classes.
Defines the clang::Module class, which describes a module in the source code.
Defines the PreprocessorLexer interface.
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
Defines the clang::Preprocessor interface.
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an #include directive.
virtual ~CodeCompletionHandler()
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
Concrete class used by the front-end to report problems and issues.
virtual ~EmptylineHandler()
virtual ~ExternalPreprocessorSource()
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
virtual void ReadDefinedMacros()=0
Read the set of macros defined by this external macro source.
Cached information about one file (either on disk or in the virtual file system).
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Provides lookups to, and iteration over, IdentiferInfo objects.
One of these records is kept for each identifier that is lexed.
bool isModulesImport() const
Determine whether this is the contextual keyword import.
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
bool isPoisoned() const
Return true if this token has been poisoned.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
void setIsFutureCompatKeyword(bool Val)
StringRef getName() const
Return the actual identifier string.
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
bool isExtensionToken() const
get/setExtension - Initialize information about whether or not this language token is an extension.
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
A description of the current definition of a macro.
MacroInfo * getMacroInfo()
SourceLocation getLocation() const
Encapsulates the data about a macro definition (e.g.
const_tokens_iterator tokens_begin() const
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
const Token & getReplacementToken(unsigned Tok) const
bool isObjectLike() const
Abstract interface for a module loader.
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
Represents a macro directive exported by a module.
Describes a module or submodule.
@ Hidden
All of the names in this module are hidden.
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber,...
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
void LexIncludeFilename(Token &FilenameTok)
Lex a token, producing a header-name token if possible.
void FinalizeForModelFile()
Cleanup after model file parsing.
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
bool markIncluded(const FileEntry *File)
Mark the file as included.
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
bool isSafeBufferOptOut(const SourceManager &SourceMgr, const SourceLocation &Loc) const
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
void DumpMacro(const MacroInfo &MI) const
void setCodeCompletionReached()
Note that we hit the code-completion point.
void makeModuleVisible(Module *M, SourceLocation Loc)
bool isInImportingCXXNamedModules() const
If we're importing a standard C++20 Named Modules.
void Lex(Token &Result)
Lex the next token for this preprocessor.
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
bool enterOrExitSafeBufferOptOutRegion(bool isEnter, const SourceLocation &Loc)
Alter the state of whether this PP currently is in a "-Wunsafe-buffer-usage" opt-out region.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
macro_iterator macro_end(bool IncludeExternalMacros=true) const
SourceManager & getSourceManager() const
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
bool getCommentRetentionState() const
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
MacroMap::const_iterator macro_iterator
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
bool isPPInSafeBufferOptOutRegion()
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
const TargetInfo & getTargetInfo() const
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void DumpLocation(SourceLocation Loc) const
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleComment(Token &result, SourceRange Comment)
HeaderSearch & getHeaderSearchInfo() const
ExternalPreprocessorSource * getExternalSource() const
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
IdentifierTable & getIdentifierTable()
const LangOptions & getLangOpts() const
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void PoisonSEHIdentifiers(bool Poison=true)
size_t getTotalMemory() const
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
DiagnosticsEngine & getDiagnostics() const
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop.
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens.
Encodes a location in the source.
bool isValid() const
Return true if this is a valid SourceLocation object.
void print(raw_ostream &OS, const SourceManager &SM) const
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
This class handles loading and caching of source files into memory.
void overrideFileContents(const FileEntry *SourceFile, const llvm::MemoryBufferRef &Buffer)
Override the contents of the given source file by providing an already-allocated buffer.
FileID getMainFileID() const
Returns the FileID of the main source file.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer.
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
std::optional< llvm::MemoryBufferRef > getMemoryBufferForFileOrNone(const FileEntry *File)
Retrieve the memory buffer associated with the given file.
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
SourceLocation createExpansionLoc(SourceLocation SpellingLoc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned Length, bool ExpansionIsTokenRange=true, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Creates an expansion SLocEntry for a macro use.
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
A trivial tuple used to represent a source range.
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
Exposes information about the current target.
Token - This structure provides full information about a lexed token.
IdentifierInfo * getIdentifierInfo() const
void setLiteralData(const char *Ptr)
bool hasUCN() const
Returns true if this token contains a universal character name.
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
unsigned getLength() const
void setLength(unsigned Len)
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6....
void setKind(tok::TokenKind K)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
tok::TokenKind getKind() const
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
void setLocation(SourceLocation L)
bool hasLeadingEmptyMacro() const
Return true if this token has an empty macro before it.
void setRawIdentifierData(const char *Ptr)
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
bool isNot(tok::TokenKind K) const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
void startToken()
Reset all flags to cleared.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
void setIdentifierInfo(IdentifierInfo *II)
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Defines the clang::TargetInfo interface.
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
@ Result
The result type of a method or function.
TranslationUnitKind
Describes the kind of translation unit being processed.
@ TU_Prefix
The translation unit is a prefix to a translation unit, and is not complete.