21#include "llvm/ADT/StringSwitch.h"
22#include "llvm/Support/FileSystem.h"
23#include "llvm/Support/MemoryBufferRef.h"
24#include "llvm/Support/Path.h"
37 return IncludeMacroStack.empty();
40 assert(IsFileLexer(IncludeMacroStack[0]) &&
41 "Top level include stack isn't our primary lexer?");
43 llvm::drop_begin(IncludeMacroStack),
44 [&](
const IncludeStackInfo &ISI) ->
bool {
return IsFileLexer(ISI); });
55 for (
const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
57 return ISI.ThePPLexer;
71 bool IsFirstIncludeOfFile) {
72 assert(!CurTokenLexer &&
"Cannot #include a file inside a macro!");
73 ++NumEnteredSourceFiles;
75 if (MaxIncludeStackDepth < IncludeMacroStack.size())
76 MaxIncludeStackDepth = IncludeMacroStack.size();
79 std::optional<llvm::MemoryBufferRef> InputFile =
83 Diag(Loc, diag::err_pp_error_opening_file)
95 Lexer *TheLexer =
new Lexer(FID, *InputFile, *
this, IsFirstIncludeOfFile);
97 FID != PredefinesFileID) {
102 TheLexer->DepDirectives = *DepDirectives;
107 EnterSourceFileWithLexer(TheLexer, CurDir);
113void Preprocessor::EnterSourceFileWithLexer(
Lexer *TheLexer,
118 if (CurPPLexer || CurTokenLexer)
119 PushIncludeMacroStack();
121 CurLexer.reset(TheLexer);
122 CurPPLexer = TheLexer;
123 CurDirLookup = CurDir;
124 CurLexerSubmodule =
nullptr;
125 if (CurLexerCallback != CLK_LexAfterModuleImport)
126 CurLexerCallback = TheLexer->isDependencyDirectivesLexer()
127 ? CLK_DependencyDirectivesLexer
131 if (Callbacks && !CurLexer->Is_PragmaLexer) {
143 Callbacks->LexedFileChanged(CurLexer->getFileID(),
145 FileType, PrevFID, EnterLoc);
153 std::unique_ptr<TokenLexer> TokLexer;
154 if (NumCachedTokenLexers == 0) {
155 TokLexer = std::make_unique<TokenLexer>(Tok, ILEnd, Macro, Args, *
this);
157 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
158 TokLexer->Init(Tok, ILEnd, Macro, Args);
161 PushIncludeMacroStack();
162 CurDirLookup =
nullptr;
163 CurTokenLexer = std::move(TokLexer);
164 if (CurLexerCallback != CLK_LexAfterModuleImport)
165 CurLexerCallback = CLK_TokenLexer;
180void Preprocessor::EnterTokenStream(
const Token *Toks,
unsigned NumToks,
181 bool DisableMacroExpansion,
bool OwnsTokens,
183 if (CurLexerCallback == CLK_CachingLexer) {
185 assert(IsReinject &&
"new tokens in the middle of cached stream");
189 Toks, Toks + NumToks);
197 ExitCachingLexMode();
198 EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
200 EnterCachingLexMode();
205 std::unique_ptr<TokenLexer> TokLexer;
206 if (NumCachedTokenLexers == 0) {
207 TokLexer = std::make_unique<TokenLexer>(
208 Toks, NumToks, DisableMacroExpansion, OwnsTokens, IsReinject, *
this);
210 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
211 TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
216 PushIncludeMacroStack();
217 CurDirLookup =
nullptr;
218 CurTokenLexer = std::move(TokLexer);
219 if (CurLexerCallback != CLK_LexAfterModuleImport)
220 CurLexerCallback = CLK_TokenLexer;
229 StringRef FilePath =
File.getDir().getName();
230 StringRef Path = FilePath;
231 while (!Path.empty()) {
233 if (*CurDir == Dir) {
234 Result = FilePath.substr(Path.size());
235 llvm::sys::path::append(
Result,
236 llvm::sys::path::filename(
File.getName()));
241 Path = llvm::sys::path::parent_path(Path);
247void Preprocessor::PropagateLineStartLeadingSpaceInfo(
Token &
Result) {
249 CurTokenLexer->PropagateLineStartLeadingSpaceInfo(
Result);
253 CurLexer->PropagateLineStartLeadingSpaceInfo(
Result);
266const char *Preprocessor::getCurLexerEndPos() {
267 const char *EndPos = CurLexer->BufferEnd;
268 if (EndPos != CurLexer->BufferStart &&
269 (EndPos[-1] ==
'\n' || EndPos[-1] ==
'\r')) {
273 if (EndPos != CurLexer->BufferStart &&
274 (EndPos[-1] ==
'\n' || EndPos[-1] ==
'\r') &&
275 EndPos[-1] != EndPos[0])
285 SubMods.push_back(&Mod);
290void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(
const Module &Mod) {
291 std::optional<Module::Header> UmbrellaHeader =
293 assert(UmbrellaHeader &&
"Module must use umbrella header");
296 if (
getDiagnostics().isIgnored(diag::warn_uncovered_module_header,
304 for (llvm::vfs::recursive_directory_iterator Entry(FS, Dir->
getName(), EC),
306 Entry != End && !EC; Entry.increment(EC)) {
307 using llvm::StringSwitch;
311 if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->path()))
312 .Cases(
".h",
".H",
".hh",
".hpp",
true)
316 if (
auto Header =
getFileManager().getOptionalFileRef(Entry->path()))
322 Diag(ExpectedHeadersLoc, diag::warn_uncovered_module_header)
333 assert(!CurTokenLexer &&
334 "Ending a file when currently in a macro!");
338 if (IncludeMacroStack.empty() &&
342 Diag(UnclosedSafeBufferOptOutLoc,
343 diag::err_pp_unclosed_pragma_unsafe_buffer_usage);
347 const bool LeavingSubmodule = CurLexer && CurLexerSubmodule;
348 if ((LeavingSubmodule || IncludeMacroStack.empty()) &&
349 !BuildingSubmoduleStack.empty() &&
350 BuildingSubmoduleStack.back().IsPragma) {
351 Diag(BuildingSubmoduleStack.back().ImportLoc,
352 diag::err_pp_module_begin_without_module_end);
356 const char *EndPos = getCurLexerEndPos();
357 CurLexer->BufferPtr = EndPos;
358 CurLexer->FormTokenWithChars(
Result, EndPos, tok::annot_module_end);
360 Result.setAnnotationValue(M);
373 MI->setUsedForHeaderGuard(
true);
377 DefinedMacro != ControllingMacro &&
378 CurLexer->isFirstTimeLexingFile()) {
386 const StringRef ControllingMacroName = ControllingMacro->getName();
387 const StringRef DefinedMacroName = DefinedMacro->getName();
388 const size_t MaxHalfLength = std::max(ControllingMacroName.size(),
389 DefinedMacroName.size()) / 2;
390 const unsigned ED = ControllingMacroName.edit_distance(
391 DefinedMacroName,
true, MaxHalfLength);
392 if (ED <= MaxHalfLength) {
395 diag::warn_header_guard)
398 diag::note_header_guard)
403 ControllingMacro->getName());
414 if (PragmaARCCFCodeAuditedInfo.second.isValid() && !isEndOfMacro &&
415 !(CurLexer && CurLexer->Is_PragmaLexer)) {
416 Diag(PragmaARCCFCodeAuditedInfo.second,
417 diag::err_pp_eof_in_arc_cf_code_audited);
426 if (PragmaAssumeNonNullLoc.
isValid() &&
427 !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
432 PreambleRecordedPragmaAssumeNonNullLoc = PragmaAssumeNonNullLoc;
434 Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
439 bool LeavingPCHThroughHeader =
false;
443 if (!IncludeMacroStack.empty()) {
448 CodeCompletionFileLoc) {
449 assert(CurLexer &&
"Got EOF but no current lexer set!");
451 CurLexer->FormTokenWithChars(
Result, CurLexer->BufferEnd, tok::eof);
454 CurPPLexer =
nullptr;
459 if (!isEndOfMacro && CurPPLexer &&
463 CurPPLexer->
getFileID() == PredefinesFileID))) {
472 bool ExitedFromPredefinesFile =
false;
474 if (!isEndOfMacro && CurPPLexer) {
477 assert(PredefinesFileID.
isValid() &&
478 "HandleEndOfFile is called before PredefinesFileId is set");
479 ExitedFromPredefinesFile = (PredefinesFileID == ExitedFID);
482 if (LeavingSubmodule) {
487 const char *EndPos = getCurLexerEndPos();
489 CurLexer->BufferPtr = EndPos;
490 CurLexer->FormTokenWithChars(
Result, EndPos, tok::annot_module_end);
492 Result.setAnnotationValue(M);
495 bool FoundPCHThroughHeader =
false;
499 FoundPCHThroughHeader =
true;
505 PropagateLineStartLeadingSpaceInfo(
Result);
508 if (Callbacks && !isEndOfMacro && CurPPLexer) {
513 Callbacks->LexedFileChanged(CurPPLexer->
getFileID(),
515 FileType, ExitedFID, Loc);
521 if (ExitedFromPredefinesFile) {
522 replayPreambleConditionalStack();
523 if (PreambleRecordedPragmaAssumeNonNullLoc.
isValid())
524 PragmaAssumeNonNullLoc = PreambleRecordedPragmaAssumeNonNullLoc;
527 if (!isEndOfMacro && CurPPLexer && FoundPCHThroughHeader &&
532 LeavingPCHThroughHeader =
true;
535 return LeavingSubmodule;
539 assert(CurLexer &&
"Got EOF but no current lexer set!");
540 const char *EndPos = getCurLexerEndPos();
542 CurLexer->BufferPtr = EndPos;
545 CurLexer->FormTokenWithChars(
Result, EndPos, tok::annot_repl_input_end);
547 Result.setAnnotationValue(
nullptr);
549 CurLexer->FormTokenWithChars(
Result, EndPos, tok::eof);
559 if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
560 Result.setLocation(
Result.getLocation().getLocWithOffset(-1));
565 Diag(CurLexer->getFileLoc(), diag::err_pp_through_header_not_seen)
566 << PPOpts->PCHThroughHeader << 0;
574 CurPPLexer =
nullptr;
580 for (WarnUnusedMacroLocsTy::iterator
581 I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
583 Diag(*I, diag::pp_macro_not_used);
593 for (
auto *M : AllMods)
594 diagnoseMissingHeaderInUmbrellaDir(*M);
603 assert(CurTokenLexer && !CurPPLexer &&
604 "Ending a macro when currently in a #include file!");
606 if (!MacroExpandingLexersStack.empty() &&
607 MacroExpandingLexersStack.back().first == CurTokenLexer.get())
608 removeCachedMacroExpandedTokensOfLastLexer();
611 if (NumCachedTokenLexers == TokenLexerCacheSize)
612 CurTokenLexer.reset();
614 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
624 assert(!IncludeMacroStack.empty() &&
"Ran out of stack entries to load");
628 if (NumCachedTokenLexers == TokenLexerCacheSize)
629 CurTokenLexer.reset();
631 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
634 PopIncludeMacroStack();
641 assert(CurTokenLexer && !CurPPLexer &&
642 "Pasted comment can only be formed from macro");
647 bool LexerWasInPPMode =
false;
648 for (
const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
649 if (ISI.ThePPLexer ==
nullptr)
continue;
657 FoundLexer = ISI.ThePPLexer;
675 while (Tok.
isNot(tok::eod) && Tok.
isNot(tok::eof))
679 if (Tok.
is(tok::eod)) {
680 assert(FoundLexer &&
"Can't get end of line without an active lexer");
686 if (LexerWasInPPMode)
return;
697 assert(!FoundLexer &&
"Lexer should return EOD before EOF in PP mode");
704 BuildingSubmoduleStack.push_back(
705 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
706 PendingModuleMacroNames.size()));
708 Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma);
722 auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
723 auto &State = R.first->second;
724 bool FirstTime = R.second;
732 auto &StartingMacros = NullSubmoduleState.Macros;
736 for (
auto &Macro : StartingMacros) {
738 if (!Macro.second.getLatest() &&
739 Macro.second.getOverriddenMacros().empty())
742 MacroState MS(Macro.second.getLatest());
743 MS.setOverriddenMacros(*
this, Macro.second.getOverriddenMacros());
744 State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
749 BuildingSubmoduleStack.push_back(
750 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
751 PendingModuleMacroNames.size()));
754 Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma);
757 CurSubmoduleState = &State;
764bool Preprocessor::needModuleMacros()
const {
766 if (BuildingSubmoduleStack.empty())
778 if (BuildingSubmoduleStack.empty() ||
779 BuildingSubmoduleStack.back().IsPragma != ForPragma) {
780 assert(ForPragma &&
"non-pragma module enter/leave mismatch");
784 auto &Info = BuildingSubmoduleStack.back();
786 Module *LeavingMod = Info.M;
789 if (!needModuleMacros() ||
795 BuildingSubmoduleStack.pop_back();
798 Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma);
806 for (
unsigned I = Info.OuterPendingModuleMacroNames;
807 I != PendingModuleMacroNames.size(); ++I) {
808 auto *II =
const_cast<IdentifierInfo*
>(PendingModuleMacroNames[I]);
809 if (!VisitedMacros.insert(II).second)
812 auto MacroIt = CurSubmoduleState->Macros.find(II);
813 if (MacroIt == CurSubmoduleState->Macros.end())
815 auto &Macro = MacroIt->second;
819 auto *OldState = Info.OuterSubmoduleState;
821 OldState = &NullSubmoduleState;
822 if (OldState && OldState != CurSubmoduleState) {
825 auto &OldMacros = OldState->Macros;
826 auto OldMacroIt = OldMacros.find(II);
827 if (OldMacroIt == OldMacros.end())
830 OldMD = OldMacroIt->second.getLatest();
835 bool ExplicitlyPublic =
false;
836 for (
auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->
getPrevious()) {
837 assert(MD &&
"broken macro directive chain");
839 if (
auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
842 if (VisMD->isPublic())
843 ExplicitlyPublic =
true;
844 else if (!ExplicitlyPublic)
850 Def = DefMD->getInfo();
857 if (Def || !Macro.getOverriddenMacros().empty())
859 Macro.getOverriddenMacros(), IsNew);
864 Macro.setLatest(
nullptr);
865 Macro.setOverriddenMacros(*
this, {});
871 PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames);
880 CurSubmoduleState = Info.OuterSubmoduleState;
882 BuildingSubmoduleStack.pop_back();
885 Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma);
Defines the clang::FileManager interface and associated types.
Defines the clang::MacroInfo and clang::MacroDirective classes.
static void collectAllSubModulesWithUmbrellaHeader(const Module &Mod, SmallVectorImpl< const Module * > &SubMods)
static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir, FileEntryRef File, SmallString< 128 > &Result)
Compute the relative path that names the given file relative to the given directory.
Defines the clang::Preprocessor interface.
Defines the SourceManager interface.
A directive for a defined macro or a macro imported from a module.
StringRef getName() const
Cached information about one directory (either on disk or in the virtual file system).
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Implements support for file system lookup, file system caching, and directory search management.
llvm::vfs::FileSystem & getVirtualFileSystem() const
llvm::ErrorOr< const DirectoryEntry * > getDirectory(StringRef DirName, bool CacheFailure=true)
Lookup, cache, and verify the specified directory (real or virtual).
static FixItHint CreateReplacement(CharSourceRange RemoveRange, StringRef Code)
Create a code modification hint that replaces the given source range with the given code string.
One of these records is kept for each identifier that is lexed.
bool isCompilingModule() const
Are we compiling a module?
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Encapsulates changes to the "macros namespace" (the location where the macro name became active,...
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Encapsulates the data about a macro definition (e.g.
bool resolveExports(Module *Mod, bool Complain)
Resolve all of the unresolved exports in the given module.
bool resolveConflicts(Module *Mod, bool Complain)
Resolve all of the unresolved conflicts in the given module.
bool isHeaderInUnavailableModule(FileEntryRef Header) const
Determine whether the given header is part of a module marked 'unavailable'.
bool resolveUses(Module *Mod, bool Complain)
Resolve all of the unresolved uses in the given module.
Describes a module or submodule.
StringRef getTopLevelModuleName() const
Retrieve the name of the top-level module.
llvm::iterator_range< submodule_iterator > submodules()
std::optional< Header > getUmbrellaHeaderAsWritten() const
Retrieve the umbrella header as written.
std::string getFullModuleName(bool AllowStringLiterals=false) const
Retrieve the full name of this module, including the path from its top-level module.
OptionalDirectoryEntryRef getEffectiveUmbrellaDir() const
Get the effective umbrella directory for this module: either the one explicitly written in the module...
const IdentifierInfo * GetDefinedMacro() const
If the ControllingMacro is followed by a macro definition, return the macro that was defined.
const IdentifierInfo * GetControllingMacroAtEndOfFile() const
Once the entire file has been lexed, if there is a controlling macro, return it.
SourceLocation GetDefinedLocation() const
SourceLocation GetMacroLocation() const
OptionalFileEntryRefDegradesToFileEntryPtr getFileEntry() const
getFileEntry - Return the FileEntry corresponding to this FileID.
unsigned getInitialNumSLocEntries() const
Number of SLocEntries before lexing the file.
bool LexingRawMode
True if in raw mode.
bool ParsingPreprocessorDirective
True when parsing #XXX; turns '\n' into a tok::eod token.
MultipleIncludeOpt MIOpt
A state machine that detects the #ifndef-wrapping a file idiom for the multiple-include optimization.
virtual SourceLocation getSourceLocation()=0
Return the source location for the next observable location.
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
bool isRecordingPreamble() const
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
void makeModuleVisible(Module *M, SourceLocation Loc)
void Lex(Token &Result)
Lex the next token for this preprocessor.
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
SourceManager & getSourceManager() const
bool isMacroDefined(StringRef Id)
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
bool isPPInSafeBufferOptOutRegion()
FileManager & getFileManager() const
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
HeaderSearch & getHeaderSearchInfo() const
Module * LeaveSubmodule(bool ForPragma)
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
const LangOptions & getLangOpts() const
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
DiagnosticsEngine & getDiagnostics() const
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
Encodes a location in the source.
bool isValid() const
Return true if this is a valid SourceLocation object.
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
OptionalFileEntryRef getFileEntryRefForID(FileID FID) const
Returns the FileEntryRef for the provided FileID.
StringRef getBufferName(SourceLocation Loc, bool *Invalid=nullptr) const
Return the filename or buffer identifier of the buffer the location is in.
FileID translateFile(const FileEntry *SourceFile) const
Get the FileID for the given file.
unsigned local_sloc_entry_size() const
Get the number of local SLocEntries we have.
SourceLocation getLocForEndOfFile(FileID FID) const
Return the source location corresponding to the last byte of the specified file.
SourceLocation getIncludeLoc(FileID FID) const
Returns the include location if FID is a #include'd file otherwise it returns an invalid location.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file.
SrcMgr::CharacteristicKind getFileCharacteristic(SourceLocation Loc) const
Return the file characteristic of the specified source location, indicating whether this is a normal ...
void setNumCreatedFIDsForFileID(FileID FID, unsigned NumFIDs, bool Force=false) const
Set the number of FileIDs (files and macros) that were created during preprocessing of FID,...
std::optional< llvm::MemoryBufferRef > getBufferOrNone(FileID FID, SourceLocation Loc=SourceLocation()) const
Return the buffer for the specified FileID.
Token - This structure provides full information about a lexed token.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
bool isNot(tok::TokenKind K) const
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
@ Result
The result type of a method or function.
@ TU_Complete
The translation unit is a complete translation unit.