21 #include "llvm/ADT/StringSwitch.h" 22 #include "llvm/Support/FileSystem.h" 23 #include "llvm/Support/MemoryBuffer.h" 24 #include "llvm/Support/Path.h" 25 using namespace clang;
37 return IncludeMacroStack.empty();
40 assert(IsFileLexer(IncludeMacroStack[0]) &&
41 "Top level include stack isn't our primary lexer?");
43 IncludeMacroStack.begin() + 1, IncludeMacroStack.end(),
44 [&](
const IncludeStackInfo &ISI) ->
bool {
return IsFileLexer(ISI); });
55 for (
const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
57 return ISI.ThePPLexer;
71 assert(!CurTokenLexer &&
"Cannot #include a file inside a macro!");
72 ++NumEnteredSourceFiles;
74 if (MaxIncludeStackDepth < IncludeMacroStack.size())
75 MaxIncludeStackDepth = IncludeMacroStack.size();
79 const llvm::MemoryBuffer *InputFile =
80 getSourceManager().getBuffer(FID, Loc, &Invalid);
83 Diag(Loc, diag::err_pp_error_opening_file)
84 << std::string(SourceMgr.getBufferName(FileStart)) <<
"";
88 if (isCodeCompletionEnabled() &&
89 SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
90 CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
92 CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
95 EnterSourceFileWithLexer(
new Lexer(FID, InputFile, *
this), CurDir);
101 void Preprocessor::EnterSourceFileWithLexer(
Lexer *TheLexer,
105 if (CurPPLexer || CurTokenLexer)
106 PushIncludeMacroStack();
108 CurLexer.reset(TheLexer);
109 CurPPLexer = TheLexer;
110 CurDirLookup = CurDir;
111 CurLexerSubmodule =
nullptr;
112 if (CurLexerKind != CLK_LexAfterModuleImport)
113 CurLexerKind = CLK_Lexer;
116 if (Callbacks && !CurLexer->Is_PragmaLexer) {
118 SourceMgr.getFileCharacteristic(CurLexer->getFileLoc());
120 Callbacks->FileChanged(CurLexer->getFileLoc(),
129 std::unique_ptr<TokenLexer> TokLexer;
130 if (NumCachedTokenLexers == 0) {
131 TokLexer = std::make_unique<TokenLexer>(
Tok, ILEnd, Macro, Args, *
this);
133 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
134 TokLexer->Init(Tok, ILEnd, Macro, Args);
137 PushIncludeMacroStack();
138 CurDirLookup =
nullptr;
139 CurTokenLexer = std::move(TokLexer);
140 if (CurLexerKind != CLK_LexAfterModuleImport)
141 CurLexerKind = CLK_TokenLexer;
156 void Preprocessor::EnterTokenStream(
const Token *Toks,
unsigned NumToks,
157 bool DisableMacroExpansion,
bool OwnsTokens,
159 if (CurLexerKind == CLK_CachingLexer) {
161 assert(IsReinject &&
"new tokens in the middle of cached stream");
165 Toks, Toks + NumToks);
173 ExitCachingLexMode();
174 EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
176 EnterCachingLexMode();
181 std::unique_ptr<TokenLexer> TokLexer;
182 if (NumCachedTokenLexers == 0) {
183 TokLexer = std::make_unique<TokenLexer>(
184 Toks, NumToks, DisableMacroExpansion, OwnsTokens, IsReinject, *
this);
186 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
187 TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
192 PushIncludeMacroStack();
193 CurDirLookup =
nullptr;
194 CurTokenLexer = std::move(TokLexer);
195 if (CurLexerKind != CLK_LexAfterModuleImport)
196 CurLexerKind = CLK_TokenLexer;
207 StringRef Path = FilePath;
208 while (!Path.empty()) {
210 if (*CurDir == Dir) {
211 Result = FilePath.substr(Path.size());
212 llvm::sys::path::append(Result,
213 llvm::sys::path::filename(File->
getName()));
218 Path = llvm::sys::path::parent_path(Path);
224 void Preprocessor::PropagateLineStartLeadingSpaceInfo(
Token &
Result) {
226 CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result);
230 CurLexer->PropagateLineStartLeadingSpaceInfo(Result);
243 const char *Preprocessor::getCurLexerEndPos() {
244 const char *EndPos = CurLexer->BufferEnd;
245 if (EndPos != CurLexer->BufferStart &&
246 (EndPos[-1] ==
'\n' || EndPos[-1] ==
'\r')) {
250 if (EndPos != CurLexer->BufferStart &&
251 (EndPos[-1] ==
'\n' || EndPos[-1] ==
'\r') &&
252 EndPos[-1] != EndPos[0])
262 SubMods.push_back(&Mod);
267 void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(
const Module &Mod) {
270 SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());
271 if (getDiagnostics().isIgnored(diag::warn_uncovered_module_header, StartLoc))
274 ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
276 llvm::vfs::FileSystem &FS = FileMgr.getVirtualFileSystem();
278 for (llvm::vfs::recursive_directory_iterator Entry(FS, Dir->
getName(), EC),
280 Entry !=
End && !EC; Entry.increment(EC)) {
281 using llvm::StringSwitch;
285 if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->path()))
286 .Cases(
".h",
".H",
".hh",
".hpp",
true)
290 if (
auto Header = getFileManager().getFile(Entry->path()))
291 if (!getSourceManager().hasFileInfo(*Header)) {
296 Diag(StartLoc, diag::warn_uncovered_module_header)
307 assert(!CurTokenLexer &&
308 "Ending a file when currently in a macro!");
312 const bool LeavingSubmodule = CurLexer && CurLexerSubmodule;
313 if ((LeavingSubmodule || IncludeMacroStack.empty()) &&
314 !BuildingSubmoduleStack.empty() &&
315 BuildingSubmoduleStack.back().IsPragma) {
316 Diag(BuildingSubmoduleStack.back().ImportLoc,
317 diag::err_pp_module_begin_without_module_end);
318 Module *M = LeaveSubmodule(
true);
321 const char *EndPos = getCurLexerEndPos();
322 CurLexer->BufferPtr = EndPos;
323 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
332 CurPPLexer->MIOpt.GetControllingMacroAtEndOfFile()) {
334 if (
const FileEntry *FE = CurPPLexer->getFileEntry()) {
335 HeaderInfo.SetFileControllingMacro(FE, ControllingMacro);
337 getMacroInfo(const_cast<IdentifierInfo*>(ControllingMacro)))
338 MI->setUsedForHeaderGuard(
true);
340 CurPPLexer->MIOpt.GetDefinedMacro()) {
342 DefinedMacro != ControllingMacro &&
343 HeaderInfo.FirstTimeLexingFile(FE)) {
351 const StringRef ControllingMacroName = ControllingMacro->getName();
352 const StringRef DefinedMacroName = DefinedMacro->getName();
353 const size_t MaxHalfLength =
std::max(ControllingMacroName.size(),
354 DefinedMacroName.size()) / 2;
355 const unsigned ED = ControllingMacroName.edit_distance(
356 DefinedMacroName,
true, MaxHalfLength);
357 if (ED <= MaxHalfLength) {
359 Diag(CurPPLexer->MIOpt.GetMacroLocation(),
360 diag::warn_header_guard)
361 << CurPPLexer->MIOpt.GetMacroLocation() << ControllingMacro;
362 Diag(CurPPLexer->MIOpt.GetDefinedLocation(),
363 diag::note_header_guard)
364 << CurPPLexer->MIOpt.GetDefinedLocation() << DefinedMacro
367 CurPPLexer->MIOpt.GetDefinedLocation(),
368 ControllingMacro->getName());
379 if (PragmaARCCFCodeAuditedInfo.second.isValid() && !isEndOfMacro &&
380 !(CurLexer && CurLexer->Is_PragmaLexer)) {
381 Diag(PragmaARCCFCodeAuditedInfo.second,
382 diag::err_pp_eof_in_arc_cf_code_audited);
391 if (PragmaAssumeNonNullLoc.isValid() &&
392 !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
393 Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
399 bool LeavingPCHThroughHeader =
false;
403 if (!IncludeMacroStack.empty()) {
406 if (isCodeCompletionEnabled() && CurPPLexer &&
407 SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
408 CodeCompletionFileLoc) {
409 assert(CurLexer &&
"Got EOF but no current lexer set!");
411 CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd,
tok::eof);
414 CurPPLexer =
nullptr;
415 recomputeCurLexerKind();
419 if (!isEndOfMacro && CurPPLexer &&
420 SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid()) {
424 SourceMgr.local_sloc_entry_size() -
425 CurPPLexer->getInitialNumSLocEntries() + 1;
426 SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
429 bool ExitedFromPredefinesFile =
false;
431 if (!isEndOfMacro && CurPPLexer) {
432 ExitedFID = CurPPLexer->getFileID();
434 assert(PredefinesFileID.isValid() &&
435 "HandleEndOfFile is called before PredefinesFileId is set");
436 ExitedFromPredefinesFile = (PredefinesFileID == ExitedFID);
439 if (LeavingSubmodule) {
441 Module *M = LeaveSubmodule(
false);
444 const char *EndPos = getCurLexerEndPos();
446 CurLexer->BufferPtr = EndPos;
447 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
452 bool FoundPCHThroughHeader =
false;
453 if (CurPPLexer && creatingPCHWithThroughHeader() &&
455 SourceMgr.getFileEntryForID(CurPPLexer->getFileID())))
456 FoundPCHThroughHeader =
true;
459 RemoveTopOfLexerStack();
462 PropagateLineStartLeadingSpaceInfo(Result);
465 if (Callbacks && !isEndOfMacro && CurPPLexer) {
467 SourceMgr.getFileCharacteristic(CurPPLexer->getSourceLocation());
468 Callbacks->FileChanged(CurPPLexer->getSourceLocation(),
474 if (ExitedFromPredefinesFile)
475 replayPreambleConditionalStack();
477 if (!isEndOfMacro && CurPPLexer && FoundPCHThroughHeader &&
478 (isInPrimaryFile() ||
479 CurPPLexer->getFileID() == getPredefinesFileID())) {
482 LeavingPCHThroughHeader =
true;
485 return LeavingSubmodule;
490 assert(CurLexer &&
"Got EOF but no current lexer set!");
491 const char *EndPos = getCurLexerEndPos();
493 CurLexer->BufferPtr = EndPos;
494 CurLexer->FormTokenWithChars(Result, EndPos,
tok::eof);
496 if (isCodeCompletionEnabled()) {
503 if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
507 if (creatingPCHWithThroughHeader() && !LeavingPCHThroughHeader) {
509 Diag(CurLexer->getFileLoc(), diag::err_pp_through_header_not_seen)
510 << PPOpts->PCHThroughHeader << 0;
513 if (!isIncrementalProcessingEnabled())
517 if (!isIncrementalProcessingEnabled())
518 CurPPLexer =
nullptr;
524 for (WarnUnusedMacroLocsTy::iterator
525 I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
527 Diag(*I, diag::pp_macro_not_used);
534 if (
Module *Mod = getCurrentModule()) {
537 for (
auto *M : AllMods)
538 diagnoseMissingHeaderInUmbrellaDir(*M);
547 assert(CurTokenLexer && !CurPPLexer &&
548 "Ending a macro when currently in a #include file!");
550 if (!MacroExpandingLexersStack.empty() &&
551 MacroExpandingLexersStack.back().first == CurTokenLexer.get())
552 removeCachedMacroExpandedTokensOfLastLexer();
555 if (NumCachedTokenLexers == TokenLexerCacheSize)
556 CurTokenLexer.reset();
558 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
561 return HandleEndOfFile(Result,
true);
568 assert(!IncludeMacroStack.empty() &&
"Ran out of stack entries to load");
572 if (NumCachedTokenLexers == TokenLexerCacheSize)
573 CurTokenLexer.reset();
575 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
578 PopIncludeMacroStack();
585 assert(CurTokenLexer && !CurPPLexer &&
586 "Pasted comment can only be formed from macro");
591 bool LexerWasInPPMode =
false;
592 for (
const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
593 if (ISI.ThePPLexer ==
nullptr)
continue;
601 FoundLexer = ISI.ThePPLexer;
611 if (!HandleEndOfTokenLexer(Tok)) Lex(Tok);
623 if (Tok.
is(tok::eod)) {
624 assert(FoundLexer &&
"Can't get end of line without an active lexer");
630 if (LexerWasInPPMode)
return;
641 assert(!FoundLexer &&
"Lexer should return EOD before EOF in PP mode");
646 if (!getLangOpts().ModulesLocalVisibility) {
648 BuildingSubmoduleStack.push_back(
649 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
650 PendingModuleMacroNames.size()));
652 Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma);
660 ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
666 auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
667 auto &
State = R.first->second;
668 bool FirstTime = R.second;
676 auto &StartingMacros = NullSubmoduleState.Macros;
680 for (
auto &Macro : StartingMacros) {
682 if (!Macro.second.getLatest() &&
683 Macro.second.getOverriddenMacros().empty())
686 MacroState MS(Macro.second.getLatest());
687 MS.setOverriddenMacros(*
this, Macro.second.getOverriddenMacros());
688 State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
693 BuildingSubmoduleStack.push_back(
694 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
695 PendingModuleMacroNames.size()));
698 Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma);
701 CurSubmoduleState = &
State;
705 makeModuleVisible(M, ImportLoc);
708 bool Preprocessor::needModuleMacros()
const {
710 if (BuildingSubmoduleStack.empty())
714 if (getLangOpts().ModulesLocalVisibility)
718 return getLangOpts().isCompilingModule();
722 if (BuildingSubmoduleStack.empty() ||
723 BuildingSubmoduleStack.back().IsPragma != ForPragma) {
724 assert(ForPragma &&
"non-pragma module enter/leave mismatch");
728 auto &Info = BuildingSubmoduleStack.back();
730 Module *LeavingMod = Info.M;
733 if (!needModuleMacros() ||
734 (!getLangOpts().ModulesLocalVisibility &&
739 BuildingSubmoduleStack.pop_back();
742 Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma);
744 makeModuleVisible(LeavingMod, ImportLoc);
749 llvm::SmallPtrSet<const IdentifierInfo*, 8> VisitedMacros;
750 for (
unsigned I = Info.OuterPendingModuleMacroNames;
751 I != PendingModuleMacroNames.size(); ++I) {
752 auto *II =
const_cast<IdentifierInfo*
>(PendingModuleMacroNames[I]);
753 if (!VisitedMacros.insert(II).second)
756 auto MacroIt = CurSubmoduleState->Macros.find(II);
757 if (MacroIt == CurSubmoduleState->Macros.end())
759 auto &Macro = MacroIt->second;
763 auto *OldState = Info.OuterSubmoduleState;
764 if (getLangOpts().ModulesLocalVisibility)
765 OldState = &NullSubmoduleState;
766 if (OldState && OldState != CurSubmoduleState) {
769 auto &OldMacros = OldState->Macros;
770 auto OldMacroIt = OldMacros.find(II);
771 if (OldMacroIt == OldMacros.end())
774 OldMD = OldMacroIt->second.getLatest();
779 bool ExplicitlyPublic =
false;
780 for (
auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->
getPrevious()) {
781 assert(MD &&
"broken macro directive chain");
783 if (
auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
786 if (VisMD->isPublic())
787 ExplicitlyPublic =
true;
788 else if (!ExplicitlyPublic)
794 Def = DefMD->getInfo();
801 if (Def || !Macro.getOverriddenMacros().empty())
802 addModuleMacro(LeavingMod, II, Def,
803 Macro.getOverriddenMacros(), IsNew);
805 if (!getLangOpts().ModulesLocalVisibility) {
808 Macro.setLatest(
nullptr);
809 Macro.setOverriddenMacros(*
this, {});
815 PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames);
823 if (getLangOpts().ModulesLocalVisibility)
824 CurSubmoduleState = Info.OuterSubmoduleState;
826 BuildingSubmoduleStack.pop_back();
829 Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma);
832 makeModuleVisible(LeavingMod, ImportLoc);
static DiagnosticBuilder Diag(DiagnosticsEngine *Diags, const LangOptions &Features, FullSourceLoc TokLoc, const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd, unsigned DiagID)
Produce a diagnostic highlighting some portion of a literal.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
Implements support for file system lookup, file system caching, and directory search management...
Defines the clang::FileManager interface and associated types.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Defines the SourceManager interface.
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool resolveUses(Module *Mod, bool Complain)
Resolve all of the unresolved uses in the given module.
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
One of these records is kept for each identifier that is lexed.
A directive for a defined macro or a macro imported from a module.
bool ParsingPreprocessorDirective
True when parsing #XXX; turns '\n' into a tok::eod token.
bool resolveConflicts(Module *Mod, bool Complain)
Resolve all of the unresolved conflicts in the given module.
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
Module * LeaveSubmodule(bool ForPragma)
Header getUmbrellaHeader() const
Retrieve the header that serves as the umbrella header for this module.
Token - This structure provides full information about a lexed token.
__DEVICE__ int max(int __a, int __b)
Describes a module or submodule.
StringRef getTopLevelModuleName() const
Retrieve the name of the top-level module.
llvm::ErrorOr< const DirectoryEntry * > getDirectory(StringRef DirName, bool CacheFailure=true)
Lookup, cache, and verify the specified directory (real or virtual).
std::string getFullModuleName(bool AllowStringLiterals=false) const
Retrieve the full name of this module, including the path from its top-level module.
const DirectoryEntry * Entry
void setAnnotationValue(void *val)
bool LexingRawMode
True if in raw mode.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
bool resolveExports(Module *Mod, bool Complain)
Resolve all of the unresolved exports in the given module.
static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir, const FileEntry *File, SmallString< 128 > &Result)
Compute the relative path that names the given file relative to the given directory.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
The result type of a method or function.
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
const DirectoryEntry * getDir() const
Return the directory the file lives in.
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Encodes a location in the source.
StringRef getName() const
void setAnnotationEndLoc(SourceLocation L)
Cached information about one file (either on disk or in the virtual file system). ...
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
Dataflow Directional Tag Classes.
llvm::iterator_range< submodule_iterator > submodules()
Encapsulates the data about a macro definition (e.g.
DirectoryName getUmbrellaDir() const
Retrieve the directory for which this module serves as the umbrella.
Cached information about one directory (either on disk or in the virtual file system).
static void collectAllSubModulesWithUmbrellaHeader(const Module &Mod, SmallVectorImpl< const Module *> &SubMods)
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
static FixItHint CreateReplacement(CharSourceRange RemoveRange, StringRef Code)
Create a code modification hint that replaces the given source range with the given code string...
The translation unit is a complete translation unit.
bool isHeaderInUnavailableModule(const FileEntry *Header) const
Determine whether the given header is part of a module marked 'unavailable'.
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
void setLocation(SourceLocation L)
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
void startToken()
Reset all flags to cleared.
StringRef getName() const