clang 17.0.0git
PPLexerChange.cpp
Go to the documentation of this file.
1//===--- PPLexerChange.cpp - Handle changing lexers in the preprocessor ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements pieces of the Preprocessor interface that manage the
10// current lexer stack.
11//
12//===----------------------------------------------------------------------===//
13
18#include "clang/Lex/MacroInfo.h"
21#include "llvm/ADT/StringSwitch.h"
22#include "llvm/Support/FileSystem.h"
23#include "llvm/Support/MemoryBufferRef.h"
24#include "llvm/Support/Path.h"
25#include <optional>
26
27using namespace clang;
28
29//===----------------------------------------------------------------------===//
30// Miscellaneous Methods.
31//===----------------------------------------------------------------------===//
32
33/// isInPrimaryFile - Return true if we're in the top-level file, not in a
34/// \#include. This looks through macro expansions and active _Pragma lexers.
36 if (IsFileLexer())
37 return IncludeMacroStack.empty();
38
39 // If there are any stacked lexers, we're in a #include.
40 assert(IsFileLexer(IncludeMacroStack[0]) &&
41 "Top level include stack isn't our primary lexer?");
42 return llvm::none_of(
43 llvm::drop_begin(IncludeMacroStack),
44 [&](const IncludeStackInfo &ISI) -> bool { return IsFileLexer(ISI); });
45}
46
47/// getCurrentLexer - Return the current file lexer being lexed from. Note
48/// that this ignores any potentially active macro expansions and _Pragma
49/// expansions going on at the time.
51 if (IsFileLexer())
52 return CurPPLexer;
53
54 // Look for a stacked lexer.
55 for (const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
56 if (IsFileLexer(ISI))
57 return ISI.ThePPLexer;
58 }
59 return nullptr;
60}
61
62
63//===----------------------------------------------------------------------===//
64// Methods for Entering and Callbacks for leaving various contexts
65//===----------------------------------------------------------------------===//
66
67/// EnterSourceFile - Add a source file to the top of the include stack and
68/// start lexing tokens from it instead of the current buffer.
71 bool IsFirstIncludeOfFile) {
72 assert(!CurTokenLexer && "Cannot #include a file inside a macro!");
73 ++NumEnteredSourceFiles;
74
75 if (MaxIncludeStackDepth < IncludeMacroStack.size())
76 MaxIncludeStackDepth = IncludeMacroStack.size();
77
78 // Get the MemoryBuffer for this FID, if it fails, we fail.
79 std::optional<llvm::MemoryBufferRef> InputFile =
81 if (!InputFile) {
82 SourceLocation FileStart = SourceMgr.getLocForStartOfFile(FID);
83 Diag(Loc, diag::err_pp_error_opening_file)
84 << std::string(SourceMgr.getBufferName(FileStart)) << "";
85 return true;
86 }
87
89 SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
90 CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
91 CodeCompletionLoc =
92 CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
93 }
94
95 Lexer *TheLexer = new Lexer(FID, *InputFile, *this, IsFirstIncludeOfFile);
96 if (getPreprocessorOpts().DependencyDirectivesForFile &&
97 FID != PredefinesFileID) {
98 if (OptionalFileEntryRef File = SourceMgr.getFileEntryRefForID(FID)) {
100 DepDirectives =
101 getPreprocessorOpts().DependencyDirectivesForFile(*File)) {
102 TheLexer->DepDirectives = *DepDirectives;
103 }
104 }
105 }
106
107 EnterSourceFileWithLexer(TheLexer, CurDir);
108 return false;
109}
110
111/// EnterSourceFileWithLexer - Add a source file to the top of the include stack
112/// and start lexing tokens from it instead of the current buffer.
113void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
114 ConstSearchDirIterator CurDir) {
115 PreprocessorLexer *PrevPPLexer = CurPPLexer;
116
117 // Add the current lexer to the include stack.
118 if (CurPPLexer || CurTokenLexer)
119 PushIncludeMacroStack();
120
121 CurLexer.reset(TheLexer);
122 CurPPLexer = TheLexer;
123 CurDirLookup = CurDir;
124 CurLexerSubmodule = nullptr;
125 if (CurLexerKind != CLK_LexAfterModuleImport)
126 CurLexerKind = TheLexer->isDependencyDirectivesLexer()
127 ? CLK_DependencyDirectivesLexer
128 : CLK_Lexer;
129
130 // Notify the client, if desired, that we are in a new source file.
131 if (Callbacks && !CurLexer->Is_PragmaLexer) {
133 SourceMgr.getFileCharacteristic(CurLexer->getFileLoc());
134
135 FileID PrevFID;
136 SourceLocation EnterLoc;
137 if (PrevPPLexer) {
138 PrevFID = PrevPPLexer->getFileID();
139 EnterLoc = PrevPPLexer->getSourceLocation();
140 }
141 Callbacks->FileChanged(CurLexer->getFileLoc(), PPCallbacks::EnterFile,
142 FileType, PrevFID);
143 Callbacks->LexedFileChanged(CurLexer->getFileID(),
145 FileType, PrevFID, EnterLoc);
146 }
147}
148
149/// EnterMacro - Add a Macro to the top of the include stack and start lexing
150/// tokens from it instead of the current buffer.
152 MacroInfo *Macro, MacroArgs *Args) {
153 std::unique_ptr<TokenLexer> TokLexer;
154 if (NumCachedTokenLexers == 0) {
155 TokLexer = std::make_unique<TokenLexer>(Tok, ILEnd, Macro, Args, *this);
156 } else {
157 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
158 TokLexer->Init(Tok, ILEnd, Macro, Args);
159 }
160
161 PushIncludeMacroStack();
162 CurDirLookup = nullptr;
163 CurTokenLexer = std::move(TokLexer);
164 if (CurLexerKind != CLK_LexAfterModuleImport)
165 CurLexerKind = CLK_TokenLexer;
166}
167
168/// EnterTokenStream - Add a "macro" context to the top of the include stack,
169/// which will cause the lexer to start returning the specified tokens.
170///
171/// If DisableMacroExpansion is true, tokens lexed from the token stream will
172/// not be subject to further macro expansion. Otherwise, these tokens will
173/// be re-macro-expanded when/if expansion is enabled.
174///
175/// If OwnsTokens is false, this method assumes that the specified stream of
176/// tokens has a permanent owner somewhere, so they do not need to be copied.
177/// If it is true, it assumes the array of tokens is allocated with new[] and
178/// must be freed.
179///
180void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
181 bool DisableMacroExpansion, bool OwnsTokens,
182 bool IsReinject) {
183 if (CurLexerKind == CLK_CachingLexer) {
184 if (CachedLexPos < CachedTokens.size()) {
185 assert(IsReinject && "new tokens in the middle of cached stream");
186 // We're entering tokens into the middle of our cached token stream. We
187 // can't represent that, so just insert the tokens into the buffer.
188 CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
189 Toks, Toks + NumToks);
190 if (OwnsTokens)
191 delete [] Toks;
192 return;
193 }
194
195 // New tokens are at the end of the cached token sequnece; insert the
196 // token stream underneath the caching lexer.
197 ExitCachingLexMode();
198 EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
199 IsReinject);
200 EnterCachingLexMode();
201 return;
202 }
203
204 // Create a macro expander to expand from the specified token stream.
205 std::unique_ptr<TokenLexer> TokLexer;
206 if (NumCachedTokenLexers == 0) {
207 TokLexer = std::make_unique<TokenLexer>(
208 Toks, NumToks, DisableMacroExpansion, OwnsTokens, IsReinject, *this);
209 } else {
210 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
211 TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
212 IsReinject);
213 }
214
215 // Save our current state.
216 PushIncludeMacroStack();
217 CurDirLookup = nullptr;
218 CurTokenLexer = std::move(TokLexer);
219 if (CurLexerKind != CLK_LexAfterModuleImport)
220 CurLexerKind = CLK_TokenLexer;
221}
222
223/// Compute the relative path that names the given file relative to
224/// the given directory.
227 Result.clear();
228
229 StringRef FilePath = File.getDir().getName();
230 StringRef Path = FilePath;
231 while (!Path.empty()) {
232 if (auto CurDir = FM.getDirectory(Path)) {
233 if (*CurDir == Dir) {
234 Result = FilePath.substr(Path.size());
235 llvm::sys::path::append(Result,
236 llvm::sys::path::filename(File.getName()));
237 return;
238 }
239 }
240
241 Path = llvm::sys::path::parent_path(Path);
242 }
243
244 Result = File.getName();
245}
246
247void Preprocessor::PropagateLineStartLeadingSpaceInfo(Token &Result) {
248 if (CurTokenLexer) {
249 CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result);
250 return;
251 }
252 if (CurLexer) {
253 CurLexer->PropagateLineStartLeadingSpaceInfo(Result);
254 return;
255 }
256 // FIXME: Handle other kinds of lexers? It generally shouldn't matter,
257 // but it might if they're empty?
258}
259
260/// Determine the location to use as the end of the buffer for a lexer.
261///
262/// If the file ends with a newline, form the EOF token on the newline itself,
263/// rather than "on the line following it", which doesn't exist. This makes
264/// diagnostics relating to the end of file include the last file that the user
265/// actually typed, which is goodness.
266const char *Preprocessor::getCurLexerEndPos() {
267 const char *EndPos = CurLexer->BufferEnd;
268 if (EndPos != CurLexer->BufferStart &&
269 (EndPos[-1] == '\n' || EndPos[-1] == '\r')) {
270 --EndPos;
271
272 // Handle \n\r and \r\n:
273 if (EndPos != CurLexer->BufferStart &&
274 (EndPos[-1] == '\n' || EndPos[-1] == '\r') &&
275 EndPos[-1] != EndPos[0])
276 --EndPos;
277 }
278
279 return EndPos;
280}
281
283 const Module &Mod, SmallVectorImpl<const Module *> &SubMods) {
285 SubMods.push_back(&Mod);
286 for (auto *M : Mod.submodules())
288}
289
290void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(const Module &Mod) {
291 std::optional<Module::Header> UmbrellaHeader =
293 assert(UmbrellaHeader && "Module must use umbrella header");
294 const FileID &File = SourceMgr.translateFile(UmbrellaHeader->Entry);
295 SourceLocation ExpectedHeadersLoc = SourceMgr.getLocForEndOfFile(File);
296 if (getDiagnostics().isIgnored(diag::warn_uncovered_module_header,
297 ExpectedHeadersLoc))
298 return;
299
302 llvm::vfs::FileSystem &FS = FileMgr.getVirtualFileSystem();
303 std::error_code EC;
304 for (llvm::vfs::recursive_directory_iterator Entry(FS, Dir->getName(), EC),
305 End;
306 Entry != End && !EC; Entry.increment(EC)) {
307 using llvm::StringSwitch;
308
309 // Check whether this entry has an extension typically associated with
310 // headers.
311 if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->path()))
312 .Cases(".h", ".H", ".hh", ".hpp", true)
313 .Default(false))
314 continue;
315
316 if (auto Header = getFileManager().getOptionalFileRef(Entry->path()))
317 if (!getSourceManager().hasFileInfo(*Header)) {
318 if (!ModMap.isHeaderInUnavailableModule(*Header)) {
319 // Find the relative path that would access this header.
320 SmallString<128> RelativePath;
321 computeRelativePath(FileMgr, *Dir, *Header, RelativePath);
322 Diag(ExpectedHeadersLoc, diag::warn_uncovered_module_header)
323 << Mod.getFullModuleName() << RelativePath;
324 }
325 }
326 }
327}
328
329/// HandleEndOfFile - This callback is invoked when the lexer hits the end of
330/// the current file. This either returns the EOF token or pops a level off
331/// the include stack and keeps going.
332bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
333 assert(!CurTokenLexer &&
334 "Ending a file when currently in a macro!");
335
336 SourceLocation UnclosedSafeBufferOptOutLoc;
337
338 if (IncludeMacroStack.empty() &&
339 isPPInSafeBufferOptOutRegion(UnclosedSafeBufferOptOutLoc)) {
340 // To warn if a "-Wunsafe-buffer-usage" opt-out region is still open by the
341 // end of a file.
342 Diag(UnclosedSafeBufferOptOutLoc,
343 diag::err_pp_unclosed_pragma_unsafe_buffer_usage);
344 }
345 // If we have an unclosed module region from a pragma at the end of a
346 // module, complain and close it now.
347 const bool LeavingSubmodule = CurLexer && CurLexerSubmodule;
348 if ((LeavingSubmodule || IncludeMacroStack.empty()) &&
349 !BuildingSubmoduleStack.empty() &&
350 BuildingSubmoduleStack.back().IsPragma) {
351 Diag(BuildingSubmoduleStack.back().ImportLoc,
352 diag::err_pp_module_begin_without_module_end);
353 Module *M = LeaveSubmodule(/*ForPragma*/true);
354
355 Result.startToken();
356 const char *EndPos = getCurLexerEndPos();
357 CurLexer->BufferPtr = EndPos;
358 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
359 Result.setAnnotationEndLoc(Result.getLocation());
360 Result.setAnnotationValue(M);
361 return true;
362 }
363
364 // See if this file had a controlling macro.
365 if (CurPPLexer) { // Not ending a macro, ignore it.
366 if (const IdentifierInfo *ControllingMacro =
368 // Okay, this has a controlling macro, remember in HeaderFileInfo.
369 if (const FileEntry *FE = CurPPLexer->getFileEntry()) {
370 HeaderInfo.SetFileControllingMacro(FE, ControllingMacro);
371 if (MacroInfo *MI =
372 getMacroInfo(const_cast<IdentifierInfo*>(ControllingMacro)))
373 MI->setUsedForHeaderGuard(true);
374 if (const IdentifierInfo *DefinedMacro =
375 CurPPLexer->MIOpt.GetDefinedMacro()) {
376 if (!isMacroDefined(ControllingMacro) &&
377 DefinedMacro != ControllingMacro &&
378 CurLexer->isFirstTimeLexingFile()) {
379
380 // If the edit distance between the two macros is more than 50%,
381 // DefinedMacro may not be header guard, or can be header guard of
382 // another header file. Therefore, it maybe defining something
383 // completely different. This can be observed in the wild when
384 // handling feature macros or header guards in different files.
385
386 const StringRef ControllingMacroName = ControllingMacro->getName();
387 const StringRef DefinedMacroName = DefinedMacro->getName();
388 const size_t MaxHalfLength = std::max(ControllingMacroName.size(),
389 DefinedMacroName.size()) / 2;
390 const unsigned ED = ControllingMacroName.edit_distance(
391 DefinedMacroName, true, MaxHalfLength);
392 if (ED <= MaxHalfLength) {
393 // Emit a warning for a bad header guard.
394 Diag(CurPPLexer->MIOpt.GetMacroLocation(),
395 diag::warn_header_guard)
396 << CurPPLexer->MIOpt.GetMacroLocation() << ControllingMacro;
397 Diag(CurPPLexer->MIOpt.GetDefinedLocation(),
398 diag::note_header_guard)
399 << CurPPLexer->MIOpt.GetDefinedLocation() << DefinedMacro
400 << ControllingMacro
402 CurPPLexer->MIOpt.GetDefinedLocation(),
403 ControllingMacro->getName());
404 }
405 }
406 }
407 }
408 }
409 }
410
411 // Complain about reaching a true EOF within arc_cf_code_audited.
412 // We don't want to complain about reaching the end of a macro
413 // instantiation or a _Pragma.
414 if (PragmaARCCFCodeAuditedInfo.second.isValid() && !isEndOfMacro &&
415 !(CurLexer && CurLexer->Is_PragmaLexer)) {
416 Diag(PragmaARCCFCodeAuditedInfo.second,
417 diag::err_pp_eof_in_arc_cf_code_audited);
418
419 // Recover by leaving immediately.
420 PragmaARCCFCodeAuditedInfo = {nullptr, SourceLocation()};
421 }
422
423 // Complain about reaching a true EOF within assume_nonnull.
424 // We don't want to complain about reaching the end of a macro
425 // instantiation or a _Pragma.
426 if (PragmaAssumeNonNullLoc.isValid() &&
427 !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
428 // If we're at the end of generating a preamble, we should record the
429 // unterminated \#pragma clang assume_nonnull so we can restore it later
430 // when the preamble is loaded into the main file.
432 PreambleRecordedPragmaAssumeNonNullLoc = PragmaAssumeNonNullLoc;
433 else
434 Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
435 // Recover by leaving immediately.
436 PragmaAssumeNonNullLoc = SourceLocation();
437 }
438
439 bool LeavingPCHThroughHeader = false;
440
441 // If this is a #include'd file, pop it off the include stack and continue
442 // lexing the #includer file.
443 if (!IncludeMacroStack.empty()) {
444
445 // If we lexed the code-completion file, act as if we reached EOF.
446 if (isCodeCompletionEnabled() && CurPPLexer &&
447 SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
448 CodeCompletionFileLoc) {
449 assert(CurLexer && "Got EOF but no current lexer set!");
450 Result.startToken();
451 CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof);
452 CurLexer.reset();
453
454 CurPPLexer = nullptr;
456 return true;
457 }
458
459 if (!isEndOfMacro && CurPPLexer &&
460 (SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid() ||
461 // Predefines file doesn't have a valid include location.
462 (PredefinesFileID.isValid() &&
463 CurPPLexer->getFileID() == PredefinesFileID))) {
464 // Notify SourceManager to record the number of FileIDs that were created
465 // during lexing of the #include'd file.
466 unsigned NumFIDs =
467 SourceMgr.local_sloc_entry_size() -
468 CurPPLexer->getInitialNumSLocEntries() + 1/*#include'd file*/;
469 SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
470 }
471
472 bool ExitedFromPredefinesFile = false;
473 FileID ExitedFID;
474 if (!isEndOfMacro && CurPPLexer) {
475 ExitedFID = CurPPLexer->getFileID();
476
477 assert(PredefinesFileID.isValid() &&
478 "HandleEndOfFile is called before PredefinesFileId is set");
479 ExitedFromPredefinesFile = (PredefinesFileID == ExitedFID);
480 }
481
482 if (LeavingSubmodule) {
483 // We're done with this submodule.
484 Module *M = LeaveSubmodule(/*ForPragma*/false);
485
486 // Notify the parser that we've left the module.
487 const char *EndPos = getCurLexerEndPos();
488 Result.startToken();
489 CurLexer->BufferPtr = EndPos;
490 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
491 Result.setAnnotationEndLoc(Result.getLocation());
492 Result.setAnnotationValue(M);
493 }
494
495 bool FoundPCHThroughHeader = false;
496 if (CurPPLexer && creatingPCHWithThroughHeader() &&
498 SourceMgr.getFileEntryForID(CurPPLexer->getFileID())))
499 FoundPCHThroughHeader = true;
500
501 // We're done with the #included file.
503
504 // Propagate info about start-of-line/leading white-space/etc.
505 PropagateLineStartLeadingSpaceInfo(Result);
506
507 // Notify the client, if desired, that we are in a new source file.
508 if (Callbacks && !isEndOfMacro && CurPPLexer) {
509 SourceLocation Loc = CurPPLexer->getSourceLocation();
511 SourceMgr.getFileCharacteristic(Loc);
512 Callbacks->FileChanged(Loc, PPCallbacks::ExitFile, FileType, ExitedFID);
513 Callbacks->LexedFileChanged(CurPPLexer->getFileID(),
515 FileType, ExitedFID, Loc);
516 }
517
518 // Restore conditional stack as well as the recorded
519 // \#pragma clang assume_nonnull from the preamble right after exiting
520 // from the predefines file.
521 if (ExitedFromPredefinesFile) {
522 replayPreambleConditionalStack();
523 if (PreambleRecordedPragmaAssumeNonNullLoc.isValid())
524 PragmaAssumeNonNullLoc = PreambleRecordedPragmaAssumeNonNullLoc;
525 }
526
527 if (!isEndOfMacro && CurPPLexer && FoundPCHThroughHeader &&
528 (isInPrimaryFile() ||
529 CurPPLexer->getFileID() == getPredefinesFileID())) {
530 // Leaving the through header. Continue directly to end of main file
531 // processing.
532 LeavingPCHThroughHeader = true;
533 } else {
534 // Client should lex another token unless we generated an EOM.
535 return LeavingSubmodule;
536 }
537 }
538 // If this is the end of the main file, form an EOF token.
539 assert(CurLexer && "Got EOF but no current lexer set!");
540 const char *EndPos = getCurLexerEndPos();
541 Result.startToken();
542 CurLexer->BufferPtr = EndPos;
543
545 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_repl_input_end);
546 Result.setAnnotationEndLoc(Result.getLocation());
547 Result.setAnnotationValue(nullptr);
548 } else {
549 CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
550 }
551
553 // Inserting the code-completion point increases the source buffer by 1,
554 // but the main FileID was created before inserting the point.
555 // Compensate by reducing the EOF location by 1, otherwise the location
556 // will point to the next FileID.
557 // FIXME: This is hacky, the code-completion point should probably be
558 // inserted before the main FileID is created.
559 if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
560 Result.setLocation(Result.getLocation().getLocWithOffset(-1));
561 }
562
563 if (creatingPCHWithThroughHeader() && !LeavingPCHThroughHeader) {
564 // Reached the end of the compilation without finding the through header.
565 Diag(CurLexer->getFileLoc(), diag::err_pp_through_header_not_seen)
566 << PPOpts->PCHThroughHeader << 0;
567 }
568
570 // We're done with lexing.
571 CurLexer.reset();
572
574 CurPPLexer = nullptr;
575
576 if (TUKind == TU_Complete) {
577 // This is the end of the top-level file. 'WarnUnusedMacroLocs' has
578 // collected all macro locations that we need to warn because they are not
579 // used.
580 for (WarnUnusedMacroLocsTy::iterator
581 I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
582 I!=E; ++I)
583 Diag(*I, diag::pp_macro_not_used);
584 }
585
586 // If we are building a module that has an umbrella header, make sure that
587 // each of the headers within the directory, including all submodules, is
588 // covered by the umbrella header was actually included by the umbrella
589 // header.
590 if (Module *Mod = getCurrentModule()) {
593 for (auto *M : AllMods)
594 diagnoseMissingHeaderInUmbrellaDir(*M);
595 }
596
597 return true;
598}
599
600/// HandleEndOfTokenLexer - This callback is invoked when the current TokenLexer
601/// hits the end of its token stream.
603 assert(CurTokenLexer && !CurPPLexer &&
604 "Ending a macro when currently in a #include file!");
605
606 if (!MacroExpandingLexersStack.empty() &&
607 MacroExpandingLexersStack.back().first == CurTokenLexer.get())
608 removeCachedMacroExpandedTokensOfLastLexer();
609
610 // Delete or cache the now-dead macro expander.
611 if (NumCachedTokenLexers == TokenLexerCacheSize)
612 CurTokenLexer.reset();
613 else
614 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
615
616 // Handle this like a #include file being popped off the stack.
617 return HandleEndOfFile(Result, true);
618}
619
620/// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
621/// lexer stack. This should only be used in situations where the current
622/// state of the top-of-stack lexer is unknown.
624 assert(!IncludeMacroStack.empty() && "Ran out of stack entries to load");
625
626 if (CurTokenLexer) {
627 // Delete or cache the now-dead macro expander.
628 if (NumCachedTokenLexers == TokenLexerCacheSize)
629 CurTokenLexer.reset();
630 else
631 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
632 }
633
634 PopIncludeMacroStack();
635}
636
637/// HandleMicrosoftCommentPaste - When the macro expander pastes together a
638/// comment (/##/) in microsoft mode, this method handles updating the current
639/// state, returning the token on the next source line.
641 assert(CurTokenLexer && !CurPPLexer &&
642 "Pasted comment can only be formed from macro");
643 // We handle this by scanning for the closest real lexer, switching it to
644 // raw mode and preprocessor mode. This will cause it to return \n as an
645 // explicit EOD token.
646 PreprocessorLexer *FoundLexer = nullptr;
647 bool LexerWasInPPMode = false;
648 for (const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
649 if (ISI.ThePPLexer == nullptr) continue; // Scan for a real lexer.
650
651 // Once we find a real lexer, mark it as raw mode (disabling macro
652 // expansions) and preprocessor mode (return EOD). We know that the lexer
653 // was *not* in raw mode before, because the macro that the comment came
654 // from was expanded. However, it could have already been in preprocessor
655 // mode (#if COMMENT) in which case we have to return it to that mode and
656 // return EOD.
657 FoundLexer = ISI.ThePPLexer;
658 FoundLexer->LexingRawMode = true;
659 LexerWasInPPMode = FoundLexer->ParsingPreprocessorDirective;
660 FoundLexer->ParsingPreprocessorDirective = true;
661 break;
662 }
663
664 // Okay, we either found and switched over the lexer, or we didn't find a
665 // lexer. In either case, finish off the macro the comment came from, getting
666 // the next token.
667 if (!HandleEndOfTokenLexer(Tok)) Lex(Tok);
668
669 // Discarding comments as long as we don't have EOF or EOD. This 'comments
670 // out' the rest of the line, including any tokens that came from other macros
671 // that were active, as in:
672 // #define submacro a COMMENT b
673 // submacro c
674 // which should lex to 'a' only: 'b' and 'c' should be removed.
675 while (Tok.isNot(tok::eod) && Tok.isNot(tok::eof))
676 Lex(Tok);
677
678 // If we got an eod token, then we successfully found the end of the line.
679 if (Tok.is(tok::eod)) {
680 assert(FoundLexer && "Can't get end of line without an active lexer");
681 // Restore the lexer back to normal mode instead of raw mode.
682 FoundLexer->LexingRawMode = false;
683
684 // If the lexer was already in preprocessor mode, just return the EOD token
685 // to finish the preprocessor line.
686 if (LexerWasInPPMode) return;
687
688 // Otherwise, switch out of PP mode and return the next lexed token.
689 FoundLexer->ParsingPreprocessorDirective = false;
690 return Lex(Tok);
691 }
692
693 // If we got an EOF token, then we reached the end of the token stream but
694 // didn't find an explicit \n. This can only happen if there was no lexer
695 // active (an active lexer would return EOD at EOF if there was no \n in
696 // preprocessor directive mode), so just return EOF as our token.
697 assert(!FoundLexer && "Lexer should return EOD before EOF in PP mode");
698}
699
701 bool ForPragma) {
702 if (!getLangOpts().ModulesLocalVisibility) {
703 // Just track that we entered this submodule.
704 BuildingSubmoduleStack.push_back(
705 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
706 PendingModuleMacroNames.size()));
707 if (Callbacks)
708 Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma);
709 return;
710 }
711
712 // Resolve as much of the module definition as we can now, before we enter
713 // one of its headers.
714 // FIXME: Can we enable Complain here?
715 // FIXME: Can we do this when local visibility is disabled?
717 ModMap.resolveExports(M, /*Complain=*/false);
718 ModMap.resolveUses(M, /*Complain=*/false);
719 ModMap.resolveConflicts(M, /*Complain=*/false);
720
721 // If this is the first time we've entered this module, set up its state.
722 auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
723 auto &State = R.first->second;
724 bool FirstTime = R.second;
725 if (FirstTime) {
726 // Determine the set of starting macros for this submodule; take these
727 // from the "null" module (the predefines buffer).
728 //
729 // FIXME: If we have local visibility but not modules enabled, the
730 // NullSubmoduleState is polluted by #defines in the top-level source
731 // file.
732 auto &StartingMacros = NullSubmoduleState.Macros;
733
734 // Restore to the starting state.
735 // FIXME: Do this lazily, when each macro name is first referenced.
736 for (auto &Macro : StartingMacros) {
737 // Skip uninteresting macros.
738 if (!Macro.second.getLatest() &&
739 Macro.second.getOverriddenMacros().empty())
740 continue;
741
742 MacroState MS(Macro.second.getLatest());
743 MS.setOverriddenMacros(*this, Macro.second.getOverriddenMacros());
744 State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
745 }
746 }
747
748 // Track that we entered this module.
749 BuildingSubmoduleStack.push_back(
750 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
751 PendingModuleMacroNames.size()));
752
753 if (Callbacks)
754 Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma);
755
756 // Switch to this submodule as the current submodule.
757 CurSubmoduleState = &State;
758
759 // This module is visible to itself.
760 if (FirstTime)
761 makeModuleVisible(M, ImportLoc);
762}
763
764bool Preprocessor::needModuleMacros() const {
765 // If we're not within a submodule, we never need to create ModuleMacros.
766 if (BuildingSubmoduleStack.empty())
767 return false;
768 // If we are tracking module macro visibility even for textually-included
769 // headers, we need ModuleMacros.
770 if (getLangOpts().ModulesLocalVisibility)
771 return true;
772 // Otherwise, we only need module macros if we're actually compiling a module
773 // interface.
775}
776
778 if (BuildingSubmoduleStack.empty() ||
779 BuildingSubmoduleStack.back().IsPragma != ForPragma) {
780 assert(ForPragma && "non-pragma module enter/leave mismatch");
781 return nullptr;
782 }
783
784 auto &Info = BuildingSubmoduleStack.back();
785
786 Module *LeavingMod = Info.M;
787 SourceLocation ImportLoc = Info.ImportLoc;
788
789 if (!needModuleMacros() ||
790 (!getLangOpts().ModulesLocalVisibility &&
791 LeavingMod->getTopLevelModuleName() != getLangOpts().CurrentModule)) {
792 // If we don't need module macros, or this is not a module for which we
793 // are tracking macro visibility, don't build any, and preserve the list
794 // of pending names for the surrounding submodule.
795 BuildingSubmoduleStack.pop_back();
796
797 if (Callbacks)
798 Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma);
799
800 makeModuleVisible(LeavingMod, ImportLoc);
801 return LeavingMod;
802 }
803
804 // Create ModuleMacros for any macros defined in this submodule.
806 for (unsigned I = Info.OuterPendingModuleMacroNames;
807 I != PendingModuleMacroNames.size(); ++I) {
808 auto *II = const_cast<IdentifierInfo*>(PendingModuleMacroNames[I]);
809 if (!VisitedMacros.insert(II).second)
810 continue;
811
812 auto MacroIt = CurSubmoduleState->Macros.find(II);
813 if (MacroIt == CurSubmoduleState->Macros.end())
814 continue;
815 auto &Macro = MacroIt->second;
816
817 // Find the starting point for the MacroDirective chain in this submodule.
818 MacroDirective *OldMD = nullptr;
819 auto *OldState = Info.OuterSubmoduleState;
820 if (getLangOpts().ModulesLocalVisibility)
821 OldState = &NullSubmoduleState;
822 if (OldState && OldState != CurSubmoduleState) {
823 // FIXME: It'd be better to start at the state from when we most recently
824 // entered this submodule, but it doesn't really matter.
825 auto &OldMacros = OldState->Macros;
826 auto OldMacroIt = OldMacros.find(II);
827 if (OldMacroIt == OldMacros.end())
828 OldMD = nullptr;
829 else
830 OldMD = OldMacroIt->second.getLatest();
831 }
832
833 // This module may have exported a new macro. If so, create a ModuleMacro
834 // representing that fact.
835 bool ExplicitlyPublic = false;
836 for (auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->getPrevious()) {
837 assert(MD && "broken macro directive chain");
838
839 if (auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
840 // The latest visibility directive for a name in a submodule affects
841 // all the directives that come before it.
842 if (VisMD->isPublic())
843 ExplicitlyPublic = true;
844 else if (!ExplicitlyPublic)
845 // Private with no following public directive: not exported.
846 break;
847 } else {
848 MacroInfo *Def = nullptr;
849 if (DefMacroDirective *DefMD = dyn_cast<DefMacroDirective>(MD))
850 Def = DefMD->getInfo();
851
852 // FIXME: Issue a warning if multiple headers for the same submodule
853 // define a macro, rather than silently ignoring all but the first.
854 bool IsNew;
855 // Don't bother creating a module macro if it would represent a #undef
856 // that doesn't override anything.
857 if (Def || !Macro.getOverriddenMacros().empty())
858 addModuleMacro(LeavingMod, II, Def,
859 Macro.getOverriddenMacros(), IsNew);
860
861 if (!getLangOpts().ModulesLocalVisibility) {
862 // This macro is exposed to the rest of this compilation as a
863 // ModuleMacro; we don't need to track its MacroDirective any more.
864 Macro.setLatest(nullptr);
865 Macro.setOverriddenMacros(*this, {});
866 }
867 break;
868 }
869 }
870 }
871 PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames);
872
873 // FIXME: Before we leave this submodule, we should parse all the other
874 // headers within it. Otherwise, we're left with an inconsistent state
875 // where we've made the module visible but don't yet have its complete
876 // contents.
877
878 // Put back the outer module's state, if we're tracking it.
879 if (getLangOpts().ModulesLocalVisibility)
880 CurSubmoduleState = Info.OuterSubmoduleState;
881
882 BuildingSubmoduleStack.pop_back();
883
884 if (Callbacks)
885 Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma);
886
887 // A nested #include makes the included submodule visible.
888 makeModuleVisible(LeavingMod, ImportLoc);
889 return LeavingMod;
890}
Defines the clang::FileManager interface and associated types.
Defines the clang::MacroInfo and clang::MacroDirective classes.
static void collectAllSubModulesWithUmbrellaHeader(const Module &Mod, SmallVectorImpl< const Module * > &SubMods)
static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir, FileEntryRef File, SmallString< 128 > &Result)
Compute the relative path that names the given file relative to the given directory.
Defines the clang::Preprocessor interface.
Defines the SourceManager interface.
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:429
StringRef getName() const
Cached information about one directory (either on disk or in the virtual file system).
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
Definition: FileEntry.h:57
Cached information about one file (either on disk or in the virtual file system).
Definition: FileEntry.h:353
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isValid() const
Implements support for file system lookup, file system caching, and directory search management.
Definition: FileManager.h:53
llvm::vfs::FileSystem & getVirtualFileSystem() const
Definition: FileManager.h:245
llvm::ErrorOr< const DirectoryEntry * > getDirectory(StringRef DirName, bool CacheFailure=true)
Lookup, cache, and verify the specified directory (real or virtual).
static FixItHint CreateReplacement(CharSourceRange RemoveRange, StringRef Code)
Create a code modification hint that replaces the given source range with the given code string.
Definition: Diagnostic.h:134
void SetFileControllingMacro(const FileEntry *File, const IdentifierInfo *ControllingMacro)
Mark the specified file as having a controlling macro.
Definition: HeaderSearch.h:546
ModuleMap & getModuleMap()
Retrieve the module map.
Definition: HeaderSearch.h:798
One of these records is kept for each identifier that is lexed.
bool isCompilingModule() const
Are we compiling a module?
Definition: LangOptions.h:514
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.
Definition: Lexer.h:78
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
Encapsulates changes to the "macros namespace" (the location where the macro name became active,...
Definition: MacroInfo.h:313
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Definition: MacroInfo.h:351
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
bool resolveExports(Module *Mod, bool Complain)
Resolve all of the unresolved exports in the given module.
Definition: ModuleMap.cpp:1384
bool resolveConflicts(Module *Mod, bool Complain)
Resolve all of the unresolved conflicts in the given module.
Definition: ModuleMap.cpp:1410
bool isHeaderInUnavailableModule(const FileEntry *Header) const
Determine whether the given header is part of a module marked 'unavailable'.
Definition: ModuleMap.cpp:703
bool resolveUses(Module *Mod, bool Complain)
Resolve all of the unresolved uses in the given module.
Definition: ModuleMap.cpp:1397
Describes a module or submodule.
Definition: Module.h:98
StringRef getTopLevelModuleName() const
Retrieve the name of the top-level module.
Definition: Module.h:636
llvm::iterator_range< submodule_iterator > submodules()
Definition: Module.h:745
std::optional< Header > getUmbrellaHeaderAsWritten() const
Retrieve the umbrella header as written.
Definition: Module.h:662
std::string getFullModuleName(bool AllowStringLiterals=false) const
Retrieve the full name of this module, including the path from its top-level module.
Definition: Module.cpp:241
OptionalDirectoryEntryRef getEffectiveUmbrellaDir() const
Get the effective umbrella directory for this module: either the one explicitly written in the module...
Definition: Module.cpp:266
const IdentifierInfo * GetDefinedMacro() const
If the ControllingMacro is followed by a macro definition, return the macro that was defined.
const IdentifierInfo * GetControllingMacroAtEndOfFile() const
Once the entire file has been lexed, if there is a controlling macro, return it.
SourceLocation GetDefinedLocation() const
SourceLocation GetMacroLocation() const
OptionalFileEntryRefDegradesToFileEntryPtr getFileEntry() const
getFileEntry - Return the FileEntry corresponding to this FileID.
unsigned getInitialNumSLocEntries() const
Number of SLocEntries before lexing the file.
bool LexingRawMode
True if in raw mode.
bool ParsingPreprocessorDirective
True when parsing #XXX; turns '\n' into a tok::eod token.
MultipleIncludeOpt MIOpt
A state machine that detects the #ifndef-wrapping a file idiom for the multiple-include optimization.
virtual SourceLocation getSourceLocation()=0
Return the source location for the next observable location.
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
bool isRecordingPreamble() const
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
void makeModuleVisible(Module *M, SourceLocation Loc)
void Lex(Token &Result)
Lex the next token for this preprocessor.
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
Definition: Preprocessor.h:282
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
SourceManager & getSourceManager() const
bool isMacroDefined(StringRef Id)
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
bool isPPInSafeBufferOptOutRegion()
FileManager & getFileManager() const
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
HeaderSearch & getHeaderSearchInfo() const
Module * LeaveSubmodule(bool ForPragma)
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
const LangOptions & getLangOpts() const
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
DiagnosticsEngine & getDiagnostics() const
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
Encodes a location in the source.
bool isValid() const
Return true if this is a valid SourceLocation object.
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
OptionalFileEntryRef getFileEntryRefForID(FileID FID) const
Returns the FileEntryRef for the provided FileID.
StringRef getBufferName(SourceLocation Loc, bool *Invalid=nullptr) const
Return the filename or buffer identifier of the buffer the location is in.
FileID translateFile(const FileEntry *SourceFile) const
Get the FileID for the given file.
unsigned local_sloc_entry_size() const
Get the number of local SLocEntries we have.
SourceLocation getLocForEndOfFile(FileID FID) const
Return the source location corresponding to the last byte of the specified file.
SourceLocation getIncludeLoc(FileID FID) const
Returns the include location if FID is a #include'd file otherwise it returns an invalid location.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file.
SrcMgr::CharacteristicKind getFileCharacteristic(SourceLocation Loc) const
Return the file characteristic of the specified source location, indicating whether this is a normal ...
void setNumCreatedFIDsForFileID(FileID FID, unsigned NumFIDs, bool Force=false) const
Set the number of FileIDs (files and macros) that were created during preprocessing of FID,...
std::optional< llvm::MemoryBufferRef > getBufferOrNone(FileID FID, SourceLocation Loc=SourceLocation()) const
Return the buffer for the specified FileID.
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:98
bool isNot(tok::TokenKind K) const
Definition: Token.h:99
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
Definition: SourceManager.h:80
@ Result
The result type of a method or function.
@ TU_Complete
The translation unit is a complete translation unit.
Definition: LangOptions.h:919