clang 20.0.0git
PPLexerChange.cpp
Go to the documentation of this file.
1//===--- PPLexerChange.cpp - Handle changing lexers in the preprocessor ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements pieces of the Preprocessor interface that manage the
10// current lexer stack.
11//
12//===----------------------------------------------------------------------===//
13
18#include "clang/Lex/MacroInfo.h"
21#include "llvm/ADT/StringSwitch.h"
22#include "llvm/Support/FileSystem.h"
23#include "llvm/Support/MemoryBufferRef.h"
24#include "llvm/Support/Path.h"
25#include <optional>
26
27using namespace clang;
28
29//===----------------------------------------------------------------------===//
30// Miscellaneous Methods.
31//===----------------------------------------------------------------------===//
32
33/// isInPrimaryFile - Return true if we're in the top-level file, not in a
34/// \#include. This looks through macro expansions and active _Pragma lexers.
36 if (IsFileLexer())
37 return IncludeMacroStack.empty();
38
39 // If there are any stacked lexers, we're in a #include.
40 assert(IsFileLexer(IncludeMacroStack[0]) &&
41 "Top level include stack isn't our primary lexer?");
42 return llvm::none_of(
43 llvm::drop_begin(IncludeMacroStack),
44 [&](const IncludeStackInfo &ISI) -> bool { return IsFileLexer(ISI); });
45}
46
47/// getCurrentLexer - Return the current file lexer being lexed from. Note
48/// that this ignores any potentially active macro expansions and _Pragma
49/// expansions going on at the time.
51 if (IsFileLexer())
52 return CurPPLexer;
53
54 // Look for a stacked lexer.
55 for (const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
56 if (IsFileLexer(ISI))
57 return ISI.ThePPLexer;
58 }
59 return nullptr;
60}
61
62
63//===----------------------------------------------------------------------===//
64// Methods for Entering and Callbacks for leaving various contexts
65//===----------------------------------------------------------------------===//
66
67/// EnterSourceFile - Add a source file to the top of the include stack and
68/// start lexing tokens from it instead of the current buffer.
71 bool IsFirstIncludeOfFile) {
72 assert(!CurTokenLexer && "Cannot #include a file inside a macro!");
73 ++NumEnteredSourceFiles;
74
75 if (MaxIncludeStackDepth < IncludeMacroStack.size())
76 MaxIncludeStackDepth = IncludeMacroStack.size();
77
78 // Get the MemoryBuffer for this FID, if it fails, we fail.
79 std::optional<llvm::MemoryBufferRef> InputFile =
81 if (!InputFile) {
82 SourceLocation FileStart = SourceMgr.getLocForStartOfFile(FID);
83 Diag(Loc, diag::err_pp_error_opening_file)
84 << std::string(SourceMgr.getBufferName(FileStart)) << "";
85 return true;
86 }
87
89 SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
90 CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
91 CodeCompletionLoc =
92 CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
93 }
94
95 Lexer *TheLexer = new Lexer(FID, *InputFile, *this, IsFirstIncludeOfFile);
96 if (getPreprocessorOpts().DependencyDirectivesForFile &&
97 FID != PredefinesFileID) {
98 if (OptionalFileEntryRef File = SourceMgr.getFileEntryRefForID(FID)) {
100 DepDirectives =
101 getPreprocessorOpts().DependencyDirectivesForFile(*File)) {
102 TheLexer->DepDirectives = *DepDirectives;
103 }
104 }
105 }
106
107 EnterSourceFileWithLexer(TheLexer, CurDir);
108 return false;
109}
110
111/// EnterSourceFileWithLexer - Add a source file to the top of the include stack
112/// and start lexing tokens from it instead of the current buffer.
113void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
114 ConstSearchDirIterator CurDir) {
115 PreprocessorLexer *PrevPPLexer = CurPPLexer;
116
117 // Add the current lexer to the include stack.
118 if (CurPPLexer || CurTokenLexer)
119 PushIncludeMacroStack();
120
121 CurLexer.reset(TheLexer);
122 CurPPLexer = TheLexer;
123 CurDirLookup = CurDir;
124 CurLexerSubmodule = nullptr;
125 if (CurLexerCallback != CLK_LexAfterModuleImport)
126 CurLexerCallback = TheLexer->isDependencyDirectivesLexer()
127 ? CLK_DependencyDirectivesLexer
128 : CLK_Lexer;
129
130 // Notify the client, if desired, that we are in a new source file.
131 if (Callbacks && !CurLexer->Is_PragmaLexer) {
133 SourceMgr.getFileCharacteristic(CurLexer->getFileLoc());
134
135 FileID PrevFID;
136 SourceLocation EnterLoc;
137 if (PrevPPLexer) {
138 PrevFID = PrevPPLexer->getFileID();
139 EnterLoc = PrevPPLexer->getSourceLocation();
140 }
141 Callbacks->FileChanged(CurLexer->getFileLoc(), PPCallbacks::EnterFile,
142 FileType, PrevFID);
143 Callbacks->LexedFileChanged(CurLexer->getFileID(),
145 FileType, PrevFID, EnterLoc);
146 }
147}
148
149/// EnterMacro - Add a Macro to the top of the include stack and start lexing
150/// tokens from it instead of the current buffer.
152 MacroInfo *Macro, MacroArgs *Args) {
153 std::unique_ptr<TokenLexer> TokLexer;
154 if (NumCachedTokenLexers == 0) {
155 TokLexer = std::make_unique<TokenLexer>(Tok, ILEnd, Macro, Args, *this);
156 } else {
157 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
158 TokLexer->Init(Tok, ILEnd, Macro, Args);
159 }
160
161 PushIncludeMacroStack();
162 CurDirLookup = nullptr;
163 CurTokenLexer = std::move(TokLexer);
164 if (CurLexerCallback != CLK_LexAfterModuleImport)
165 CurLexerCallback = CLK_TokenLexer;
166}
167
168/// EnterTokenStream - Add a "macro" context to the top of the include stack,
169/// which will cause the lexer to start returning the specified tokens.
170///
171/// If DisableMacroExpansion is true, tokens lexed from the token stream will
172/// not be subject to further macro expansion. Otherwise, these tokens will
173/// be re-macro-expanded when/if expansion is enabled.
174///
175/// If OwnsTokens is false, this method assumes that the specified stream of
176/// tokens has a permanent owner somewhere, so they do not need to be copied.
177/// If it is true, it assumes the array of tokens is allocated with new[] and
178/// must be freed.
179///
180void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
181 bool DisableMacroExpansion, bool OwnsTokens,
182 bool IsReinject) {
183 if (CurLexerCallback == CLK_CachingLexer) {
184 if (CachedLexPos < CachedTokens.size()) {
185 assert(IsReinject && "new tokens in the middle of cached stream");
186 // We're entering tokens into the middle of our cached token stream. We
187 // can't represent that, so just insert the tokens into the buffer.
188 CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
189 Toks, Toks + NumToks);
190 if (OwnsTokens)
191 delete [] Toks;
192 return;
193 }
194
195 // New tokens are at the end of the cached token sequnece; insert the
196 // token stream underneath the caching lexer.
197 ExitCachingLexMode();
198 EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
199 IsReinject);
200 EnterCachingLexMode();
201 return;
202 }
203
204 // Create a macro expander to expand from the specified token stream.
205 std::unique_ptr<TokenLexer> TokLexer;
206 if (NumCachedTokenLexers == 0) {
207 TokLexer = std::make_unique<TokenLexer>(
208 Toks, NumToks, DisableMacroExpansion, OwnsTokens, IsReinject, *this);
209 } else {
210 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
211 TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
212 IsReinject);
213 }
214
215 // Save our current state.
216 PushIncludeMacroStack();
217 CurDirLookup = nullptr;
218 CurTokenLexer = std::move(TokLexer);
219 if (CurLexerCallback != CLK_LexAfterModuleImport)
220 CurLexerCallback = CLK_TokenLexer;
221}
222
223/// Compute the relative path that names the given file relative to
224/// the given directory.
227 Result.clear();
228
229 StringRef FilePath = File.getDir().getName();
230 StringRef Path = FilePath;
231 while (!Path.empty()) {
232 if (auto CurDir = FM.getDirectory(Path)) {
233 if (*CurDir == Dir) {
234 Result = FilePath.substr(Path.size());
235 llvm::sys::path::append(Result,
236 llvm::sys::path::filename(File.getName()));
237 return;
238 }
239 }
240
241 Path = llvm::sys::path::parent_path(Path);
242 }
243
244 Result = File.getName();
245}
246
247void Preprocessor::PropagateLineStartLeadingSpaceInfo(Token &Result) {
248 if (CurTokenLexer) {
249 CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result);
250 return;
251 }
252 if (CurLexer) {
253 CurLexer->PropagateLineStartLeadingSpaceInfo(Result);
254 return;
255 }
256 // FIXME: Handle other kinds of lexers? It generally shouldn't matter,
257 // but it might if they're empty?
258}
259
260/// Determine the location to use as the end of the buffer for a lexer.
261///
262/// If the file ends with a newline, form the EOF token on the newline itself,
263/// rather than "on the line following it", which doesn't exist. This makes
264/// diagnostics relating to the end of file include the last file that the user
265/// actually typed, which is goodness.
266const char *Preprocessor::getCurLexerEndPos() {
267 const char *EndPos = CurLexer->BufferEnd;
268 if (EndPos != CurLexer->BufferStart &&
269 (EndPos[-1] == '\n' || EndPos[-1] == '\r')) {
270 --EndPos;
271
272 // Handle \n\r and \r\n:
273 if (EndPos != CurLexer->BufferStart &&
274 (EndPos[-1] == '\n' || EndPos[-1] == '\r') &&
275 EndPos[-1] != EndPos[0])
276 --EndPos;
277 }
278
279 return EndPos;
280}
281
283 const Module &Mod, SmallVectorImpl<const Module *> &SubMods) {
285 SubMods.push_back(&Mod);
286 for (auto *M : Mod.submodules())
288}
289
290void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(const Module &Mod) {
291 std::optional<Module::Header> UmbrellaHeader =
293 assert(UmbrellaHeader && "Module must use umbrella header");
294 const FileID &File = SourceMgr.translateFile(UmbrellaHeader->Entry);
295 SourceLocation ExpectedHeadersLoc = SourceMgr.getLocForEndOfFile(File);
296 if (getDiagnostics().isIgnored(diag::warn_uncovered_module_header,
297 ExpectedHeadersLoc))
298 return;
299
302 llvm::vfs::FileSystem &FS = FileMgr.getVirtualFileSystem();
303 std::error_code EC;
304 for (llvm::vfs::recursive_directory_iterator Entry(FS, Dir->getName(), EC),
305 End;
306 Entry != End && !EC; Entry.increment(EC)) {
307 using llvm::StringSwitch;
308
309 // Check whether this entry has an extension typically associated with
310 // headers.
311 if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->path()))
312 .Cases(".h", ".H", ".hh", ".hpp", true)
313 .Default(false))
314 continue;
315
316 if (auto Header = getFileManager().getOptionalFileRef(Entry->path()))
317 if (!getSourceManager().hasFileInfo(*Header)) {
318 if (!ModMap.isHeaderInUnavailableModule(*Header)) {
319 // Find the relative path that would access this header.
320 SmallString<128> RelativePath;
321 computeRelativePath(FileMgr, *Dir, *Header, RelativePath);
322 Diag(ExpectedHeadersLoc, diag::warn_uncovered_module_header)
323 << Mod.getFullModuleName() << RelativePath;
324 }
325 }
326 }
327}
328
329/// HandleEndOfFile - This callback is invoked when the lexer hits the end of
330/// the current file. This either returns the EOF token or pops a level off
331/// the include stack and keeps going.
332bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
333 assert(!CurTokenLexer &&
334 "Ending a file when currently in a macro!");
335
336 SourceLocation UnclosedSafeBufferOptOutLoc;
337
338 if (IncludeMacroStack.empty() &&
339 isPPInSafeBufferOptOutRegion(UnclosedSafeBufferOptOutLoc)) {
340 // To warn if a "-Wunsafe-buffer-usage" opt-out region is still open by the
341 // end of a file.
342 Diag(UnclosedSafeBufferOptOutLoc,
343 diag::err_pp_unclosed_pragma_unsafe_buffer_usage);
344 }
345 // If we have an unclosed module region from a pragma at the end of a
346 // module, complain and close it now.
347 const bool LeavingSubmodule = CurLexer && CurLexerSubmodule;
348 if ((LeavingSubmodule || IncludeMacroStack.empty()) &&
349 !BuildingSubmoduleStack.empty() &&
350 BuildingSubmoduleStack.back().IsPragma) {
351 Diag(BuildingSubmoduleStack.back().ImportLoc,
352 diag::err_pp_module_begin_without_module_end);
353 Module *M = LeaveSubmodule(/*ForPragma*/true);
354
355 Result.startToken();
356 const char *EndPos = getCurLexerEndPos();
357 CurLexer->BufferPtr = EndPos;
358 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
359 Result.setAnnotationEndLoc(Result.getLocation());
360 Result.setAnnotationValue(M);
361 return true;
362 }
363
364 // See if this file had a controlling macro.
365 if (CurPPLexer) { // Not ending a macro, ignore it.
366 if (const IdentifierInfo *ControllingMacro =
368 // Okay, this has a controlling macro, remember in HeaderFileInfo.
369 if (OptionalFileEntryRef FE = CurPPLexer->getFileEntry()) {
370 HeaderInfo.SetFileControllingMacro(*FE, ControllingMacro);
371 if (MacroInfo *MI = getMacroInfo(ControllingMacro))
372 MI->setUsedForHeaderGuard(true);
373 if (const IdentifierInfo *DefinedMacro =
374 CurPPLexer->MIOpt.GetDefinedMacro()) {
375 if (!isMacroDefined(ControllingMacro) &&
376 DefinedMacro != ControllingMacro &&
377 CurLexer->isFirstTimeLexingFile()) {
378
379 // If the edit distance between the two macros is more than 50%,
380 // DefinedMacro may not be header guard, or can be header guard of
381 // another header file. Therefore, it maybe defining something
382 // completely different. This can be observed in the wild when
383 // handling feature macros or header guards in different files.
384
385 const StringRef ControllingMacroName = ControllingMacro->getName();
386 const StringRef DefinedMacroName = DefinedMacro->getName();
387 const size_t MaxHalfLength = std::max(ControllingMacroName.size(),
388 DefinedMacroName.size()) / 2;
389 const unsigned ED = ControllingMacroName.edit_distance(
390 DefinedMacroName, true, MaxHalfLength);
391 if (ED <= MaxHalfLength) {
392 // Emit a warning for a bad header guard.
393 Diag(CurPPLexer->MIOpt.GetMacroLocation(),
394 diag::warn_header_guard)
395 << CurPPLexer->MIOpt.GetMacroLocation() << ControllingMacro;
396 Diag(CurPPLexer->MIOpt.GetDefinedLocation(),
397 diag::note_header_guard)
398 << CurPPLexer->MIOpt.GetDefinedLocation() << DefinedMacro
399 << ControllingMacro
401 CurPPLexer->MIOpt.GetDefinedLocation(),
402 ControllingMacro->getName());
403 }
404 }
405 }
406 }
407 }
408 }
409
410 // Complain about reaching a true EOF within arc_cf_code_audited.
411 // We don't want to complain about reaching the end of a macro
412 // instantiation or a _Pragma.
413 if (PragmaARCCFCodeAuditedInfo.second.isValid() && !isEndOfMacro &&
414 !(CurLexer && CurLexer->Is_PragmaLexer)) {
415 Diag(PragmaARCCFCodeAuditedInfo.second,
416 diag::err_pp_eof_in_arc_cf_code_audited);
417
418 // Recover by leaving immediately.
419 PragmaARCCFCodeAuditedInfo = {nullptr, SourceLocation()};
420 }
421
422 // Complain about reaching a true EOF within assume_nonnull.
423 // We don't want to complain about reaching the end of a macro
424 // instantiation or a _Pragma.
425 if (PragmaAssumeNonNullLoc.isValid() &&
426 !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
427 // If we're at the end of generating a preamble, we should record the
428 // unterminated \#pragma clang assume_nonnull so we can restore it later
429 // when the preamble is loaded into the main file.
431 PreambleRecordedPragmaAssumeNonNullLoc = PragmaAssumeNonNullLoc;
432 else
433 Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
434 // Recover by leaving immediately.
435 PragmaAssumeNonNullLoc = SourceLocation();
436 }
437
438 bool LeavingPCHThroughHeader = false;
439
440 // If this is a #include'd file, pop it off the include stack and continue
441 // lexing the #includer file.
442 if (!IncludeMacroStack.empty()) {
443
444 // If we lexed the code-completion file, act as if we reached EOF.
445 if (isCodeCompletionEnabled() && CurPPLexer &&
446 SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
447 CodeCompletionFileLoc) {
448 assert(CurLexer && "Got EOF but no current lexer set!");
449 Result.startToken();
450 CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof);
451 CurLexer.reset();
452
453 CurPPLexer = nullptr;
455 return true;
456 }
457
458 if (!isEndOfMacro && CurPPLexer &&
459 (SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid() ||
460 // Predefines file doesn't have a valid include location.
461 (PredefinesFileID.isValid() &&
462 CurPPLexer->getFileID() == PredefinesFileID))) {
463 // Notify SourceManager to record the number of FileIDs that were created
464 // during lexing of the #include'd file.
465 unsigned NumFIDs =
466 SourceMgr.local_sloc_entry_size() -
467 CurPPLexer->getInitialNumSLocEntries() + 1/*#include'd file*/;
468 SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
469 }
470
471 bool ExitedFromPredefinesFile = false;
472 FileID ExitedFID;
473 if (!isEndOfMacro && CurPPLexer) {
474 ExitedFID = CurPPLexer->getFileID();
475
476 assert(PredefinesFileID.isValid() &&
477 "HandleEndOfFile is called before PredefinesFileId is set");
478 ExitedFromPredefinesFile = (PredefinesFileID == ExitedFID);
479 }
480
481 if (LeavingSubmodule) {
482 // We're done with this submodule.
483 Module *M = LeaveSubmodule(/*ForPragma*/false);
484
485 // Notify the parser that we've left the module.
486 const char *EndPos = getCurLexerEndPos();
487 Result.startToken();
488 CurLexer->BufferPtr = EndPos;
489 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
490 Result.setAnnotationEndLoc(Result.getLocation());
491 Result.setAnnotationValue(M);
492 }
493
494 bool FoundPCHThroughHeader = false;
495 if (CurPPLexer && creatingPCHWithThroughHeader() &&
497 SourceMgr.getFileEntryForID(CurPPLexer->getFileID())))
498 FoundPCHThroughHeader = true;
499
500 // We're done with the #included file.
502
503 // Propagate info about start-of-line/leading white-space/etc.
504 PropagateLineStartLeadingSpaceInfo(Result);
505
506 // Notify the client, if desired, that we are in a new source file.
507 if (Callbacks && !isEndOfMacro && CurPPLexer) {
508 SourceLocation Loc = CurPPLexer->getSourceLocation();
510 SourceMgr.getFileCharacteristic(Loc);
511 Callbacks->FileChanged(Loc, PPCallbacks::ExitFile, FileType, ExitedFID);
512 Callbacks->LexedFileChanged(CurPPLexer->getFileID(),
514 FileType, ExitedFID, Loc);
515 }
516
517 // Restore conditional stack as well as the recorded
518 // \#pragma clang assume_nonnull from the preamble right after exiting
519 // from the predefines file.
520 if (ExitedFromPredefinesFile) {
521 replayPreambleConditionalStack();
522 if (PreambleRecordedPragmaAssumeNonNullLoc.isValid())
523 PragmaAssumeNonNullLoc = PreambleRecordedPragmaAssumeNonNullLoc;
524 }
525
526 if (!isEndOfMacro && CurPPLexer && FoundPCHThroughHeader &&
527 (isInPrimaryFile() ||
528 CurPPLexer->getFileID() == getPredefinesFileID())) {
529 // Leaving the through header. Continue directly to end of main file
530 // processing.
531 LeavingPCHThroughHeader = true;
532 } else {
533 // Client should lex another token unless we generated an EOM.
534 return LeavingSubmodule;
535 }
536 }
537 // If this is the end of the main file, form an EOF token.
538 assert(CurLexer && "Got EOF but no current lexer set!");
539 const char *EndPos = getCurLexerEndPos();
540 Result.startToken();
541 CurLexer->BufferPtr = EndPos;
542
543 if (getLangOpts().IncrementalExtensions) {
544 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_repl_input_end);
545 Result.setAnnotationEndLoc(Result.getLocation());
546 Result.setAnnotationValue(nullptr);
547 } else {
548 CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
549 }
550
552 // Inserting the code-completion point increases the source buffer by 1,
553 // but the main FileID was created before inserting the point.
554 // Compensate by reducing the EOF location by 1, otherwise the location
555 // will point to the next FileID.
556 // FIXME: This is hacky, the code-completion point should probably be
557 // inserted before the main FileID is created.
558 if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
559 Result.setLocation(Result.getLocation().getLocWithOffset(-1));
560 }
561
562 if (creatingPCHWithThroughHeader() && !LeavingPCHThroughHeader) {
563 // Reached the end of the compilation without finding the through header.
564 Diag(CurLexer->getFileLoc(), diag::err_pp_through_header_not_seen)
565 << PPOpts->PCHThroughHeader << 0;
566 }
567
569 // We're done with lexing.
570 CurLexer.reset();
571
573 CurPPLexer = nullptr;
574
575 if (TUKind == TU_Complete) {
576 // This is the end of the top-level file. 'WarnUnusedMacroLocs' has
577 // collected all macro locations that we need to warn because they are not
578 // used.
579 for (WarnUnusedMacroLocsTy::iterator
580 I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
581 I!=E; ++I)
582 Diag(*I, diag::pp_macro_not_used);
583 }
584
585 // If we are building a module that has an umbrella header, make sure that
586 // each of the headers within the directory, including all submodules, is
587 // covered by the umbrella header was actually included by the umbrella
588 // header.
589 if (Module *Mod = getCurrentModule()) {
592 for (auto *M : AllMods)
593 diagnoseMissingHeaderInUmbrellaDir(*M);
594 }
595
596 return true;
597}
598
599/// HandleEndOfTokenLexer - This callback is invoked when the current TokenLexer
600/// hits the end of its token stream.
602 assert(CurTokenLexer && !CurPPLexer &&
603 "Ending a macro when currently in a #include file!");
604
605 if (!MacroExpandingLexersStack.empty() &&
606 MacroExpandingLexersStack.back().first == CurTokenLexer.get())
607 removeCachedMacroExpandedTokensOfLastLexer();
608
609 // Delete or cache the now-dead macro expander.
610 if (NumCachedTokenLexers == TokenLexerCacheSize)
611 CurTokenLexer.reset();
612 else
613 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
614
615 // Handle this like a #include file being popped off the stack.
616 return HandleEndOfFile(Result, true);
617}
618
619/// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
620/// lexer stack. This should only be used in situations where the current
621/// state of the top-of-stack lexer is unknown.
623 assert(!IncludeMacroStack.empty() && "Ran out of stack entries to load");
624
625 if (CurTokenLexer) {
626 // Delete or cache the now-dead macro expander.
627 if (NumCachedTokenLexers == TokenLexerCacheSize)
628 CurTokenLexer.reset();
629 else
630 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
631 }
632
633 PopIncludeMacroStack();
634}
635
636/// HandleMicrosoftCommentPaste - When the macro expander pastes together a
637/// comment (/##/) in microsoft mode, this method handles updating the current
638/// state, returning the token on the next source line.
640 assert(CurTokenLexer && !CurPPLexer &&
641 "Pasted comment can only be formed from macro");
642 // We handle this by scanning for the closest real lexer, switching it to
643 // raw mode and preprocessor mode. This will cause it to return \n as an
644 // explicit EOD token.
645 PreprocessorLexer *FoundLexer = nullptr;
646 bool LexerWasInPPMode = false;
647 for (const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
648 if (ISI.ThePPLexer == nullptr) continue; // Scan for a real lexer.
649
650 // Once we find a real lexer, mark it as raw mode (disabling macro
651 // expansions) and preprocessor mode (return EOD). We know that the lexer
652 // was *not* in raw mode before, because the macro that the comment came
653 // from was expanded. However, it could have already been in preprocessor
654 // mode (#if COMMENT) in which case we have to return it to that mode and
655 // return EOD.
656 FoundLexer = ISI.ThePPLexer;
657 FoundLexer->LexingRawMode = true;
658 LexerWasInPPMode = FoundLexer->ParsingPreprocessorDirective;
659 FoundLexer->ParsingPreprocessorDirective = true;
660 break;
661 }
662
663 // Okay, we either found and switched over the lexer, or we didn't find a
664 // lexer. In either case, finish off the macro the comment came from, getting
665 // the next token.
666 if (!HandleEndOfTokenLexer(Tok)) Lex(Tok);
667
668 // Discarding comments as long as we don't have EOF or EOD. This 'comments
669 // out' the rest of the line, including any tokens that came from other macros
670 // that were active, as in:
671 // #define submacro a COMMENT b
672 // submacro c
673 // which should lex to 'a' only: 'b' and 'c' should be removed.
674 while (Tok.isNot(tok::eod) && Tok.isNot(tok::eof))
675 Lex(Tok);
676
677 // If we got an eod token, then we successfully found the end of the line.
678 if (Tok.is(tok::eod)) {
679 assert(FoundLexer && "Can't get end of line without an active lexer");
680 // Restore the lexer back to normal mode instead of raw mode.
681 FoundLexer->LexingRawMode = false;
682
683 // If the lexer was already in preprocessor mode, just return the EOD token
684 // to finish the preprocessor line.
685 if (LexerWasInPPMode) return;
686
687 // Otherwise, switch out of PP mode and return the next lexed token.
688 FoundLexer->ParsingPreprocessorDirective = false;
689 return Lex(Tok);
690 }
691
692 // If we got an EOF token, then we reached the end of the token stream but
693 // didn't find an explicit \n. This can only happen if there was no lexer
694 // active (an active lexer would return EOD at EOF if there was no \n in
695 // preprocessor directive mode), so just return EOF as our token.
696 assert(!FoundLexer && "Lexer should return EOD before EOF in PP mode");
697}
698
700 bool ForPragma) {
701 if (!getLangOpts().ModulesLocalVisibility) {
702 // Just track that we entered this submodule.
703 BuildingSubmoduleStack.push_back(
704 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
705 PendingModuleMacroNames.size()));
706 if (Callbacks)
707 Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma);
708 return;
709 }
710
711 // Resolve as much of the module definition as we can now, before we enter
712 // one of its headers.
713 // FIXME: Can we enable Complain here?
714 // FIXME: Can we do this when local visibility is disabled?
716 ModMap.resolveExports(M, /*Complain=*/false);
717 ModMap.resolveUses(M, /*Complain=*/false);
718 ModMap.resolveConflicts(M, /*Complain=*/false);
719
720 // If this is the first time we've entered this module, set up its state.
721 auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
722 auto &State = R.first->second;
723 bool FirstTime = R.second;
724 if (FirstTime) {
725 // Determine the set of starting macros for this submodule; take these
726 // from the "null" module (the predefines buffer).
727 //
728 // FIXME: If we have local visibility but not modules enabled, the
729 // NullSubmoduleState is polluted by #defines in the top-level source
730 // file.
731 auto &StartingMacros = NullSubmoduleState.Macros;
732
733 // Restore to the starting state.
734 // FIXME: Do this lazily, when each macro name is first referenced.
735 for (auto &Macro : StartingMacros) {
736 // Skip uninteresting macros.
737 if (!Macro.second.getLatest() &&
738 Macro.second.getOverriddenMacros().empty())
739 continue;
740
741 MacroState MS(Macro.second.getLatest());
742 MS.setOverriddenMacros(*this, Macro.second.getOverriddenMacros());
743 State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
744 }
745 }
746
747 // Track that we entered this module.
748 BuildingSubmoduleStack.push_back(
749 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
750 PendingModuleMacroNames.size()));
751
752 if (Callbacks)
753 Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma);
754
755 // Switch to this submodule as the current submodule.
756 CurSubmoduleState = &State;
757
758 // This module is visible to itself.
759 if (FirstTime)
760 makeModuleVisible(M, ImportLoc);
761}
762
763bool Preprocessor::needModuleMacros() const {
764 // If we're not within a submodule, we never need to create ModuleMacros.
765 if (BuildingSubmoduleStack.empty())
766 return false;
767 // If we are tracking module macro visibility even for textually-included
768 // headers, we need ModuleMacros.
769 if (getLangOpts().ModulesLocalVisibility)
770 return true;
771 // Otherwise, we only need module macros if we're actually compiling a module
772 // interface.
774}
775
777 if (BuildingSubmoduleStack.empty() ||
778 BuildingSubmoduleStack.back().IsPragma != ForPragma) {
779 assert(ForPragma && "non-pragma module enter/leave mismatch");
780 return nullptr;
781 }
782
783 auto &Info = BuildingSubmoduleStack.back();
784
785 Module *LeavingMod = Info.M;
786 SourceLocation ImportLoc = Info.ImportLoc;
787
788 if (!needModuleMacros() ||
789 (!getLangOpts().ModulesLocalVisibility &&
790 LeavingMod->getTopLevelModuleName() != getLangOpts().CurrentModule)) {
791 // If we don't need module macros, or this is not a module for which we
792 // are tracking macro visibility, don't build any, and preserve the list
793 // of pending names for the surrounding submodule.
794 BuildingSubmoduleStack.pop_back();
795
796 if (Callbacks)
797 Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma);
798
799 makeModuleVisible(LeavingMod, ImportLoc);
800 return LeavingMod;
801 }
802
803 // Create ModuleMacros for any macros defined in this submodule.
805 for (unsigned I = Info.OuterPendingModuleMacroNames;
806 I != PendingModuleMacroNames.size(); ++I) {
807 auto *II = PendingModuleMacroNames[I];
808 if (!VisitedMacros.insert(II).second)
809 continue;
810
811 auto MacroIt = CurSubmoduleState->Macros.find(II);
812 if (MacroIt == CurSubmoduleState->Macros.end())
813 continue;
814 auto &Macro = MacroIt->second;
815
816 // Find the starting point for the MacroDirective chain in this submodule.
817 MacroDirective *OldMD = nullptr;
818 auto *OldState = Info.OuterSubmoduleState;
819 if (getLangOpts().ModulesLocalVisibility)
820 OldState = &NullSubmoduleState;
821 if (OldState && OldState != CurSubmoduleState) {
822 // FIXME: It'd be better to start at the state from when we most recently
823 // entered this submodule, but it doesn't really matter.
824 auto &OldMacros = OldState->Macros;
825 auto OldMacroIt = OldMacros.find(II);
826 if (OldMacroIt == OldMacros.end())
827 OldMD = nullptr;
828 else
829 OldMD = OldMacroIt->second.getLatest();
830 }
831
832 // This module may have exported a new macro. If so, create a ModuleMacro
833 // representing that fact.
834 bool ExplicitlyPublic = false;
835 for (auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->getPrevious()) {
836 assert(MD && "broken macro directive chain");
837
838 if (auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
839 // The latest visibility directive for a name in a submodule affects
840 // all the directives that come before it.
841 if (VisMD->isPublic())
842 ExplicitlyPublic = true;
843 else if (!ExplicitlyPublic)
844 // Private with no following public directive: not exported.
845 break;
846 } else {
847 MacroInfo *Def = nullptr;
848 if (DefMacroDirective *DefMD = dyn_cast<DefMacroDirective>(MD))
849 Def = DefMD->getInfo();
850
851 // FIXME: Issue a warning if multiple headers for the same submodule
852 // define a macro, rather than silently ignoring all but the first.
853 bool IsNew;
854 // Don't bother creating a module macro if it would represent a #undef
855 // that doesn't override anything.
856 if (Def || !Macro.getOverriddenMacros().empty())
857 addModuleMacro(LeavingMod, II, Def, Macro.getOverriddenMacros(),
858 IsNew);
859
860 if (!getLangOpts().ModulesLocalVisibility) {
861 // This macro is exposed to the rest of this compilation as a
862 // ModuleMacro; we don't need to track its MacroDirective any more.
863 Macro.setLatest(nullptr);
864 Macro.setOverriddenMacros(*this, {});
865 }
866 break;
867 }
868 }
869 }
870 PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames);
871
872 // FIXME: Before we leave this submodule, we should parse all the other
873 // headers within it. Otherwise, we're left with an inconsistent state
874 // where we've made the module visible but don't yet have its complete
875 // contents.
876
877 // Put back the outer module's state, if we're tracking it.
878 if (getLangOpts().ModulesLocalVisibility)
879 CurSubmoduleState = Info.OuterSubmoduleState;
880
881 BuildingSubmoduleStack.pop_back();
882
883 if (Callbacks)
884 Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma);
885
886 // A nested #include makes the included submodule visible.
887 makeModuleVisible(LeavingMod, ImportLoc);
888 return LeavingMod;
889}
IndirectLocalPath & Path
Expr * E
Defines the clang::FileManager interface and associated types.
llvm::MachO::FileType FileType
Definition: MachO.h:46
Defines the clang::MacroInfo and clang::MacroDirective classes.
static void collectAllSubModulesWithUmbrellaHeader(const Module &Mod, SmallVectorImpl< const Module * > &SubMods)
static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir, FileEntryRef File, SmallString< 128 > &Result)
Compute the relative path that names the given file relative to the given directory.
Defines the clang::Preprocessor interface.
SourceLocation Loc
Definition: SemaObjC.cpp:758
Defines the SourceManager interface.
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:432
StringRef getName() const
Cached information about one directory (either on disk or in the virtual file system).
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
Definition: FileEntry.h:57
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isValid() const
Implements support for file system lookup, file system caching, and directory search management.
Definition: FileManager.h:53
llvm::vfs::FileSystem & getVirtualFileSystem() const
Definition: FileManager.h:251
llvm::ErrorOr< const DirectoryEntry * > getDirectory(StringRef DirName, bool CacheFailure=true)
Lookup, cache, and verify the specified directory (real or virtual).
static FixItHint CreateReplacement(CharSourceRange RemoveRange, StringRef Code)
Create a code modification hint that replaces the given source range with the given code string.
Definition: Diagnostic.h:134
void SetFileControllingMacro(FileEntryRef File, const IdentifierInfo *ControllingMacro)
Mark the specified file as having a controlling macro.
Definition: HeaderSearch.h:577
ModuleMap & getModuleMap()
Retrieve the module map.
Definition: HeaderSearch.h:833
One of these records is kept for each identifier that is lexed.
bool isCompilingModule() const
Are we compiling a module?
Definition: LangOptions.h:606
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.
Definition: Lexer.h:78
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
Encapsulates changes to the "macros namespace" (the location where the macro name became active,...
Definition: MacroInfo.h:313
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Definition: MacroInfo.h:354
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
bool resolveExports(Module *Mod, bool Complain)
Resolve all of the unresolved exports in the given module.
Definition: ModuleMap.cpp:1399
bool resolveConflicts(Module *Mod, bool Complain)
Resolve all of the unresolved conflicts in the given module.
Definition: ModuleMap.cpp:1426
bool isHeaderInUnavailableModule(FileEntryRef Header) const
Determine whether the given header is part of a module marked 'unavailable'.
Definition: ModuleMap.cpp:728
bool resolveUses(Module *Mod, bool Complain)
Resolve all of the unresolved uses in the given module.
Definition: ModuleMap.cpp:1412
Describes a module or submodule.
Definition: Module.h:105
StringRef getTopLevelModuleName() const
Retrieve the name of the top-level module.
Definition: Module.h:676
llvm::iterator_range< submodule_iterator > submodules()
Definition: Module.h:783
std::optional< Header > getUmbrellaHeaderAsWritten() const
Retrieve the umbrella header as written.
Definition: Module.h:700
std::string getFullModuleName(bool AllowStringLiterals=false) const
Retrieve the full name of this module, including the path from its top-level module.
Definition: Module.cpp:244
OptionalDirectoryEntryRef getEffectiveUmbrellaDir() const
Get the effective umbrella directory for this module: either the one explicitly written in the module...
Definition: Module.cpp:269
const IdentifierInfo * GetDefinedMacro() const
If the ControllingMacro is followed by a macro definition, return the macro that was defined.
const IdentifierInfo * GetControllingMacroAtEndOfFile() const
Once the entire file has been lexed, if there is a controlling macro, return it.
SourceLocation GetDefinedLocation() const
SourceLocation GetMacroLocation() const
unsigned getInitialNumSLocEntries() const
Number of SLocEntries before lexing the file.
bool LexingRawMode
True if in raw mode.
bool ParsingPreprocessorDirective
True when parsing #XXX; turns '\n' into a tok::eod token.
MultipleIncludeOpt MIOpt
A state machine that detects the #ifndef-wrapping a file idiom for the multiple-include optimization.
OptionalFileEntryRef getFileEntry() const
getFileEntry - Return the FileEntry corresponding to this FileID.
virtual SourceLocation getSourceLocation()=0
Return the source location for the next observable location.
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
bool isRecordingPreamble() const
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
void makeModuleVisible(Module *M, SourceLocation Loc)
void Lex(Token &Result)
Lex the next token for this preprocessor.
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
Definition: Preprocessor.h:295
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
SourceManager & getSourceManager() const
bool isMacroDefined(StringRef Id)
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
bool isPPInSafeBufferOptOutRegion()
FileManager & getFileManager() const
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
HeaderSearch & getHeaderSearchInfo() const
Module * LeaveSubmodule(bool ForPragma)
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
const LangOptions & getLangOpts() const
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
DiagnosticsEngine & getDiagnostics() const
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
Encodes a location in the source.
bool isValid() const
Return true if this is a valid SourceLocation object.
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
OptionalFileEntryRef getFileEntryRefForID(FileID FID) const
Returns the FileEntryRef for the provided FileID.
StringRef getBufferName(SourceLocation Loc, bool *Invalid=nullptr) const
Return the filename or buffer identifier of the buffer the location is in.
FileID translateFile(const FileEntry *SourceFile) const
Get the FileID for the given file.
unsigned local_sloc_entry_size() const
Get the number of local SLocEntries we have.
SourceLocation getLocForEndOfFile(FileID FID) const
Return the source location corresponding to the last byte of the specified file.
SourceLocation getIncludeLoc(FileID FID) const
Returns the include location if FID is a #include'd file otherwise it returns an invalid location.
void setNumCreatedFIDsForFileID(FileID FID, unsigned NumFIDs, bool Force=false)
Set the number of FileIDs (files and macros) that were created during preprocessing of FID,...
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file.
SrcMgr::CharacteristicKind getFileCharacteristic(SourceLocation Loc) const
Return the file characteristic of the specified source location, indicating whether this is a normal ...
std::optional< llvm::MemoryBufferRef > getBufferOrNone(FileID FID, SourceLocation Loc=SourceLocation()) const
Return the buffer for the specified FileID.
Token - This structure provides full information about a lexed token.
Definition: Token.h:36
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:99
bool isNot(tok::TokenKind K) const
Definition: Token.h:100
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
Definition: SourceManager.h:81
The JSON file list parser is used to communicate input to InstallAPI.
@ Result
The result type of a method or function.
@ TU_Complete
The translation unit is a complete translation unit.
Definition: LangOptions.h:1039