clang  8.0.0svn
PPLexerChange.cpp
Go to the documentation of this file.
1 //===--- PPLexerChange.cpp - Handle changing lexers in the preprocessor ---===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements pieces of the Preprocessor interface that manage the
11 // current lexer stack.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "clang/Lex/Preprocessor.h"
19 #include "clang/Lex/HeaderSearch.h"
21 #include "clang/Lex/MacroInfo.h"
22 #include "clang/Lex/PTHManager.h"
23 #include "llvm/ADT/StringSwitch.h"
24 #include "llvm/Support/FileSystem.h"
25 #include "llvm/Support/MemoryBuffer.h"
26 #include "llvm/Support/Path.h"
27 using namespace clang;
28 
30 
31 //===----------------------------------------------------------------------===//
32 // Miscellaneous Methods.
33 //===----------------------------------------------------------------------===//
34 
35 /// isInPrimaryFile - Return true if we're in the top-level file, not in a
36 /// \#include. This looks through macro expansions and active _Pragma lexers.
38  if (IsFileLexer())
39  return IncludeMacroStack.empty();
40 
41  // If there are any stacked lexers, we're in a #include.
42  assert(IsFileLexer(IncludeMacroStack[0]) &&
43  "Top level include stack isn't our primary lexer?");
44  return std::none_of(
45  IncludeMacroStack.begin() + 1, IncludeMacroStack.end(),
46  [&](const IncludeStackInfo &ISI) -> bool { return IsFileLexer(ISI); });
47 }
48 
49 /// getCurrentLexer - Return the current file lexer being lexed from. Note
50 /// that this ignores any potentially active macro expansions and _Pragma
51 /// expansions going on at the time.
53  if (IsFileLexer())
54  return CurPPLexer;
55 
56  // Look for a stacked lexer.
57  for (const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
58  if (IsFileLexer(ISI))
59  return ISI.ThePPLexer;
60  }
61  return nullptr;
62 }
63 
64 
65 //===----------------------------------------------------------------------===//
66 // Methods for Entering and Callbacks for leaving various contexts
67 //===----------------------------------------------------------------------===//
68 
69 /// EnterSourceFile - Add a source file to the top of the include stack and
70 /// start lexing tokens from it instead of the current buffer.
72  SourceLocation Loc) {
73  assert(!CurTokenLexer && "Cannot #include a file inside a macro!");
74  ++NumEnteredSourceFiles;
75 
76  if (MaxIncludeStackDepth < IncludeMacroStack.size())
77  MaxIncludeStackDepth = IncludeMacroStack.size();
78 
79  if (PTH) {
80  if (PTHLexer *PL = PTH->CreateLexer(FID)) {
81  EnterSourceFileWithPTH(PL, CurDir);
82  return false;
83  }
84  }
85 
86  // Get the MemoryBuffer for this FID, if it fails, we fail.
87  bool Invalid = false;
88  const llvm::MemoryBuffer *InputFile =
89  getSourceManager().getBuffer(FID, Loc, &Invalid);
90  if (Invalid) {
91  SourceLocation FileStart = SourceMgr.getLocForStartOfFile(FID);
92  Diag(Loc, diag::err_pp_error_opening_file)
93  << std::string(SourceMgr.getBufferName(FileStart)) << "";
94  return true;
95  }
96 
97  if (isCodeCompletionEnabled() &&
98  SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
99  CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
100  CodeCompletionLoc =
101  CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
102  }
103 
104  EnterSourceFileWithLexer(new Lexer(FID, InputFile, *this), CurDir);
105  return false;
106 }
107 
108 /// EnterSourceFileWithLexer - Add a source file to the top of the include stack
109 /// and start lexing tokens from it instead of the current buffer.
110 void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
111  const DirectoryLookup *CurDir) {
112 
113  // Add the current lexer to the include stack.
114  if (CurPPLexer || CurTokenLexer)
115  PushIncludeMacroStack();
116 
117  CurLexer.reset(TheLexer);
118  CurPPLexer = TheLexer;
119  CurDirLookup = CurDir;
120  CurLexerSubmodule = nullptr;
121  if (CurLexerKind != CLK_LexAfterModuleImport)
122  CurLexerKind = CLK_Lexer;
123 
124  // Notify the client, if desired, that we are in a new source file.
125  if (Callbacks && !CurLexer->Is_PragmaLexer) {
126  SrcMgr::CharacteristicKind FileType =
127  SourceMgr.getFileCharacteristic(CurLexer->getFileLoc());
128 
129  Callbacks->FileChanged(CurLexer->getFileLoc(),
130  PPCallbacks::EnterFile, FileType);
131  }
132 }
133 
134 /// EnterSourceFileWithPTH - Add a source file to the top of the include stack
135 /// and start getting tokens from it using the PTH cache.
136 void Preprocessor::EnterSourceFileWithPTH(PTHLexer *PL,
137  const DirectoryLookup *CurDir) {
138 
139  if (CurPPLexer || CurTokenLexer)
140  PushIncludeMacroStack();
141 
142  CurDirLookup = CurDir;
143  CurPTHLexer.reset(PL);
144  CurPPLexer = CurPTHLexer.get();
145  CurLexerSubmodule = nullptr;
146  if (CurLexerKind != CLK_LexAfterModuleImport)
147  CurLexerKind = CLK_PTHLexer;
148 
149  // Notify the client, if desired, that we are in a new source file.
150  if (Callbacks) {
151  FileID FID = CurPPLexer->getFileID();
152  SourceLocation EnterLoc = SourceMgr.getLocForStartOfFile(FID);
153  SrcMgr::CharacteristicKind FileType =
154  SourceMgr.getFileCharacteristic(EnterLoc);
155  Callbacks->FileChanged(EnterLoc, PPCallbacks::EnterFile, FileType);
156  }
157 }
158 
159 /// EnterMacro - Add a Macro to the top of the include stack and start lexing
160 /// tokens from it instead of the current buffer.
162  MacroInfo *Macro, MacroArgs *Args) {
163  std::unique_ptr<TokenLexer> TokLexer;
164  if (NumCachedTokenLexers == 0) {
165  TokLexer = llvm::make_unique<TokenLexer>(Tok, ILEnd, Macro, Args, *this);
166  } else {
167  TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
168  TokLexer->Init(Tok, ILEnd, Macro, Args);
169  }
170 
171  PushIncludeMacroStack();
172  CurDirLookup = nullptr;
173  CurTokenLexer = std::move(TokLexer);
174  if (CurLexerKind != CLK_LexAfterModuleImport)
175  CurLexerKind = CLK_TokenLexer;
176 }
177 
178 /// EnterTokenStream - Add a "macro" context to the top of the include stack,
179 /// which will cause the lexer to start returning the specified tokens.
180 ///
181 /// If DisableMacroExpansion is true, tokens lexed from the token stream will
182 /// not be subject to further macro expansion. Otherwise, these tokens will
183 /// be re-macro-expanded when/if expansion is enabled.
184 ///
185 /// If OwnsTokens is false, this method assumes that the specified stream of
186 /// tokens has a permanent owner somewhere, so they do not need to be copied.
187 /// If it is true, it assumes the array of tokens is allocated with new[] and
188 /// must be freed.
189 ///
190 void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
191  bool DisableMacroExpansion,
192  bool OwnsTokens) {
193  if (CurLexerKind == CLK_CachingLexer) {
194  if (CachedLexPos < CachedTokens.size()) {
195  // We're entering tokens into the middle of our cached token stream. We
196  // can't represent that, so just insert the tokens into the buffer.
197  CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
198  Toks, Toks + NumToks);
199  if (OwnsTokens)
200  delete [] Toks;
201  return;
202  }
203 
204  // New tokens are at the end of the cached token sequnece; insert the
205  // token stream underneath the caching lexer.
206  ExitCachingLexMode();
207  EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
208  EnterCachingLexMode();
209  return;
210  }
211 
212  // Create a macro expander to expand from the specified token stream.
213  std::unique_ptr<TokenLexer> TokLexer;
214  if (NumCachedTokenLexers == 0) {
215  TokLexer = llvm::make_unique<TokenLexer>(
216  Toks, NumToks, DisableMacroExpansion, OwnsTokens, *this);
217  } else {
218  TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
219  TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
220  }
221 
222  // Save our current state.
223  PushIncludeMacroStack();
224  CurDirLookup = nullptr;
225  CurTokenLexer = std::move(TokLexer);
226  if (CurLexerKind != CLK_LexAfterModuleImport)
227  CurLexerKind = CLK_TokenLexer;
228 }
229 
230 /// Compute the relative path that names the given file relative to
231 /// the given directory.
232 static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir,
233  const FileEntry *File,
235  Result.clear();
236 
237  StringRef FilePath = File->getDir()->getName();
238  StringRef Path = FilePath;
239  while (!Path.empty()) {
240  if (const DirectoryEntry *CurDir = FM.getDirectory(Path)) {
241  if (CurDir == Dir) {
242  Result = FilePath.substr(Path.size());
243  llvm::sys::path::append(Result,
244  llvm::sys::path::filename(File->getName()));
245  return;
246  }
247  }
248 
249  Path = llvm::sys::path::parent_path(Path);
250  }
251 
252  Result = File->getName();
253 }
254 
255 void Preprocessor::PropagateLineStartLeadingSpaceInfo(Token &Result) {
256  if (CurTokenLexer) {
257  CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result);
258  return;
259  }
260  if (CurLexer) {
261  CurLexer->PropagateLineStartLeadingSpaceInfo(Result);
262  return;
263  }
264  // FIXME: Handle other kinds of lexers? It generally shouldn't matter,
265  // but it might if they're empty?
266 }
267 
268 /// Determine the location to use as the end of the buffer for a lexer.
269 ///
270 /// If the file ends with a newline, form the EOF token on the newline itself,
271 /// rather than "on the line following it", which doesn't exist. This makes
272 /// diagnostics relating to the end of file include the last file that the user
273 /// actually typed, which is goodness.
274 const char *Preprocessor::getCurLexerEndPos() {
275  const char *EndPos = CurLexer->BufferEnd;
276  if (EndPos != CurLexer->BufferStart &&
277  (EndPos[-1] == '\n' || EndPos[-1] == '\r')) {
278  --EndPos;
279 
280  // Handle \n\r and \r\n:
281  if (EndPos != CurLexer->BufferStart &&
282  (EndPos[-1] == '\n' || EndPos[-1] == '\r') &&
283  EndPos[-1] != EndPos[0])
284  --EndPos;
285  }
286 
287  return EndPos;
288 }
289 
291  const Module &Mod, SmallVectorImpl<const Module *> &SubMods) {
292  if (Mod.getUmbrellaHeader())
293  SubMods.push_back(&Mod);
294  for (auto *M : Mod.submodules())
296 }
297 
298 void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(const Module &Mod) {
299  assert(Mod.getUmbrellaHeader() && "Module must use umbrella header");
300  SourceLocation StartLoc =
301  SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());
302  if (getDiagnostics().isIgnored(diag::warn_uncovered_module_header, StartLoc))
303  return;
304 
305  ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
306  const DirectoryEntry *Dir = Mod.getUmbrellaDir().Entry;
307  llvm::vfs::FileSystem &FS = *FileMgr.getVirtualFileSystem();
308  std::error_code EC;
309  for (llvm::vfs::recursive_directory_iterator Entry(FS, Dir->getName(), EC),
310  End;
311  Entry != End && !EC; Entry.increment(EC)) {
312  using llvm::StringSwitch;
313 
314  // Check whether this entry has an extension typically associated with
315  // headers.
316  if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->path()))
317  .Cases(".h", ".H", ".hh", ".hpp", true)
318  .Default(false))
319  continue;
320 
321  if (const FileEntry *Header = getFileManager().getFile(Entry->path()))
322  if (!getSourceManager().hasFileInfo(Header)) {
323  if (!ModMap.isHeaderInUnavailableModule(Header)) {
324  // Find the relative path that would access this header.
325  SmallString<128> RelativePath;
326  computeRelativePath(FileMgr, Dir, Header, RelativePath);
327  Diag(StartLoc, diag::warn_uncovered_module_header)
328  << Mod.getFullModuleName() << RelativePath;
329  }
330  }
331  }
332 }
333 
334 /// HandleEndOfFile - This callback is invoked when the lexer hits the end of
335 /// the current file. This either returns the EOF token or pops a level off
336 /// the include stack and keeps going.
337 bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
338  assert(!CurTokenLexer &&
339  "Ending a file when currently in a macro!");
340 
341  // If we have an unclosed module region from a pragma at the end of a
342  // module, complain and close it now.
343  // FIXME: This is not correct if we are building a module from PTH.
344  const bool LeavingSubmodule = CurLexer && CurLexerSubmodule;
345  if ((LeavingSubmodule || IncludeMacroStack.empty()) &&
346  !BuildingSubmoduleStack.empty() &&
347  BuildingSubmoduleStack.back().IsPragma) {
348  Diag(BuildingSubmoduleStack.back().ImportLoc,
349  diag::err_pp_module_begin_without_module_end);
350  Module *M = LeaveSubmodule(/*ForPragma*/true);
351 
352  Result.startToken();
353  const char *EndPos = getCurLexerEndPos();
354  CurLexer->BufferPtr = EndPos;
355  CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
356  Result.setAnnotationEndLoc(Result.getLocation());
357  Result.setAnnotationValue(M);
358  return true;
359  }
360 
361  // See if this file had a controlling macro.
362  if (CurPPLexer) { // Not ending a macro, ignore it.
363  if (const IdentifierInfo *ControllingMacro =
364  CurPPLexer->MIOpt.GetControllingMacroAtEndOfFile()) {
365  // Okay, this has a controlling macro, remember in HeaderFileInfo.
366  if (const FileEntry *FE = CurPPLexer->getFileEntry()) {
367  HeaderInfo.SetFileControllingMacro(FE, ControllingMacro);
368  if (MacroInfo *MI =
369  getMacroInfo(const_cast<IdentifierInfo*>(ControllingMacro)))
370  MI->setUsedForHeaderGuard(true);
371  if (const IdentifierInfo *DefinedMacro =
372  CurPPLexer->MIOpt.GetDefinedMacro()) {
373  if (!isMacroDefined(ControllingMacro) &&
374  DefinedMacro != ControllingMacro &&
375  HeaderInfo.FirstTimeLexingFile(FE)) {
376 
377  // If the edit distance between the two macros is more than 50%,
378  // DefinedMacro may not be header guard, or can be header guard of
379  // another header file. Therefore, it maybe defining something
380  // completely different. This can be observed in the wild when
381  // handling feature macros or header guards in different files.
382 
383  const StringRef ControllingMacroName = ControllingMacro->getName();
384  const StringRef DefinedMacroName = DefinedMacro->getName();
385  const size_t MaxHalfLength = std::max(ControllingMacroName.size(),
386  DefinedMacroName.size()) / 2;
387  const unsigned ED = ControllingMacroName.edit_distance(
388  DefinedMacroName, true, MaxHalfLength);
389  if (ED <= MaxHalfLength) {
390  // Emit a warning for a bad header guard.
391  Diag(CurPPLexer->MIOpt.GetMacroLocation(),
392  diag::warn_header_guard)
393  << CurPPLexer->MIOpt.GetMacroLocation() << ControllingMacro;
394  Diag(CurPPLexer->MIOpt.GetDefinedLocation(),
395  diag::note_header_guard)
396  << CurPPLexer->MIOpt.GetDefinedLocation() << DefinedMacro
397  << ControllingMacro
399  CurPPLexer->MIOpt.GetDefinedLocation(),
400  ControllingMacro->getName());
401  }
402  }
403  }
404  }
405  }
406  }
407 
408  // Complain about reaching a true EOF within arc_cf_code_audited.
409  // We don't want to complain about reaching the end of a macro
410  // instantiation or a _Pragma.
411  if (PragmaARCCFCodeAuditedLoc.isValid() &&
412  !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
413  Diag(PragmaARCCFCodeAuditedLoc, diag::err_pp_eof_in_arc_cf_code_audited);
414 
415  // Recover by leaving immediately.
416  PragmaARCCFCodeAuditedLoc = SourceLocation();
417  }
418 
419  // Complain about reaching a true EOF within assume_nonnull.
420  // We don't want to complain about reaching the end of a macro
421  // instantiation or a _Pragma.
422  if (PragmaAssumeNonNullLoc.isValid() &&
423  !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
424  Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
425 
426  // Recover by leaving immediately.
427  PragmaAssumeNonNullLoc = SourceLocation();
428  }
429 
430  bool LeavingPCHThroughHeader = false;
431 
432  // If this is a #include'd file, pop it off the include stack and continue
433  // lexing the #includer file.
434  if (!IncludeMacroStack.empty()) {
435 
436  // If we lexed the code-completion file, act as if we reached EOF.
437  if (isCodeCompletionEnabled() && CurPPLexer &&
438  SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
439  CodeCompletionFileLoc) {
440  if (CurLexer) {
441  Result.startToken();
442  CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof);
443  CurLexer.reset();
444  } else {
445  assert(CurPTHLexer && "Got EOF but no current lexer set!");
446  CurPTHLexer->getEOF(Result);
447  CurPTHLexer.reset();
448  }
449 
450  CurPPLexer = nullptr;
451  recomputeCurLexerKind();
452  return true;
453  }
454 
455  if (!isEndOfMacro && CurPPLexer &&
456  SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid()) {
457  // Notify SourceManager to record the number of FileIDs that were created
458  // during lexing of the #include'd file.
459  unsigned NumFIDs =
460  SourceMgr.local_sloc_entry_size() -
461  CurPPLexer->getInitialNumSLocEntries() + 1/*#include'd file*/;
462  SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
463  }
464 
465  bool ExitedFromPredefinesFile = false;
466  FileID ExitedFID;
467  if (!isEndOfMacro && CurPPLexer) {
468  ExitedFID = CurPPLexer->getFileID();
469 
470  assert(PredefinesFileID.isValid() &&
471  "HandleEndOfFile is called before PredefinesFileId is set");
472  ExitedFromPredefinesFile = (PredefinesFileID == ExitedFID);
473  }
474 
475  if (LeavingSubmodule) {
476  // We're done with this submodule.
477  Module *M = LeaveSubmodule(/*ForPragma*/false);
478 
479  // Notify the parser that we've left the module.
480  const char *EndPos = getCurLexerEndPos();
481  Result.startToken();
482  CurLexer->BufferPtr = EndPos;
483  CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
484  Result.setAnnotationEndLoc(Result.getLocation());
485  Result.setAnnotationValue(M);
486  }
487 
488  bool FoundPCHThroughHeader = false;
489  if (CurPPLexer && creatingPCHWithThroughHeader() &&
490  isPCHThroughHeader(
491  SourceMgr.getFileEntryForID(CurPPLexer->getFileID())))
492  FoundPCHThroughHeader = true;
493 
494  // We're done with the #included file.
495  RemoveTopOfLexerStack();
496 
497  // Propagate info about start-of-line/leading white-space/etc.
498  PropagateLineStartLeadingSpaceInfo(Result);
499 
500  // Notify the client, if desired, that we are in a new source file.
501  if (Callbacks && !isEndOfMacro && CurPPLexer) {
502  SrcMgr::CharacteristicKind FileType =
503  SourceMgr.getFileCharacteristic(CurPPLexer->getSourceLocation());
504  Callbacks->FileChanged(CurPPLexer->getSourceLocation(),
505  PPCallbacks::ExitFile, FileType, ExitedFID);
506  }
507 
508  // Restore conditional stack from the preamble right after exiting from the
509  // predefines file.
510  if (ExitedFromPredefinesFile)
511  replayPreambleConditionalStack();
512 
513  if (!isEndOfMacro && CurPPLexer && FoundPCHThroughHeader &&
514  (isInPrimaryFile() ||
515  CurPPLexer->getFileID() == getPredefinesFileID())) {
516  // Leaving the through header. Continue directly to end of main file
517  // processing.
518  LeavingPCHThroughHeader = true;
519  } else {
520  // Client should lex another token unless we generated an EOM.
521  return LeavingSubmodule;
522  }
523  }
524 
525  // If this is the end of the main file, form an EOF token.
526  if (CurLexer) {
527  const char *EndPos = getCurLexerEndPos();
528  Result.startToken();
529  CurLexer->BufferPtr = EndPos;
530  CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
531 
532  if (isCodeCompletionEnabled()) {
533  // Inserting the code-completion point increases the source buffer by 1,
534  // but the main FileID was created before inserting the point.
535  // Compensate by reducing the EOF location by 1, otherwise the location
536  // will point to the next FileID.
537  // FIXME: This is hacky, the code-completion point should probably be
538  // inserted before the main FileID is created.
539  if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
540  Result.setLocation(Result.getLocation().getLocWithOffset(-1));
541  }
542 
543  if (creatingPCHWithThroughHeader() && !LeavingPCHThroughHeader) {
544  // Reached the end of the compilation without finding the through header.
545  Diag(CurLexer->getFileLoc(), diag::err_pp_through_header_not_seen)
546  << PPOpts->PCHThroughHeader << 0;
547  }
548 
549  if (!isIncrementalProcessingEnabled())
550  // We're done with lexing.
551  CurLexer.reset();
552  } else {
553  assert(CurPTHLexer && "Got EOF but no current lexer set!");
554  CurPTHLexer->getEOF(Result);
555  CurPTHLexer.reset();
556  }
557 
558  if (!isIncrementalProcessingEnabled())
559  CurPPLexer = nullptr;
560 
561  if (TUKind == TU_Complete) {
562  // This is the end of the top-level file. 'WarnUnusedMacroLocs' has
563  // collected all macro locations that we need to warn because they are not
564  // used.
565  for (WarnUnusedMacroLocsTy::iterator
566  I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
567  I!=E; ++I)
568  Diag(*I, diag::pp_macro_not_used);
569  }
570 
571  // If we are building a module that has an umbrella header, make sure that
572  // each of the headers within the directory, including all submodules, is
573  // covered by the umbrella header was actually included by the umbrella
574  // header.
575  if (Module *Mod = getCurrentModule()) {
578  for (auto *M : AllMods)
579  diagnoseMissingHeaderInUmbrellaDir(*M);
580  }
581 
582  return true;
583 }
584 
585 /// HandleEndOfTokenLexer - This callback is invoked when the current TokenLexer
586 /// hits the end of its token stream.
588  assert(CurTokenLexer && !CurPPLexer &&
589  "Ending a macro when currently in a #include file!");
590 
591  if (!MacroExpandingLexersStack.empty() &&
592  MacroExpandingLexersStack.back().first == CurTokenLexer.get())
593  removeCachedMacroExpandedTokensOfLastLexer();
594 
595  // Delete or cache the now-dead macro expander.
596  if (NumCachedTokenLexers == TokenLexerCacheSize)
597  CurTokenLexer.reset();
598  else
599  TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
600 
601  // Handle this like a #include file being popped off the stack.
602  return HandleEndOfFile(Result, true);
603 }
604 
605 /// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
606 /// lexer stack. This should only be used in situations where the current
607 /// state of the top-of-stack lexer is unknown.
609  assert(!IncludeMacroStack.empty() && "Ran out of stack entries to load");
610 
611  if (CurTokenLexer) {
612  // Delete or cache the now-dead macro expander.
613  if (NumCachedTokenLexers == TokenLexerCacheSize)
614  CurTokenLexer.reset();
615  else
616  TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
617  }
618 
619  PopIncludeMacroStack();
620 }
621 
622 /// HandleMicrosoftCommentPaste - When the macro expander pastes together a
623 /// comment (/##/) in microsoft mode, this method handles updating the current
624 /// state, returning the token on the next source line.
626  assert(CurTokenLexer && !CurPPLexer &&
627  "Pasted comment can only be formed from macro");
628  // We handle this by scanning for the closest real lexer, switching it to
629  // raw mode and preprocessor mode. This will cause it to return \n as an
630  // explicit EOD token.
631  PreprocessorLexer *FoundLexer = nullptr;
632  bool LexerWasInPPMode = false;
633  for (const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
634  if (ISI.ThePPLexer == nullptr) continue; // Scan for a real lexer.
635 
636  // Once we find a real lexer, mark it as raw mode (disabling macro
637  // expansions) and preprocessor mode (return EOD). We know that the lexer
638  // was *not* in raw mode before, because the macro that the comment came
639  // from was expanded. However, it could have already been in preprocessor
640  // mode (#if COMMENT) in which case we have to return it to that mode and
641  // return EOD.
642  FoundLexer = ISI.ThePPLexer;
643  FoundLexer->LexingRawMode = true;
644  LexerWasInPPMode = FoundLexer->ParsingPreprocessorDirective;
645  FoundLexer->ParsingPreprocessorDirective = true;
646  break;
647  }
648 
649  // Okay, we either found and switched over the lexer, or we didn't find a
650  // lexer. In either case, finish off the macro the comment came from, getting
651  // the next token.
652  if (!HandleEndOfTokenLexer(Tok)) Lex(Tok);
653 
654  // Discarding comments as long as we don't have EOF or EOD. This 'comments
655  // out' the rest of the line, including any tokens that came from other macros
656  // that were active, as in:
657  // #define submacro a COMMENT b
658  // submacro c
659  // which should lex to 'a' only: 'b' and 'c' should be removed.
660  while (Tok.isNot(tok::eod) && Tok.isNot(tok::eof))
661  Lex(Tok);
662 
663  // If we got an eod token, then we successfully found the end of the line.
664  if (Tok.is(tok::eod)) {
665  assert(FoundLexer && "Can't get end of line without an active lexer");
666  // Restore the lexer back to normal mode instead of raw mode.
667  FoundLexer->LexingRawMode = false;
668 
669  // If the lexer was already in preprocessor mode, just return the EOD token
670  // to finish the preprocessor line.
671  if (LexerWasInPPMode) return;
672 
673  // Otherwise, switch out of PP mode and return the next lexed token.
674  FoundLexer->ParsingPreprocessorDirective = false;
675  return Lex(Tok);
676  }
677 
678  // If we got an EOF token, then we reached the end of the token stream but
679  // didn't find an explicit \n. This can only happen if there was no lexer
680  // active (an active lexer would return EOD at EOF if there was no \n in
681  // preprocessor directive mode), so just return EOF as our token.
682  assert(!FoundLexer && "Lexer should return EOD before EOF in PP mode");
683 }
684 
686  bool ForPragma) {
687  if (!getLangOpts().ModulesLocalVisibility) {
688  // Just track that we entered this submodule.
689  BuildingSubmoduleStack.push_back(
690  BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
691  PendingModuleMacroNames.size()));
692  return;
693  }
694 
695  // Resolve as much of the module definition as we can now, before we enter
696  // one of its headers.
697  // FIXME: Can we enable Complain here?
698  // FIXME: Can we do this when local visibility is disabled?
699  ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
700  ModMap.resolveExports(M, /*Complain=*/false);
701  ModMap.resolveUses(M, /*Complain=*/false);
702  ModMap.resolveConflicts(M, /*Complain=*/false);
703 
704  // If this is the first time we've entered this module, set up its state.
705  auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
706  auto &State = R.first->second;
707  bool FirstTime = R.second;
708  if (FirstTime) {
709  // Determine the set of starting macros for this submodule; take these
710  // from the "null" module (the predefines buffer).
711  //
712  // FIXME: If we have local visibility but not modules enabled, the
713  // NullSubmoduleState is polluted by #defines in the top-level source
714  // file.
715  auto &StartingMacros = NullSubmoduleState.Macros;
716 
717  // Restore to the starting state.
718  // FIXME: Do this lazily, when each macro name is first referenced.
719  for (auto &Macro : StartingMacros) {
720  // Skip uninteresting macros.
721  if (!Macro.second.getLatest() &&
722  Macro.second.getOverriddenMacros().empty())
723  continue;
724 
725  MacroState MS(Macro.second.getLatest());
726  MS.setOverriddenMacros(*this, Macro.second.getOverriddenMacros());
727  State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
728  }
729  }
730 
731  // Track that we entered this module.
732  BuildingSubmoduleStack.push_back(
733  BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
734  PendingModuleMacroNames.size()));
735 
736  // Switch to this submodule as the current submodule.
737  CurSubmoduleState = &State;
738 
739  // This module is visible to itself.
740  if (FirstTime)
741  makeModuleVisible(M, ImportLoc);
742 }
743 
744 bool Preprocessor::needModuleMacros() const {
745  // If we're not within a submodule, we never need to create ModuleMacros.
746  if (BuildingSubmoduleStack.empty())
747  return false;
748  // If we are tracking module macro visibility even for textually-included
749  // headers, we need ModuleMacros.
750  if (getLangOpts().ModulesLocalVisibility)
751  return true;
752  // Otherwise, we only need module macros if we're actually compiling a module
753  // interface.
754  return getLangOpts().isCompilingModule();
755 }
756 
758  if (BuildingSubmoduleStack.empty() ||
759  BuildingSubmoduleStack.back().IsPragma != ForPragma) {
760  assert(ForPragma && "non-pragma module enter/leave mismatch");
761  return nullptr;
762  }
763 
764  auto &Info = BuildingSubmoduleStack.back();
765 
766  Module *LeavingMod = Info.M;
767  SourceLocation ImportLoc = Info.ImportLoc;
768 
769  if (!needModuleMacros() ||
770  (!getLangOpts().ModulesLocalVisibility &&
771  LeavingMod->getTopLevelModuleName() != getLangOpts().CurrentModule)) {
772  // If we don't need module macros, or this is not a module for which we
773  // are tracking macro visibility, don't build any, and preserve the list
774  // of pending names for the surrounding submodule.
775  BuildingSubmoduleStack.pop_back();
776  makeModuleVisible(LeavingMod, ImportLoc);
777  return LeavingMod;
778  }
779 
780  // Create ModuleMacros for any macros defined in this submodule.
781  llvm::SmallPtrSet<const IdentifierInfo*, 8> VisitedMacros;
782  for (unsigned I = Info.OuterPendingModuleMacroNames;
783  I != PendingModuleMacroNames.size(); ++I) {
784  auto *II = const_cast<IdentifierInfo*>(PendingModuleMacroNames[I]);
785  if (!VisitedMacros.insert(II).second)
786  continue;
787 
788  auto MacroIt = CurSubmoduleState->Macros.find(II);
789  if (MacroIt == CurSubmoduleState->Macros.end())
790  continue;
791  auto &Macro = MacroIt->second;
792 
793  // Find the starting point for the MacroDirective chain in this submodule.
794  MacroDirective *OldMD = nullptr;
795  auto *OldState = Info.OuterSubmoduleState;
796  if (getLangOpts().ModulesLocalVisibility)
797  OldState = &NullSubmoduleState;
798  if (OldState && OldState != CurSubmoduleState) {
799  // FIXME: It'd be better to start at the state from when we most recently
800  // entered this submodule, but it doesn't really matter.
801  auto &OldMacros = OldState->Macros;
802  auto OldMacroIt = OldMacros.find(II);
803  if (OldMacroIt == OldMacros.end())
804  OldMD = nullptr;
805  else
806  OldMD = OldMacroIt->second.getLatest();
807  }
808 
809  // This module may have exported a new macro. If so, create a ModuleMacro
810  // representing that fact.
811  bool ExplicitlyPublic = false;
812  for (auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->getPrevious()) {
813  assert(MD && "broken macro directive chain");
814 
815  if (auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
816  // The latest visibility directive for a name in a submodule affects
817  // all the directives that come before it.
818  if (VisMD->isPublic())
819  ExplicitlyPublic = true;
820  else if (!ExplicitlyPublic)
821  // Private with no following public directive: not exported.
822  break;
823  } else {
824  MacroInfo *Def = nullptr;
825  if (DefMacroDirective *DefMD = dyn_cast<DefMacroDirective>(MD))
826  Def = DefMD->getInfo();
827 
828  // FIXME: Issue a warning if multiple headers for the same submodule
829  // define a macro, rather than silently ignoring all but the first.
830  bool IsNew;
831  // Don't bother creating a module macro if it would represent a #undef
832  // that doesn't override anything.
833  if (Def || !Macro.getOverriddenMacros().empty())
834  addModuleMacro(LeavingMod, II, Def,
835  Macro.getOverriddenMacros(), IsNew);
836 
837  if (!getLangOpts().ModulesLocalVisibility) {
838  // This macro is exposed to the rest of this compilation as a
839  // ModuleMacro; we don't need to track its MacroDirective any more.
840  Macro.setLatest(nullptr);
841  Macro.setOverriddenMacros(*this, {});
842  }
843  break;
844  }
845  }
846  }
847  PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames);
848 
849  // FIXME: Before we leave this submodule, we should parse all the other
850  // headers within it. Otherwise, we're left with an inconsistent state
851  // where we've made the module visible but don't yet have its complete
852  // contents.
853 
854  // Put back the outer module's state, if we're tracking it.
855  if (getLangOpts().ModulesLocalVisibility)
856  CurSubmoduleState = Info.OuterSubmoduleState;
857 
858  BuildingSubmoduleStack.pop_back();
859 
860  // A nested #include makes the included submodule visible.
861  makeModuleVisible(LeavingMod, ImportLoc);
862  return LeavingMod;
863 }
static DiagnosticBuilder Diag(DiagnosticsEngine *Diags, const LangOptions &Features, FullSourceLoc TokLoc, const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd, unsigned DiagID)
Produce a diagnostic highlighting some portion of a literal.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:77
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:117
Defines the clang::FileManager interface and associated types.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
Defines the SourceManager interface.
bool isInPrimaryFile() const
Return true if we&#39;re in the top-level file, not in a #include.
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool resolveUses(Module *Mod, bool Complain)
Resolve all of the unresolved uses in the given module.
Definition: ModuleMap.cpp:1260
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
Definition: SourceManager.h:78
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Definition: MacroInfo.h:329
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
One of these records is kept for each identifier that is lexed.
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:406
bool ParsingPreprocessorDirective
True when parsing #XXX; turns &#39;\n&#39; into a tok::eod token.
bool resolveConflicts(Module *Mod, bool Complain)
Resolve all of the unresolved conflicts in the given module.
Definition: ModuleMap.cpp:1273
LineState State
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
Module * LeaveSubmodule(bool ForPragma)
Header getUmbrellaHeader() const
Retrieve the header that serves as the umbrella header for this module.
Definition: Module.h:483
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
Describes a module or submodule.
Definition: Module.h:65
StringRef getTopLevelModuleName() const
Retrieve the name of the top-level module.
Definition: Module.h:461
std::string getFullModuleName(bool AllowStringLiterals=false) const
Retrieve the full name of this module, including the path from its top-level module.
Definition: Module.cpp:214
const FormatToken & Tok
const DirectoryEntry * getDirectory(StringRef DirName, bool CacheFailure=true)
Lookup, cache, and verify the specified directory (real or virtual).
const DirectoryEntry * Entry
Definition: Module.h:162
void setAnnotationValue(void *val)
Definition: Token.h:228
bool LexingRawMode
True if in raw mode.
SourceLocation End
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
bool resolveExports(Module *Mod, bool Complain)
Resolve all of the unresolved exports in the given module.
Definition: ModuleMap.cpp:1247
static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir, const FileEntry *File, SmallString< 128 > &Result)
Compute the relative path that names the given file relative to the given directory.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
The result type of a method or function.
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
const DirectoryEntry * getDir() const
Return the directory the file lives in.
Definition: FileManager.h:95
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:291
Encodes a location in the source.
StringRef getName() const
Definition: FileManager.h:85
void setAnnotationEndLoc(SourceLocation L)
Definition: Token.h:142
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:60
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:96
bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
Dataflow Directional Tag Classes.
llvm::iterator_range< submodule_iterator > submodules()
Definition: Module.h:561
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:40
DirectoryName getUmbrellaDir() const
Retrieve the directory for which this module serves as the umbrella.
Definition: Module.cpp:239
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:46
static void collectAllSubModulesWithUmbrellaHeader(const Module &Mod, SmallVectorImpl< const Module *> &SubMods)
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
static FixItHint CreateReplacement(CharSourceRange RemoveRange, StringRef Code)
Create a code modification hint that replaces the given source range with the given code string...
Definition: Diagnostic.h:129
The translation unit is a complete translation unit.
Definition: LangOptions.h:331
bool isHeaderInUnavailableModule(const FileEntry *Header) const
Determine whether the given header is part of a module marked &#39;unavailable&#39;.
Definition: ModuleMap.cpp:666
__DEVICE__ int max(int __a, int __b)
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
void setLocation(SourceLocation L)
Definition: Token.h:132
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
void startToken()
Reset all flags to cleared.
Definition: Token.h:169
StringRef getName() const
Definition: FileManager.h:52