clang  8.0.0svn
InclusionRewriter.cpp
Go to the documentation of this file.
1 //===--- InclusionRewriter.cpp - Rewrite includes into their expansions ---===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This code rewrites include invocations into their expansions. This gives you
11 // a file with all included files merged into it.
12 //
13 //===----------------------------------------------------------------------===//
14 
18 #include "clang/Lex/HeaderSearch.h"
19 #include "clang/Lex/Pragma.h"
20 #include "clang/Lex/Preprocessor.h"
21 #include "llvm/ADT/SmallString.h"
22 #include "llvm/Support/raw_ostream.h"
23 
24 using namespace clang;
25 using namespace llvm;
26 
27 namespace {
28 
29 class InclusionRewriter : public PPCallbacks {
30  /// Information about which #includes were actually performed,
31  /// created by preprocessor callbacks.
32  struct IncludedFile {
33  FileID Id;
35  const DirectoryLookup *DirLookup;
36  IncludedFile(FileID Id, SrcMgr::CharacteristicKind FileType,
37  const DirectoryLookup *DirLookup)
38  : Id(Id), FileType(FileType), DirLookup(DirLookup) {}
39  };
40  Preprocessor &PP; ///< Used to find inclusion directives.
41  SourceManager &SM; ///< Used to read and manage source files.
42  raw_ostream &OS; ///< The destination stream for rewritten contents.
43  StringRef MainEOL; ///< The line ending marker to use.
44  const llvm::MemoryBuffer *PredefinesBuffer; ///< The preprocessor predefines.
45  bool ShowLineMarkers; ///< Show #line markers.
46  bool UseLineDirectives; ///< Use of line directives or line markers.
47  /// Tracks where inclusions that change the file are found.
48  std::map<unsigned, IncludedFile> FileIncludes;
49  /// Tracks where inclusions that import modules are found.
50  std::map<unsigned, const Module *> ModuleIncludes;
51  /// Tracks where inclusions that enter modules (in a module build) are found.
52  std::map<unsigned, const Module *> ModuleEntryIncludes;
53  /// Used transitively for building up the FileIncludes mapping over the
54  /// various \c PPCallbacks callbacks.
55  SourceLocation LastInclusionLocation;
56 public:
57  InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers,
58  bool UseLineDirectives);
59  void Process(FileID FileId, SrcMgr::CharacteristicKind FileType,
60  const DirectoryLookup *DirLookup);
61  void setPredefinesBuffer(const llvm::MemoryBuffer *Buf) {
62  PredefinesBuffer = Buf;
63  }
64  void detectMainFileEOL();
65  void handleModuleBegin(Token &Tok) {
66  assert(Tok.getKind() == tok::annot_module_begin);
67  ModuleEntryIncludes.insert({Tok.getLocation().getRawEncoding(),
68  (Module *)Tok.getAnnotationValue()});
69  }
70 private:
71  void FileChanged(SourceLocation Loc, FileChangeReason Reason,
73  FileID PrevFID) override;
74  void FileSkipped(const FileEntry &SkippedFile, const Token &FilenameTok,
75  SrcMgr::CharacteristicKind FileType) override;
76  void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
77  StringRef FileName, bool IsAngled,
78  CharSourceRange FilenameRange, const FileEntry *File,
79  StringRef SearchPath, StringRef RelativePath,
80  const Module *Imported,
81  SrcMgr::CharacteristicKind FileType) override;
82  void WriteLineInfo(StringRef Filename, int Line,
84  StringRef Extra = StringRef());
85  void WriteImplicitModuleImport(const Module *Mod);
86  void OutputContentUpTo(const MemoryBuffer &FromFile,
87  unsigned &WriteFrom, unsigned WriteTo,
88  StringRef EOL, int &lines,
89  bool EnsureNewline);
90  void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken,
91  const MemoryBuffer &FromFile, StringRef EOL,
92  unsigned &NextToWrite, int &Lines);
93  bool HandleHasInclude(FileID FileId, Lexer &RawLex,
94  const DirectoryLookup *Lookup, Token &Tok,
95  bool &FileExists);
96  const IncludedFile *FindIncludeAtLocation(SourceLocation Loc) const;
97  const Module *FindModuleAtLocation(SourceLocation Loc) const;
98  const Module *FindEnteredModule(SourceLocation Loc) const;
99  StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken);
100 };
101 
102 } // end anonymous namespace
103 
104 /// Initializes an InclusionRewriter with a \p PP source and \p OS destination.
105 InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS,
106  bool ShowLineMarkers,
107  bool UseLineDirectives)
108  : PP(PP), SM(PP.getSourceManager()), OS(OS), MainEOL("\n"),
109  PredefinesBuffer(nullptr), ShowLineMarkers(ShowLineMarkers),
110  UseLineDirectives(UseLineDirectives),
111  LastInclusionLocation(SourceLocation()) {}
112 
113 /// Write appropriate line information as either #line directives or GNU line
114 /// markers depending on what mode we're in, including the \p Filename and
115 /// \p Line we are located at, using the specified \p EOL line separator, and
116 /// any \p Extra context specifiers in GNU line directives.
117 void InclusionRewriter::WriteLineInfo(StringRef Filename, int Line,
119  StringRef Extra) {
120  if (!ShowLineMarkers)
121  return;
122  if (UseLineDirectives) {
123  OS << "#line" << ' ' << Line << ' ' << '"';
124  OS.write_escaped(Filename);
125  OS << '"';
126  } else {
127  // Use GNU linemarkers as described here:
128  // http://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html
129  OS << '#' << ' ' << Line << ' ' << '"';
130  OS.write_escaped(Filename);
131  OS << '"';
132  if (!Extra.empty())
133  OS << Extra;
134  if (FileType == SrcMgr::C_System)
135  // "`3' This indicates that the following text comes from a system header
136  // file, so certain warnings should be suppressed."
137  OS << " 3";
138  else if (FileType == SrcMgr::C_ExternCSystem)
139  // as above for `3', plus "`4' This indicates that the following text
140  // should be treated as being wrapped in an implicit extern "C" block."
141  OS << " 3 4";
142  }
143  OS << MainEOL;
144 }
145 
146 void InclusionRewriter::WriteImplicitModuleImport(const Module *Mod) {
147  OS << "#pragma clang module import " << Mod->getFullModuleName(true)
148  << " /* clang -frewrite-includes: implicit import */" << MainEOL;
149 }
150 
151 /// FileChanged - Whenever the preprocessor enters or exits a #include file
152 /// it invokes this handler.
153 void InclusionRewriter::FileChanged(SourceLocation Loc,
154  FileChangeReason Reason,
155  SrcMgr::CharacteristicKind NewFileType,
156  FileID) {
157  if (Reason != EnterFile)
158  return;
159  if (LastInclusionLocation.isInvalid())
160  // we didn't reach this file (eg: the main file) via an inclusion directive
161  return;
162  FileID Id = FullSourceLoc(Loc, SM).getFileID();
163  auto P = FileIncludes.insert(
164  std::make_pair(LastInclusionLocation.getRawEncoding(),
165  IncludedFile(Id, NewFileType, PP.GetCurDirLookup())));
166  (void)P;
167  assert(P.second && "Unexpected revisitation of the same include directive");
168  LastInclusionLocation = SourceLocation();
169 }
170 
171 /// Called whenever an inclusion is skipped due to canonical header protection
172 /// macros.
173 void InclusionRewriter::FileSkipped(const FileEntry &/*SkippedFile*/,
174  const Token &/*FilenameTok*/,
175  SrcMgr::CharacteristicKind /*FileType*/) {
176  assert(LastInclusionLocation.isValid() &&
177  "A file, that wasn't found via an inclusion directive, was skipped");
178  LastInclusionLocation = SourceLocation();
179 }
180 
181 /// This should be called whenever the preprocessor encounters include
182 /// directives. It does not say whether the file has been included, but it
183 /// provides more information about the directive (hash location instead
184 /// of location inside the included file). It is assumed that the matching
185 /// FileChanged() or FileSkipped() is called after this (or neither is
186 /// called if this #include results in an error or does not textually include
187 /// anything).
188 void InclusionRewriter::InclusionDirective(SourceLocation HashLoc,
189  const Token &/*IncludeTok*/,
190  StringRef /*FileName*/,
191  bool /*IsAngled*/,
192  CharSourceRange /*FilenameRange*/,
193  const FileEntry * /*File*/,
194  StringRef /*SearchPath*/,
195  StringRef /*RelativePath*/,
196  const Module *Imported,
197  SrcMgr::CharacteristicKind FileType){
198  if (Imported) {
199  auto P = ModuleIncludes.insert(
200  std::make_pair(HashLoc.getRawEncoding(), Imported));
201  (void)P;
202  assert(P.second && "Unexpected revisitation of the same include directive");
203  } else
204  LastInclusionLocation = HashLoc;
205 }
206 
207 /// Simple lookup for a SourceLocation (specifically one denoting the hash in
208 /// an inclusion directive) in the map of inclusion information, FileChanges.
209 const InclusionRewriter::IncludedFile *
210 InclusionRewriter::FindIncludeAtLocation(SourceLocation Loc) const {
211  const auto I = FileIncludes.find(Loc.getRawEncoding());
212  if (I != FileIncludes.end())
213  return &I->second;
214  return nullptr;
215 }
216 
217 /// Simple lookup for a SourceLocation (specifically one denoting the hash in
218 /// an inclusion directive) in the map of module inclusion information.
219 const Module *
220 InclusionRewriter::FindModuleAtLocation(SourceLocation Loc) const {
221  const auto I = ModuleIncludes.find(Loc.getRawEncoding());
222  if (I != ModuleIncludes.end())
223  return I->second;
224  return nullptr;
225 }
226 
227 /// Simple lookup for a SourceLocation (specifically one denoting the hash in
228 /// an inclusion directive) in the map of module entry information.
229 const Module *
230 InclusionRewriter::FindEnteredModule(SourceLocation Loc) const {
231  const auto I = ModuleEntryIncludes.find(Loc.getRawEncoding());
232  if (I != ModuleEntryIncludes.end())
233  return I->second;
234  return nullptr;
235 }
236 
237 /// Detect the likely line ending style of \p FromFile by examining the first
238 /// newline found within it.
239 static StringRef DetectEOL(const MemoryBuffer &FromFile) {
240  // Detect what line endings the file uses, so that added content does not mix
241  // the style. We need to check for "\r\n" first because "\n\r" will match
242  // "\r\n\r\n".
243  const char *Pos = strchr(FromFile.getBufferStart(), '\n');
244  if (!Pos)
245  return "\n";
246  if (Pos - 1 >= FromFile.getBufferStart() && Pos[-1] == '\r')
247  return "\r\n";
248  if (Pos + 1 < FromFile.getBufferEnd() && Pos[1] == '\r')
249  return "\n\r";
250  return "\n";
251 }
252 
253 void InclusionRewriter::detectMainFileEOL() {
254  bool Invalid;
255  const MemoryBuffer &FromFile = *SM.getBuffer(SM.getMainFileID(), &Invalid);
256  assert(!Invalid);
257  if (Invalid)
258  return; // Should never happen, but whatever.
259  MainEOL = DetectEOL(FromFile);
260 }
261 
262 /// Writes out bytes from \p FromFile, starting at \p NextToWrite and ending at
263 /// \p WriteTo - 1.
264 void InclusionRewriter::OutputContentUpTo(const MemoryBuffer &FromFile,
265  unsigned &WriteFrom, unsigned WriteTo,
266  StringRef LocalEOL, int &Line,
267  bool EnsureNewline) {
268  if (WriteTo <= WriteFrom)
269  return;
270  if (&FromFile == PredefinesBuffer) {
271  // Ignore the #defines of the predefines buffer.
272  WriteFrom = WriteTo;
273  return;
274  }
275 
276  // If we would output half of a line ending, advance one character to output
277  // the whole line ending. All buffers are null terminated, so looking ahead
278  // one byte is safe.
279  if (LocalEOL.size() == 2 &&
280  LocalEOL[0] == (FromFile.getBufferStart() + WriteTo)[-1] &&
281  LocalEOL[1] == (FromFile.getBufferStart() + WriteTo)[0])
282  WriteTo++;
283 
284  StringRef TextToWrite(FromFile.getBufferStart() + WriteFrom,
285  WriteTo - WriteFrom);
286 
287  if (MainEOL == LocalEOL) {
288  OS << TextToWrite;
289  // count lines manually, it's faster than getPresumedLoc()
290  Line += TextToWrite.count(LocalEOL);
291  if (EnsureNewline && !TextToWrite.endswith(LocalEOL))
292  OS << MainEOL;
293  } else {
294  // Output the file one line at a time, rewriting the line endings as we go.
295  StringRef Rest = TextToWrite;
296  while (!Rest.empty()) {
297  StringRef LineText;
298  std::tie(LineText, Rest) = Rest.split(LocalEOL);
299  OS << LineText;
300  Line++;
301  if (!Rest.empty())
302  OS << MainEOL;
303  }
304  if (TextToWrite.endswith(LocalEOL) || EnsureNewline)
305  OS << MainEOL;
306  }
307  WriteFrom = WriteTo;
308 }
309 
310 /// Print characters from \p FromFile starting at \p NextToWrite up until the
311 /// inclusion directive at \p StartToken, then print out the inclusion
312 /// inclusion directive disabled by a #if directive, updating \p NextToWrite
313 /// and \p Line to track the number of source lines visited and the progress
314 /// through the \p FromFile buffer.
315 void InclusionRewriter::CommentOutDirective(Lexer &DirectiveLex,
316  const Token &StartToken,
317  const MemoryBuffer &FromFile,
318  StringRef LocalEOL,
319  unsigned &NextToWrite, int &Line) {
320  OutputContentUpTo(FromFile, NextToWrite,
321  SM.getFileOffset(StartToken.getLocation()), LocalEOL, Line,
322  false);
323  Token DirectiveToken;
324  do {
325  DirectiveLex.LexFromRawLexer(DirectiveToken);
326  } while (!DirectiveToken.is(tok::eod) && DirectiveToken.isNot(tok::eof));
327  if (&FromFile == PredefinesBuffer) {
328  // OutputContentUpTo() would not output anything anyway.
329  return;
330  }
331  OS << "#if 0 /* expanded by -frewrite-includes */" << MainEOL;
332  OutputContentUpTo(FromFile, NextToWrite,
333  SM.getFileOffset(DirectiveToken.getLocation()) +
334  DirectiveToken.getLength(),
335  LocalEOL, Line, true);
336  OS << "#endif /* expanded by -frewrite-includes */" << MainEOL;
337 }
338 
339 /// Find the next identifier in the pragma directive specified by \p RawToken.
340 StringRef InclusionRewriter::NextIdentifierName(Lexer &RawLex,
341  Token &RawToken) {
342  RawLex.LexFromRawLexer(RawToken);
343  if (RawToken.is(tok::raw_identifier))
344  PP.LookUpIdentifierInfo(RawToken);
345  if (RawToken.is(tok::identifier))
346  return RawToken.getIdentifierInfo()->getName();
347  return StringRef();
348 }
349 
350 // Expand __has_include and __has_include_next if possible. If there's no
351 // definitive answer return false.
352 bool InclusionRewriter::HandleHasInclude(
353  FileID FileId, Lexer &RawLex, const DirectoryLookup *Lookup, Token &Tok,
354  bool &FileExists) {
355  // Lex the opening paren.
356  RawLex.LexFromRawLexer(Tok);
357  if (Tok.isNot(tok::l_paren))
358  return false;
359 
360  RawLex.LexFromRawLexer(Tok);
361 
362  SmallString<128> FilenameBuffer;
363  StringRef Filename;
364  // Since the raw lexer doesn't give us angle_literals we have to parse them
365  // ourselves.
366  // FIXME: What to do if the file name is a macro?
367  if (Tok.is(tok::less)) {
368  RawLex.LexFromRawLexer(Tok);
369 
370  FilenameBuffer += '<';
371  do {
372  if (Tok.is(tok::eod)) // Sanity check.
373  return false;
374 
375  if (Tok.is(tok::raw_identifier))
376  PP.LookUpIdentifierInfo(Tok);
377 
378  // Get the string piece.
379  SmallVector<char, 128> TmpBuffer;
380  bool Invalid = false;
381  StringRef TmpName = PP.getSpelling(Tok, TmpBuffer, &Invalid);
382  if (Invalid)
383  return false;
384 
385  FilenameBuffer += TmpName;
386 
387  RawLex.LexFromRawLexer(Tok);
388  } while (Tok.isNot(tok::greater));
389 
390  FilenameBuffer += '>';
391  Filename = FilenameBuffer;
392  } else {
393  if (Tok.isNot(tok::string_literal))
394  return false;
395 
396  bool Invalid = false;
397  Filename = PP.getSpelling(Tok, FilenameBuffer, &Invalid);
398  if (Invalid)
399  return false;
400  }
401 
402  // Lex the closing paren.
403  RawLex.LexFromRawLexer(Tok);
404  if (Tok.isNot(tok::r_paren))
405  return false;
406 
407  // Now ask HeaderInfo if it knows about the header.
408  // FIXME: Subframeworks aren't handled here. Do we care?
409  bool isAngled = PP.GetIncludeFilenameSpelling(Tok.getLocation(), Filename);
410  const DirectoryLookup *CurDir;
411  const FileEntry *FileEnt = PP.getSourceManager().getFileEntryForID(FileId);
413  Includers;
414  Includers.push_back(std::make_pair(FileEnt, FileEnt->getDir()));
415  // FIXME: Why don't we call PP.LookupFile here?
416  const FileEntry *File = PP.getHeaderSearchInfo().LookupFile(
417  Filename, SourceLocation(), isAngled, Lookup, CurDir, Includers, nullptr,
418  nullptr, nullptr, nullptr, nullptr);
419 
420  FileExists = File != nullptr;
421  return true;
422 }
423 
424 /// Use a raw lexer to analyze \p FileId, incrementally copying parts of it
425 /// and including content of included files recursively.
426 void InclusionRewriter::Process(FileID FileId,
428  const DirectoryLookup *DirLookup) {
429  bool Invalid;
430  const MemoryBuffer &FromFile = *SM.getBuffer(FileId, &Invalid);
431  assert(!Invalid && "Attempting to process invalid inclusion");
432  StringRef FileName = FromFile.getBufferIdentifier();
433  Lexer RawLex(FileId, &FromFile, PP.getSourceManager(), PP.getLangOpts());
434  RawLex.SetCommentRetentionState(false);
435 
436  StringRef LocalEOL = DetectEOL(FromFile);
437 
438  // Per the GNU docs: "1" indicates entering a new file.
439  if (FileId == SM.getMainFileID() || FileId == PP.getPredefinesFileID())
440  WriteLineInfo(FileName, 1, FileType, "");
441  else
442  WriteLineInfo(FileName, 1, FileType, " 1");
443 
444  if (SM.getFileIDSize(FileId) == 0)
445  return;
446 
447  // The next byte to be copied from the source file, which may be non-zero if
448  // the lexer handled a BOM.
449  unsigned NextToWrite = SM.getFileOffset(RawLex.getSourceLocation());
450  assert(SM.getLineNumber(FileId, NextToWrite) == 1);
451  int Line = 1; // The current input file line number.
452 
453  Token RawToken;
454  RawLex.LexFromRawLexer(RawToken);
455 
456  // TODO: Consider adding a switch that strips possibly unimportant content,
457  // such as comments, to reduce the size of repro files.
458  while (RawToken.isNot(tok::eof)) {
459  if (RawToken.is(tok::hash) && RawToken.isAtStartOfLine()) {
460  RawLex.setParsingPreprocessorDirective(true);
461  Token HashToken = RawToken;
462  RawLex.LexFromRawLexer(RawToken);
463  if (RawToken.is(tok::raw_identifier))
464  PP.LookUpIdentifierInfo(RawToken);
465  if (RawToken.getIdentifierInfo() != nullptr) {
466  switch (RawToken.getIdentifierInfo()->getPPKeywordID()) {
467  case tok::pp_include:
468  case tok::pp_include_next:
469  case tok::pp_import: {
470  CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL, NextToWrite,
471  Line);
472  if (FileId != PP.getPredefinesFileID())
473  WriteLineInfo(FileName, Line - 1, FileType, "");
474  StringRef LineInfoExtra;
475  SourceLocation Loc = HashToken.getLocation();
476  if (const Module *Mod = FindModuleAtLocation(Loc))
477  WriteImplicitModuleImport(Mod);
478  else if (const IncludedFile *Inc = FindIncludeAtLocation(Loc)) {
479  const Module *Mod = FindEnteredModule(Loc);
480  if (Mod)
481  OS << "#pragma clang module begin "
482  << Mod->getFullModuleName(true) << "\n";
483 
484  // Include and recursively process the file.
485  Process(Inc->Id, Inc->FileType, Inc->DirLookup);
486 
487  if (Mod)
488  OS << "#pragma clang module end /*"
489  << Mod->getFullModuleName(true) << "*/\n";
490 
491  // Add line marker to indicate we're returning from an included
492  // file.
493  LineInfoExtra = " 2";
494  }
495  // fix up lineinfo (since commented out directive changed line
496  // numbers) for inclusions that were skipped due to header guards
497  WriteLineInfo(FileName, Line, FileType, LineInfoExtra);
498  break;
499  }
500  case tok::pp_pragma: {
501  StringRef Identifier = NextIdentifierName(RawLex, RawToken);
502  if (Identifier == "clang" || Identifier == "GCC") {
503  if (NextIdentifierName(RawLex, RawToken) == "system_header") {
504  // keep the directive in, commented out
505  CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL,
506  NextToWrite, Line);
507  // update our own type
508  FileType = SM.getFileCharacteristic(RawToken.getLocation());
509  WriteLineInfo(FileName, Line, FileType);
510  }
511  } else if (Identifier == "once") {
512  // keep the directive in, commented out
513  CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL,
514  NextToWrite, Line);
515  WriteLineInfo(FileName, Line, FileType);
516  }
517  break;
518  }
519  case tok::pp_if:
520  case tok::pp_elif: {
521  bool elif = (RawToken.getIdentifierInfo()->getPPKeywordID() ==
522  tok::pp_elif);
523  // Rewrite special builtin macros to avoid pulling in host details.
524  do {
525  // Walk over the directive.
526  RawLex.LexFromRawLexer(RawToken);
527  if (RawToken.is(tok::raw_identifier))
528  PP.LookUpIdentifierInfo(RawToken);
529 
530  if (RawToken.is(tok::identifier)) {
531  bool HasFile;
532  SourceLocation Loc = RawToken.getLocation();
533 
534  // Rewrite __has_include(x)
535  if (RawToken.getIdentifierInfo()->isStr("__has_include")) {
536  if (!HandleHasInclude(FileId, RawLex, nullptr, RawToken,
537  HasFile))
538  continue;
539  // Rewrite __has_include_next(x)
540  } else if (RawToken.getIdentifierInfo()->isStr(
541  "__has_include_next")) {
542  if (DirLookup)
543  ++DirLookup;
544 
545  if (!HandleHasInclude(FileId, RawLex, DirLookup, RawToken,
546  HasFile))
547  continue;
548  } else {
549  continue;
550  }
551  // Replace the macro with (0) or (1), followed by the commented
552  // out macro for reference.
553  OutputContentUpTo(FromFile, NextToWrite, SM.getFileOffset(Loc),
554  LocalEOL, Line, false);
555  OS << '(' << (int) HasFile << ")/*";
556  OutputContentUpTo(FromFile, NextToWrite,
557  SM.getFileOffset(RawToken.getLocation()) +
558  RawToken.getLength(),
559  LocalEOL, Line, false);
560  OS << "*/";
561  }
562  } while (RawToken.isNot(tok::eod));
563  if (elif) {
564  OutputContentUpTo(FromFile, NextToWrite,
565  SM.getFileOffset(RawToken.getLocation()) +
566  RawToken.getLength(),
567  LocalEOL, Line, /*EnsureNewline=*/ true);
568  WriteLineInfo(FileName, Line, FileType);
569  }
570  break;
571  }
572  case tok::pp_endif:
573  case tok::pp_else: {
574  // We surround every #include by #if 0 to comment it out, but that
575  // changes line numbers. These are fixed up right after that, but
576  // the whole #include could be inside a preprocessor conditional
577  // that is not processed. So it is necessary to fix the line
578  // numbers one the next line after each #else/#endif as well.
579  RawLex.SetKeepWhitespaceMode(true);
580  do {
581  RawLex.LexFromRawLexer(RawToken);
582  } while (RawToken.isNot(tok::eod) && RawToken.isNot(tok::eof));
583  OutputContentUpTo(FromFile, NextToWrite,
584  SM.getFileOffset(RawToken.getLocation()) +
585  RawToken.getLength(),
586  LocalEOL, Line, /*EnsureNewline=*/ true);
587  WriteLineInfo(FileName, Line, FileType);
588  RawLex.SetKeepWhitespaceMode(false);
589  }
590  default:
591  break;
592  }
593  }
594  RawLex.setParsingPreprocessorDirective(false);
595  }
596  RawLex.LexFromRawLexer(RawToken);
597  }
598  OutputContentUpTo(FromFile, NextToWrite,
599  SM.getFileOffset(SM.getLocForEndOfFile(FileId)), LocalEOL,
600  Line, /*EnsureNewline=*/true);
601 }
602 
603 /// InclusionRewriterInInput - Implement -frewrite-includes mode.
604 void clang::RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS,
605  const PreprocessorOutputOptions &Opts) {
606  SourceManager &SM = PP.getSourceManager();
607  InclusionRewriter *Rewrite = new InclusionRewriter(
608  PP, *OS, Opts.ShowLineMarkers, Opts.UseLineDirectives);
609  Rewrite->detectMainFileEOL();
610 
611  PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Rewrite));
612  PP.IgnorePragmas();
613 
614  // First let the preprocessor process the entire file and call callbacks.
615  // Callbacks will record which #include's were actually performed.
616  PP.EnterMainSourceFile();
617  Token Tok;
618  // Only preprocessor directives matter here, so disable macro expansion
619  // everywhere else as an optimization.
620  // TODO: It would be even faster if the preprocessor could be switched
621  // to a mode where it would parse only preprocessor directives and comments,
622  // nothing else matters for parsing or processing.
624  do {
625  PP.Lex(Tok);
626  if (Tok.is(tok::annot_module_begin))
627  Rewrite->handleModuleBegin(Tok);
628  } while (Tok.isNot(tok::eof));
629  Rewrite->setPredefinesBuffer(SM.getBuffer(PP.getPredefinesFileID()));
630  Rewrite->Process(PP.getPredefinesFileID(), SrcMgr::C_User, nullptr);
631  Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User, nullptr);
632  OS->flush();
633 }
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:77
unsigned getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it...
SourceLocation getLocForEndOfFile(FileID FID) const
Return the source location corresponding to the last byte of the specified file.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:30
bool LexFromRawLexer(Token &Result)
LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...
Definition: Lexer.h:195
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
Defines the SourceManager interface.
StringRef P
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
Definition: Pragma.cpp:1855
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
Definition: SourceManager.h:78
FileID getFileID() const
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:36
tok::TokenKind getKind() const
Definition: Token.h:90
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:827
Describes a module or submodule.
Definition: Module.h:65
std::string getFullModuleName(bool AllowStringLiterals=false) const
Retrieve the full name of this module, including the path from its top-level module.
Definition: Module.cpp:214
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:833
const FormatToken & Tok
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Buffer)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
void setParsingPreprocessorDirective(bool f)
Inform the lexer whether or not we are currently lexing a preprocessor directive. ...
unsigned getFileIDSize(FileID FID) const
The size of the SLocEntry that FID represents.
static StringRef DetectEOL(const MemoryBuffer &FromFile)
Detect the likely line ending style of FromFile by examining the first newline found within it...
StringRef Filename
Definition: Format.cpp:1602
PreprocessorOutputOptions - Options for controlling the C preprocessor output (e.g., -E).
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Represents a character-granular source range.
int Id
Definition: ASTDiff.cpp:191
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
const AnnotatedLine * Line
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
SourceLocation getSourceLocation(const char *Loc, unsigned TokLen=1) const
getSourceLocation - Return a source location identifier for the specified offset in the current file...
Definition: Lexer.cpp:1119
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
Defines the clang::Preprocessor interface.
SrcMgr::CharacteristicKind getFileCharacteristic(SourceLocation Loc) const
Return the file characteristic of the specified source location, indicating whether this is a normal ...
Record the location of an inclusion directive, such as an #include or #import statement.
const SourceManager & SM
Definition: Format.cpp:1472
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
SourceManager & getSourceManager() const
Definition: Preprocessor.h:831
const DirectoryEntry * getDir() const
Return the directory the file lives in.
Definition: FileManager.h:94
unsigned getFileOffset(SourceLocation SpellingLoc) const
Returns the offset from the start of the file that the specified SourceLocation represents.
llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
Encodes a location in the source.
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:177
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
void Lex(Token &Result)
Lex the next token for this preprocessor.
unsigned getLineNumber(FileID FID, unsigned FilePos, bool *Invalid=nullptr) const
Given a SourceLocation, return the spelling line number for the position indicated.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:266
unsigned UseLineDirectives
Use #line instead of GCC-style # N.
const FileEntry * LookupFile(StringRef Filename, SourceLocation IncludeLoc, bool isAngled, const DirectoryLookup *FromDir, const DirectoryLookup *&CurDir, ArrayRef< std::pair< const FileEntry *, const DirectoryEntry *>> Includers, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, Module *RequestingModule, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool SkipCache=false, bool BuildSystemModule=false)
Given a "foo" or <foo> reference, look up the indicated file, return null on failure.
StringRef getName() const
Return the actual identifier string.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:96
void RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS, const PreprocessorOutputOptions &Opts)
RewriteIncludesInInput - Implement -frewrite-includes mode.
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
FileID getMainFileID() const
Returns the FileID of the main source file.
tok::PPKeywordKind getPPKeywordID() const
Return the preprocessor keyword ID for this identifier.
unsigned getLength() const
Definition: Token.h:127
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:916
unsigned ShowLineMarkers
Show #line markers.
void SetCommentRetentionState(bool Mode)
SetCommentRetentionMode - Change the comment retention mode of the lexer to the specified mode...
Definition: Lexer.h:228
A SourceLocation and its associated SourceManager.
void * getAnnotationValue() const
Definition: Token.h:224
void SetKeepWhitespaceMode(bool Val)
SetKeepWhitespaceMode - This method lets clients enable or disable whitespace retention mode...
Definition: Lexer.h:213
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:924
This class handles loading and caching of source files into memory.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:127