clang  6.0.0svn
InclusionRewriter.cpp
Go to the documentation of this file.
1 //===--- InclusionRewriter.cpp - Rewrite includes into their expansions ---===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This code rewrites include invocations into their expansions. This gives you
11 // a file with all included files merged into it.
12 //
13 //===----------------------------------------------------------------------===//
14 
18 #include "clang/Lex/HeaderSearch.h"
19 #include "clang/Lex/Pragma.h"
20 #include "clang/Lex/Preprocessor.h"
21 #include "llvm/ADT/SmallString.h"
22 #include "llvm/Support/raw_ostream.h"
23 
24 using namespace clang;
25 using namespace llvm;
26 
27 namespace {
28 
29 class InclusionRewriter : public PPCallbacks {
30  /// Information about which #includes were actually performed,
31  /// created by preprocessor callbacks.
32  struct IncludedFile {
33  FileID Id;
35  IncludedFile(FileID Id, SrcMgr::CharacteristicKind FileType)
36  : Id(Id), FileType(FileType) {}
37  };
38  Preprocessor &PP; ///< Used to find inclusion directives.
39  SourceManager &SM; ///< Used to read and manage source files.
40  raw_ostream &OS; ///< The destination stream for rewritten contents.
41  StringRef MainEOL; ///< The line ending marker to use.
42  const llvm::MemoryBuffer *PredefinesBuffer; ///< The preprocessor predefines.
43  bool ShowLineMarkers; ///< Show #line markers.
44  bool UseLineDirectives; ///< Use of line directives or line markers.
45  /// Tracks where inclusions that change the file are found.
46  std::map<unsigned, IncludedFile> FileIncludes;
47  /// Tracks where inclusions that import modules are found.
48  std::map<unsigned, const Module *> ModuleIncludes;
49  /// Tracks where inclusions that enter modules (in a module build) are found.
50  std::map<unsigned, const Module *> ModuleEntryIncludes;
51  /// Used transitively for building up the FileIncludes mapping over the
52  /// various \c PPCallbacks callbacks.
53  SourceLocation LastInclusionLocation;
54 public:
55  InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers,
56  bool UseLineDirectives);
57  void Process(FileID FileId, SrcMgr::CharacteristicKind FileType);
58  void setPredefinesBuffer(const llvm::MemoryBuffer *Buf) {
59  PredefinesBuffer = Buf;
60  }
61  void detectMainFileEOL();
62  void handleModuleBegin(Token &Tok) {
63  assert(Tok.getKind() == tok::annot_module_begin);
64  ModuleEntryIncludes.insert({Tok.getLocation().getRawEncoding(),
65  (Module *)Tok.getAnnotationValue()});
66  }
67 private:
68  void FileChanged(SourceLocation Loc, FileChangeReason Reason,
70  FileID PrevFID) override;
71  void FileSkipped(const FileEntry &SkippedFile, const Token &FilenameTok,
72  SrcMgr::CharacteristicKind FileType) override;
73  void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
74  StringRef FileName, bool IsAngled,
75  CharSourceRange FilenameRange, const FileEntry *File,
76  StringRef SearchPath, StringRef RelativePath,
77  const Module *Imported) override;
78  void WriteLineInfo(StringRef Filename, int Line,
80  StringRef Extra = StringRef());
81  void WriteImplicitModuleImport(const Module *Mod);
82  void OutputContentUpTo(const MemoryBuffer &FromFile,
83  unsigned &WriteFrom, unsigned WriteTo,
84  StringRef EOL, int &lines,
85  bool EnsureNewline);
86  void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken,
87  const MemoryBuffer &FromFile, StringRef EOL,
88  unsigned &NextToWrite, int &Lines);
89  bool HandleHasInclude(FileID FileId, Lexer &RawLex,
90  const DirectoryLookup *Lookup, Token &Tok,
91  bool &FileExists);
92  const IncludedFile *FindIncludeAtLocation(SourceLocation Loc) const;
93  const Module *FindModuleAtLocation(SourceLocation Loc) const;
94  const Module *FindEnteredModule(SourceLocation Loc) const;
95  StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken);
96 };
97 
98 } // end anonymous namespace
99 
100 /// Initializes an InclusionRewriter with a \p PP source and \p OS destination.
101 InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS,
102  bool ShowLineMarkers,
103  bool UseLineDirectives)
104  : PP(PP), SM(PP.getSourceManager()), OS(OS), MainEOL("\n"),
105  PredefinesBuffer(nullptr), ShowLineMarkers(ShowLineMarkers),
106  UseLineDirectives(UseLineDirectives),
107  LastInclusionLocation(SourceLocation()) {}
108 
109 /// Write appropriate line information as either #line directives or GNU line
110 /// markers depending on what mode we're in, including the \p Filename and
111 /// \p Line we are located at, using the specified \p EOL line separator, and
112 /// any \p Extra context specifiers in GNU line directives.
113 void InclusionRewriter::WriteLineInfo(StringRef Filename, int Line,
115  StringRef Extra) {
116  if (!ShowLineMarkers)
117  return;
118  if (UseLineDirectives) {
119  OS << "#line" << ' ' << Line << ' ' << '"';
120  OS.write_escaped(Filename);
121  OS << '"';
122  } else {
123  // Use GNU linemarkers as described here:
124  // http://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html
125  OS << '#' << ' ' << Line << ' ' << '"';
126  OS.write_escaped(Filename);
127  OS << '"';
128  if (!Extra.empty())
129  OS << Extra;
130  if (FileType == SrcMgr::C_System)
131  // "`3' This indicates that the following text comes from a system header
132  // file, so certain warnings should be suppressed."
133  OS << " 3";
134  else if (FileType == SrcMgr::C_ExternCSystem)
135  // as above for `3', plus "`4' This indicates that the following text
136  // should be treated as being wrapped in an implicit extern "C" block."
137  OS << " 3 4";
138  }
139  OS << MainEOL;
140 }
141 
142 void InclusionRewriter::WriteImplicitModuleImport(const Module *Mod) {
143  OS << "#pragma clang module import " << Mod->getFullModuleName(true)
144  << " /* clang -frewrite-includes: implicit import */" << MainEOL;
145 }
146 
147 /// FileChanged - Whenever the preprocessor enters or exits a #include file
148 /// it invokes this handler.
149 void InclusionRewriter::FileChanged(SourceLocation Loc,
150  FileChangeReason Reason,
151  SrcMgr::CharacteristicKind NewFileType,
152  FileID) {
153  if (Reason != EnterFile)
154  return;
155  if (LastInclusionLocation.isInvalid())
156  // we didn't reach this file (eg: the main file) via an inclusion directive
157  return;
158  FileID Id = FullSourceLoc(Loc, SM).getFileID();
159  auto P = FileIncludes.insert(std::make_pair(
160  LastInclusionLocation.getRawEncoding(), IncludedFile(Id, NewFileType)));
161  (void)P;
162  assert(P.second && "Unexpected revisitation of the same include directive");
163  LastInclusionLocation = SourceLocation();
164 }
165 
166 /// Called whenever an inclusion is skipped due to canonical header protection
167 /// macros.
168 void InclusionRewriter::FileSkipped(const FileEntry &/*SkippedFile*/,
169  const Token &/*FilenameTok*/,
170  SrcMgr::CharacteristicKind /*FileType*/) {
171  assert(LastInclusionLocation.isValid() &&
172  "A file, that wasn't found via an inclusion directive, was skipped");
173  LastInclusionLocation = SourceLocation();
174 }
175 
176 /// This should be called whenever the preprocessor encounters include
177 /// directives. It does not say whether the file has been included, but it
178 /// provides more information about the directive (hash location instead
179 /// of location inside the included file). It is assumed that the matching
180 /// FileChanged() or FileSkipped() is called after this (or neither is
181 /// called if this #include results in an error or does not textually include
182 /// anything).
183 void InclusionRewriter::InclusionDirective(SourceLocation HashLoc,
184  const Token &/*IncludeTok*/,
185  StringRef /*FileName*/,
186  bool /*IsAngled*/,
187  CharSourceRange /*FilenameRange*/,
188  const FileEntry * /*File*/,
189  StringRef /*SearchPath*/,
190  StringRef /*RelativePath*/,
191  const Module *Imported) {
192  if (Imported) {
193  auto P = ModuleIncludes.insert(
194  std::make_pair(HashLoc.getRawEncoding(), Imported));
195  (void)P;
196  assert(P.second && "Unexpected revisitation of the same include directive");
197  } else
198  LastInclusionLocation = HashLoc;
199 }
200 
201 /// Simple lookup for a SourceLocation (specifically one denoting the hash in
202 /// an inclusion directive) in the map of inclusion information, FileChanges.
203 const InclusionRewriter::IncludedFile *
204 InclusionRewriter::FindIncludeAtLocation(SourceLocation Loc) const {
205  const auto I = FileIncludes.find(Loc.getRawEncoding());
206  if (I != FileIncludes.end())
207  return &I->second;
208  return nullptr;
209 }
210 
211 /// Simple lookup for a SourceLocation (specifically one denoting the hash in
212 /// an inclusion directive) in the map of module inclusion information.
213 const Module *
214 InclusionRewriter::FindModuleAtLocation(SourceLocation Loc) const {
215  const auto I = ModuleIncludes.find(Loc.getRawEncoding());
216  if (I != ModuleIncludes.end())
217  return I->second;
218  return nullptr;
219 }
220 
221 /// Simple lookup for a SourceLocation (specifically one denoting the hash in
222 /// an inclusion directive) in the map of module entry information.
223 const Module *
224 InclusionRewriter::FindEnteredModule(SourceLocation Loc) const {
225  const auto I = ModuleEntryIncludes.find(Loc.getRawEncoding());
226  if (I != ModuleEntryIncludes.end())
227  return I->second;
228  return nullptr;
229 }
230 
231 /// Detect the likely line ending style of \p FromFile by examining the first
232 /// newline found within it.
233 static StringRef DetectEOL(const MemoryBuffer &FromFile) {
234  // Detect what line endings the file uses, so that added content does not mix
235  // the style. We need to check for "\r\n" first because "\n\r" will match
236  // "\r\n\r\n".
237  const char *Pos = strchr(FromFile.getBufferStart(), '\n');
238  if (!Pos)
239  return "\n";
240  if (Pos - 1 >= FromFile.getBufferStart() && Pos[-1] == '\r')
241  return "\r\n";
242  if (Pos + 1 < FromFile.getBufferEnd() && Pos[1] == '\r')
243  return "\n\r";
244  return "\n";
245 }
246 
247 void InclusionRewriter::detectMainFileEOL() {
248  bool Invalid;
249  const MemoryBuffer &FromFile = *SM.getBuffer(SM.getMainFileID(), &Invalid);
250  assert(!Invalid);
251  if (Invalid)
252  return; // Should never happen, but whatever.
253  MainEOL = DetectEOL(FromFile);
254 }
255 
256 /// Writes out bytes from \p FromFile, starting at \p NextToWrite and ending at
257 /// \p WriteTo - 1.
258 void InclusionRewriter::OutputContentUpTo(const MemoryBuffer &FromFile,
259  unsigned &WriteFrom, unsigned WriteTo,
260  StringRef LocalEOL, int &Line,
261  bool EnsureNewline) {
262  if (WriteTo <= WriteFrom)
263  return;
264  if (&FromFile == PredefinesBuffer) {
265  // Ignore the #defines of the predefines buffer.
266  WriteFrom = WriteTo;
267  return;
268  }
269 
270  // If we would output half of a line ending, advance one character to output
271  // the whole line ending. All buffers are null terminated, so looking ahead
272  // one byte is safe.
273  if (LocalEOL.size() == 2 &&
274  LocalEOL[0] == (FromFile.getBufferStart() + WriteTo)[-1] &&
275  LocalEOL[1] == (FromFile.getBufferStart() + WriteTo)[0])
276  WriteTo++;
277 
278  StringRef TextToWrite(FromFile.getBufferStart() + WriteFrom,
279  WriteTo - WriteFrom);
280 
281  if (MainEOL == LocalEOL) {
282  OS << TextToWrite;
283  // count lines manually, it's faster than getPresumedLoc()
284  Line += TextToWrite.count(LocalEOL);
285  if (EnsureNewline && !TextToWrite.endswith(LocalEOL))
286  OS << MainEOL;
287  } else {
288  // Output the file one line at a time, rewriting the line endings as we go.
289  StringRef Rest = TextToWrite;
290  while (!Rest.empty()) {
291  StringRef LineText;
292  std::tie(LineText, Rest) = Rest.split(LocalEOL);
293  OS << LineText;
294  Line++;
295  if (!Rest.empty())
296  OS << MainEOL;
297  }
298  if (TextToWrite.endswith(LocalEOL) || EnsureNewline)
299  OS << MainEOL;
300  }
301  WriteFrom = WriteTo;
302 }
303 
304 /// Print characters from \p FromFile starting at \p NextToWrite up until the
305 /// inclusion directive at \p StartToken, then print out the inclusion
306 /// inclusion directive disabled by a #if directive, updating \p NextToWrite
307 /// and \p Line to track the number of source lines visited and the progress
308 /// through the \p FromFile buffer.
309 void InclusionRewriter::CommentOutDirective(Lexer &DirectiveLex,
310  const Token &StartToken,
311  const MemoryBuffer &FromFile,
312  StringRef LocalEOL,
313  unsigned &NextToWrite, int &Line) {
314  OutputContentUpTo(FromFile, NextToWrite,
315  SM.getFileOffset(StartToken.getLocation()), LocalEOL, Line,
316  false);
317  Token DirectiveToken;
318  do {
319  DirectiveLex.LexFromRawLexer(DirectiveToken);
320  } while (!DirectiveToken.is(tok::eod) && DirectiveToken.isNot(tok::eof));
321  if (&FromFile == PredefinesBuffer) {
322  // OutputContentUpTo() would not output anything anyway.
323  return;
324  }
325  OS << "#if 0 /* expanded by -frewrite-includes */" << MainEOL;
326  OutputContentUpTo(FromFile, NextToWrite,
327  SM.getFileOffset(DirectiveToken.getLocation()) +
328  DirectiveToken.getLength(),
329  LocalEOL, Line, true);
330  OS << "#endif /* expanded by -frewrite-includes */" << MainEOL;
331 }
332 
333 /// Find the next identifier in the pragma directive specified by \p RawToken.
334 StringRef InclusionRewriter::NextIdentifierName(Lexer &RawLex,
335  Token &RawToken) {
336  RawLex.LexFromRawLexer(RawToken);
337  if (RawToken.is(tok::raw_identifier))
338  PP.LookUpIdentifierInfo(RawToken);
339  if (RawToken.is(tok::identifier))
340  return RawToken.getIdentifierInfo()->getName();
341  return StringRef();
342 }
343 
344 // Expand __has_include and __has_include_next if possible. If there's no
345 // definitive answer return false.
346 bool InclusionRewriter::HandleHasInclude(
347  FileID FileId, Lexer &RawLex, const DirectoryLookup *Lookup, Token &Tok,
348  bool &FileExists) {
349  // Lex the opening paren.
350  RawLex.LexFromRawLexer(Tok);
351  if (Tok.isNot(tok::l_paren))
352  return false;
353 
354  RawLex.LexFromRawLexer(Tok);
355 
356  SmallString<128> FilenameBuffer;
357  StringRef Filename;
358  // Since the raw lexer doesn't give us angle_literals we have to parse them
359  // ourselves.
360  // FIXME: What to do if the file name is a macro?
361  if (Tok.is(tok::less)) {
362  RawLex.LexFromRawLexer(Tok);
363 
364  FilenameBuffer += '<';
365  do {
366  if (Tok.is(tok::eod)) // Sanity check.
367  return false;
368 
369  if (Tok.is(tok::raw_identifier))
370  PP.LookUpIdentifierInfo(Tok);
371 
372  // Get the string piece.
373  SmallVector<char, 128> TmpBuffer;
374  bool Invalid = false;
375  StringRef TmpName = PP.getSpelling(Tok, TmpBuffer, &Invalid);
376  if (Invalid)
377  return false;
378 
379  FilenameBuffer += TmpName;
380 
381  RawLex.LexFromRawLexer(Tok);
382  } while (Tok.isNot(tok::greater));
383 
384  FilenameBuffer += '>';
385  Filename = FilenameBuffer;
386  } else {
387  if (Tok.isNot(tok::string_literal))
388  return false;
389 
390  bool Invalid = false;
391  Filename = PP.getSpelling(Tok, FilenameBuffer, &Invalid);
392  if (Invalid)
393  return false;
394  }
395 
396  // Lex the closing paren.
397  RawLex.LexFromRawLexer(Tok);
398  if (Tok.isNot(tok::r_paren))
399  return false;
400 
401  // Now ask HeaderInfo if it knows about the header.
402  // FIXME: Subframeworks aren't handled here. Do we care?
403  bool isAngled = PP.GetIncludeFilenameSpelling(Tok.getLocation(), Filename);
404  const DirectoryLookup *CurDir;
405  const FileEntry *FileEnt = PP.getSourceManager().getFileEntryForID(FileId);
407  Includers;
408  Includers.push_back(std::make_pair(FileEnt, FileEnt->getDir()));
409  // FIXME: Why don't we call PP.LookupFile here?
410  const FileEntry *File = PP.getHeaderSearchInfo().LookupFile(
411  Filename, SourceLocation(), isAngled, nullptr, CurDir, Includers, nullptr,
412  nullptr, nullptr, nullptr, nullptr);
413 
414  FileExists = File != nullptr;
415  return true;
416 }
417 
418 /// Use a raw lexer to analyze \p FileId, incrementally copying parts of it
419 /// and including content of included files recursively.
420 void InclusionRewriter::Process(FileID FileId,
421  SrcMgr::CharacteristicKind FileType) {
422  bool Invalid;
423  const MemoryBuffer &FromFile = *SM.getBuffer(FileId, &Invalid);
424  assert(!Invalid && "Attempting to process invalid inclusion");
425  StringRef FileName = FromFile.getBufferIdentifier();
426  Lexer RawLex(FileId, &FromFile, PP.getSourceManager(), PP.getLangOpts());
427  RawLex.SetCommentRetentionState(false);
428 
429  StringRef LocalEOL = DetectEOL(FromFile);
430 
431  // Per the GNU docs: "1" indicates entering a new file.
432  if (FileId == SM.getMainFileID() || FileId == PP.getPredefinesFileID())
433  WriteLineInfo(FileName, 1, FileType, "");
434  else
435  WriteLineInfo(FileName, 1, FileType, " 1");
436 
437  if (SM.getFileIDSize(FileId) == 0)
438  return;
439 
440  // The next byte to be copied from the source file, which may be non-zero if
441  // the lexer handled a BOM.
442  unsigned NextToWrite = SM.getFileOffset(RawLex.getSourceLocation());
443  assert(SM.getLineNumber(FileId, NextToWrite) == 1);
444  int Line = 1; // The current input file line number.
445 
446  Token RawToken;
447  RawLex.LexFromRawLexer(RawToken);
448 
449  // TODO: Consider adding a switch that strips possibly unimportant content,
450  // such as comments, to reduce the size of repro files.
451  while (RawToken.isNot(tok::eof)) {
452  if (RawToken.is(tok::hash) && RawToken.isAtStartOfLine()) {
453  RawLex.setParsingPreprocessorDirective(true);
454  Token HashToken = RawToken;
455  RawLex.LexFromRawLexer(RawToken);
456  if (RawToken.is(tok::raw_identifier))
457  PP.LookUpIdentifierInfo(RawToken);
458  if (RawToken.getIdentifierInfo() != nullptr) {
459  switch (RawToken.getIdentifierInfo()->getPPKeywordID()) {
460  case tok::pp_include:
461  case tok::pp_include_next:
462  case tok::pp_import: {
463  CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL, NextToWrite,
464  Line);
465  if (FileId != PP.getPredefinesFileID())
466  WriteLineInfo(FileName, Line - 1, FileType, "");
467  StringRef LineInfoExtra;
468  SourceLocation Loc = HashToken.getLocation();
469  if (const Module *Mod = FindModuleAtLocation(Loc))
470  WriteImplicitModuleImport(Mod);
471  else if (const IncludedFile *Inc = FindIncludeAtLocation(Loc)) {
472  const Module *Mod = FindEnteredModule(Loc);
473  if (Mod)
474  OS << "#pragma clang module begin "
475  << Mod->getFullModuleName(true) << "\n";
476 
477  // Include and recursively process the file.
478  Process(Inc->Id, Inc->FileType);
479 
480  if (Mod)
481  OS << "#pragma clang module end /*"
482  << Mod->getFullModuleName(true) << "*/\n";
483 
484  // Add line marker to indicate we're returning from an included
485  // file.
486  LineInfoExtra = " 2";
487  }
488  // fix up lineinfo (since commented out directive changed line
489  // numbers) for inclusions that were skipped due to header guards
490  WriteLineInfo(FileName, Line, FileType, LineInfoExtra);
491  break;
492  }
493  case tok::pp_pragma: {
494  StringRef Identifier = NextIdentifierName(RawLex, RawToken);
495  if (Identifier == "clang" || Identifier == "GCC") {
496  if (NextIdentifierName(RawLex, RawToken) == "system_header") {
497  // keep the directive in, commented out
498  CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL,
499  NextToWrite, Line);
500  // update our own type
501  FileType = SM.getFileCharacteristic(RawToken.getLocation());
502  WriteLineInfo(FileName, Line, FileType);
503  }
504  } else if (Identifier == "once") {
505  // keep the directive in, commented out
506  CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL,
507  NextToWrite, Line);
508  WriteLineInfo(FileName, Line, FileType);
509  }
510  break;
511  }
512  case tok::pp_if:
513  case tok::pp_elif: {
514  bool elif = (RawToken.getIdentifierInfo()->getPPKeywordID() ==
515  tok::pp_elif);
516  // Rewrite special builtin macros to avoid pulling in host details.
517  do {
518  // Walk over the directive.
519  RawLex.LexFromRawLexer(RawToken);
520  if (RawToken.is(tok::raw_identifier))
521  PP.LookUpIdentifierInfo(RawToken);
522 
523  if (RawToken.is(tok::identifier)) {
524  bool HasFile;
525  SourceLocation Loc = RawToken.getLocation();
526 
527  // Rewrite __has_include(x)
528  if (RawToken.getIdentifierInfo()->isStr("__has_include")) {
529  if (!HandleHasInclude(FileId, RawLex, nullptr, RawToken,
530  HasFile))
531  continue;
532  // Rewrite __has_include_next(x)
533  } else if (RawToken.getIdentifierInfo()->isStr(
534  "__has_include_next")) {
535  const DirectoryLookup *Lookup = PP.GetCurDirLookup();
536  if (Lookup)
537  ++Lookup;
538 
539  if (!HandleHasInclude(FileId, RawLex, Lookup, RawToken,
540  HasFile))
541  continue;
542  } else {
543  continue;
544  }
545  // Replace the macro with (0) or (1), followed by the commented
546  // out macro for reference.
547  OutputContentUpTo(FromFile, NextToWrite, SM.getFileOffset(Loc),
548  LocalEOL, Line, false);
549  OS << '(' << (int) HasFile << ")/*";
550  OutputContentUpTo(FromFile, NextToWrite,
551  SM.getFileOffset(RawToken.getLocation()) +
552  RawToken.getLength(),
553  LocalEOL, Line, false);
554  OS << "*/";
555  }
556  } while (RawToken.isNot(tok::eod));
557  if (elif) {
558  OutputContentUpTo(FromFile, NextToWrite,
559  SM.getFileOffset(RawToken.getLocation()) +
560  RawToken.getLength(),
561  LocalEOL, Line, /*EnsureNewline=*/ true);
562  WriteLineInfo(FileName, Line, FileType);
563  }
564  break;
565  }
566  case tok::pp_endif:
567  case tok::pp_else: {
568  // We surround every #include by #if 0 to comment it out, but that
569  // changes line numbers. These are fixed up right after that, but
570  // the whole #include could be inside a preprocessor conditional
571  // that is not processed. So it is necessary to fix the line
572  // numbers one the next line after each #else/#endif as well.
573  RawLex.SetKeepWhitespaceMode(true);
574  do {
575  RawLex.LexFromRawLexer(RawToken);
576  } while (RawToken.isNot(tok::eod) && RawToken.isNot(tok::eof));
577  OutputContentUpTo(FromFile, NextToWrite,
578  SM.getFileOffset(RawToken.getLocation()) +
579  RawToken.getLength(),
580  LocalEOL, Line, /*EnsureNewline=*/ true);
581  WriteLineInfo(FileName, Line, FileType);
582  RawLex.SetKeepWhitespaceMode(false);
583  }
584  default:
585  break;
586  }
587  }
588  RawLex.setParsingPreprocessorDirective(false);
589  }
590  RawLex.LexFromRawLexer(RawToken);
591  }
592  OutputContentUpTo(FromFile, NextToWrite,
593  SM.getFileOffset(SM.getLocForEndOfFile(FileId)), LocalEOL,
594  Line, /*EnsureNewline=*/true);
595 }
596 
597 /// InclusionRewriterInInput - Implement -frewrite-includes mode.
598 void clang::RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS,
599  const PreprocessorOutputOptions &Opts) {
600  SourceManager &SM = PP.getSourceManager();
601  InclusionRewriter *Rewrite = new InclusionRewriter(
602  PP, *OS, Opts.ShowLineMarkers, Opts.UseLineDirectives);
603  Rewrite->detectMainFileEOL();
604 
605  PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Rewrite));
606  PP.IgnorePragmas();
607 
608  // First let the preprocessor process the entire file and call callbacks.
609  // Callbacks will record which #include's were actually performed.
610  PP.EnterMainSourceFile();
611  Token Tok;
612  // Only preprocessor directives matter here, so disable macro expansion
613  // everywhere else as an optimization.
614  // TODO: It would be even faster if the preprocessor could be switched
615  // to a mode where it would parse only preprocessor directives and comments,
616  // nothing else matters for parsing or processing.
618  do {
619  PP.Lex(Tok);
620  if (Tok.is(tok::annot_module_begin))
621  Rewrite->handleModuleBegin(Tok);
622  } while (Tok.isNot(tok::eof));
623  Rewrite->setPredefinesBuffer(SM.getBuffer(PP.getPredefinesFileID()));
624  Rewrite->Process(PP.getPredefinesFileID(), SrcMgr::C_User);
625  Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User);
626  OS->flush();
627 }
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:77
unsigned getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it...
SourceLocation getLocForEndOfFile(FileID FID) const
Return the source location corresponding to the last byte of the specified file.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:26
bool LexFromRawLexer(Token &Result)
LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...
Definition: Lexer.h:195
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
Defines the SourceManager interface.
StringRef P
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
Definition: Pragma.cpp:1840
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
Definition: SourceManager.h:78
FileID getFileID() const
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:36
tok::TokenKind getKind() const
Definition: Token.h:90
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:815
Describes a module or submodule.
Definition: Module.h:65
std::string getFullModuleName(bool AllowStringLiterals=false) const
Retrieve the full name of this module, including the path from its top-level module.
Definition: Module.cpp:169
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:821
const FormatToken & Tok
void setParsingPreprocessorDirective(bool f)
Inform the lexer whether or not we are currently lexing a preprocessor directive. ...
unsigned getFileIDSize(FileID FID) const
The size of the SLocEntry that FID represents.
static StringRef DetectEOL(const MemoryBuffer &FromFile)
Detect the likely line ending style of FromFile by examining the first newline found within it...
StringRef Filename
Definition: Format.cpp:1345
PreprocessorOutputOptions - Options for controlling the C preprocessor output (e.g., -E).
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Represents a character-granular source range.
int Id
Definition: ASTDiff.cpp:191
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
const AnnotatedLine * Line
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
SourceLocation getSourceLocation(const char *Loc, unsigned TokLen=1) const
getSourceLocation - Return a source location identifier for the specified offset in the current file...
Definition: Lexer.cpp:1123
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
Defines the clang::Preprocessor interface.
SrcMgr::CharacteristicKind getFileCharacteristic(SourceLocation Loc) const
Return the file characteristic of the specified source location, indicating whether this is a normal ...
Record the location of an inclusion directive, such as an #include or #import statement.
const SourceManager & SM
Definition: Format.cpp:1337
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
SourceManager & getSourceManager() const
Definition: Preprocessor.h:819
const DirectoryEntry * getDir() const
Return the directory the file lives in.
Definition: FileManager.h:94
unsigned getFileOffset(SourceLocation SpellingLoc) const
Returns the offset from the start of the file that the specified SourceLocation represents.
llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
Encodes a location in the source.
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:177
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
void Lex(Token &Result)
Lex the next token for this preprocessor.
unsigned getLineNumber(FileID FID, unsigned FilePos, bool *Invalid=nullptr) const
Given a SourceLocation, return the spelling line number for the position indicated.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:266
unsigned UseLineDirectives
Use #line instead of GCC-style # N.
const FileEntry * LookupFile(StringRef Filename, SourceLocation IncludeLoc, bool isAngled, const DirectoryLookup *FromDir, const DirectoryLookup *&CurDir, ArrayRef< std::pair< const FileEntry *, const DirectoryEntry *>> Includers, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, Module *RequestingModule, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool SkipCache=false, bool BuildSystemModule=false)
Given a "foo" or <foo> reference, look up the indicated file, return null on failure.
StringRef getName() const
Return the actual identifier string.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:96
void RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS, const PreprocessorOutputOptions &Opts)
RewriteIncludesInInput - Implement -frewrite-includes mode.
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
FileID getMainFileID() const
Returns the FileID of the main source file.
tok::PPKeywordKind getPPKeywordID() const
Return the preprocessor keyword ID for this identifier.
unsigned getLength() const
Definition: Token.h:127
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Filename)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:904
unsigned ShowLineMarkers
Show #line markers.
void SetCommentRetentionState(bool Mode)
SetCommentRetentionMode - Change the comment retention mode of the lexer to the specified mode...
Definition: Lexer.h:228
A SourceLocation and its associated SourceManager.
void * getAnnotationValue() const
Definition: Token.h:224
void SetKeepWhitespaceMode(bool Val)
SetKeepWhitespaceMode - This method lets clients enable or disable whitespace retention mode...
Definition: Lexer.h:213
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:912
This class handles loading and caching of source files into memory.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:127