clang  9.0.0svn
InclusionRewriter.cpp
Go to the documentation of this file.
1 //===--- InclusionRewriter.cpp - Rewrite includes into their expansions ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This code rewrites include invocations into their expansions. This gives you
10 // a file with all included files merged into it.
11 //
12 //===----------------------------------------------------------------------===//
13 
17 #include "clang/Lex/HeaderSearch.h"
18 #include "clang/Lex/Pragma.h"
19 #include "clang/Lex/Preprocessor.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/Support/raw_ostream.h"
22 
23 using namespace clang;
24 using namespace llvm;
25 
26 namespace {
27 
28 class InclusionRewriter : public PPCallbacks {
29  /// Information about which #includes were actually performed,
30  /// created by preprocessor callbacks.
31  struct IncludedFile {
32  FileID Id;
34  const DirectoryLookup *DirLookup;
35  IncludedFile(FileID Id, SrcMgr::CharacteristicKind FileType,
36  const DirectoryLookup *DirLookup)
37  : Id(Id), FileType(FileType), DirLookup(DirLookup) {}
38  };
39  Preprocessor &PP; ///< Used to find inclusion directives.
40  SourceManager &SM; ///< Used to read and manage source files.
41  raw_ostream &OS; ///< The destination stream for rewritten contents.
42  StringRef MainEOL; ///< The line ending marker to use.
43  const llvm::MemoryBuffer *PredefinesBuffer; ///< The preprocessor predefines.
44  bool ShowLineMarkers; ///< Show #line markers.
45  bool UseLineDirectives; ///< Use of line directives or line markers.
46  /// Tracks where inclusions that change the file are found.
47  std::map<unsigned, IncludedFile> FileIncludes;
48  /// Tracks where inclusions that import modules are found.
49  std::map<unsigned, const Module *> ModuleIncludes;
50  /// Tracks where inclusions that enter modules (in a module build) are found.
51  std::map<unsigned, const Module *> ModuleEntryIncludes;
52  /// Used transitively for building up the FileIncludes mapping over the
53  /// various \c PPCallbacks callbacks.
54  SourceLocation LastInclusionLocation;
55 public:
56  InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers,
57  bool UseLineDirectives);
58  void Process(FileID FileId, SrcMgr::CharacteristicKind FileType,
59  const DirectoryLookup *DirLookup);
60  void setPredefinesBuffer(const llvm::MemoryBuffer *Buf) {
61  PredefinesBuffer = Buf;
62  }
63  void detectMainFileEOL();
64  void handleModuleBegin(Token &Tok) {
65  assert(Tok.getKind() == tok::annot_module_begin);
66  ModuleEntryIncludes.insert({Tok.getLocation().getRawEncoding(),
67  (Module *)Tok.getAnnotationValue()});
68  }
69 private:
70  void FileChanged(SourceLocation Loc, FileChangeReason Reason,
72  FileID PrevFID) override;
73  void FileSkipped(const FileEntry &SkippedFile, const Token &FilenameTok,
74  SrcMgr::CharacteristicKind FileType) override;
75  void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
76  StringRef FileName, bool IsAngled,
77  CharSourceRange FilenameRange, const FileEntry *File,
78  StringRef SearchPath, StringRef RelativePath,
79  const Module *Imported,
80  SrcMgr::CharacteristicKind FileType) override;
81  void WriteLineInfo(StringRef Filename, int Line,
83  StringRef Extra = StringRef());
84  void WriteImplicitModuleImport(const Module *Mod);
85  void OutputContentUpTo(const MemoryBuffer &FromFile,
86  unsigned &WriteFrom, unsigned WriteTo,
87  StringRef EOL, int &lines,
88  bool EnsureNewline);
89  void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken,
90  const MemoryBuffer &FromFile, StringRef EOL,
91  unsigned &NextToWrite, int &Lines);
92  bool HandleHasInclude(FileID FileId, Lexer &RawLex,
93  const DirectoryLookup *Lookup, Token &Tok,
94  bool &FileExists);
95  const IncludedFile *FindIncludeAtLocation(SourceLocation Loc) const;
96  const Module *FindModuleAtLocation(SourceLocation Loc) const;
97  const Module *FindEnteredModule(SourceLocation Loc) const;
98  StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken);
99 };
100 
101 } // end anonymous namespace
102 
103 /// Initializes an InclusionRewriter with a \p PP source and \p OS destination.
104 InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS,
105  bool ShowLineMarkers,
106  bool UseLineDirectives)
107  : PP(PP), SM(PP.getSourceManager()), OS(OS), MainEOL("\n"),
108  PredefinesBuffer(nullptr), ShowLineMarkers(ShowLineMarkers),
109  UseLineDirectives(UseLineDirectives),
110  LastInclusionLocation(SourceLocation()) {}
111 
112 /// Write appropriate line information as either #line directives or GNU line
113 /// markers depending on what mode we're in, including the \p Filename and
114 /// \p Line we are located at, using the specified \p EOL line separator, and
115 /// any \p Extra context specifiers in GNU line directives.
116 void InclusionRewriter::WriteLineInfo(StringRef Filename, int Line,
118  StringRef Extra) {
119  if (!ShowLineMarkers)
120  return;
121  if (UseLineDirectives) {
122  OS << "#line" << ' ' << Line << ' ' << '"';
123  OS.write_escaped(Filename);
124  OS << '"';
125  } else {
126  // Use GNU linemarkers as described here:
127  // http://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html
128  OS << '#' << ' ' << Line << ' ' << '"';
129  OS.write_escaped(Filename);
130  OS << '"';
131  if (!Extra.empty())
132  OS << Extra;
133  if (FileType == SrcMgr::C_System)
134  // "`3' This indicates that the following text comes from a system header
135  // file, so certain warnings should be suppressed."
136  OS << " 3";
137  else if (FileType == SrcMgr::C_ExternCSystem)
138  // as above for `3', plus "`4' This indicates that the following text
139  // should be treated as being wrapped in an implicit extern "C" block."
140  OS << " 3 4";
141  }
142  OS << MainEOL;
143 }
144 
145 void InclusionRewriter::WriteImplicitModuleImport(const Module *Mod) {
146  OS << "#pragma clang module import " << Mod->getFullModuleName(true)
147  << " /* clang -frewrite-includes: implicit import */" << MainEOL;
148 }
149 
150 /// FileChanged - Whenever the preprocessor enters or exits a #include file
151 /// it invokes this handler.
152 void InclusionRewriter::FileChanged(SourceLocation Loc,
153  FileChangeReason Reason,
154  SrcMgr::CharacteristicKind NewFileType,
155  FileID) {
156  if (Reason != EnterFile)
157  return;
158  if (LastInclusionLocation.isInvalid())
159  // we didn't reach this file (eg: the main file) via an inclusion directive
160  return;
161  FileID Id = FullSourceLoc(Loc, SM).getFileID();
162  auto P = FileIncludes.insert(
163  std::make_pair(LastInclusionLocation.getRawEncoding(),
164  IncludedFile(Id, NewFileType, PP.GetCurDirLookup())));
165  (void)P;
166  assert(P.second && "Unexpected revisitation of the same include directive");
167  LastInclusionLocation = SourceLocation();
168 }
169 
170 /// Called whenever an inclusion is skipped due to canonical header protection
171 /// macros.
172 void InclusionRewriter::FileSkipped(const FileEntry &/*SkippedFile*/,
173  const Token &/*FilenameTok*/,
174  SrcMgr::CharacteristicKind /*FileType*/) {
175  assert(LastInclusionLocation.isValid() &&
176  "A file, that wasn't found via an inclusion directive, was skipped");
177  LastInclusionLocation = SourceLocation();
178 }
179 
180 /// This should be called whenever the preprocessor encounters include
181 /// directives. It does not say whether the file has been included, but it
182 /// provides more information about the directive (hash location instead
183 /// of location inside the included file). It is assumed that the matching
184 /// FileChanged() or FileSkipped() is called after this (or neither is
185 /// called if this #include results in an error or does not textually include
186 /// anything).
187 void InclusionRewriter::InclusionDirective(SourceLocation HashLoc,
188  const Token &/*IncludeTok*/,
189  StringRef /*FileName*/,
190  bool /*IsAngled*/,
191  CharSourceRange /*FilenameRange*/,
192  const FileEntry * /*File*/,
193  StringRef /*SearchPath*/,
194  StringRef /*RelativePath*/,
195  const Module *Imported,
196  SrcMgr::CharacteristicKind FileType){
197  if (Imported) {
198  auto P = ModuleIncludes.insert(
199  std::make_pair(HashLoc.getRawEncoding(), Imported));
200  (void)P;
201  assert(P.second && "Unexpected revisitation of the same include directive");
202  } else
203  LastInclusionLocation = HashLoc;
204 }
205 
206 /// Simple lookup for a SourceLocation (specifically one denoting the hash in
207 /// an inclusion directive) in the map of inclusion information, FileChanges.
208 const InclusionRewriter::IncludedFile *
209 InclusionRewriter::FindIncludeAtLocation(SourceLocation Loc) const {
210  const auto I = FileIncludes.find(Loc.getRawEncoding());
211  if (I != FileIncludes.end())
212  return &I->second;
213  return nullptr;
214 }
215 
216 /// Simple lookup for a SourceLocation (specifically one denoting the hash in
217 /// an inclusion directive) in the map of module inclusion information.
218 const Module *
219 InclusionRewriter::FindModuleAtLocation(SourceLocation Loc) const {
220  const auto I = ModuleIncludes.find(Loc.getRawEncoding());
221  if (I != ModuleIncludes.end())
222  return I->second;
223  return nullptr;
224 }
225 
226 /// Simple lookup for a SourceLocation (specifically one denoting the hash in
227 /// an inclusion directive) in the map of module entry information.
228 const Module *
229 InclusionRewriter::FindEnteredModule(SourceLocation Loc) const {
230  const auto I = ModuleEntryIncludes.find(Loc.getRawEncoding());
231  if (I != ModuleEntryIncludes.end())
232  return I->second;
233  return nullptr;
234 }
235 
236 /// Detect the likely line ending style of \p FromFile by examining the first
237 /// newline found within it.
238 static StringRef DetectEOL(const MemoryBuffer &FromFile) {
239  // Detect what line endings the file uses, so that added content does not mix
240  // the style. We need to check for "\r\n" first because "\n\r" will match
241  // "\r\n\r\n".
242  const char *Pos = strchr(FromFile.getBufferStart(), '\n');
243  if (!Pos)
244  return "\n";
245  if (Pos - 1 >= FromFile.getBufferStart() && Pos[-1] == '\r')
246  return "\r\n";
247  if (Pos + 1 < FromFile.getBufferEnd() && Pos[1] == '\r')
248  return "\n\r";
249  return "\n";
250 }
251 
252 void InclusionRewriter::detectMainFileEOL() {
253  bool Invalid;
254  const MemoryBuffer &FromFile = *SM.getBuffer(SM.getMainFileID(), &Invalid);
255  assert(!Invalid);
256  if (Invalid)
257  return; // Should never happen, but whatever.
258  MainEOL = DetectEOL(FromFile);
259 }
260 
261 /// Writes out bytes from \p FromFile, starting at \p NextToWrite and ending at
262 /// \p WriteTo - 1.
263 void InclusionRewriter::OutputContentUpTo(const MemoryBuffer &FromFile,
264  unsigned &WriteFrom, unsigned WriteTo,
265  StringRef LocalEOL, int &Line,
266  bool EnsureNewline) {
267  if (WriteTo <= WriteFrom)
268  return;
269  if (&FromFile == PredefinesBuffer) {
270  // Ignore the #defines of the predefines buffer.
271  WriteFrom = WriteTo;
272  return;
273  }
274 
275  // If we would output half of a line ending, advance one character to output
276  // the whole line ending. All buffers are null terminated, so looking ahead
277  // one byte is safe.
278  if (LocalEOL.size() == 2 &&
279  LocalEOL[0] == (FromFile.getBufferStart() + WriteTo)[-1] &&
280  LocalEOL[1] == (FromFile.getBufferStart() + WriteTo)[0])
281  WriteTo++;
282 
283  StringRef TextToWrite(FromFile.getBufferStart() + WriteFrom,
284  WriteTo - WriteFrom);
285 
286  if (MainEOL == LocalEOL) {
287  OS << TextToWrite;
288  // count lines manually, it's faster than getPresumedLoc()
289  Line += TextToWrite.count(LocalEOL);
290  if (EnsureNewline && !TextToWrite.endswith(LocalEOL))
291  OS << MainEOL;
292  } else {
293  // Output the file one line at a time, rewriting the line endings as we go.
294  StringRef Rest = TextToWrite;
295  while (!Rest.empty()) {
296  StringRef LineText;
297  std::tie(LineText, Rest) = Rest.split(LocalEOL);
298  OS << LineText;
299  Line++;
300  if (!Rest.empty())
301  OS << MainEOL;
302  }
303  if (TextToWrite.endswith(LocalEOL) || EnsureNewline)
304  OS << MainEOL;
305  }
306  WriteFrom = WriteTo;
307 }
308 
309 /// Print characters from \p FromFile starting at \p NextToWrite up until the
310 /// inclusion directive at \p StartToken, then print out the inclusion
311 /// inclusion directive disabled by a #if directive, updating \p NextToWrite
312 /// and \p Line to track the number of source lines visited and the progress
313 /// through the \p FromFile buffer.
314 void InclusionRewriter::CommentOutDirective(Lexer &DirectiveLex,
315  const Token &StartToken,
316  const MemoryBuffer &FromFile,
317  StringRef LocalEOL,
318  unsigned &NextToWrite, int &Line) {
319  OutputContentUpTo(FromFile, NextToWrite,
320  SM.getFileOffset(StartToken.getLocation()), LocalEOL, Line,
321  false);
322  Token DirectiveToken;
323  do {
324  DirectiveLex.LexFromRawLexer(DirectiveToken);
325  } while (!DirectiveToken.is(tok::eod) && DirectiveToken.isNot(tok::eof));
326  if (&FromFile == PredefinesBuffer) {
327  // OutputContentUpTo() would not output anything anyway.
328  return;
329  }
330  OS << "#if 0 /* expanded by -frewrite-includes */" << MainEOL;
331  OutputContentUpTo(FromFile, NextToWrite,
332  SM.getFileOffset(DirectiveToken.getLocation()) +
333  DirectiveToken.getLength(),
334  LocalEOL, Line, true);
335  OS << "#endif /* expanded by -frewrite-includes */" << MainEOL;
336 }
337 
338 /// Find the next identifier in the pragma directive specified by \p RawToken.
339 StringRef InclusionRewriter::NextIdentifierName(Lexer &RawLex,
340  Token &RawToken) {
341  RawLex.LexFromRawLexer(RawToken);
342  if (RawToken.is(tok::raw_identifier))
343  PP.LookUpIdentifierInfo(RawToken);
344  if (RawToken.is(tok::identifier))
345  return RawToken.getIdentifierInfo()->getName();
346  return StringRef();
347 }
348 
349 // Expand __has_include and __has_include_next if possible. If there's no
350 // definitive answer return false.
351 bool InclusionRewriter::HandleHasInclude(
352  FileID FileId, Lexer &RawLex, const DirectoryLookup *Lookup, Token &Tok,
353  bool &FileExists) {
354  // Lex the opening paren.
355  RawLex.LexFromRawLexer(Tok);
356  if (Tok.isNot(tok::l_paren))
357  return false;
358 
359  RawLex.LexFromRawLexer(Tok);
360 
361  SmallString<128> FilenameBuffer;
362  StringRef Filename;
363  // Since the raw lexer doesn't give us angle_literals we have to parse them
364  // ourselves.
365  // FIXME: What to do if the file name is a macro?
366  if (Tok.is(tok::less)) {
367  RawLex.LexFromRawLexer(Tok);
368 
369  FilenameBuffer += '<';
370  do {
371  if (Tok.is(tok::eod)) // Sanity check.
372  return false;
373 
374  if (Tok.is(tok::raw_identifier))
375  PP.LookUpIdentifierInfo(Tok);
376 
377  // Get the string piece.
378  SmallVector<char, 128> TmpBuffer;
379  bool Invalid = false;
380  StringRef TmpName = PP.getSpelling(Tok, TmpBuffer, &Invalid);
381  if (Invalid)
382  return false;
383 
384  FilenameBuffer += TmpName;
385 
386  RawLex.LexFromRawLexer(Tok);
387  } while (Tok.isNot(tok::greater));
388 
389  FilenameBuffer += '>';
390  Filename = FilenameBuffer;
391  } else {
392  if (Tok.isNot(tok::string_literal))
393  return false;
394 
395  bool Invalid = false;
396  Filename = PP.getSpelling(Tok, FilenameBuffer, &Invalid);
397  if (Invalid)
398  return false;
399  }
400 
401  // Lex the closing paren.
402  RawLex.LexFromRawLexer(Tok);
403  if (Tok.isNot(tok::r_paren))
404  return false;
405 
406  // Now ask HeaderInfo if it knows about the header.
407  // FIXME: Subframeworks aren't handled here. Do we care?
408  bool isAngled = PP.GetIncludeFilenameSpelling(Tok.getLocation(), Filename);
409  const DirectoryLookup *CurDir;
410  const FileEntry *FileEnt = PP.getSourceManager().getFileEntryForID(FileId);
412  Includers;
413  Includers.push_back(std::make_pair(FileEnt, FileEnt->getDir()));
414  // FIXME: Why don't we call PP.LookupFile here?
415  const FileEntry *File = PP.getHeaderSearchInfo().LookupFile(
416  Filename, SourceLocation(), isAngled, Lookup, CurDir, Includers, nullptr,
417  nullptr, nullptr, nullptr, nullptr, nullptr);
418 
419  FileExists = File != nullptr;
420  return true;
421 }
422 
423 /// Use a raw lexer to analyze \p FileId, incrementally copying parts of it
424 /// and including content of included files recursively.
425 void InclusionRewriter::Process(FileID FileId,
427  const DirectoryLookup *DirLookup) {
428  bool Invalid;
429  const MemoryBuffer &FromFile = *SM.getBuffer(FileId, &Invalid);
430  assert(!Invalid && "Attempting to process invalid inclusion");
431  StringRef FileName = FromFile.getBufferIdentifier();
432  Lexer RawLex(FileId, &FromFile, PP.getSourceManager(), PP.getLangOpts());
433  RawLex.SetCommentRetentionState(false);
434 
435  StringRef LocalEOL = DetectEOL(FromFile);
436 
437  // Per the GNU docs: "1" indicates entering a new file.
438  if (FileId == SM.getMainFileID() || FileId == PP.getPredefinesFileID())
439  WriteLineInfo(FileName, 1, FileType, "");
440  else
441  WriteLineInfo(FileName, 1, FileType, " 1");
442 
443  if (SM.getFileIDSize(FileId) == 0)
444  return;
445 
446  // The next byte to be copied from the source file, which may be non-zero if
447  // the lexer handled a BOM.
448  unsigned NextToWrite = SM.getFileOffset(RawLex.getSourceLocation());
449  assert(SM.getLineNumber(FileId, NextToWrite) == 1);
450  int Line = 1; // The current input file line number.
451 
452  Token RawToken;
453  RawLex.LexFromRawLexer(RawToken);
454 
455  // TODO: Consider adding a switch that strips possibly unimportant content,
456  // such as comments, to reduce the size of repro files.
457  while (RawToken.isNot(tok::eof)) {
458  if (RawToken.is(tok::hash) && RawToken.isAtStartOfLine()) {
459  RawLex.setParsingPreprocessorDirective(true);
460  Token HashToken = RawToken;
461  RawLex.LexFromRawLexer(RawToken);
462  if (RawToken.is(tok::raw_identifier))
463  PP.LookUpIdentifierInfo(RawToken);
464  if (RawToken.getIdentifierInfo() != nullptr) {
465  switch (RawToken.getIdentifierInfo()->getPPKeywordID()) {
466  case tok::pp_include:
467  case tok::pp_include_next:
468  case tok::pp_import: {
469  CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL, NextToWrite,
470  Line);
471  if (FileId != PP.getPredefinesFileID())
472  WriteLineInfo(FileName, Line - 1, FileType, "");
473  StringRef LineInfoExtra;
474  SourceLocation Loc = HashToken.getLocation();
475  if (const Module *Mod = FindModuleAtLocation(Loc))
476  WriteImplicitModuleImport(Mod);
477  else if (const IncludedFile *Inc = FindIncludeAtLocation(Loc)) {
478  const Module *Mod = FindEnteredModule(Loc);
479  if (Mod)
480  OS << "#pragma clang module begin "
481  << Mod->getFullModuleName(true) << "\n";
482 
483  // Include and recursively process the file.
484  Process(Inc->Id, Inc->FileType, Inc->DirLookup);
485 
486  if (Mod)
487  OS << "#pragma clang module end /*"
488  << Mod->getFullModuleName(true) << "*/\n";
489 
490  // Add line marker to indicate we're returning from an included
491  // file.
492  LineInfoExtra = " 2";
493  }
494  // fix up lineinfo (since commented out directive changed line
495  // numbers) for inclusions that were skipped due to header guards
496  WriteLineInfo(FileName, Line, FileType, LineInfoExtra);
497  break;
498  }
499  case tok::pp_pragma: {
500  StringRef Identifier = NextIdentifierName(RawLex, RawToken);
501  if (Identifier == "clang" || Identifier == "GCC") {
502  if (NextIdentifierName(RawLex, RawToken) == "system_header") {
503  // keep the directive in, commented out
504  CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL,
505  NextToWrite, Line);
506  // update our own type
507  FileType = SM.getFileCharacteristic(RawToken.getLocation());
508  WriteLineInfo(FileName, Line, FileType);
509  }
510  } else if (Identifier == "once") {
511  // keep the directive in, commented out
512  CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL,
513  NextToWrite, Line);
514  WriteLineInfo(FileName, Line, FileType);
515  }
516  break;
517  }
518  case tok::pp_if:
519  case tok::pp_elif: {
520  bool elif = (RawToken.getIdentifierInfo()->getPPKeywordID() ==
521  tok::pp_elif);
522  // Rewrite special builtin macros to avoid pulling in host details.
523  do {
524  // Walk over the directive.
525  RawLex.LexFromRawLexer(RawToken);
526  if (RawToken.is(tok::raw_identifier))
527  PP.LookUpIdentifierInfo(RawToken);
528 
529  if (RawToken.is(tok::identifier)) {
530  bool HasFile;
531  SourceLocation Loc = RawToken.getLocation();
532 
533  // Rewrite __has_include(x)
534  if (RawToken.getIdentifierInfo()->isStr("__has_include")) {
535  if (!HandleHasInclude(FileId, RawLex, nullptr, RawToken,
536  HasFile))
537  continue;
538  // Rewrite __has_include_next(x)
539  } else if (RawToken.getIdentifierInfo()->isStr(
540  "__has_include_next")) {
541  if (DirLookup)
542  ++DirLookup;
543 
544  if (!HandleHasInclude(FileId, RawLex, DirLookup, RawToken,
545  HasFile))
546  continue;
547  } else {
548  continue;
549  }
550  // Replace the macro with (0) or (1), followed by the commented
551  // out macro for reference.
552  OutputContentUpTo(FromFile, NextToWrite, SM.getFileOffset(Loc),
553  LocalEOL, Line, false);
554  OS << '(' << (int) HasFile << ")/*";
555  OutputContentUpTo(FromFile, NextToWrite,
556  SM.getFileOffset(RawToken.getLocation()) +
557  RawToken.getLength(),
558  LocalEOL, Line, false);
559  OS << "*/";
560  }
561  } while (RawToken.isNot(tok::eod));
562  if (elif) {
563  OutputContentUpTo(FromFile, NextToWrite,
564  SM.getFileOffset(RawToken.getLocation()) +
565  RawToken.getLength(),
566  LocalEOL, Line, /*EnsureNewline=*/ true);
567  WriteLineInfo(FileName, Line, FileType);
568  }
569  break;
570  }
571  case tok::pp_endif:
572  case tok::pp_else: {
573  // We surround every #include by #if 0 to comment it out, but that
574  // changes line numbers. These are fixed up right after that, but
575  // the whole #include could be inside a preprocessor conditional
576  // that is not processed. So it is necessary to fix the line
577  // numbers one the next line after each #else/#endif as well.
578  RawLex.SetKeepWhitespaceMode(true);
579  do {
580  RawLex.LexFromRawLexer(RawToken);
581  } while (RawToken.isNot(tok::eod) && RawToken.isNot(tok::eof));
582  OutputContentUpTo(FromFile, NextToWrite,
583  SM.getFileOffset(RawToken.getLocation()) +
584  RawToken.getLength(),
585  LocalEOL, Line, /*EnsureNewline=*/ true);
586  WriteLineInfo(FileName, Line, FileType);
587  RawLex.SetKeepWhitespaceMode(false);
588  break;
589  }
590  default:
591  break;
592  }
593  }
594  RawLex.setParsingPreprocessorDirective(false);
595  }
596  RawLex.LexFromRawLexer(RawToken);
597  }
598  OutputContentUpTo(FromFile, NextToWrite,
599  SM.getFileOffset(SM.getLocForEndOfFile(FileId)), LocalEOL,
600  Line, /*EnsureNewline=*/true);
601 }
602 
603 /// InclusionRewriterInInput - Implement -frewrite-includes mode.
604 void clang::RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS,
605  const PreprocessorOutputOptions &Opts) {
606  SourceManager &SM = PP.getSourceManager();
607  InclusionRewriter *Rewrite = new InclusionRewriter(
608  PP, *OS, Opts.ShowLineMarkers, Opts.UseLineDirectives);
609  Rewrite->detectMainFileEOL();
610 
611  PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Rewrite));
612  PP.IgnorePragmas();
613 
614  // First let the preprocessor process the entire file and call callbacks.
615  // Callbacks will record which #include's were actually performed.
616  PP.EnterMainSourceFile();
617  Token Tok;
618  // Only preprocessor directives matter here, so disable macro expansion
619  // everywhere else as an optimization.
620  // TODO: It would be even faster if the preprocessor could be switched
621  // to a mode where it would parse only preprocessor directives and comments,
622  // nothing else matters for parsing or processing.
624  do {
625  PP.Lex(Tok);
626  if (Tok.is(tok::annot_module_begin))
627  Rewrite->handleModuleBegin(Tok);
628  } while (Tok.isNot(tok::eof));
629  Rewrite->setPredefinesBuffer(SM.getBuffer(PP.getPredefinesFileID()));
630  Rewrite->Process(PP.getPredefinesFileID(), SrcMgr::C_User, nullptr);
631  Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User, nullptr);
632  OS->flush();
633 }
StringRef Identifier
Definition: Format.cpp:1714
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:76
unsigned getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it...
SourceLocation getLocForEndOfFile(FileID FID) const
Return the source location corresponding to the last byte of the specified file.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:29
bool LexFromRawLexer(Token &Result)
LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...
Definition: Lexer.h:194
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:94
Defines the SourceManager interface.
StringRef P
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
Definition: Pragma.cpp:1893
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
Definition: SourceManager.h:77
FileID getFileID() const
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:35
tok::TokenKind getKind() const
Definition: Token.h:89
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:900
Describes a module or submodule.
Definition: Module.h:64
std::string getFullModuleName(bool AllowStringLiterals=false) const
Retrieve the full name of this module, including the path from its top-level module.
Definition: Module.cpp:213
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:905
const FormatToken & Tok
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Buffer)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
void setParsingPreprocessorDirective(bool f)
Inform the lexer whether or not we are currently lexing a preprocessor directive. ...
unsigned getFileIDSize(FileID FID) const
The size of the SLocEntry that FID represents.
static StringRef DetectEOL(const MemoryBuffer &FromFile)
Detect the likely line ending style of FromFile by examining the first newline found within it...
StringRef Filename
Definition: Format.cpp:1707
PreprocessorOutputOptions - Options for controlling the C preprocessor output (e.g., -E).
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Represents a character-granular source range.
int Id
Definition: ASTDiff.cpp:190
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
const AnnotatedLine * Line
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
SourceLocation getSourceLocation(const char *Loc, unsigned TokLen=1) const
getSourceLocation - Return a source location identifier for the specified offset in the current file...
Definition: Lexer.cpp:1118
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:123
Defines the clang::Preprocessor interface.
SrcMgr::CharacteristicKind getFileCharacteristic(SourceLocation Loc) const
Return the file characteristic of the specified source location, indicating whether this is a normal ...
Record the location of an inclusion directive, such as an #include or #import statement.
const FileEntry * LookupFile(StringRef Filename, SourceLocation IncludeLoc, bool isAngled, const DirectoryLookup *FromDir, const DirectoryLookup *&CurDir, ArrayRef< std::pair< const FileEntry *, const DirectoryEntry *>> Includers, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, Module *RequestingModule, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool BuildSystemModule=false)
Given a "foo" or <foo> reference, look up the indicated file, return null on failure.
const SourceManager & SM
Definition: Format.cpp:1568
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
SourceManager & getSourceManager() const
Definition: Preprocessor.h:904
const DirectoryEntry * getDir() const
Return the directory the file lives in.
Definition: FileManager.h:92
unsigned getFileOffset(SourceLocation SpellingLoc) const
Returns the offset from the start of the file that the specified SourceLocation represents.
Encodes a location in the source.
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:176
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
void Lex(Token &Result)
Lex the next token for this preprocessor.
unsigned getLineNumber(FileID FID, unsigned FilePos, bool *Invalid=nullptr) const
Given a SourceLocation, return the spelling line number for the position indicated.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:265
unsigned UseLineDirectives
Use #line instead of GCC-style # N.
StringRef getName() const
Return the actual identifier string.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:95
const llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
void RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS, const PreprocessorOutputOptions &Opts)
RewriteIncludesInInput - Implement -frewrite-includes mode.
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
FileID getMainFileID() const
Returns the FileID of the main source file.
tok::PPKeywordKind getPPKeywordID() const
Return the preprocessor keyword ID for this identifier.
unsigned getLength() const
Definition: Token.h:126
Indicates that the tracking object is a descendant of a referenced-counted OSObject, used in the Darwin kernel.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:984
unsigned ShowLineMarkers
Show #line markers.
void SetCommentRetentionState(bool Mode)
SetCommentRetentionMode - Change the comment retention mode of the lexer to the specified mode...
Definition: Lexer.h:227
A SourceLocation and its associated SourceManager.
void * getAnnotationValue() const
Definition: Token.h:223
void SetKeepWhitespaceMode(bool Val)
SetKeepWhitespaceMode - This method lets clients enable or disable whitespace retention mode...
Definition: Lexer.h:212
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:992
This class handles loading and caching of source files into memory.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:123