clang  15.0.0git
DependencyDirectivesScanner.cpp
Go to the documentation of this file.
1 //===- DependencyDirectivesScanner.cpp ------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This is the interface for scanning header and source files to get the
11 /// minimum necessary preprocessor directives for evaluating includes. It
12 /// reduces the source down to #define, #include, #import, @import, and any
13 /// conditional preprocessor logic that contains one of those.
14 ///
15 //===----------------------------------------------------------------------===//
16 
18 #include "clang/Basic/CharInfo.h"
19 #include "clang/Basic/Diagnostic.h"
21 #include "clang/Lex/Lexer.h"
22 #include "llvm/ADT/ScopeExit.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/ADT/StringMap.h"
25 #include "llvm/ADT/StringSwitch.h"
26 
27 using namespace clang;
28 using namespace clang::dependency_directives_scan;
29 using namespace llvm;
30 
31 namespace {
32 
33 struct DirectiveWithTokens {
35  unsigned NumTokens;
36 
37  DirectiveWithTokens(DirectiveKind Kind, unsigned NumTokens)
38  : Kind(Kind), NumTokens(NumTokens) {}
39 };
40 
41 /// Does an efficient "scan" of the sources to detect the presence of
42 /// preprocessor (or module import) directives and collects the raw lexed tokens
43 /// for those directives so that the \p Lexer can "replay" them when the file is
44 /// included.
45 ///
46 /// Note that the behavior of the raw lexer is affected by the language mode,
47 /// while at this point we want to do a scan and collect tokens once,
48 /// irrespective of the language mode that the file will get included in. To
49 /// compensate for that the \p Lexer, while "replaying", will adjust a token
50 /// where appropriate, when it could affect the preprocessor's state.
51 /// For example in a directive like
52 ///
53 /// \code
54 /// #if __has_cpp_attribute(clang::fallthrough)
55 /// \endcode
56 ///
57 /// The preprocessor needs to see '::' as 'tok::coloncolon' instead of 2
58 /// 'tok::colon'. The \p Lexer will adjust if it sees consecutive 'tok::colon'
59 /// while in C++ mode.
60 struct Scanner {
61  Scanner(StringRef Input,
63  DiagnosticsEngine *Diags, SourceLocation InputSourceLoc)
64  : Input(Input), Tokens(Tokens), Diags(Diags),
65  InputSourceLoc(InputSourceLoc), LangOpts(getLangOptsForDepScanning()),
66  TheLexer(InputSourceLoc, LangOpts, Input.begin(), Input.begin(),
67  Input.end()) {}
68 
69  static LangOptions getLangOptsForDepScanning() {
70  LangOptions LangOpts;
71  // Set the lexer to use 'tok::at' for '@', instead of 'tok::unknown'.
72  LangOpts.ObjC = true;
73  LangOpts.LineComment = true;
74  return LangOpts;
75  }
76 
77  /// Lex the provided source and emit the directive tokens.
78  ///
79  /// \returns True on error.
80  bool scan(SmallVectorImpl<Directive> &Directives);
81 
82 private:
83  /// Lexes next token and advances \p First and the \p Lexer.
84  LLVM_NODISCARD dependency_directives_scan::Token &
85  lexToken(const char *&First, const char *const End);
86 
87  dependency_directives_scan::Token &lexIncludeFilename(const char *&First,
88  const char *const End);
89 
90  /// Lexes next token and if it is identifier returns its string, otherwise
91  /// it skips the current line and returns \p None.
92  ///
93  /// In any case (whatever the token kind) \p First and the \p Lexer will
94  /// advance beyond the token.
95  LLVM_NODISCARD Optional<StringRef>
96  tryLexIdentifierOrSkipLine(const char *&First, const char *const End);
97 
98  /// Used when it is certain that next token is an identifier.
99  LLVM_NODISCARD StringRef lexIdentifier(const char *&First,
100  const char *const End);
101 
102  /// Lexes next token and returns true iff it is an identifier that matches \p
103  /// Id, otherwise it skips the current line and returns false.
104  ///
105  /// In any case (whatever the token kind) \p First and the \p Lexer will
106  /// advance beyond the token.
107  LLVM_NODISCARD bool isNextIdentifierOrSkipLine(StringRef Id,
108  const char *&First,
109  const char *const End);
110 
111  LLVM_NODISCARD bool scanImpl(const char *First, const char *const End);
112  LLVM_NODISCARD bool lexPPLine(const char *&First, const char *const End);
113  LLVM_NODISCARD bool lexAt(const char *&First, const char *const End);
114  LLVM_NODISCARD bool lexModule(const char *&First, const char *const End);
115  LLVM_NODISCARD bool lexDefine(const char *HashLoc, const char *&First,
116  const char *const End);
117  LLVM_NODISCARD bool lexPragma(const char *&First, const char *const End);
118  LLVM_NODISCARD bool lexEndif(const char *&First, const char *const End);
119  LLVM_NODISCARD bool lexDefault(DirectiveKind Kind, const char *&First,
120  const char *const End);
121  LLVM_NODISCARD bool lexModuleDirectiveBody(DirectiveKind Kind,
122  const char *&First,
123  const char *const End);
124  void lexPPDirectiveBody(const char *&First, const char *const End);
125 
126  DirectiveWithTokens &pushDirective(DirectiveKind Kind) {
127  Tokens.append(CurDirToks);
128  DirsWithToks.emplace_back(Kind, CurDirToks.size());
129  CurDirToks.clear();
130  return DirsWithToks.back();
131  }
132  void popDirective() {
133  Tokens.pop_back_n(DirsWithToks.pop_back_val().NumTokens);
134  }
135  DirectiveKind topDirective() const {
136  return DirsWithToks.empty() ? pp_none : DirsWithToks.back().Kind;
137  }
138 
139  unsigned getOffsetAt(const char *CurPtr) const {
140  return CurPtr - Input.data();
141  }
142 
143  /// Reports a diagnostic if the diagnostic engine is provided. Always returns
144  /// true at the end.
145  bool reportError(const char *CurPtr, unsigned Err);
146 
147  StringMap<char> SplitIds;
148  StringRef Input;
150  DiagnosticsEngine *Diags;
151  SourceLocation InputSourceLoc;
152 
153  /// Keeps track of the tokens for the currently lexed directive. Once a
154  /// directive is fully lexed and "committed" then the tokens get appended to
155  /// \p Tokens and \p CurDirToks is cleared for the next directive.
157  /// The directives that were lexed along with the number of tokens that each
158  /// directive contains. The tokens of all the directives are kept in \p Tokens
159  /// vector, in the same order as the directives order in \p DirsWithToks.
161  LangOptions LangOpts;
162  Lexer TheLexer;
163 };
164 
165 } // end anonymous namespace
166 
167 bool Scanner::reportError(const char *CurPtr, unsigned Err) {
168  if (!Diags)
169  return true;
170  assert(CurPtr >= Input.data() && "invalid buffer ptr");
171  Diags->Report(InputSourceLoc.getLocWithOffset(getOffsetAt(CurPtr)), Err);
172  return true;
173 }
174 
175 static void skipOverSpaces(const char *&First, const char *const End) {
176  while (First != End && isHorizontalWhitespace(*First))
177  ++First;
178 }
179 
180 LLVM_NODISCARD static bool isRawStringLiteral(const char *First,
181  const char *Current) {
182  assert(First <= Current);
183 
184  // Check if we can even back up.
185  if (*Current != '"' || First == Current)
186  return false;
187 
188  // Check for an "R".
189  --Current;
190  if (*Current != 'R')
191  return false;
192  if (First == Current || !isAsciiIdentifierContinue(*--Current))
193  return true;
194 
195  // Check for a prefix of "u", "U", or "L".
196  if (*Current == 'u' || *Current == 'U' || *Current == 'L')
197  return First == Current || !isAsciiIdentifierContinue(*--Current);
198 
199  // Check for a prefix of "u8".
200  if (*Current != '8' || First == Current || *Current-- != 'u')
201  return false;
202  return First == Current || !isAsciiIdentifierContinue(*--Current);
203 }
204 
205 static void skipRawString(const char *&First, const char *const End) {
206  assert(First[0] == '"');
207  assert(First[-1] == 'R');
208 
209  const char *Last = ++First;
210  while (Last != End && *Last != '(')
211  ++Last;
212  if (Last == End) {
213  First = Last; // Hit the end... just give up.
214  return;
215  }
216 
217  StringRef Terminator(First, Last - First);
218  for (;;) {
219  // Move First to just past the next ")".
220  First = Last;
221  while (First != End && *First != ')')
222  ++First;
223  if (First == End)
224  return;
225  ++First;
226 
227  // Look ahead for the terminator sequence.
228  Last = First;
229  while (Last != End && size_t(Last - First) < Terminator.size() &&
230  Terminator[Last - First] == *Last)
231  ++Last;
232 
233  // Check if we hit it (or the end of the file).
234  if (Last == End) {
235  First = Last;
236  return;
237  }
238  if (size_t(Last - First) < Terminator.size())
239  continue;
240  if (*Last != '"')
241  continue;
242  First = Last + 1;
243  return;
244  }
245 }
246 
247 // Returns the length of EOL, either 0 (no end-of-line), 1 (\n) or 2 (\r\n)
248 static unsigned isEOL(const char *First, const char *const End) {
249  if (First == End)
250  return 0;
251  if (End - First > 1 && isVerticalWhitespace(First[0]) &&
252  isVerticalWhitespace(First[1]) && First[0] != First[1])
253  return 2;
254  return !!isVerticalWhitespace(First[0]);
255 }
256 
257 static void skipString(const char *&First, const char *const End) {
258  assert(*First == '\'' || *First == '"' || *First == '<');
259  const char Terminator = *First == '<' ? '>' : *First;
260  for (++First; First != End && *First != Terminator; ++First) {
261  // String and character literals don't extend past the end of the line.
263  return;
264  if (*First != '\\')
265  continue;
266  // Skip past backslash to the next character. This ensures that the
267  // character right after it is skipped as well, which matters if it's
268  // the terminator.
269  if (++First == End)
270  return;
271  if (!isWhitespace(*First))
272  continue;
273  // Whitespace after the backslash might indicate a line continuation.
274  const char *FirstAfterBackslashPastSpace = First;
275  skipOverSpaces(FirstAfterBackslashPastSpace, End);
276  if (unsigned NLSize = isEOL(FirstAfterBackslashPastSpace, End)) {
277  // Advance the character pointer to the next line for the next
278  // iteration.
279  First = FirstAfterBackslashPastSpace + NLSize - 1;
280  }
281  }
282  if (First != End)
283  ++First; // Finish off the string.
284 }
285 
286 // Returns the length of the skipped newline
287 static unsigned skipNewline(const char *&First, const char *End) {
288  if (First == End)
289  return 0;
290  assert(isVerticalWhitespace(*First));
291  unsigned Len = isEOL(First, End);
292  assert(Len && "expected newline");
293  First += Len;
294  return Len;
295 }
296 
297 static bool wasLineContinuation(const char *First, unsigned EOLLen) {
298  return *(First - (int)EOLLen - 1) == '\\';
299 }
300 
301 static void skipToNewlineRaw(const char *&First, const char *const End) {
302  for (;;) {
303  if (First == End)
304  return;
305 
306  unsigned Len = isEOL(First, End);
307  if (Len)
308  return;
309 
310  do {
311  if (++First == End)
312  return;
313  Len = isEOL(First, End);
314  } while (!Len);
315 
316  if (First[-1] != '\\')
317  return;
318 
319  First += Len;
320  // Keep skipping lines...
321  }
322 }
323 
324 static void skipLineComment(const char *&First, const char *const End) {
325  assert(First[0] == '/' && First[1] == '/');
326  First += 2;
328 }
329 
330 static void skipBlockComment(const char *&First, const char *const End) {
331  assert(First[0] == '/' && First[1] == '*');
332  if (End - First < 4) {
333  First = End;
334  return;
335  }
336  for (First += 3; First != End; ++First)
337  if (First[-1] == '*' && First[0] == '/') {
338  ++First;
339  return;
340  }
341 }
342 
343 /// \returns True if the current single quotation mark character is a C++ 14
344 /// digit separator.
345 static bool isQuoteCppDigitSeparator(const char *const Start,
346  const char *const Cur,
347  const char *const End) {
348  assert(*Cur == '\'' && "expected quotation character");
349  // skipLine called in places where we don't expect a valid number
350  // body before `start` on the same line, so always return false at the start.
351  if (Start == Cur)
352  return false;
353  // The previous character must be a valid PP number character.
354  // Make sure that the L, u, U, u8 prefixes don't get marked as a
355  // separator though.
356  char Prev = *(Cur - 1);
357  if (Prev == 'L' || Prev == 'U' || Prev == 'u')
358  return false;
359  if (Prev == '8' && (Cur - 1 != Start) && *(Cur - 2) == 'u')
360  return false;
361  if (!isPreprocessingNumberBody(Prev))
362  return false;
363  // The next character should be a valid identifier body character.
364  return (Cur + 1) < End && isAsciiIdentifierContinue(*(Cur + 1));
365 }
366 
367 static void skipLine(const char *&First, const char *const End) {
368  for (;;) {
369  assert(First <= End);
370  if (First == End)
371  return;
372 
373  if (isVerticalWhitespace(*First)) {
375  return;
376  }
377  const char *Start = First;
378  while (First != End && !isVerticalWhitespace(*First)) {
379  // Iterate over strings correctly to avoid comments and newlines.
380  if (*First == '"' ||
381  (*First == '\'' && !isQuoteCppDigitSeparator(Start, First, End))) {
382  if (isRawStringLiteral(Start, First))
384  else
385  skipString(First, End);
386  continue;
387  }
388 
389  // Iterate over comments correctly.
390  if (*First != '/' || End - First < 2) {
391  ++First;
392  continue;
393  }
394 
395  if (First[1] == '/') {
396  // "//...".
398  continue;
399  }
400 
401  if (First[1] != '*') {
402  ++First;
403  continue;
404  }
405 
406  // "/*...*/".
408  }
409  if (First == End)
410  return;
411 
412  // Skip over the newline.
413  unsigned Len = skipNewline(First, End);
414  if (!wasLineContinuation(First, Len)) // Continue past line-continuations.
415  break;
416  }
417 }
418 
419 static void skipDirective(StringRef Name, const char *&First,
420  const char *const End) {
421  if (llvm::StringSwitch<bool>(Name)
422  .Case("warning", true)
423  .Case("error", true)
424  .Default(false))
425  // Do not process quotes or comments.
427  else
428  skipLine(First, End);
429 }
430 
431 static void skipWhitespace(const char *&First, const char *const End) {
432  for (;;) {
433  assert(First <= End);
435 
436  if (End - First < 2)
437  return;
438 
439  if (First[0] == '\\' && isVerticalWhitespace(First[1])) {
440  skipNewline(++First, End);
441  continue;
442  }
443 
444  // Check for a non-comment character.
445  if (First[0] != '/')
446  return;
447 
448  // "// ...".
449  if (First[1] == '/') {
451  return;
452  }
453 
454  // Cannot be a comment.
455  if (First[1] != '*')
456  return;
457 
458  // "/*...*/".
460  }
461 }
462 
463 bool Scanner::lexModuleDirectiveBody(DirectiveKind Kind, const char *&First,
464  const char *const End) {
465  const char *DirectiveLoc = Input.data() + CurDirToks.front().Offset;
466  for (;;) {
467  const dependency_directives_scan::Token &Tok = lexToken(First, End);
468  if (Tok.is(tok::eof))
469  return reportError(
470  DirectiveLoc,
471  diag::err_dep_source_scanner_missing_semi_after_at_import);
472  if (Tok.is(tok::semi))
473  break;
474  }
475  pushDirective(Kind);
476  skipWhitespace(First, End);
477  if (First == End)
478  return false;
479  if (!isVerticalWhitespace(*First))
480  return reportError(
481  DirectiveLoc, diag::err_dep_source_scanner_unexpected_tokens_at_import);
482  skipNewline(First, End);
483  return false;
484 }
485 
486 dependency_directives_scan::Token &Scanner::lexToken(const char *&First,
487  const char *const End) {
488  clang::Token Tok;
489  TheLexer.LexFromRawLexer(Tok);
490  First = Input.data() + TheLexer.getCurrentBufferOffset();
491  assert(First <= End);
492 
493  unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.getLength();
494  CurDirToks.emplace_back(Offset, Tok.getLength(), Tok.getKind(),
495  Tok.getFlags());
496  return CurDirToks.back();
497 }
498 
500 Scanner::lexIncludeFilename(const char *&First, const char *const End) {
501  clang::Token Tok;
502  TheLexer.LexIncludeFilename(Tok);
503  First = Input.data() + TheLexer.getCurrentBufferOffset();
504  assert(First <= End);
505 
506  unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.getLength();
507  CurDirToks.emplace_back(Offset, Tok.getLength(), Tok.getKind(),
508  Tok.getFlags());
509  return CurDirToks.back();
510 }
511 
512 void Scanner::lexPPDirectiveBody(const char *&First, const char *const End) {
513  while (true) {
514  const dependency_directives_scan::Token &Tok = lexToken(First, End);
515  if (Tok.is(tok::eod))
516  break;
517  }
518 }
519 
520 LLVM_NODISCARD Optional<StringRef>
521 Scanner::tryLexIdentifierOrSkipLine(const char *&First, const char *const End) {
522  const dependency_directives_scan::Token &Tok = lexToken(First, End);
523  if (Tok.isNot(tok::raw_identifier)) {
524  if (!Tok.is(tok::eod))
525  skipLine(First, End);
526  return None;
527  }
528 
529  bool NeedsCleaning = Tok.Flags & clang::Token::NeedsCleaning;
530  if (LLVM_LIKELY(!NeedsCleaning))
531  return Input.slice(Tok.Offset, Tok.getEnd());
532 
533  SmallString<64> Spelling;
534  Spelling.resize(Tok.Length);
535 
536  unsigned SpellingLength = 0;
537  const char *BufPtr = Input.begin() + Tok.Offset;
538  const char *AfterIdent = Input.begin() + Tok.getEnd();
539  while (BufPtr < AfterIdent) {
540  unsigned Size;
541  Spelling[SpellingLength++] =
542  Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts);
543  BufPtr += Size;
544  }
545 
546  return SplitIds.try_emplace(StringRef(Spelling.begin(), SpellingLength), 0)
547  .first->first();
548 }
549 
550 StringRef Scanner::lexIdentifier(const char *&First, const char *const End) {
551  Optional<StringRef> Id = tryLexIdentifierOrSkipLine(First, End);
552  assert(Id && "expected identifier token");
553  return Id.value();
554 }
555 
556 bool Scanner::isNextIdentifierOrSkipLine(StringRef Id, const char *&First,
557  const char *const End) {
558  if (Optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End)) {
559  if (*FoundId == Id)
560  return true;
561  skipLine(First, End);
562  }
563  return false;
564 }
565 
566 bool Scanner::lexAt(const char *&First, const char *const End) {
567  // Handle "@import".
568 
569  // Lex '@'.
570  const dependency_directives_scan::Token &AtTok = lexToken(First, End);
571  assert(AtTok.is(tok::at));
572  (void)AtTok;
573 
574  if (!isNextIdentifierOrSkipLine("import", First, End))
575  return false;
576  return lexModuleDirectiveBody(decl_at_import, First, End);
577 }
578 
579 bool Scanner::lexModule(const char *&First, const char *const End) {
580  StringRef Id = lexIdentifier(First, End);
581  bool Export = false;
582  if (Id == "export") {
583  Export = true;
584  Optional<StringRef> NextId = tryLexIdentifierOrSkipLine(First, End);
585  if (!NextId)
586  return false;
587  Id = *NextId;
588  }
589 
590  if (Id != "module" && Id != "import") {
591  skipLine(First, End);
592  return false;
593  }
594 
595  skipWhitespace(First, End);
596 
597  // Ignore this as a module directive if the next character can't be part of
598  // an import.
599 
600  switch (*First) {
601  case ':':
602  case '<':
603  case '"':
604  break;
605  default:
606  if (!isAsciiIdentifierContinue(*First)) {
607  skipLine(First, End);
608  return false;
609  }
610  }
611 
612  TheLexer.seek(getOffsetAt(First), /*IsAtStartOfLine*/ false);
613 
615  if (Id == "module")
617  else
619 
620  return lexModuleDirectiveBody(Kind, First, End);
621 }
622 
623 bool Scanner::lexPragma(const char *&First, const char *const End) {
624  Optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);
625  if (!FoundId)
626  return false;
627 
628  StringRef Id = *FoundId;
629  auto Kind = llvm::StringSwitch<DirectiveKind>(Id)
630  .Case("once", pp_pragma_once)
631  .Case("push_macro", pp_pragma_push_macro)
632  .Case("pop_macro", pp_pragma_pop_macro)
633  .Case("include_alias", pp_pragma_include_alias)
634  .Default(pp_none);
635  if (Kind != pp_none) {
636  lexPPDirectiveBody(First, End);
637  pushDirective(Kind);
638  return false;
639  }
640 
641  if (Id != "clang") {
642  skipLine(First, End);
643  return false;
644  }
645 
646  // #pragma clang.
647  if (!isNextIdentifierOrSkipLine("module", First, End))
648  return false;
649 
650  // #pragma clang module.
651  if (!isNextIdentifierOrSkipLine("import", First, End))
652  return false;
653 
654  // #pragma clang module import.
655  lexPPDirectiveBody(First, End);
656  pushDirective(pp_pragma_import);
657  return false;
658 }
659 
660 bool Scanner::lexEndif(const char *&First, const char *const End) {
661  // Strip out "#else" if it's empty.
662  if (topDirective() == pp_else)
663  popDirective();
664 
665  // If "#ifdef" is empty, strip it and skip the "#endif".
666  //
667  // FIXME: Once/if Clang starts disallowing __has_include in macro expansions,
668  // we can skip empty `#if` and `#elif` blocks as well after scanning for a
669  // literal __has_include in the condition. Even without that rule we could
670  // drop the tokens if we scan for identifiers in the condition and find none.
671  if (topDirective() == pp_ifdef || topDirective() == pp_ifndef) {
672  popDirective();
673  skipLine(First, End);
674  return false;
675  }
676 
677  return lexDefault(pp_endif, First, End);
678 }
679 
680 bool Scanner::lexDefault(DirectiveKind Kind, const char *&First,
681  const char *const End) {
682  lexPPDirectiveBody(First, End);
683  pushDirective(Kind);
684  return false;
685 }
686 
687 static bool isStartOfRelevantLine(char First) {
688  switch (First) {
689  case '#':
690  case '@':
691  case 'i':
692  case 'e':
693  case 'm':
694  return true;
695  }
696  return false;
697 }
698 
699 bool Scanner::lexPPLine(const char *&First, const char *const End) {
700  assert(First != End);
701 
702  skipWhitespace(First, End);
703  assert(First <= End);
704  if (First == End)
705  return false;
706 
707  if (!isStartOfRelevantLine(*First)) {
708  skipLine(First, End);
709  assert(First <= End);
710  return false;
711  }
712 
713  TheLexer.seek(getOffsetAt(First), /*IsAtStartOfLine*/ true);
714 
715  auto ScEx1 = make_scope_exit([&]() {
716  /// Clear Scanner's CurDirToks before returning, in case we didn't push a
717  /// new directive.
718  CurDirToks.clear();
719  });
720 
721  // Handle "@import".
722  if (*First == '@')
723  return lexAt(First, End);
724 
725  if (*First == 'i' || *First == 'e' || *First == 'm')
726  return lexModule(First, End);
727 
728  // Handle preprocessing directives.
729 
730  TheLexer.setParsingPreprocessorDirective(true);
731  auto ScEx2 = make_scope_exit(
732  [&]() { TheLexer.setParsingPreprocessorDirective(false); });
733 
734  // Lex '#'.
735  const dependency_directives_scan::Token &HashTok = lexToken(First, End);
736  assert(HashTok.is(tok::hash));
737  (void)HashTok;
738 
739  Optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);
740  if (!FoundId)
741  return false;
742 
743  StringRef Id = *FoundId;
744 
745  if (Id == "pragma")
746  return lexPragma(First, End);
747 
748  auto Kind = llvm::StringSwitch<DirectiveKind>(Id)
749  .Case("include", pp_include)
750  .Case("__include_macros", pp___include_macros)
751  .Case("define", pp_define)
752  .Case("undef", pp_undef)
753  .Case("import", pp_import)
754  .Case("include_next", pp_include_next)
755  .Case("if", pp_if)
756  .Case("ifdef", pp_ifdef)
757  .Case("ifndef", pp_ifndef)
758  .Case("elif", pp_elif)
759  .Case("elifdef", pp_elifdef)
760  .Case("elifndef", pp_elifndef)
761  .Case("else", pp_else)
762  .Case("endif", pp_endif)
763  .Default(pp_none);
764  if (Kind == pp_none) {
765  skipDirective(Id, First, End);
766  return false;
767  }
768 
769  if (Kind == pp_endif)
770  return lexEndif(First, End);
771 
772  switch (Kind) {
773  case pp_include:
774  case pp___include_macros:
775  case pp_include_next:
776  case pp_import:
777  lexIncludeFilename(First, End);
778  break;
779  default:
780  break;
781  }
782 
783  // Everything else.
784  return lexDefault(Kind, First, End);
785 }
786 
787 static void skipUTF8ByteOrderMark(const char *&First, const char *const End) {
788  if ((End - First) >= 3 && First[0] == '\xef' && First[1] == '\xbb' &&
789  First[2] == '\xbf')
790  First += 3;
791 }
792 
793 bool Scanner::scanImpl(const char *First, const char *const End) {
794  skipUTF8ByteOrderMark(First, End);
795  while (First != End)
796  if (lexPPLine(First, End))
797  return true;
798  return false;
799 }
800 
801 bool Scanner::scan(SmallVectorImpl<Directive> &Directives) {
802  bool Error = scanImpl(Input.begin(), Input.end());
803 
804  if (!Error) {
805  // Add an EOF on success.
806  pushDirective(pp_eof);
807  }
808 
809  ArrayRef<dependency_directives_scan::Token> RemainingTokens = Tokens;
810  for (const DirectiveWithTokens &DirWithToks : DirsWithToks) {
811  assert(RemainingTokens.size() >= DirWithToks.NumTokens);
812  Directives.emplace_back(DirWithToks.Kind,
813  RemainingTokens.take_front(DirWithToks.NumTokens));
814  RemainingTokens = RemainingTokens.drop_front(DirWithToks.NumTokens);
815  }
816  assert(RemainingTokens.empty());
817 
818  return Error;
819 }
820 
823  SmallVectorImpl<Directive> &Directives, DiagnosticsEngine *Diags,
824  SourceLocation InputSourceLoc) {
825  return Scanner(Input, Tokens, Diags, InputSourceLoc).scan(Directives);
826 }
827 
829  StringRef Source,
831  llvm::raw_ostream &OS) {
832  // Add a space separator where it is convenient for testing purposes.
833  auto needsSpaceSeparator =
834  [](tok::TokenKind Prev,
835  const dependency_directives_scan::Token &Tok) -> bool {
836  if (Prev == Tok.Kind)
837  return !Tok.isOneOf(tok::l_paren, tok::r_paren, tok::l_square,
838  tok::r_square);
839  if (Prev == tok::raw_identifier &&
840  Tok.isOneOf(tok::hash, tok::numeric_constant, tok::string_literal,
841  tok::char_constant, tok::header_name))
842  return true;
843  if (Prev == tok::r_paren &&
844  Tok.isOneOf(tok::raw_identifier, tok::hash, tok::string_literal,
845  tok::char_constant, tok::unknown))
846  return true;
847  if (Prev == tok::comma &&
848  Tok.isOneOf(tok::l_paren, tok::string_literal, tok::less))
849  return true;
850  return false;
851  };
852 
853  for (const dependency_directives_scan::Directive &Directive : Directives) {
854  Optional<tok::TokenKind> PrevTokenKind;
856  if (PrevTokenKind && needsSpaceSeparator(*PrevTokenKind, Tok))
857  OS << ' ';
858  PrevTokenKind = Tok.Kind;
859  OS << Source.slice(Tok.Offset, Tok.getEnd());
860  }
861  }
862 }
clang::scanSourceForDependencyDirectives
bool scanSourceForDependencyDirectives(StringRef Input, SmallVectorImpl< dependency_directives_scan::Token > &Tokens, SmallVectorImpl< dependency_directives_scan::Directive > &Directives, DiagnosticsEngine *Diags=nullptr, SourceLocation InputSourceLoc=SourceLocation())
Scan the input for the preprocessor directives that might have an effect on the dependencies for a co...
Definition: DependencyDirectivesScanner.cpp:821
clang::dependency_directives_scan::pp_undef
@ pp_undef
Definition: DependencyDirectivesScanner.h:64
llvm
YAML serialization mapping.
Definition: Dominators.h:30
clang::dependency_directives_scan::cxx_import_decl
@ cxx_import_decl
Definition: DependencyDirectivesScanner.h:82
skipLineComment
static void skipLineComment(const char *&First, const char *const End)
Definition: DependencyDirectivesScanner.cpp:324
clang::dependency_directives_scan::cxx_export_module_decl
@ cxx_export_module_decl
Definition: DependencyDirectivesScanner.h:83
skipUTF8ByteOrderMark
static void skipUTF8ByteOrderMark(const char *&First, const char *const End)
Definition: DependencyDirectivesScanner.cpp:787
Error
llvm::Error Error
Definition: ByteCodeEmitter.cpp:20
llvm::SmallVector
Definition: LLVM.h:38
clang::SourceLocation
Encodes a location in the source.
Definition: SourceLocation.h:86
clang::dependency_directives_scan::pp_ifndef
@ pp_ifndef
Definition: DependencyDirectivesScanner.h:74
clang::dependency_directives_scan::Token::getEnd
unsigned getEnd() const
Definition: DependencyDirectivesScanner.h:45
isStartOfRelevantLine
static bool isStartOfRelevantLine(char First)
Definition: DependencyDirectivesScanner.cpp:687
clang::DiagnosticsEngine
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
clang::dependency_directives_scan::Token::is
bool is(tok::TokenKind K) const
Definition: DependencyDirectivesScanner.h:47
int
__device__ int
Definition: __clang_hip_libdevice_declares.h:63
llvm::Optional
Definition: LLVM.h:40
clang::ComparisonCategoryType::First
@ First
clang::dependency_directives_scan::Token::isNot
bool isNot(tok::TokenKind K) const
Definition: DependencyDirectivesScanner.h:48
clang::Token
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
End
SourceLocation End
Definition: USRLocFinder.cpp:167
clang::isPreprocessingNumberBody
LLVM_READONLY bool isPreprocessingNumberBody(unsigned char c)
Return true if this is the body character of a C preprocessing number, which is [a-zA-Z0-9_.
Definition: CharInfo.h:153
clang::dependency_directives_scan::Token::Offset
unsigned Offset
Offset into the original source input.
Definition: DependencyDirectivesScanner.h:36
clang::Lexer
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.
Definition: Lexer.h:78
Offset
unsigned Offset
Definition: Format.cpp:2579
clang::StructuralEquivalenceKind::Default
@ Default
clang::dependency_directives_scan::pp_endif
@ pp_endif
Definition: DependencyDirectivesScanner.h:79
clang::dependency_directives_scan::pp_if
@ pp_if
Definition: DependencyDirectivesScanner.h:72
clang::dependency_directives_scan::Token::Kind
tok::TokenKind Kind
Definition: DependencyDirectivesScanner.h:38
clang::dependency_directives_scan::pp_eof
@ pp_eof
Definition: DependencyDirectivesScanner.h:85
wasLineContinuation
static bool wasLineContinuation(const char *First, unsigned EOLLen)
Definition: DependencyDirectivesScanner.cpp:297
Id
int Id
Definition: ASTDiff.cpp:191
clang::XRayInstrKind::None
constexpr XRayInstrMask None
Definition: XRayInstr.h:38
clang::dependency_directives_scan::Token::Length
unsigned Length
Definition: DependencyDirectivesScanner.h:37
clang::Token::getKind
tok::TokenKind getKind() const
Definition: Token.h:92
Diagnostic.h
skipOverSpaces
static void skipOverSpaces(const char *&First, const char *const End)
Definition: DependencyDirectivesScanner.cpp:175
LexDiagnostic.h
clang::dependency_directives_scan::pp_define
@ pp_define
Definition: DependencyDirectivesScanner.h:63
skipWhitespace
static void skipWhitespace(const char *&First, const char *const End)
Definition: DependencyDirectivesScanner.cpp:431
clang::dependency_directives_scan::pp_elif
@ pp_elif
Definition: DependencyDirectivesScanner.h:75
llvm::SmallString
Definition: LLVM.h:37
clang::dependency_directives_scan::pp_elifdef
@ pp_elifdef
Definition: DependencyDirectivesScanner.h:76
skipDirective
static void skipDirective(StringRef Name, const char *&First, const char *const End)
Definition: DependencyDirectivesScanner.cpp:419
DependencyDirectivesScanner.h
clang::dependency_directives_scan::pp___include_macros
@ pp___include_macros
Definition: DependencyDirectivesScanner.h:62
clang::dependency_directives_scan::cxx_module_decl
@ cxx_module_decl
Definition: DependencyDirectivesScanner.h:81
clang::dependency_directives_scan::pp_pragma_once
@ pp_pragma_once
Definition: DependencyDirectivesScanner.h:67
clang::isAsciiIdentifierContinue
LLVM_READONLY bool isAsciiIdentifierContinue(unsigned char c, bool AllowDollar=false)
Returns true if this is a body character of a C identifier, which is [a-zA-Z0-9_].
Definition: CharInfo.h:64
clang::Token::getLength
unsigned getLength() const
Definition: Token.h:128
skipToNewlineRaw
static void skipToNewlineRaw(const char *&First, const char *const End)
Definition: DependencyDirectivesScanner.cpp:301
isQuoteCppDigitSeparator
static bool isQuoteCppDigitSeparator(const char *const Start, const char *const Cur, const char *const End)
Definition: DependencyDirectivesScanner.cpp:345
clang::dependency_directives_scan::Directive::Tokens
ArrayRef< Token > Tokens
Definition: DependencyDirectivesScanner.h:92
clang::isWhitespace
LLVM_READONLY bool isWhitespace(unsigned char c)
Return true if this character is horizontal or vertical ASCII whitespace: ' ', '\t',...
Definition: CharInfo.h:93
clang::tok::TokenKind
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
clang::dependency_directives_scan::DirectiveKind
DirectiveKind
Represents the kind of preprocessor directive or a module declaration that is tracked by the scanner ...
Definition: DependencyDirectivesScanner.h:59
clang::dependency_directives_scan::Token::Flags
unsigned short Flags
Definition: DependencyDirectivesScanner.h:39
CharInfo.h
clang::syntax::NodeRole::Size
@ Size
clang::Token::getFlags
unsigned getFlags() const
Return the internal represtation of the flags.
Definition: Token.h:253
clang::dependency_directives_scan::pp_pragma_import
@ pp_pragma_import
Definition: DependencyDirectivesScanner.h:66
clang::dependency_directives_scan::pp_elifndef
@ pp_elifndef
Definition: DependencyDirectivesScanner.h:77
clang::dependency_directives_scan::pp_pragma_push_macro
@ pp_pragma_push_macro
Definition: DependencyDirectivesScanner.h:68
clang::dependency_directives_scan::pp_else
@ pp_else
Definition: DependencyDirectivesScanner.h:78
clang::printDependencyDirectivesAsSource
void printDependencyDirectivesAsSource(StringRef Source, ArrayRef< dependency_directives_scan::Directive > Directives, llvm::raw_ostream &OS)
Print the previously scanned dependency directives as minimized source text.
Definition: DependencyDirectivesScanner.cpp:828
llvm::ArrayRef
Definition: LLVM.h:34
clang::dependency_directives_scan::pp_import
@ pp_import
Definition: DependencyDirectivesScanner.h:65
Lexer.h
clang::dependency_directives_scan::pp_include
@ pp_include
Definition: DependencyDirectivesScanner.h:61
isEOL
static unsigned isEOL(const char *First, const char *const End)
Definition: DependencyDirectivesScanner.cpp:248
clang::LangOptions
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:78
clang::ObjCPropertyAttribute::Kind
Kind
Definition: DeclObjCCommon.h:22
clang::dependency_directives_scan::Token
Token lexed as part of dependency directive scanning.
Definition: DependencyDirectivesScanner.h:34
clang::dependency_directives_scan::decl_at_import
@ decl_at_import
Definition: DependencyDirectivesScanner.h:80
clang::dependency_directives_scan::pp_include_next
@ pp_include_next
Definition: DependencyDirectivesScanner.h:71
clang::isVerticalWhitespace
LLVM_READONLY bool isVerticalWhitespace(unsigned char c)
Returns true if this character is vertical ASCII whitespace: '\n', '\r'.
Definition: CharInfo.h:84
clang
Definition: CalledOnceCheck.h:17
skipBlockComment
static void skipBlockComment(const char *&First, const char *const End)
Definition: DependencyDirectivesScanner.cpp:330
clang::dependency_directives_scan::pp_ifdef
@ pp_ifdef
Definition: DependencyDirectivesScanner.h:73
clang::ComparisonCategoryType::Last
@ Last
skipString
static void skipString(const char *&First, const char *const End)
Definition: DependencyDirectivesScanner.cpp:257
skipLine
static void skipLine(const char *&First, const char *const End)
Definition: DependencyDirectivesScanner.cpp:367
clang::dependency_directives_scan::Directive
Represents a directive that's lexed as part of the dependency directives scanning.
Definition: DependencyDirectivesScanner.h:91
clang::dependency_directives_scan::pp_pragma_include_alias
@ pp_pragma_include_alias
Definition: DependencyDirectivesScanner.h:70
clang::Token::NeedsCleaning
@ NeedsCleaning
Definition: Token.h:78
clang::dependency_directives_scan::cxx_export_import_decl
@ cxx_export_import_decl
Definition: DependencyDirectivesScanner.h:84
skipNewline
static unsigned skipNewline(const char *&First, const char *End)
Definition: DependencyDirectivesScanner.cpp:287
clang::isHorizontalWhitespace
LLVM_READONLY bool isHorizontalWhitespace(unsigned char c)
Returns true if this character is horizontal ASCII whitespace: ' ', '\t', '\f', '\v'.
Definition: CharInfo.h:76
clang::comments::tok::eof
@ eof
Definition: CommentLexer.h:33
llvm::SmallVectorImpl
Definition: Randstruct.h:18
isRawStringLiteral
static LLVM_NODISCARD bool isRawStringLiteral(const char *First, const char *Current)
Definition: DependencyDirectivesScanner.cpp:180
clang::dependency_directives_scan
Definition: DependencyDirectivesScanner.h:31
clang::Lexer::getCharAndSizeNoWarn
static char getCharAndSizeNoWarn(const char *Ptr, unsigned &Size, const LangOptions &LangOpts)
getCharAndSizeNoWarn - Like the getCharAndSize method, but does not ever emit a warning.
Definition: Lexer.h:580
clang::dependency_directives_scan::Token::isOneOf
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: DependencyDirectivesScanner.h:49
clang::dependency_directives_scan::pp_pragma_pop_macro
@ pp_pragma_pop_macro
Definition: DependencyDirectivesScanner.h:69
skipRawString
static void skipRawString(const char *&First, const char *const End)
Definition: DependencyDirectivesScanner.cpp:205
clang::dependency_directives_scan::pp_none
@ pp_none
Definition: DependencyDirectivesScanner.h:60