22#include "llvm/ADT/ScopeExit.h"
23#include "llvm/ADT/SmallString.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/StringSwitch.h"
34struct DirectiveWithTokens {
39 :
Kind(
Kind), NumTokens(NumTokens) {}
62 Scanner(StringRef Input,
65 : Input(Input), Tokens(Tokens), Diags(Diags),
66 InputSourceLoc(InputSourceLoc), LangOpts(getLangOptsForDepScanning()),
67 TheLexer(InputSourceLoc, LangOpts, Input.begin(), Input.begin(),
74 LangOpts.LineComment =
true;
86 lexToken(
const char *&
First,
const char *
const End);
89 const char *
const End);
91 void skipLine(
const char *&
First,
const char *
const End);
92 void skipDirective(StringRef Name,
const char *&
First,
const char *
const End);
99 [[nodiscard]] std::optional<StringRef>
100 tryLexIdentifierOrSkipLine(
const char *&
First,
const char *
const End);
103 [[nodiscard]] StringRef lexIdentifier(
const char *&
First,
104 const char *
const End);
111 [[nodiscard]]
bool isNextIdentifierOrSkipLine(StringRef
Id,
113 const char *
const End);
115 [[nodiscard]]
bool scanImpl(
const char *
First,
const char *
const End);
116 [[nodiscard]]
bool lexPPLine(
const char *&
First,
const char *
const End);
117 [[nodiscard]]
bool lexAt(
const char *&
First,
const char *
const End);
118 [[nodiscard]]
bool lexModule(
const char *&
First,
const char *
const End);
119 [[nodiscard]]
bool lexDefine(
const char *HashLoc,
const char *&
First,
120 const char *
const End);
121 [[nodiscard]]
bool lexPragma(
const char *&
First,
const char *
const End);
122 [[nodiscard]]
bool lexEndif(
const char *&
First,
const char *
const End);
124 const char *
const End);
125 [[nodiscard]]
bool lexModuleDirectiveBody(
DirectiveKind Kind,
127 const char *
const End);
128 void lexPPDirectiveBody(
const char *&
First,
const char *
const End);
131 Tokens.append(CurDirToks);
132 DirsWithToks.emplace_back(Kind, CurDirToks.size());
134 return DirsWithToks.back();
136 void popDirective() {
137 Tokens.pop_back_n(DirsWithToks.pop_back_val().NumTokens);
140 return DirsWithToks.empty() ?
pp_none : DirsWithToks.back().Kind;
143 unsigned getOffsetAt(
const char *CurPtr)
const {
144 return CurPtr - Input.data();
149 bool reportError(
const char *CurPtr,
unsigned Err);
151 StringMap<char> SplitIds;
157 const char *LastTokenPtr =
nullptr;
172bool Scanner::reportError(
const char *CurPtr,
unsigned Err) {
175 assert(CurPtr >= Input.data() &&
"invalid buffer ptr");
176 Diags->Report(InputSourceLoc.getLocWithOffset(getOffsetAt(CurPtr)), Err);
186 const char *Current) {
187 assert(
First <= Current);
190 if (*Current !=
'"' ||
First == Current)
201 if (*Current ==
'u' || *Current ==
'U' || *Current ==
'L')
205 if (*Current !=
'8' ||
First == Current || *Current-- !=
'u')
211 assert(
First[0] ==
'"');
212 assert(
First[-1] ==
'R');
234 while (
Last != End &&
size_t(
Last -
First) < Terminator.size() &&
243 if (
size_t(
Last -
First) < Terminator.size())
253static unsigned isEOL(
const char *
First,
const char *
const End) {
264 const char Terminator = *
First ==
'<' ?
'>' : *
First;
279 const char *FirstAfterBackslashPastSpace =
First;
281 if (
unsigned NLSize =
isEOL(FirstAfterBackslashPastSpace, End)) {
284 First = FirstAfterBackslashPastSpace + NLSize - 1;
297 assert(Len &&
"expected newline");
303 return *(
First - (
int)EOLLen - 1) ==
'\\';
321 if (
First[-1] !=
'\\')
337 if (End -
First < 4) {
351 const char *
const Cur,
352 const char *
const End) {
353 assert(*Cur ==
'\'' &&
"expected quotation character");
361 char Prev = *(Cur - 1);
362 if (Prev ==
'L' || Prev ==
'U' || Prev ==
'u')
364 if (Prev ==
'8' && (Cur - 1 != Start) && *(Cur - 2) ==
'u')
372void Scanner::skipLine(
const char *&
First,
const char *
const End) {
374 assert(
First <= End);
382 const char *Start =
First;
387 LastTokenPtr =
First;
397 LastTokenPtr =
First;
402 if (
First[1] ==
'/') {
408 if (
First[1] !=
'*') {
409 LastTokenPtr =
First;
427void Scanner::skipDirective(StringRef Name,
const char *&
First,
428 const char *
const End) {
429 if (llvm::StringSwitch<bool>(Name)
430 .Case(
"warning",
true)
436 skipLine(
First, End);
441 assert(
First <= End);
457 if (
First[1] ==
'/') {
472 const char *
const End) {
473 const char *DirectiveLoc = Input.data() + CurDirToks.front().Offset;
476 if (Tok.
is(tok::eof))
479 diag::err_dep_source_scanner_missing_semi_after_at_import);
480 if (Tok.
is(tok::semi))
489 DirectiveLoc, diag::err_dep_source_scanner_unexpected_tokens_at_import);
495 const char *
const End) {
497 TheLexer.LexFromRawLexer(Tok);
498 First = Input.data() + TheLexer.getCurrentBufferOffset();
499 assert(
First <= End);
504 return CurDirToks.back();
508Scanner::lexIncludeFilename(
const char *&
First,
const char *
const End) {
510 TheLexer.LexIncludeFilename(Tok);
511 First = Input.data() + TheLexer.getCurrentBufferOffset();
512 assert(
First <= End);
517 return CurDirToks.back();
520void Scanner::lexPPDirectiveBody(
const char *&
First,
const char *
const End) {
523 if (Tok.
is(tok::eod))
528[[nodiscard]] std::optional<StringRef>
529Scanner::tryLexIdentifierOrSkipLine(
const char *&
First,
const char *
const End) {
531 if (Tok.
isNot(tok::raw_identifier)) {
532 if (!Tok.
is(tok::eod))
533 skipLine(
First, End);
538 if (LLVM_LIKELY(!NeedsCleaning))
542 Spelling.resize(Tok.
Length);
544 unsigned SpellingLength = 0;
545 const char *BufPtr = Input.begin() + Tok.
Offset;
546 const char *AfterIdent = Input.begin() + Tok.
getEnd();
547 while (BufPtr < AfterIdent) {
549 Spelling[SpellingLength++] =
554 return SplitIds.try_emplace(StringRef(Spelling.begin(), SpellingLength), 0)
558StringRef Scanner::lexIdentifier(
const char *&
First,
const char *
const End) {
559 std::optional<StringRef>
Id = tryLexIdentifierOrSkipLine(
First, End);
560 assert(
Id &&
"expected identifier token");
564bool Scanner::isNextIdentifierOrSkipLine(StringRef
Id,
const char *&
First,
565 const char *
const End) {
566 if (std::optional<StringRef> FoundId =
567 tryLexIdentifierOrSkipLine(
First, End)) {
570 skipLine(
First, End);
575bool Scanner::lexAt(
const char *&
First,
const char *
const End) {
580 assert(AtTok.
is(tok::at));
583 if (!isNextIdentifierOrSkipLine(
"import",
First, End))
588bool Scanner::lexModule(
const char *&
First,
const char *
const End) {
589 StringRef
Id = lexIdentifier(
First, End);
591 if (
Id ==
"export") {
593 std::optional<StringRef> NextId = tryLexIdentifierOrSkipLine(
First, End);
599 if (
Id !=
"module" &&
Id !=
"import") {
600 skipLine(
First, End);
616 skipLine(
First, End);
621 TheLexer.seek(getOffsetAt(
First),
false);
629 return lexModuleDirectiveBody(Kind,
First, End);
632bool Scanner::lexPragma(
const char *&
First,
const char *
const End) {
633 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(
First, End);
637 StringRef
Id = *FoundId;
638 auto Kind = llvm::StringSwitch<DirectiveKind>(
Id)
645 lexPPDirectiveBody(
First, End);
651 skipLine(
First, End);
656 if (!isNextIdentifierOrSkipLine(
"module",
First, End))
660 if (!isNextIdentifierOrSkipLine(
"import",
First, End))
664 lexPPDirectiveBody(
First, End);
669bool Scanner::lexEndif(
const char *&
First,
const char *
const End) {
682 skipLine(
First, End);
690 const char *
const End) {
691 lexPPDirectiveBody(
First, End);
708bool Scanner::lexPPLine(
const char *&
First,
const char *
const End) {
709 assert(
First != End);
712 assert(
First <= End);
717 skipLine(
First, End);
718 assert(
First <= End);
722 LastTokenPtr =
First;
724 TheLexer.seek(getOffsetAt(
First),
true);
726 auto ScEx1 = make_scope_exit([&]() {
734 return lexAt(
First, End);
737 return lexModule(
First, End);
741 TheLexer.setParsingPreprocessorDirective(
true);
742 auto ScEx2 = make_scope_exit(
743 [&]() { TheLexer.setParsingPreprocessorDirective(
false); });
747 if (HashTok.is(tok::hashhash)) {
751 skipLine(
First, End);
752 assert(
First <= End);
755 assert(HashTok.is(tok::hash));
758 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(
First, End);
762 StringRef
Id = *FoundId;
765 return lexPragma(
First, End);
767 auto Kind = llvm::StringSwitch<DirectiveKind>(
Id)
789 return lexEndif(
First, End);
796 lexIncludeFilename(
First, End);
803 return lexDefault(Kind,
First, End);
812bool Scanner::scanImpl(
const char *
First,
const char *
const End) {
815 if (lexPPLine(
First, End))
821 bool Error = scanImpl(Input.begin(), Input.end());
826 (Tokens.empty() || LastTokenPtr > Input.begin() + Tokens.back().Offset))
832 for (
const DirectiveWithTokens &DirWithToks : DirsWithToks) {
833 assert(RemainingTokens.size() >= DirWithToks.NumTokens);
834 Directives.emplace_back(DirWithToks.Kind,
835 RemainingTokens.take_front(DirWithToks.NumTokens));
836 RemainingTokens = RemainingTokens.drop_front(DirWithToks.NumTokens);
838 assert(RemainingTokens.empty());
847 return Scanner(Input, Tokens, Diags, InputSourceLoc).scan(Directives);
853 llvm::raw_ostream &
OS) {
855 auto needsSpaceSeparator =
858 if (Prev == Tok.
Kind)
859 return !Tok.
isOneOf(tok::l_paren, tok::r_paren, tok::l_square,
861 if (Prev == tok::raw_identifier &&
862 Tok.
isOneOf(tok::hash, tok::numeric_constant, tok::string_literal,
863 tok::char_constant, tok::header_name))
865 if (Prev == tok::r_paren &&
866 Tok.
isOneOf(tok::raw_identifier, tok::hash, tok::string_literal,
867 tok::char_constant, tok::unknown))
869 if (Prev == tok::comma &&
870 Tok.
isOneOf(tok::l_paren, tok::string_literal, tok::less))
877 OS <<
"<TokBeforeEOF>";
878 std::optional<tok::TokenKind> PrevTokenKind;
880 if (PrevTokenKind && needsSpaceSeparator(*PrevTokenKind, Tok))
882 PrevTokenKind = Tok.
Kind;
Defines the Diagnostic-related interfaces.
static void skipBlockComment(const char *&First, const char *const End)
static void skipRawString(const char *&First, const char *const End)
static void skipString(const char *&First, const char *const End)
static bool isStartOfRelevantLine(char First)
static void skipWhitespace(const char *&First, const char *const End)
static bool isRawStringLiteral(const char *First, const char *Current)
static void skipOverSpaces(const char *&First, const char *const End)
static unsigned isEOL(const char *First, const char *const End)
static void skipToNewlineRaw(const char *&First, const char *const End)
static unsigned skipNewline(const char *&First, const char *End)
static void skipUTF8ByteOrderMark(const char *&First, const char *const End)
static void skipLineComment(const char *&First, const char *const End)
static bool isQuoteCppDigitSeparator(const char *const Start, const char *const Cur, const char *const End)
static bool wasLineContinuation(const char *First, unsigned EOLLen)
This is the interface for scanning header and source files to get the minimum necessary preprocessor ...
Concrete class used by the front-end to report problems and issues.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.
static char getCharAndSizeNoWarn(const char *Ptr, unsigned &Size, const LangOptions &LangOpts)
getCharAndSizeNoWarn - Like the getCharAndSize method, but does not ever emit a warning.
Encodes a location in the source.
Token - This structure provides full information about a lexed token.
unsigned getFlags() const
Return the internal represtation of the flags.
unsigned getLength() const
tok::TokenKind getKind() const
DirectiveKind
Represents the kind of preprocessor directive or a module declaration that is tracked by the scanner ...
@ tokens_present_before_eof
Indicates that there are tokens present between the last scanned directive and eof.
@ pp_pragma_include_alias
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
LLVM_READONLY bool isVerticalWhitespace(unsigned char c)
Returns true if this character is vertical ASCII whitespace: '\n', '\r'.
void printDependencyDirectivesAsSource(StringRef Source, ArrayRef< dependency_directives_scan::Directive > Directives, llvm::raw_ostream &OS)
Print the previously scanned dependency directives as minimized source text.
LLVM_READONLY bool isHorizontalWhitespace(unsigned char c)
Returns true if this character is horizontal ASCII whitespace: ' ', '\t', '\f', '\v'.
bool scanSourceForDependencyDirectives(StringRef Input, SmallVectorImpl< dependency_directives_scan::Token > &Tokens, SmallVectorImpl< dependency_directives_scan::Directive > &Directives, DiagnosticsEngine *Diags=nullptr, SourceLocation InputSourceLoc=SourceLocation())
Scan the input for the preprocessor directives that might have an effect on the dependencies for a co...
LLVM_READONLY bool isWhitespace(unsigned char c)
Return true if this character is horizontal or vertical ASCII whitespace: ' ', '\t',...
LLVM_READONLY bool isPreprocessingNumberBody(unsigned char c)
Return true if this is the body character of a C preprocessing number, which is [a-zA-Z0-9_.
LLVM_READONLY bool isAsciiIdentifierContinue(unsigned char c, bool AllowDollar=false)
Returns true if this is a body character of a C identifier, which is [a-zA-Z0-9_].
YAML serialization mapping.
Represents a directive that's lexed as part of the dependency directives scanning.
DirectiveKind Kind
The kind of token.
Token lexed as part of dependency directive scanning.
bool isNot(tok::TokenKind K) const
unsigned Offset
Offset into the original source input.
bool is(tok::TokenKind K) const
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const