23#include "llvm/ADT/ScopeExit.h"
24#include "llvm/ADT/SmallString.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringSwitch.h"
35struct DirectiveWithTokens {
40 :
Kind(
Kind), NumTokens(NumTokens) {}
63 Scanner(StringRef Input,
66 : Input(Input), Tokens(Tokens), Diags(Diags),
67 InputSourceLoc(InputSourceLoc), LangOpts(getLangOptsForDepScanning()),
68 TheLexer(InputSourceLoc, LangOpts, Input.begin(), Input.begin(),
75 LangOpts.LineComment =
true;
89 lexToken(
const char *&
First,
const char *
const End);
92 const char *
const End);
94 void skipLine(
const char *&
First,
const char *
const End);
95 void skipDirective(StringRef Name,
const char *&
First,
const char *
const End);
106 [[nodiscard]] std::optional<StringRef>
107 tryLexIdentifierOrSkipLine(
const char *&
First,
const char *
const End);
110 [[nodiscard]] StringRef lexIdentifier(
const char *&
First,
111 const char *
const End);
118 [[nodiscard]]
bool isNextIdentifierOrSkipLine(StringRef
Id,
120 const char *
const End);
128 const char *
const End);
135 [[nodiscard]] std::optional<StringRef>
136 tryLexStringLiteralOrSkipLine(
const char *&
First,
const char *
const End);
138 [[nodiscard]]
bool scanImpl(
const char *
First,
const char *
const End);
139 [[nodiscard]]
bool lexPPLine(
const char *&
First,
const char *
const End);
140 [[nodiscard]]
bool lexAt(
const char *&
First,
const char *
const End);
141 [[nodiscard]]
bool lexModule(
const char *&
First,
const char *
const End);
142 [[nodiscard]]
bool lexDefine(
const char *HashLoc,
const char *&
First,
143 const char *
const End);
144 [[nodiscard]]
bool lexPragma(
const char *&
First,
const char *
const End);
145 [[nodiscard]]
bool lex_Pragma(
const char *&
First,
const char *
const End);
146 [[nodiscard]]
bool lexEndif(
const char *&
First,
const char *
const End);
148 const char *
const End);
149 [[nodiscard]]
bool lexModuleDirectiveBody(
DirectiveKind Kind,
151 const char *
const End);
152 void lexPPDirectiveBody(
const char *&
First,
const char *
const End);
155 Tokens.append(CurDirToks);
156 DirsWithToks.emplace_back(Kind, CurDirToks.size());
158 return DirsWithToks.back();
160 void popDirective() {
161 Tokens.pop_back_n(DirsWithToks.pop_back_val().NumTokens);
164 return DirsWithToks.empty() ?
pp_none : DirsWithToks.back().Kind;
167 unsigned getOffsetAt(
const char *CurPtr)
const {
168 return CurPtr - Input.data();
173 bool reportError(
const char *CurPtr,
unsigned Err);
175 StringMap<char> SplitIds;
181 const char *LastTokenPtr =
nullptr;
196bool Scanner::reportError(
const char *CurPtr,
unsigned Err) {
199 assert(CurPtr >= Input.data() &&
"invalid buffer ptr");
200 Diags->Report(InputSourceLoc.getLocWithOffset(getOffsetAt(CurPtr)), Err);
210 const char *Current) {
211 assert(
First <= Current);
214 if (*Current !=
'"' ||
First == Current)
225 if (*Current ==
'u' || *Current ==
'U' || *Current ==
'L')
229 if (*Current !=
'8' ||
First == Current || *Current-- !=
'u')
235 assert(
First[0] ==
'"');
236 assert(
First[-1] ==
'R');
258 while (
Last != End &&
size_t(
Last -
First) < Terminator.size() &&
267 if (
size_t(
Last -
First) < Terminator.size())
277static unsigned isEOL(
const char *
First,
const char *
const End) {
288 const char Terminator = *
First ==
'<' ?
'>' : *
First;
303 const char *FirstAfterBackslashPastSpace =
First;
305 if (
unsigned NLSize =
isEOL(FirstAfterBackslashPastSpace, End)) {
308 First = FirstAfterBackslashPastSpace + NLSize - 1;
321 assert(Len &&
"expected newline");
327 return *(
First - (
int)EOLLen - 1) ==
'\\';
345 if (
First[-1] !=
'\\')
361 if (End -
First < 4) {
375 const char *
const Cur,
376 const char *
const End) {
377 assert(*Cur ==
'\'' &&
"expected quotation character");
385 char Prev = *(Cur - 1);
386 if (Prev ==
'L' || Prev ==
'U' || Prev ==
'u')
388 if (Prev ==
'8' && (Cur - 1 != Start) && *(Cur - 2) ==
'u')
396void Scanner::skipLine(
const char *&
First,
const char *
const End) {
398 assert(
First <= End);
406 const char *Start =
First;
411 LastTokenPtr =
First;
421 LastTokenPtr =
First;
426 if (
First[1] ==
'/') {
432 if (
First[1] !=
'*') {
433 LastTokenPtr =
First;
451void Scanner::skipDirective(StringRef Name,
const char *&
First,
452 const char *
const End) {
453 if (llvm::StringSwitch<bool>(Name)
454 .Case(
"warning",
true)
460 skipLine(
First, End);
465 assert(
First <= End);
481 if (
First[1] ==
'/') {
496 const char *
const End) {
497 const char *DirectiveLoc = Input.data() + CurDirToks.front().Offset;
500 if (Tok.
is(tok::eof))
503 diag::err_dep_source_scanner_missing_semi_after_at_import);
504 if (Tok.
is(tok::semi))
513 DirectiveLoc, diag::err_dep_source_scanner_unexpected_tokens_at_import);
519 const char *
const End) {
521 TheLexer.LexFromRawLexer(Tok);
522 First = Input.data() + TheLexer.getCurrentBufferOffset();
523 assert(
First <= End);
525 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.
getLength();
528 return CurDirToks.back();
532Scanner::lexIncludeFilename(
const char *&
First,
const char *
const End) {
534 TheLexer.LexIncludeFilename(Tok);
535 First = Input.data() + TheLexer.getCurrentBufferOffset();
536 assert(
First <= End);
538 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.
getLength();
541 return CurDirToks.back();
544void Scanner::lexPPDirectiveBody(
const char *&
First,
const char *
const End) {
547 if (Tok.
is(tok::eod))
555 if (LLVM_LIKELY(!NeedsCleaning))
559 Spelling.resize(Tok.
Length);
564 unsigned SpellingLength = 0;
565 const char *BufPtr = Input.begin() + Tok.
Offset;
566 const char *AfterIdent = Input.begin() + Tok.
getEnd();
567 while (BufPtr < AfterIdent) {
569 Spelling[SpellingLength++] =
574 return SplitIds.try_emplace(StringRef(Spelling.begin(), SpellingLength), 0)
578std::optional<StringRef>
579Scanner::tryLexIdentifierOrSkipLine(
const char *&
First,
const char *
const End) {
581 if (Tok.
isNot(tok::raw_identifier)) {
582 if (!Tok.
is(tok::eod))
583 skipLine(
First, End);
587 return cleanStringIfNeeded(Tok);
590StringRef Scanner::lexIdentifier(
const char *&
First,
const char *
const End) {
591 std::optional<StringRef>
Id = tryLexIdentifierOrSkipLine(
First, End);
592 assert(
Id &&
"expected identifier token");
596bool Scanner::isNextIdentifierOrSkipLine(StringRef
Id,
const char *&
First,
597 const char *
const End) {
598 if (std::optional<StringRef> FoundId =
599 tryLexIdentifierOrSkipLine(
First, End)) {
602 skipLine(
First, End);
608 const char *
const End) {
612 skipLine(
First, End);
616std::optional<StringRef>
617Scanner::tryLexStringLiteralOrSkipLine(
const char *&
First,
618 const char *
const End) {
621 if (!Tok.
is(tok::eod))
622 skipLine(
First, End);
626 return cleanStringIfNeeded(Tok);
629bool Scanner::lexAt(
const char *&
First,
const char *
const End) {
634 assert(AtTok.
is(tok::at));
637 if (!isNextIdentifierOrSkipLine(
"import",
First, End))
642bool Scanner::lexModule(
const char *&
First,
const char *
const End) {
643 StringRef
Id = lexIdentifier(
First, End);
645 if (
Id ==
"export") {
647 std::optional<StringRef> NextId = tryLexIdentifierOrSkipLine(
First, End);
653 if (
Id !=
"module" &&
Id !=
"import") {
654 skipLine(
First, End);
670 skipLine(
First, End);
675 TheLexer.seek(getOffsetAt(
First),
false);
683 return lexModuleDirectiveBody(Kind,
First, End);
686bool Scanner::lex_Pragma(
const char *&
First,
const char *
const End) {
687 if (!isNextTokenOrSkipLine(tok::l_paren,
First, End))
690 std::optional<StringRef> Str = tryLexStringLiteralOrSkipLine(
First, End);
692 if (!Str || !isNextTokenOrSkipLine(tok::r_paren,
First, End))
702 const char *
Begin = Buffer.c_str();
703 Scanner PragmaScanner{StringRef(
Begin, Buffer.size()), DiscardTokens, Diags,
706 PragmaScanner.TheLexer.setParsingPreprocessorDirective(
true);
707 if (PragmaScanner.lexPragma(
Begin, Buffer.end()))
712 skipLine(
First, End);
716 assert(
Begin == Buffer.end());
721bool Scanner::lexPragma(
const char *&
First,
const char *
const End) {
722 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(
First, End);
726 StringRef
Id = *FoundId;
727 auto Kind = llvm::StringSwitch<DirectiveKind>(
Id)
734 lexPPDirectiveBody(
First, End);
740 skipLine(
First, End);
744 FoundId = tryLexIdentifierOrSkipLine(
First, End);
750 if (
Id ==
"system_header") {
751 lexPPDirectiveBody(
First, End);
756 if (
Id !=
"module") {
757 skipLine(
First, End);
762 if (!isNextIdentifierOrSkipLine(
"import",
First, End))
766 lexPPDirectiveBody(
First, End);
771bool Scanner::lexEndif(
const char *&
First,
const char *
const End) {
784 skipLine(
First, End);
792 const char *
const End) {
793 lexPPDirectiveBody(
First, End);
811bool Scanner::lexPPLine(
const char *&
First,
const char *
const End) {
812 assert(
First != End);
815 assert(
First <= End);
820 skipLine(
First, End);
821 assert(
First <= End);
825 LastTokenPtr =
First;
827 TheLexer.seek(getOffsetAt(
First),
true);
829 auto ScEx1 = make_scope_exit([&]() {
837 return lexAt(
First, End);
840 return lexModule(
First, End);
843 if (isNextIdentifierOrSkipLine(
"_Pragma",
First, End))
844 return lex_Pragma(
First, End);
850 TheLexer.setParsingPreprocessorDirective(
true);
851 auto ScEx2 = make_scope_exit(
852 [&]() { TheLexer.setParsingPreprocessorDirective(
false); });
856 if (HashTok.is(tok::hashhash)) {
860 skipLine(
First, End);
861 assert(
First <= End);
864 assert(HashTok.is(tok::hash));
867 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(
First, End);
871 StringRef
Id = *FoundId;
874 return lexPragma(
First, End);
876 auto Kind = llvm::StringSwitch<DirectiveKind>(
Id)
898 return lexEndif(
First, End);
905 lexIncludeFilename(
First, End);
912 return lexDefault(Kind,
First, End);
921bool Scanner::scanImpl(
const char *
First,
const char *
const End) {
924 if (lexPPLine(
First, End))
930 bool Error = scanImpl(Input.begin(), Input.end());
935 (Tokens.empty() || LastTokenPtr > Input.begin() + Tokens.back().Offset))
941 for (
const DirectiveWithTokens &DirWithToks : DirsWithToks) {
942 assert(RemainingTokens.size() >= DirWithToks.NumTokens);
943 Directives.emplace_back(DirWithToks.Kind,
944 RemainingTokens.take_front(DirWithToks.NumTokens));
945 RemainingTokens = RemainingTokens.drop_front(DirWithToks.NumTokens);
947 assert(RemainingTokens.empty());
956 return Scanner(Input, Tokens, Diags, InputSourceLoc).scan(Directives);
962 llvm::raw_ostream &OS) {
964 auto needsSpaceSeparator =
967 if (Prev == Tok.
Kind)
968 return !Tok.
isOneOf(tok::l_paren, tok::r_paren, tok::l_square,
970 if (Prev == tok::raw_identifier &&
971 Tok.
isOneOf(tok::hash, tok::numeric_constant, tok::string_literal,
972 tok::char_constant, tok::header_name))
974 if (Prev == tok::r_paren &&
975 Tok.
isOneOf(tok::raw_identifier, tok::hash, tok::string_literal,
976 tok::char_constant, tok::unknown))
978 if (Prev == tok::comma &&
979 Tok.
isOneOf(tok::l_paren, tok::string_literal, tok::less))
986 OS <<
"<TokBeforeEOF>";
987 std::optional<tok::TokenKind> PrevTokenKind;
989 if (PrevTokenKind && needsSpaceSeparator(*PrevTokenKind, Tok))
991 PrevTokenKind = Tok.
Kind;
Defines the Diagnostic-related interfaces.
static void skipBlockComment(const char *&First, const char *const End)
static void skipRawString(const char *&First, const char *const End)
static void skipString(const char *&First, const char *const End)
static bool isStartOfRelevantLine(char First)
static void skipWhitespace(const char *&First, const char *const End)
static bool isRawStringLiteral(const char *First, const char *Current)
static void skipOverSpaces(const char *&First, const char *const End)
static unsigned isEOL(const char *First, const char *const End)
static void skipToNewlineRaw(const char *&First, const char *const End)
static unsigned skipNewline(const char *&First, const char *End)
static void skipUTF8ByteOrderMark(const char *&First, const char *const End)
static void skipLineComment(const char *&First, const char *const End)
static bool isQuoteCppDigitSeparator(const char *const Start, const char *const Cur, const char *const End)
static bool wasLineContinuation(const char *First, unsigned EOLLen)
This is the interface for scanning header and source files to get the minimum necessary preprocessor ...
Concrete class used by the front-end to report problems and issues.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.
static char getCharAndSizeNoWarn(const char *Ptr, unsigned &Size, const LangOptions &LangOpts)
getCharAndSizeNoWarn - Like the getCharAndSize method, but does not ever emit a warning.
Encodes a location in the source.
Token - This structure provides full information about a lexed token.
unsigned getFlags() const
Return the internal represtation of the flags.
unsigned getLength() const
tok::TokenKind getKind() const
DirectiveKind
Represents the kind of preprocessor directive or a module declaration that is tracked by the scanner ...
@ tokens_present_before_eof
Indicates that there are tokens present between the last scanned directive and eof.
@ pp_pragma_system_header
@ pp_pragma_include_alias
bool isStringLiteral(TokenKind K)
Return true if this is a C or C++ string-literal (or C++11 user-defined-string-literal) token.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
LLVM_READONLY bool isVerticalWhitespace(unsigned char c)
Returns true if this character is vertical ASCII whitespace: '\n', '\r'.
void printDependencyDirectivesAsSource(StringRef Source, ArrayRef< dependency_directives_scan::Directive > Directives, llvm::raw_ostream &OS)
Print the previously scanned dependency directives as minimized source text.
LLVM_READONLY bool isHorizontalWhitespace(unsigned char c)
Returns true if this character is horizontal ASCII whitespace: ' ', '\t', '\f', '\v'.
bool scanSourceForDependencyDirectives(StringRef Input, SmallVectorImpl< dependency_directives_scan::Token > &Tokens, SmallVectorImpl< dependency_directives_scan::Directive > &Directives, DiagnosticsEngine *Diags=nullptr, SourceLocation InputSourceLoc=SourceLocation())
Scan the input for the preprocessor directives that might have an effect on the dependencies for a co...
LLVM_READONLY bool isWhitespace(unsigned char c)
Return true if this character is horizontal or vertical ASCII whitespace: ' ', '\t',...
LLVM_READONLY bool isPreprocessingNumberBody(unsigned char c)
Return true if this is the body character of a C preprocessing number, which is [a-zA-Z0-9_.
LLVM_READONLY bool isAsciiIdentifierContinue(unsigned char c, bool AllowDollar=false)
Returns true if this is a body character of a C identifier, which is [a-zA-Z0-9_].
void prepare_PragmaString(SmallVectorImpl< char > &StrVal)
Destringize a _Pragma("") string according to C11 6.10.9.1: "The string literal is destringized by de...
YAML serialization mapping.
Represents a directive that's lexed as part of the dependency directives scanning.
DirectiveKind Kind
The kind of token.
Token lexed as part of dependency directive scanning.
bool isNot(tok::TokenKind K) const
unsigned Offset
Offset into the original source input.
bool is(tok::TokenKind K) const
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const