16 #include "llvm/Support/ErrorHandling.h" 17 using namespace clang;
25 (CPlusPlus11 && (Str[0] ==
'u' || Str[0] ==
'U' || Str[0] ==
'R'))) {
32 if (Str[1] ==
'R' && Str[0] !=
'R' && Str.size() == 2 &&
CPlusPlus11)
36 if (Str[0] ==
'u' && Str[1] ==
'8') {
37 if (Str.size() == 2)
return true;
38 if (Str.size() == 3 && Str[2] ==
'R')
return true;
47 bool TokenConcatenation::IsIdentifierStringPrefix(
const Token &
Tok)
const {
56 LangOpts.CPlusPlus11);
61 const char *TokPtr = Buffer;
63 return IsStringPrefix(StringRef(TokPtr, length), LangOpts.CPlusPlus11);
70 memset(TokenInfo, 0,
sizeof(TokenInfo));
73 TokenInfo[tok::identifier ] |= aci_custom;
74 TokenInfo[tok::numeric_constant] |= aci_custom_firstchar;
75 TokenInfo[tok::period ] |= aci_custom_firstchar;
76 TokenInfo[tok::amp ] |= aci_custom_firstchar;
77 TokenInfo[tok::plus ] |= aci_custom_firstchar;
78 TokenInfo[tok::minus ] |= aci_custom_firstchar;
79 TokenInfo[tok::slash ] |= aci_custom_firstchar;
80 TokenInfo[tok::less ] |= aci_custom_firstchar;
81 TokenInfo[tok::greater ] |= aci_custom_firstchar;
82 TokenInfo[tok::pipe ] |= aci_custom_firstchar;
83 TokenInfo[tok::percent ] |= aci_custom_firstchar;
84 TokenInfo[tok::colon ] |= aci_custom_firstchar;
85 TokenInfo[tok::hash ] |= aci_custom_firstchar;
86 TokenInfo[tok::arrow ] |= aci_custom_firstchar;
90 TokenInfo[tok::string_literal ] |= aci_custom;
91 TokenInfo[tok::wide_string_literal ] |= aci_custom;
92 TokenInfo[tok::utf8_string_literal ] |= aci_custom;
93 TokenInfo[tok::utf16_string_literal] |= aci_custom;
94 TokenInfo[tok::utf32_string_literal] |= aci_custom;
95 TokenInfo[tok::char_constant ] |= aci_custom;
96 TokenInfo[tok::wide_char_constant ] |= aci_custom;
97 TokenInfo[tok::utf16_char_constant ] |= aci_custom;
98 TokenInfo[tok::utf32_char_constant ] |= aci_custom;
103 TokenInfo[tok::utf8_char_constant] |= aci_custom;
107 TokenInfo[tok::lessequal ] |= aci_custom_firstchar;
110 TokenInfo[tok::amp ] |= aci_avoid_equal;
111 TokenInfo[tok::plus ] |= aci_avoid_equal;
112 TokenInfo[tok::minus ] |= aci_avoid_equal;
113 TokenInfo[tok::slash ] |= aci_avoid_equal;
114 TokenInfo[tok::less ] |= aci_avoid_equal;
115 TokenInfo[tok::greater ] |= aci_avoid_equal;
116 TokenInfo[tok::pipe ] |= aci_avoid_equal;
117 TokenInfo[tok::percent ] |= aci_avoid_equal;
118 TokenInfo[tok::star ] |= aci_avoid_equal;
119 TokenInfo[tok::exclaim ] |= aci_avoid_equal;
120 TokenInfo[tok::lessless ] |= aci_avoid_equal;
121 TokenInfo[tok::greatergreater] |= aci_avoid_equal;
122 TokenInfo[tok::caret ] |= aci_avoid_equal;
123 TokenInfo[tok::equal ] |= aci_avoid_equal;
131 return II->getNameStart()[0];
141 const char *TokPtr = Buffer;
161 const Token &PrevTok,
162 const Token &Tok)
const {
174 PrevKind = tok::identifier;
177 unsigned ConcatInfo = TokenInfo[PrevKind];
180 if (ConcatInfo == 0)
return false;
182 if (ConcatInfo & aci_avoid_equal) {
184 if (Tok.
isOneOf(tok::equal, tok::equalequal))
186 ConcatInfo &= ~aci_avoid_equal;
190 assert(Tok.
isOneOf(tok::annot_module_include, tok::annot_module_begin,
191 tok::annot_module_end) &&
192 "unexpected annotation in AvoidConcat");
203 if (ConcatInfo & aci_custom) {
211 llvm_unreachable(
"InitAvoidConcatTokenInfo built wrong");
213 case tok::raw_identifier:
214 llvm_unreachable(
"tok::raw_identifier in non-raw lexing mode!");
216 case tok::string_literal:
217 case tok::wide_string_literal:
218 case tok::utf8_string_literal:
219 case tok::utf16_string_literal:
220 case tok::utf32_string_literal:
221 case tok::char_constant:
222 case tok::wide_char_constant:
223 case tok::utf8_char_constant:
224 case tok::utf16_char_constant:
225 case tok::utf32_char_constant:
239 case tok::identifier:
241 if (Tok.
is(tok::numeric_constant))
245 Tok.
isOneOf(tok::wide_string_literal, tok::utf8_string_literal,
246 tok::utf16_string_literal, tok::utf32_string_literal,
247 tok::wide_char_constant, tok::utf8_char_constant,
248 tok::utf16_char_constant, tok::utf32_char_constant))
252 if (Tok.
isNot(tok::char_constant) && Tok.
isNot(tok::string_literal))
257 return IsIdentifierStringPrefix(PrevTok);
259 case tok::numeric_constant:
261 FirstChar ==
'+' || FirstChar ==
'-';
263 return (FirstChar ==
'.' && PrevPrevTok.
is(tok::period)) ||
267 return FirstChar ==
'&';
269 return FirstChar ==
'+';
271 return FirstChar ==
'-' || FirstChar ==
'>';
273 return FirstChar ==
'*' || FirstChar ==
'/';
275 return FirstChar ==
'<' || FirstChar ==
':' || FirstChar ==
'%';
277 return FirstChar ==
'>';
279 return FirstChar ==
'|';
281 return FirstChar ==
'>' || FirstChar ==
':';
283 return FirstChar ==
'>' ||
286 return FirstChar ==
'#' || FirstChar ==
'@' || FirstChar ==
'%';
288 return PP.
getLangOpts().CPlusPlus && FirstChar ==
'*';
290 return PP.
getLangOpts().CPlusPlus2a && FirstChar ==
'>';
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
TokenConcatenation(const Preprocessor &PP)
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
tok::TokenKind getKind() const
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
One of these records is kept for each identifier that is lexed.
Token - This structure provides full information about a lexed token.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
const LangOptions & getLangOpts() const
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool hasUDSuffix() const
Return true if this token is a string or character literal which has a ud-suffix. ...
SourceLocation getSpellingLoc(SourceLocation Loc) const
Given a SourceLocation object, return the spelling location referenced by the ID. ...
__DEVICE__ void * memset(void *__a, int __b, size_t __c)
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::Preprocessor interface.
float __ovld __cnfn length(float p)
Return the length of vector p, i.e., sqrt(p.x2 + p.y 2 + ...)
SourceManager & getSourceManager() const
Encodes a location in the source.
IdentifierInfo * getIdentifierInfo() const
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
bool isNot(tok::TokenKind K) const
Dataflow Directional Tag Classes.
static char GetFirstChar(const Preprocessor &PP, const Token &Tok)
GetFirstChar - Get the first character of the token.
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
unsigned getLength() const
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
LLVM_READONLY bool isDigit(unsigned char c)
Return true if this character is an ASCII digit: [0-9].
bool AvoidConcat(const Token &PrevPrevTok, const Token &PrevTok, const Token &Tok) const
AvoidConcat - If printing PrevTok immediately followed by Tok would cause the two individual tokens t...
static bool IsStringPrefix(StringRef Str, bool CPlusPlus11)
IsStringPrefix - Return true if Str is a string prefix.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
LLVM_READONLY bool isPreprocessingNumberBody(unsigned char c)
Return true if this is the body character of a C preprocessing number, which is [a-zA-Z0-9_.
This class handles loading and caching of source files into memory.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.