13 #ifndef LLVM_CLANG_AST_COMMENTLEXER_H
14 #define LLVM_CLANG_AST_COMMENTLEXER_H
18 #include "llvm/ADT/SmallString.h"
19 #include "llvm/ADT/StringRef.h"
20 #include "llvm/Support/Allocator.h"
21 #include "llvm/Support/raw_ostream.h"
27 class TextTokenRetokenizer;
84 if (Length == 0 || Length == 1)
95 unsigned getLength() const LLVM_READONLY {
return Length; }
100 return StringRef(TextPtr, IntVal);
105 TextPtr =
Text.data();
106 IntVal =
Text.size();
111 return StringRef(TextPtr, IntVal);
116 TextPtr = Name.data();
117 IntVal = Name.size();
142 return StringRef(TextPtr, IntVal);
147 TextPtr =
Text.data();
148 IntVal =
Text.size();
163 return StringRef(TextPtr, IntVal);
168 TextPtr =
Text.data();
169 IntVal =
Text.size();
174 return StringRef(TextPtr, IntVal);
179 TextPtr = Name.data();
180 IntVal = Name.size();
185 return StringRef(TextPtr, IntVal);
190 TextPtr = Name.data();
191 IntVal = Name.size();
196 return StringRef(TextPtr, IntVal);
201 TextPtr = Str.data();
207 return StringRef(TextPtr, IntVal);
212 TextPtr = Name.data();
213 IntVal = Name.size();
223 void operator=(
const Lexer &) =
delete;
227 llvm::BumpPtrAllocator &Allocator;
233 const char *
const BufferStart;
234 const char *
const BufferEnd;
236 const char *BufferPtr;
240 const char *CommentEnd;
249 enum LexerCommentState : uint8_t {
251 LCS_InsideBCPLComment,
257 LexerCommentState CommentState;
259 enum LexerState : uint8_t {
265 LS_VerbatimBlockFirstLine,
269 LS_VerbatimBlockBody,
291 StringRef resolveHTMLNamedCharacterReference(StringRef Name)
const;
294 StringRef resolveHTMLDecimalCharacterReference(StringRef Name)
const;
297 StringRef resolveHTMLHexCharacterReference(StringRef Name)
const;
299 void formTokenWithChars(
Token &Result,
const char *TokEnd,
302 void formTextToken(
Token &Result,
const char *TokEnd) {
303 StringRef
Text(BufferPtr, TokEnd - BufferPtr);
304 formTokenWithChars(Result, TokEnd,
tok::text);
305 Result.setText(
Text);
309 assert(Loc >= BufferStart && Loc <= BufferEnd &&
310 "Location out of range for this buffer!");
312 const unsigned CharNo = Loc - BufferStart;
317 return Diags.
Report(Loc, DiagID);
321 void skipLineStartingDecorations();
324 const char *skipTextToken();
327 void lexCommentText(
Token &T);
329 void setupAndLexVerbatimBlock(
Token &T,
const char *TextBegin,
char Marker,
332 void lexVerbatimBlockFirstLine(
Token &T);
334 void lexVerbatimBlockBody(
Token &T);
336 void setupAndLexVerbatimLine(
Token &T,
const char *TextBegin,
339 void lexVerbatimLineText(
Token &T);
341 void lexHTMLCharacterReference(
Token &T);
343 void setupAndLexHTMLStartTag(
Token &T);
345 void lexHTMLStartTag(
Token &T);
347 void setupAndLexHTMLEndTag(
Token &T);
349 void lexHTMLEndTag(
Token &T);
354 const char *BufferStart,
const char *BufferEnd,
355 bool ParseCommands =
true);