Go to the documentation of this file.
27 #ifndef LLVM_CLANG_TOOLING_SYNTAX_TOKENS_H
28 #define LLVM_CLANG_TOOLING_SYNTAX_TOKENS_H
36 #include "llvm/ADT/ArrayRef.h"
37 #include "llvm/ADT/DenseMap.h"
38 #include "llvm/ADT/Optional.h"
39 #include "llvm/ADT/StringRef.h"
40 #include "llvm/Support/Compiler.h"
41 #include "llvm/Support/raw_ostream.h"
68 unsigned length()
const {
return End - Begin; }
87 return std::tie(L.File, L.Begin, L.End) == std::tie(R.File, R.Begin, R.End);
100 llvm::raw_ostream &
operator<<(llvm::raw_ostream &OS,
const FileRange &R);
118 unsigned length()
const {
return Length; }
193 return ExpandedTokens;
286 std::vector<Expansion>
327 unsigned BeginSpelled = 0;
328 unsigned EndSpelled = 0;
331 unsigned BeginExpanded = 0;
332 unsigned EndExpanded = 0;
344 std::vector<syntax::Token> SpelledTokens;
346 std::vector<Mapping> Mappings;
348 unsigned BeginExpanded = 0;
349 unsigned EndExpanded = 0;
356 std::pair<const syntax::Token *, const Mapping *>
357 spelledForExpandedToken(
const syntax::Token *Expanded)
const;
361 static const Mapping *
362 mappingStartingBeforeSpelled(
const MarkedFile &F,
366 Expansion makeExpansion(
const MarkedFile &,
const Mapping &)
const;
374 std::vector<syntax::Token> ExpandedTokens;
376 llvm::DenseMap<SourceLocation, unsigned> ExpandedTokIndex;
377 llvm::DenseMap<FileID, MarkedFile> Files;
413 std::vector<syntax::Token>
449 using PPExpansions = llvm::DenseMap<SourceLocation, SourceLocation>;
451 class CollectPPExpansions;
453 std::vector<syntax::Token> Expanded;
455 PPExpansions Expansions;
457 CollectPPExpansions *Collector;
std::vector< const syntax::Token * > macroExpansions(FileID FID) const
Get all tokens that expand a macro in FID.
SourceLocation location() const
Location of the first character of a token.
tok::TokenKind kind() const
llvm::StringRef text(const SourceManager &SM) const
Get the substring covered by the token.
A trivial tuple used to represent a source range.
raw_ostream & operator<<(raw_ostream &OS, NodeKind K)
For debugging purposes.
const syntax::Token * spelledTokenAt(SourceLocation Loc) const
Returns the spelled Token starting at Loc, if there are no such tokens returns nullptr.
Encodes a location in the source.
friend bool operator!=(const FileRange &L, const FileRange &R)
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
TokenCollector(Preprocessor &P)
Adds the hooks to collect the tokens.
llvm::Optional< llvm::ArrayRef< syntax::Token > > spelledForExpanded(llvm::ArrayRef< syntax::Token > Expanded) const
Returns the subrange of spelled tokens corresponding to AST node spanning Expanded.
FileRange range(const SourceManager &SM) const
Gets a range of this token.
Token - This structure provides full information about a lexed token.
FileRange(FileID File, unsigned BeginOffset, unsigned EndOffset)
EXPECTS: File.isValid() && Begin <= End.
const SourceManager & sourceManager() const
This class handles loading and caching of source files into memory.
Token(SourceLocation Location, unsigned Length, tok::TokenKind Kind)
void indexExpandedTokens()
Builds a cache to make future calls to expandedToken(SourceRange) faster.
std::vector< Expansion > expansionsOverlapping(llvm::ArrayRef< syntax::Token > Spelled) const
Returns all expansions (partially) expanded from the specified tokens.
llvm::ArrayRef< syntax::Token > Expanded
CharSourceRange toCharRange(const SourceManager &SM) const
Convert to the clang range.
llvm::ArrayRef< syntax::Token > Spelled
A list of tokens obtained by preprocessing a text buffer and operations to map between the expanded a...
llvm::ArrayRef< syntax::Token > spelledTokens(FileID FID) const
Lexed tokens of a file before preprocessing.
bool touches(unsigned Offset) const
Check Offset is inside the range or equal to its endpoint.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
llvm::Optional< Expansion > expansionStartingAt(const syntax::Token *Spelled) const
If Spelled starts a mapping (e.g.
std::string str() const
For debugging purposes.
friend bool operator==(const FileRange &L, const FileRange &R)
Represents a character-granular source range.
unsigned endOffset() const
End offset (exclusive) in the corresponding file.
std::string dumpForTests(const SourceManager &SM) const
const syntax::Token * spelledIdentifierTouching(SourceLocation Loc, llvm::ArrayRef< syntax::Token > Tokens)
The identifier token that overlaps or touches a spelling location Loc.
Collects tokens for the main file while running the frontend action.
unsigned beginOffset() const
Start is a start offset (inclusive) in the corresponding file.
TokenBuffer(const SourceManager &SourceMgr)
SourceLocation endLocation() const
Location right after the last character of a token.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
A token coming directly from a file or from a macro invocation.
llvm::SmallVector< llvm::ArrayRef< syntax::Token >, 1 > expandedForSpelled(llvm::ArrayRef< syntax::Token > Spelled) const
Find the subranges of expanded tokens, corresponding to Spelled.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
llvm::StringRef text(const SourceManager &SM) const
Gets the substring that this FileRange refers to.
llvm::ArrayRef< syntax::Token > spelledTokensTouching(SourceLocation Loc, const syntax::TokenBuffer &Tokens)
The spelled tokens that overlap or touch a spelling location Loc.
std::string dumpForTests() const
LLVM_NODISCARD TokenBuffer consume() &&
Finalizes token collection.
An expansion produced by the preprocessor, includes macro expansions and preprocessor directives.
A half-open character range inside a particular file, the start offset is included and the end offset...
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
bool contains(unsigned Offset) const
Check if Offset is inside the range.
std::vector< syntax::Token > tokenize(FileID FID, const SourceManager &SM, const LangOptions &LO)
Lex the text buffer, corresponding to FID, in raw mode and record the resulting spelled tokens.
TokenBuffer & operator=(TokenBuffer &&)=default
llvm::ArrayRef< syntax::Token > expandedTokens() const
All tokens produced by the preprocessor after all macro replacements, directives, etc.