20#include "llvm/ADT/ArrayRef.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/Support/Debug.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/FormatVariadic.h"
25#include "llvm/Support/raw_ostream.h"
77 assert(
SM.getSLocEntry(TargetFile).isFile());
88 while (
First.isMacroID() &&
Last.isMacroID()) {
89 auto DecFirst =
SM.getDecomposedLoc(
First);
90 auto DecLast =
SM.getDecomposedLoc(
Last);
91 auto &ExpFirst =
SM.getSLocEntry(DecFirst.first).getExpansion();
92 auto &ExpLast =
SM.getSLocEntry(DecLast.first).getExpansion();
94 if (!ExpFirst.isMacroArgExpansion() || !ExpLast.isMacroArgExpansion())
98 if (ExpFirst.getExpansionLocStart() != ExpLast.getExpansionLocStart())
106 auto ExpFileID =
SM.getFileID(ExpFirst.getExpansionLocStart());
107 if (ExpFileID == TargetFile)
111 First = ExpFirst.getSpellingLoc().getLocWithOffset(DecFirst.second);
112 Last = ExpLast.getSpellingLoc().getLocWithOffset(DecLast.second);
119 auto DecFirst =
SM.getDecomposedExpansionLoc(Candidate.
getBegin());
120 auto DecLast =
SM.getDecomposedExpansionLoc(Candidate.
getEnd());
122 if (Candidate.
isInvalid() || DecFirst.first != TargetFile ||
123 DecLast.first != TargetFile)
127 auto Dec =
SM.getDecomposedLoc(
SM.getExpansionRange(Prev).getBegin());
128 if (
Dec.first != DecFirst.first ||
Dec.second >= DecFirst.second)
131 if (Next.isValid()) {
132 auto Dec =
SM.getDecomposedLoc(
SM.getExpansionRange(Next).getEnd());
133 if (
Dec.first != DecLast.first ||
Dec.second <= DecLast.second)
145 : Location(Location), Length(Length), Kind(Kind) {
156 const char *Start =
SM.getCharacterData(location(), &
Invalid);
158 return llvm::StringRef(Start,
length());
162 assert(location().isFileID() &&
"must be a spelled token");
164 unsigned StartOffset;
165 std::tie(
File, StartOffset) =
SM.getDecomposedLoc(location());
174 assert(F.file() == L.file() &&
"tokens from different files");
175 assert((F == L || F.endOffset() <= L.beginOffset()) &&
176 "wrong order of tokens");
177 return FileRange(F.file(), F.beginOffset(), L.endOffset());
181 return OS << T.
str();
186 assert(
File.isValid());
187 assert(BeginOffset <= EndOffset);
195 std::tie(File,
Begin) =
SM.getDecomposedLoc(BeginLoc);
196 End =
Begin + Length;
204 assert(
SM.getFileID(BeginLoc) ==
SM.getFileID(EndLoc));
205 assert(
SM.getFileOffset(BeginLoc) <=
SM.getFileOffset(EndLoc));
207 std::tie(File,
Begin) =
SM.getDecomposedLoc(BeginLoc);
208 End =
SM.getFileOffset(EndLoc);
213 return OS << llvm::formatv(
"FileRange(file = {0}, offsets = {1}-{2})",
224 assert(End <=
Text.size());
230 if (!ExpandedTokIndex.empty())
232 ExpandedTokIndex.reserve(ExpandedTokens.size());
234 for (
size_t I = 0, E = ExpandedTokens.size(); I != E; ++I) {
237 ExpandedTokIndex[Loc] = I;
244 if (!ExpandedTokIndex.empty()) {
248 const auto B = ExpandedTokIndex.find(R.
getBegin());
249 const auto E = ExpandedTokIndex.find(R.
getEnd());
250 if (B != ExpandedTokIndex.end() && E != ExpandedTokIndex.end()) {
251 const Token *L = ExpandedTokens.data() + B->getSecond();
253 const Token *R = ExpandedTokens.data() + E->getSecond() + 1;
270std::pair<const syntax::Token *, const TokenBuffer::Mapping *>
271TokenBuffer::spelledForExpandedToken(
const syntax::Token *Expanded)
const {
273 assert(ExpandedTokens.data() <= Expanded &&
274 Expanded < ExpandedTokens.data() + ExpandedTokens.size());
276 auto FileIt = Files.find(
278 assert(FileIt != Files.end() &&
"no file for an expanded token");
280 const MarkedFile &
File = FileIt->second;
282 unsigned ExpandedIndex = Expanded - ExpandedTokens.data();
284 auto It = llvm::partition_point(
File.Mappings, [&](
const Mapping &M) {
285 return M.BeginExpanded <= ExpandedIndex;
288 if (It ==
File.Mappings.begin()) {
290 return {&
File.SpelledTokens[ExpandedIndex -
File.BeginExpanded],
296 if (ExpandedIndex < It->EndExpanded)
297 return {&
File.SpelledTokens[It->BeginSpelled], &*It};
302 &
File.SpelledTokens[It->EndSpelled + (ExpandedIndex - It->EndExpanded)],
306const TokenBuffer::Mapping *
307TokenBuffer::mappingStartingBeforeSpelled(
const MarkedFile &F,
309 assert(F.SpelledTokens.data() <= Spelled);
310 unsigned SpelledI = Spelled - F.SpelledTokens.data();
311 assert(SpelledI < F.SpelledTokens.size());
313 auto It = llvm::partition_point(F.Mappings, [SpelledI](
const Mapping &M) {
314 return M.BeginSpelled <= SpelledI;
316 if (It == F.Mappings.begin())
326 const auto &
File = fileForSpelled(Spelled);
328 auto *FrontMapping = mappingStartingBeforeSpelled(
File, &Spelled.front());
329 unsigned SpelledFrontI = &Spelled.front() -
File.SpelledTokens.data();
330 assert(SpelledFrontI <
File.SpelledTokens.size());
331 unsigned ExpandedBegin;
335 ExpandedBegin =
File.BeginExpanded + SpelledFrontI;
336 }
else if (SpelledFrontI < FrontMapping->EndSpelled) {
338 if (SpelledFrontI != FrontMapping->BeginSpelled) {
343 ExpandedBegin = FrontMapping->BeginExpanded;
348 FrontMapping->EndExpanded + (SpelledFrontI - FrontMapping->EndSpelled);
351 auto *BackMapping = mappingStartingBeforeSpelled(
File, &Spelled.back());
352 unsigned SpelledBackI = &Spelled.back() -
File.SpelledTokens.data();
353 unsigned ExpandedEnd;
357 ExpandedEnd =
File.BeginExpanded + SpelledBackI + 1;
358 }
else if (SpelledBackI < BackMapping->EndSpelled) {
360 if (SpelledBackI + 1 != BackMapping->EndSpelled) {
364 ExpandedEnd = BackMapping->EndExpanded;
368 BackMapping->EndExpanded + (SpelledBackI - BackMapping->EndSpelled) + 1;
371 assert(ExpandedBegin < ExpandedTokens.size());
372 assert(ExpandedEnd < ExpandedTokens.size());
374 if (ExpandedBegin == ExpandedEnd)
377 ExpandedTokens.data() + ExpandedEnd)};
381 auto It = Files.find(FID);
382 assert(It != Files.end());
383 return It->second.SpelledTokens;
388 const auto *Tok = llvm::partition_point(
390 [&](
const syntax::Token &Tok) { return Tok.location() < Loc; });
396std::string TokenBuffer::Mapping::str()
const {
398 llvm::formatv(
"spelled tokens: [{0},{1}), expanded tokens: [{2},{3})",
399 BeginSpelled, EndSpelled, BeginExpanded, EndExpanded));
402std::optional<llvm::ArrayRef<syntax::Token>>
406 if (Expanded.empty())
410 auto [FirstSpelled, FirstMapping] = spelledForExpandedToken(
First);
411 auto [LastSpelled, LastMapping] = spelledForExpandedToken(
Last);
415 if (FID != SourceMgr->
getFileID(LastSpelled->location()))
418 const MarkedFile &
File = Files.find(FID)->second;
422 if (FirstMapping && FirstMapping == LastMapping &&
428 : (
First - 1)->location();
431 : (
Last + 1)->location();
433 First->location(),
Last->location(), Prev, Next, FID, *SourceMgr);
434 if (Range.isInvalid())
436 return getTokensCovering(
File.SpelledTokens, Range, *SourceMgr);
441 unsigned FirstExpanded = Expanded.begin() - ExpandedTokens.data();
442 unsigned LastExpanded = Expanded.end() - ExpandedTokens.data();
443 if (FirstMapping && FirstExpanded != FirstMapping->BeginExpanded)
445 if (LastMapping && LastMapping->EndExpanded != LastExpanded)
448 FirstMapping ?
File.SpelledTokens.data() + FirstMapping->BeginSpelled
450 LastMapping ?
File.SpelledTokens.data() + LastMapping->EndSpelled
455 const Mapping &M)
const {
458 F.SpelledTokens.data() + M.EndSpelled);
459 E.Expanded =
llvm::ArrayRef(ExpandedTokens.data() + M.BeginExpanded,
460 ExpandedTokens.data() + M.EndExpanded);
464const TokenBuffer::MarkedFile &
466 assert(!Spelled.empty());
467 assert(Spelled.front().location().isFileID() &&
"not a spelled token");
468 auto FileIt = Files.find(SourceMgr->
getFileID(Spelled.front().location()));
469 assert(FileIt != Files.end() &&
"file not tracked by token buffer");
470 const auto &
File = FileIt->second;
471 assert(
File.SpelledTokens.data() <= Spelled.data() &&
473 (
File.SpelledTokens.data() +
File.SpelledTokens.size()) &&
474 "Tokens not in spelled range");
476 auto T1 = Spelled.back().location();
477 auto T2 =
File.SpelledTokens.back().location();
478 assert(T1 == T2 ||
sourceManager().isBeforeInTranslationUnit(T1, T2));
483std::optional<TokenBuffer::Expansion>
486 const auto &
File = fileForSpelled(*Spelled);
488 unsigned SpelledIndex = Spelled -
File.SpelledTokens.data();
489 auto M = llvm::partition_point(
File.Mappings, [&](
const Mapping &M) {
490 return M.BeginSpelled < SpelledIndex;
492 if (M ==
File.Mappings.end() || M->BeginSpelled != SpelledIndex)
494 return makeExpansion(
File, *M);
501 const auto &
File = fileForSpelled(Spelled);
504 unsigned SpelledBeginIndex = Spelled.begin() -
File.SpelledTokens.data();
505 unsigned SpelledEndIndex = Spelled.end() -
File.SpelledTokens.data();
506 auto M = llvm::partition_point(
File.Mappings, [&](
const Mapping &M) {
507 return M.EndSpelled <= SpelledBeginIndex;
509 std::vector<TokenBuffer::Expansion> Expansions;
510 for (; M !=
File.Mappings.end() && M->BeginSpelled < SpelledEndIndex; ++M)
511 Expansions.push_back(makeExpansion(
File, *M));
520 auto *Right = llvm::partition_point(
522 bool AcceptRight = Right != Tokens.end() && Right->location() <= Loc;
524 Right != Tokens.begin() && (Right - 1)->endLocation() >= Loc;
526 Right + (AcceptRight ? 1 : 0));
533 Loc, Tokens.spelledTokens(Tokens.sourceManager().getFileID(Loc)));
540 if (Tok.kind() == tok::identifier)
550 Loc, Tokens.spelledTokens(Tokens.sourceManager().getFileID(Loc)));
553std::vector<const syntax::Token *>
555 auto FileIt = Files.find(FID);
556 assert(FileIt != Files.end() &&
"file not tracked by token buffer");
557 auto &
File = FileIt->second;
558 std::vector<const syntax::Token *> Expansions;
559 auto &Spelled =
File.SpelledTokens;
560 for (
auto Mapping :
File.Mappings) {
562 if (
Token->
kind() == tok::TokenKind::identifier)
563 Expansions.push_back(
Token);
571 std::vector<syntax::Token> Tokens;
575 if (T.getKind() == tok::raw_identifier && !T.needsCleaning() &&
578 T.setIdentifierInfo(&II);
584 auto SrcBuffer =
SM.getBufferData(FR.
file());
585 Lexer L(
SM.getLocForStartOfFile(FR.
file()), LO, SrcBuffer.data(),
589 SrcBuffer.data() + SrcBuffer.size());
621 const auto &
SM = Collector->PP.getSourceManager();
636 if (!Range.getEnd().isFileID())
640 if (LastExpansionEnd.isValid() &&
641 !
SM.isBeforeInTranslationUnit(LastExpansionEnd, Range.getEnd()))
647 if (!Range.getBegin().isFileID()) {
648 Range.setBegin(
SM.getExpansionLoc(Range.getBegin()));
649 assert(Collector->Expansions.count(Range.getBegin()) &&
650 "Overlapping macros should have same expansion location");
653 Collector->Expansions[Range.getBegin()] = Range.getEnd();
654 LastExpansionEnd = Range.getEnd();
681 DEBUG_WITH_TYPE(
"collect-tokens", llvm::dbgs()
692 auto CB = std::make_unique<CollectPPExpansions>(*
this);
693 this->Collector = CB.get();
701 Builder(std::vector<syntax::Token> Expanded, PPExpansions CollectedExpansions,
703 : Result(
SM), CollectedExpansions(
std::move(CollectedExpansions)),
SM(
SM),
705 Result.ExpandedTokens = std::move(Expanded);
709 assert(!Result.ExpandedTokens.empty());
710 assert(Result.ExpandedTokens.back().kind() == tok::eof);
713 buildSpelledTokens();
719 while (NextExpanded < Result.ExpandedTokens.size() - 1 ) {
725 unsigned OldPosition = NextExpanded;
727 if (NextExpanded == OldPosition)
728 diagnoseAdvanceFailure();
732 for (
const auto &
File : Result.Files)
736 for (
auto &pair : Result.Files) {
737 auto &mappings = pair.second.Mappings;
738 assert(llvm::is_sorted(mappings, [](
const TokenBuffer::Mapping &M1,
739 const TokenBuffer::Mapping &M2) {
740 return M1.BeginSpelled < M2.BeginSpelled &&
741 M1.EndSpelled < M2.EndSpelled &&
742 M1.BeginExpanded < M2.BeginExpanded &&
743 M1.EndExpanded < M2.EndExpanded;
748 return std::move(Result);
756 void discard(std::optional<FileID> Drain = std::nullopt) {
758 Drain ?
SM.getLocForEndOfFile(*Drain)
759 :
SM.getExpansionLoc(
760 Result.ExpandedTokens[NextExpanded].location());
762 const auto &SpelledTokens = Result.Files[
File].SpelledTokens;
763 auto &NextSpelled = this->NextSpelled[
File];
765 TokenBuffer::Mapping Mapping;
766 Mapping.BeginSpelled = NextSpelled;
769 Mapping.BeginExpanded = Mapping.EndExpanded =
770 Drain ? Result.Files[*Drain].EndExpanded : NextExpanded;
773 auto FlushMapping = [&,
this] {
774 Mapping.EndSpelled = NextSpelled;
775 if (Mapping.BeginSpelled != Mapping.EndSpelled)
776 Result.Files[
File].Mappings.push_back(Mapping);
777 Mapping.BeginSpelled = NextSpelled;
780 while (NextSpelled < SpelledTokens.size() &&
781 SpelledTokens[NextSpelled].location() < Target) {
786 CollectedExpansions.lookup(SpelledTokens[NextSpelled].location());
789 while (NextSpelled < SpelledTokens.size() &&
790 SpelledTokens[NextSpelled].location() <= KnownEnd)
805 const syntax::Token &Tok = Result.ExpandedTokens[NextExpanded];
808 const auto &SpelledTokens = Result.Files[
File].SpelledTokens;
809 auto &NextSpelled = this->NextSpelled[
File];
813 while (NextSpelled < SpelledTokens.size() &&
814 NextExpanded < Result.ExpandedTokens.size() &&
815 SpelledTokens[NextSpelled].location() ==
816 Result.ExpandedTokens[NextExpanded].location()) {
823 auto End = CollectedExpansions.lookup(Expansion);
824 assert(End.isValid() &&
"Macro expansion wasn't captured?");
827 TokenBuffer::Mapping Mapping;
828 Mapping.BeginExpanded = NextExpanded;
829 Mapping.BeginSpelled = NextSpelled;
831 while (NextSpelled < SpelledTokens.size() &&
832 SpelledTokens[NextSpelled].location() <= End)
835 while (NextExpanded < Result.ExpandedTokens.size() &&
837 Result.ExpandedTokens[NextExpanded].location()) == Expansion)
840 Mapping.EndExpanded = NextExpanded;
841 Mapping.EndSpelled = NextSpelled;
842 Result.Files[
File].Mappings.push_back(Mapping);
847 void diagnoseAdvanceFailure() {
850 for (
unsigned I = (NextExpanded < 10) ? 0 : NextExpanded - 10;
851 I < NextExpanded + 5 && I < Result.ExpandedTokens.size(); ++I) {
853 (I == NextExpanded) ?
"!! " : (I < NextExpanded) ?
"ok " :
" ";
854 llvm::errs() << L << Result.ExpandedTokens[I].dumpForTests(
SM) <<
"\n";
857 llvm_unreachable(
"Couldn't map expanded token to spelled tokens!");
862 void buildSpelledTokens() {
863 for (
unsigned I = 0; I < Result.ExpandedTokens.size(); ++I) {
864 const auto &Tok = Result.ExpandedTokens[I];
865 auto FID =
SM.getFileID(
SM.getExpansionLoc(Tok.
location()));
866 auto It = Result.Files.try_emplace(FID);
867 TokenBuffer::MarkedFile &
File = It.first->second;
870 File.EndExpanded = Tok.
kind() == tok::eof ? I : I + 1;
875 File.BeginExpanded = I;
881 unsigned NextExpanded = 0;
882 llvm::DenseMap<FileID, unsigned> NextSpelled;
883 PPExpansions CollectedExpansions;
889 PP.setTokenWatcher(
nullptr);
890 Collector->disable();
891 return Builder(std::move(Expanded), std::move(Expansions),
892 PP.getSourceManager(), PP.getLangOpts())
897 return std::string(llvm::formatv(
"Token({0}, length = {1})",
902 return std::string(llvm::formatv(
"Token(`{0}`, {1}, length = {2})", text(
SM),
907 auto PrintToken = [
this](
const syntax::Token &T) -> std::string {
908 if (T.
kind() == tok::eof)
910 return std::string(T.
text(*SourceMgr));
913 auto DumpTokens = [
this, &PrintToken](llvm::raw_ostream &OS,
915 if (Tokens.empty()) {
919 OS << Tokens[0].text(*SourceMgr);
920 for (
unsigned I = 1; I < Tokens.size(); ++I) {
921 if (Tokens[I].kind() == tok::eof)
923 OS <<
" " << PrintToken(Tokens[I]);
928 llvm::raw_string_ostream OS(Dump);
930 OS <<
"expanded tokens:\n"
936 std::vector<FileID> Keys;
937 for (
const auto &F : Files)
938 Keys.push_back(F.first);
942 const MarkedFile &
File = Files.find(ID)->second;
946 std::string Path = llvm::sys::path::convert_to_slash(Entry->getName());
947 OS << llvm::formatv(
"file '{0}'\n", Path) <<
" spelled tokens:\n"
949 DumpTokens(OS,
File.SpelledTokens);
952 if (
File.Mappings.empty()) {
953 OS <<
" no mappings.\n";
956 OS <<
" mappings:\n";
957 for (
auto &M :
File.Mappings) {
959 " ['{0}'_{1}, '{2}'_{3}) => ['{4}'_{5}, '{6}'_{7})\n",
960 PrintToken(
File.SpelledTokens[M.BeginSpelled]), M.BeginSpelled,
961 M.EndSpelled ==
File.SpelledTokens.size()
963 : PrintToken(
File.SpelledTokens[M.EndSpelled]),
964 M.EndSpelled, PrintToken(ExpandedTokens[M.BeginExpanded]),
965 M.BeginExpanded, PrintToken(ExpandedTokens[M.EndExpanded]),
Defines the Diagnostic-related interfaces.
static Decl::Kind getKind(const Decl *D)
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
Defines the PPCallbacks interface.
static ParseState advance(ParseState S, size_t N)
Defines the clang::Preprocessor interface.
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
Defines the clang::TokenKind enum and support functions.
Builds mappings and spelled tokens in the TokenBuffer based on the expanded token stream.
Builder(std::vector< syntax::Token > Expanded, PPExpansions CollectedExpansions, const SourceManager &SM, const LangOptions &LangOpts)
Records information reqired to construct mappings for the token buffer that we are collecting.
CollectPPExpansions(TokenCollector &C)
void disable()
Disabled instance will stop reporting anything to TokenCollector.
void MacroExpands(const clang::Token &MacroNameTok, const MacroDefinition &MD, SourceRange Range, const MacroArgs *Args) override
Called by Preprocessor::HandleMacroExpandedIdentifier when a macro invocation is found.
Represents a character-granular source range.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
unsigned getHashValue() const
One of these records is kept for each identifier that is lexed.
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
Implements an efficient mapping from strings to IdentifierInfo nodes.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.
bool LexFromRawLexer(Token &Result)
LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...
unsigned getCurrentBufferOffset()
Returns the current lexing offset.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
A description of the current definition of a macro.
This interface provides a way to observe the actions of the preprocessor as it does its thing.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
SourceManager & getSourceManager() const
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream.
Encodes a location in the source.
bool isValid() const
Return true if this is a valid SourceLocation object.
This class handles loading and caching of source files into memory.
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
OptionalFileEntryRef getFileEntryRefForID(FileID FID) const
Returns the FileEntryRef for the provided FileID.
bool isMacroArgExpansion(SourceLocation Loc, SourceLocation *StartLoc=nullptr) const
Tests whether the given source location represents a macro argument's expansion into the function-lik...
SourceLocation getExpansionLoc(SourceLocation Loc) const
Given a SourceLocation object Loc, return the expansion location referenced by the ID.
A trivial tuple used to represent a source range.
SourceLocation getEnd() const
SourceLocation getBegin() const
Token - This structure provides full information about a lexed token.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
A list of tokens obtained by preprocessing a text buffer and operations to map between the expanded a...
const SourceManager & sourceManager() const
void indexExpandedTokens()
Builds a cache to make future calls to expandedToken(SourceRange) faster.
llvm::SmallVector< llvm::ArrayRef< syntax::Token >, 1 > expandedForSpelled(llvm::ArrayRef< syntax::Token > Spelled) const
Find the subranges of expanded tokens, corresponding to Spelled.
llvm::ArrayRef< syntax::Token > expandedTokens() const
All tokens produced by the preprocessor after all macro replacements, directives, etc.
std::string dumpForTests() const
std::optional< llvm::ArrayRef< syntax::Token > > spelledForExpanded(llvm::ArrayRef< syntax::Token > Expanded) const
Returns the subrange of spelled tokens corresponding to AST node spanning Expanded.
const syntax::Token * spelledTokenAt(SourceLocation Loc) const
Returns the spelled Token starting at Loc, if there are no such tokens returns nullptr.
std::vector< Expansion > expansionsOverlapping(llvm::ArrayRef< syntax::Token > Spelled) const
Returns all expansions (partially) expanded from the specified tokens.
std::optional< Expansion > expansionStartingAt(const syntax::Token *Spelled) const
If Spelled starts a mapping (e.g.
llvm::ArrayRef< syntax::Token > spelledTokens(FileID FID) const
Lexed tokens of a file before preprocessing.
std::vector< const syntax::Token * > macroExpansions(FileID FID) const
Get all tokens that expand a macro in FID.
Collects tokens for the main file while running the frontend action.
TokenBuffer consume() &&
Finalizes token collection.
TokenCollector(Preprocessor &P)
Adds the hooks to collect the tokens.
A token coming directly from a file or from a macro invocation.
std::string str() const
For debugging purposes.
llvm::StringRef text(const SourceManager &SM) const
Get the substring covered by the token.
tok::TokenKind kind() const
FileRange range(const SourceManager &SM) const
Gets a range of this token.
Token(SourceLocation Location, unsigned Length, tok::TokenKind Kind)
std::string dumpForTests(const SourceManager &SM) const
SourceLocation location() const
Location of the first character of a token.
bool Dec(InterpState &S, CodePtr OpPC)
1) Pops a pointer from the stack 2) Load the value from the pointer 3) Writes the value decreased by ...
const syntax::Token * spelledIdentifierTouching(SourceLocation Loc, llvm::ArrayRef< syntax::Token > Tokens)
The identifier token that overlaps or touches a spelling location Loc.
std::vector< syntax::Token > tokenize(FileID FID, const SourceManager &SM, const LangOptions &LO)
Lex the text buffer, corresponding to FID, in raw mode and record the resulting spelled tokens.
raw_ostream & operator<<(raw_ostream &OS, NodeKind K)
For debugging purposes.
llvm::ArrayRef< syntax::Token > spelledTokensTouching(SourceLocation Loc, const syntax::TokenBuffer &Tokens)
The spelled tokens that overlap or touch a spelling location Loc.
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
float __ovld __cnfn length(float)
Return the length of vector p, i.e., sqrt(p.x2 + p.y 2 + ...)
A half-open character range inside a particular file, the start offset is included and the end offset...
CharSourceRange toCharRange(const SourceManager &SM) const
Convert to the clang range.
FileRange(FileID File, unsigned BeginOffset, unsigned EndOffset)
EXPECTS: File.isValid() && Begin <= End.
unsigned beginOffset() const
Start is a start offset (inclusive) in the corresponding file.
llvm::StringRef text(const SourceManager &SM) const
Gets the substring that this FileRange refers to.
unsigned endOffset() const
End offset (exclusive) in the corresponding file.
An expansion produced by the preprocessor, includes macro expansions and preprocessor directives.
llvm::ArrayRef< syntax::Token > Spelled