clang 20.0.0git
TokenRewriter.cpp
Go to the documentation of this file.
1//===- TokenRewriter.cpp - Token-based code rewriting interface -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the TokenRewriter class, which is used for code
10// transformations.
11//
12//===----------------------------------------------------------------------===//
13
16#include "clang/Lex/Lexer.h"
18#include "clang/Lex/Token.h"
19#include <cassert>
20#include <cstring>
21#include <map>
22#include <utility>
23
24using namespace clang;
25
27 const LangOptions &LangOpts) {
28 ScratchBuf.reset(new ScratchBuffer(SM));
29
30 // Create a lexer to lex all the tokens of the main file in raw mode.
31 llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(FID);
32 Lexer RawLex(FID, FromFile, SM, LangOpts);
33
34 // Return all comments and whitespace as tokens.
35 RawLex.SetKeepWhitespaceMode(true);
36
37 // Lex the file, populating our datastructures.
38 Token RawTok;
39 RawLex.LexFromRawLexer(RawTok);
40 while (RawTok.isNot(tok::eof)) {
41#if 0
42 if (Tok.is(tok::raw_identifier)) {
43 // Look up the identifier info for the token. This should use
44 // IdentifierTable directly instead of PP.
45 PP.LookUpIdentifierInfo(Tok);
46 }
47#endif
48
49 AddToken(RawTok, TokenList.end());
50 RawLex.LexFromRawLexer(RawTok);
51 }
52}
53
55
56/// RemapIterator - Convert from token_iterator (a const iterator) to
57/// TokenRefTy (a non-const iterator).
58TokenRewriter::TokenRefTy TokenRewriter::RemapIterator(token_iterator I) {
59 if (I == token_end()) return TokenList.end();
60
61 // FIXME: This is horrible, we should use our own list or something to avoid
62 // this.
63 std::map<SourceLocation, TokenRefTy>::iterator MapIt =
64 TokenAtLoc.find(I->getLocation());
65 assert(MapIt != TokenAtLoc.end() && "iterator not in rewriter?");
66 return MapIt->second;
67}
68
69/// AddToken - Add the specified token into the Rewriter before the other
70/// position.
71TokenRewriter::TokenRefTy
72TokenRewriter::AddToken(const Token &T, TokenRefTy Where) {
73 Where = TokenList.insert(Where, T);
74
75 bool InsertSuccess = TokenAtLoc.insert(std::make_pair(T.getLocation(),
76 Where)).second;
77 assert(InsertSuccess && "Token location already in rewriter!");
78 (void)InsertSuccess;
79 return Where;
80}
81
84 unsigned Len = strlen(Val);
85
86 // Plop the string into the scratch buffer, then create a token for this
87 // string.
88 Token Tok;
89 Tok.startToken();
90 const char *Spelling;
91 Tok.setLocation(ScratchBuf->getToken(Val, Len, Spelling));
92 Tok.setLength(Len);
93
94 // TODO: Form a whole lexer around this and relex the token! For now, just
95 // set kind to tok::unknown.
96 Tok.setKind(tok::unknown);
97
98 return AddToken(Tok, RemapIterator(I));
99}
#define SM(sm)
Definition: Cuda.cpp:83
Defines the SourceManager interface.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:461
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.
Definition: Lexer.h:78
void SetKeepWhitespaceMode(bool Val)
SetKeepWhitespaceMode - This method lets clients enable or disable whitespace retention mode.
Definition: Lexer.h:254
bool LexFromRawLexer(Token &Result)
LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...
Definition: Lexer.h:236
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens.
Definition: ScratchBuffer.h:24
This class handles loading and caching of source files into memory.
TokenRewriter(FileID FID, SourceManager &SM, const LangOptions &LO)
TokenRewriter - This creates a TokenRewriter for the file with the specified FileID.
token_iterator AddTokenBefore(token_iterator I, const char *Val)
token_iterator token_end() const
Definition: TokenRewriter.h:59
std::list< Token >::const_iterator token_iterator
Definition: TokenRewriter.h:56
Token - This structure provides full information about a lexed token.
Definition: Token.h:36
void setLength(unsigned Len)
Definition: Token.h:141
void setKind(tok::TokenKind K)
Definition: Token.h:95
void setLocation(SourceLocation L)
Definition: Token.h:140
bool isNot(tok::TokenKind K) const
Definition: Token.h:100
void startToken()
Reset all flags to cleared.
Definition: Token.h:177
The JSON file list parser is used to communicate input to InstallAPI.
const FunctionProtoType * T