clang  10.0.0svn
TokenRewriter.cpp
Go to the documentation of this file.
1 //===- TokenRewriter.cpp - Token-based code rewriting interface -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the TokenRewriter class, which is used for code
10 // transformations.
11 //
12 //===----------------------------------------------------------------------===//
13 
16 #include "clang/Lex/Lexer.h"
18 #include "clang/Lex/Token.h"
19 #include <cassert>
20 #include <cstring>
21 #include <map>
22 #include <utility>
23 
24 using namespace clang;
25 
27  const LangOptions &LangOpts) {
28  ScratchBuf.reset(new ScratchBuffer(SM));
29 
30  // Create a lexer to lex all the tokens of the main file in raw mode.
31  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
32  Lexer RawLex(FID, FromFile, SM, LangOpts);
33 
34  // Return all comments and whitespace as tokens.
35  RawLex.SetKeepWhitespaceMode(true);
36 
37  // Lex the file, populating our datastructures.
38  Token RawTok;
39  RawLex.LexFromRawLexer(RawTok);
40  while (RawTok.isNot(tok::eof)) {
41 #if 0
42  if (Tok.is(tok::raw_identifier)) {
43  // Look up the identifier info for the token. This should use
44  // IdentifierTable directly instead of PP.
45  PP.LookUpIdentifierInfo(Tok);
46  }
47 #endif
48 
49  AddToken(RawTok, TokenList.end());
50  RawLex.LexFromRawLexer(RawTok);
51  }
52 }
53 
55 
56 /// RemapIterator - Convert from token_iterator (a const iterator) to
57 /// TokenRefTy (a non-const iterator).
58 TokenRewriter::TokenRefTy TokenRewriter::RemapIterator(token_iterator I) {
59  if (I == token_end()) return TokenList.end();
60 
61  // FIXME: This is horrible, we should use our own list or something to avoid
62  // this.
63  std::map<SourceLocation, TokenRefTy>::iterator MapIt =
64  TokenAtLoc.find(I->getLocation());
65  assert(MapIt != TokenAtLoc.end() && "iterator not in rewriter?");
66  return MapIt->second;
67 }
68 
69 /// AddToken - Add the specified token into the Rewriter before the other
70 /// position.
71 TokenRewriter::TokenRefTy
72 TokenRewriter::AddToken(const Token &T, TokenRefTy Where) {
73  Where = TokenList.insert(Where, T);
74 
75  bool InsertSuccess = TokenAtLoc.insert(std::make_pair(T.getLocation(),
76  Where)).second;
77  assert(InsertSuccess && "Token location already in rewriter!");
78  (void)InsertSuccess;
79  return Where;
80 }
81 
84  unsigned Len = strlen(Val);
85 
86  // Plop the string into the scratch buffer, then create a token for this
87  // string.
88  Token Tok;
89  Tok.startToken();
90  const char *Spelling;
91  Tok.setLocation(ScratchBuf->getToken(Val, Len, Spelling));
92  Tok.setLength(Len);
93 
94  // TODO: Form a whole lexer around this and relex the token! For now, just
95  // set kind to tok::unknown.
96  Tok.setKind(tok::unknown);
97 
98  return AddToken(Tok, RemapIterator(I));
99 }
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:76
bool LexFromRawLexer(Token &Result)
LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...
Definition: Lexer.h:194
Defines the SourceManager interface.
token_iterator token_end() const
Definition: TokenRewriter.h:59
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
void setKind(tok::TokenKind K)
Definition: Token.h:93
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:49
const FormatToken & Tok
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:126
const SourceManager & SM
Definition: Format.cpp:1667
token_iterator AddTokenBefore(token_iterator I, const char *Val)
std::list< Token >::const_iterator token_iterator
Definition: TokenRewriter.h:56
void setLength(unsigned Len)
Definition: Token.h:135
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:312
TokenRewriter(FileID FID, SourceManager &SM, const LangOptions &LO)
TokenRewriter - This creates a TokenRewriter for the file with the specified FileID.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:98
const llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
Dataflow Directional Tag Classes.
void setLocation(SourceLocation L)
Definition: Token.h:134
void SetKeepWhitespaceMode(bool Val)
SetKeepWhitespaceMode - This method lets clients enable or disable whitespace retention mode...
Definition: Lexer.h:212
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens...
Definition: ScratchBuffer.h:24
This class handles loading and caching of source files into memory.
void startToken()
Reset all flags to cleared.
Definition: Token.h:171