clang  6.0.0svn
TokenRewriter.cpp
Go to the documentation of this file.
1 //===--- TokenRewriter.cpp - Token-based code rewriting interface ---------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the TokenRewriter class, which is used for code
11 // transformations.
12 //
13 //===----------------------------------------------------------------------===//
14 
17 #include "clang/Lex/Lexer.h"
19 using namespace clang;
20 
21 TokenRewriter::TokenRewriter(FileID FID, SourceManager &SM,
22  const LangOptions &LangOpts) {
23  ScratchBuf.reset(new ScratchBuffer(SM));
24 
25  // Create a lexer to lex all the tokens of the main file in raw mode.
26  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
27  Lexer RawLex(FID, FromFile, SM, LangOpts);
28 
29  // Return all comments and whitespace as tokens.
30  RawLex.SetKeepWhitespaceMode(true);
31 
32  // Lex the file, populating our datastructures.
33  Token RawTok;
34  RawLex.LexFromRawLexer(RawTok);
35  while (RawTok.isNot(tok::eof)) {
36 #if 0
37  if (Tok.is(tok::raw_identifier)) {
38  // Look up the identifier info for the token. This should use
39  // IdentifierTable directly instead of PP.
40  PP.LookUpIdentifierInfo(Tok);
41  }
42 #endif
43 
44  AddToken(RawTok, TokenList.end());
45  RawLex.LexFromRawLexer(RawTok);
46  }
47 }
48 
50 }
51 
52 
53 /// RemapIterator - Convert from token_iterator (a const iterator) to
54 /// TokenRefTy (a non-const iterator).
55 TokenRewriter::TokenRefTy TokenRewriter::RemapIterator(token_iterator I) {
56  if (I == token_end()) return TokenList.end();
57 
58  // FIXME: This is horrible, we should use our own list or something to avoid
59  // this.
60  std::map<SourceLocation, TokenRefTy>::iterator MapIt =
61  TokenAtLoc.find(I->getLocation());
62  assert(MapIt != TokenAtLoc.end() && "iterator not in rewriter?");
63  return MapIt->second;
64 }
65 
66 
67 /// AddToken - Add the specified token into the Rewriter before the other
68 /// position.
69 TokenRewriter::TokenRefTy
70 TokenRewriter::AddToken(const Token &T, TokenRefTy Where) {
71  Where = TokenList.insert(Where, T);
72 
73  bool InsertSuccess = TokenAtLoc.insert(std::make_pair(T.getLocation(),
74  Where)).second;
75  assert(InsertSuccess && "Token location already in rewriter!");
76  (void)InsertSuccess;
77  return Where;
78 }
79 
80 
83  unsigned Len = strlen(Val);
84 
85  // Plop the string into the scratch buffer, then create a token for this
86  // string.
87  Token Tok;
88  Tok.startToken();
89  const char *Spelling;
90  Tok.setLocation(ScratchBuf->getToken(Val, Len, Spelling));
91  Tok.setLength(Len);
92 
93  // TODO: Form a whole lexer around this and relex the token! For now, just
94  // set kind to tok::unknown.
95  Tok.setKind(tok::unknown);
96 
97  return AddToken(Tok, RemapIterator(I));
98 }
99 
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:77
bool LexFromRawLexer(Token &Result)
LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...
Definition: Lexer.h:195
Defines the SourceManager interface.
token_iterator token_end() const
Definition: TokenRewriter.h:56
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
void setKind(tok::TokenKind K)
Definition: Token.h:91
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:48
const FormatToken & Tok
std::list< Token >::const_iterator token_iterator
Definition: TokenRewriter.h:54
const FunctionProtoType * T
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
const SourceManager & SM
Definition: Format.cpp:1337
token_iterator AddTokenBefore(token_iterator I, const char *Val)
llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
void setLength(unsigned Len)
Definition: Token.h:133
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:297
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:96
Dataflow Directional Tag Classes.
void setLocation(SourceLocation L)
Definition: Token.h:132
void SetKeepWhitespaceMode(bool Val)
SetKeepWhitespaceMode - This method lets clients enable or disable whitespace retention mode...
Definition: Lexer.h:213
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens...
Definition: ScratchBuffer.h:25
This class handles loading and caching of source files into memory.
void startToken()
Reset all flags to cleared.
Definition: Token.h:169