clang  6.0.0svn
PPCaching.cpp
Go to the documentation of this file.
1 //===--- PPCaching.cpp - Handle caching lexed tokens ----------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements pieces of the Preprocessor interface that manage the
11 // caching of lexed tokens.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "clang/Lex/Preprocessor.h"
16 using namespace clang;
17 
18 // EnableBacktrackAtThisPos - From the point that this method is called, and
19 // until CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
20 // keeps track of the lexed tokens so that a subsequent Backtrack() call will
21 // make the Preprocessor re-lex the same tokens.
22 //
23 // Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
24 // be called multiple times and CommitBacktrackedTokens/Backtrack calls will
25 // be combined with the EnableBacktrackAtThisPos calls in reverse order.
27  BacktrackPositions.push_back(CachedLexPos);
28  EnterCachingLexMode();
29 }
30 
31 // Disable the last EnableBacktrackAtThisPos call.
33  assert(!BacktrackPositions.empty()
34  && "EnableBacktrackAtThisPos was not called!");
35  BacktrackPositions.pop_back();
36 }
37 
39  assert(isBacktrackEnabled());
40  auto PrevCachedLexPos = BacktrackPositions.back();
41  return CachedTokensRange{PrevCachedLexPos, CachedLexPos};
42 }
43 
45  assert(TokenRange.Begin <= TokenRange.End);
46  if (CachedLexPos == TokenRange.Begin && TokenRange.Begin != TokenRange.End) {
47  // We have backtracked to the start of the token range as we want to consume
48  // them again. Erase the tokens only after consuming then.
49  assert(!CachedTokenRangeToErase);
50  CachedTokenRangeToErase = TokenRange;
51  return;
52  }
53  // The cached tokens were committed, so they should be erased now.
54  assert(TokenRange.End == CachedLexPos);
55  CachedTokens.erase(CachedTokens.begin() + TokenRange.Begin,
56  CachedTokens.begin() + TokenRange.End);
57  CachedLexPos = TokenRange.Begin;
58  ExitCachingLexMode();
59 }
60 
61 // Make Preprocessor re-lex the tokens that were lexed since
62 // EnableBacktrackAtThisPos() was previously called.
64  assert(!BacktrackPositions.empty()
65  && "EnableBacktrackAtThisPos was not called!");
66  CachedLexPos = BacktrackPositions.back();
67  BacktrackPositions.pop_back();
69 }
70 
71 void Preprocessor::CachingLex(Token &Result) {
72  if (!InCachingLexMode())
73  return;
74 
75  if (CachedLexPos < CachedTokens.size()) {
76  Result = CachedTokens[CachedLexPos++];
77  // Erase the some of the cached tokens after they are consumed when
78  // asked to do so.
79  if (CachedTokenRangeToErase &&
80  CachedTokenRangeToErase->End == CachedLexPos) {
81  EraseCachedTokens(*CachedTokenRangeToErase);
82  CachedTokenRangeToErase = None;
83  }
84  return;
85  }
86 
87  ExitCachingLexMode();
88  Lex(Result);
89 
90  if (isBacktrackEnabled()) {
91  // Cache the lexed token.
92  EnterCachingLexMode();
93  CachedTokens.push_back(Result);
94  ++CachedLexPos;
95  return;
96  }
97 
98  if (CachedLexPos < CachedTokens.size()) {
99  EnterCachingLexMode();
100  } else {
101  // All cached tokens were consumed.
102  CachedTokens.clear();
103  CachedLexPos = 0;
104  }
105 }
106 
107 void Preprocessor::EnterCachingLexMode() {
108  if (InCachingLexMode())
109  return;
110 
111  PushIncludeMacroStack();
112  CurLexerKind = CLK_CachingLexer;
113 }
114 
115 
116 const Token &Preprocessor::PeekAhead(unsigned N) {
117  assert(CachedLexPos + N > CachedTokens.size() && "Confused caching.");
118  ExitCachingLexMode();
119  for (size_t C = CachedLexPos + N - CachedTokens.size(); C > 0; --C) {
120  CachedTokens.push_back(Token());
121  Lex(CachedTokens.back());
122  }
123  EnterCachingLexMode();
124  return CachedTokens.back();
125 }
126 
127 void Preprocessor::AnnotatePreviousCachedTokens(const Token &Tok) {
128  assert(Tok.isAnnotation() && "Expected annotation token");
129  assert(CachedLexPos != 0 && "Expected to have some cached tokens");
130  assert(CachedTokens[CachedLexPos-1].getLastLoc() == Tok.getAnnotationEndLoc()
131  && "The annotation should be until the most recent cached token");
132 
133  // Start from the end of the cached tokens list and look for the token
134  // that is the beginning of the annotation token.
135  for (CachedTokensTy::size_type i = CachedLexPos; i != 0; --i) {
136  CachedTokensTy::iterator AnnotBegin = CachedTokens.begin() + i-1;
137  if (AnnotBegin->getLocation() == Tok.getLocation()) {
138  assert((BacktrackPositions.empty() || BacktrackPositions.back() <= i) &&
139  "The backtrack pos points inside the annotated tokens!");
140  // Replace the cached tokens with the single annotation token.
141  if (i < CachedLexPos)
142  CachedTokens.erase(AnnotBegin + 1, CachedTokens.begin() + CachedLexPos);
143  *AnnotBegin = Tok;
144  CachedLexPos = i;
145  return;
146  }
147  }
148 }
149 
151  // There's currently no cached token...
152  if (!CachedLexPos)
153  return false;
154 
155  const Token LastCachedTok = CachedTokens[CachedLexPos - 1];
156  if (LastCachedTok.getKind() != Tok.getKind())
157  return false;
158 
159  int RelOffset = 0;
160  if ((!getSourceManager().isInSameSLocAddrSpace(
161  Tok.getLocation(), getLastCachedTokenLocation(), &RelOffset)) ||
162  RelOffset)
163  return false;
164 
165  return true;
166 }
167 
169  assert(CachedLexPos != 0 && "Expected to have some cached tokens");
170  CachedTokens.insert(CachedTokens.begin() + CachedLexPos - 1, NewToks.begin(),
171  NewToks.end());
172  CachedTokens.erase(CachedTokens.begin() + CachedLexPos - 1 + NewToks.size());
173  CachedLexPos += NewToks.size() - 1;
174 }
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
Definition: PPCaching.cpp:168
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:118
tok::TokenKind getKind() const
Definition: Token.h:90
SourceLocation getAnnotationEndLoc() const
Definition: Token.h:138
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
CachedTokensTy::size_type Begin
const FormatToken & Tok
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
Definition: PPCaching.cpp:32
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
Definition: PPCaching.cpp:150
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/CurPTHLexer/ CurTokenLexer pointers...
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
Definition: PPCaching.cpp:63
void EnableBacktrackAtThisPos()
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
Definition: PPCaching.cpp:26
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
Defines the clang::Preprocessor interface.
The result type of a method or function.
SourceManager & getSourceManager() const
Definition: Preprocessor.h:819
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
Dataflow Directional Tag Classes.
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
void EraseCachedTokens(CachedTokensRange TokenRange)
Erase the range of cached tokens that were lexed since EnableBacktrackAtThisPos() was previously call...
Definition: PPCaching.cpp:44
CachedTokensRange LastCachedTokenRange()
Returns the range of cached tokens that were lexed since EnableBacktrackAtThisPos() was previously ca...
Definition: PPCaching.cpp:38