clang  9.0.0svn
PPCaching.cpp
Go to the documentation of this file.
1 //===--- PPCaching.cpp - Handle caching lexed tokens ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements pieces of the Preprocessor interface that manage the
10 // caching of lexed tokens.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "clang/Lex/Preprocessor.h"
15 using namespace clang;
16 
17 // EnableBacktrackAtThisPos - From the point that this method is called, and
18 // until CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
19 // keeps track of the lexed tokens so that a subsequent Backtrack() call will
20 // make the Preprocessor re-lex the same tokens.
21 //
22 // Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
23 // be called multiple times and CommitBacktrackedTokens/Backtrack calls will
24 // be combined with the EnableBacktrackAtThisPos calls in reverse order.
26  BacktrackPositions.push_back(CachedLexPos);
27  EnterCachingLexMode();
28 }
29 
30 // Disable the last EnableBacktrackAtThisPos call.
32  assert(!BacktrackPositions.empty()
33  && "EnableBacktrackAtThisPos was not called!");
34  BacktrackPositions.pop_back();
35 }
36 
38  assert(isBacktrackEnabled());
39  auto PrevCachedLexPos = BacktrackPositions.back();
40  return CachedTokensRange{PrevCachedLexPos, CachedLexPos};
41 }
42 
44  assert(TokenRange.Begin <= TokenRange.End);
45  if (CachedLexPos == TokenRange.Begin && TokenRange.Begin != TokenRange.End) {
46  // We have backtracked to the start of the token range as we want to consume
47  // them again. Erase the tokens only after consuming then.
48  assert(!CachedTokenRangeToErase);
49  CachedTokenRangeToErase = TokenRange;
50  return;
51  }
52  // The cached tokens were committed, so they should be erased now.
53  assert(TokenRange.End == CachedLexPos);
54  CachedTokens.erase(CachedTokens.begin() + TokenRange.Begin,
55  CachedTokens.begin() + TokenRange.End);
56  CachedLexPos = TokenRange.Begin;
57  ExitCachingLexMode();
58 }
59 
60 // Make Preprocessor re-lex the tokens that were lexed since
61 // EnableBacktrackAtThisPos() was previously called.
63  assert(!BacktrackPositions.empty()
64  && "EnableBacktrackAtThisPos was not called!");
65  CachedLexPos = BacktrackPositions.back();
66  BacktrackPositions.pop_back();
68 }
69 
70 void Preprocessor::CachingLex(Token &Result) {
71  if (!InCachingLexMode())
72  return;
73 
74  if (CachedLexPos < CachedTokens.size()) {
75  Result = CachedTokens[CachedLexPos++];
76  // Erase the some of the cached tokens after they are consumed when
77  // asked to do so.
78  if (CachedTokenRangeToErase &&
79  CachedTokenRangeToErase->End == CachedLexPos) {
80  EraseCachedTokens(*CachedTokenRangeToErase);
81  CachedTokenRangeToErase = None;
82  }
83  return;
84  }
85 
86  ExitCachingLexMode();
87  Lex(Result);
88 
89  if (isBacktrackEnabled()) {
90  // Cache the lexed token.
91  EnterCachingLexMode();
92  CachedTokens.push_back(Result);
93  ++CachedLexPos;
94  return;
95  }
96 
97  if (CachedLexPos < CachedTokens.size()) {
98  EnterCachingLexMode();
99  } else {
100  // All cached tokens were consumed.
101  CachedTokens.clear();
102  CachedLexPos = 0;
103  }
104 }
105 
106 void Preprocessor::EnterCachingLexMode() {
107  if (InCachingLexMode()) {
108  assert(CurLexerKind == CLK_CachingLexer && "Unexpected lexer kind");
109  return;
110  }
111 
112  PushIncludeMacroStack();
113  CurLexerKind = CLK_CachingLexer;
114 }
115 
116 
117 const Token &Preprocessor::PeekAhead(unsigned N) {
118  assert(CachedLexPos + N > CachedTokens.size() && "Confused caching.");
119  ExitCachingLexMode();
120  for (size_t C = CachedLexPos + N - CachedTokens.size(); C > 0; --C) {
121  CachedTokens.push_back(Token());
122  Lex(CachedTokens.back());
123  }
124  EnterCachingLexMode();
125  return CachedTokens.back();
126 }
127 
128 void Preprocessor::AnnotatePreviousCachedTokens(const Token &Tok) {
129  assert(Tok.isAnnotation() && "Expected annotation token");
130  assert(CachedLexPos != 0 && "Expected to have some cached tokens");
131  assert(CachedTokens[CachedLexPos-1].getLastLoc() == Tok.getAnnotationEndLoc()
132  && "The annotation should be until the most recent cached token");
133 
134  // Start from the end of the cached tokens list and look for the token
135  // that is the beginning of the annotation token.
136  for (CachedTokensTy::size_type i = CachedLexPos; i != 0; --i) {
137  CachedTokensTy::iterator AnnotBegin = CachedTokens.begin() + i-1;
138  if (AnnotBegin->getLocation() == Tok.getLocation()) {
139  assert((BacktrackPositions.empty() || BacktrackPositions.back() <= i) &&
140  "The backtrack pos points inside the annotated tokens!");
141  // Replace the cached tokens with the single annotation token.
142  if (i < CachedLexPos)
143  CachedTokens.erase(AnnotBegin + 1, CachedTokens.begin() + CachedLexPos);
144  *AnnotBegin = Tok;
145  CachedLexPos = i;
146  return;
147  }
148  }
149 }
150 
152  // There's currently no cached token...
153  if (!CachedLexPos)
154  return false;
155 
156  const Token LastCachedTok = CachedTokens[CachedLexPos - 1];
157  if (LastCachedTok.getKind() != Tok.getKind())
158  return false;
159 
160  int RelOffset = 0;
161  if ((!getSourceManager().isInSameSLocAddrSpace(
162  Tok.getLocation(), getLastCachedTokenLocation(), &RelOffset)) ||
163  RelOffset)
164  return false;
165 
166  return true;
167 }
168 
170  assert(CachedLexPos != 0 && "Expected to have some cached tokens");
171  CachedTokens.insert(CachedTokens.begin() + CachedLexPos - 1, NewToks.begin(),
172  NewToks.end());
173  CachedTokens.erase(CachedTokens.begin() + CachedLexPos - 1 + NewToks.size());
174  CachedLexPos += NewToks.size() - 1;
175 }
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
Definition: PPCaching.cpp:169
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:117
tok::TokenKind getKind() const
Definition: Token.h:89
SourceLocation getAnnotationEndLoc() const
Definition: Token.h:137
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
CachedTokensTy::size_type Begin
const FormatToken & Tok
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
Definition: PPCaching.cpp:31
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
Definition: PPCaching.cpp:151
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
Definition: PPCaching.cpp:62
void EnableBacktrackAtThisPos()
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
Definition: PPCaching.cpp:25
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:123
Defines the clang::Preprocessor interface.
The result type of a method or function.
SourceManager & getSourceManager() const
Definition: Preprocessor.h:818
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
Dataflow Directional Tag Classes.
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
void EraseCachedTokens(CachedTokensRange TokenRange)
Erase the range of cached tokens that were lexed since EnableBacktrackAtThisPos() was previously call...
Definition: PPCaching.cpp:43
CachedTokensRange LastCachedTokenRange()
Returns the range of cached tokens that were lexed since EnableBacktrackAtThisPos() was previously ca...
Definition: PPCaching.cpp:37