clang  14.0.0git
PPCaching.cpp
Go to the documentation of this file.
1 //===--- PPCaching.cpp - Handle caching lexed tokens ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements pieces of the Preprocessor interface that manage the
10 // caching of lexed tokens.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "clang/Lex/Preprocessor.h"
15 using namespace clang;
16 
17 // EnableBacktrackAtThisPos - From the point that this method is called, and
18 // until CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
19 // keeps track of the lexed tokens so that a subsequent Backtrack() call will
20 // make the Preprocessor re-lex the same tokens.
21 //
22 // Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
23 // be called multiple times and CommitBacktrackedTokens/Backtrack calls will
24 // be combined with the EnableBacktrackAtThisPos calls in reverse order.
26  assert(LexLevel == 0 && "cannot use lookahead while lexing");
27  BacktrackPositions.push_back(CachedLexPos);
28  EnterCachingLexMode();
29 }
30 
31 // Disable the last EnableBacktrackAtThisPos call.
33  assert(!BacktrackPositions.empty()
34  && "EnableBacktrackAtThisPos was not called!");
35  BacktrackPositions.pop_back();
36 }
37 
38 // Make Preprocessor re-lex the tokens that were lexed since
39 // EnableBacktrackAtThisPos() was previously called.
41  assert(!BacktrackPositions.empty()
42  && "EnableBacktrackAtThisPos was not called!");
43  CachedLexPos = BacktrackPositions.back();
44  BacktrackPositions.pop_back();
46 }
47 
48 void Preprocessor::CachingLex(Token &Result) {
49  if (!InCachingLexMode())
50  return;
51 
52  // The assert in EnterCachingLexMode should prevent this from happening.
53  assert(LexLevel == 1 &&
54  "should not use token caching within the preprocessor");
55 
56  if (CachedLexPos < CachedTokens.size()) {
57  Result = CachedTokens[CachedLexPos++];
58  Result.setFlag(Token::IsReinjected);
59  return;
60  }
61 
62  ExitCachingLexMode();
63  Lex(Result);
64 
65  if (isBacktrackEnabled()) {
66  // Cache the lexed token.
67  EnterCachingLexModeUnchecked();
68  CachedTokens.push_back(Result);
69  ++CachedLexPos;
70  return;
71  }
72 
73  if (CachedLexPos < CachedTokens.size()) {
74  EnterCachingLexModeUnchecked();
75  } else {
76  // All cached tokens were consumed.
77  CachedTokens.clear();
78  CachedLexPos = 0;
79  }
80 }
81 
82 void Preprocessor::EnterCachingLexMode() {
83  // The caching layer sits on top of all the other lexers, so it's incorrect
84  // to cache tokens while inside a nested lex action. The cached tokens would
85  // be retained after returning to the enclosing lex action and, at best,
86  // would appear at the wrong position in the token stream.
87  assert(LexLevel == 0 &&
88  "entered caching lex mode while lexing something else");
89 
90  if (InCachingLexMode()) {
91  assert(CurLexerKind == CLK_CachingLexer && "Unexpected lexer kind");
92  return;
93  }
94 
95  EnterCachingLexModeUnchecked();
96 }
97 
98 void Preprocessor::EnterCachingLexModeUnchecked() {
99  assert(CurLexerKind != CLK_CachingLexer && "already in caching lex mode");
100  PushIncludeMacroStack();
101  CurLexerKind = CLK_CachingLexer;
102 }
103 
104 
105 const Token &Preprocessor::PeekAhead(unsigned N) {
106  assert(CachedLexPos + N > CachedTokens.size() && "Confused caching.");
107  ExitCachingLexMode();
108  for (size_t C = CachedLexPos + N - CachedTokens.size(); C > 0; --C) {
109  CachedTokens.push_back(Token());
110  Lex(CachedTokens.back());
111  }
112  EnterCachingLexMode();
113  return CachedTokens.back();
114 }
115 
116 void Preprocessor::AnnotatePreviousCachedTokens(const Token &Tok) {
117  assert(Tok.isAnnotation() && "Expected annotation token");
118  assert(CachedLexPos != 0 && "Expected to have some cached tokens");
119  assert(CachedTokens[CachedLexPos-1].getLastLoc() == Tok.getAnnotationEndLoc()
120  && "The annotation should be until the most recent cached token");
121 
122  // Start from the end of the cached tokens list and look for the token
123  // that is the beginning of the annotation token.
124  for (CachedTokensTy::size_type i = CachedLexPos; i != 0; --i) {
125  CachedTokensTy::iterator AnnotBegin = CachedTokens.begin() + i-1;
126  if (AnnotBegin->getLocation() == Tok.getLocation()) {
127  assert((BacktrackPositions.empty() || BacktrackPositions.back() <= i) &&
128  "The backtrack pos points inside the annotated tokens!");
129  // Replace the cached tokens with the single annotation token.
130  if (i < CachedLexPos)
131  CachedTokens.erase(AnnotBegin + 1, CachedTokens.begin() + CachedLexPos);
132  *AnnotBegin = Tok;
133  CachedLexPos = i;
134  return;
135  }
136  }
137 }
138 
140  // There's currently no cached token...
141  if (!CachedLexPos)
142  return false;
143 
144  const Token LastCachedTok = CachedTokens[CachedLexPos - 1];
145  if (LastCachedTok.getKind() != Tok.getKind())
146  return false;
147 
148  SourceLocation::IntTy RelOffset = 0;
149  if ((!getSourceManager().isInSameSLocAddrSpace(
150  Tok.getLocation(), getLastCachedTokenLocation(), &RelOffset)) ||
151  RelOffset)
152  return false;
153 
154  return true;
155 }
156 
158  assert(CachedLexPos != 0 && "Expected to have some cached tokens");
159  CachedTokens.insert(CachedTokens.begin() + CachedLexPos - 1, NewToks.begin(),
160  NewToks.end());
161  CachedTokens.erase(CachedTokens.begin() + CachedLexPos - 1 + NewToks.size());
162  CachedLexPos += NewToks.size() - 1;
163 }
clang::Token::IsReinjected
@ IsReinjected
Definition: Token.h:87
clang::Preprocessor::Lex
void Lex(Token &Result)
Lex the next token for this preprocessor.
Definition: Preprocessor.cpp:888
llvm::SmallVector< Token, 4 >
clang::Token
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
clang::Preprocessor::ReplacePreviousCachedToken
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
Definition: PPCaching.cpp:157
Preprocessor.h
clang::SourceLocation::IntTy
int32_t IntTy
Definition: SourceLocation.h:96
clang::Token::getKind
tok::TokenKind getKind() const
Definition: Token.h:92
clang::Preprocessor::isBacktrackEnabled
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
Definition: Preprocessor.h:1444
clang::Token::getAnnotationEndLoc
SourceLocation getAnnotationEndLoc() const
Definition: Token.h:140
clang::Preprocessor::getLastCachedTokenLocation
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
Definition: Preprocessor.h:1588
clang::Preprocessor::getSourceManager
SourceManager & getSourceManager() const
Definition: Preprocessor.h:964
llvm::ArrayRef
Definition: LLVM.h:34
clang::Preprocessor::EnableBacktrackAtThisPos
void EnableBacktrackAtThisPos()
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
Definition: PPCaching.cpp:25
clang::Preprocessor::Backtrack
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
Definition: PPCaching.cpp:40
clang::Preprocessor::IsPreviousCachedToken
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
Definition: PPCaching.cpp:139
clang::Token::getLocation
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:126
clang::Token::isAnnotation
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:120
clang
Definition: CalledOnceCheck.h:17
clang::Preprocessor::recomputeCurLexerKind
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Definition: Preprocessor.cpp:378
clang::Preprocessor::CommitBacktrackedTokens
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
Definition: PPCaching.cpp:32