clang  10.0.0svn
TokenLexer.cpp
Go to the documentation of this file.
1 //===- TokenLexer.cpp - Lex from a token stream ---------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the TokenLexer interface.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "clang/Lex/TokenLexer.h"
14 #include "clang/Basic/Diagnostic.h"
19 #include "clang/Basic/TokenKinds.h"
21 #include "clang/Lex/Lexer.h"
22 #include "clang/Lex/MacroArgs.h"
23 #include "clang/Lex/MacroInfo.h"
24 #include "clang/Lex/Preprocessor.h"
25 #include "clang/Lex/Token.h"
27 #include "llvm/ADT/ArrayRef.h"
28 #include "llvm/ADT/SmallString.h"
29 #include "llvm/ADT/SmallVector.h"
30 #include "llvm/ADT/iterator_range.h"
31 #include <cassert>
32 #include <cstring>
33 
34 using namespace clang;
35 
36 /// Create a TokenLexer for the specified macro with the specified actual
37 /// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
39  MacroArgs *Actuals) {
40  // If the client is reusing a TokenLexer, make sure to free any memory
41  // associated with it.
42  destroy();
43 
44  Macro = MI;
45  ActualArgs = Actuals;
46  CurTokenIdx = 0;
47 
48  ExpandLocStart = Tok.getLocation();
49  ExpandLocEnd = ELEnd;
50  AtStartOfLine = Tok.isAtStartOfLine();
51  HasLeadingSpace = Tok.hasLeadingSpace();
52  NextTokGetsSpace = false;
53  Tokens = &*Macro->tokens_begin();
54  OwnsTokens = false;
55  DisableMacroExpansion = false;
56  IsReinject = false;
57  NumTokens = Macro->tokens_end()-Macro->tokens_begin();
58  MacroExpansionStart = SourceLocation();
59 
61  MacroStartSLocOffset = SM.getNextLocalOffset();
62 
63  if (NumTokens > 0) {
64  assert(Tokens[0].getLocation().isValid());
65  assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
66  "Macro defined in macro?");
67  assert(ExpandLocStart.isValid());
68 
69  // Reserve a source location entry chunk for the length of the macro
70  // definition. Tokens that get lexed directly from the definition will
71  // have their locations pointing inside this chunk. This is to avoid
72  // creating separate source location entries for each token.
73  MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());
74  MacroDefLength = Macro->getDefinitionLength(SM);
75  MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,
76  ExpandLocStart,
77  ExpandLocEnd,
78  MacroDefLength);
79  }
80 
81  // If this is a function-like macro, expand the arguments and change
82  // Tokens to point to the expanded tokens.
83  if (Macro->isFunctionLike() && Macro->getNumParams())
84  ExpandFunctionArguments();
85 
86  // Mark the macro as currently disabled, so that it is not recursively
87  // expanded. The macro must be disabled only after argument pre-expansion of
88  // function-like macro arguments occurs.
89  Macro->DisableMacro();
90 }
91 
92 /// Create a TokenLexer for the specified token stream. This does not
93 /// take ownership of the specified token vector.
94 void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
95  bool disableMacroExpansion, bool ownsTokens,
96  bool isReinject) {
97  assert(!isReinject || disableMacroExpansion);
98  // If the client is reusing a TokenLexer, make sure to free any memory
99  // associated with it.
100  destroy();
101 
102  Macro = nullptr;
103  ActualArgs = nullptr;
104  Tokens = TokArray;
105  OwnsTokens = ownsTokens;
106  DisableMacroExpansion = disableMacroExpansion;
107  IsReinject = isReinject;
108  NumTokens = NumToks;
109  CurTokenIdx = 0;
110  ExpandLocStart = ExpandLocEnd = SourceLocation();
111  AtStartOfLine = false;
112  HasLeadingSpace = false;
113  NextTokGetsSpace = false;
114  MacroExpansionStart = SourceLocation();
115 
116  // Set HasLeadingSpace/AtStartOfLine so that the first token will be
117  // returned unmodified.
118  if (NumToks != 0) {
119  AtStartOfLine = TokArray[0].isAtStartOfLine();
120  HasLeadingSpace = TokArray[0].hasLeadingSpace();
121  }
122 }
123 
124 void TokenLexer::destroy() {
125  // If this was a function-like macro that actually uses its arguments, delete
126  // the expanded tokens.
127  if (OwnsTokens) {
128  delete [] Tokens;
129  Tokens = nullptr;
130  OwnsTokens = false;
131  }
132 
133  // TokenLexer owns its formal arguments.
134  if (ActualArgs) ActualArgs->destroy(PP);
135 }
136 
137 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
138  SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
139  unsigned MacroArgNo, Preprocessor &PP) {
140  // Is the macro argument __VA_ARGS__?
141  if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1)
142  return false;
143 
144  // In Microsoft-compatibility mode, a comma is removed in the expansion
145  // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is
146  // not supported by gcc.
147  if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)
148  return false;
149 
150  // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if
151  // __VA_ARGS__ is empty, but not in strict C99 mode where there are no
152  // named arguments, where it remains. In all other modes, including C99
153  // with GNU extensions, it is removed regardless of named arguments.
154  // Microsoft also appears to support this extension, unofficially.
155  if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode
156  && Macro->getNumParams() < 2)
157  return false;
158 
159  // Is a comma available to be removed?
160  if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
161  return false;
162 
163  // Issue an extension diagnostic for the paste operator.
164  if (HasPasteOperator)
165  PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
166 
167  // Remove the comma.
168  ResultToks.pop_back();
169 
170  if (!ResultToks.empty()) {
171  // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
172  // then removal of the comma should produce a placemarker token (in C99
173  // terms) which we model by popping off the previous ##, giving us a plain
174  // "X" when __VA_ARGS__ is empty.
175  if (ResultToks.back().is(tok::hashhash))
176  ResultToks.pop_back();
177 
178  // Remember that this comma was elided.
179  ResultToks.back().setFlag(Token::CommaAfterElided);
180  }
181 
182  // Never add a space, even if the comma, ##, or arg had a space.
183  NextTokGetsSpace = false;
184  return true;
185 }
186 
187 void TokenLexer::stringifyVAOPTContents(
188  SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,
189  const SourceLocation VAOPTClosingParenLoc) {
190  const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt();
191  const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
192  Token *const VAOPTTokens =
193  NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr;
194 
195  SmallVector<Token, 64> ConcatenatedVAOPTResultToks;
196  // FIXME: Should we keep track within VCtx that we did or didnot
197  // encounter pasting - and only then perform this loop.
198 
199  // Perform token pasting (concatenation) prior to stringization.
200  for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
201  ++CurTokenIdx) {
202  if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {
203  assert(CurTokenIdx != 0 &&
204  "Can not have __VAOPT__ contents begin with a ##");
205  Token &LHS = VAOPTTokens[CurTokenIdx - 1];
206  pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens),
207  CurTokenIdx);
208  // Replace the token prior to the first ## in this iteration.
209  ConcatenatedVAOPTResultToks.back() = LHS;
210  if (CurTokenIdx == NumVAOptTokens)
211  break;
212  }
213  ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);
214  }
215 
216  ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok());
217  // Get the SourceLocation that represents the start location within
218  // the macro definition that marks where this string is substituted
219  // into: i.e. the __VA_OPT__ and the ')' within the spelling of the
220  // macro definition, and use it to indicate that the stringified token
221  // was generated from that location.
222  const SourceLocation ExpansionLocStartWithinMacro =
223  getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc());
224  const SourceLocation ExpansionLocEndWithinMacro =
225  getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);
226 
227  Token StringifiedVAOPT = MacroArgs::StringifyArgument(
228  &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() /*Charify*/,
229  ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);
230 
232  StringifiedVAOPT.setFlag(Token::LeadingSpace);
233 
234  StringifiedVAOPT.setFlag(Token::StringifiedInMacro);
235  // Resize (shrink) the token stream to just capture this stringified token.
236  ResultToks.resize(NumToksPriorToVAOpt + 1);
237  ResultToks.back() = StringifiedVAOPT;
238 }
239 
240 /// Expand the arguments of a function-like macro so that we can quickly
241 /// return preexpanded tokens from Tokens.
242 void TokenLexer::ExpandFunctionArguments() {
243  SmallVector<Token, 128> ResultToks;
244 
245  // Loop through 'Tokens', expanding them into ResultToks. Keep
246  // track of whether we change anything. If not, no need to keep them. If so,
247  // we install the newly expanded sequence as the new 'Tokens' list.
248  bool MadeChange = false;
249 
250  Optional<bool> CalledWithVariadicArguments;
251 
252  VAOptExpansionContext VCtx(PP);
253 
254  for (unsigned I = 0, E = NumTokens; I != E; ++I) {
255  const Token &CurTok = Tokens[I];
256  // We don't want a space for the next token after a paste
257  // operator. In valid code, the token will get smooshed onto the
258  // preceding one anyway. In assembler-with-cpp mode, invalid
259  // pastes are allowed through: in this case, we do not want the
260  // extra whitespace to be added. For example, we want ". ## foo"
261  // -> ".foo" not ". foo".
262  if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace())
263  NextTokGetsSpace = true;
264 
265  if (VCtx.isVAOptToken(CurTok)) {
266  MadeChange = true;
267  assert(Tokens[I + 1].is(tok::l_paren) &&
268  "__VA_OPT__ must be followed by '('");
269 
270  ++I; // Skip the l_paren
272  ResultToks.size());
273 
274  continue;
275  }
276 
277  // We have entered into the __VA_OPT__ context, so handle tokens
278  // appropriately.
279  if (VCtx.isInVAOpt()) {
280  // If we are about to process a token that is either an argument to
281  // __VA_OPT__ or its closing rparen, then:
282  // 1) If the token is the closing rparen that exits us out of __VA_OPT__,
283  // perform any necessary stringification or placemarker processing,
284  // and/or skip to the next token.
285  // 2) else if macro was invoked without variadic arguments skip this
286  // token.
287  // 3) else (macro was invoked with variadic arguments) process the token
288  // normally.
289 
290  if (Tokens[I].is(tok::l_paren))
291  VCtx.sawOpeningParen(Tokens[I].getLocation());
292  // Continue skipping tokens within __VA_OPT__ if the macro was not
293  // called with variadic arguments, else let the rest of the loop handle
294  // this token. Note sawClosingParen() returns true only if the r_paren matches
295  // the closing r_paren of the __VA_OPT__.
296  if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) {
297  // Lazily expand __VA_ARGS__ when we see the first __VA_OPT__.
298  if (!CalledWithVariadicArguments.hasValue()) {
299  CalledWithVariadicArguments =
300  ActualArgs->invokedWithVariadicArgument(Macro, PP);
301  }
302  if (!*CalledWithVariadicArguments) {
303  // Skip this token.
304  continue;
305  }
306  // ... else the macro was called with variadic arguments, and we do not
307  // have a closing rparen - so process this token normally.
308  } else {
309  // Current token is the closing r_paren which marks the end of the
310  // __VA_OPT__ invocation, so handle any place-marker pasting (if
311  // empty) by removing hashhash either before (if exists) or after. And
312  // also stringify the entire contents if VAOPT was preceded by a hash,
313  // but do so only after any token concatenation that needs to occur
314  // within the contents of VAOPT.
315 
316  if (VCtx.hasStringifyOrCharifyBefore()) {
317  // Replace all the tokens just added from within VAOPT into a single
318  // stringified token. This requires token-pasting to eagerly occur
319  // within these tokens. If either the contents of VAOPT were empty
320  // or the macro wasn't called with any variadic arguments, the result
321  // is a token that represents an empty string.
322  stringifyVAOPTContents(ResultToks, VCtx,
323  /*ClosingParenLoc*/ Tokens[I].getLocation());
324 
325  } else if (/*No tokens within VAOPT*/
326  ResultToks.size() == VCtx.getNumberOfTokensPriorToVAOpt()) {
327  // Treat VAOPT as a placemarker token. Eat either the '##' before the
328  // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that
329  // hashhash was not a placemarker) or the '##'
330  // after VAOPT, but not both.
331 
332  if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {
333  ResultToks.pop_back();
334  } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {
335  ++I; // Skip the following hashhash.
336  }
337  } else {
338  // If there's a ## before the __VA_OPT__, we might have discovered
339  // that the __VA_OPT__ begins with a placeholder. We delay action on
340  // that to now to avoid messing up our stashed count of tokens before
341  // __VA_OPT__.
342  if (VCtx.beginsWithPlaceholder()) {
343  assert(VCtx.getNumberOfTokensPriorToVAOpt() > 0 &&
344  ResultToks.size() >= VCtx.getNumberOfTokensPriorToVAOpt() &&
345  ResultToks[VCtx.getNumberOfTokensPriorToVAOpt() - 1].is(
346  tok::hashhash) &&
347  "no token paste before __VA_OPT__");
348  ResultToks.erase(ResultToks.begin() +
349  VCtx.getNumberOfTokensPriorToVAOpt() - 1);
350  }
351  // If the expansion of __VA_OPT__ ends with a placeholder, eat any
352  // following '##' token.
353  if (VCtx.endsWithPlaceholder() && I + 1 != E &&
354  Tokens[I + 1].is(tok::hashhash)) {
355  ++I;
356  }
357  }
358  VCtx.reset();
359  // We processed __VA_OPT__'s closing paren (and the exit out of
360  // __VA_OPT__), so skip to the next token.
361  continue;
362  }
363  }
364 
365  // If we found the stringify operator, get the argument stringified. The
366  // preprocessor already verified that the following token is a macro
367  // parameter or __VA_OPT__ when the #define was lexed.
368 
369  if (CurTok.isOneOf(tok::hash, tok::hashat)) {
370  int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());
371  assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&
372  "Token following # is not an argument or __VA_OPT__!");
373 
374  if (ArgNo == -1) {
375  // Handle the __VA_OPT__ case.
376  VCtx.sawHashOrHashAtBefore(NextTokGetsSpace,
377  CurTok.is(tok::hashat));
378  continue;
379  }
380  // Else handle the simple argument case.
381  SourceLocation ExpansionLocStart =
382  getExpansionLocForMacroDefLoc(CurTok.getLocation());
383  SourceLocation ExpansionLocEnd =
384  getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());
385 
386  bool Charify = CurTok.is(tok::hashat);
387  const Token *UnexpArg = ActualArgs->getUnexpArgument(ArgNo);
389  UnexpArg, PP, Charify, ExpansionLocStart, ExpansionLocEnd);
391 
392  // The stringified/charified string leading space flag gets set to match
393  // the #/#@ operator.
394  if (NextTokGetsSpace)
396 
397  ResultToks.push_back(Res);
398  MadeChange = true;
399  ++I; // Skip arg name.
400  NextTokGetsSpace = false;
401  continue;
402  }
403 
404  // Find out if there is a paste (##) operator before or after the token.
405  bool NonEmptyPasteBefore =
406  !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
407  bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash);
408  bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash);
409  bool RParenAfter = I+1 != E && Tokens[I+1].is(tok::r_paren);
410 
411  assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&
412  "unexpected ## in ResultToks");
413 
414  // Otherwise, if this is not an argument token, just add the token to the
415  // output buffer.
416  IdentifierInfo *II = CurTok.getIdentifierInfo();
417  int ArgNo = II ? Macro->getParameterNum(II) : -1;
418  if (ArgNo == -1) {
419  // This isn't an argument, just add it.
420  ResultToks.push_back(CurTok);
421 
422  if (NextTokGetsSpace) {
423  ResultToks.back().setFlag(Token::LeadingSpace);
424  NextTokGetsSpace = false;
425  } else if (PasteBefore && !NonEmptyPasteBefore)
426  ResultToks.back().clearFlag(Token::LeadingSpace);
427 
428  continue;
429  }
430 
431  // An argument is expanded somehow, the result is different than the
432  // input.
433  MadeChange = true;
434 
435  // Otherwise, this is a use of the argument.
436 
437  // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
438  // are no trailing commas if __VA_ARGS__ is empty.
439  if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
440  MaybeRemoveCommaBeforeVaArgs(ResultToks,
441  /*HasPasteOperator=*/false,
442  Macro, ArgNo, PP))
443  continue;
444 
445  // If it is not the LHS/RHS of a ## operator, we must pre-expand the
446  // argument and substitute the expanded tokens into the result. This is
447  // C99 6.10.3.1p1.
448  if (!PasteBefore && !PasteAfter) {
449  const Token *ResultArgToks;
450 
451  // Only preexpand the argument if it could possibly need it. This
452  // avoids some work in common cases.
453  const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
454  if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
455  ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];
456  else
457  ResultArgToks = ArgTok; // Use non-preexpanded tokens.
458 
459  // If the arg token expanded into anything, append it.
460  if (ResultArgToks->isNot(tok::eof)) {
461  size_t FirstResult = ResultToks.size();
462  unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
463  ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
464 
465  // In Microsoft-compatibility mode, we follow MSVC's preprocessing
466  // behavior by not considering single commas from nested macro
467  // expansions as argument separators. Set a flag on the token so we can
468  // test for this later when the macro expansion is processed.
469  if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&
470  ResultToks.back().is(tok::comma))
471  ResultToks.back().setFlag(Token::IgnoredComma);
472 
473  // If the '##' came from expanding an argument, turn it into 'unknown'
474  // to avoid pasting.
475  for (Token &Tok : llvm::make_range(ResultToks.begin() + FirstResult,
476  ResultToks.end())) {
477  if (Tok.is(tok::hashhash))
478  Tok.setKind(tok::unknown);
479  }
480 
481  if(ExpandLocStart.isValid()) {
482  updateLocForMacroArgTokens(CurTok.getLocation(),
483  ResultToks.begin()+FirstResult,
484  ResultToks.end());
485  }
486 
487  // If any tokens were substituted from the argument, the whitespace
488  // before the first token should match the whitespace of the arg
489  // identifier.
490  ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
491  NextTokGetsSpace);
492  ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
493  NextTokGetsSpace = false;
494  } else {
495  // We're creating a placeholder token. Usually this doesn't matter,
496  // but it can affect paste behavior when at the start or end of a
497  // __VA_OPT__.
498  if (NonEmptyPasteBefore) {
499  // We're imagining a placeholder token is inserted here. If this is
500  // the first token in a __VA_OPT__ after a ##, delete the ##.
501  assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__");
503  }
504  if (RParenAfter)
506  }
507  continue;
508  }
509 
510  // Okay, we have a token that is either the LHS or RHS of a paste (##)
511  // argument. It gets substituted as its non-pre-expanded tokens.
512  const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
513  unsigned NumToks = MacroArgs::getArgLength(ArgToks);
514  if (NumToks) { // Not an empty argument?
515  bool VaArgsPseudoPaste = false;
516  // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned
517  // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when
518  // the expander tries to paste ',' with the first token of the __VA_ARGS__
519  // expansion.
520  if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
521  ResultToks[ResultToks.size()-2].is(tok::comma) &&
522  (unsigned)ArgNo == Macro->getNumParams()-1 &&
523  Macro->isVariadic()) {
524  VaArgsPseudoPaste = true;
525  // Remove the paste operator, report use of the extension.
526  PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
527  }
528 
529  ResultToks.append(ArgToks, ArgToks+NumToks);
530 
531  // If the '##' came from expanding an argument, turn it into 'unknown'
532  // to avoid pasting.
533  for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
534  ResultToks.end())) {
535  if (Tok.is(tok::hashhash))
536  Tok.setKind(tok::unknown);
537  }
538 
539  if (ExpandLocStart.isValid()) {
540  updateLocForMacroArgTokens(CurTok.getLocation(),
541  ResultToks.end()-NumToks, ResultToks.end());
542  }
543 
544  // Transfer the leading whitespace information from the token
545  // (the macro argument) onto the first token of the
546  // expansion. Note that we don't do this for the GNU
547  // pseudo-paste extension ", ## __VA_ARGS__".
548  if (!VaArgsPseudoPaste) {
549  ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,
550  false);
551  ResultToks[ResultToks.size() - NumToks].setFlagValue(
552  Token::LeadingSpace, NextTokGetsSpace);
553  }
554 
555  NextTokGetsSpace = false;
556  continue;
557  }
558 
559  // If an empty argument is on the LHS or RHS of a paste, the standard (C99
560  // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We
561  // implement this by eating ## operators when a LHS or RHS expands to
562  // empty.
563  if (PasteAfter) {
564  // Discard the argument token and skip (don't copy to the expansion
565  // buffer) the paste operator after it.
566  ++I;
567  continue;
568  }
569 
570  if (RParenAfter)
572 
573  // If this is on the RHS of a paste operator, we've already copied the
574  // paste operator to the ResultToks list, unless the LHS was empty too.
575  // Remove it.
576  assert(PasteBefore);
577  if (NonEmptyPasteBefore) {
578  assert(ResultToks.back().is(tok::hashhash));
579  // Do not remove the paste operator if it is the one before __VA_OPT__
580  // (and we are still processing tokens within VA_OPT). We handle the case
581  // of removing the paste operator if __VA_OPT__ reduces to the notional
582  // placemarker above when we encounter the closing paren of VA_OPT.
583  if (!VCtx.isInVAOpt() ||
584  ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt())
585  ResultToks.pop_back();
586  else
588  }
589 
590  // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
591  // and if the macro had at least one real argument, and if the token before
592  // the ## was a comma, remove the comma. This is a GCC extension which is
593  // disabled when using -std=c99.
594  if (ActualArgs->isVarargsElidedUse())
595  MaybeRemoveCommaBeforeVaArgs(ResultToks,
596  /*HasPasteOperator=*/true,
597  Macro, ArgNo, PP);
598  }
599 
600  // If anything changed, install this as the new Tokens list.
601  if (MadeChange) {
602  assert(!OwnsTokens && "This would leak if we already own the token list");
603  // This is deleted in the dtor.
604  NumTokens = ResultToks.size();
605  // The tokens will be added to Preprocessor's cache and will be removed
606  // when this TokenLexer finishes lexing them.
607  Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
608 
609  // The preprocessor cache of macro expanded tokens owns these tokens,not us.
610  OwnsTokens = false;
611  }
612 }
613 
614 /// Checks if two tokens form wide string literal.
615 static bool isWideStringLiteralFromMacro(const Token &FirstTok,
616  const Token &SecondTok) {
617  return FirstTok.is(tok::identifier) &&
618  FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral() &&
619  SecondTok.stringifiedInMacro();
620 }
621 
622 /// Lex - Lex and return a token from this macro stream.
624  // Lexing off the end of the macro, pop this macro off the expansion stack.
625  if (isAtEnd()) {
626  // If this is a macro (not a token stream), mark the macro enabled now
627  // that it is no longer being expanded.
628  if (Macro) Macro->EnableMacro();
629 
630  Tok.startToken();
631  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
632  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);
633  if (CurTokenIdx == 0)
635  return PP.HandleEndOfTokenLexer(Tok);
636  }
637 
639 
640  // If this is the first token of the expanded result, we inherit spacing
641  // properties later.
642  bool isFirstToken = CurTokenIdx == 0;
643 
644  // Get the next token to return.
645  Tok = Tokens[CurTokenIdx++];
646  if (IsReinject)
648 
649  bool TokenIsFromPaste = false;
650 
651  // If this token is followed by a token paste (##) operator, paste the tokens!
652  // Note that ## is a normal token when not expanding a macro.
653  if (!isAtEnd() && Macro &&
654  (Tokens[CurTokenIdx].is(tok::hashhash) ||
655  // Special processing of L#x macros in -fms-compatibility mode.
656  // Microsoft compiler is able to form a wide string literal from
657  // 'L#macro_arg' construct in a function-like macro.
658  (PP.getLangOpts().MSVCCompat &&
659  isWideStringLiteralFromMacro(Tok, Tokens[CurTokenIdx])))) {
660  // When handling the microsoft /##/ extension, the final token is
661  // returned by pasteTokens, not the pasted token.
662  if (pasteTokens(Tok))
663  return true;
664 
665  TokenIsFromPaste = true;
666  }
667 
668  // The token's current location indicate where the token was lexed from. We
669  // need this information to compute the spelling of the token, but any
670  // diagnostics for the expanded token should appear as if they came from
671  // ExpansionLoc. Pull this information together into a new SourceLocation
672  // that captures all of this.
673  if (ExpandLocStart.isValid() && // Don't do this for token streams.
674  // Check that the token's location was not already set properly.
675  SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {
676  SourceLocation instLoc;
677  if (Tok.is(tok::comment)) {
678  instLoc = SM.createExpansionLoc(Tok.getLocation(),
679  ExpandLocStart,
680  ExpandLocEnd,
681  Tok.getLength());
682  } else {
683  instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());
684  }
685 
686  Tok.setLocation(instLoc);
687  }
688 
689  // If this is the first token, set the lexical properties of the token to
690  // match the lexical properties of the macro identifier.
691  if (isFirstToken) {
692  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
693  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
694  } else {
695  // If this is not the first token, we may still need to pass through
696  // leading whitespace if we've expanded a macro.
697  if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);
698  if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);
699  }
700  AtStartOfLine = false;
701  HasLeadingSpace = false;
702 
703  // Handle recursive expansion!
704  if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) {
705  // Change the kind of this identifier to the appropriate token kind, e.g.
706  // turning "for" into a keyword.
707  IdentifierInfo *II = Tok.getIdentifierInfo();
708  Tok.setKind(II->getTokenID());
709 
710  // If this identifier was poisoned and from a paste, emit an error. This
711  // won't be handled by Preprocessor::HandleIdentifier because this is coming
712  // from a macro expansion.
713  if (II->isPoisoned() && TokenIsFromPaste) {
714  PP.HandlePoisonedIdentifier(Tok);
715  }
716 
717  if (!DisableMacroExpansion && II->isHandleIdentifierCase())
718  return PP.HandleIdentifier(Tok);
719  }
720 
721  // Otherwise, return a normal token.
722  return true;
723 }
724 
725 bool TokenLexer::pasteTokens(Token &Tok) {
726  return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx);
727 }
728 
729 /// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ##
730 /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there
731 /// are more ## after it, chomp them iteratively. Return the result as LHSTok.
732 /// If this returns true, the caller should immediately return the token.
733 bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
734  unsigned int &CurIdx) {
735  assert(CurIdx > 0 && "## can not be the first token within tokens");
736  assert((TokenStream[CurIdx].is(tok::hashhash) ||
737  (PP.getLangOpts().MSVCCompat &&
738  isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) &&
739  "Token at this Index must be ## or part of the MSVC 'L "
740  "#macro-arg' pasting pair");
741 
742  // MSVC: If previous token was pasted, this must be a recovery from an invalid
743  // paste operation. Ignore spaces before this token to mimic MSVC output.
744  // Required for generating valid UUID strings in some MS headers.
745  if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) &&
746  TokenStream[CurIdx - 2].is(tok::hashhash))
748 
749  SmallString<128> Buffer;
750  const char *ResultTokStrPtr = nullptr;
751  SourceLocation StartLoc = LHSTok.getLocation();
752  SourceLocation PasteOpLoc;
753 
754  auto IsAtEnd = [&TokenStream, &CurIdx] {
755  return TokenStream.size() == CurIdx;
756  };
757 
758  do {
759  // Consume the ## operator if any.
760  PasteOpLoc = TokenStream[CurIdx].getLocation();
761  if (TokenStream[CurIdx].is(tok::hashhash))
762  ++CurIdx;
763  assert(!IsAtEnd() && "No token on the RHS of a paste operator!");
764 
765  // Get the RHS token.
766  const Token &RHS = TokenStream[CurIdx];
767 
768  // Allocate space for the result token. This is guaranteed to be enough for
769  // the two tokens.
770  Buffer.resize(LHSTok.getLength() + RHS.getLength());
771 
772  // Get the spelling of the LHS token in Buffer.
773  const char *BufPtr = &Buffer[0];
774  bool Invalid = false;
775  unsigned LHSLen = PP.getSpelling(LHSTok, BufPtr, &Invalid);
776  if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer!
777  memcpy(&Buffer[0], BufPtr, LHSLen);
778  if (Invalid)
779  return true;
780 
781  BufPtr = Buffer.data() + LHSLen;
782  unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid);
783  if (Invalid)
784  return true;
785  if (RHSLen && BufPtr != &Buffer[LHSLen])
786  // Really, we want the chars in Buffer!
787  memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
788 
789  // Trim excess space.
790  Buffer.resize(LHSLen+RHSLen);
791 
792  // Plop the pasted result (including the trailing newline and null) into a
793  // scratch buffer where we can lex it.
794  Token ResultTokTmp;
795  ResultTokTmp.startToken();
796 
797  // Claim that the tmp token is a string_literal so that we can get the
798  // character pointer back from CreateString in getLiteralData().
799  ResultTokTmp.setKind(tok::string_literal);
800  PP.CreateString(Buffer, ResultTokTmp);
801  SourceLocation ResultTokLoc = ResultTokTmp.getLocation();
802  ResultTokStrPtr = ResultTokTmp.getLiteralData();
803 
804  // Lex the resultant pasted token into Result.
805  Token Result;
806 
807  if (LHSTok.isAnyIdentifier() && RHS.isAnyIdentifier()) {
808  // Common paste case: identifier+identifier = identifier. Avoid creating
809  // a lexer and other overhead.
810  PP.IncrementPasteCounter(true);
811  Result.startToken();
812  Result.setKind(tok::raw_identifier);
813  Result.setRawIdentifierData(ResultTokStrPtr);
814  Result.setLocation(ResultTokLoc);
815  Result.setLength(LHSLen+RHSLen);
816  } else {
817  PP.IncrementPasteCounter(false);
818 
819  assert(ResultTokLoc.isFileID() &&
820  "Should be a raw location into scratch buffer");
821  SourceManager &SourceMgr = PP.getSourceManager();
822  FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
823 
824  bool Invalid = false;
825  const char *ScratchBufStart
826  = SourceMgr.getBufferData(LocFileID, &Invalid).data();
827  if (Invalid)
828  return false;
829 
830  // Make a lexer to lex this string from. Lex just this one token.
831  // Make a lexer object so that we lex and expand the paste result.
832  Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
833  PP.getLangOpts(), ScratchBufStart,
834  ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
835 
836  // Lex a token in raw mode. This way it won't look up identifiers
837  // automatically, lexing off the end will return an eof token, and
838  // warnings are disabled. This returns true if the result token is the
839  // entire buffer.
840  bool isInvalid = !TL.LexFromRawLexer(Result);
841 
842  // If we got an EOF token, we didn't form even ONE token. For example, we
843  // did "/ ## /" to get "//".
844  isInvalid |= Result.is(tok::eof);
845 
846  // If pasting the two tokens didn't form a full new token, this is an
847  // error. This occurs with "x ## +" and other stuff. Return with LHSTok
848  // unmodified and with RHS as the next token to lex.
849  if (isInvalid) {
850  // Explicitly convert the token location to have proper expansion
851  // information so that the user knows where it came from.
853  SourceLocation Loc =
854  SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);
855 
856  // Test for the Microsoft extension of /##/ turning into // here on the
857  // error path.
858  if (PP.getLangOpts().MicrosoftExt && LHSTok.is(tok::slash) &&
859  RHS.is(tok::slash)) {
860  HandleMicrosoftCommentPaste(LHSTok, Loc);
861  return true;
862  }
863 
864  // Do not emit the error when preprocessing assembler code.
865  if (!PP.getLangOpts().AsmPreprocessor) {
866  // If we're in microsoft extensions mode, downgrade this from a hard
867  // error to an extension that defaults to an error. This allows
868  // disabling it.
869  PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms
870  : diag::err_pp_bad_paste)
871  << Buffer;
872  }
873 
874  // An error has occurred so exit loop.
875  break;
876  }
877 
878  // Turn ## into 'unknown' to avoid # ## # from looking like a paste
879  // operator.
880  if (Result.is(tok::hashhash))
881  Result.setKind(tok::unknown);
882  }
883 
884  // Transfer properties of the LHS over the Result.
887 
888  // Finally, replace LHS with the result, consume the RHS, and iterate.
889  ++CurIdx;
890  LHSTok = Result;
891  } while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));
892 
893  SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation();
894 
895  // The token's current location indicate where the token was lexed from. We
896  // need this information to compute the spelling of the token, but any
897  // diagnostics for the expanded token should appear as if the token was
898  // expanded from the full ## expression. Pull this information together into
899  // a new SourceLocation that captures all of this.
901  if (StartLoc.isFileID())
902  StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
903  if (EndLoc.isFileID())
904  EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
905  FileID MacroFID = SM.getFileID(MacroExpansionStart);
906  while (SM.getFileID(StartLoc) != MacroFID)
907  StartLoc = SM.getImmediateExpansionRange(StartLoc).getBegin();
908  while (SM.getFileID(EndLoc) != MacroFID)
909  EndLoc = SM.getImmediateExpansionRange(EndLoc).getEnd();
910 
911  LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc,
912  LHSTok.getLength()));
913 
914  // Now that we got the result token, it will be subject to expansion. Since
915  // token pasting re-lexes the result token in raw mode, identifier information
916  // isn't looked up. As such, if the result is an identifier, look up id info.
917  if (LHSTok.is(tok::raw_identifier)) {
918  // Look up the identifier info for the token. We disabled identifier lookup
919  // by saying we're skipping contents, so we need to do this manually.
920  PP.LookUpIdentifierInfo(LHSTok);
921  }
922  return false;
923 }
924 
925 /// isNextTokenLParen - If the next token lexed will pop this macro off the
926 /// expansion stack, return 2. If the next unexpanded token is a '(', return
927 /// 1, otherwise return 0.
929  // Out of tokens?
930  if (isAtEnd())
931  return 2;
932  return Tokens[CurTokenIdx].is(tok::l_paren);
933 }
934 
935 /// isParsingPreprocessorDirective - Return true if we are in the middle of a
936 /// preprocessor directive.
938  return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd();
939 }
940 
941 /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
942 /// together to form a comment that comments out everything in the current
943 /// macro, other active macros, and anything left on the current physical
944 /// source line of the expanded buffer. Handle this by returning the
945 /// first token on the next line.
946 void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {
947  PP.Diag(OpLoc, diag::ext_comment_paste_microsoft);
948 
949  // We 'comment out' the rest of this macro by just ignoring the rest of the
950  // tokens that have not been lexed yet, if any.
951 
952  // Since this must be a macro, mark the macro enabled now that it is no longer
953  // being expanded.
954  assert(Macro && "Token streams can't paste comments");
955  Macro->EnableMacro();
956 
958 }
959 
960 /// If \arg loc is a file ID and points inside the current macro
961 /// definition, returns the appropriate source location pointing at the
962 /// macro expansion source location entry, otherwise it returns an invalid
963 /// SourceLocation.
965 TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {
966  assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&
967  "Not appropriate for token streams");
968  assert(loc.isValid() && loc.isFileID());
969 
971  assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
972  "Expected loc to come from the macro definition");
973 
974  unsigned relativeOffset = 0;
975  SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);
976  return MacroExpansionStart.getLocWithOffset(relativeOffset);
977 }
978 
979 /// Finds the tokens that are consecutive (from the same FileID)
980 /// creates a single SLocEntry, and assigns SourceLocations to each token that
981 /// point to that SLocEntry. e.g for
982 /// assert(foo == bar);
983 /// There will be a single SLocEntry for the "foo == bar" chunk and locations
984 /// for the 'foo', '==', 'bar' tokens will point inside that chunk.
985 ///
986 /// \arg begin_tokens will be updated to a position past all the found
987 /// consecutive tokens.
989  SourceLocation InstLoc,
990  Token *&begin_tokens,
991  Token * end_tokens) {
992  assert(begin_tokens < end_tokens);
993 
994  SourceLocation FirstLoc = begin_tokens->getLocation();
995  SourceLocation CurLoc = FirstLoc;
996 
997  // Compare the source location offset of tokens and group together tokens that
998  // are close, even if their locations point to different FileIDs. e.g.
999  //
1000  // |bar | foo | cake | (3 tokens from 3 consecutive FileIDs)
1001  // ^ ^
1002  // |bar foo cake| (one SLocEntry chunk for all tokens)
1003  //
1004  // we can perform this "merge" since the token's spelling location depends
1005  // on the relative offset.
1006 
1007  Token *NextTok = begin_tokens + 1;
1008  for (; NextTok < end_tokens; ++NextTok) {
1009  SourceLocation NextLoc = NextTok->getLocation();
1010  if (CurLoc.isFileID() != NextLoc.isFileID())
1011  break; // Token from different kind of FileID.
1012 
1013  int RelOffs;
1014  if (!SM.isInSameSLocAddrSpace(CurLoc, NextLoc, &RelOffs))
1015  break; // Token from different local/loaded location.
1016  // Check that token is not before the previous token or more than 50
1017  // "characters" away.
1018  if (RelOffs < 0 || RelOffs > 50)
1019  break;
1020 
1021  if (CurLoc.isMacroID() && !SM.isWrittenInSameFile(CurLoc, NextLoc))
1022  break; // Token from a different macro.
1023 
1024  CurLoc = NextLoc;
1025  }
1026 
1027  // For the consecutive tokens, find the length of the SLocEntry to contain
1028  // all of them.
1029  Token &LastConsecutiveTok = *(NextTok-1);
1030  int LastRelOffs = 0;
1031  SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(),
1032  &LastRelOffs);
1033  unsigned FullLength = LastRelOffs + LastConsecutiveTok.getLength();
1034 
1035  // Create a macro expansion SLocEntry that will "contain" all of the tokens.
1036  SourceLocation Expansion =
1037  SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength);
1038 
1039  // Change the location of the tokens from the spelling location to the new
1040  // expanded location.
1041  for (; begin_tokens < NextTok; ++begin_tokens) {
1042  Token &Tok = *begin_tokens;
1043  int RelOffs = 0;
1044  SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs);
1045  Tok.setLocation(Expansion.getLocWithOffset(RelOffs));
1046  }
1047 }
1048 
1049 /// Creates SLocEntries and updates the locations of macro argument
1050 /// tokens to their new expanded locations.
1051 ///
1052 /// \param ArgIdSpellLoc the location of the macro argument id inside the macro
1053 /// definition.
1054 void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
1055  Token *begin_tokens,
1056  Token *end_tokens) {
1058 
1059  SourceLocation InstLoc =
1060  getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
1061 
1062  while (begin_tokens < end_tokens) {
1063  // If there's only one token just create a SLocEntry for it.
1064  if (end_tokens - begin_tokens == 1) {
1065  Token &Tok = *begin_tokens;
1067  InstLoc,
1068  Tok.getLength()));
1069  return;
1070  }
1071 
1072  updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens);
1073  }
1074 }
1075 
1076 void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
1077  AtStartOfLine = Result.isAtStartOfLine();
1078  HasLeadingSpace = Result.hasLeadingSpace();
1079 }
bool isWrittenInSameFile(SourceLocation Loc1, SourceLocation Loc2) const
Returns true if the spelling locations for both SourceLocations are part of the same file buffer...
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:76
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void DisableMacro()
Definition: MacroInfo.h:265
void sawOpeningParen(SourceLocation LParenLoc)
Call this function each time an lparen is seen.
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Definition: Token.h:259
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
ArgNeedsPreexpansion - If we can prove that the argument won&#39;t be affected by pre-expansion, return false.
Definition: MacroArgs.cpp:146
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:97
bool sawClosingParen()
Call this function each time an rparen is seen.
Defines the SourceManager interface.
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation InstLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry, and assigns SourceLocations to each token that point to that SLocEntry.
Definition: TokenLexer.cpp:988
A class for tracking whether we&#39;re inside a VA_OPT during a traversal of the tokens of a macro during...
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool isBeforeInSLocAddrSpace(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the "source location address space".
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:236
bool isVarargsElidedUse() const
isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...
Definition: MacroArgs.h:102
bool isVariadic() const
Definition: MacroInfo.h:207
bool isInSameSLocAddrSpace(SourceLocation LHS, SourceLocation RHS, int *RelativeOffset) const
Return true if both LHS and RHS are in the local source location address space or the loaded one...
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
Definition: Token.h:109
unsigned getNextLocalOffset() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:120
tokens_iterator tokens_begin() const
Definition: MacroInfo.h:242
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:115
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
One of these records is kept for each identifier that is lexed.
SourceLocation getBegin() const
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:209
void sawVAOptFollowedByOpeningParens(const SourceLocation VAOptLoc, const unsigned int NumPriorTokens)
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
void setKind(tok::TokenKind K)
Definition: Token.h:93
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:904
bool Lex(Token &Tok)
Lex and return a token from this macro stream.
Definition: TokenLexer.cpp:623
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
const FormatToken & Tok
SourceLocation getExpansionLoc(SourceLocation Loc) const
Given a SourceLocation object Loc, return the expansion location referenced by the ID...
Defines the Diagnostic-related interfaces.
void destroy(Preprocessor &PP)
destroy - Destroy and deallocate the memory for this object.
Definition: MacroArgs.cpp:78
void Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, MacroArgs *Actuals)
Initialize this TokenLexer to expand from the specified macro with the specified argument information...
Definition: TokenLexer.cpp:38
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
Definition: TokenLexer.cpp:615
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...
Definition: MacroArgs.cpp:208
unsigned getNumParams() const
Definition: MacroInfo.h:182
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
bool invokedWithVariadicArgument(const MacroInfo *const MI, Preprocessor &PP)
Returns true if the macro was defined with a variadic (ellipsis) parameter AND was invoked with at le...
Definition: MacroArgs.cpp:136
Defines the clang::LangOptions interface.
bool isInSLocAddrSpace(SourceLocation Loc, SourceLocation Start, unsigned Length, unsigned *RelativeOffset=nullptr) const
Returns true if Loc is inside the [Start, +Length) chunk of the source location address space...
SourceLocation getVAOptLoc() const
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
SourceLocation createMacroArgExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLoc, unsigned TokLength)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:126
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
CharSourceRange getImmediateExpansionRange(SourceLocation Loc) const
Return the start/end of the expansion information for an expansion location.
The result type of a method or function.
const SourceManager & SM
Definition: Format.cpp:1667
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
SourceManager & getSourceManager() const
Definition: Preprocessor.h:908
bool isVAOptToken(const Token &T) const
Encodes a location in the source.
void setLength(unsigned Len)
Definition: Token.h:135
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, bool ExpansionIsTokenRange=true, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:312
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:179
bool isInVAOpt() const
Returns true if we have seen the VA_OPT and &#39;(&#39; but before having seen the matching &#39;)&#39;...
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:268
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:98
static bool isInvalid(LocType Loc, bool *Invalid)
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
bool isFunctionLike() const
Definition: MacroInfo.h:199
const std::vector< Token > & getPreExpArgument(unsigned Arg, Preprocessor &PP)
getPreExpArgument - Return the pre-expanded form of the specified argument.
Definition: MacroArgs.cpp:161
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:99
tokens_iterator tokens_end() const
Definition: MacroInfo.h:243
static unsigned getArgLength(const Token *ArgPtr)
getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens...
Definition: MacroArgs.cpp:108
void sawHashOrHashAtBefore(const bool HasLeadingSpace, const bool IsHashAt)
unsigned getLength() const
Definition: Token.h:129
bool isMacroID() const
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:217
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
int getParameterNum(const IdentifierInfo *Arg) const
Return the parameter number of the specified identifier, or -1 if the identifier is not a formal para...
Definition: MacroInfo.h:189
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
unsigned int getNumberOfTokensPriorToVAOpt() const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token&#39;s location and length to ...
SourceLocation getEnd() const
unsigned isNextTokenLParen() const
If the next token lexed will pop this macro off the expansion stack, return 2.
Definition: TokenLexer.cpp:928
Defines the clang::TokenKind enum and support functions.
unsigned getDefinitionLength(const SourceManager &SM) const
Get length in characters of the macro definition.
Definition: MacroInfo.h:132
Defines the clang::SourceLocation class and associated facilities.
bool stringifiedInMacro() const
Returns true if this token is formed by macro by stringizing or charizing operator.
Definition: Token.h:300
void setLocation(SourceLocation L)
Definition: Token.h:134
void EnableMacro()
Definition: MacroInfo.h:260
void clearFlag(TokenFlags Flag)
Unset the specified flag.
Definition: Token.h:246
bool isParsingPreprocessorDirective() const
isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive...
Definition: TokenLexer.cpp:937
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:272
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
This class handles loading and caching of source files into memory.
const Token * getUnexpArgument(unsigned Arg) const
getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...
Definition: MacroArgs.cpp:118
void startToken()
Reset all flags to cleared.
Definition: Token.h:171
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:125