clang  9.0.0svn
TokenLexer.cpp
Go to the documentation of this file.
1 //===- TokenLexer.cpp - Lex from a token stream ---------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the TokenLexer interface.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "clang/Lex/TokenLexer.h"
14 #include "clang/Basic/Diagnostic.h"
19 #include "clang/Basic/TokenKinds.h"
21 #include "clang/Lex/Lexer.h"
22 #include "clang/Lex/MacroArgs.h"
23 #include "clang/Lex/MacroInfo.h"
24 #include "clang/Lex/Preprocessor.h"
25 #include "clang/Lex/Token.h"
27 #include "llvm/ADT/ArrayRef.h"
28 #include "llvm/ADT/SmallString.h"
29 #include "llvm/ADT/SmallVector.h"
30 #include "llvm/ADT/iterator_range.h"
31 #include <cassert>
32 #include <cstring>
33 
34 using namespace clang;
35 
36 /// Create a TokenLexer for the specified macro with the specified actual
37 /// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
39  MacroArgs *Actuals) {
40  // If the client is reusing a TokenLexer, make sure to free any memory
41  // associated with it.
42  destroy();
43 
44  Macro = MI;
45  ActualArgs = Actuals;
46  CurTokenIdx = 0;
47 
48  ExpandLocStart = Tok.getLocation();
49  ExpandLocEnd = ELEnd;
50  AtStartOfLine = Tok.isAtStartOfLine();
51  HasLeadingSpace = Tok.hasLeadingSpace();
52  NextTokGetsSpace = false;
53  Tokens = &*Macro->tokens_begin();
54  OwnsTokens = false;
55  DisableMacroExpansion = false;
56  IsReinject = false;
57  NumTokens = Macro->tokens_end()-Macro->tokens_begin();
58  MacroExpansionStart = SourceLocation();
59 
61  MacroStartSLocOffset = SM.getNextLocalOffset();
62 
63  if (NumTokens > 0) {
64  assert(Tokens[0].getLocation().isValid());
65  assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
66  "Macro defined in macro?");
67  assert(ExpandLocStart.isValid());
68 
69  // Reserve a source location entry chunk for the length of the macro
70  // definition. Tokens that get lexed directly from the definition will
71  // have their locations pointing inside this chunk. This is to avoid
72  // creating separate source location entries for each token.
73  MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());
74  MacroDefLength = Macro->getDefinitionLength(SM);
75  MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,
76  ExpandLocStart,
77  ExpandLocEnd,
78  MacroDefLength);
79  }
80 
81  // If this is a function-like macro, expand the arguments and change
82  // Tokens to point to the expanded tokens.
83  if (Macro->isFunctionLike() && Macro->getNumParams())
84  ExpandFunctionArguments();
85 
86  // Mark the macro as currently disabled, so that it is not recursively
87  // expanded. The macro must be disabled only after argument pre-expansion of
88  // function-like macro arguments occurs.
89  Macro->DisableMacro();
90 }
91 
92 /// Create a TokenLexer for the specified token stream. This does not
93 /// take ownership of the specified token vector.
94 void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
95  bool disableMacroExpansion, bool ownsTokens,
96  bool isReinject) {
97  assert(!isReinject || disableMacroExpansion);
98  // If the client is reusing a TokenLexer, make sure to free any memory
99  // associated with it.
100  destroy();
101 
102  Macro = nullptr;
103  ActualArgs = nullptr;
104  Tokens = TokArray;
105  OwnsTokens = ownsTokens;
106  DisableMacroExpansion = disableMacroExpansion;
107  IsReinject = isReinject;
108  NumTokens = NumToks;
109  CurTokenIdx = 0;
110  ExpandLocStart = ExpandLocEnd = SourceLocation();
111  AtStartOfLine = false;
112  HasLeadingSpace = false;
113  NextTokGetsSpace = false;
114  MacroExpansionStart = SourceLocation();
115 
116  // Set HasLeadingSpace/AtStartOfLine so that the first token will be
117  // returned unmodified.
118  if (NumToks != 0) {
119  AtStartOfLine = TokArray[0].isAtStartOfLine();
120  HasLeadingSpace = TokArray[0].hasLeadingSpace();
121  }
122 }
123 
124 void TokenLexer::destroy() {
125  // If this was a function-like macro that actually uses its arguments, delete
126  // the expanded tokens.
127  if (OwnsTokens) {
128  delete [] Tokens;
129  Tokens = nullptr;
130  OwnsTokens = false;
131  }
132 
133  // TokenLexer owns its formal arguments.
134  if (ActualArgs) ActualArgs->destroy(PP);
135 }
136 
137 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
138  SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
139  unsigned MacroArgNo, Preprocessor &PP) {
140  // Is the macro argument __VA_ARGS__?
141  if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1)
142  return false;
143 
144  // In Microsoft-compatibility mode, a comma is removed in the expansion
145  // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is
146  // not supported by gcc.
147  if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)
148  return false;
149 
150  // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if
151  // __VA_ARGS__ is empty, but not in strict C99 mode where there are no
152  // named arguments, where it remains. In all other modes, including C99
153  // with GNU extensions, it is removed regardless of named arguments.
154  // Microsoft also appears to support this extension, unofficially.
155  if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode
156  && Macro->getNumParams() < 2)
157  return false;
158 
159  // Is a comma available to be removed?
160  if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
161  return false;
162 
163  // Issue an extension diagnostic for the paste operator.
164  if (HasPasteOperator)
165  PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
166 
167  // Remove the comma.
168  ResultToks.pop_back();
169 
170  if (!ResultToks.empty()) {
171  // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
172  // then removal of the comma should produce a placemarker token (in C99
173  // terms) which we model by popping off the previous ##, giving us a plain
174  // "X" when __VA_ARGS__ is empty.
175  if (ResultToks.back().is(tok::hashhash))
176  ResultToks.pop_back();
177 
178  // Remember that this comma was elided.
179  ResultToks.back().setFlag(Token::CommaAfterElided);
180  }
181 
182  // Never add a space, even if the comma, ##, or arg had a space.
183  NextTokGetsSpace = false;
184  return true;
185 }
186 
187 void TokenLexer::stringifyVAOPTContents(
188  SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,
189  const SourceLocation VAOPTClosingParenLoc) {
190  const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt();
191  const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
192  Token *const VAOPTTokens =
193  NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr;
194 
195  SmallVector<Token, 64> ConcatenatedVAOPTResultToks;
196  // FIXME: Should we keep track within VCtx that we did or didnot
197  // encounter pasting - and only then perform this loop.
198 
199  // Perform token pasting (concatenation) prior to stringization.
200  for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
201  ++CurTokenIdx) {
202  if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {
203  assert(CurTokenIdx != 0 &&
204  "Can not have __VAOPT__ contents begin with a ##");
205  Token &LHS = VAOPTTokens[CurTokenIdx - 1];
206  pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens),
207  CurTokenIdx);
208  // Replace the token prior to the first ## in this iteration.
209  ConcatenatedVAOPTResultToks.back() = LHS;
210  if (CurTokenIdx == NumVAOptTokens)
211  break;
212  }
213  ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);
214  }
215 
216  ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok());
217  // Get the SourceLocation that represents the start location within
218  // the macro definition that marks where this string is substituted
219  // into: i.e. the __VA_OPT__ and the ')' within the spelling of the
220  // macro definition, and use it to indicate that the stringified token
221  // was generated from that location.
222  const SourceLocation ExpansionLocStartWithinMacro =
223  getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc());
224  const SourceLocation ExpansionLocEndWithinMacro =
225  getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);
226 
227  Token StringifiedVAOPT = MacroArgs::StringifyArgument(
228  &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() /*Charify*/,
229  ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);
230 
232  StringifiedVAOPT.setFlag(Token::LeadingSpace);
233 
234  StringifiedVAOPT.setFlag(Token::StringifiedInMacro);
235  // Resize (shrink) the token stream to just capture this stringified token.
236  ResultToks.resize(NumToksPriorToVAOpt + 1);
237  ResultToks.back() = StringifiedVAOPT;
238 }
239 
240 /// Expand the arguments of a function-like macro so that we can quickly
241 /// return preexpanded tokens from Tokens.
242 void TokenLexer::ExpandFunctionArguments() {
243  SmallVector<Token, 128> ResultToks;
244 
245  // Loop through 'Tokens', expanding them into ResultToks. Keep
246  // track of whether we change anything. If not, no need to keep them. If so,
247  // we install the newly expanded sequence as the new 'Tokens' list.
248  bool MadeChange = false;
249 
250  Optional<bool> CalledWithVariadicArguments;
251 
252  VAOptExpansionContext VCtx(PP);
253 
254  for (unsigned I = 0, E = NumTokens; I != E; ++I) {
255  const Token &CurTok = Tokens[I];
256  // We don't want a space for the next token after a paste
257  // operator. In valid code, the token will get smooshed onto the
258  // preceding one anyway. In assembler-with-cpp mode, invalid
259  // pastes are allowed through: in this case, we do not want the
260  // extra whitespace to be added. For example, we want ". ## foo"
261  // -> ".foo" not ". foo".
262  if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace())
263  NextTokGetsSpace = true;
264 
265  if (VCtx.isVAOptToken(CurTok)) {
266  MadeChange = true;
267  assert(Tokens[I + 1].is(tok::l_paren) &&
268  "__VA_OPT__ must be followed by '('");
269 
270  ++I; // Skip the l_paren
272  ResultToks.size());
273 
274  continue;
275  }
276 
277  // We have entered into the __VA_OPT__ context, so handle tokens
278  // appropriately.
279  if (VCtx.isInVAOpt()) {
280  // If we are about to process a token that is either an argument to
281  // __VA_OPT__ or its closing rparen, then:
282  // 1) If the token is the closing rparen that exits us out of __VA_OPT__,
283  // perform any necessary stringification or placemarker processing,
284  // and/or skip to the next token.
285  // 2) else if macro was invoked without variadic arguments skip this
286  // token.
287  // 3) else (macro was invoked with variadic arguments) process the token
288  // normally.
289 
290  if (Tokens[I].is(tok::l_paren))
291  VCtx.sawOpeningParen(Tokens[I].getLocation());
292  // Continue skipping tokens within __VA_OPT__ if the macro was not
293  // called with variadic arguments, else let the rest of the loop handle
294  // this token. Note sawClosingParen() returns true only if the r_paren matches
295  // the closing r_paren of the __VA_OPT__.
296  if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) {
297  // Lazily expand __VA_ARGS__ when we see the first __VA_OPT__.
298  if (!CalledWithVariadicArguments.hasValue()) {
299  CalledWithVariadicArguments =
300  ActualArgs->invokedWithVariadicArgument(Macro, PP);
301  }
302  if (!*CalledWithVariadicArguments) {
303  // Skip this token.
304  continue;
305  }
306  // ... else the macro was called with variadic arguments, and we do not
307  // have a closing rparen - so process this token normally.
308  } else {
309  // Current token is the closing r_paren which marks the end of the
310  // __VA_OPT__ invocation, so handle any place-marker pasting (if
311  // empty) by removing hashhash either before (if exists) or after. And
312  // also stringify the entire contents if VAOPT was preceded by a hash,
313  // but do so only after any token concatenation that needs to occur
314  // within the contents of VAOPT.
315 
316  if (VCtx.hasStringifyOrCharifyBefore()) {
317  // Replace all the tokens just added from within VAOPT into a single
318  // stringified token. This requires token-pasting to eagerly occur
319  // within these tokens. If either the contents of VAOPT were empty
320  // or the macro wasn't called with any variadic arguments, the result
321  // is a token that represents an empty string.
322  stringifyVAOPTContents(ResultToks, VCtx,
323  /*ClosingParenLoc*/ Tokens[I].getLocation());
324 
325  } else if (/*No tokens within VAOPT*/
326  ResultToks.size() == VCtx.getNumberOfTokensPriorToVAOpt()) {
327  // Treat VAOPT as a placemarker token. Eat either the '##' before the
328  // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that
329  // hashhash was not a placemarker) or the '##'
330  // after VAOPT, but not both.
331 
332  if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {
333  ResultToks.pop_back();
334  } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {
335  ++I; // Skip the following hashhash.
336  }
337  } else {
338  // If there's a ## before the __VA_OPT__, we might have discovered
339  // that the __VA_OPT__ begins with a placeholder. We delay action on
340  // that to now to avoid messing up our stashed count of tokens before
341  // __VA_OPT__.
342  if (VCtx.beginsWithPlaceholder()) {
343  assert(VCtx.getNumberOfTokensPriorToVAOpt() > 0 &&
344  ResultToks.size() >= VCtx.getNumberOfTokensPriorToVAOpt() &&
345  ResultToks[VCtx.getNumberOfTokensPriorToVAOpt() - 1].is(
346  tok::hashhash) &&
347  "no token paste before __VA_OPT__");
348  ResultToks.erase(ResultToks.begin() +
349  VCtx.getNumberOfTokensPriorToVAOpt() - 1);
350  }
351  // If the expansion of __VA_OPT__ ends with a placeholder, eat any
352  // following '##' token.
353  if (VCtx.endsWithPlaceholder() && I + 1 != E &&
354  Tokens[I + 1].is(tok::hashhash)) {
355  ++I;
356  }
357  }
358  VCtx.reset();
359  // We processed __VA_OPT__'s closing paren (and the exit out of
360  // __VA_OPT__), so skip to the next token.
361  continue;
362  }
363  }
364 
365  // If we found the stringify operator, get the argument stringified. The
366  // preprocessor already verified that the following token is a macro
367  // parameter or __VA_OPT__ when the #define was lexed.
368 
369  if (CurTok.isOneOf(tok::hash, tok::hashat)) {
370  int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());
371  assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&
372  "Token following # is not an argument or __VA_OPT__!");
373 
374  if (ArgNo == -1) {
375  // Handle the __VA_OPT__ case.
376  VCtx.sawHashOrHashAtBefore(NextTokGetsSpace,
377  CurTok.is(tok::hashat));
378  continue;
379  }
380  // Else handle the simple argument case.
381  SourceLocation ExpansionLocStart =
382  getExpansionLocForMacroDefLoc(CurTok.getLocation());
383  SourceLocation ExpansionLocEnd =
384  getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());
385 
386  Token Res;
387  if (CurTok.is(tok::hash)) // Stringify
388  Res = ActualArgs->getStringifiedArgument(ArgNo, PP,
389  ExpansionLocStart,
390  ExpansionLocEnd);
391  else {
392  // 'charify': don't bother caching these.
393  Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo),
394  PP, true,
395  ExpansionLocStart,
396  ExpansionLocEnd);
397  }
399 
400  // The stringified/charified string leading space flag gets set to match
401  // the #/#@ operator.
402  if (NextTokGetsSpace)
404 
405  ResultToks.push_back(Res);
406  MadeChange = true;
407  ++I; // Skip arg name.
408  NextTokGetsSpace = false;
409  continue;
410  }
411 
412  // Find out if there is a paste (##) operator before or after the token.
413  bool NonEmptyPasteBefore =
414  !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
415  bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash);
416  bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash);
417  bool RParenAfter = I+1 != E && Tokens[I+1].is(tok::r_paren);
418 
419  assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&
420  "unexpected ## in ResultToks");
421 
422  // Otherwise, if this is not an argument token, just add the token to the
423  // output buffer.
424  IdentifierInfo *II = CurTok.getIdentifierInfo();
425  int ArgNo = II ? Macro->getParameterNum(II) : -1;
426  if (ArgNo == -1) {
427  // This isn't an argument, just add it.
428  ResultToks.push_back(CurTok);
429 
430  if (NextTokGetsSpace) {
431  ResultToks.back().setFlag(Token::LeadingSpace);
432  NextTokGetsSpace = false;
433  } else if (PasteBefore && !NonEmptyPasteBefore)
434  ResultToks.back().clearFlag(Token::LeadingSpace);
435 
436  continue;
437  }
438 
439  // An argument is expanded somehow, the result is different than the
440  // input.
441  MadeChange = true;
442 
443  // Otherwise, this is a use of the argument.
444 
445  // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
446  // are no trailing commas if __VA_ARGS__ is empty.
447  if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
448  MaybeRemoveCommaBeforeVaArgs(ResultToks,
449  /*HasPasteOperator=*/false,
450  Macro, ArgNo, PP))
451  continue;
452 
453  // If it is not the LHS/RHS of a ## operator, we must pre-expand the
454  // argument and substitute the expanded tokens into the result. This is
455  // C99 6.10.3.1p1.
456  if (!PasteBefore && !PasteAfter) {
457  const Token *ResultArgToks;
458 
459  // Only preexpand the argument if it could possibly need it. This
460  // avoids some work in common cases.
461  const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
462  if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
463  ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];
464  else
465  ResultArgToks = ArgTok; // Use non-preexpanded tokens.
466 
467  // If the arg token expanded into anything, append it.
468  if (ResultArgToks->isNot(tok::eof)) {
469  size_t FirstResult = ResultToks.size();
470  unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
471  ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
472 
473  // In Microsoft-compatibility mode, we follow MSVC's preprocessing
474  // behavior by not considering single commas from nested macro
475  // expansions as argument separators. Set a flag on the token so we can
476  // test for this later when the macro expansion is processed.
477  if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&
478  ResultToks.back().is(tok::comma))
479  ResultToks.back().setFlag(Token::IgnoredComma);
480 
481  // If the '##' came from expanding an argument, turn it into 'unknown'
482  // to avoid pasting.
483  for (Token &Tok : llvm::make_range(ResultToks.begin() + FirstResult,
484  ResultToks.end())) {
485  if (Tok.is(tok::hashhash))
486  Tok.setKind(tok::unknown);
487  }
488 
489  if(ExpandLocStart.isValid()) {
490  updateLocForMacroArgTokens(CurTok.getLocation(),
491  ResultToks.begin()+FirstResult,
492  ResultToks.end());
493  }
494 
495  // If any tokens were substituted from the argument, the whitespace
496  // before the first token should match the whitespace of the arg
497  // identifier.
498  ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
499  NextTokGetsSpace);
500  ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
501  NextTokGetsSpace = false;
502  } else {
503  // We're creating a placeholder token. Usually this doesn't matter,
504  // but it can affect paste behavior when at the start or end of a
505  // __VA_OPT__.
506  if (NonEmptyPasteBefore) {
507  // We're imagining a placeholder token is inserted here. If this is
508  // the first token in a __VA_OPT__ after a ##, delete the ##.
509  assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__");
511  }
512  if (RParenAfter)
514  }
515  continue;
516  }
517 
518  // Okay, we have a token that is either the LHS or RHS of a paste (##)
519  // argument. It gets substituted as its non-pre-expanded tokens.
520  const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
521  unsigned NumToks = MacroArgs::getArgLength(ArgToks);
522  if (NumToks) { // Not an empty argument?
523  bool VaArgsPseudoPaste = false;
524  // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned
525  // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when
526  // the expander tries to paste ',' with the first token of the __VA_ARGS__
527  // expansion.
528  if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
529  ResultToks[ResultToks.size()-2].is(tok::comma) &&
530  (unsigned)ArgNo == Macro->getNumParams()-1 &&
531  Macro->isVariadic()) {
532  VaArgsPseudoPaste = true;
533  // Remove the paste operator, report use of the extension.
534  PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
535  }
536 
537  ResultToks.append(ArgToks, ArgToks+NumToks);
538 
539  // If the '##' came from expanding an argument, turn it into 'unknown'
540  // to avoid pasting.
541  for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
542  ResultToks.end())) {
543  if (Tok.is(tok::hashhash))
544  Tok.setKind(tok::unknown);
545  }
546 
547  if (ExpandLocStart.isValid()) {
548  updateLocForMacroArgTokens(CurTok.getLocation(),
549  ResultToks.end()-NumToks, ResultToks.end());
550  }
551 
552  // Transfer the leading whitespace information from the token
553  // (the macro argument) onto the first token of the
554  // expansion. Note that we don't do this for the GNU
555  // pseudo-paste extension ", ## __VA_ARGS__".
556  if (!VaArgsPseudoPaste) {
557  ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,
558  false);
559  ResultToks[ResultToks.size() - NumToks].setFlagValue(
560  Token::LeadingSpace, NextTokGetsSpace);
561  }
562 
563  NextTokGetsSpace = false;
564  continue;
565  }
566 
567  // If an empty argument is on the LHS or RHS of a paste, the standard (C99
568  // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We
569  // implement this by eating ## operators when a LHS or RHS expands to
570  // empty.
571  if (PasteAfter) {
572  // Discard the argument token and skip (don't copy to the expansion
573  // buffer) the paste operator after it.
574  ++I;
575  continue;
576  }
577 
578  if (RParenAfter)
580 
581  // If this is on the RHS of a paste operator, we've already copied the
582  // paste operator to the ResultToks list, unless the LHS was empty too.
583  // Remove it.
584  assert(PasteBefore);
585  if (NonEmptyPasteBefore) {
586  assert(ResultToks.back().is(tok::hashhash));
587  // Do not remove the paste operator if it is the one before __VA_OPT__
588  // (and we are still processing tokens within VA_OPT). We handle the case
589  // of removing the paste operator if __VA_OPT__ reduces to the notional
590  // placemarker above when we encounter the closing paren of VA_OPT.
591  if (!VCtx.isInVAOpt() ||
592  ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt())
593  ResultToks.pop_back();
594  else
596  }
597 
598  // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
599  // and if the macro had at least one real argument, and if the token before
600  // the ## was a comma, remove the comma. This is a GCC extension which is
601  // disabled when using -std=c99.
602  if (ActualArgs->isVarargsElidedUse())
603  MaybeRemoveCommaBeforeVaArgs(ResultToks,
604  /*HasPasteOperator=*/true,
605  Macro, ArgNo, PP);
606  }
607 
608  // If anything changed, install this as the new Tokens list.
609  if (MadeChange) {
610  assert(!OwnsTokens && "This would leak if we already own the token list");
611  // This is deleted in the dtor.
612  NumTokens = ResultToks.size();
613  // The tokens will be added to Preprocessor's cache and will be removed
614  // when this TokenLexer finishes lexing them.
615  Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
616 
617  // The preprocessor cache of macro expanded tokens owns these tokens,not us.
618  OwnsTokens = false;
619  }
620 }
621 
622 /// Checks if two tokens form wide string literal.
623 static bool isWideStringLiteralFromMacro(const Token &FirstTok,
624  const Token &SecondTok) {
625  return FirstTok.is(tok::identifier) &&
626  FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral() &&
627  SecondTok.stringifiedInMacro();
628 }
629 
630 /// Lex - Lex and return a token from this macro stream.
632  // Lexing off the end of the macro, pop this macro off the expansion stack.
633  if (isAtEnd()) {
634  // If this is a macro (not a token stream), mark the macro enabled now
635  // that it is no longer being expanded.
636  if (Macro) Macro->EnableMacro();
637 
638  Tok.startToken();
639  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
640  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);
641  if (CurTokenIdx == 0)
643  return PP.HandleEndOfTokenLexer(Tok);
644  }
645 
647 
648  // If this is the first token of the expanded result, we inherit spacing
649  // properties later.
650  bool isFirstToken = CurTokenIdx == 0;
651 
652  // Get the next token to return.
653  Tok = Tokens[CurTokenIdx++];
654  if (IsReinject)
656 
657  bool TokenIsFromPaste = false;
658 
659  // If this token is followed by a token paste (##) operator, paste the tokens!
660  // Note that ## is a normal token when not expanding a macro.
661  if (!isAtEnd() && Macro &&
662  (Tokens[CurTokenIdx].is(tok::hashhash) ||
663  // Special processing of L#x macros in -fms-compatibility mode.
664  // Microsoft compiler is able to form a wide string literal from
665  // 'L#macro_arg' construct in a function-like macro.
666  (PP.getLangOpts().MSVCCompat &&
667  isWideStringLiteralFromMacro(Tok, Tokens[CurTokenIdx])))) {
668  // When handling the microsoft /##/ extension, the final token is
669  // returned by pasteTokens, not the pasted token.
670  if (pasteTokens(Tok))
671  return true;
672 
673  TokenIsFromPaste = true;
674  }
675 
676  // The token's current location indicate where the token was lexed from. We
677  // need this information to compute the spelling of the token, but any
678  // diagnostics for the expanded token should appear as if they came from
679  // ExpansionLoc. Pull this information together into a new SourceLocation
680  // that captures all of this.
681  if (ExpandLocStart.isValid() && // Don't do this for token streams.
682  // Check that the token's location was not already set properly.
683  SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {
684  SourceLocation instLoc;
685  if (Tok.is(tok::comment)) {
686  instLoc = SM.createExpansionLoc(Tok.getLocation(),
687  ExpandLocStart,
688  ExpandLocEnd,
689  Tok.getLength());
690  } else {
691  instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());
692  }
693 
694  Tok.setLocation(instLoc);
695  }
696 
697  // If this is the first token, set the lexical properties of the token to
698  // match the lexical properties of the macro identifier.
699  if (isFirstToken) {
700  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
701  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
702  } else {
703  // If this is not the first token, we may still need to pass through
704  // leading whitespace if we've expanded a macro.
705  if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);
706  if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);
707  }
708  AtStartOfLine = false;
709  HasLeadingSpace = false;
710 
711  // Handle recursive expansion!
712  if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) {
713  // Change the kind of this identifier to the appropriate token kind, e.g.
714  // turning "for" into a keyword.
715  IdentifierInfo *II = Tok.getIdentifierInfo();
716  Tok.setKind(II->getTokenID());
717 
718  // If this identifier was poisoned and from a paste, emit an error. This
719  // won't be handled by Preprocessor::HandleIdentifier because this is coming
720  // from a macro expansion.
721  if (II->isPoisoned() && TokenIsFromPaste) {
722  PP.HandlePoisonedIdentifier(Tok);
723  }
724 
725  if (!DisableMacroExpansion && II->isHandleIdentifierCase())
726  return PP.HandleIdentifier(Tok);
727  }
728 
729  // Otherwise, return a normal token.
730  return true;
731 }
732 
733 bool TokenLexer::pasteTokens(Token &Tok) {
734  return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx);
735 }
736 
737 /// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ##
738 /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there
739 /// are more ## after it, chomp them iteratively. Return the result as LHSTok.
740 /// If this returns true, the caller should immediately return the token.
741 bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
742  unsigned int &CurIdx) {
743  assert(CurIdx > 0 && "## can not be the first token within tokens");
744  assert((TokenStream[CurIdx].is(tok::hashhash) ||
745  (PP.getLangOpts().MSVCCompat &&
746  isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) &&
747  "Token at this Index must be ## or part of the MSVC 'L "
748  "#macro-arg' pasting pair");
749 
750  // MSVC: If previous token was pasted, this must be a recovery from an invalid
751  // paste operation. Ignore spaces before this token to mimic MSVC output.
752  // Required for generating valid UUID strings in some MS headers.
753  if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) &&
754  TokenStream[CurIdx - 2].is(tok::hashhash))
756 
757  SmallString<128> Buffer;
758  const char *ResultTokStrPtr = nullptr;
759  SourceLocation StartLoc = LHSTok.getLocation();
760  SourceLocation PasteOpLoc;
761 
762  auto IsAtEnd = [&TokenStream, &CurIdx] {
763  return TokenStream.size() == CurIdx;
764  };
765 
766  do {
767  // Consume the ## operator if any.
768  PasteOpLoc = TokenStream[CurIdx].getLocation();
769  if (TokenStream[CurIdx].is(tok::hashhash))
770  ++CurIdx;
771  assert(!IsAtEnd() && "No token on the RHS of a paste operator!");
772 
773  // Get the RHS token.
774  const Token &RHS = TokenStream[CurIdx];
775 
776  // Allocate space for the result token. This is guaranteed to be enough for
777  // the two tokens.
778  Buffer.resize(LHSTok.getLength() + RHS.getLength());
779 
780  // Get the spelling of the LHS token in Buffer.
781  const char *BufPtr = &Buffer[0];
782  bool Invalid = false;
783  unsigned LHSLen = PP.getSpelling(LHSTok, BufPtr, &Invalid);
784  if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer!
785  memcpy(&Buffer[0], BufPtr, LHSLen);
786  if (Invalid)
787  return true;
788 
789  BufPtr = Buffer.data() + LHSLen;
790  unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid);
791  if (Invalid)
792  return true;
793  if (RHSLen && BufPtr != &Buffer[LHSLen])
794  // Really, we want the chars in Buffer!
795  memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
796 
797  // Trim excess space.
798  Buffer.resize(LHSLen+RHSLen);
799 
800  // Plop the pasted result (including the trailing newline and null) into a
801  // scratch buffer where we can lex it.
802  Token ResultTokTmp;
803  ResultTokTmp.startToken();
804 
805  // Claim that the tmp token is a string_literal so that we can get the
806  // character pointer back from CreateString in getLiteralData().
807  ResultTokTmp.setKind(tok::string_literal);
808  PP.CreateString(Buffer, ResultTokTmp);
809  SourceLocation ResultTokLoc = ResultTokTmp.getLocation();
810  ResultTokStrPtr = ResultTokTmp.getLiteralData();
811 
812  // Lex the resultant pasted token into Result.
813  Token Result;
814 
815  if (LHSTok.isAnyIdentifier() && RHS.isAnyIdentifier()) {
816  // Common paste case: identifier+identifier = identifier. Avoid creating
817  // a lexer and other overhead.
818  PP.IncrementPasteCounter(true);
819  Result.startToken();
820  Result.setKind(tok::raw_identifier);
821  Result.setRawIdentifierData(ResultTokStrPtr);
822  Result.setLocation(ResultTokLoc);
823  Result.setLength(LHSLen+RHSLen);
824  } else {
825  PP.IncrementPasteCounter(false);
826 
827  assert(ResultTokLoc.isFileID() &&
828  "Should be a raw location into scratch buffer");
829  SourceManager &SourceMgr = PP.getSourceManager();
830  FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
831 
832  bool Invalid = false;
833  const char *ScratchBufStart
834  = SourceMgr.getBufferData(LocFileID, &Invalid).data();
835  if (Invalid)
836  return false;
837 
838  // Make a lexer to lex this string from. Lex just this one token.
839  // Make a lexer object so that we lex and expand the paste result.
840  Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
841  PP.getLangOpts(), ScratchBufStart,
842  ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
843 
844  // Lex a token in raw mode. This way it won't look up identifiers
845  // automatically, lexing off the end will return an eof token, and
846  // warnings are disabled. This returns true if the result token is the
847  // entire buffer.
848  bool isInvalid = !TL.LexFromRawLexer(Result);
849 
850  // If we got an EOF token, we didn't form even ONE token. For example, we
851  // did "/ ## /" to get "//".
852  isInvalid |= Result.is(tok::eof);
853 
854  // If pasting the two tokens didn't form a full new token, this is an
855  // error. This occurs with "x ## +" and other stuff. Return with LHSTok
856  // unmodified and with RHS as the next token to lex.
857  if (isInvalid) {
858  // Explicitly convert the token location to have proper expansion
859  // information so that the user knows where it came from.
861  SourceLocation Loc =
862  SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);
863 
864  // Test for the Microsoft extension of /##/ turning into // here on the
865  // error path.
866  if (PP.getLangOpts().MicrosoftExt && LHSTok.is(tok::slash) &&
867  RHS.is(tok::slash)) {
868  HandleMicrosoftCommentPaste(LHSTok, Loc);
869  return true;
870  }
871 
872  // Do not emit the error when preprocessing assembler code.
873  if (!PP.getLangOpts().AsmPreprocessor) {
874  // If we're in microsoft extensions mode, downgrade this from a hard
875  // error to an extension that defaults to an error. This allows
876  // disabling it.
877  PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms
878  : diag::err_pp_bad_paste)
879  << Buffer;
880  }
881 
882  // An error has occurred so exit loop.
883  break;
884  }
885 
886  // Turn ## into 'unknown' to avoid # ## # from looking like a paste
887  // operator.
888  if (Result.is(tok::hashhash))
889  Result.setKind(tok::unknown);
890  }
891 
892  // Transfer properties of the LHS over the Result.
895 
896  // Finally, replace LHS with the result, consume the RHS, and iterate.
897  ++CurIdx;
898  LHSTok = Result;
899  } while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));
900 
901  SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation();
902 
903  // The token's current location indicate where the token was lexed from. We
904  // need this information to compute the spelling of the token, but any
905  // diagnostics for the expanded token should appear as if the token was
906  // expanded from the full ## expression. Pull this information together into
907  // a new SourceLocation that captures all of this.
909  if (StartLoc.isFileID())
910  StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
911  if (EndLoc.isFileID())
912  EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
913  FileID MacroFID = SM.getFileID(MacroExpansionStart);
914  while (SM.getFileID(StartLoc) != MacroFID)
915  StartLoc = SM.getImmediateExpansionRange(StartLoc).getBegin();
916  while (SM.getFileID(EndLoc) != MacroFID)
917  EndLoc = SM.getImmediateExpansionRange(EndLoc).getEnd();
918 
919  LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc,
920  LHSTok.getLength()));
921 
922  // Now that we got the result token, it will be subject to expansion. Since
923  // token pasting re-lexes the result token in raw mode, identifier information
924  // isn't looked up. As such, if the result is an identifier, look up id info.
925  if (LHSTok.is(tok::raw_identifier)) {
926  // Look up the identifier info for the token. We disabled identifier lookup
927  // by saying we're skipping contents, so we need to do this manually.
928  PP.LookUpIdentifierInfo(LHSTok);
929  }
930  return false;
931 }
932 
933 /// isNextTokenLParen - If the next token lexed will pop this macro off the
934 /// expansion stack, return 2. If the next unexpanded token is a '(', return
935 /// 1, otherwise return 0.
937  // Out of tokens?
938  if (isAtEnd())
939  return 2;
940  return Tokens[CurTokenIdx].is(tok::l_paren);
941 }
942 
943 /// isParsingPreprocessorDirective - Return true if we are in the middle of a
944 /// preprocessor directive.
946  return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd();
947 }
948 
949 /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
950 /// together to form a comment that comments out everything in the current
951 /// macro, other active macros, and anything left on the current physical
952 /// source line of the expanded buffer. Handle this by returning the
953 /// first token on the next line.
954 void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {
955  PP.Diag(OpLoc, diag::ext_comment_paste_microsoft);
956 
957  // We 'comment out' the rest of this macro by just ignoring the rest of the
958  // tokens that have not been lexed yet, if any.
959 
960  // Since this must be a macro, mark the macro enabled now that it is no longer
961  // being expanded.
962  assert(Macro && "Token streams can't paste comments");
963  Macro->EnableMacro();
964 
966 }
967 
968 /// If \arg loc is a file ID and points inside the current macro
969 /// definition, returns the appropriate source location pointing at the
970 /// macro expansion source location entry, otherwise it returns an invalid
971 /// SourceLocation.
973 TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {
974  assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&
975  "Not appropriate for token streams");
976  assert(loc.isValid() && loc.isFileID());
977 
979  assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
980  "Expected loc to come from the macro definition");
981 
982  unsigned relativeOffset = 0;
983  SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);
984  return MacroExpansionStart.getLocWithOffset(relativeOffset);
985 }
986 
987 /// Finds the tokens that are consecutive (from the same FileID)
988 /// creates a single SLocEntry, and assigns SourceLocations to each token that
989 /// point to that SLocEntry. e.g for
990 /// assert(foo == bar);
991 /// There will be a single SLocEntry for the "foo == bar" chunk and locations
992 /// for the 'foo', '==', 'bar' tokens will point inside that chunk.
993 ///
994 /// \arg begin_tokens will be updated to a position past all the found
995 /// consecutive tokens.
997  SourceLocation InstLoc,
998  Token *&begin_tokens,
999  Token * end_tokens) {
1000  assert(begin_tokens < end_tokens);
1001 
1002  SourceLocation FirstLoc = begin_tokens->getLocation();
1003  SourceLocation CurLoc = FirstLoc;
1004 
1005  // Compare the source location offset of tokens and group together tokens that
1006  // are close, even if their locations point to different FileIDs. e.g.
1007  //
1008  // |bar | foo | cake | (3 tokens from 3 consecutive FileIDs)
1009  // ^ ^
1010  // |bar foo cake| (one SLocEntry chunk for all tokens)
1011  //
1012  // we can perform this "merge" since the token's spelling location depends
1013  // on the relative offset.
1014 
1015  Token *NextTok = begin_tokens + 1;
1016  for (; NextTok < end_tokens; ++NextTok) {
1017  SourceLocation NextLoc = NextTok->getLocation();
1018  if (CurLoc.isFileID() != NextLoc.isFileID())
1019  break; // Token from different kind of FileID.
1020 
1021  int RelOffs;
1022  if (!SM.isInSameSLocAddrSpace(CurLoc, NextLoc, &RelOffs))
1023  break; // Token from different local/loaded location.
1024  // Check that token is not before the previous token or more than 50
1025  // "characters" away.
1026  if (RelOffs < 0 || RelOffs > 50)
1027  break;
1028 
1029  if (CurLoc.isMacroID() && !SM.isWrittenInSameFile(CurLoc, NextLoc))
1030  break; // Token from a different macro.
1031 
1032  CurLoc = NextLoc;
1033  }
1034 
1035  // For the consecutive tokens, find the length of the SLocEntry to contain
1036  // all of them.
1037  Token &LastConsecutiveTok = *(NextTok-1);
1038  int LastRelOffs = 0;
1039  SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(),
1040  &LastRelOffs);
1041  unsigned FullLength = LastRelOffs + LastConsecutiveTok.getLength();
1042 
1043  // Create a macro expansion SLocEntry that will "contain" all of the tokens.
1044  SourceLocation Expansion =
1045  SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength);
1046 
1047  // Change the location of the tokens from the spelling location to the new
1048  // expanded location.
1049  for (; begin_tokens < NextTok; ++begin_tokens) {
1050  Token &Tok = *begin_tokens;
1051  int RelOffs = 0;
1052  SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs);
1053  Tok.setLocation(Expansion.getLocWithOffset(RelOffs));
1054  }
1055 }
1056 
1057 /// Creates SLocEntries and updates the locations of macro argument
1058 /// tokens to their new expanded locations.
1059 ///
1060 /// \param ArgIdSpellLoc the location of the macro argument id inside the macro
1061 /// definition.
1062 void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
1063  Token *begin_tokens,
1064  Token *end_tokens) {
1066 
1067  SourceLocation InstLoc =
1068  getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
1069 
1070  while (begin_tokens < end_tokens) {
1071  // If there's only one token just create a SLocEntry for it.
1072  if (end_tokens - begin_tokens == 1) {
1073  Token &Tok = *begin_tokens;
1075  InstLoc,
1076  Tok.getLength()));
1077  return;
1078  }
1079 
1080  updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens);
1081  }
1082 }
1083 
1084 void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
1085  AtStartOfLine = Result.isAtStartOfLine();
1086  HasLeadingSpace = Result.hasLeadingSpace();
1087 }
bool isWrittenInSameFile(SourceLocation Loc1, SourceLocation Loc2) const
Returns true if the spelling locations for both SourceLocations are part of the same file buffer...
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:76
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void DisableMacro()
Definition: MacroInfo.h:265
void sawOpeningParen(SourceLocation LParenLoc)
Call this function each time an lparen is seen.
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Definition: Token.h:259
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
ArgNeedsPreexpansion - If we can prove that the argument won&#39;t be affected by pre-expansion, return false.
Definition: MacroArgs.cpp:148
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:97
bool sawClosingParen()
Call this function each time an rparen is seen.
Defines the SourceManager interface.
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation InstLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry, and assigns SourceLocations to each token that point to that SLocEntry.
Definition: TokenLexer.cpp:996
A class for tracking whether we&#39;re inside a VA_OPT during a traversal of the tokens of a macro during...
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool isBeforeInSLocAddrSpace(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the "source location address space".
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:236
bool isVarargsElidedUse() const
isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...
Definition: MacroArgs.h:112
bool isVariadic() const
Definition: MacroInfo.h:207
bool isInSameSLocAddrSpace(SourceLocation LHS, SourceLocation RHS, int *RelativeOffset) const
Return true if both LHS and RHS are in the local source location address space or the loaded one...
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
Definition: Token.h:109
unsigned getNextLocalOffset() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:120
tokens_iterator tokens_begin() const
Definition: MacroInfo.h:242
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:115
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
One of these records is kept for each identifier that is lexed.
SourceLocation getBegin() const
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:209
void sawVAOptFollowedByOpeningParens(const SourceLocation VAOptLoc, const unsigned int NumPriorTokens)
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
void setKind(tok::TokenKind K)
Definition: Token.h:93
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:903
bool Lex(Token &Tok)
Lex and return a token from this macro stream.
Definition: TokenLexer.cpp:631
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
const FormatToken & Tok
SourceLocation getExpansionLoc(SourceLocation Loc) const
Given a SourceLocation object Loc, return the expansion location referenced by the ID...
Defines the Diagnostic-related interfaces.
void destroy(Preprocessor &PP)
destroy - Destroy and deallocate the memory for this object.
Definition: MacroArgs.cpp:78
void Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, MacroArgs *Actuals)
Initialize this TokenLexer to expand from the specified macro with the specified argument information...
Definition: TokenLexer.cpp:38
const Token & getStringifiedArgument(unsigned ArgNo, Preprocessor &PP, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
getStringifiedArgument - Compute, cache, and return the specified argument that has been &#39;stringified...
Definition: MacroArgs.cpp:313
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
Definition: TokenLexer.cpp:623
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...
Definition: MacroArgs.cpp:210
unsigned getNumParams() const
Definition: MacroInfo.h:182
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
bool invokedWithVariadicArgument(const MacroInfo *const MI, Preprocessor &PP)
Returns true if the macro was defined with a variadic (ellipsis) parameter AND was invoked with at le...
Definition: MacroArgs.cpp:138
Defines the clang::LangOptions interface.
bool isInSLocAddrSpace(SourceLocation Loc, SourceLocation Start, unsigned Length, unsigned *RelativeOffset=nullptr) const
Returns true if Loc is inside the [Start, +Length) chunk of the source location address space...
SourceLocation getVAOptLoc() const
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
SourceLocation createMacroArgExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLoc, unsigned TokLength)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:126
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
CharSourceRange getImmediateExpansionRange(SourceLocation Loc) const
Return the start/end of the expansion information for an expansion location.
The result type of a method or function.
const SourceManager & SM
Definition: Format.cpp:1570
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
SourceManager & getSourceManager() const
Definition: Preprocessor.h:907
bool isVAOptToken(const Token &T) const
Encodes a location in the source.
void setLength(unsigned Len)
Definition: Token.h:135
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, bool ExpansionIsTokenRange=true, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:312
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:179
bool isInVAOpt() const
Returns true if we have seen the VA_OPT and &#39;(&#39; but before having seen the matching &#39;)&#39;...
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:268
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:98
static bool isInvalid(LocType Loc, bool *Invalid)
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
bool isFunctionLike() const
Definition: MacroInfo.h:199
const std::vector< Token > & getPreExpArgument(unsigned Arg, Preprocessor &PP)
getPreExpArgument - Return the pre-expanded form of the specified argument.
Definition: MacroArgs.cpp:163
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:99
tokens_iterator tokens_end() const
Definition: MacroInfo.h:243
static unsigned getArgLength(const Token *ArgPtr)
getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens...
Definition: MacroArgs.cpp:110
void sawHashOrHashAtBefore(const bool HasLeadingSpace, const bool IsHashAt)
unsigned getLength() const
Definition: Token.h:129
bool isMacroID() const
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:217
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
int getParameterNum(const IdentifierInfo *Arg) const
Return the parameter number of the specified identifier, or -1 if the identifier is not a formal para...
Definition: MacroInfo.h:189
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
unsigned int getNumberOfTokensPriorToVAOpt() const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token&#39;s location and length to ...
SourceLocation getEnd() const
unsigned isNextTokenLParen() const
If the next token lexed will pop this macro off the expansion stack, return 2.
Definition: TokenLexer.cpp:936
Defines the clang::TokenKind enum and support functions.
unsigned getDefinitionLength(const SourceManager &SM) const
Get length in characters of the macro definition.
Definition: MacroInfo.h:132
Defines the clang::SourceLocation class and associated facilities.
bool stringifiedInMacro() const
Returns true if this token is formed by macro by stringizing or charizing operator.
Definition: Token.h:300
void setLocation(SourceLocation L)
Definition: Token.h:134
void EnableMacro()
Definition: MacroInfo.h:260
void clearFlag(TokenFlags Flag)
Unset the specified flag.
Definition: Token.h:246
bool isParsingPreprocessorDirective() const
isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive...
Definition: TokenLexer.cpp:945
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:272
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
This class handles loading and caching of source files into memory.
const Token * getUnexpArgument(unsigned Arg) const
getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...
Definition: MacroArgs.cpp:120
void startToken()
Reset all flags to cleared.
Definition: Token.h:171
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:124