clang  16.0.0git
TokenLexer.cpp
Go to the documentation of this file.
1 //===- TokenLexer.cpp - Lex from a token stream ---------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the TokenLexer interface.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "clang/Lex/TokenLexer.h"
14 #include "clang/Basic/Diagnostic.h"
19 #include "clang/Basic/TokenKinds.h"
21 #include "clang/Lex/Lexer.h"
22 #include "clang/Lex/MacroArgs.h"
23 #include "clang/Lex/MacroInfo.h"
24 #include "clang/Lex/Preprocessor.h"
25 #include "clang/Lex/Token.h"
27 #include "llvm/ADT/ArrayRef.h"
28 #include "llvm/ADT/STLExtras.h"
29 #include "llvm/ADT/SmallString.h"
30 #include "llvm/ADT/SmallVector.h"
31 #include "llvm/ADT/iterator_range.h"
32 #include <cassert>
33 #include <cstring>
34 
35 using namespace clang;
36 
37 /// Create a TokenLexer for the specified macro with the specified actual
38 /// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
40  MacroArgs *Actuals) {
41  // If the client is reusing a TokenLexer, make sure to free any memory
42  // associated with it.
43  destroy();
44 
45  Macro = MI;
46  ActualArgs = Actuals;
47  CurTokenIdx = 0;
48 
49  ExpandLocStart = Tok.getLocation();
50  ExpandLocEnd = ELEnd;
51  AtStartOfLine = Tok.isAtStartOfLine();
52  HasLeadingSpace = Tok.hasLeadingSpace();
53  NextTokGetsSpace = false;
54  Tokens = &*Macro->tokens_begin();
55  OwnsTokens = false;
56  DisableMacroExpansion = false;
57  IsReinject = false;
58  NumTokens = Macro->tokens_end()-Macro->tokens_begin();
59  MacroExpansionStart = SourceLocation();
60 
62  MacroStartSLocOffset = SM.getNextLocalOffset();
63 
64  if (NumTokens > 0) {
65  assert(Tokens[0].getLocation().isValid());
66  assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
67  "Macro defined in macro?");
68  assert(ExpandLocStart.isValid());
69 
70  // Reserve a source location entry chunk for the length of the macro
71  // definition. Tokens that get lexed directly from the definition will
72  // have their locations pointing inside this chunk. This is to avoid
73  // creating separate source location entries for each token.
74  MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());
75  MacroDefLength = Macro->getDefinitionLength(SM);
76  MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,
77  ExpandLocStart,
78  ExpandLocEnd,
79  MacroDefLength);
80  }
81 
82  // If this is a function-like macro, expand the arguments and change
83  // Tokens to point to the expanded tokens.
84  if (Macro->isFunctionLike() && Macro->getNumParams())
85  ExpandFunctionArguments();
86 
87  // Mark the macro as currently disabled, so that it is not recursively
88  // expanded. The macro must be disabled only after argument pre-expansion of
89  // function-like macro arguments occurs.
90  Macro->DisableMacro();
91 }
92 
93 /// Create a TokenLexer for the specified token stream. This does not
94 /// take ownership of the specified token vector.
95 void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
96  bool disableMacroExpansion, bool ownsTokens,
97  bool isReinject) {
98  assert(!isReinject || disableMacroExpansion);
99  // If the client is reusing a TokenLexer, make sure to free any memory
100  // associated with it.
101  destroy();
102 
103  Macro = nullptr;
104  ActualArgs = nullptr;
105  Tokens = TokArray;
106  OwnsTokens = ownsTokens;
107  DisableMacroExpansion = disableMacroExpansion;
108  IsReinject = isReinject;
109  NumTokens = NumToks;
110  CurTokenIdx = 0;
111  ExpandLocStart = ExpandLocEnd = SourceLocation();
112  AtStartOfLine = false;
113  HasLeadingSpace = false;
114  NextTokGetsSpace = false;
115  MacroExpansionStart = SourceLocation();
116 
117  // Set HasLeadingSpace/AtStartOfLine so that the first token will be
118  // returned unmodified.
119  if (NumToks != 0) {
120  AtStartOfLine = TokArray[0].isAtStartOfLine();
121  HasLeadingSpace = TokArray[0].hasLeadingSpace();
122  }
123 }
124 
125 void TokenLexer::destroy() {
126  // If this was a function-like macro that actually uses its arguments, delete
127  // the expanded tokens.
128  if (OwnsTokens) {
129  delete [] Tokens;
130  Tokens = nullptr;
131  OwnsTokens = false;
132  }
133 
134  // TokenLexer owns its formal arguments.
135  if (ActualArgs) ActualArgs->destroy(PP);
136 }
137 
138 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
139  SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
140  unsigned MacroArgNo, Preprocessor &PP) {
141  // Is the macro argument __VA_ARGS__?
142  if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1)
143  return false;
144 
145  // In Microsoft-compatibility mode, a comma is removed in the expansion
146  // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is
147  // not supported by gcc.
148  if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)
149  return false;
150 
151  // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if
152  // __VA_ARGS__ is empty, but not in strict C99 mode where there are no
153  // named arguments, where it remains. In all other modes, including C99
154  // with GNU extensions, it is removed regardless of named arguments.
155  // Microsoft also appears to support this extension, unofficially.
156  if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode
157  && Macro->getNumParams() < 2)
158  return false;
159 
160  // Is a comma available to be removed?
161  if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
162  return false;
163 
164  // Issue an extension diagnostic for the paste operator.
165  if (HasPasteOperator)
166  PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
167 
168  // Remove the comma.
169  ResultToks.pop_back();
170 
171  if (!ResultToks.empty()) {
172  // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
173  // then removal of the comma should produce a placemarker token (in C99
174  // terms) which we model by popping off the previous ##, giving us a plain
175  // "X" when __VA_ARGS__ is empty.
176  if (ResultToks.back().is(tok::hashhash))
177  ResultToks.pop_back();
178 
179  // Remember that this comma was elided.
180  ResultToks.back().setFlag(Token::CommaAfterElided);
181  }
182 
183  // Never add a space, even if the comma, ##, or arg had a space.
184  NextTokGetsSpace = false;
185  return true;
186 }
187 
188 void TokenLexer::stringifyVAOPTContents(
189  SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,
190  const SourceLocation VAOPTClosingParenLoc) {
191  const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt();
192  const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
193  Token *const VAOPTTokens =
194  NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr;
195 
196  SmallVector<Token, 64> ConcatenatedVAOPTResultToks;
197  // FIXME: Should we keep track within VCtx that we did or didnot
198  // encounter pasting - and only then perform this loop.
199 
200  // Perform token pasting (concatenation) prior to stringization.
201  for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
202  ++CurTokenIdx) {
203  if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {
204  assert(CurTokenIdx != 0 &&
205  "Can not have __VAOPT__ contents begin with a ##");
206  Token &LHS = VAOPTTokens[CurTokenIdx - 1];
207  pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens),
208  CurTokenIdx);
209  // Replace the token prior to the first ## in this iteration.
210  ConcatenatedVAOPTResultToks.back() = LHS;
211  if (CurTokenIdx == NumVAOptTokens)
212  break;
213  }
214  ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);
215  }
216 
217  ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok());
218  // Get the SourceLocation that represents the start location within
219  // the macro definition that marks where this string is substituted
220  // into: i.e. the __VA_OPT__ and the ')' within the spelling of the
221  // macro definition, and use it to indicate that the stringified token
222  // was generated from that location.
223  const SourceLocation ExpansionLocStartWithinMacro =
224  getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc());
225  const SourceLocation ExpansionLocEndWithinMacro =
226  getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);
227 
228  Token StringifiedVAOPT = MacroArgs::StringifyArgument(
229  &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() /*Charify*/,
230  ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);
231 
233  StringifiedVAOPT.setFlag(Token::LeadingSpace);
234 
235  StringifiedVAOPT.setFlag(Token::StringifiedInMacro);
236  // Resize (shrink) the token stream to just capture this stringified token.
237  ResultToks.resize(NumToksPriorToVAOpt + 1);
238  ResultToks.back() = StringifiedVAOPT;
239 }
240 
241 /// Expand the arguments of a function-like macro so that we can quickly
242 /// return preexpanded tokens from Tokens.
243 void TokenLexer::ExpandFunctionArguments() {
244  SmallVector<Token, 128> ResultToks;
245 
246  // Loop through 'Tokens', expanding them into ResultToks. Keep
247  // track of whether we change anything. If not, no need to keep them. If so,
248  // we install the newly expanded sequence as the new 'Tokens' list.
249  bool MadeChange = false;
250 
251  Optional<bool> CalledWithVariadicArguments;
252 
253  VAOptExpansionContext VCtx(PP);
254 
255  for (unsigned I = 0, E = NumTokens; I != E; ++I) {
256  const Token &CurTok = Tokens[I];
257  // We don't want a space for the next token after a paste
258  // operator. In valid code, the token will get smooshed onto the
259  // preceding one anyway. In assembler-with-cpp mode, invalid
260  // pastes are allowed through: in this case, we do not want the
261  // extra whitespace to be added. For example, we want ". ## foo"
262  // -> ".foo" not ". foo".
263  if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace())
264  NextTokGetsSpace = true;
265 
266  if (VCtx.isVAOptToken(CurTok)) {
267  MadeChange = true;
268  assert(Tokens[I + 1].is(tok::l_paren) &&
269  "__VA_OPT__ must be followed by '('");
270 
271  ++I; // Skip the l_paren
273  ResultToks.size());
274 
275  continue;
276  }
277 
278  // We have entered into the __VA_OPT__ context, so handle tokens
279  // appropriately.
280  if (VCtx.isInVAOpt()) {
281  // If we are about to process a token that is either an argument to
282  // __VA_OPT__ or its closing rparen, then:
283  // 1) If the token is the closing rparen that exits us out of __VA_OPT__,
284  // perform any necessary stringification or placemarker processing,
285  // and/or skip to the next token.
286  // 2) else if macro was invoked without variadic arguments skip this
287  // token.
288  // 3) else (macro was invoked with variadic arguments) process the token
289  // normally.
290 
291  if (Tokens[I].is(tok::l_paren))
292  VCtx.sawOpeningParen(Tokens[I].getLocation());
293  // Continue skipping tokens within __VA_OPT__ if the macro was not
294  // called with variadic arguments, else let the rest of the loop handle
295  // this token. Note sawClosingParen() returns true only if the r_paren matches
296  // the closing r_paren of the __VA_OPT__.
297  if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) {
298  // Lazily expand __VA_ARGS__ when we see the first __VA_OPT__.
299  if (!CalledWithVariadicArguments) {
300  CalledWithVariadicArguments =
301  ActualArgs->invokedWithVariadicArgument(Macro, PP);
302  }
303  if (!*CalledWithVariadicArguments) {
304  // Skip this token.
305  continue;
306  }
307  // ... else the macro was called with variadic arguments, and we do not
308  // have a closing rparen - so process this token normally.
309  } else {
310  // Current token is the closing r_paren which marks the end of the
311  // __VA_OPT__ invocation, so handle any place-marker pasting (if
312  // empty) by removing hashhash either before (if exists) or after. And
313  // also stringify the entire contents if VAOPT was preceded by a hash,
314  // but do so only after any token concatenation that needs to occur
315  // within the contents of VAOPT.
316 
317  if (VCtx.hasStringifyOrCharifyBefore()) {
318  // Replace all the tokens just added from within VAOPT into a single
319  // stringified token. This requires token-pasting to eagerly occur
320  // within these tokens. If either the contents of VAOPT were empty
321  // or the macro wasn't called with any variadic arguments, the result
322  // is a token that represents an empty string.
323  stringifyVAOPTContents(ResultToks, VCtx,
324  /*ClosingParenLoc*/ Tokens[I].getLocation());
325 
326  } else if (/*No tokens within VAOPT*/
327  ResultToks.size() == VCtx.getNumberOfTokensPriorToVAOpt()) {
328  // Treat VAOPT as a placemarker token. Eat either the '##' before the
329  // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that
330  // hashhash was not a placemarker) or the '##'
331  // after VAOPT, but not both.
332 
333  if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {
334  ResultToks.pop_back();
335  } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {
336  ++I; // Skip the following hashhash.
337  }
338  } else {
339  // If there's a ## before the __VA_OPT__, we might have discovered
340  // that the __VA_OPT__ begins with a placeholder. We delay action on
341  // that to now to avoid messing up our stashed count of tokens before
342  // __VA_OPT__.
343  if (VCtx.beginsWithPlaceholder()) {
344  assert(VCtx.getNumberOfTokensPriorToVAOpt() > 0 &&
345  ResultToks.size() >= VCtx.getNumberOfTokensPriorToVAOpt() &&
346  ResultToks[VCtx.getNumberOfTokensPriorToVAOpt() - 1].is(
347  tok::hashhash) &&
348  "no token paste before __VA_OPT__");
349  ResultToks.erase(ResultToks.begin() +
350  VCtx.getNumberOfTokensPriorToVAOpt() - 1);
351  }
352  // If the expansion of __VA_OPT__ ends with a placeholder, eat any
353  // following '##' token.
354  if (VCtx.endsWithPlaceholder() && I + 1 != E &&
355  Tokens[I + 1].is(tok::hashhash)) {
356  ++I;
357  }
358  }
359  VCtx.reset();
360  // We processed __VA_OPT__'s closing paren (and the exit out of
361  // __VA_OPT__), so skip to the next token.
362  continue;
363  }
364  }
365 
366  // If we found the stringify operator, get the argument stringified. The
367  // preprocessor already verified that the following token is a macro
368  // parameter or __VA_OPT__ when the #define was lexed.
369 
370  if (CurTok.isOneOf(tok::hash, tok::hashat)) {
371  int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());
372  assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&
373  "Token following # is not an argument or __VA_OPT__!");
374 
375  if (ArgNo == -1) {
376  // Handle the __VA_OPT__ case.
377  VCtx.sawHashOrHashAtBefore(NextTokGetsSpace,
378  CurTok.is(tok::hashat));
379  continue;
380  }
381  // Else handle the simple argument case.
382  SourceLocation ExpansionLocStart =
383  getExpansionLocForMacroDefLoc(CurTok.getLocation());
384  SourceLocation ExpansionLocEnd =
385  getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());
386 
387  bool Charify = CurTok.is(tok::hashat);
388  const Token *UnexpArg = ActualArgs->getUnexpArgument(ArgNo);
390  UnexpArg, PP, Charify, ExpansionLocStart, ExpansionLocEnd);
392 
393  // The stringified/charified string leading space flag gets set to match
394  // the #/#@ operator.
395  if (NextTokGetsSpace)
397 
398  ResultToks.push_back(Res);
399  MadeChange = true;
400  ++I; // Skip arg name.
401  NextTokGetsSpace = false;
402  continue;
403  }
404 
405  // Find out if there is a paste (##) operator before or after the token.
406  bool NonEmptyPasteBefore =
407  !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
408  bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash);
409  bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash);
410  bool RParenAfter = I+1 != E && Tokens[I+1].is(tok::r_paren);
411 
412  assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&
413  "unexpected ## in ResultToks");
414 
415  // Otherwise, if this is not an argument token, just add the token to the
416  // output buffer.
417  IdentifierInfo *II = CurTok.getIdentifierInfo();
418  int ArgNo = II ? Macro->getParameterNum(II) : -1;
419  if (ArgNo == -1) {
420  // This isn't an argument, just add it.
421  ResultToks.push_back(CurTok);
422 
423  if (NextTokGetsSpace) {
424  ResultToks.back().setFlag(Token::LeadingSpace);
425  NextTokGetsSpace = false;
426  } else if (PasteBefore && !NonEmptyPasteBefore)
427  ResultToks.back().clearFlag(Token::LeadingSpace);
428 
429  continue;
430  }
431 
432  // An argument is expanded somehow, the result is different than the
433  // input.
434  MadeChange = true;
435 
436  // Otherwise, this is a use of the argument.
437 
438  // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
439  // are no trailing commas if __VA_ARGS__ is empty.
440  if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
441  MaybeRemoveCommaBeforeVaArgs(ResultToks,
442  /*HasPasteOperator=*/false,
443  Macro, ArgNo, PP))
444  continue;
445 
446  // If it is not the LHS/RHS of a ## operator, we must pre-expand the
447  // argument and substitute the expanded tokens into the result. This is
448  // C99 6.10.3.1p1.
449  if (!PasteBefore && !PasteAfter) {
450  const Token *ResultArgToks;
451 
452  // Only preexpand the argument if it could possibly need it. This
453  // avoids some work in common cases.
454  const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
455  if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
456  ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];
457  else
458  ResultArgToks = ArgTok; // Use non-preexpanded tokens.
459 
460  // If the arg token expanded into anything, append it.
461  if (ResultArgToks->isNot(tok::eof)) {
462  size_t FirstResult = ResultToks.size();
463  unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
464  ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
465 
466  // In Microsoft-compatibility mode, we follow MSVC's preprocessing
467  // behavior by not considering single commas from nested macro
468  // expansions as argument separators. Set a flag on the token so we can
469  // test for this later when the macro expansion is processed.
470  if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&
471  ResultToks.back().is(tok::comma))
472  ResultToks.back().setFlag(Token::IgnoredComma);
473 
474  // If the '##' came from expanding an argument, turn it into 'unknown'
475  // to avoid pasting.
476  for (Token &Tok : llvm::drop_begin(ResultToks, FirstResult))
477  if (Tok.is(tok::hashhash))
478  Tok.setKind(tok::unknown);
479 
480  if(ExpandLocStart.isValid()) {
481  updateLocForMacroArgTokens(CurTok.getLocation(),
482  ResultToks.begin()+FirstResult,
483  ResultToks.end());
484  }
485 
486  // If any tokens were substituted from the argument, the whitespace
487  // before the first token should match the whitespace of the arg
488  // identifier.
489  ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
490  NextTokGetsSpace);
491  ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
492  NextTokGetsSpace = false;
493  } else {
494  // We're creating a placeholder token. Usually this doesn't matter,
495  // but it can affect paste behavior when at the start or end of a
496  // __VA_OPT__.
497  if (NonEmptyPasteBefore) {
498  // We're imagining a placeholder token is inserted here. If this is
499  // the first token in a __VA_OPT__ after a ##, delete the ##.
500  assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__");
502  }
503  if (RParenAfter)
505  }
506  continue;
507  }
508 
509  // Okay, we have a token that is either the LHS or RHS of a paste (##)
510  // argument. It gets substituted as its non-pre-expanded tokens.
511  const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
512  unsigned NumToks = MacroArgs::getArgLength(ArgToks);
513  if (NumToks) { // Not an empty argument?
514  bool VaArgsPseudoPaste = false;
515  // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned
516  // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when
517  // the expander tries to paste ',' with the first token of the __VA_ARGS__
518  // expansion.
519  if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
520  ResultToks[ResultToks.size()-2].is(tok::comma) &&
521  (unsigned)ArgNo == Macro->getNumParams()-1 &&
522  Macro->isVariadic()) {
523  VaArgsPseudoPaste = true;
524  // Remove the paste operator, report use of the extension.
525  PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
526  }
527 
528  ResultToks.append(ArgToks, ArgToks+NumToks);
529 
530  // If the '##' came from expanding an argument, turn it into 'unknown'
531  // to avoid pasting.
532  for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
533  ResultToks.end())) {
534  if (Tok.is(tok::hashhash))
535  Tok.setKind(tok::unknown);
536  }
537 
538  if (ExpandLocStart.isValid()) {
539  updateLocForMacroArgTokens(CurTok.getLocation(),
540  ResultToks.end()-NumToks, ResultToks.end());
541  }
542 
543  // Transfer the leading whitespace information from the token
544  // (the macro argument) onto the first token of the
545  // expansion. Note that we don't do this for the GNU
546  // pseudo-paste extension ", ## __VA_ARGS__".
547  if (!VaArgsPseudoPaste) {
548  ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,
549  false);
550  ResultToks[ResultToks.size() - NumToks].setFlagValue(
551  Token::LeadingSpace, NextTokGetsSpace);
552  }
553 
554  NextTokGetsSpace = false;
555  continue;
556  }
557 
558  // If an empty argument is on the LHS or RHS of a paste, the standard (C99
559  // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We
560  // implement this by eating ## operators when a LHS or RHS expands to
561  // empty.
562  if (PasteAfter) {
563  // Discard the argument token and skip (don't copy to the expansion
564  // buffer) the paste operator after it.
565  ++I;
566  continue;
567  }
568 
569  if (RParenAfter)
571 
572  // If this is on the RHS of a paste operator, we've already copied the
573  // paste operator to the ResultToks list, unless the LHS was empty too.
574  // Remove it.
575  assert(PasteBefore);
576  if (NonEmptyPasteBefore) {
577  assert(ResultToks.back().is(tok::hashhash));
578  // Do not remove the paste operator if it is the one before __VA_OPT__
579  // (and we are still processing tokens within VA_OPT). We handle the case
580  // of removing the paste operator if __VA_OPT__ reduces to the notional
581  // placemarker above when we encounter the closing paren of VA_OPT.
582  if (!VCtx.isInVAOpt() ||
583  ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt())
584  ResultToks.pop_back();
585  else
587  }
588 
589  // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
590  // and if the macro had at least one real argument, and if the token before
591  // the ## was a comma, remove the comma. This is a GCC extension which is
592  // disabled when using -std=c99.
593  if (ActualArgs->isVarargsElidedUse())
594  MaybeRemoveCommaBeforeVaArgs(ResultToks,
595  /*HasPasteOperator=*/true,
596  Macro, ArgNo, PP);
597  }
598 
599  // If anything changed, install this as the new Tokens list.
600  if (MadeChange) {
601  assert(!OwnsTokens && "This would leak if we already own the token list");
602  // This is deleted in the dtor.
603  NumTokens = ResultToks.size();
604  // The tokens will be added to Preprocessor's cache and will be removed
605  // when this TokenLexer finishes lexing them.
606  Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
607 
608  // The preprocessor cache of macro expanded tokens owns these tokens,not us.
609  OwnsTokens = false;
610  }
611 }
612 
613 /// Checks if two tokens form wide string literal.
614 static bool isWideStringLiteralFromMacro(const Token &FirstTok,
615  const Token &SecondTok) {
616  return FirstTok.is(tok::identifier) &&
617  FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral() &&
618  SecondTok.stringifiedInMacro();
619 }
620 
621 /// Lex - Lex and return a token from this macro stream.
622 bool TokenLexer::Lex(Token &Tok) {
623  // Lexing off the end of the macro, pop this macro off the expansion stack.
624  if (isAtEnd()) {
625  // If this is a macro (not a token stream), mark the macro enabled now
626  // that it is no longer being expanded.
627  if (Macro) Macro->EnableMacro();
628 
629  Tok.startToken();
630  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
631  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);
632  if (CurTokenIdx == 0)
634  return PP.HandleEndOfTokenLexer(Tok);
635  }
636 
638 
639  // If this is the first token of the expanded result, we inherit spacing
640  // properties later.
641  bool isFirstToken = CurTokenIdx == 0;
642 
643  // Get the next token to return.
644  Tok = Tokens[CurTokenIdx++];
645  if (IsReinject)
647 
648  bool TokenIsFromPaste = false;
649 
650  // If this token is followed by a token paste (##) operator, paste the tokens!
651  // Note that ## is a normal token when not expanding a macro.
652  if (!isAtEnd() && Macro &&
653  (Tokens[CurTokenIdx].is(tok::hashhash) ||
654  // Special processing of L#x macros in -fms-compatibility mode.
655  // Microsoft compiler is able to form a wide string literal from
656  // 'L#macro_arg' construct in a function-like macro.
657  (PP.getLangOpts().MSVCCompat &&
658  isWideStringLiteralFromMacro(Tok, Tokens[CurTokenIdx])))) {
659  // When handling the microsoft /##/ extension, the final token is
660  // returned by pasteTokens, not the pasted token.
661  if (pasteTokens(Tok))
662  return true;
663 
664  TokenIsFromPaste = true;
665  }
666 
667  // The token's current location indicate where the token was lexed from. We
668  // need this information to compute the spelling of the token, but any
669  // diagnostics for the expanded token should appear as if they came from
670  // ExpansionLoc. Pull this information together into a new SourceLocation
671  // that captures all of this.
672  if (ExpandLocStart.isValid() && // Don't do this for token streams.
673  // Check that the token's location was not already set properly.
674  SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {
675  SourceLocation instLoc;
676  if (Tok.is(tok::comment)) {
677  instLoc = SM.createExpansionLoc(Tok.getLocation(),
678  ExpandLocStart,
679  ExpandLocEnd,
680  Tok.getLength());
681  } else {
682  instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());
683  }
684 
685  Tok.setLocation(instLoc);
686  }
687 
688  // If this is the first token, set the lexical properties of the token to
689  // match the lexical properties of the macro identifier.
690  if (isFirstToken) {
691  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
692  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
693  } else {
694  // If this is not the first token, we may still need to pass through
695  // leading whitespace if we've expanded a macro.
696  if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);
697  if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);
698  }
699  AtStartOfLine = false;
700  HasLeadingSpace = false;
701 
702  // Handle recursive expansion!
703  if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) {
704  // Change the kind of this identifier to the appropriate token kind, e.g.
705  // turning "for" into a keyword.
706  IdentifierInfo *II = Tok.getIdentifierInfo();
707  Tok.setKind(II->getTokenID());
708 
709  // If this identifier was poisoned and from a paste, emit an error. This
710  // won't be handled by Preprocessor::HandleIdentifier because this is coming
711  // from a macro expansion.
712  if (II->isPoisoned() && TokenIsFromPaste) {
713  PP.HandlePoisonedIdentifier(Tok);
714  }
715 
716  if (!DisableMacroExpansion && II->isHandleIdentifierCase())
717  return PP.HandleIdentifier(Tok);
718  }
719 
720  // Otherwise, return a normal token.
721  return true;
722 }
723 
724 bool TokenLexer::pasteTokens(Token &Tok) {
725  return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx);
726 }
727 
728 /// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ##
729 /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there
730 /// are more ## after it, chomp them iteratively. Return the result as LHSTok.
731 /// If this returns true, the caller should immediately return the token.
732 bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
733  unsigned int &CurIdx) {
734  assert(CurIdx > 0 && "## can not be the first token within tokens");
735  assert((TokenStream[CurIdx].is(tok::hashhash) ||
736  (PP.getLangOpts().MSVCCompat &&
737  isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) &&
738  "Token at this Index must be ## or part of the MSVC 'L "
739  "#macro-arg' pasting pair");
740 
741  // MSVC: If previous token was pasted, this must be a recovery from an invalid
742  // paste operation. Ignore spaces before this token to mimic MSVC output.
743  // Required for generating valid UUID strings in some MS headers.
744  if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) &&
745  TokenStream[CurIdx - 2].is(tok::hashhash))
747 
748  SmallString<128> Buffer;
749  const char *ResultTokStrPtr = nullptr;
750  SourceLocation StartLoc = LHSTok.getLocation();
751  SourceLocation PasteOpLoc;
752 
753  auto IsAtEnd = [&TokenStream, &CurIdx] {
754  return TokenStream.size() == CurIdx;
755  };
756 
757  do {
758  // Consume the ## operator if any.
759  PasteOpLoc = TokenStream[CurIdx].getLocation();
760  if (TokenStream[CurIdx].is(tok::hashhash))
761  ++CurIdx;
762  assert(!IsAtEnd() && "No token on the RHS of a paste operator!");
763 
764  // Get the RHS token.
765  const Token &RHS = TokenStream[CurIdx];
766 
767  // Allocate space for the result token. This is guaranteed to be enough for
768  // the two tokens.
769  Buffer.resize(LHSTok.getLength() + RHS.getLength());
770 
771  // Get the spelling of the LHS token in Buffer.
772  const char *BufPtr = &Buffer[0];
773  bool Invalid = false;
774  unsigned LHSLen = PP.getSpelling(LHSTok, BufPtr, &Invalid);
775  if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer!
776  memcpy(&Buffer[0], BufPtr, LHSLen);
777  if (Invalid)
778  return true;
779 
780  BufPtr = Buffer.data() + LHSLen;
781  unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid);
782  if (Invalid)
783  return true;
784  if (RHSLen && BufPtr != &Buffer[LHSLen])
785  // Really, we want the chars in Buffer!
786  memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
787 
788  // Trim excess space.
789  Buffer.resize(LHSLen+RHSLen);
790 
791  // Plop the pasted result (including the trailing newline and null) into a
792  // scratch buffer where we can lex it.
793  Token ResultTokTmp;
794  ResultTokTmp.startToken();
795 
796  // Claim that the tmp token is a string_literal so that we can get the
797  // character pointer back from CreateString in getLiteralData().
798  ResultTokTmp.setKind(tok::string_literal);
799  PP.CreateString(Buffer, ResultTokTmp);
800  SourceLocation ResultTokLoc = ResultTokTmp.getLocation();
801  ResultTokStrPtr = ResultTokTmp.getLiteralData();
802 
803  // Lex the resultant pasted token into Result.
804  Token Result;
805 
806  if (LHSTok.isAnyIdentifier() && RHS.isAnyIdentifier()) {
807  // Common paste case: identifier+identifier = identifier. Avoid creating
808  // a lexer and other overhead.
809  PP.IncrementPasteCounter(true);
810  Result.startToken();
811  Result.setKind(tok::raw_identifier);
812  Result.setRawIdentifierData(ResultTokStrPtr);
813  Result.setLocation(ResultTokLoc);
814  Result.setLength(LHSLen+RHSLen);
815  } else {
816  PP.IncrementPasteCounter(false);
817 
818  assert(ResultTokLoc.isFileID() &&
819  "Should be a raw location into scratch buffer");
820  SourceManager &SourceMgr = PP.getSourceManager();
821  FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
822 
823  bool Invalid = false;
824  const char *ScratchBufStart
825  = SourceMgr.getBufferData(LocFileID, &Invalid).data();
826  if (Invalid)
827  return false;
828 
829  // Make a lexer to lex this string from. Lex just this one token.
830  // Make a lexer object so that we lex and expand the paste result.
831  Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
832  PP.getLangOpts(), ScratchBufStart,
833  ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
834 
835  // Lex a token in raw mode. This way it won't look up identifiers
836  // automatically, lexing off the end will return an eof token, and
837  // warnings are disabled. This returns true if the result token is the
838  // entire buffer.
839  bool isInvalid = !TL.LexFromRawLexer(Result);
840 
841  // If we got an EOF token, we didn't form even ONE token. For example, we
842  // did "/ ## /" to get "//".
843  isInvalid |= Result.is(tok::eof);
844 
845  // If pasting the two tokens didn't form a full new token, this is an
846  // error. This occurs with "x ## +" and other stuff. Return with LHSTok
847  // unmodified and with RHS as the next token to lex.
848  if (isInvalid) {
849  // Explicitly convert the token location to have proper expansion
850  // information so that the user knows where it came from.
852  SourceLocation Loc =
853  SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);
854 
855  // Test for the Microsoft extension of /##/ turning into // here on the
856  // error path.
857  if (PP.getLangOpts().MicrosoftExt && LHSTok.is(tok::slash) &&
858  RHS.is(tok::slash)) {
859  HandleMicrosoftCommentPaste(LHSTok, Loc);
860  return true;
861  }
862 
863  // Do not emit the error when preprocessing assembler code.
864  if (!PP.getLangOpts().AsmPreprocessor) {
865  // If we're in microsoft extensions mode, downgrade this from a hard
866  // error to an extension that defaults to an error. This allows
867  // disabling it.
868  PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms
869  : diag::err_pp_bad_paste)
870  << Buffer;
871  }
872 
873  // An error has occurred so exit loop.
874  break;
875  }
876 
877  // Turn ## into 'unknown' to avoid # ## # from looking like a paste
878  // operator.
879  if (Result.is(tok::hashhash))
880  Result.setKind(tok::unknown);
881  }
882 
883  // Transfer properties of the LHS over the Result.
884  Result.setFlagValue(Token::StartOfLine , LHSTok.isAtStartOfLine());
885  Result.setFlagValue(Token::LeadingSpace, LHSTok.hasLeadingSpace());
886 
887  // Finally, replace LHS with the result, consume the RHS, and iterate.
888  ++CurIdx;
889  LHSTok = Result;
890  } while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));
891 
892  SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation();
893 
894  // The token's current location indicate where the token was lexed from. We
895  // need this information to compute the spelling of the token, but any
896  // diagnostics for the expanded token should appear as if the token was
897  // expanded from the full ## expression. Pull this information together into
898  // a new SourceLocation that captures all of this.
900  if (StartLoc.isFileID())
901  StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
902  if (EndLoc.isFileID())
903  EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
904  FileID MacroFID = SM.getFileID(MacroExpansionStart);
905  while (SM.getFileID(StartLoc) != MacroFID)
906  StartLoc = SM.getImmediateExpansionRange(StartLoc).getBegin();
907  while (SM.getFileID(EndLoc) != MacroFID)
908  EndLoc = SM.getImmediateExpansionRange(EndLoc).getEnd();
909 
910  LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc,
911  LHSTok.getLength()));
912 
913  // Now that we got the result token, it will be subject to expansion. Since
914  // token pasting re-lexes the result token in raw mode, identifier information
915  // isn't looked up. As such, if the result is an identifier, look up id info.
916  if (LHSTok.is(tok::raw_identifier)) {
917  // Look up the identifier info for the token. We disabled identifier lookup
918  // by saying we're skipping contents, so we need to do this manually.
919  PP.LookUpIdentifierInfo(LHSTok);
920  }
921  return false;
922 }
923 
924 /// isNextTokenLParen - If the next token lexed will pop this macro off the
925 /// expansion stack, return 2. If the next unexpanded token is a '(', return
926 /// 1, otherwise return 0.
928  // Out of tokens?
929  if (isAtEnd())
930  return 2;
931  return Tokens[CurTokenIdx].is(tok::l_paren);
932 }
933 
934 /// isParsingPreprocessorDirective - Return true if we are in the middle of a
935 /// preprocessor directive.
937  return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd();
938 }
939 
940 /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
941 /// together to form a comment that comments out everything in the current
942 /// macro, other active macros, and anything left on the current physical
943 /// source line of the expanded buffer. Handle this by returning the
944 /// first token on the next line.
945 void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {
946  PP.Diag(OpLoc, diag::ext_comment_paste_microsoft);
947 
948  // We 'comment out' the rest of this macro by just ignoring the rest of the
949  // tokens that have not been lexed yet, if any.
950 
951  // Since this must be a macro, mark the macro enabled now that it is no longer
952  // being expanded.
953  assert(Macro && "Token streams can't paste comments");
954  Macro->EnableMacro();
955 
957 }
958 
959 /// If \arg loc is a file ID and points inside the current macro
960 /// definition, returns the appropriate source location pointing at the
961 /// macro expansion source location entry, otherwise it returns an invalid
962 /// SourceLocation.
964 TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {
965  assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&
966  "Not appropriate for token streams");
967  assert(loc.isValid() && loc.isFileID());
968 
970  assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
971  "Expected loc to come from the macro definition");
972 
973  SourceLocation::UIntTy relativeOffset = 0;
974  SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);
975  return MacroExpansionStart.getLocWithOffset(relativeOffset);
976 }
977 
978 /// Finds the tokens that are consecutive (from the same FileID)
979 /// creates a single SLocEntry, and assigns SourceLocations to each token that
980 /// point to that SLocEntry. e.g for
981 /// assert(foo == bar);
982 /// There will be a single SLocEntry for the "foo == bar" chunk and locations
983 /// for the 'foo', '==', 'bar' tokens will point inside that chunk.
984 ///
985 /// \arg begin_tokens will be updated to a position past all the found
986 /// consecutive tokens.
988  SourceLocation ExpandLoc,
989  Token *&begin_tokens,
990  Token * end_tokens) {
991  assert(begin_tokens + 1 < end_tokens);
992  SourceLocation BeginLoc = begin_tokens->getLocation();
993  llvm::MutableArrayRef<Token> All(begin_tokens, end_tokens);
995 
996  auto NearLast = [&, Last = BeginLoc](SourceLocation Loc) mutable {
997  // The maximum distance between two consecutive tokens in a partition.
998  // This is an important trick to avoid using too much SourceLocation address
999  // space!
1000  static constexpr SourceLocation::IntTy MaxDistance = 50;
1001  auto Distance = Loc.getRawEncoding() - Last.getRawEncoding();
1002  Last = Loc;
1003  return Distance <= MaxDistance;
1004  };
1005 
1006  // Partition the tokens by their FileID.
1007  // This is a hot function, and calling getFileID can be expensive, the
1008  // implementation is optimized by reducing the number of getFileID.
1009  if (BeginLoc.isFileID()) {
1010  // Consecutive tokens not written in macros must be from the same file.
1011  // (Neither #include nor eof can occur inside a macro argument.)
1012  Partition = All.take_while([&](const Token &T) {
1013  return T.getLocation().isFileID() && NearLast(T.getLocation());
1014  });
1015  } else {
1016  // Call getFileID once to calculate the bounds, and use the cheaper
1017  // sourcelocation-against-bounds comparison.
1018  FileID BeginFID = SM.getFileID(BeginLoc);
1019  SourceLocation Limit =
1020  SM.getComposedLoc(BeginFID, SM.getFileIDSize(BeginFID));
1021  Partition = All.take_while([&](const Token &T) {
1022  return T.getLocation() >= BeginLoc && T.getLocation() < Limit &&
1023  NearLast(T.getLocation());
1024  });
1025  }
1026  assert(!Partition.empty());
1027 
1028  // For the consecutive tokens, find the length of the SLocEntry to contain
1029  // all of them.
1030  SourceLocation::UIntTy FullLength =
1031  Partition.back().getEndLoc().getRawEncoding() -
1032  Partition.front().getLocation().getRawEncoding();
1033  // Create a macro expansion SLocEntry that will "contain" all of the tokens.
1034  SourceLocation Expansion =
1035  SM.createMacroArgExpansionLoc(BeginLoc, ExpandLoc, FullLength);
1036 
1037 #ifdef EXPENSIVE_CHECKS
1038  assert(llvm::all_of(Partition.drop_front(),
1039  [&SM, ID = SM.getFileID(Partition.front().getLocation())](
1040  const Token &T) {
1041  return ID == SM.getFileID(T.getLocation());
1042  }) &&
1043  "Must have the same FIleID!");
1044 #endif
1045  // Change the location of the tokens from the spelling location to the new
1046  // expanded location.
1047  for (Token& T : Partition) {
1048  SourceLocation::IntTy RelativeOffset =
1049  T.getLocation().getRawEncoding() - BeginLoc.getRawEncoding();
1050  T.setLocation(Expansion.getLocWithOffset(RelativeOffset));
1051  }
1052  begin_tokens = &Partition.back() + 1;
1053 }
1054 
1055 /// Creates SLocEntries and updates the locations of macro argument
1056 /// tokens to their new expanded locations.
1057 ///
1058 /// \param ArgIdSpellLoc the location of the macro argument id inside the macro
1059 /// definition.
1060 void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
1061  Token *begin_tokens,
1062  Token *end_tokens) {
1064 
1065  SourceLocation ExpandLoc =
1066  getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
1067 
1068  while (begin_tokens < end_tokens) {
1069  // If there's only one token just create a SLocEntry for it.
1070  if (end_tokens - begin_tokens == 1) {
1071  Token &Tok = *begin_tokens;
1072  Tok.setLocation(SM.createMacroArgExpansionLoc(Tok.getLocation(),
1073  ExpandLoc,
1074  Tok.getLength()));
1075  return;
1076  }
1077 
1078  updateConsecutiveMacroArgTokens(SM, ExpandLoc, begin_tokens, end_tokens);
1079  }
1080 }
1081 
1082 void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
1083  AtStartOfLine = Result.isAtStartOfLine();
1084  HasLeadingSpace = Result.hasLeadingSpace();
1085 }
clang::Token::startToken
void startToken()
Reset all flags to cleared.
Definition: Token.h:171
clang::Preprocessor::CreateString
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
Definition: Preprocessor.cpp:488
clang::Token::IgnoredComma
@ IgnoredComma
Definition: Token.h:83
clang::Token::IsReinjected
@ IsReinjected
Definition: Token.h:88
clang::MacroInfo::tokens_end
const_tokens_iterator tokens_end() const
Definition: MacroInfo.h:245
clang::VAOptExpansionContext::hasPlaceholderBeforeRParen
void hasPlaceholderBeforeRParen()
Definition: VariadicMacroSupport.h:180
clang::Token::hasLeadingSpace
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:274
clang::VAOptExpansionContext::sawOpeningParen
void sawOpeningParen(SourceLocation LParenLoc)
Call this function each time an lparen is seen.
Definition: VariadicMacroSupport.h:107
clang::VAOptExpansionContext::sawHashOrHashAtBefore
void sawHashOrHashAtBefore(const bool HasLeadingSpace, const bool IsHashAt)
Definition: VariadicMacroSupport.h:171
clang::Preprocessor::HandleEndOfTokenLexer
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
Definition: PPLexerChange.cpp:586
clang::VAOptExpansionContext::getNumberOfTokensPriorToVAOpt
unsigned int getNumberOfTokensPriorToVAOpt() const
Definition: VariadicMacroSupport.h:206
clang::Preprocessor::HandleIdentifier
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
Definition: Preprocessor.cpp:789
clang::Token::getLiteralData
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:219
llvm::SmallVector
Definition: LLVM.h:38
clang::SourceLocation
Encodes a location in the source.
Definition: SourceLocation.h:86
clang::MacroInfo::tokens_begin
const_tokens_iterator tokens_begin() const
Definition: MacroInfo.h:244
clang::SourceLocation::getLocWithOffset
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
Definition: SourceLocation.h:134
clang::Token::getIdentifierInfo
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:181
clang::MacroArgs
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
clang::VAOptExpansionContext::beginsWithPlaceholder
bool beginsWithPlaceholder() const
Definition: VariadicMacroSupport.h:186
isWideStringLiteralFromMacro
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
Definition: TokenLexer.cpp:614
clang::VAOptExpansionContext
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a macro during...
Definition: VariadicMacroSupport.h:118
updateConsecutiveMacroArgTokens
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation ExpandLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry,...
Definition: TokenLexer.cpp:987
memcpy
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
Definition: __clang_cuda_device_functions.h:1549
clang::TokenLexer::Lex
bool Lex(Token &Tok)
Lex and return a token from this macro stream.
Definition: TokenLexer.cpp:622
clang::Token::isAtStartOfLine
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:270
llvm::Optional< bool >
clang::MacroInfo
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
SourceManager.h
clang::Preprocessor::HandleMicrosoftCommentPaste
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
Definition: PPLexerChange.cpp:624
clang::index::SymbolKind::Macro
@ Macro
clang::Preprocessor::IncrementPasteCounter
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Definition: Preprocessor.h:2065
clang::IdentifierInfo::getTokenID
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
Definition: IdentifierTable.h:262
clang::Token
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
TokenLexer.h
clang::SourceManager
This class handles loading and caching of source files into memory.
Definition: SourceManager.h:636
clang::Preprocessor::getLangOpts
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:1065
Preprocessor.h
clang::Lexer
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.
Definition: Lexer.h:78
clang::Token::StringifiedInMacro
@ StringifiedInMacro
Definition: Token.h:84
clang::SourceLocation::IntTy
int32_t IntTy
Definition: SourceLocation.h:94
TokenKinds.h
clang::IdentifierInfo::isStr
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
Definition: IdentifierTable.h:177
clang::MacroArgs::getPreExpArgument
const std::vector< Token > & getPreExpArgument(unsigned Arg, Preprocessor &PP)
getPreExpArgument - Return the pre-expanded form of the specified argument.
Definition: MacroArgs.cpp:161
llvm::MutableArrayRef
Definition: LLVM.h:35
clang::MacroInfo::DisableMacro
void DisableMacro()
Definition: MacroInfo.h:288
clang::VAOptExpansionContext::hasCharifyBefore
bool hasCharifyBefore() const
Definition: VariadicMacroSupport.h:197
clang::VAOptExpansionContext::getLeadingSpaceForStringifiedToken
bool getLeadingSpaceForStringifiedToken() const
Definition: VariadicMacroSupport.h:212
clang::VAOptExpansionContext::isVAOptToken
bool isVAOptToken(const Token &T) const
Definition: VariadicMacroSupport.h:77
clang::Token::setFlagValue
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Definition: Token.h:261
LangOptions.h
Diagnostic.h
LexDiagnostic.h
clang::Preprocessor::Diag
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
Definition: Preprocessor.h:1899
clang::Token::is
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:98
clang::MacroArgs::isVarargsElidedUse
bool isVarargsElidedUse() const
isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...
Definition: MacroArgs.h:102
IdentifierTable.h
clang::SourceManager::getFileID
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
Definition: SourceManager.h:1116
llvm::SmallString< 128 >
clang::Token::LeadingEmptyMacro
@ LeadingEmptyMacro
Definition: Token.h:80
clang::VAOptExpansionContext::reset
void reset()
Definition: VariadicMacroSupport.h:159
clang::VAOptExpansionContext::isInVAOpt
bool isInVAOpt() const
Returns true if we have seen the VA_OPT and '(' but before having seen the matching ')'.
Definition: VariadicMacroSupport.h:83
isInvalid
static bool isInvalid(LocType Loc, bool *Invalid)
Definition: SourceManager.cpp:1233
clang::SourceLocation::isFileID
bool isFileID() const
Definition: SourceLocation.h:102
clang::Token::isAnyIdentifier
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
Definition: Token.h:109
clang::TokenLexer::isNextTokenLParen
unsigned isNextTokenLParen() const
If the next token lexed will pop this macro off the expansion stack, return 2.
Definition: TokenLexer.cpp:927
clang::Token::isNot
bool isNot(tok::TokenKind K) const
Definition: Token.h:99
clang::MacroArgs::getUnexpArgument
const Token * getUnexpArgument(unsigned Arg) const
getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...
Definition: MacroArgs.cpp:118
clang::MacroArgs::getArgLength
static unsigned getArgLength(const Token *ArgPtr)
getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens,...
Definition: MacroArgs.cpp:108
clang::VAOptExpansionContext::hasPlaceholderAfterHashhashAtStart
void hasPlaceholderAfterHashhashAtStart()
Definition: VariadicMacroSupport.h:179
clang::Token::isLiteral
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:115
clang::Token::getLength
unsigned getLength() const
Definition: Token.h:129
clang::Token::LeadingSpace
@ LeadingSpace
Definition: Token.h:76
clang::SourceManager::getLocForStartOfFile
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file.
Definition: SourceManager.h:1125
clang::Preprocessor::getSourceManager
SourceManager & getSourceManager() const
Definition: Preprocessor.h:1069
clang::MacroArgs::destroy
void destroy(Preprocessor &PP)
destroy - Destroy and deallocate the memory for this object.
Definition: MacroArgs.cpp:78
clang::Preprocessor::getSpelling
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
Definition: Preprocessor.h:1914
MacroArgs.h
clang::Decl::setLocation
void setLocation(SourceLocation L)
Definition: DeclBase.h:433
SourceLocation.h
clang::VAOptExpansionContext::endsWithPlaceholder
bool endsWithPlaceholder() const
Definition: VariadicMacroSupport.h:191
clang::MacroArgs::invokedWithVariadicArgument
bool invokedWithVariadicArgument(const MacroInfo *const MI, Preprocessor &PP)
Returns true if the macro was defined with a variadic (ellipsis) parameter AND was invoked with at le...
Definition: MacroArgs.cpp:136
clang::IdentifierInfo::isPoisoned
bool isPoisoned() const
Return true if this token has been poisoned.
Definition: IdentifierTable.h:355
clang::TokenLexer::isParsingPreprocessorDirective
bool isParsingPreprocessorDirective() const
isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive.
Definition: TokenLexer.cpp:936
llvm::ArrayRef
Definition: LLVM.h:34
Lexer.h
clang::TokenLexer::Init
void Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, MacroArgs *Actuals)
Initialize this TokenLexer to expand from the specified macro with the specified argument information...
Definition: TokenLexer.cpp:39
clang::Token::StartOfLine
@ StartOfLine
Definition: Token.h:74
clang::SourceManager::getBufferData
StringRef getBufferData(FileID FID, bool *Invalid=nullptr) const
Return a StringRef to the source buffer data for the specified FileID.
Definition: SourceManager.cpp:741
clang::IdentifierInfo
One of these records is kept for each identifier that is lexed.
Definition: IdentifierTable.h:85
clang::Token::clearFlag
void clearFlag(TokenFlags Flag)
Unset the specified flag.
Definition: Token.h:248
clang::Token::getLocation
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:126
clang::IdentifierInfo::isHandleIdentifierCase
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier.
Definition: IdentifierTable.h:381
clang::MacroInfo::getDefinitionLength
unsigned getDefinitionLength(const SourceManager &SM) const
Get length in characters of the macro definition.
Definition: MacroInfo.h:134
clang::Token::isAnnotation
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:120
clang::VAOptExpansionContext::sawClosingParen
bool sawClosingParen()
Call this function each time an rparen is seen.
Definition: VariadicMacroSupport.h:100
clang::Token::setLocation
void setLocation(SourceLocation L)
Definition: Token.h:134
clang::Builtin::ID
ID
Definition: Builtins.h:52
clang::MacroInfo::getNumParams
unsigned getNumParams() const
Definition: MacroInfo.h:184
clang
Definition: CalledOnceCheck.h:17
clang::Preprocessor::LookUpIdentifierInfo
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Definition: Preprocessor.cpp:712
MacroInfo.h
clang::ComparisonCategoryType::Last
@ Last
clang::VAOptExpansionContext::getEOFTok
const Token & getEOFTok() const
Definition: VariadicMacroSupport.h:169
clang::Token::CommaAfterElided
@ CommaAfterElided
Definition: Token.h:86
clang::VAOptExpansionContext::hasStringifyOrCharifyBefore
bool hasStringifyOrCharifyBefore() const
Definition: VariadicMacroSupport.h:202
clang::Token::stringifiedInMacro
bool stringifiedInMacro() const
Returns true if this token is formed by macro by stringizing or charizing operator.
Definition: Token.h:302
clang::FileID
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Definition: SourceLocation.h:38
clang::SourceLocation::isValid
bool isValid() const
Return true if this is a valid SourceLocation object.
Definition: SourceLocation.h:110
VariadicMacroSupport.h
clang::Token::setKind
void setKind(tok::TokenKind K)
Definition: Token.h:94
clang::Token::isOneOf
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:100
clang::VAOptExpansionContext::getVAOptLoc
SourceLocation getVAOptLoc() const
Definition: VariadicMacroSupport.h:229
clang::Token::setFlag
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:238
clang::MacroArgs::ArgNeedsPreexpansion
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
ArgNeedsPreexpansion - If we can prove that the argument won't be affected by pre-expansion,...
Definition: MacroArgs.cpp:146
clang::comments::tok::eof
@ eof
Definition: CommentLexer.h:33
llvm::SmallVectorImpl
Definition: Randstruct.h:18
clang::MacroInfo::isFunctionLike
bool isFunctionLike() const
Definition: MacroInfo.h:201
clang::Preprocessor
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:129
SM
#define SM(sm)
Definition: Cuda.cpp:79
clang::Preprocessor::HandlePoisonedIdentifier
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
Definition: Preprocessor.cpp:765
clang::MacroArgs::StringifyArgument
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...
Definition: MacroArgs.cpp:208
Token.h
clang::RISCV::Invalid
@ Invalid
Definition: RISCVVIntrinsicUtils.h:171
clang::VAOptExpansionContext::sawVAOptFollowedByOpeningParens
void sawVAOptFollowedByOpeningParens(const SourceLocation VAOptLoc, const unsigned int NumPriorTokens)
Definition: VariadicMacroSupport.h:218
clang::SourceLocation::UIntTy
uint32_t UIntTy
Definition: SourceLocation.h:93
clang::SourceLocation::getRawEncoding
UIntTy getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it.
Definition: SourceLocation.h:146