clang  15.0.0git
TokenLexer.cpp
Go to the documentation of this file.
1 //===- TokenLexer.cpp - Lex from a token stream ---------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the TokenLexer interface.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "clang/Lex/TokenLexer.h"
14 #include "clang/Basic/Diagnostic.h"
19 #include "clang/Basic/TokenKinds.h"
21 #include "clang/Lex/Lexer.h"
22 #include "clang/Lex/MacroArgs.h"
23 #include "clang/Lex/MacroInfo.h"
24 #include "clang/Lex/Preprocessor.h"
25 #include "clang/Lex/Token.h"
27 #include "llvm/ADT/ArrayRef.h"
28 #include "llvm/ADT/SmallString.h"
29 #include "llvm/ADT/SmallVector.h"
30 #include "llvm/ADT/iterator_range.h"
31 #include <cassert>
32 #include <cstring>
33 
34 using namespace clang;
35 
36 /// Create a TokenLexer for the specified macro with the specified actual
37 /// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
39  MacroArgs *Actuals) {
40  // If the client is reusing a TokenLexer, make sure to free any memory
41  // associated with it.
42  destroy();
43 
44  Macro = MI;
45  ActualArgs = Actuals;
46  CurTokenIdx = 0;
47 
48  ExpandLocStart = Tok.getLocation();
49  ExpandLocEnd = ELEnd;
50  AtStartOfLine = Tok.isAtStartOfLine();
51  HasLeadingSpace = Tok.hasLeadingSpace();
52  NextTokGetsSpace = false;
53  Tokens = &*Macro->tokens_begin();
54  OwnsTokens = false;
55  DisableMacroExpansion = false;
56  IsReinject = false;
57  NumTokens = Macro->tokens_end()-Macro->tokens_begin();
58  MacroExpansionStart = SourceLocation();
59 
61  MacroStartSLocOffset = SM.getNextLocalOffset();
62 
63  if (NumTokens > 0) {
64  assert(Tokens[0].getLocation().isValid());
65  assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
66  "Macro defined in macro?");
67  assert(ExpandLocStart.isValid());
68 
69  // Reserve a source location entry chunk for the length of the macro
70  // definition. Tokens that get lexed directly from the definition will
71  // have their locations pointing inside this chunk. This is to avoid
72  // creating separate source location entries for each token.
73  MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());
74  MacroDefLength = Macro->getDefinitionLength(SM);
75  MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,
76  ExpandLocStart,
77  ExpandLocEnd,
78  MacroDefLength);
79  }
80 
81  // If this is a function-like macro, expand the arguments and change
82  // Tokens to point to the expanded tokens.
83  if (Macro->isFunctionLike() && Macro->getNumParams())
84  ExpandFunctionArguments();
85 
86  // Mark the macro as currently disabled, so that it is not recursively
87  // expanded. The macro must be disabled only after argument pre-expansion of
88  // function-like macro arguments occurs.
89  Macro->DisableMacro();
90 }
91 
92 /// Create a TokenLexer for the specified token stream. This does not
93 /// take ownership of the specified token vector.
94 void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
95  bool disableMacroExpansion, bool ownsTokens,
96  bool isReinject) {
97  assert(!isReinject || disableMacroExpansion);
98  // If the client is reusing a TokenLexer, make sure to free any memory
99  // associated with it.
100  destroy();
101 
102  Macro = nullptr;
103  ActualArgs = nullptr;
104  Tokens = TokArray;
105  OwnsTokens = ownsTokens;
106  DisableMacroExpansion = disableMacroExpansion;
107  IsReinject = isReinject;
108  NumTokens = NumToks;
109  CurTokenIdx = 0;
110  ExpandLocStart = ExpandLocEnd = SourceLocation();
111  AtStartOfLine = false;
112  HasLeadingSpace = false;
113  NextTokGetsSpace = false;
114  MacroExpansionStart = SourceLocation();
115 
116  // Set HasLeadingSpace/AtStartOfLine so that the first token will be
117  // returned unmodified.
118  if (NumToks != 0) {
119  AtStartOfLine = TokArray[0].isAtStartOfLine();
120  HasLeadingSpace = TokArray[0].hasLeadingSpace();
121  }
122 }
123 
124 void TokenLexer::destroy() {
125  // If this was a function-like macro that actually uses its arguments, delete
126  // the expanded tokens.
127  if (OwnsTokens) {
128  delete [] Tokens;
129  Tokens = nullptr;
130  OwnsTokens = false;
131  }
132 
133  // TokenLexer owns its formal arguments.
134  if (ActualArgs) ActualArgs->destroy(PP);
135 }
136 
137 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
138  SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
139  unsigned MacroArgNo, Preprocessor &PP) {
140  // Is the macro argument __VA_ARGS__?
141  if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1)
142  return false;
143 
144  // In Microsoft-compatibility mode, a comma is removed in the expansion
145  // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is
146  // not supported by gcc.
147  if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)
148  return false;
149 
150  // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if
151  // __VA_ARGS__ is empty, but not in strict C99 mode where there are no
152  // named arguments, where it remains. In all other modes, including C99
153  // with GNU extensions, it is removed regardless of named arguments.
154  // Microsoft also appears to support this extension, unofficially.
155  if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode
156  && Macro->getNumParams() < 2)
157  return false;
158 
159  // Is a comma available to be removed?
160  if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
161  return false;
162 
163  // Issue an extension diagnostic for the paste operator.
164  if (HasPasteOperator)
165  PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
166 
167  // Remove the comma.
168  ResultToks.pop_back();
169 
170  if (!ResultToks.empty()) {
171  // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
172  // then removal of the comma should produce a placemarker token (in C99
173  // terms) which we model by popping off the previous ##, giving us a plain
174  // "X" when __VA_ARGS__ is empty.
175  if (ResultToks.back().is(tok::hashhash))
176  ResultToks.pop_back();
177 
178  // Remember that this comma was elided.
179  ResultToks.back().setFlag(Token::CommaAfterElided);
180  }
181 
182  // Never add a space, even if the comma, ##, or arg had a space.
183  NextTokGetsSpace = false;
184  return true;
185 }
186 
187 void TokenLexer::stringifyVAOPTContents(
188  SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,
189  const SourceLocation VAOPTClosingParenLoc) {
190  const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt();
191  const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
192  Token *const VAOPTTokens =
193  NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr;
194 
195  SmallVector<Token, 64> ConcatenatedVAOPTResultToks;
196  // FIXME: Should we keep track within VCtx that we did or didnot
197  // encounter pasting - and only then perform this loop.
198 
199  // Perform token pasting (concatenation) prior to stringization.
200  for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
201  ++CurTokenIdx) {
202  if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {
203  assert(CurTokenIdx != 0 &&
204  "Can not have __VAOPT__ contents begin with a ##");
205  Token &LHS = VAOPTTokens[CurTokenIdx - 1];
206  pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens),
207  CurTokenIdx);
208  // Replace the token prior to the first ## in this iteration.
209  ConcatenatedVAOPTResultToks.back() = LHS;
210  if (CurTokenIdx == NumVAOptTokens)
211  break;
212  }
213  ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);
214  }
215 
216  ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok());
217  // Get the SourceLocation that represents the start location within
218  // the macro definition that marks where this string is substituted
219  // into: i.e. the __VA_OPT__ and the ')' within the spelling of the
220  // macro definition, and use it to indicate that the stringified token
221  // was generated from that location.
222  const SourceLocation ExpansionLocStartWithinMacro =
223  getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc());
224  const SourceLocation ExpansionLocEndWithinMacro =
225  getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);
226 
227  Token StringifiedVAOPT = MacroArgs::StringifyArgument(
228  &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() /*Charify*/,
229  ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);
230 
232  StringifiedVAOPT.setFlag(Token::LeadingSpace);
233 
234  StringifiedVAOPT.setFlag(Token::StringifiedInMacro);
235  // Resize (shrink) the token stream to just capture this stringified token.
236  ResultToks.resize(NumToksPriorToVAOpt + 1);
237  ResultToks.back() = StringifiedVAOPT;
238 }
239 
240 /// Expand the arguments of a function-like macro so that we can quickly
241 /// return preexpanded tokens from Tokens.
242 void TokenLexer::ExpandFunctionArguments() {
243  SmallVector<Token, 128> ResultToks;
244 
245  // Loop through 'Tokens', expanding them into ResultToks. Keep
246  // track of whether we change anything. If not, no need to keep them. If so,
247  // we install the newly expanded sequence as the new 'Tokens' list.
248  bool MadeChange = false;
249 
250  Optional<bool> CalledWithVariadicArguments;
251 
252  VAOptExpansionContext VCtx(PP);
253 
254  for (unsigned I = 0, E = NumTokens; I != E; ++I) {
255  const Token &CurTok = Tokens[I];
256  // We don't want a space for the next token after a paste
257  // operator. In valid code, the token will get smooshed onto the
258  // preceding one anyway. In assembler-with-cpp mode, invalid
259  // pastes are allowed through: in this case, we do not want the
260  // extra whitespace to be added. For example, we want ". ## foo"
261  // -> ".foo" not ". foo".
262  if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace())
263  NextTokGetsSpace = true;
264 
265  if (VCtx.isVAOptToken(CurTok)) {
266  MadeChange = true;
267  assert(Tokens[I + 1].is(tok::l_paren) &&
268  "__VA_OPT__ must be followed by '('");
269 
270  ++I; // Skip the l_paren
272  ResultToks.size());
273 
274  continue;
275  }
276 
277  // We have entered into the __VA_OPT__ context, so handle tokens
278  // appropriately.
279  if (VCtx.isInVAOpt()) {
280  // If we are about to process a token that is either an argument to
281  // __VA_OPT__ or its closing rparen, then:
282  // 1) If the token is the closing rparen that exits us out of __VA_OPT__,
283  // perform any necessary stringification or placemarker processing,
284  // and/or skip to the next token.
285  // 2) else if macro was invoked without variadic arguments skip this
286  // token.
287  // 3) else (macro was invoked with variadic arguments) process the token
288  // normally.
289 
290  if (Tokens[I].is(tok::l_paren))
291  VCtx.sawOpeningParen(Tokens[I].getLocation());
292  // Continue skipping tokens within __VA_OPT__ if the macro was not
293  // called with variadic arguments, else let the rest of the loop handle
294  // this token. Note sawClosingParen() returns true only if the r_paren matches
295  // the closing r_paren of the __VA_OPT__.
296  if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) {
297  // Lazily expand __VA_ARGS__ when we see the first __VA_OPT__.
298  if (!CalledWithVariadicArguments.hasValue()) {
299  CalledWithVariadicArguments =
300  ActualArgs->invokedWithVariadicArgument(Macro, PP);
301  }
302  if (!*CalledWithVariadicArguments) {
303  // Skip this token.
304  continue;
305  }
306  // ... else the macro was called with variadic arguments, and we do not
307  // have a closing rparen - so process this token normally.
308  } else {
309  // Current token is the closing r_paren which marks the end of the
310  // __VA_OPT__ invocation, so handle any place-marker pasting (if
311  // empty) by removing hashhash either before (if exists) or after. And
312  // also stringify the entire contents if VAOPT was preceded by a hash,
313  // but do so only after any token concatenation that needs to occur
314  // within the contents of VAOPT.
315 
316  if (VCtx.hasStringifyOrCharifyBefore()) {
317  // Replace all the tokens just added from within VAOPT into a single
318  // stringified token. This requires token-pasting to eagerly occur
319  // within these tokens. If either the contents of VAOPT were empty
320  // or the macro wasn't called with any variadic arguments, the result
321  // is a token that represents an empty string.
322  stringifyVAOPTContents(ResultToks, VCtx,
323  /*ClosingParenLoc*/ Tokens[I].getLocation());
324 
325  } else if (/*No tokens within VAOPT*/
326  ResultToks.size() == VCtx.getNumberOfTokensPriorToVAOpt()) {
327  // Treat VAOPT as a placemarker token. Eat either the '##' before the
328  // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that
329  // hashhash was not a placemarker) or the '##'
330  // after VAOPT, but not both.
331 
332  if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {
333  ResultToks.pop_back();
334  } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {
335  ++I; // Skip the following hashhash.
336  }
337  } else {
338  // If there's a ## before the __VA_OPT__, we might have discovered
339  // that the __VA_OPT__ begins with a placeholder. We delay action on
340  // that to now to avoid messing up our stashed count of tokens before
341  // __VA_OPT__.
342  if (VCtx.beginsWithPlaceholder()) {
343  assert(VCtx.getNumberOfTokensPriorToVAOpt() > 0 &&
344  ResultToks.size() >= VCtx.getNumberOfTokensPriorToVAOpt() &&
345  ResultToks[VCtx.getNumberOfTokensPriorToVAOpt() - 1].is(
346  tok::hashhash) &&
347  "no token paste before __VA_OPT__");
348  ResultToks.erase(ResultToks.begin() +
349  VCtx.getNumberOfTokensPriorToVAOpt() - 1);
350  }
351  // If the expansion of __VA_OPT__ ends with a placeholder, eat any
352  // following '##' token.
353  if (VCtx.endsWithPlaceholder() && I + 1 != E &&
354  Tokens[I + 1].is(tok::hashhash)) {
355  ++I;
356  }
357  }
358  VCtx.reset();
359  // We processed __VA_OPT__'s closing paren (and the exit out of
360  // __VA_OPT__), so skip to the next token.
361  continue;
362  }
363  }
364 
365  // If we found the stringify operator, get the argument stringified. The
366  // preprocessor already verified that the following token is a macro
367  // parameter or __VA_OPT__ when the #define was lexed.
368 
369  if (CurTok.isOneOf(tok::hash, tok::hashat)) {
370  int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());
371  assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&
372  "Token following # is not an argument or __VA_OPT__!");
373 
374  if (ArgNo == -1) {
375  // Handle the __VA_OPT__ case.
376  VCtx.sawHashOrHashAtBefore(NextTokGetsSpace,
377  CurTok.is(tok::hashat));
378  continue;
379  }
380  // Else handle the simple argument case.
381  SourceLocation ExpansionLocStart =
382  getExpansionLocForMacroDefLoc(CurTok.getLocation());
383  SourceLocation ExpansionLocEnd =
384  getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());
385 
386  bool Charify = CurTok.is(tok::hashat);
387  const Token *UnexpArg = ActualArgs->getUnexpArgument(ArgNo);
389  UnexpArg, PP, Charify, ExpansionLocStart, ExpansionLocEnd);
391 
392  // The stringified/charified string leading space flag gets set to match
393  // the #/#@ operator.
394  if (NextTokGetsSpace)
396 
397  ResultToks.push_back(Res);
398  MadeChange = true;
399  ++I; // Skip arg name.
400  NextTokGetsSpace = false;
401  continue;
402  }
403 
404  // Find out if there is a paste (##) operator before or after the token.
405  bool NonEmptyPasteBefore =
406  !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
407  bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash);
408  bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash);
409  bool RParenAfter = I+1 != E && Tokens[I+1].is(tok::r_paren);
410 
411  assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&
412  "unexpected ## in ResultToks");
413 
414  // Otherwise, if this is not an argument token, just add the token to the
415  // output buffer.
416  IdentifierInfo *II = CurTok.getIdentifierInfo();
417  int ArgNo = II ? Macro->getParameterNum(II) : -1;
418  if (ArgNo == -1) {
419  // This isn't an argument, just add it.
420  ResultToks.push_back(CurTok);
421 
422  if (NextTokGetsSpace) {
423  ResultToks.back().setFlag(Token::LeadingSpace);
424  NextTokGetsSpace = false;
425  } else if (PasteBefore && !NonEmptyPasteBefore)
426  ResultToks.back().clearFlag(Token::LeadingSpace);
427 
428  continue;
429  }
430 
431  // An argument is expanded somehow, the result is different than the
432  // input.
433  MadeChange = true;
434 
435  // Otherwise, this is a use of the argument.
436 
437  // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
438  // are no trailing commas if __VA_ARGS__ is empty.
439  if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
440  MaybeRemoveCommaBeforeVaArgs(ResultToks,
441  /*HasPasteOperator=*/false,
442  Macro, ArgNo, PP))
443  continue;
444 
445  // If it is not the LHS/RHS of a ## operator, we must pre-expand the
446  // argument and substitute the expanded tokens into the result. This is
447  // C99 6.10.3.1p1.
448  if (!PasteBefore && !PasteAfter) {
449  const Token *ResultArgToks;
450 
451  // Only preexpand the argument if it could possibly need it. This
452  // avoids some work in common cases.
453  const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
454  if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
455  ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];
456  else
457  ResultArgToks = ArgTok; // Use non-preexpanded tokens.
458 
459  // If the arg token expanded into anything, append it.
460  if (ResultArgToks->isNot(tok::eof)) {
461  size_t FirstResult = ResultToks.size();
462  unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
463  ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
464 
465  // In Microsoft-compatibility mode, we follow MSVC's preprocessing
466  // behavior by not considering single commas from nested macro
467  // expansions as argument separators. Set a flag on the token so we can
468  // test for this later when the macro expansion is processed.
469  if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&
470  ResultToks.back().is(tok::comma))
471  ResultToks.back().setFlag(Token::IgnoredComma);
472 
473  // If the '##' came from expanding an argument, turn it into 'unknown'
474  // to avoid pasting.
475  for (Token &Tok : llvm::drop_begin(ResultToks, FirstResult))
476  if (Tok.is(tok::hashhash))
477  Tok.setKind(tok::unknown);
478 
479  if(ExpandLocStart.isValid()) {
480  updateLocForMacroArgTokens(CurTok.getLocation(),
481  ResultToks.begin()+FirstResult,
482  ResultToks.end());
483  }
484 
485  // If any tokens were substituted from the argument, the whitespace
486  // before the first token should match the whitespace of the arg
487  // identifier.
488  ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
489  NextTokGetsSpace);
490  ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
491  NextTokGetsSpace = false;
492  } else {
493  // We're creating a placeholder token. Usually this doesn't matter,
494  // but it can affect paste behavior when at the start or end of a
495  // __VA_OPT__.
496  if (NonEmptyPasteBefore) {
497  // We're imagining a placeholder token is inserted here. If this is
498  // the first token in a __VA_OPT__ after a ##, delete the ##.
499  assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__");
501  }
502  if (RParenAfter)
504  }
505  continue;
506  }
507 
508  // Okay, we have a token that is either the LHS or RHS of a paste (##)
509  // argument. It gets substituted as its non-pre-expanded tokens.
510  const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
511  unsigned NumToks = MacroArgs::getArgLength(ArgToks);
512  if (NumToks) { // Not an empty argument?
513  bool VaArgsPseudoPaste = false;
514  // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned
515  // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when
516  // the expander tries to paste ',' with the first token of the __VA_ARGS__
517  // expansion.
518  if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
519  ResultToks[ResultToks.size()-2].is(tok::comma) &&
520  (unsigned)ArgNo == Macro->getNumParams()-1 &&
521  Macro->isVariadic()) {
522  VaArgsPseudoPaste = true;
523  // Remove the paste operator, report use of the extension.
524  PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
525  }
526 
527  ResultToks.append(ArgToks, ArgToks+NumToks);
528 
529  // If the '##' came from expanding an argument, turn it into 'unknown'
530  // to avoid pasting.
531  for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
532  ResultToks.end())) {
533  if (Tok.is(tok::hashhash))
534  Tok.setKind(tok::unknown);
535  }
536 
537  if (ExpandLocStart.isValid()) {
538  updateLocForMacroArgTokens(CurTok.getLocation(),
539  ResultToks.end()-NumToks, ResultToks.end());
540  }
541 
542  // Transfer the leading whitespace information from the token
543  // (the macro argument) onto the first token of the
544  // expansion. Note that we don't do this for the GNU
545  // pseudo-paste extension ", ## __VA_ARGS__".
546  if (!VaArgsPseudoPaste) {
547  ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,
548  false);
549  ResultToks[ResultToks.size() - NumToks].setFlagValue(
550  Token::LeadingSpace, NextTokGetsSpace);
551  }
552 
553  NextTokGetsSpace = false;
554  continue;
555  }
556 
557  // If an empty argument is on the LHS or RHS of a paste, the standard (C99
558  // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We
559  // implement this by eating ## operators when a LHS or RHS expands to
560  // empty.
561  if (PasteAfter) {
562  // Discard the argument token and skip (don't copy to the expansion
563  // buffer) the paste operator after it.
564  ++I;
565  continue;
566  }
567 
568  if (RParenAfter)
570 
571  // If this is on the RHS of a paste operator, we've already copied the
572  // paste operator to the ResultToks list, unless the LHS was empty too.
573  // Remove it.
574  assert(PasteBefore);
575  if (NonEmptyPasteBefore) {
576  assert(ResultToks.back().is(tok::hashhash));
577  // Do not remove the paste operator if it is the one before __VA_OPT__
578  // (and we are still processing tokens within VA_OPT). We handle the case
579  // of removing the paste operator if __VA_OPT__ reduces to the notional
580  // placemarker above when we encounter the closing paren of VA_OPT.
581  if (!VCtx.isInVAOpt() ||
582  ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt())
583  ResultToks.pop_back();
584  else
586  }
587 
588  // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
589  // and if the macro had at least one real argument, and if the token before
590  // the ## was a comma, remove the comma. This is a GCC extension which is
591  // disabled when using -std=c99.
592  if (ActualArgs->isVarargsElidedUse())
593  MaybeRemoveCommaBeforeVaArgs(ResultToks,
594  /*HasPasteOperator=*/true,
595  Macro, ArgNo, PP);
596  }
597 
598  // If anything changed, install this as the new Tokens list.
599  if (MadeChange) {
600  assert(!OwnsTokens && "This would leak if we already own the token list");
601  // This is deleted in the dtor.
602  NumTokens = ResultToks.size();
603  // The tokens will be added to Preprocessor's cache and will be removed
604  // when this TokenLexer finishes lexing them.
605  Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
606 
607  // The preprocessor cache of macro expanded tokens owns these tokens,not us.
608  OwnsTokens = false;
609  }
610 }
611 
612 /// Checks if two tokens form wide string literal.
613 static bool isWideStringLiteralFromMacro(const Token &FirstTok,
614  const Token &SecondTok) {
615  return FirstTok.is(tok::identifier) &&
616  FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral() &&
617  SecondTok.stringifiedInMacro();
618 }
619 
620 /// Lex - Lex and return a token from this macro stream.
621 bool TokenLexer::Lex(Token &Tok) {
622  // Lexing off the end of the macro, pop this macro off the expansion stack.
623  if (isAtEnd()) {
624  // If this is a macro (not a token stream), mark the macro enabled now
625  // that it is no longer being expanded.
626  if (Macro) Macro->EnableMacro();
627 
628  Tok.startToken();
629  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
630  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);
631  if (CurTokenIdx == 0)
633  return PP.HandleEndOfTokenLexer(Tok);
634  }
635 
637 
638  // If this is the first token of the expanded result, we inherit spacing
639  // properties later.
640  bool isFirstToken = CurTokenIdx == 0;
641 
642  // Get the next token to return.
643  Tok = Tokens[CurTokenIdx++];
644  if (IsReinject)
646 
647  bool TokenIsFromPaste = false;
648 
649  // If this token is followed by a token paste (##) operator, paste the tokens!
650  // Note that ## is a normal token when not expanding a macro.
651  if (!isAtEnd() && Macro &&
652  (Tokens[CurTokenIdx].is(tok::hashhash) ||
653  // Special processing of L#x macros in -fms-compatibility mode.
654  // Microsoft compiler is able to form a wide string literal from
655  // 'L#macro_arg' construct in a function-like macro.
656  (PP.getLangOpts().MSVCCompat &&
657  isWideStringLiteralFromMacro(Tok, Tokens[CurTokenIdx])))) {
658  // When handling the microsoft /##/ extension, the final token is
659  // returned by pasteTokens, not the pasted token.
660  if (pasteTokens(Tok))
661  return true;
662 
663  TokenIsFromPaste = true;
664  }
665 
666  // The token's current location indicate where the token was lexed from. We
667  // need this information to compute the spelling of the token, but any
668  // diagnostics for the expanded token should appear as if they came from
669  // ExpansionLoc. Pull this information together into a new SourceLocation
670  // that captures all of this.
671  if (ExpandLocStart.isValid() && // Don't do this for token streams.
672  // Check that the token's location was not already set properly.
673  SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {
674  SourceLocation instLoc;
675  if (Tok.is(tok::comment)) {
676  instLoc = SM.createExpansionLoc(Tok.getLocation(),
677  ExpandLocStart,
678  ExpandLocEnd,
679  Tok.getLength());
680  } else {
681  instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());
682  }
683 
684  Tok.setLocation(instLoc);
685  }
686 
687  // If this is the first token, set the lexical properties of the token to
688  // match the lexical properties of the macro identifier.
689  if (isFirstToken) {
690  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
691  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
692  } else {
693  // If this is not the first token, we may still need to pass through
694  // leading whitespace if we've expanded a macro.
695  if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);
696  if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);
697  }
698  AtStartOfLine = false;
699  HasLeadingSpace = false;
700 
701  // Handle recursive expansion!
702  if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) {
703  // Change the kind of this identifier to the appropriate token kind, e.g.
704  // turning "for" into a keyword.
705  IdentifierInfo *II = Tok.getIdentifierInfo();
706  Tok.setKind(II->getTokenID());
707 
708  // If this identifier was poisoned and from a paste, emit an error. This
709  // won't be handled by Preprocessor::HandleIdentifier because this is coming
710  // from a macro expansion.
711  if (II->isPoisoned() && TokenIsFromPaste) {
712  PP.HandlePoisonedIdentifier(Tok);
713  }
714 
715  if (!DisableMacroExpansion && II->isHandleIdentifierCase())
716  return PP.HandleIdentifier(Tok);
717  }
718 
719  // Otherwise, return a normal token.
720  return true;
721 }
722 
723 bool TokenLexer::pasteTokens(Token &Tok) {
724  return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx);
725 }
726 
727 /// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ##
728 /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there
729 /// are more ## after it, chomp them iteratively. Return the result as LHSTok.
730 /// If this returns true, the caller should immediately return the token.
731 bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
732  unsigned int &CurIdx) {
733  assert(CurIdx > 0 && "## can not be the first token within tokens");
734  assert((TokenStream[CurIdx].is(tok::hashhash) ||
735  (PP.getLangOpts().MSVCCompat &&
736  isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) &&
737  "Token at this Index must be ## or part of the MSVC 'L "
738  "#macro-arg' pasting pair");
739 
740  // MSVC: If previous token was pasted, this must be a recovery from an invalid
741  // paste operation. Ignore spaces before this token to mimic MSVC output.
742  // Required for generating valid UUID strings in some MS headers.
743  if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) &&
744  TokenStream[CurIdx - 2].is(tok::hashhash))
746 
747  SmallString<128> Buffer;
748  const char *ResultTokStrPtr = nullptr;
749  SourceLocation StartLoc = LHSTok.getLocation();
750  SourceLocation PasteOpLoc;
751 
752  auto IsAtEnd = [&TokenStream, &CurIdx] {
753  return TokenStream.size() == CurIdx;
754  };
755 
756  do {
757  // Consume the ## operator if any.
758  PasteOpLoc = TokenStream[CurIdx].getLocation();
759  if (TokenStream[CurIdx].is(tok::hashhash))
760  ++CurIdx;
761  assert(!IsAtEnd() && "No token on the RHS of a paste operator!");
762 
763  // Get the RHS token.
764  const Token &RHS = TokenStream[CurIdx];
765 
766  // Allocate space for the result token. This is guaranteed to be enough for
767  // the two tokens.
768  Buffer.resize(LHSTok.getLength() + RHS.getLength());
769 
770  // Get the spelling of the LHS token in Buffer.
771  const char *BufPtr = &Buffer[0];
772  bool Invalid = false;
773  unsigned LHSLen = PP.getSpelling(LHSTok, BufPtr, &Invalid);
774  if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer!
775  memcpy(&Buffer[0], BufPtr, LHSLen);
776  if (Invalid)
777  return true;
778 
779  BufPtr = Buffer.data() + LHSLen;
780  unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid);
781  if (Invalid)
782  return true;
783  if (RHSLen && BufPtr != &Buffer[LHSLen])
784  // Really, we want the chars in Buffer!
785  memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
786 
787  // Trim excess space.
788  Buffer.resize(LHSLen+RHSLen);
789 
790  // Plop the pasted result (including the trailing newline and null) into a
791  // scratch buffer where we can lex it.
792  Token ResultTokTmp;
793  ResultTokTmp.startToken();
794 
795  // Claim that the tmp token is a string_literal so that we can get the
796  // character pointer back from CreateString in getLiteralData().
797  ResultTokTmp.setKind(tok::string_literal);
798  PP.CreateString(Buffer, ResultTokTmp);
799  SourceLocation ResultTokLoc = ResultTokTmp.getLocation();
800  ResultTokStrPtr = ResultTokTmp.getLiteralData();
801 
802  // Lex the resultant pasted token into Result.
803  Token Result;
804 
805  if (LHSTok.isAnyIdentifier() && RHS.isAnyIdentifier()) {
806  // Common paste case: identifier+identifier = identifier. Avoid creating
807  // a lexer and other overhead.
808  PP.IncrementPasteCounter(true);
809  Result.startToken();
810  Result.setKind(tok::raw_identifier);
811  Result.setRawIdentifierData(ResultTokStrPtr);
812  Result.setLocation(ResultTokLoc);
813  Result.setLength(LHSLen+RHSLen);
814  } else {
815  PP.IncrementPasteCounter(false);
816 
817  assert(ResultTokLoc.isFileID() &&
818  "Should be a raw location into scratch buffer");
819  SourceManager &SourceMgr = PP.getSourceManager();
820  FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
821 
822  bool Invalid = false;
823  const char *ScratchBufStart
824  = SourceMgr.getBufferData(LocFileID, &Invalid).data();
825  if (Invalid)
826  return false;
827 
828  // Make a lexer to lex this string from. Lex just this one token.
829  // Make a lexer object so that we lex and expand the paste result.
830  Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
831  PP.getLangOpts(), ScratchBufStart,
832  ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
833 
834  // Lex a token in raw mode. This way it won't look up identifiers
835  // automatically, lexing off the end will return an eof token, and
836  // warnings are disabled. This returns true if the result token is the
837  // entire buffer.
838  bool isInvalid = !TL.LexFromRawLexer(Result);
839 
840  // If we got an EOF token, we didn't form even ONE token. For example, we
841  // did "/ ## /" to get "//".
842  isInvalid |= Result.is(tok::eof);
843 
844  // If pasting the two tokens didn't form a full new token, this is an
845  // error. This occurs with "x ## +" and other stuff. Return with LHSTok
846  // unmodified and with RHS as the next token to lex.
847  if (isInvalid) {
848  // Explicitly convert the token location to have proper expansion
849  // information so that the user knows where it came from.
851  SourceLocation Loc =
852  SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);
853 
854  // Test for the Microsoft extension of /##/ turning into // here on the
855  // error path.
856  if (PP.getLangOpts().MicrosoftExt && LHSTok.is(tok::slash) &&
857  RHS.is(tok::slash)) {
858  HandleMicrosoftCommentPaste(LHSTok, Loc);
859  return true;
860  }
861 
862  // Do not emit the error when preprocessing assembler code.
863  if (!PP.getLangOpts().AsmPreprocessor) {
864  // If we're in microsoft extensions mode, downgrade this from a hard
865  // error to an extension that defaults to an error. This allows
866  // disabling it.
867  PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms
868  : diag::err_pp_bad_paste)
869  << Buffer;
870  }
871 
872  // An error has occurred so exit loop.
873  break;
874  }
875 
876  // Turn ## into 'unknown' to avoid # ## # from looking like a paste
877  // operator.
878  if (Result.is(tok::hashhash))
879  Result.setKind(tok::unknown);
880  }
881 
882  // Transfer properties of the LHS over the Result.
883  Result.setFlagValue(Token::StartOfLine , LHSTok.isAtStartOfLine());
884  Result.setFlagValue(Token::LeadingSpace, LHSTok.hasLeadingSpace());
885 
886  // Finally, replace LHS with the result, consume the RHS, and iterate.
887  ++CurIdx;
888  LHSTok = Result;
889  } while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));
890 
891  SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation();
892 
893  // The token's current location indicate where the token was lexed from. We
894  // need this information to compute the spelling of the token, but any
895  // diagnostics for the expanded token should appear as if the token was
896  // expanded from the full ## expression. Pull this information together into
897  // a new SourceLocation that captures all of this.
899  if (StartLoc.isFileID())
900  StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
901  if (EndLoc.isFileID())
902  EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
903  FileID MacroFID = SM.getFileID(MacroExpansionStart);
904  while (SM.getFileID(StartLoc) != MacroFID)
905  StartLoc = SM.getImmediateExpansionRange(StartLoc).getBegin();
906  while (SM.getFileID(EndLoc) != MacroFID)
907  EndLoc = SM.getImmediateExpansionRange(EndLoc).getEnd();
908 
909  LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc,
910  LHSTok.getLength()));
911 
912  // Now that we got the result token, it will be subject to expansion. Since
913  // token pasting re-lexes the result token in raw mode, identifier information
914  // isn't looked up. As such, if the result is an identifier, look up id info.
915  if (LHSTok.is(tok::raw_identifier)) {
916  // Look up the identifier info for the token. We disabled identifier lookup
917  // by saying we're skipping contents, so we need to do this manually.
918  PP.LookUpIdentifierInfo(LHSTok);
919  }
920  return false;
921 }
922 
923 /// isNextTokenLParen - If the next token lexed will pop this macro off the
924 /// expansion stack, return 2. If the next unexpanded token is a '(', return
925 /// 1, otherwise return 0.
927  // Out of tokens?
928  if (isAtEnd())
929  return 2;
930  return Tokens[CurTokenIdx].is(tok::l_paren);
931 }
932 
933 /// isParsingPreprocessorDirective - Return true if we are in the middle of a
934 /// preprocessor directive.
936  return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd();
937 }
938 
939 /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
940 /// together to form a comment that comments out everything in the current
941 /// macro, other active macros, and anything left on the current physical
942 /// source line of the expanded buffer. Handle this by returning the
943 /// first token on the next line.
944 void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {
945  PP.Diag(OpLoc, diag::ext_comment_paste_microsoft);
946 
947  // We 'comment out' the rest of this macro by just ignoring the rest of the
948  // tokens that have not been lexed yet, if any.
949 
950  // Since this must be a macro, mark the macro enabled now that it is no longer
951  // being expanded.
952  assert(Macro && "Token streams can't paste comments");
953  Macro->EnableMacro();
954 
956 }
957 
958 /// If \arg loc is a file ID and points inside the current macro
959 /// definition, returns the appropriate source location pointing at the
960 /// macro expansion source location entry, otherwise it returns an invalid
961 /// SourceLocation.
963 TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {
964  assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&
965  "Not appropriate for token streams");
966  assert(loc.isValid() && loc.isFileID());
967 
969  assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
970  "Expected loc to come from the macro definition");
971 
972  SourceLocation::UIntTy relativeOffset = 0;
973  SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);
974  return MacroExpansionStart.getLocWithOffset(relativeOffset);
975 }
976 
977 /// Finds the tokens that are consecutive (from the same FileID)
978 /// creates a single SLocEntry, and assigns SourceLocations to each token that
979 /// point to that SLocEntry. e.g for
980 /// assert(foo == bar);
981 /// There will be a single SLocEntry for the "foo == bar" chunk and locations
982 /// for the 'foo', '==', 'bar' tokens will point inside that chunk.
983 ///
984 /// \arg begin_tokens will be updated to a position past all the found
985 /// consecutive tokens.
987  SourceLocation InstLoc,
988  Token *&begin_tokens,
989  Token * end_tokens) {
990  assert(begin_tokens < end_tokens);
991 
992  SourceLocation FirstLoc = begin_tokens->getLocation();
993  SourceLocation CurLoc = FirstLoc;
994 
995  // Compare the source location offset of tokens and group together tokens that
996  // are close, even if their locations point to different FileIDs. e.g.
997  //
998  // |bar | foo | cake | (3 tokens from 3 consecutive FileIDs)
999  // ^ ^
1000  // |bar foo cake| (one SLocEntry chunk for all tokens)
1001  //
1002  // we can perform this "merge" since the token's spelling location depends
1003  // on the relative offset.
1004 
1005  Token *NextTok = begin_tokens + 1;
1006  for (; NextTok < end_tokens; ++NextTok) {
1007  SourceLocation NextLoc = NextTok->getLocation();
1008  if (CurLoc.isFileID() != NextLoc.isFileID())
1009  break; // Token from different kind of FileID.
1010 
1011  SourceLocation::IntTy RelOffs;
1012  if (!SM.isInSameSLocAddrSpace(CurLoc, NextLoc, &RelOffs))
1013  break; // Token from different local/loaded location.
1014  // Check that token is not before the previous token or more than 50
1015  // "characters" away.
1016  if (RelOffs < 0 || RelOffs > 50)
1017  break;
1018 
1019  if (CurLoc.isMacroID() && !SM.isWrittenInSameFile(CurLoc, NextLoc))
1020  break; // Token from a different macro.
1021 
1022  CurLoc = NextLoc;
1023  }
1024 
1025  // For the consecutive tokens, find the length of the SLocEntry to contain
1026  // all of them.
1027  Token &LastConsecutiveTok = *(NextTok-1);
1028  SourceLocation::IntTy LastRelOffs = 0;
1029  SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(),
1030  &LastRelOffs);
1031  SourceLocation::UIntTy FullLength =
1032  LastRelOffs + LastConsecutiveTok.getLength();
1033 
1034  // Create a macro expansion SLocEntry that will "contain" all of the tokens.
1035  SourceLocation Expansion =
1036  SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength);
1037 
1038  // Change the location of the tokens from the spelling location to the new
1039  // expanded location.
1040  for (; begin_tokens < NextTok; ++begin_tokens) {
1041  Token &Tok = *begin_tokens;
1042  SourceLocation::IntTy RelOffs = 0;
1043  SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs);
1044  Tok.setLocation(Expansion.getLocWithOffset(RelOffs));
1045  }
1046 }
1047 
1048 /// Creates SLocEntries and updates the locations of macro argument
1049 /// tokens to their new expanded locations.
1050 ///
1051 /// \param ArgIdSpellLoc the location of the macro argument id inside the macro
1052 /// definition.
1053 void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
1054  Token *begin_tokens,
1055  Token *end_tokens) {
1057 
1058  SourceLocation InstLoc =
1059  getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
1060 
1061  while (begin_tokens < end_tokens) {
1062  // If there's only one token just create a SLocEntry for it.
1063  if (end_tokens - begin_tokens == 1) {
1064  Token &Tok = *begin_tokens;
1065  Tok.setLocation(SM.createMacroArgExpansionLoc(Tok.getLocation(),
1066  InstLoc,
1067  Tok.getLength()));
1068  return;
1069  }
1070 
1071  updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens);
1072  }
1073 }
1074 
1075 void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
1076  AtStartOfLine = Result.isAtStartOfLine();
1077  HasLeadingSpace = Result.hasLeadingSpace();
1078 }
clang::Token::startToken
void startToken()
Reset all flags to cleared.
Definition: Token.h:170
clang::Preprocessor::CreateString
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
Definition: Preprocessor.cpp:486
clang::Token::IgnoredComma
@ IgnoredComma
Definition: Token.h:82
clang::Token::IsReinjected
@ IsReinjected
Definition: Token.h:87
clang::MacroInfo::tokens_end
const_tokens_iterator tokens_end() const
Definition: MacroInfo.h:246
clang::VAOptExpansionContext::hasPlaceholderBeforeRParen
void hasPlaceholderBeforeRParen()
Definition: VariadicMacroSupport.h:180
clang::Token::hasLeadingSpace
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:271
clang::VAOptExpansionContext::sawOpeningParen
void sawOpeningParen(SourceLocation LParenLoc)
Call this function each time an lparen is seen.
Definition: VariadicMacroSupport.h:107
clang::VAOptExpansionContext::sawHashOrHashAtBefore
void sawHashOrHashAtBefore(const bool HasLeadingSpace, const bool IsHashAt)
Definition: VariadicMacroSupport.h:171
clang::Preprocessor::HandleEndOfTokenLexer
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
Definition: PPLexerChange.cpp:560
clang::VAOptExpansionContext::getNumberOfTokensPriorToVAOpt
unsigned int getNumberOfTokensPriorToVAOpt() const
Definition: VariadicMacroSupport.h:206
clang::Preprocessor::HandleIdentifier
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
Definition: Preprocessor.cpp:800
clang::Token::getLiteralData
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:216
llvm::SmallVector
Definition: LLVM.h:38
clang::SourceLocation
Encodes a location in the source.
Definition: SourceLocation.h:86
clang::MacroInfo::tokens_begin
const_tokens_iterator tokens_begin() const
Definition: MacroInfo.h:245
clang::SourceLocation::getLocWithOffset
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
Definition: SourceLocation.h:134
clang::Token::getIdentifierInfo
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:178
clang::MacroArgs
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
clang::VAOptExpansionContext::beginsWithPlaceholder
bool beginsWithPlaceholder() const
Definition: VariadicMacroSupport.h:186
isWideStringLiteralFromMacro
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
Definition: TokenLexer.cpp:613
clang::VAOptExpansionContext
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a macro during...
Definition: VariadicMacroSupport.h:118
memcpy
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
Definition: __clang_cuda_device_functions.h:1549
clang::TokenLexer::Lex
bool Lex(Token &Tok)
Lex and return a token from this macro stream.
Definition: TokenLexer.cpp:621
clang::Token::isAtStartOfLine
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:267
llvm::Optional< bool >
clang::MacroInfo
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
SourceManager.h
clang::Preprocessor::HandleMicrosoftCommentPaste
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
Definition: PPLexerChange.cpp:598
clang::index::SymbolKind::Macro
@ Macro
clang::Preprocessor::IncrementPasteCounter
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Definition: Preprocessor.h:1972
clang::IdentifierInfo::getTokenID
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
Definition: IdentifierTable.h:261
clang::Token
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
TokenLexer.h
clang::SourceManager
This class handles loading and caching of source files into memory.
Definition: SourceManager.h:627
clang::Preprocessor::getLangOpts
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:994
Preprocessor.h
clang::Lexer
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.
Definition: Lexer.h:77
clang::Token::StringifiedInMacro
@ StringifiedInMacro
Definition: Token.h:83
clang::SourceLocation::IntTy
int32_t IntTy
Definition: SourceLocation.h:94
TokenKinds.h
clang::IdentifierInfo::isStr
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
Definition: IdentifierTable.h:176
clang::MacroArgs::getPreExpArgument
const std::vector< Token > & getPreExpArgument(unsigned Arg, Preprocessor &PP)
getPreExpArgument - Return the pre-expanded form of the specified argument.
Definition: MacroArgs.cpp:161
clang::MacroInfo::DisableMacro
void DisableMacro()
Definition: MacroInfo.h:289
clang::VAOptExpansionContext::hasCharifyBefore
bool hasCharifyBefore() const
Definition: VariadicMacroSupport.h:197
clang::VAOptExpansionContext::getLeadingSpaceForStringifiedToken
bool getLeadingSpaceForStringifiedToken() const
Definition: VariadicMacroSupport.h:212
clang::VAOptExpansionContext::isVAOptToken
bool isVAOptToken(const Token &T) const
Definition: VariadicMacroSupport.h:77
clang::Token::setFlagValue
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Definition: Token.h:258
LangOptions.h
Diagnostic.h
LexDiagnostic.h
clang::Preprocessor::Diag
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
Definition: Preprocessor.h:1806
clang::Token::is
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:97
clang::MacroArgs::isVarargsElidedUse
bool isVarargsElidedUse() const
isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...
Definition: MacroArgs.h:102
IdentifierTable.h
clang::SourceManager::getFileID
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
Definition: SourceManager.h:1108
llvm::SmallString< 128 >
clang::Token::LeadingEmptyMacro
@ LeadingEmptyMacro
Definition: Token.h:79
clang::VAOptExpansionContext::reset
void reset()
Definition: VariadicMacroSupport.h:159
clang::VAOptExpansionContext::isInVAOpt
bool isInVAOpt() const
Returns true if we have seen the VA_OPT and '(' but before having seen the matching ')'.
Definition: VariadicMacroSupport.h:83
isInvalid
static bool isInvalid(LocType Loc, bool *Invalid)
Definition: SourceManager.cpp:1225
clang::SourceLocation::isFileID
bool isFileID() const
Definition: SourceLocation.h:102
clang::Token::isAnyIdentifier
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
Definition: Token.h:108
clang::TokenLexer::isNextTokenLParen
unsigned isNextTokenLParen() const
If the next token lexed will pop this macro off the expansion stack, return 2.
Definition: TokenLexer.cpp:926
clang::Token::isNot
bool isNot(tok::TokenKind K) const
Definition: Token.h:98
clang::MacroArgs::getUnexpArgument
const Token * getUnexpArgument(unsigned Arg) const
getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...
Definition: MacroArgs.cpp:118
clang::MacroArgs::getArgLength
static unsigned getArgLength(const Token *ArgPtr)
getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens,...
Definition: MacroArgs.cpp:108
clang::VAOptExpansionContext::hasPlaceholderAfterHashhashAtStart
void hasPlaceholderAfterHashhashAtStart()
Definition: VariadicMacroSupport.h:179
clang::Token::isLiteral
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:114
clang::Token::getLength
unsigned getLength() const
Definition: Token.h:128
clang::Token::LeadingSpace
@ LeadingSpace
Definition: Token.h:75
clang::SourceManager::getLocForStartOfFile
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file.
Definition: SourceManager.h:1123
clang::Preprocessor::getSourceManager
SourceManager & getSourceManager() const
Definition: Preprocessor.h:998
clang::MacroArgs::destroy
void destroy(Preprocessor &PP)
destroy - Destroy and deallocate the memory for this object.
Definition: MacroArgs.cpp:78
clang::Preprocessor::getSpelling
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
Definition: Preprocessor.h:1821
MacroArgs.h
clang::Decl::setLocation
void setLocation(SourceLocation L)
Definition: DeclBase.h:426
SourceLocation.h
clang::VAOptExpansionContext::endsWithPlaceholder
bool endsWithPlaceholder() const
Definition: VariadicMacroSupport.h:191
clang::MacroArgs::invokedWithVariadicArgument
bool invokedWithVariadicArgument(const MacroInfo *const MI, Preprocessor &PP)
Returns true if the macro was defined with a variadic (ellipsis) parameter AND was invoked with at le...
Definition: MacroArgs.cpp:136
clang::IdentifierInfo::isPoisoned
bool isPoisoned() const
Return true if this token has been poisoned.
Definition: IdentifierTable.h:354
clang::TokenLexer::isParsingPreprocessorDirective
bool isParsingPreprocessorDirective() const
isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive.
Definition: TokenLexer.cpp:935
llvm::ArrayRef
Definition: LLVM.h:34
Lexer.h
clang::TokenLexer::Init
void Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, MacroArgs *Actuals)
Initialize this TokenLexer to expand from the specified macro with the specified argument information...
Definition: TokenLexer.cpp:38
clang::Token::StartOfLine
@ StartOfLine
Definition: Token.h:73
clang::SourceManager::getBufferData
StringRef getBufferData(FileID FID, bool *Invalid=nullptr) const
Return a StringRef to the source buffer data for the specified FileID.
Definition: SourceManager.cpp:733
clang::IdentifierInfo
One of these records is kept for each identifier that is lexed.
Definition: IdentifierTable.h:84
clang::Token::clearFlag
void clearFlag(TokenFlags Flag)
Unset the specified flag.
Definition: Token.h:245
clang::Token::getLocation
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:125
clang::SourceLocation::isMacroID
bool isMacroID() const
Definition: SourceLocation.h:103
clang::IdentifierInfo::isHandleIdentifierCase
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier.
Definition: IdentifierTable.h:380
clang::MacroInfo::getDefinitionLength
unsigned getDefinitionLength(const SourceManager &SM) const
Get length in characters of the macro definition.
Definition: MacroInfo.h:135
clang::Token::isAnnotation
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:119
clang::VAOptExpansionContext::sawClosingParen
bool sawClosingParen()
Call this function each time an rparen is seen.
Definition: VariadicMacroSupport.h:100
clang::Token::setLocation
void setLocation(SourceLocation L)
Definition: Token.h:133
clang::MacroInfo::getNumParams
unsigned getNumParams() const
Definition: MacroInfo.h:185
clang
Definition: CalledOnceCheck.h:17
clang::Preprocessor::LookUpIdentifierInfo
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Definition: Preprocessor.cpp:700
MacroInfo.h
clang::VAOptExpansionContext::getEOFTok
const Token & getEOFTok() const
Definition: VariadicMacroSupport.h:169
clang::Token::CommaAfterElided
@ CommaAfterElided
Definition: Token.h:85
clang::VAOptExpansionContext::hasStringifyOrCharifyBefore
bool hasStringifyOrCharifyBefore() const
Definition: VariadicMacroSupport.h:202
clang::Token::stringifiedInMacro
bool stringifiedInMacro() const
Returns true if this token is formed by macro by stringizing or charizing operator.
Definition: Token.h:299
clang::FileID
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Definition: SourceLocation.h:38
clang::SourceLocation::isValid
bool isValid() const
Return true if this is a valid SourceLocation object.
Definition: SourceLocation.h:110
updateConsecutiveMacroArgTokens
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation InstLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry,...
Definition: TokenLexer.cpp:986
VariadicMacroSupport.h
clang::Token::setKind
void setKind(tok::TokenKind K)
Definition: Token.h:93
clang::Token::isOneOf
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:99
clang::VAOptExpansionContext::getVAOptLoc
SourceLocation getVAOptLoc() const
Definition: VariadicMacroSupport.h:229
clang::Token::setFlag
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:235
clang::MacroArgs::ArgNeedsPreexpansion
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
ArgNeedsPreexpansion - If we can prove that the argument won't be affected by pre-expansion,...
Definition: MacroArgs.cpp:146
clang::comments::tok::eof
@ eof
Definition: CommentLexer.h:33
llvm::SmallVectorImpl
Definition: Randstruct.h:18
clang::MacroInfo::isFunctionLike
bool isFunctionLike() const
Definition: MacroInfo.h:202
clang::Preprocessor
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:131
SM
#define SM(sm)
Definition: Cuda.cpp:81
clang::Preprocessor::HandlePoisonedIdentifier
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
Definition: Preprocessor.cpp:753
clang::MacroArgs::StringifyArgument
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...
Definition: MacroArgs.cpp:208
Token.h
clang::RISCV::Invalid
@ Invalid
Definition: RISCVVIntrinsicUtils.h:148
clang::VAOptExpansionContext::sawVAOptFollowedByOpeningParens
void sawVAOptFollowedByOpeningParens(const SourceLocation VAOptLoc, const unsigned int NumPriorTokens)
Definition: VariadicMacroSupport.h:218
clang::SourceLocation::UIntTy
uint32_t UIntTy
Definition: SourceLocation.h:93