clang  14.0.0git
TokenLexer.cpp
Go to the documentation of this file.
1 //===- TokenLexer.cpp - Lex from a token stream ---------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the TokenLexer interface.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "clang/Lex/TokenLexer.h"
14 #include "clang/Basic/Diagnostic.h"
19 #include "clang/Basic/TokenKinds.h"
21 #include "clang/Lex/Lexer.h"
22 #include "clang/Lex/MacroArgs.h"
23 #include "clang/Lex/MacroInfo.h"
24 #include "clang/Lex/Preprocessor.h"
25 #include "clang/Lex/Token.h"
27 #include "llvm/ADT/ArrayRef.h"
28 #include "llvm/ADT/SmallString.h"
29 #include "llvm/ADT/SmallVector.h"
30 #include "llvm/ADT/iterator_range.h"
31 #include <cassert>
32 #include <cstring>
33 
34 using namespace clang;
35 
36 /// Create a TokenLexer for the specified macro with the specified actual
37 /// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
39  MacroArgs *Actuals) {
40  // If the client is reusing a TokenLexer, make sure to free any memory
41  // associated with it.
42  destroy();
43 
44  Macro = MI;
45  ActualArgs = Actuals;
46  CurTokenIdx = 0;
47 
48  ExpandLocStart = Tok.getLocation();
49  ExpandLocEnd = ELEnd;
50  AtStartOfLine = Tok.isAtStartOfLine();
51  HasLeadingSpace = Tok.hasLeadingSpace();
52  NextTokGetsSpace = false;
53  Tokens = &*Macro->tokens_begin();
54  OwnsTokens = false;
55  DisableMacroExpansion = false;
56  IsReinject = false;
57  NumTokens = Macro->tokens_end()-Macro->tokens_begin();
58  MacroExpansionStart = SourceLocation();
59 
61  MacroStartSLocOffset = SM.getNextLocalOffset();
62 
63  if (NumTokens > 0) {
64  assert(Tokens[0].getLocation().isValid());
65  assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
66  "Macro defined in macro?");
67  assert(ExpandLocStart.isValid());
68 
69  // Reserve a source location entry chunk for the length of the macro
70  // definition. Tokens that get lexed directly from the definition will
71  // have their locations pointing inside this chunk. This is to avoid
72  // creating separate source location entries for each token.
73  MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());
74  MacroDefLength = Macro->getDefinitionLength(SM);
75  MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,
76  ExpandLocStart,
77  ExpandLocEnd,
78  MacroDefLength);
79  }
80 
81  // If this is a function-like macro, expand the arguments and change
82  // Tokens to point to the expanded tokens.
83  if (Macro->isFunctionLike() && Macro->getNumParams())
84  ExpandFunctionArguments();
85 
86  // Mark the macro as currently disabled, so that it is not recursively
87  // expanded. The macro must be disabled only after argument pre-expansion of
88  // function-like macro arguments occurs.
89  Macro->DisableMacro();
90 }
91 
92 /// Create a TokenLexer for the specified token stream. This does not
93 /// take ownership of the specified token vector.
94 void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
95  bool disableMacroExpansion, bool ownsTokens,
96  bool isReinject) {
97  assert(!isReinject || disableMacroExpansion);
98  // If the client is reusing a TokenLexer, make sure to free any memory
99  // associated with it.
100  destroy();
101 
102  Macro = nullptr;
103  ActualArgs = nullptr;
104  Tokens = TokArray;
105  OwnsTokens = ownsTokens;
106  DisableMacroExpansion = disableMacroExpansion;
107  IsReinject = isReinject;
108  NumTokens = NumToks;
109  CurTokenIdx = 0;
110  ExpandLocStart = ExpandLocEnd = SourceLocation();
111  AtStartOfLine = false;
112  HasLeadingSpace = false;
113  NextTokGetsSpace = false;
114  MacroExpansionStart = SourceLocation();
115 
116  // Set HasLeadingSpace/AtStartOfLine so that the first token will be
117  // returned unmodified.
118  if (NumToks != 0) {
119  AtStartOfLine = TokArray[0].isAtStartOfLine();
120  HasLeadingSpace = TokArray[0].hasLeadingSpace();
121  }
122 }
123 
124 void TokenLexer::destroy() {
125  // If this was a function-like macro that actually uses its arguments, delete
126  // the expanded tokens.
127  if (OwnsTokens) {
128  delete [] Tokens;
129  Tokens = nullptr;
130  OwnsTokens = false;
131  }
132 
133  // TokenLexer owns its formal arguments.
134  if (ActualArgs) ActualArgs->destroy(PP);
135 }
136 
137 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
138  SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
139  unsigned MacroArgNo, Preprocessor &PP) {
140  // Is the macro argument __VA_ARGS__?
141  if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1)
142  return false;
143 
144  // In Microsoft-compatibility mode, a comma is removed in the expansion
145  // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is
146  // not supported by gcc.
147  if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)
148  return false;
149 
150  // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if
151  // __VA_ARGS__ is empty, but not in strict C99 mode where there are no
152  // named arguments, where it remains. In all other modes, including C99
153  // with GNU extensions, it is removed regardless of named arguments.
154  // Microsoft also appears to support this extension, unofficially.
155  if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode
156  && Macro->getNumParams() < 2)
157  return false;
158 
159  // Is a comma available to be removed?
160  if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
161  return false;
162 
163  // Issue an extension diagnostic for the paste operator.
164  if (HasPasteOperator)
165  PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
166 
167  // Remove the comma.
168  ResultToks.pop_back();
169 
170  if (!ResultToks.empty()) {
171  // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
172  // then removal of the comma should produce a placemarker token (in C99
173  // terms) which we model by popping off the previous ##, giving us a plain
174  // "X" when __VA_ARGS__ is empty.
175  if (ResultToks.back().is(tok::hashhash))
176  ResultToks.pop_back();
177 
178  // Remember that this comma was elided.
179  ResultToks.back().setFlag(Token::CommaAfterElided);
180  }
181 
182  // Never add a space, even if the comma, ##, or arg had a space.
183  NextTokGetsSpace = false;
184  return true;
185 }
186 
187 void TokenLexer::stringifyVAOPTContents(
188  SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,
189  const SourceLocation VAOPTClosingParenLoc) {
190  const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt();
191  const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
192  Token *const VAOPTTokens =
193  NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr;
194 
195  SmallVector<Token, 64> ConcatenatedVAOPTResultToks;
196  // FIXME: Should we keep track within VCtx that we did or didnot
197  // encounter pasting - and only then perform this loop.
198 
199  // Perform token pasting (concatenation) prior to stringization.
200  for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
201  ++CurTokenIdx) {
202  if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {
203  assert(CurTokenIdx != 0 &&
204  "Can not have __VAOPT__ contents begin with a ##");
205  Token &LHS = VAOPTTokens[CurTokenIdx - 1];
206  pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens),
207  CurTokenIdx);
208  // Replace the token prior to the first ## in this iteration.
209  ConcatenatedVAOPTResultToks.back() = LHS;
210  if (CurTokenIdx == NumVAOptTokens)
211  break;
212  }
213  ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);
214  }
215 
216  ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok());
217  // Get the SourceLocation that represents the start location within
218  // the macro definition that marks where this string is substituted
219  // into: i.e. the __VA_OPT__ and the ')' within the spelling of the
220  // macro definition, and use it to indicate that the stringified token
221  // was generated from that location.
222  const SourceLocation ExpansionLocStartWithinMacro =
223  getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc());
224  const SourceLocation ExpansionLocEndWithinMacro =
225  getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);
226 
227  Token StringifiedVAOPT = MacroArgs::StringifyArgument(
228  &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() /*Charify*/,
229  ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);
230 
232  StringifiedVAOPT.setFlag(Token::LeadingSpace);
233 
234  StringifiedVAOPT.setFlag(Token::StringifiedInMacro);
235  // Resize (shrink) the token stream to just capture this stringified token.
236  ResultToks.resize(NumToksPriorToVAOpt + 1);
237  ResultToks.back() = StringifiedVAOPT;
238 }
239 
240 /// Expand the arguments of a function-like macro so that we can quickly
241 /// return preexpanded tokens from Tokens.
242 void TokenLexer::ExpandFunctionArguments() {
243  SmallVector<Token, 128> ResultToks;
244 
245  // Loop through 'Tokens', expanding them into ResultToks. Keep
246  // track of whether we change anything. If not, no need to keep them. If so,
247  // we install the newly expanded sequence as the new 'Tokens' list.
248  bool MadeChange = false;
249 
250  Optional<bool> CalledWithVariadicArguments;
251 
252  VAOptExpansionContext VCtx(PP);
253 
254  for (unsigned I = 0, E = NumTokens; I != E; ++I) {
255  const Token &CurTok = Tokens[I];
256  // We don't want a space for the next token after a paste
257  // operator. In valid code, the token will get smooshed onto the
258  // preceding one anyway. In assembler-with-cpp mode, invalid
259  // pastes are allowed through: in this case, we do not want the
260  // extra whitespace to be added. For example, we want ". ## foo"
261  // -> ".foo" not ". foo".
262  if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace())
263  NextTokGetsSpace = true;
264 
265  if (VCtx.isVAOptToken(CurTok)) {
266  MadeChange = true;
267  assert(Tokens[I + 1].is(tok::l_paren) &&
268  "__VA_OPT__ must be followed by '('");
269 
270  ++I; // Skip the l_paren
272  ResultToks.size());
273 
274  continue;
275  }
276 
277  // We have entered into the __VA_OPT__ context, so handle tokens
278  // appropriately.
279  if (VCtx.isInVAOpt()) {
280  // If we are about to process a token that is either an argument to
281  // __VA_OPT__ or its closing rparen, then:
282  // 1) If the token is the closing rparen that exits us out of __VA_OPT__,
283  // perform any necessary stringification or placemarker processing,
284  // and/or skip to the next token.
285  // 2) else if macro was invoked without variadic arguments skip this
286  // token.
287  // 3) else (macro was invoked with variadic arguments) process the token
288  // normally.
289 
290  if (Tokens[I].is(tok::l_paren))
291  VCtx.sawOpeningParen(Tokens[I].getLocation());
292  // Continue skipping tokens within __VA_OPT__ if the macro was not
293  // called with variadic arguments, else let the rest of the loop handle
294  // this token. Note sawClosingParen() returns true only if the r_paren matches
295  // the closing r_paren of the __VA_OPT__.
296  if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) {
297  // Lazily expand __VA_ARGS__ when we see the first __VA_OPT__.
298  if (!CalledWithVariadicArguments.hasValue()) {
299  CalledWithVariadicArguments =
300  ActualArgs->invokedWithVariadicArgument(Macro, PP);
301  }
302  if (!*CalledWithVariadicArguments) {
303  // Skip this token.
304  continue;
305  }
306  // ... else the macro was called with variadic arguments, and we do not
307  // have a closing rparen - so process this token normally.
308  } else {
309  // Current token is the closing r_paren which marks the end of the
310  // __VA_OPT__ invocation, so handle any place-marker pasting (if
311  // empty) by removing hashhash either before (if exists) or after. And
312  // also stringify the entire contents if VAOPT was preceded by a hash,
313  // but do so only after any token concatenation that needs to occur
314  // within the contents of VAOPT.
315 
316  if (VCtx.hasStringifyOrCharifyBefore()) {
317  // Replace all the tokens just added from within VAOPT into a single
318  // stringified token. This requires token-pasting to eagerly occur
319  // within these tokens. If either the contents of VAOPT were empty
320  // or the macro wasn't called with any variadic arguments, the result
321  // is a token that represents an empty string.
322  stringifyVAOPTContents(ResultToks, VCtx,
323  /*ClosingParenLoc*/ Tokens[I].getLocation());
324 
325  } else if (/*No tokens within VAOPT*/
326  ResultToks.size() == VCtx.getNumberOfTokensPriorToVAOpt()) {
327  // Treat VAOPT as a placemarker token. Eat either the '##' before the
328  // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that
329  // hashhash was not a placemarker) or the '##'
330  // after VAOPT, but not both.
331 
332  if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {
333  ResultToks.pop_back();
334  } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {
335  ++I; // Skip the following hashhash.
336  }
337  } else {
338  // If there's a ## before the __VA_OPT__, we might have discovered
339  // that the __VA_OPT__ begins with a placeholder. We delay action on
340  // that to now to avoid messing up our stashed count of tokens before
341  // __VA_OPT__.
342  if (VCtx.beginsWithPlaceholder()) {
343  assert(VCtx.getNumberOfTokensPriorToVAOpt() > 0 &&
344  ResultToks.size() >= VCtx.getNumberOfTokensPriorToVAOpt() &&
345  ResultToks[VCtx.getNumberOfTokensPriorToVAOpt() - 1].is(
346  tok::hashhash) &&
347  "no token paste before __VA_OPT__");
348  ResultToks.erase(ResultToks.begin() +
349  VCtx.getNumberOfTokensPriorToVAOpt() - 1);
350  }
351  // If the expansion of __VA_OPT__ ends with a placeholder, eat any
352  // following '##' token.
353  if (VCtx.endsWithPlaceholder() && I + 1 != E &&
354  Tokens[I + 1].is(tok::hashhash)) {
355  ++I;
356  }
357  }
358  VCtx.reset();
359  // We processed __VA_OPT__'s closing paren (and the exit out of
360  // __VA_OPT__), so skip to the next token.
361  continue;
362  }
363  }
364 
365  // If we found the stringify operator, get the argument stringified. The
366  // preprocessor already verified that the following token is a macro
367  // parameter or __VA_OPT__ when the #define was lexed.
368 
369  if (CurTok.isOneOf(tok::hash, tok::hashat)) {
370  int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());
371  assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&
372  "Token following # is not an argument or __VA_OPT__!");
373 
374  if (ArgNo == -1) {
375  // Handle the __VA_OPT__ case.
376  VCtx.sawHashOrHashAtBefore(NextTokGetsSpace,
377  CurTok.is(tok::hashat));
378  continue;
379  }
380  // Else handle the simple argument case.
381  SourceLocation ExpansionLocStart =
382  getExpansionLocForMacroDefLoc(CurTok.getLocation());
383  SourceLocation ExpansionLocEnd =
384  getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());
385 
386  bool Charify = CurTok.is(tok::hashat);
387  const Token *UnexpArg = ActualArgs->getUnexpArgument(ArgNo);
389  UnexpArg, PP, Charify, ExpansionLocStart, ExpansionLocEnd);
391 
392  // The stringified/charified string leading space flag gets set to match
393  // the #/#@ operator.
394  if (NextTokGetsSpace)
396 
397  ResultToks.push_back(Res);
398  MadeChange = true;
399  ++I; // Skip arg name.
400  NextTokGetsSpace = false;
401  continue;
402  }
403 
404  // Find out if there is a paste (##) operator before or after the token.
405  bool NonEmptyPasteBefore =
406  !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
407  bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash);
408  bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash);
409  bool RParenAfter = I+1 != E && Tokens[I+1].is(tok::r_paren);
410 
411  assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&
412  "unexpected ## in ResultToks");
413 
414  // Otherwise, if this is not an argument token, just add the token to the
415  // output buffer.
416  IdentifierInfo *II = CurTok.getIdentifierInfo();
417  int ArgNo = II ? Macro->getParameterNum(II) : -1;
418  if (ArgNo == -1) {
419  // This isn't an argument, just add it.
420  ResultToks.push_back(CurTok);
421 
422  if (NextTokGetsSpace) {
423  ResultToks.back().setFlag(Token::LeadingSpace);
424  NextTokGetsSpace = false;
425  } else if (PasteBefore && !NonEmptyPasteBefore)
426  ResultToks.back().clearFlag(Token::LeadingSpace);
427 
428  continue;
429  }
430 
431  // An argument is expanded somehow, the result is different than the
432  // input.
433  MadeChange = true;
434 
435  // Otherwise, this is a use of the argument.
436 
437  // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
438  // are no trailing commas if __VA_ARGS__ is empty.
439  if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
440  MaybeRemoveCommaBeforeVaArgs(ResultToks,
441  /*HasPasteOperator=*/false,
442  Macro, ArgNo, PP))
443  continue;
444 
445  // If it is not the LHS/RHS of a ## operator, we must pre-expand the
446  // argument and substitute the expanded tokens into the result. This is
447  // C99 6.10.3.1p1.
448  if (!PasteBefore && !PasteAfter) {
449  const Token *ResultArgToks;
450 
451  // Only preexpand the argument if it could possibly need it. This
452  // avoids some work in common cases.
453  const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
454  if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
455  ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];
456  else
457  ResultArgToks = ArgTok; // Use non-preexpanded tokens.
458 
459  // If the arg token expanded into anything, append it.
460  if (ResultArgToks->isNot(tok::eof)) {
461  size_t FirstResult = ResultToks.size();
462  unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
463  ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
464 
465  // In Microsoft-compatibility mode, we follow MSVC's preprocessing
466  // behavior by not considering single commas from nested macro
467  // expansions as argument separators. Set a flag on the token so we can
468  // test for this later when the macro expansion is processed.
469  if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&
470  ResultToks.back().is(tok::comma))
471  ResultToks.back().setFlag(Token::IgnoredComma);
472 
473  // If the '##' came from expanding an argument, turn it into 'unknown'
474  // to avoid pasting.
475  for (Token &Tok : llvm::make_range(ResultToks.begin() + FirstResult,
476  ResultToks.end())) {
477  if (Tok.is(tok::hashhash))
478  Tok.setKind(tok::unknown);
479  }
480 
481  if(ExpandLocStart.isValid()) {
482  updateLocForMacroArgTokens(CurTok.getLocation(),
483  ResultToks.begin()+FirstResult,
484  ResultToks.end());
485  }
486 
487  // If any tokens were substituted from the argument, the whitespace
488  // before the first token should match the whitespace of the arg
489  // identifier.
490  ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
491  NextTokGetsSpace);
492  ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
493  NextTokGetsSpace = false;
494  } else {
495  // We're creating a placeholder token. Usually this doesn't matter,
496  // but it can affect paste behavior when at the start or end of a
497  // __VA_OPT__.
498  if (NonEmptyPasteBefore) {
499  // We're imagining a placeholder token is inserted here. If this is
500  // the first token in a __VA_OPT__ after a ##, delete the ##.
501  assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__");
503  }
504  if (RParenAfter)
506  }
507  continue;
508  }
509 
510  // Okay, we have a token that is either the LHS or RHS of a paste (##)
511  // argument. It gets substituted as its non-pre-expanded tokens.
512  const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
513  unsigned NumToks = MacroArgs::getArgLength(ArgToks);
514  if (NumToks) { // Not an empty argument?
515  bool VaArgsPseudoPaste = false;
516  // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned
517  // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when
518  // the expander tries to paste ',' with the first token of the __VA_ARGS__
519  // expansion.
520  if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
521  ResultToks[ResultToks.size()-2].is(tok::comma) &&
522  (unsigned)ArgNo == Macro->getNumParams()-1 &&
523  Macro->isVariadic()) {
524  VaArgsPseudoPaste = true;
525  // Remove the paste operator, report use of the extension.
526  PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
527  }
528 
529  ResultToks.append(ArgToks, ArgToks+NumToks);
530 
531  // If the '##' came from expanding an argument, turn it into 'unknown'
532  // to avoid pasting.
533  for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
534  ResultToks.end())) {
535  if (Tok.is(tok::hashhash))
536  Tok.setKind(tok::unknown);
537  }
538 
539  if (ExpandLocStart.isValid()) {
540  updateLocForMacroArgTokens(CurTok.getLocation(),
541  ResultToks.end()-NumToks, ResultToks.end());
542  }
543 
544  // Transfer the leading whitespace information from the token
545  // (the macro argument) onto the first token of the
546  // expansion. Note that we don't do this for the GNU
547  // pseudo-paste extension ", ## __VA_ARGS__".
548  if (!VaArgsPseudoPaste) {
549  ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,
550  false);
551  ResultToks[ResultToks.size() - NumToks].setFlagValue(
552  Token::LeadingSpace, NextTokGetsSpace);
553  }
554 
555  NextTokGetsSpace = false;
556  continue;
557  }
558 
559  // If an empty argument is on the LHS or RHS of a paste, the standard (C99
560  // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We
561  // implement this by eating ## operators when a LHS or RHS expands to
562  // empty.
563  if (PasteAfter) {
564  // Discard the argument token and skip (don't copy to the expansion
565  // buffer) the paste operator after it.
566  ++I;
567  continue;
568  }
569 
570  if (RParenAfter)
572 
573  // If this is on the RHS of a paste operator, we've already copied the
574  // paste operator to the ResultToks list, unless the LHS was empty too.
575  // Remove it.
576  assert(PasteBefore);
577  if (NonEmptyPasteBefore) {
578  assert(ResultToks.back().is(tok::hashhash));
579  // Do not remove the paste operator if it is the one before __VA_OPT__
580  // (and we are still processing tokens within VA_OPT). We handle the case
581  // of removing the paste operator if __VA_OPT__ reduces to the notional
582  // placemarker above when we encounter the closing paren of VA_OPT.
583  if (!VCtx.isInVAOpt() ||
584  ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt())
585  ResultToks.pop_back();
586  else
588  }
589 
590  // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
591  // and if the macro had at least one real argument, and if the token before
592  // the ## was a comma, remove the comma. This is a GCC extension which is
593  // disabled when using -std=c99.
594  if (ActualArgs->isVarargsElidedUse())
595  MaybeRemoveCommaBeforeVaArgs(ResultToks,
596  /*HasPasteOperator=*/true,
597  Macro, ArgNo, PP);
598  }
599 
600  // If anything changed, install this as the new Tokens list.
601  if (MadeChange) {
602  assert(!OwnsTokens && "This would leak if we already own the token list");
603  // This is deleted in the dtor.
604  NumTokens = ResultToks.size();
605  // The tokens will be added to Preprocessor's cache and will be removed
606  // when this TokenLexer finishes lexing them.
607  Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
608 
609  // The preprocessor cache of macro expanded tokens owns these tokens,not us.
610  OwnsTokens = false;
611  }
612 }
613 
614 /// Checks if two tokens form wide string literal.
615 static bool isWideStringLiteralFromMacro(const Token &FirstTok,
616  const Token &SecondTok) {
617  return FirstTok.is(tok::identifier) &&
618  FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral() &&
619  SecondTok.stringifiedInMacro();
620 }
621 
622 /// Lex - Lex and return a token from this macro stream.
623 bool TokenLexer::Lex(Token &Tok) {
624  // Lexing off the end of the macro, pop this macro off the expansion stack.
625  if (isAtEnd()) {
626  // If this is a macro (not a token stream), mark the macro enabled now
627  // that it is no longer being expanded.
628  if (Macro) Macro->EnableMacro();
629 
630  Tok.startToken();
631  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
632  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);
633  if (CurTokenIdx == 0)
635  return PP.HandleEndOfTokenLexer(Tok);
636  }
637 
639 
640  // If this is the first token of the expanded result, we inherit spacing
641  // properties later.
642  bool isFirstToken = CurTokenIdx == 0;
643 
644  // Get the next token to return.
645  Tok = Tokens[CurTokenIdx++];
646  if (IsReinject)
648 
649  bool TokenIsFromPaste = false;
650 
651  // If this token is followed by a token paste (##) operator, paste the tokens!
652  // Note that ## is a normal token when not expanding a macro.
653  if (!isAtEnd() && Macro &&
654  (Tokens[CurTokenIdx].is(tok::hashhash) ||
655  // Special processing of L#x macros in -fms-compatibility mode.
656  // Microsoft compiler is able to form a wide string literal from
657  // 'L#macro_arg' construct in a function-like macro.
658  (PP.getLangOpts().MSVCCompat &&
659  isWideStringLiteralFromMacro(Tok, Tokens[CurTokenIdx])))) {
660  // When handling the microsoft /##/ extension, the final token is
661  // returned by pasteTokens, not the pasted token.
662  if (pasteTokens(Tok))
663  return true;
664 
665  TokenIsFromPaste = true;
666  }
667 
668  // The token's current location indicate where the token was lexed from. We
669  // need this information to compute the spelling of the token, but any
670  // diagnostics for the expanded token should appear as if they came from
671  // ExpansionLoc. Pull this information together into a new SourceLocation
672  // that captures all of this.
673  if (ExpandLocStart.isValid() && // Don't do this for token streams.
674  // Check that the token's location was not already set properly.
675  SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {
676  SourceLocation instLoc;
677  if (Tok.is(tok::comment)) {
678  instLoc = SM.createExpansionLoc(Tok.getLocation(),
679  ExpandLocStart,
680  ExpandLocEnd,
681  Tok.getLength());
682  } else {
683  instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());
684  }
685 
686  Tok.setLocation(instLoc);
687  }
688 
689  // If this is the first token, set the lexical properties of the token to
690  // match the lexical properties of the macro identifier.
691  if (isFirstToken) {
692  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
693  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
694  } else {
695  // If this is not the first token, we may still need to pass through
696  // leading whitespace if we've expanded a macro.
697  if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);
698  if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);
699  }
700  AtStartOfLine = false;
701  HasLeadingSpace = false;
702 
703  // Handle recursive expansion!
704  if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) {
705  // Change the kind of this identifier to the appropriate token kind, e.g.
706  // turning "for" into a keyword.
707  IdentifierInfo *II = Tok.getIdentifierInfo();
708  Tok.setKind(II->getTokenID());
709 
710  // If this identifier was poisoned and from a paste, emit an error. This
711  // won't be handled by Preprocessor::HandleIdentifier because this is coming
712  // from a macro expansion.
713  if (II->isPoisoned() && TokenIsFromPaste) {
714  PP.HandlePoisonedIdentifier(Tok);
715  }
716 
717  if (!DisableMacroExpansion && II->isHandleIdentifierCase())
718  return PP.HandleIdentifier(Tok);
719  }
720 
721  // Otherwise, return a normal token.
722  return true;
723 }
724 
725 bool TokenLexer::pasteTokens(Token &Tok) {
726  return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx);
727 }
728 
729 /// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ##
730 /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there
731 /// are more ## after it, chomp them iteratively. Return the result as LHSTok.
732 /// If this returns true, the caller should immediately return the token.
733 bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
734  unsigned int &CurIdx) {
735  assert(CurIdx > 0 && "## can not be the first token within tokens");
736  assert((TokenStream[CurIdx].is(tok::hashhash) ||
737  (PP.getLangOpts().MSVCCompat &&
738  isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) &&
739  "Token at this Index must be ## or part of the MSVC 'L "
740  "#macro-arg' pasting pair");
741 
742  // MSVC: If previous token was pasted, this must be a recovery from an invalid
743  // paste operation. Ignore spaces before this token to mimic MSVC output.
744  // Required for generating valid UUID strings in some MS headers.
745  if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) &&
746  TokenStream[CurIdx - 2].is(tok::hashhash))
748 
749  SmallString<128> Buffer;
750  const char *ResultTokStrPtr = nullptr;
751  SourceLocation StartLoc = LHSTok.getLocation();
752  SourceLocation PasteOpLoc;
753 
754  auto IsAtEnd = [&TokenStream, &CurIdx] {
755  return TokenStream.size() == CurIdx;
756  };
757 
758  do {
759  // Consume the ## operator if any.
760  PasteOpLoc = TokenStream[CurIdx].getLocation();
761  if (TokenStream[CurIdx].is(tok::hashhash))
762  ++CurIdx;
763  assert(!IsAtEnd() && "No token on the RHS of a paste operator!");
764 
765  // Get the RHS token.
766  const Token &RHS = TokenStream[CurIdx];
767 
768  // Allocate space for the result token. This is guaranteed to be enough for
769  // the two tokens.
770  Buffer.resize(LHSTok.getLength() + RHS.getLength());
771 
772  // Get the spelling of the LHS token in Buffer.
773  const char *BufPtr = &Buffer[0];
774  bool Invalid = false;
775  unsigned LHSLen = PP.getSpelling(LHSTok, BufPtr, &Invalid);
776  if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer!
777  memcpy(&Buffer[0], BufPtr, LHSLen);
778  if (Invalid)
779  return true;
780 
781  BufPtr = Buffer.data() + LHSLen;
782  unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid);
783  if (Invalid)
784  return true;
785  if (RHSLen && BufPtr != &Buffer[LHSLen])
786  // Really, we want the chars in Buffer!
787  memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
788 
789  // Trim excess space.
790  Buffer.resize(LHSLen+RHSLen);
791 
792  // Plop the pasted result (including the trailing newline and null) into a
793  // scratch buffer where we can lex it.
794  Token ResultTokTmp;
795  ResultTokTmp.startToken();
796 
797  // Claim that the tmp token is a string_literal so that we can get the
798  // character pointer back from CreateString in getLiteralData().
799  ResultTokTmp.setKind(tok::string_literal);
800  PP.CreateString(Buffer, ResultTokTmp);
801  SourceLocation ResultTokLoc = ResultTokTmp.getLocation();
802  ResultTokStrPtr = ResultTokTmp.getLiteralData();
803 
804  // Lex the resultant pasted token into Result.
805  Token Result;
806 
807  if (LHSTok.isAnyIdentifier() && RHS.isAnyIdentifier()) {
808  // Common paste case: identifier+identifier = identifier. Avoid creating
809  // a lexer and other overhead.
810  PP.IncrementPasteCounter(true);
811  Result.startToken();
812  Result.setKind(tok::raw_identifier);
813  Result.setRawIdentifierData(ResultTokStrPtr);
814  Result.setLocation(ResultTokLoc);
815  Result.setLength(LHSLen+RHSLen);
816  } else {
817  PP.IncrementPasteCounter(false);
818 
819  assert(ResultTokLoc.isFileID() &&
820  "Should be a raw location into scratch buffer");
821  SourceManager &SourceMgr = PP.getSourceManager();
822  FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
823 
824  bool Invalid = false;
825  const char *ScratchBufStart
826  = SourceMgr.getBufferData(LocFileID, &Invalid).data();
827  if (Invalid)
828  return false;
829 
830  // Make a lexer to lex this string from. Lex just this one token.
831  // Make a lexer object so that we lex and expand the paste result.
832  Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
833  PP.getLangOpts(), ScratchBufStart,
834  ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
835 
836  // Lex a token in raw mode. This way it won't look up identifiers
837  // automatically, lexing off the end will return an eof token, and
838  // warnings are disabled. This returns true if the result token is the
839  // entire buffer.
840  bool isInvalid = !TL.LexFromRawLexer(Result);
841 
842  // If we got an EOF token, we didn't form even ONE token. For example, we
843  // did "/ ## /" to get "//".
844  isInvalid |= Result.is(tok::eof);
845 
846  // If pasting the two tokens didn't form a full new token, this is an
847  // error. This occurs with "x ## +" and other stuff. Return with LHSTok
848  // unmodified and with RHS as the next token to lex.
849  if (isInvalid) {
850  // Explicitly convert the token location to have proper expansion
851  // information so that the user knows where it came from.
853  SourceLocation Loc =
854  SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);
855 
856  // Test for the Microsoft extension of /##/ turning into // here on the
857  // error path.
858  if (PP.getLangOpts().MicrosoftExt && LHSTok.is(tok::slash) &&
859  RHS.is(tok::slash)) {
860  HandleMicrosoftCommentPaste(LHSTok, Loc);
861  return true;
862  }
863 
864  // Do not emit the error when preprocessing assembler code.
865  if (!PP.getLangOpts().AsmPreprocessor) {
866  // If we're in microsoft extensions mode, downgrade this from a hard
867  // error to an extension that defaults to an error. This allows
868  // disabling it.
869  PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms
870  : diag::err_pp_bad_paste)
871  << Buffer;
872  }
873 
874  // An error has occurred so exit loop.
875  break;
876  }
877 
878  // Turn ## into 'unknown' to avoid # ## # from looking like a paste
879  // operator.
880  if (Result.is(tok::hashhash))
881  Result.setKind(tok::unknown);
882  }
883 
884  // Transfer properties of the LHS over the Result.
885  Result.setFlagValue(Token::StartOfLine , LHSTok.isAtStartOfLine());
886  Result.setFlagValue(Token::LeadingSpace, LHSTok.hasLeadingSpace());
887 
888  // Finally, replace LHS with the result, consume the RHS, and iterate.
889  ++CurIdx;
890  LHSTok = Result;
891  } while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));
892 
893  SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation();
894 
895  // The token's current location indicate where the token was lexed from. We
896  // need this information to compute the spelling of the token, but any
897  // diagnostics for the expanded token should appear as if the token was
898  // expanded from the full ## expression. Pull this information together into
899  // a new SourceLocation that captures all of this.
901  if (StartLoc.isFileID())
902  StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
903  if (EndLoc.isFileID())
904  EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
905  FileID MacroFID = SM.getFileID(MacroExpansionStart);
906  while (SM.getFileID(StartLoc) != MacroFID)
907  StartLoc = SM.getImmediateExpansionRange(StartLoc).getBegin();
908  while (SM.getFileID(EndLoc) != MacroFID)
909  EndLoc = SM.getImmediateExpansionRange(EndLoc).getEnd();
910 
911  LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc,
912  LHSTok.getLength()));
913 
914  // Now that we got the result token, it will be subject to expansion. Since
915  // token pasting re-lexes the result token in raw mode, identifier information
916  // isn't looked up. As such, if the result is an identifier, look up id info.
917  if (LHSTok.is(tok::raw_identifier)) {
918  // Look up the identifier info for the token. We disabled identifier lookup
919  // by saying we're skipping contents, so we need to do this manually.
920  PP.LookUpIdentifierInfo(LHSTok);
921  }
922  return false;
923 }
924 
925 /// isNextTokenLParen - If the next token lexed will pop this macro off the
926 /// expansion stack, return 2. If the next unexpanded token is a '(', return
927 /// 1, otherwise return 0.
929  // Out of tokens?
930  if (isAtEnd())
931  return 2;
932  return Tokens[CurTokenIdx].is(tok::l_paren);
933 }
934 
935 /// isParsingPreprocessorDirective - Return true if we are in the middle of a
936 /// preprocessor directive.
938  return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd();
939 }
940 
941 /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
942 /// together to form a comment that comments out everything in the current
943 /// macro, other active macros, and anything left on the current physical
944 /// source line of the expanded buffer. Handle this by returning the
945 /// first token on the next line.
946 void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {
947  PP.Diag(OpLoc, diag::ext_comment_paste_microsoft);
948 
949  // We 'comment out' the rest of this macro by just ignoring the rest of the
950  // tokens that have not been lexed yet, if any.
951 
952  // Since this must be a macro, mark the macro enabled now that it is no longer
953  // being expanded.
954  assert(Macro && "Token streams can't paste comments");
955  Macro->EnableMacro();
956 
958 }
959 
960 /// If \arg loc is a file ID and points inside the current macro
961 /// definition, returns the appropriate source location pointing at the
962 /// macro expansion source location entry, otherwise it returns an invalid
963 /// SourceLocation.
965 TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {
966  assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&
967  "Not appropriate for token streams");
968  assert(loc.isValid() && loc.isFileID());
969 
971  assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
972  "Expected loc to come from the macro definition");
973 
974  SourceLocation::UIntTy relativeOffset = 0;
975  SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);
976  return MacroExpansionStart.getLocWithOffset(relativeOffset);
977 }
978 
979 /// Finds the tokens that are consecutive (from the same FileID)
980 /// creates a single SLocEntry, and assigns SourceLocations to each token that
981 /// point to that SLocEntry. e.g for
982 /// assert(foo == bar);
983 /// There will be a single SLocEntry for the "foo == bar" chunk and locations
984 /// for the 'foo', '==', 'bar' tokens will point inside that chunk.
985 ///
986 /// \arg begin_tokens will be updated to a position past all the found
987 /// consecutive tokens.
989  SourceLocation InstLoc,
990  Token *&begin_tokens,
991  Token * end_tokens) {
992  assert(begin_tokens < end_tokens);
993 
994  SourceLocation FirstLoc = begin_tokens->getLocation();
995  SourceLocation CurLoc = FirstLoc;
996 
997  // Compare the source location offset of tokens and group together tokens that
998  // are close, even if their locations point to different FileIDs. e.g.
999  //
1000  // |bar | foo | cake | (3 tokens from 3 consecutive FileIDs)
1001  // ^ ^
1002  // |bar foo cake| (one SLocEntry chunk for all tokens)
1003  //
1004  // we can perform this "merge" since the token's spelling location depends
1005  // on the relative offset.
1006 
1007  Token *NextTok = begin_tokens + 1;
1008  for (; NextTok < end_tokens; ++NextTok) {
1009  SourceLocation NextLoc = NextTok->getLocation();
1010  if (CurLoc.isFileID() != NextLoc.isFileID())
1011  break; // Token from different kind of FileID.
1012 
1013  SourceLocation::IntTy RelOffs;
1014  if (!SM.isInSameSLocAddrSpace(CurLoc, NextLoc, &RelOffs))
1015  break; // Token from different local/loaded location.
1016  // Check that token is not before the previous token or more than 50
1017  // "characters" away.
1018  if (RelOffs < 0 || RelOffs > 50)
1019  break;
1020 
1021  if (CurLoc.isMacroID() && !SM.isWrittenInSameFile(CurLoc, NextLoc))
1022  break; // Token from a different macro.
1023 
1024  CurLoc = NextLoc;
1025  }
1026 
1027  // For the consecutive tokens, find the length of the SLocEntry to contain
1028  // all of them.
1029  Token &LastConsecutiveTok = *(NextTok-1);
1030  SourceLocation::IntTy LastRelOffs = 0;
1031  SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(),
1032  &LastRelOffs);
1033  SourceLocation::UIntTy FullLength =
1034  LastRelOffs + LastConsecutiveTok.getLength();
1035 
1036  // Create a macro expansion SLocEntry that will "contain" all of the tokens.
1037  SourceLocation Expansion =
1038  SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength);
1039 
1040  // Change the location of the tokens from the spelling location to the new
1041  // expanded location.
1042  for (; begin_tokens < NextTok; ++begin_tokens) {
1043  Token &Tok = *begin_tokens;
1044  SourceLocation::IntTy RelOffs = 0;
1045  SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs);
1046  Tok.setLocation(Expansion.getLocWithOffset(RelOffs));
1047  }
1048 }
1049 
1050 /// Creates SLocEntries and updates the locations of macro argument
1051 /// tokens to their new expanded locations.
1052 ///
1053 /// \param ArgIdSpellLoc the location of the macro argument id inside the macro
1054 /// definition.
1055 void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
1056  Token *begin_tokens,
1057  Token *end_tokens) {
1059 
1060  SourceLocation InstLoc =
1061  getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
1062 
1063  while (begin_tokens < end_tokens) {
1064  // If there's only one token just create a SLocEntry for it.
1065  if (end_tokens - begin_tokens == 1) {
1066  Token &Tok = *begin_tokens;
1067  Tok.setLocation(SM.createMacroArgExpansionLoc(Tok.getLocation(),
1068  InstLoc,
1069  Tok.getLength()));
1070  return;
1071  }
1072 
1073  updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens);
1074  }
1075 }
1076 
1077 void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
1078  AtStartOfLine = Result.isAtStartOfLine();
1079  HasLeadingSpace = Result.hasLeadingSpace();
1080 }
clang::Token::startToken
void startToken()
Reset all flags to cleared.
Definition: Token.h:171
clang::Preprocessor::CreateString
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
Definition: Preprocessor.cpp:481
clang::Token::IgnoredComma
@ IgnoredComma
Definition: Token.h:82
clang::Token::IsReinjected
@ IsReinjected
Definition: Token.h:87
clang::VAOptExpansionContext::hasPlaceholderBeforeRParen
void hasPlaceholderBeforeRParen()
Definition: VariadicMacroSupport.h:180
clang::Token::hasLeadingSpace
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:272
clang::VAOptExpansionContext::sawOpeningParen
void sawOpeningParen(SourceLocation LParenLoc)
Call this function each time an lparen is seen.
Definition: VariadicMacroSupport.h:107
clang::VAOptExpansionContext::sawHashOrHashAtBefore
void sawHashOrHashAtBefore(const bool HasLeadingSpace, const bool IsHashAt)
Definition: VariadicMacroSupport.h:171
clang::Preprocessor::HandleEndOfTokenLexer
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
Definition: PPLexerChange.cpp:589
clang::VAOptExpansionContext::getNumberOfTokensPriorToVAOpt
unsigned int getNumberOfTokensPriorToVAOpt() const
Definition: VariadicMacroSupport.h:206
clang::Preprocessor::HandleIdentifier
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
Definition: Preprocessor.cpp:796
clang::Token::getLiteralData
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:217
llvm::SmallVector
Definition: LLVM.h:38
clang::SourceLocation
Encodes a location in the source.
Definition: SourceLocation.h:88
clang::SourceLocation::getLocWithOffset
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
Definition: SourceLocation.h:136
clang::Token::getIdentifierInfo
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:179
clang::MacroArgs
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
clang::VAOptExpansionContext::beginsWithPlaceholder
bool beginsWithPlaceholder() const
Definition: VariadicMacroSupport.h:186
isWideStringLiteralFromMacro
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
Definition: TokenLexer.cpp:615
clang::VAOptExpansionContext
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a macro during...
Definition: VariadicMacroSupport.h:118
memcpy
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
Definition: __clang_cuda_device_functions.h:1549
clang::TokenLexer::Lex
bool Lex(Token &Tok)
Lex and return a token from this macro stream.
Definition: TokenLexer.cpp:623
clang::Token::isAtStartOfLine
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:268
llvm::Optional< bool >
clang::MacroInfo
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
SourceManager.h
clang::Preprocessor::HandleMicrosoftCommentPaste
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
Definition: PPLexerChange.cpp:627
clang::index::SymbolKind::Macro
@ Macro
clang::Preprocessor::IncrementPasteCounter
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Definition: Preprocessor.h:1910
clang::IdentifierInfo::getTokenID
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
Definition: IdentifierTable.h:261
clang::Token
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
TokenLexer.h
clang::SourceManager
This class handles loading and caching of source files into memory.
Definition: SourceManager.h:626
clang::Preprocessor::getLangOpts
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:960
Preprocessor.h
clang::Lexer
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.
Definition: Lexer.h:76
clang::driver::tools::arm::ReadTPMode::Invalid
@ Invalid
clang::Token::StringifiedInMacro
@ StringifiedInMacro
Definition: Token.h:83
clang::SourceLocation::IntTy
int32_t IntTy
Definition: SourceLocation.h:96
TokenKinds.h
clang::MacroInfo::tokens_begin
tokens_iterator tokens_begin() const
Definition: MacroInfo.h:242
clang::IdentifierInfo::isStr
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
Definition: IdentifierTable.h:176
clang::MacroArgs::getPreExpArgument
const std::vector< Token > & getPreExpArgument(unsigned Arg, Preprocessor &PP)
getPreExpArgument - Return the pre-expanded form of the specified argument.
Definition: MacroArgs.cpp:161
clang::MacroInfo::DisableMacro
void DisableMacro()
Definition: MacroInfo.h:265
clang::MacroInfo::tokens_end
tokens_iterator tokens_end() const
Definition: MacroInfo.h:243
clang::VAOptExpansionContext::hasCharifyBefore
bool hasCharifyBefore() const
Definition: VariadicMacroSupport.h:197
clang::VAOptExpansionContext::getLeadingSpaceForStringifiedToken
bool getLeadingSpaceForStringifiedToken() const
Definition: VariadicMacroSupport.h:212
clang::VAOptExpansionContext::isVAOptToken
bool isVAOptToken(const Token &T) const
Definition: VariadicMacroSupport.h:77
clang::Token::setFlagValue
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Definition: Token.h:259
LangOptions.h
Diagnostic.h
LexDiagnostic.h
clang::Preprocessor::Diag
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
Definition: Preprocessor.h:1744
clang::Token::is
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:97
clang::MacroArgs::isVarargsElidedUse
bool isVarargsElidedUse() const
isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...
Definition: MacroArgs.h:102
IdentifierTable.h
clang::SourceManager::getFileID
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
Definition: SourceManager.h:1103
llvm::SmallString< 128 >
clang::Token::LeadingEmptyMacro
@ LeadingEmptyMacro
Definition: Token.h:79
clang::VAOptExpansionContext::reset
void reset()
Definition: VariadicMacroSupport.h:159
clang::VAOptExpansionContext::isInVAOpt
bool isInVAOpt() const
Returns true if we have seen the VA_OPT and '(' but before having seen the matching ')'.
Definition: VariadicMacroSupport.h:83
isInvalid
static bool isInvalid(LocType Loc, bool *Invalid)
Definition: SourceManager.cpp:1230
clang::SourceLocation::isFileID
bool isFileID() const
Definition: SourceLocation.h:104
clang::Token::isAnyIdentifier
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
Definition: Token.h:109
clang::TokenLexer::isNextTokenLParen
unsigned isNextTokenLParen() const
If the next token lexed will pop this macro off the expansion stack, return 2.
Definition: TokenLexer.cpp:928
clang::Token::isNot
bool isNot(tok::TokenKind K) const
Definition: Token.h:98
clang::MacroArgs::getUnexpArgument
const Token * getUnexpArgument(unsigned Arg) const
getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...
Definition: MacroArgs.cpp:118
clang::MacroArgs::getArgLength
static unsigned getArgLength(const Token *ArgPtr)
getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens,...
Definition: MacroArgs.cpp:108
clang::VAOptExpansionContext::hasPlaceholderAfterHashhashAtStart
void hasPlaceholderAfterHashhashAtStart()
Definition: VariadicMacroSupport.h:179
clang::Token::isLiteral
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:115
clang::Token::getLength
unsigned getLength() const
Definition: Token.h:129
clang::Token::LeadingSpace
@ LeadingSpace
Definition: Token.h:75
clang::SourceManager::getLocForStartOfFile
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file.
Definition: SourceManager.h:1118
clang::Preprocessor::getSourceManager
SourceManager & getSourceManager() const
Definition: Preprocessor.h:964
clang::MacroArgs::destroy
void destroy(Preprocessor &PP)
destroy - Destroy and deallocate the memory for this object.
Definition: MacroArgs.cpp:78
clang::Preprocessor::getSpelling
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
Definition: Preprocessor.h:1759
MacroArgs.h
clang::Decl::setLocation
void setLocation(SourceLocation L)
Definition: DeclBase.h:431
SourceLocation.h
clang::VAOptExpansionContext::endsWithPlaceholder
bool endsWithPlaceholder() const
Definition: VariadicMacroSupport.h:191
clang::MacroArgs::invokedWithVariadicArgument
bool invokedWithVariadicArgument(const MacroInfo *const MI, Preprocessor &PP)
Returns true if the macro was defined with a variadic (ellipsis) parameter AND was invoked with at le...
Definition: MacroArgs.cpp:136
clang::IdentifierInfo::isPoisoned
bool isPoisoned() const
Return true if this token has been poisoned.
Definition: IdentifierTable.h:354
clang::TokenLexer::isParsingPreprocessorDirective
bool isParsingPreprocessorDirective() const
isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive.
Definition: TokenLexer.cpp:937
llvm::ArrayRef
Definition: LLVM.h:34
Lexer.h
clang::TokenLexer::Init
void Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, MacroArgs *Actuals)
Initialize this TokenLexer to expand from the specified macro with the specified argument information...
Definition: TokenLexer.cpp:38
clang::Token::StartOfLine
@ StartOfLine
Definition: Token.h:73
clang::SourceManager::getBufferData
StringRef getBufferData(FileID FID, bool *Invalid=nullptr) const
Return a StringRef to the source buffer data for the specified FileID.
Definition: SourceManager.cpp:735
clang::IdentifierInfo
One of these records is kept for each identifier that is lexed.
Definition: IdentifierTable.h:84
clang::Token::clearFlag
void clearFlag(TokenFlags Flag)
Unset the specified flag.
Definition: Token.h:246
clang::Token::getLocation
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:126
clang::SourceLocation::isMacroID
bool isMacroID() const
Definition: SourceLocation.h:105
clang::IdentifierInfo::isHandleIdentifierCase
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier.
Definition: IdentifierTable.h:380
clang::MacroInfo::getDefinitionLength
unsigned getDefinitionLength(const SourceManager &SM) const
Get length in characters of the macro definition.
Definition: MacroInfo.h:132
clang::Token::isAnnotation
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:120
clang::VAOptExpansionContext::sawClosingParen
bool sawClosingParen()
Call this function each time an rparen is seen.
Definition: VariadicMacroSupport.h:100
clang::Token::setLocation
void setLocation(SourceLocation L)
Definition: Token.h:134
clang::MacroInfo::getNumParams
unsigned getNumParams() const
Definition: MacroInfo.h:182
clang
Definition: CalledOnceCheck.h:17
clang::Preprocessor::LookUpIdentifierInfo
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Definition: Preprocessor.cpp:696
MacroInfo.h
clang::VAOptExpansionContext::getEOFTok
const Token & getEOFTok() const
Definition: VariadicMacroSupport.h:169
clang::Token::CommaAfterElided
@ CommaAfterElided
Definition: Token.h:85
clang::VAOptExpansionContext::hasStringifyOrCharifyBefore
bool hasStringifyOrCharifyBefore() const
Definition: VariadicMacroSupport.h:202
clang::Token::stringifiedInMacro
bool stringifiedInMacro() const
Returns true if this token is formed by macro by stringizing or charizing operator.
Definition: Token.h:300
clang::FileID
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Definition: SourceLocation.h:40
clang::SourceLocation::isValid
bool isValid() const
Return true if this is a valid SourceLocation object.
Definition: SourceLocation.h:112
updateConsecutiveMacroArgTokens
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation InstLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry,...
Definition: TokenLexer.cpp:988
VariadicMacroSupport.h
clang::Token::setKind
void setKind(tok::TokenKind K)
Definition: Token.h:93
clang::Token::isOneOf
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:99
clang::VAOptExpansionContext::getVAOptLoc
SourceLocation getVAOptLoc() const
Definition: VariadicMacroSupport.h:229
clang::Token::setFlag
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:236
clang::MacroArgs::ArgNeedsPreexpansion
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
ArgNeedsPreexpansion - If we can prove that the argument won't be affected by pre-expansion,...
Definition: MacroArgs.cpp:146
clang::comments::tok::eof
@ eof
Definition: CommentLexer.h:33
llvm::SmallVectorImpl
Definition: LLVM.h:39
clang::MacroInfo::isFunctionLike
bool isFunctionLike() const
Definition: MacroInfo.h:199
clang::Preprocessor
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:130
SM
#define SM(sm)
Definition: Cuda.cpp:78
clang::Preprocessor::HandlePoisonedIdentifier
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
Definition: Preprocessor.cpp:749
clang::MacroArgs::StringifyArgument
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...
Definition: MacroArgs.cpp:208
Token.h
clang::VAOptExpansionContext::sawVAOptFollowedByOpeningParens
void sawVAOptFollowedByOpeningParens(const SourceLocation VAOptLoc, const unsigned int NumPriorTokens)
Definition: VariadicMacroSupport.h:218
clang::SourceLocation::UIntTy
uint32_t UIntTy
Definition: SourceLocation.h:95