clang  8.0.0svn
TokenLexer.cpp
Go to the documentation of this file.
1 //===- TokenLexer.cpp - Lex from a token stream ---------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the TokenLexer interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "clang/Lex/TokenLexer.h"
15 #include "clang/Basic/Diagnostic.h"
20 #include "clang/Basic/TokenKinds.h"
22 #include "clang/Lex/Lexer.h"
23 #include "clang/Lex/MacroArgs.h"
24 #include "clang/Lex/MacroInfo.h"
25 #include "clang/Lex/Preprocessor.h"
26 #include "clang/Lex/Token.h"
28 #include "llvm/ADT/ArrayRef.h"
29 #include "llvm/ADT/SmallString.h"
30 #include "llvm/ADT/SmallVector.h"
31 #include "llvm/ADT/iterator_range.h"
32 #include <cassert>
33 #include <cstring>
34 
35 using namespace clang;
36 
37 /// Create a TokenLexer for the specified macro with the specified actual
38 /// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
40  MacroArgs *Actuals) {
41  // If the client is reusing a TokenLexer, make sure to free any memory
42  // associated with it.
43  destroy();
44 
45  Macro = MI;
46  ActualArgs = Actuals;
47  CurTokenIdx = 0;
48 
49  ExpandLocStart = Tok.getLocation();
50  ExpandLocEnd = ELEnd;
51  AtStartOfLine = Tok.isAtStartOfLine();
52  HasLeadingSpace = Tok.hasLeadingSpace();
53  NextTokGetsSpace = false;
54  Tokens = &*Macro->tokens_begin();
55  OwnsTokens = false;
56  DisableMacroExpansion = false;
57  NumTokens = Macro->tokens_end()-Macro->tokens_begin();
58  MacroExpansionStart = SourceLocation();
59 
61  MacroStartSLocOffset = SM.getNextLocalOffset();
62 
63  if (NumTokens > 0) {
64  assert(Tokens[0].getLocation().isValid());
65  assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
66  "Macro defined in macro?");
67  assert(ExpandLocStart.isValid());
68 
69  // Reserve a source location entry chunk for the length of the macro
70  // definition. Tokens that get lexed directly from the definition will
71  // have their locations pointing inside this chunk. This is to avoid
72  // creating separate source location entries for each token.
73  MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());
74  MacroDefLength = Macro->getDefinitionLength(SM);
75  MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,
76  ExpandLocStart,
77  ExpandLocEnd,
78  MacroDefLength);
79  }
80 
81  // If this is a function-like macro, expand the arguments and change
82  // Tokens to point to the expanded tokens.
83  if (Macro->isFunctionLike() && Macro->getNumParams())
84  ExpandFunctionArguments();
85 
86  // Mark the macro as currently disabled, so that it is not recursively
87  // expanded. The macro must be disabled only after argument pre-expansion of
88  // function-like macro arguments occurs.
89  Macro->DisableMacro();
90 }
91 
92 /// Create a TokenLexer for the specified token stream. This does not
93 /// take ownership of the specified token vector.
94 void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
95  bool disableMacroExpansion, bool ownsTokens) {
96  // If the client is reusing a TokenLexer, make sure to free any memory
97  // associated with it.
98  destroy();
99 
100  Macro = nullptr;
101  ActualArgs = nullptr;
102  Tokens = TokArray;
103  OwnsTokens = ownsTokens;
104  DisableMacroExpansion = disableMacroExpansion;
105  NumTokens = NumToks;
106  CurTokenIdx = 0;
107  ExpandLocStart = ExpandLocEnd = SourceLocation();
108  AtStartOfLine = false;
109  HasLeadingSpace = false;
110  NextTokGetsSpace = false;
111  MacroExpansionStart = SourceLocation();
112 
113  // Set HasLeadingSpace/AtStartOfLine so that the first token will be
114  // returned unmodified.
115  if (NumToks != 0) {
116  AtStartOfLine = TokArray[0].isAtStartOfLine();
117  HasLeadingSpace = TokArray[0].hasLeadingSpace();
118  }
119 }
120 
121 void TokenLexer::destroy() {
122  // If this was a function-like macro that actually uses its arguments, delete
123  // the expanded tokens.
124  if (OwnsTokens) {
125  delete [] Tokens;
126  Tokens = nullptr;
127  OwnsTokens = false;
128  }
129 
130  // TokenLexer owns its formal arguments.
131  if (ActualArgs) ActualArgs->destroy(PP);
132 }
133 
134 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
135  SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
136  unsigned MacroArgNo, Preprocessor &PP) {
137  // Is the macro argument __VA_ARGS__?
138  if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1)
139  return false;
140 
141  // In Microsoft-compatibility mode, a comma is removed in the expansion
142  // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is
143  // not supported by gcc.
144  if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)
145  return false;
146 
147  // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if
148  // __VA_ARGS__ is empty, but not in strict C99 mode where there are no
149  // named arguments, where it remains. In all other modes, including C99
150  // with GNU extensions, it is removed regardless of named arguments.
151  // Microsoft also appears to support this extension, unofficially.
152  if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode
153  && Macro->getNumParams() < 2)
154  return false;
155 
156  // Is a comma available to be removed?
157  if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
158  return false;
159 
160  // Issue an extension diagnostic for the paste operator.
161  if (HasPasteOperator)
162  PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
163 
164  // Remove the comma.
165  ResultToks.pop_back();
166 
167  if (!ResultToks.empty()) {
168  // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
169  // then removal of the comma should produce a placemarker token (in C99
170  // terms) which we model by popping off the previous ##, giving us a plain
171  // "X" when __VA_ARGS__ is empty.
172  if (ResultToks.back().is(tok::hashhash))
173  ResultToks.pop_back();
174 
175  // Remember that this comma was elided.
176  ResultToks.back().setFlag(Token::CommaAfterElided);
177  }
178 
179  // Never add a space, even if the comma, ##, or arg had a space.
180  NextTokGetsSpace = false;
181  return true;
182 }
183 
184 void TokenLexer::stringifyVAOPTContents(
185  SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,
186  const SourceLocation VAOPTClosingParenLoc) {
187  const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt();
188  const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
189  Token *const VAOPTTokens =
190  NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr;
191 
192  SmallVector<Token, 64> ConcatenatedVAOPTResultToks;
193  // FIXME: Should we keep track within VCtx that we did or didnot
194  // encounter pasting - and only then perform this loop.
195 
196  // Perform token pasting (concatenation) prior to stringization.
197  for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
198  ++CurTokenIdx) {
199  if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {
200  assert(CurTokenIdx != 0 &&
201  "Can not have __VAOPT__ contents begin with a ##");
202  Token &LHS = VAOPTTokens[CurTokenIdx - 1];
203  pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens),
204  CurTokenIdx);
205  // Replace the token prior to the first ## in this iteration.
206  ConcatenatedVAOPTResultToks.back() = LHS;
207  if (CurTokenIdx == NumVAOptTokens)
208  break;
209  }
210  ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);
211  }
212 
213  ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok());
214  // Get the SourceLocation that represents the start location within
215  // the macro definition that marks where this string is substituted
216  // into: i.e. the __VA_OPT__ and the ')' within the spelling of the
217  // macro definition, and use it to indicate that the stringified token
218  // was generated from that location.
219  const SourceLocation ExpansionLocStartWithinMacro =
220  getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc());
221  const SourceLocation ExpansionLocEndWithinMacro =
222  getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);
223 
224  Token StringifiedVAOPT = MacroArgs::StringifyArgument(
225  &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() /*Charify*/,
226  ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);
227 
229  StringifiedVAOPT.setFlag(Token::LeadingSpace);
230 
231  StringifiedVAOPT.setFlag(Token::StringifiedInMacro);
232  // Resize (shrink) the token stream to just capture this stringified token.
233  ResultToks.resize(NumToksPriorToVAOpt + 1);
234  ResultToks.back() = StringifiedVAOPT;
235 }
236 
237 /// Expand the arguments of a function-like macro so that we can quickly
238 /// return preexpanded tokens from Tokens.
239 void TokenLexer::ExpandFunctionArguments() {
240  SmallVector<Token, 128> ResultToks;
241 
242  // Loop through 'Tokens', expanding them into ResultToks. Keep
243  // track of whether we change anything. If not, no need to keep them. If so,
244  // we install the newly expanded sequence as the new 'Tokens' list.
245  bool MadeChange = false;
246 
247  const bool CalledWithVariadicArguments =
248  ActualArgs->invokedWithVariadicArgument(Macro);
249 
250  VAOptExpansionContext VCtx(PP);
251 
252  for (unsigned I = 0, E = NumTokens; I != E; ++I) {
253  const Token &CurTok = Tokens[I];
254  // We don't want a space for the next token after a paste
255  // operator. In valid code, the token will get smooshed onto the
256  // preceding one anyway. In assembler-with-cpp mode, invalid
257  // pastes are allowed through: in this case, we do not want the
258  // extra whitespace to be added. For example, we want ". ## foo"
259  // -> ".foo" not ". foo".
260  if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace())
261  NextTokGetsSpace = true;
262 
263  if (VCtx.isVAOptToken(CurTok)) {
264  MadeChange = true;
265  assert(Tokens[I + 1].is(tok::l_paren) &&
266  "__VA_OPT__ must be followed by '('");
267 
268  ++I; // Skip the l_paren
270  ResultToks.size());
271 
272  continue;
273  }
274 
275  // We have entered into the __VA_OPT__ context, so handle tokens
276  // appropriately.
277  if (VCtx.isInVAOpt()) {
278  // If we are about to process a token that is either an argument to
279  // __VA_OPT__ or its closing rparen, then:
280  // 1) If the token is the closing rparen that exits us out of __VA_OPT__,
281  // perform any necessary stringification or placemarker processing,
282  // and/or skip to the next token.
283  // 2) else if macro was invoked without variadic arguments skip this
284  // token.
285  // 3) else (macro was invoked with variadic arguments) process the token
286  // normally.
287 
288  if (Tokens[I].is(tok::l_paren))
289  VCtx.sawOpeningParen(Tokens[I].getLocation());
290  // Continue skipping tokens within __VA_OPT__ if the macro was not
291  // called with variadic arguments, else let the rest of the loop handle
292  // this token. Note sawClosingParen() returns true only if the r_paren matches
293  // the closing r_paren of the __VA_OPT__.
294  if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) {
295  if (!CalledWithVariadicArguments) {
296  // Skip this token.
297  continue;
298  }
299  // ... else the macro was called with variadic arguments, and we do not
300  // have a closing rparen - so process this token normally.
301  } else {
302  // Current token is the closing r_paren which marks the end of the
303  // __VA_OPT__ invocation, so handle any place-marker pasting (if
304  // empty) by removing hashhash either before (if exists) or after. And
305  // also stringify the entire contents if VAOPT was preceded by a hash,
306  // but do so only after any token concatenation that needs to occur
307  // within the contents of VAOPT.
308 
309  if (VCtx.hasStringifyOrCharifyBefore()) {
310  // Replace all the tokens just added from within VAOPT into a single
311  // stringified token. This requires token-pasting to eagerly occur
312  // within these tokens. If either the contents of VAOPT were empty
313  // or the macro wasn't called with any variadic arguments, the result
314  // is a token that represents an empty string.
315  stringifyVAOPTContents(ResultToks, VCtx,
316  /*ClosingParenLoc*/ Tokens[I].getLocation());
317 
318  } else if (/*No tokens within VAOPT*/ !(
319  ResultToks.size() - VCtx.getNumberOfTokensPriorToVAOpt())) {
320  // Treat VAOPT as a placemarker token. Eat either the '##' before the
321  // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that
322  // hashhash was not a placemarker) or the '##'
323  // after VAOPT, but not both.
324 
325  if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {
326  ResultToks.pop_back();
327  } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {
328  ++I; // Skip the following hashhash.
329  }
330  }
331  VCtx.reset();
332  // We processed __VA_OPT__'s closing paren (and the exit out of
333  // __VA_OPT__), so skip to the next token.
334  continue;
335  }
336  }
337 
338  // If we found the stringify operator, get the argument stringified. The
339  // preprocessor already verified that the following token is a macro
340  // parameter or __VA_OPT__ when the #define was lexed.
341 
342  if (CurTok.isOneOf(tok::hash, tok::hashat)) {
343  int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());
344  assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&
345  "Token following # is not an argument or __VA_OPT__!");
346 
347  if (ArgNo == -1) {
348  // Handle the __VA_OPT__ case.
349  VCtx.sawHashOrHashAtBefore(NextTokGetsSpace,
350  CurTok.is(tok::hashat));
351  continue;
352  }
353  // Else handle the simple argument case.
354  SourceLocation ExpansionLocStart =
355  getExpansionLocForMacroDefLoc(CurTok.getLocation());
356  SourceLocation ExpansionLocEnd =
357  getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());
358 
359  Token Res;
360  if (CurTok.is(tok::hash)) // Stringify
361  Res = ActualArgs->getStringifiedArgument(ArgNo, PP,
362  ExpansionLocStart,
363  ExpansionLocEnd);
364  else {
365  // 'charify': don't bother caching these.
366  Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo),
367  PP, true,
368  ExpansionLocStart,
369  ExpansionLocEnd);
370  }
372 
373  // The stringified/charified string leading space flag gets set to match
374  // the #/#@ operator.
375  if (NextTokGetsSpace)
377 
378  ResultToks.push_back(Res);
379  MadeChange = true;
380  ++I; // Skip arg name.
381  NextTokGetsSpace = false;
382  continue;
383  }
384 
385  // Find out if there is a paste (##) operator before or after the token.
386  bool NonEmptyPasteBefore =
387  !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
388  bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash);
389  bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash);
390 
391  assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&
392  "unexpected ## in ResultToks");
393 
394  // Otherwise, if this is not an argument token, just add the token to the
395  // output buffer.
396  IdentifierInfo *II = CurTok.getIdentifierInfo();
397  int ArgNo = II ? Macro->getParameterNum(II) : -1;
398  if (ArgNo == -1) {
399  // This isn't an argument, just add it.
400  ResultToks.push_back(CurTok);
401 
402  if (NextTokGetsSpace) {
403  ResultToks.back().setFlag(Token::LeadingSpace);
404  NextTokGetsSpace = false;
405  } else if (PasteBefore && !NonEmptyPasteBefore)
406  ResultToks.back().clearFlag(Token::LeadingSpace);
407 
408  continue;
409  }
410 
411  // An argument is expanded somehow, the result is different than the
412  // input.
413  MadeChange = true;
414 
415  // Otherwise, this is a use of the argument.
416 
417  // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
418  // are no trailing commas if __VA_ARGS__ is empty.
419  if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
420  MaybeRemoveCommaBeforeVaArgs(ResultToks,
421  /*HasPasteOperator=*/false,
422  Macro, ArgNo, PP))
423  continue;
424 
425  // If it is not the LHS/RHS of a ## operator, we must pre-expand the
426  // argument and substitute the expanded tokens into the result. This is
427  // C99 6.10.3.1p1.
428  if (!PasteBefore && !PasteAfter) {
429  const Token *ResultArgToks;
430 
431  // Only preexpand the argument if it could possibly need it. This
432  // avoids some work in common cases.
433  const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
434  if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
435  ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];
436  else
437  ResultArgToks = ArgTok; // Use non-preexpanded tokens.
438 
439  // If the arg token expanded into anything, append it.
440  if (ResultArgToks->isNot(tok::eof)) {
441  size_t FirstResult = ResultToks.size();
442  unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
443  ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
444 
445  // In Microsoft-compatibility mode, we follow MSVC's preprocessing
446  // behavior by not considering single commas from nested macro
447  // expansions as argument separators. Set a flag on the token so we can
448  // test for this later when the macro expansion is processed.
449  if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&
450  ResultToks.back().is(tok::comma))
451  ResultToks.back().setFlag(Token::IgnoredComma);
452 
453  // If the '##' came from expanding an argument, turn it into 'unknown'
454  // to avoid pasting.
455  for (Token &Tok : llvm::make_range(ResultToks.begin() + FirstResult,
456  ResultToks.end())) {
457  if (Tok.is(tok::hashhash))
458  Tok.setKind(tok::unknown);
459  }
460 
461  if(ExpandLocStart.isValid()) {
462  updateLocForMacroArgTokens(CurTok.getLocation(),
463  ResultToks.begin()+FirstResult,
464  ResultToks.end());
465  }
466 
467  // If any tokens were substituted from the argument, the whitespace
468  // before the first token should match the whitespace of the arg
469  // identifier.
470  ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
471  NextTokGetsSpace);
472  ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
473  NextTokGetsSpace = false;
474  }
475  continue;
476  }
477 
478  // Okay, we have a token that is either the LHS or RHS of a paste (##)
479  // argument. It gets substituted as its non-pre-expanded tokens.
480  const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
481  unsigned NumToks = MacroArgs::getArgLength(ArgToks);
482  if (NumToks) { // Not an empty argument?
483  bool VaArgsPseudoPaste = false;
484  // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned
485  // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when
486  // the expander tries to paste ',' with the first token of the __VA_ARGS__
487  // expansion.
488  if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
489  ResultToks[ResultToks.size()-2].is(tok::comma) &&
490  (unsigned)ArgNo == Macro->getNumParams()-1 &&
491  Macro->isVariadic()) {
492  VaArgsPseudoPaste = true;
493  // Remove the paste operator, report use of the extension.
494  PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
495  }
496 
497  ResultToks.append(ArgToks, ArgToks+NumToks);
498 
499  // If the '##' came from expanding an argument, turn it into 'unknown'
500  // to avoid pasting.
501  for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
502  ResultToks.end())) {
503  if (Tok.is(tok::hashhash))
504  Tok.setKind(tok::unknown);
505  }
506 
507  if (ExpandLocStart.isValid()) {
508  updateLocForMacroArgTokens(CurTok.getLocation(),
509  ResultToks.end()-NumToks, ResultToks.end());
510  }
511 
512  // Transfer the leading whitespace information from the token
513  // (the macro argument) onto the first token of the
514  // expansion. Note that we don't do this for the GNU
515  // pseudo-paste extension ", ## __VA_ARGS__".
516  if (!VaArgsPseudoPaste) {
517  ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,
518  false);
519  ResultToks[ResultToks.size() - NumToks].setFlagValue(
520  Token::LeadingSpace, NextTokGetsSpace);
521  }
522 
523  NextTokGetsSpace = false;
524  continue;
525  }
526 
527  // If an empty argument is on the LHS or RHS of a paste, the standard (C99
528  // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We
529  // implement this by eating ## operators when a LHS or RHS expands to
530  // empty.
531  if (PasteAfter) {
532  // Discard the argument token and skip (don't copy to the expansion
533  // buffer) the paste operator after it.
534  ++I;
535  continue;
536  }
537 
538  // If this is on the RHS of a paste operator, we've already copied the
539  // paste operator to the ResultToks list, unless the LHS was empty too.
540  // Remove it.
541  assert(PasteBefore);
542  if (NonEmptyPasteBefore) {
543  assert(ResultToks.back().is(tok::hashhash));
544  // Do not remove the paste operator if it is the one before __VA_OPT__
545  // (and we are still processing tokens within VA_OPT). We handle the case
546  // of removing the paste operator if __VA_OPT__ reduces to the notional
547  // placemarker above when we encounter the closing paren of VA_OPT.
548  if (!VCtx.isInVAOpt() ||
549  ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt())
550  ResultToks.pop_back();
551  }
552 
553  // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
554  // and if the macro had at least one real argument, and if the token before
555  // the ## was a comma, remove the comma. This is a GCC extension which is
556  // disabled when using -std=c99.
557  if (ActualArgs->isVarargsElidedUse())
558  MaybeRemoveCommaBeforeVaArgs(ResultToks,
559  /*HasPasteOperator=*/true,
560  Macro, ArgNo, PP);
561  }
562 
563  // If anything changed, install this as the new Tokens list.
564  if (MadeChange) {
565  assert(!OwnsTokens && "This would leak if we already own the token list");
566  // This is deleted in the dtor.
567  NumTokens = ResultToks.size();
568  // The tokens will be added to Preprocessor's cache and will be removed
569  // when this TokenLexer finishes lexing them.
570  Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
571 
572  // The preprocessor cache of macro expanded tokens owns these tokens,not us.
573  OwnsTokens = false;
574  }
575 }
576 
577 /// Checks if two tokens form wide string literal.
578 static bool isWideStringLiteralFromMacro(const Token &FirstTok,
579  const Token &SecondTok) {
580  return FirstTok.is(tok::identifier) &&
581  FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral() &&
582  SecondTok.stringifiedInMacro();
583 }
584 
585 /// Lex - Lex and return a token from this macro stream.
587  // Lexing off the end of the macro, pop this macro off the expansion stack.
588  if (isAtEnd()) {
589  // If this is a macro (not a token stream), mark the macro enabled now
590  // that it is no longer being expanded.
591  if (Macro) Macro->EnableMacro();
592 
593  Tok.startToken();
594  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
595  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);
596  if (CurTokenIdx == 0)
598  return PP.HandleEndOfTokenLexer(Tok);
599  }
600 
602 
603  // If this is the first token of the expanded result, we inherit spacing
604  // properties later.
605  bool isFirstToken = CurTokenIdx == 0;
606 
607  // Get the next token to return.
608  Tok = Tokens[CurTokenIdx++];
609 
610  bool TokenIsFromPaste = false;
611 
612  // If this token is followed by a token paste (##) operator, paste the tokens!
613  // Note that ## is a normal token when not expanding a macro.
614  if (!isAtEnd() && Macro &&
615  (Tokens[CurTokenIdx].is(tok::hashhash) ||
616  // Special processing of L#x macros in -fms-compatibility mode.
617  // Microsoft compiler is able to form a wide string literal from
618  // 'L#macro_arg' construct in a function-like macro.
619  (PP.getLangOpts().MSVCCompat &&
620  isWideStringLiteralFromMacro(Tok, Tokens[CurTokenIdx])))) {
621  // When handling the microsoft /##/ extension, the final token is
622  // returned by pasteTokens, not the pasted token.
623  if (pasteTokens(Tok))
624  return true;
625 
626  TokenIsFromPaste = true;
627  }
628 
629  // The token's current location indicate where the token was lexed from. We
630  // need this information to compute the spelling of the token, but any
631  // diagnostics for the expanded token should appear as if they came from
632  // ExpansionLoc. Pull this information together into a new SourceLocation
633  // that captures all of this.
634  if (ExpandLocStart.isValid() && // Don't do this for token streams.
635  // Check that the token's location was not already set properly.
636  SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {
637  SourceLocation instLoc;
638  if (Tok.is(tok::comment)) {
639  instLoc = SM.createExpansionLoc(Tok.getLocation(),
640  ExpandLocStart,
641  ExpandLocEnd,
642  Tok.getLength());
643  } else {
644  instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());
645  }
646 
647  Tok.setLocation(instLoc);
648  }
649 
650  // If this is the first token, set the lexical properties of the token to
651  // match the lexical properties of the macro identifier.
652  if (isFirstToken) {
653  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
654  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
655  } else {
656  // If this is not the first token, we may still need to pass through
657  // leading whitespace if we've expanded a macro.
658  if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);
659  if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);
660  }
661  AtStartOfLine = false;
662  HasLeadingSpace = false;
663 
664  // Handle recursive expansion!
665  if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) {
666  // Change the kind of this identifier to the appropriate token kind, e.g.
667  // turning "for" into a keyword.
668  IdentifierInfo *II = Tok.getIdentifierInfo();
669  Tok.setKind(II->getTokenID());
670 
671  // If this identifier was poisoned and from a paste, emit an error. This
672  // won't be handled by Preprocessor::HandleIdentifier because this is coming
673  // from a macro expansion.
674  if (II->isPoisoned() && TokenIsFromPaste) {
675  PP.HandlePoisonedIdentifier(Tok);
676  }
677 
678  if (!DisableMacroExpansion && II->isHandleIdentifierCase())
679  return PP.HandleIdentifier(Tok);
680  }
681 
682  // Otherwise, return a normal token.
683  return true;
684 }
685 
686 bool TokenLexer::pasteTokens(Token &Tok) {
687  return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx);
688 }
689 
690 /// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ##
691 /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there
692 /// are more ## after it, chomp them iteratively. Return the result as LHSTok.
693 /// If this returns true, the caller should immediately return the token.
694 bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
695  unsigned int &CurIdx) {
696  assert(CurIdx > 0 && "## can not be the first token within tokens");
697  assert((TokenStream[CurIdx].is(tok::hashhash) ||
698  (PP.getLangOpts().MSVCCompat &&
699  isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) &&
700  "Token at this Index must be ## or part of the MSVC 'L "
701  "#macro-arg' pasting pair");
702 
703  // MSVC: If previous token was pasted, this must be a recovery from an invalid
704  // paste operation. Ignore spaces before this token to mimic MSVC output.
705  // Required for generating valid UUID strings in some MS headers.
706  if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) &&
707  TokenStream[CurIdx - 2].is(tok::hashhash))
709 
710  SmallString<128> Buffer;
711  const char *ResultTokStrPtr = nullptr;
712  SourceLocation StartLoc = LHSTok.getLocation();
713  SourceLocation PasteOpLoc;
714 
715  auto IsAtEnd = [&TokenStream, &CurIdx] {
716  return TokenStream.size() == CurIdx;
717  };
718 
719  do {
720  // Consume the ## operator if any.
721  PasteOpLoc = TokenStream[CurIdx].getLocation();
722  if (TokenStream[CurIdx].is(tok::hashhash))
723  ++CurIdx;
724  assert(!IsAtEnd() && "No token on the RHS of a paste operator!");
725 
726  // Get the RHS token.
727  const Token &RHS = TokenStream[CurIdx];
728 
729  // Allocate space for the result token. This is guaranteed to be enough for
730  // the two tokens.
731  Buffer.resize(LHSTok.getLength() + RHS.getLength());
732 
733  // Get the spelling of the LHS token in Buffer.
734  const char *BufPtr = &Buffer[0];
735  bool Invalid = false;
736  unsigned LHSLen = PP.getSpelling(LHSTok, BufPtr, &Invalid);
737  if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer!
738  memcpy(&Buffer[0], BufPtr, LHSLen);
739  if (Invalid)
740  return true;
741 
742  BufPtr = Buffer.data() + LHSLen;
743  unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid);
744  if (Invalid)
745  return true;
746  if (RHSLen && BufPtr != &Buffer[LHSLen])
747  // Really, we want the chars in Buffer!
748  memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
749 
750  // Trim excess space.
751  Buffer.resize(LHSLen+RHSLen);
752 
753  // Plop the pasted result (including the trailing newline and null) into a
754  // scratch buffer where we can lex it.
755  Token ResultTokTmp;
756  ResultTokTmp.startToken();
757 
758  // Claim that the tmp token is a string_literal so that we can get the
759  // character pointer back from CreateString in getLiteralData().
760  ResultTokTmp.setKind(tok::string_literal);
761  PP.CreateString(Buffer, ResultTokTmp);
762  SourceLocation ResultTokLoc = ResultTokTmp.getLocation();
763  ResultTokStrPtr = ResultTokTmp.getLiteralData();
764 
765  // Lex the resultant pasted token into Result.
766  Token Result;
767 
768  if (LHSTok.isAnyIdentifier() && RHS.isAnyIdentifier()) {
769  // Common paste case: identifier+identifier = identifier. Avoid creating
770  // a lexer and other overhead.
771  PP.IncrementPasteCounter(true);
772  Result.startToken();
773  Result.setKind(tok::raw_identifier);
774  Result.setRawIdentifierData(ResultTokStrPtr);
775  Result.setLocation(ResultTokLoc);
776  Result.setLength(LHSLen+RHSLen);
777  } else {
778  PP.IncrementPasteCounter(false);
779 
780  assert(ResultTokLoc.isFileID() &&
781  "Should be a raw location into scratch buffer");
782  SourceManager &SourceMgr = PP.getSourceManager();
783  FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
784 
785  bool Invalid = false;
786  const char *ScratchBufStart
787  = SourceMgr.getBufferData(LocFileID, &Invalid).data();
788  if (Invalid)
789  return false;
790 
791  // Make a lexer to lex this string from. Lex just this one token.
792  // Make a lexer object so that we lex and expand the paste result.
793  Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
794  PP.getLangOpts(), ScratchBufStart,
795  ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
796 
797  // Lex a token in raw mode. This way it won't look up identifiers
798  // automatically, lexing off the end will return an eof token, and
799  // warnings are disabled. This returns true if the result token is the
800  // entire buffer.
801  bool isInvalid = !TL.LexFromRawLexer(Result);
802 
803  // If we got an EOF token, we didn't form even ONE token. For example, we
804  // did "/ ## /" to get "//".
805  isInvalid |= Result.is(tok::eof);
806 
807  // If pasting the two tokens didn't form a full new token, this is an
808  // error. This occurs with "x ## +" and other stuff. Return with LHSTok
809  // unmodified and with RHS as the next token to lex.
810  if (isInvalid) {
811  // Explicitly convert the token location to have proper expansion
812  // information so that the user knows where it came from.
814  SourceLocation Loc =
815  SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);
816 
817  // Test for the Microsoft extension of /##/ turning into // here on the
818  // error path.
819  if (PP.getLangOpts().MicrosoftExt && LHSTok.is(tok::slash) &&
820  RHS.is(tok::slash)) {
821  HandleMicrosoftCommentPaste(LHSTok, Loc);
822  return true;
823  }
824 
825  // Do not emit the error when preprocessing assembler code.
826  if (!PP.getLangOpts().AsmPreprocessor) {
827  // If we're in microsoft extensions mode, downgrade this from a hard
828  // error to an extension that defaults to an error. This allows
829  // disabling it.
830  PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms
831  : diag::err_pp_bad_paste)
832  << Buffer;
833  }
834 
835  // An error has occurred so exit loop.
836  break;
837  }
838 
839  // Turn ## into 'unknown' to avoid # ## # from looking like a paste
840  // operator.
841  if (Result.is(tok::hashhash))
842  Result.setKind(tok::unknown);
843  }
844 
845  // Transfer properties of the LHS over the Result.
848 
849  // Finally, replace LHS with the result, consume the RHS, and iterate.
850  ++CurIdx;
851  LHSTok = Result;
852  } while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));
853 
854  SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation();
855 
856  // The token's current location indicate where the token was lexed from. We
857  // need this information to compute the spelling of the token, but any
858  // diagnostics for the expanded token should appear as if the token was
859  // expanded from the full ## expression. Pull this information together into
860  // a new SourceLocation that captures all of this.
862  if (StartLoc.isFileID())
863  StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
864  if (EndLoc.isFileID())
865  EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
866  FileID MacroFID = SM.getFileID(MacroExpansionStart);
867  while (SM.getFileID(StartLoc) != MacroFID)
868  StartLoc = SM.getImmediateExpansionRange(StartLoc).getBegin();
869  while (SM.getFileID(EndLoc) != MacroFID)
870  EndLoc = SM.getImmediateExpansionRange(EndLoc).getEnd();
871 
872  LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc,
873  LHSTok.getLength()));
874 
875  // Now that we got the result token, it will be subject to expansion. Since
876  // token pasting re-lexes the result token in raw mode, identifier information
877  // isn't looked up. As such, if the result is an identifier, look up id info.
878  if (LHSTok.is(tok::raw_identifier)) {
879  // Look up the identifier info for the token. We disabled identifier lookup
880  // by saying we're skipping contents, so we need to do this manually.
881  PP.LookUpIdentifierInfo(LHSTok);
882  }
883  return false;
884 }
885 
886 /// isNextTokenLParen - If the next token lexed will pop this macro off the
887 /// expansion stack, return 2. If the next unexpanded token is a '(', return
888 /// 1, otherwise return 0.
890  // Out of tokens?
891  if (isAtEnd())
892  return 2;
893  return Tokens[CurTokenIdx].is(tok::l_paren);
894 }
895 
896 /// isParsingPreprocessorDirective - Return true if we are in the middle of a
897 /// preprocessor directive.
899  return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd();
900 }
901 
902 /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
903 /// together to form a comment that comments out everything in the current
904 /// macro, other active macros, and anything left on the current physical
905 /// source line of the expanded buffer. Handle this by returning the
906 /// first token on the next line.
907 void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {
908  PP.Diag(OpLoc, diag::ext_comment_paste_microsoft);
909 
910  // We 'comment out' the rest of this macro by just ignoring the rest of the
911  // tokens that have not been lexed yet, if any.
912 
913  // Since this must be a macro, mark the macro enabled now that it is no longer
914  // being expanded.
915  assert(Macro && "Token streams can't paste comments");
916  Macro->EnableMacro();
917 
919 }
920 
921 /// If \arg loc is a file ID and points inside the current macro
922 /// definition, returns the appropriate source location pointing at the
923 /// macro expansion source location entry, otherwise it returns an invalid
924 /// SourceLocation.
926 TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {
927  assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&
928  "Not appropriate for token streams");
929  assert(loc.isValid() && loc.isFileID());
930 
932  assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
933  "Expected loc to come from the macro definition");
934 
935  unsigned relativeOffset = 0;
936  SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);
937  return MacroExpansionStart.getLocWithOffset(relativeOffset);
938 }
939 
940 /// Finds the tokens that are consecutive (from the same FileID)
941 /// creates a single SLocEntry, and assigns SourceLocations to each token that
942 /// point to that SLocEntry. e.g for
943 /// assert(foo == bar);
944 /// There will be a single SLocEntry for the "foo == bar" chunk and locations
945 /// for the 'foo', '==', 'bar' tokens will point inside that chunk.
946 ///
947 /// \arg begin_tokens will be updated to a position past all the found
948 /// consecutive tokens.
950  SourceLocation InstLoc,
951  Token *&begin_tokens,
952  Token * end_tokens) {
953  assert(begin_tokens < end_tokens);
954 
955  SourceLocation FirstLoc = begin_tokens->getLocation();
956  SourceLocation CurLoc = FirstLoc;
957 
958  // Compare the source location offset of tokens and group together tokens that
959  // are close, even if their locations point to different FileIDs. e.g.
960  //
961  // |bar | foo | cake | (3 tokens from 3 consecutive FileIDs)
962  // ^ ^
963  // |bar foo cake| (one SLocEntry chunk for all tokens)
964  //
965  // we can perform this "merge" since the token's spelling location depends
966  // on the relative offset.
967 
968  Token *NextTok = begin_tokens + 1;
969  for (; NextTok < end_tokens; ++NextTok) {
970  SourceLocation NextLoc = NextTok->getLocation();
971  if (CurLoc.isFileID() != NextLoc.isFileID())
972  break; // Token from different kind of FileID.
973 
974  int RelOffs;
975  if (!SM.isInSameSLocAddrSpace(CurLoc, NextLoc, &RelOffs))
976  break; // Token from different local/loaded location.
977  // Check that token is not before the previous token or more than 50
978  // "characters" away.
979  if (RelOffs < 0 || RelOffs > 50)
980  break;
981 
982  if (CurLoc.isMacroID() && !SM.isWrittenInSameFile(CurLoc, NextLoc))
983  break; // Token from a different macro.
984 
985  CurLoc = NextLoc;
986  }
987 
988  // For the consecutive tokens, find the length of the SLocEntry to contain
989  // all of them.
990  Token &LastConsecutiveTok = *(NextTok-1);
991  int LastRelOffs = 0;
992  SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(),
993  &LastRelOffs);
994  unsigned FullLength = LastRelOffs + LastConsecutiveTok.getLength();
995 
996  // Create a macro expansion SLocEntry that will "contain" all of the tokens.
997  SourceLocation Expansion =
998  SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength);
999 
1000  // Change the location of the tokens from the spelling location to the new
1001  // expanded location.
1002  for (; begin_tokens < NextTok; ++begin_tokens) {
1003  Token &Tok = *begin_tokens;
1004  int RelOffs = 0;
1005  SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs);
1006  Tok.setLocation(Expansion.getLocWithOffset(RelOffs));
1007  }
1008 }
1009 
1010 /// Creates SLocEntries and updates the locations of macro argument
1011 /// tokens to their new expanded locations.
1012 ///
1013 /// \param ArgIdSpellLoc the location of the macro argument id inside the macro
1014 /// definition.
1015 void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
1016  Token *begin_tokens,
1017  Token *end_tokens) {
1019 
1020  SourceLocation InstLoc =
1021  getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
1022 
1023  while (begin_tokens < end_tokens) {
1024  // If there's only one token just create a SLocEntry for it.
1025  if (end_tokens - begin_tokens == 1) {
1026  Token &Tok = *begin_tokens;
1028  InstLoc,
1029  Tok.getLength()));
1030  return;
1031  }
1032 
1033  updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens);
1034  }
1035 }
1036 
1037 void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
1038  AtStartOfLine = Result.isAtStartOfLine();
1039  HasLeadingSpace = Result.hasLeadingSpace();
1040 }
bool invokedWithVariadicArgument(const MacroInfo *const MI) const
Returns true if the macro was defined with a variadic (ellipsis) parameter AND was invoked with at le...
Definition: MacroArgs.cpp:143
bool isWrittenInSameFile(SourceLocation Loc1, SourceLocation Loc2) const
Returns true if the spelling locations for both SourceLocations are part of the same file buffer...
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:77
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void DisableMacro()
Definition: MacroInfo.h:266
void sawOpeningParen(SourceLocation LParenLoc)
Call this function each time an lparen is seen.
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Definition: Token.h:257
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
ArgNeedsPreexpansion - If we can prove that the argument won&#39;t be affected by pre-expansion, return false.
Definition: MacroArgs.cpp:152
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
bool sawClosingParen()
Call this function each time an rparen is seen.
Defines the SourceManager interface.
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation InstLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry, and assigns SourceLocations to each token that point to that SLocEntry.
Definition: TokenLexer.cpp:949
A class for tracking whether we&#39;re inside a VA_OPT during a traversal of the tokens of a macro during...
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool isBeforeInSLocAddrSpace(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the "source location address space".
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:234
bool isVarargsElidedUse() const
isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...
Definition: MacroArgs.h:113
bool isVariadic() const
Definition: MacroInfo.h:208
bool isInSameSLocAddrSpace(SourceLocation LHS, SourceLocation RHS, int *RelativeOffset) const
Return true if both LHS and RHS are in the local source location address space or the loaded one...
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
Definition: Token.h:107
unsigned getNextLocalOffset() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:118
tokens_iterator tokens_begin() const
Definition: MacroInfo.h:243
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:113
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
One of these records is kept for each identifier that is lexed.
SourceLocation getBegin() const
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:207
void sawVAOptFollowedByOpeningParens(const SourceLocation VAOptLoc, const unsigned int NumPriorTokens)
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
void setKind(tok::TokenKind K)
Definition: Token.h:91
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:827
bool Lex(Token &Tok)
Lex and return a token from this macro stream.
Definition: TokenLexer.cpp:586
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
const FormatToken & Tok
SourceLocation getExpansionLoc(SourceLocation Loc) const
Given a SourceLocation object Loc, return the expansion location referenced by the ID...
Defines the Diagnostic-related interfaces.
void destroy(Preprocessor &PP)
destroy - Destroy and deallocate the memory for this object.
Definition: MacroArgs.cpp:79
void Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, MacroArgs *Actuals)
Initialize this TokenLexer to expand from the specified macro with the specified argument information...
Definition: TokenLexer.cpp:39
const Token & getStringifiedArgument(unsigned ArgNo, Preprocessor &PP, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
getStringifiedArgument - Compute, cache, and return the specified argument that has been &#39;stringified...
Definition: MacroArgs.cpp:317
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
Definition: TokenLexer.cpp:578
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...
Definition: MacroArgs.cpp:214
unsigned getNumParams() const
Definition: MacroInfo.h:183
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Defines the clang::LangOptions interface.
bool isInSLocAddrSpace(SourceLocation Loc, SourceLocation Start, unsigned Length, unsigned *RelativeOffset=nullptr) const
Returns true if Loc is inside the [Start, +Length) chunk of the source location address space...
SourceLocation getVAOptLoc() const
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
SourceLocation createMacroArgExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLoc, unsigned TokLength)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
CharSourceRange getImmediateExpansionRange(SourceLocation Loc) const
Return the start/end of the expansion information for an expansion location.
The result type of a method or function.
const SourceManager & SM
Definition: Format.cpp:1490
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
SourceManager & getSourceManager() const
Definition: Preprocessor.h:831
bool isVAOptToken(const Token &T) const
Encodes a location in the source.
void setLength(unsigned Len)
Definition: Token.h:133
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, bool ExpansionIsTokenRange=true, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:307
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:177
bool isInVAOpt() const
Returns true if we have seen the VA_OPT and &#39;(&#39; but before having seen the matching &#39;)&#39;...
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:266
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:96
static bool isInvalid(LocType Loc, bool *Invalid)
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
bool isFunctionLike() const
Definition: MacroInfo.h:200
const std::vector< Token > & getPreExpArgument(unsigned Arg, Preprocessor &PP)
getPreExpArgument - Return the pre-expanded form of the specified argument.
Definition: MacroArgs.cpp:167
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:97
tokens_iterator tokens_end() const
Definition: MacroInfo.h:244
static unsigned getArgLength(const Token *ArgPtr)
getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens...
Definition: MacroArgs.cpp:111
void sawHashOrHashAtBefore(const bool HasLeadingSpace, const bool IsHashAt)
unsigned getLength() const
Definition: Token.h:127
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:40
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:215
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
int getParameterNum(const IdentifierInfo *Arg) const
Return the parameter number of the specified identifier, or -1 if the identifier is not a formal para...
Definition: MacroInfo.h:190
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
unsigned int getNumberOfTokensPriorToVAOpt() const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token&#39;s location and length to ...
SourceLocation getEnd() const
unsigned isNextTokenLParen() const
If the next token lexed will pop this macro off the expansion stack, return 2.
Definition: TokenLexer.cpp:889
Defines the clang::TokenKind enum and support functions.
unsigned getDefinitionLength(const SourceManager &SM) const
Get length in characters of the macro definition.
Definition: MacroInfo.h:133
Defines the clang::SourceLocation class and associated facilities.
bool stringifiedInMacro() const
Returns true if this token is formed by macro by stringizing or charizing operator.
Definition: Token.h:298
void setLocation(SourceLocation L)
Definition: Token.h:132
void EnableMacro()
Definition: MacroInfo.h:261
void clearFlag(TokenFlags Flag)
Unset the specified flag.
Definition: Token.h:244
bool isParsingPreprocessorDirective() const
isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive...
Definition: TokenLexer.cpp:898
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:270
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
This class handles loading and caching of source files into memory.
const Token * getUnexpArgument(unsigned Arg) const
getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...
Definition: MacroArgs.cpp:121
void startToken()
Reset all flags to cleared.
Definition: Token.h:169
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:127