clang  6.0.0svn
TokenLexer.cpp
Go to the documentation of this file.
1 //===--- TokenLexer.cpp - Lex from a token stream -------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the TokenLexer interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "clang/Lex/TokenLexer.h"
17 #include "clang/Lex/MacroArgs.h"
18 #include "clang/Lex/MacroInfo.h"
19 #include "clang/Lex/Preprocessor.h"
21 #include "llvm/ADT/SmallString.h"
22 
23 using namespace clang;
24 
25 /// Create a TokenLexer for the specified macro with the specified actual
26 /// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
28  MacroArgs *Actuals) {
29  // If the client is reusing a TokenLexer, make sure to free any memory
30  // associated with it.
31  destroy();
32 
33  Macro = MI;
34  ActualArgs = Actuals;
35  CurTokenIdx = 0;
36 
37  ExpandLocStart = Tok.getLocation();
38  ExpandLocEnd = ELEnd;
39  AtStartOfLine = Tok.isAtStartOfLine();
40  HasLeadingSpace = Tok.hasLeadingSpace();
41  NextTokGetsSpace = false;
42  Tokens = &*Macro->tokens_begin();
43  OwnsTokens = false;
44  DisableMacroExpansion = false;
45  NumTokens = Macro->tokens_end()-Macro->tokens_begin();
46  MacroExpansionStart = SourceLocation();
47 
49  MacroStartSLocOffset = SM.getNextLocalOffset();
50 
51  if (NumTokens > 0) {
52  assert(Tokens[0].getLocation().isValid());
53  assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
54  "Macro defined in macro?");
55  assert(ExpandLocStart.isValid());
56 
57  // Reserve a source location entry chunk for the length of the macro
58  // definition. Tokens that get lexed directly from the definition will
59  // have their locations pointing inside this chunk. This is to avoid
60  // creating separate source location entries for each token.
61  MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());
62  MacroDefLength = Macro->getDefinitionLength(SM);
63  MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,
64  ExpandLocStart,
65  ExpandLocEnd,
66  MacroDefLength);
67  }
68 
69  // If this is a function-like macro, expand the arguments and change
70  // Tokens to point to the expanded tokens.
71  if (Macro->isFunctionLike() && Macro->getNumParams())
72  ExpandFunctionArguments();
73 
74  // Mark the macro as currently disabled, so that it is not recursively
75  // expanded. The macro must be disabled only after argument pre-expansion of
76  // function-like macro arguments occurs.
77  Macro->DisableMacro();
78 }
79 
80 /// Create a TokenLexer for the specified token stream. This does not
81 /// take ownership of the specified token vector.
82 void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
83  bool disableMacroExpansion, bool ownsTokens) {
84  // If the client is reusing a TokenLexer, make sure to free any memory
85  // associated with it.
86  destroy();
87 
88  Macro = nullptr;
89  ActualArgs = nullptr;
90  Tokens = TokArray;
91  OwnsTokens = ownsTokens;
92  DisableMacroExpansion = disableMacroExpansion;
93  NumTokens = NumToks;
94  CurTokenIdx = 0;
95  ExpandLocStart = ExpandLocEnd = SourceLocation();
96  AtStartOfLine = false;
97  HasLeadingSpace = false;
98  NextTokGetsSpace = false;
99  MacroExpansionStart = SourceLocation();
100 
101  // Set HasLeadingSpace/AtStartOfLine so that the first token will be
102  // returned unmodified.
103  if (NumToks != 0) {
104  AtStartOfLine = TokArray[0].isAtStartOfLine();
105  HasLeadingSpace = TokArray[0].hasLeadingSpace();
106  }
107 }
108 
109 void TokenLexer::destroy() {
110  // If this was a function-like macro that actually uses its arguments, delete
111  // the expanded tokens.
112  if (OwnsTokens) {
113  delete [] Tokens;
114  Tokens = nullptr;
115  OwnsTokens = false;
116  }
117 
118  // TokenLexer owns its formal arguments.
119  if (ActualArgs) ActualArgs->destroy(PP);
120 }
121 
122 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
123  SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
124  unsigned MacroArgNo, Preprocessor &PP) {
125  // Is the macro argument __VA_ARGS__?
126  if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1)
127  return false;
128 
129  // In Microsoft-compatibility mode, a comma is removed in the expansion
130  // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is
131  // not supported by gcc.
132  if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)
133  return false;
134 
135  // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if
136  // __VA_ARGS__ is empty, but not in strict C99 mode where there are no
137  // named arguments, where it remains. In all other modes, including C99
138  // with GNU extensions, it is removed regardless of named arguments.
139  // Microsoft also appears to support this extension, unofficially.
140  if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode
141  && Macro->getNumParams() < 2)
142  return false;
143 
144  // Is a comma available to be removed?
145  if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
146  return false;
147 
148  // Issue an extension diagnostic for the paste operator.
149  if (HasPasteOperator)
150  PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
151 
152  // Remove the comma.
153  ResultToks.pop_back();
154 
155  if (!ResultToks.empty()) {
156  // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
157  // then removal of the comma should produce a placemarker token (in C99
158  // terms) which we model by popping off the previous ##, giving us a plain
159  // "X" when __VA_ARGS__ is empty.
160  if (ResultToks.back().is(tok::hashhash))
161  ResultToks.pop_back();
162 
163  // Remember that this comma was elided.
164  ResultToks.back().setFlag(Token::CommaAfterElided);
165  }
166 
167  // Never add a space, even if the comma, ##, or arg had a space.
168  NextTokGetsSpace = false;
169  return true;
170 }
171 
172 void TokenLexer::stringifyVAOPTContents(
173  SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,
174  const SourceLocation VAOPTClosingParenLoc) {
175  const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt();
176  const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
177  Token *const VAOPTTokens =
178  NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr;
179 
180  SmallVector<Token, 64> ConcatenatedVAOPTResultToks;
181  // FIXME: Should we keep track within VCtx that we did or didnot
182  // encounter pasting - and only then perform this loop.
183 
184  // Perform token pasting (concatenation) prior to stringization.
185  for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
186  ++CurTokenIdx) {
187  if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {
188  assert(CurTokenIdx != 0 &&
189  "Can not have __VAOPT__ contents begin with a ##");
190  Token &LHS = VAOPTTokens[CurTokenIdx - 1];
191  pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens),
192  CurTokenIdx);
193  // Replace the token prior to the first ## in this iteration.
194  ConcatenatedVAOPTResultToks.back() = LHS;
195  if (CurTokenIdx == NumVAOptTokens)
196  break;
197  }
198  ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);
199  }
200 
201  ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok());
202  // Get the SourceLocation that represents the start location within
203  // the macro definition that marks where this string is substituted
204  // into: i.e. the __VA_OPT__ and the ')' within the spelling of the
205  // macro definition, and use it to indicate that the stringified token
206  // was generated from that location.
207  const SourceLocation ExpansionLocStartWithinMacro =
208  getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc());
209  const SourceLocation ExpansionLocEndWithinMacro =
210  getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);
211 
212  Token StringifiedVAOPT = MacroArgs::StringifyArgument(
213  &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() /*Charify*/,
214  ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);
215 
217  StringifiedVAOPT.setFlag(Token::LeadingSpace);
218 
219  StringifiedVAOPT.setFlag(Token::StringifiedInMacro);
220  // Resize (shrink) the token stream to just capture this stringified token.
221  ResultToks.resize(NumToksPriorToVAOpt + 1);
222  ResultToks.back() = StringifiedVAOPT;
223 }
224 
225 /// Expand the arguments of a function-like macro so that we can quickly
226 /// return preexpanded tokens from Tokens.
227 void TokenLexer::ExpandFunctionArguments() {
228  SmallVector<Token, 128> ResultToks;
229 
230  // Loop through 'Tokens', expanding them into ResultToks. Keep
231  // track of whether we change anything. If not, no need to keep them. If so,
232  // we install the newly expanded sequence as the new 'Tokens' list.
233  bool MadeChange = false;
234 
235  const bool CalledWithVariadicArguments =
236  ActualArgs->invokedWithVariadicArgument(Macro);
237 
238  VAOptExpansionContext VCtx(PP);
239 
240  for (unsigned I = 0, E = NumTokens; I != E; ++I) {
241 
242  const Token &CurTok = Tokens[I];
243  // We don't want a space for the next token after a paste
244  // operator. In valid code, the token will get smooshed onto the
245  // preceding one anyway. In assembler-with-cpp mode, invalid
246  // pastes are allowed through: in this case, we do not want the
247  // extra whitespace to be added. For example, we want ". ## foo"
248  // -> ".foo" not ". foo".
249  if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace())
250  NextTokGetsSpace = true;
251 
252  if (VCtx.isVAOptToken(CurTok)) {
253  MadeChange = true;
254  assert(Tokens[I + 1].is(tok::l_paren) &&
255  "__VA_OPT__ must be followed by '('");
256 
257  ++I; // Skip the l_paren
259  ResultToks.size());
260 
261  continue;
262  }
263 
264  // We have entered into the __VA_OPT__ context, so handle tokens
265  // appropriately.
266  if (VCtx.isInVAOpt()) {
267  // If we are about to process a token that is either an argument to
268  // __VA_OPT__ or its closing rparen, then:
269  // 1) If the token is the closing rparen that exits us out of __VA_OPT__,
270  // perform any necessary stringification or placemarker processing,
271  // and/or skip to the next token.
272  // 2) else if macro was invoked without variadic arguments skip this
273  // token.
274  // 3) else (macro was invoked with variadic arguments) process the token
275  // normally.
276 
277  if (Tokens[I].is(tok::l_paren))
278  VCtx.sawOpeningParen(Tokens[I].getLocation());
279  // Continue skipping tokens within __VA_OPT__ if the macro was not
280  // called with variadic arguments, else let the rest of the loop handle
281  // this token. Note sawClosingParen() returns true only if the r_paren matches
282  // the closing r_paren of the __VA_OPT__.
283  if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) {
284  if (!CalledWithVariadicArguments) {
285  // Skip this token.
286  continue;
287  }
288  // ... else the macro was called with variadic arguments, and we do not
289  // have a closing rparen - so process this token normally.
290 
291  } else {
292  // Current token is the closing r_paren which marks the end of the
293  // __VA_OPT__ invocation, so handle any place-marker pasting (if
294  // empty) by removing hashhash either before (if exists) or after. And
295  // also stringify the entire contents if VAOPT was preceded by a hash,
296  // but do so only after any token concatenation that needs to occur
297  // within the contents of VAOPT.
298 
299  if (VCtx.hasStringifyOrCharifyBefore()) {
300  // Replace all the tokens just added from within VAOPT into a single
301  // stringified token. This requires token-pasting to eagerly occur
302  // within these tokens. If either the contents of VAOPT were empty
303  // or the macro wasn't called with any variadic arguments, the result
304  // is a token that represents an empty string.
305  stringifyVAOPTContents(ResultToks, VCtx,
306  /*ClosingParenLoc*/ Tokens[I].getLocation());
307 
308  } else if (/*No tokens within VAOPT*/ !(
309  ResultToks.size() - VCtx.getNumberOfTokensPriorToVAOpt())) {
310  // Treat VAOPT as a placemarker token. Eat either the '##' before the
311  // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that
312  // hashhash was not a placemarker) or the '##'
313  // after VAOPT, but not both.
314 
315  if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {
316  ResultToks.pop_back();
317  } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {
318  ++I; // Skip the following hashhash.
319  }
320  }
321  VCtx.reset();
322  // We processed __VA_OPT__'s closing paren (and the exit out of
323  // __VA_OPT__), so skip to the next token.
324  continue;
325  }
326  }
327 
328  // If we found the stringify operator, get the argument stringified. The
329  // preprocessor already verified that the following token is a macro
330  // parameter or __VA_OPT__ when the #define was lexed.
331 
332  if (CurTok.isOneOf(tok::hash, tok::hashat)) {
333  int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());
334  assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&
335  "Token following # is not an argument or __VA_OPT__!");
336 
337  if (ArgNo == -1) {
338  // Handle the __VA_OPT__ case.
339  VCtx.sawHashOrHashAtBefore(NextTokGetsSpace,
340  CurTok.is(tok::hashat));
341  continue;
342  }
343  // Else handle the simple argument case.
344  SourceLocation ExpansionLocStart =
345  getExpansionLocForMacroDefLoc(CurTok.getLocation());
346  SourceLocation ExpansionLocEnd =
347  getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());
348 
349  Token Res;
350  if (CurTok.is(tok::hash)) // Stringify
351  Res = ActualArgs->getStringifiedArgument(ArgNo, PP,
352  ExpansionLocStart,
353  ExpansionLocEnd);
354  else {
355  // 'charify': don't bother caching these.
356  Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo),
357  PP, true,
358  ExpansionLocStart,
359  ExpansionLocEnd);
360  }
362 
363  // The stringified/charified string leading space flag gets set to match
364  // the #/#@ operator.
365  if (NextTokGetsSpace)
367 
368  ResultToks.push_back(Res);
369  MadeChange = true;
370  ++I; // Skip arg name.
371  NextTokGetsSpace = false;
372  continue;
373  }
374 
375  // Find out if there is a paste (##) operator before or after the token.
376  bool NonEmptyPasteBefore =
377  !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
378  bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash);
379  bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash);
380 
381  assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&
382  "unexpected ## in ResultToks");
383 
384  // Otherwise, if this is not an argument token, just add the token to the
385  // output buffer.
386  IdentifierInfo *II = CurTok.getIdentifierInfo();
387  int ArgNo = II ? Macro->getParameterNum(II) : -1;
388  if (ArgNo == -1) {
389  // This isn't an argument, just add it.
390  ResultToks.push_back(CurTok);
391 
392  if (NextTokGetsSpace) {
393  ResultToks.back().setFlag(Token::LeadingSpace);
394  NextTokGetsSpace = false;
395  } else if (PasteBefore && !NonEmptyPasteBefore)
396  ResultToks.back().clearFlag(Token::LeadingSpace);
397 
398  continue;
399  }
400 
401  // An argument is expanded somehow, the result is different than the
402  // input.
403  MadeChange = true;
404 
405  // Otherwise, this is a use of the argument.
406 
407  // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
408  // are no trailing commas if __VA_ARGS__ is empty.
409  if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
410  MaybeRemoveCommaBeforeVaArgs(ResultToks,
411  /*HasPasteOperator=*/false,
412  Macro, ArgNo, PP))
413  continue;
414 
415  // If it is not the LHS/RHS of a ## operator, we must pre-expand the
416  // argument and substitute the expanded tokens into the result. This is
417  // C99 6.10.3.1p1.
418  if (!PasteBefore && !PasteAfter) {
419  const Token *ResultArgToks;
420 
421  // Only preexpand the argument if it could possibly need it. This
422  // avoids some work in common cases.
423  const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
424  if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
425  ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];
426  else
427  ResultArgToks = ArgTok; // Use non-preexpanded tokens.
428 
429  // If the arg token expanded into anything, append it.
430  if (ResultArgToks->isNot(tok::eof)) {
431  size_t FirstResult = ResultToks.size();
432  unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
433  ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
434 
435  // In Microsoft-compatibility mode, we follow MSVC's preprocessing
436  // behavior by not considering single commas from nested macro
437  // expansions as argument separators. Set a flag on the token so we can
438  // test for this later when the macro expansion is processed.
439  if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&
440  ResultToks.back().is(tok::comma))
441  ResultToks.back().setFlag(Token::IgnoredComma);
442 
443  // If the '##' came from expanding an argument, turn it into 'unknown'
444  // to avoid pasting.
445  for (Token &Tok : llvm::make_range(ResultToks.begin() + FirstResult,
446  ResultToks.end())) {
447  if (Tok.is(tok::hashhash))
448  Tok.setKind(tok::unknown);
449  }
450 
451  if(ExpandLocStart.isValid()) {
452  updateLocForMacroArgTokens(CurTok.getLocation(),
453  ResultToks.begin()+FirstResult,
454  ResultToks.end());
455  }
456 
457  // If any tokens were substituted from the argument, the whitespace
458  // before the first token should match the whitespace of the arg
459  // identifier.
460  ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
461  NextTokGetsSpace);
462  ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
463  NextTokGetsSpace = false;
464  }
465  continue;
466  }
467 
468  // Okay, we have a token that is either the LHS or RHS of a paste (##)
469  // argument. It gets substituted as its non-pre-expanded tokens.
470  const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
471  unsigned NumToks = MacroArgs::getArgLength(ArgToks);
472  if (NumToks) { // Not an empty argument?
473  bool VaArgsPseudoPaste = false;
474  // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned
475  // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when
476  // the expander trys to paste ',' with the first token of the __VA_ARGS__
477  // expansion.
478  if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
479  ResultToks[ResultToks.size()-2].is(tok::comma) &&
480  (unsigned)ArgNo == Macro->getNumParams()-1 &&
481  Macro->isVariadic()) {
482  VaArgsPseudoPaste = true;
483  // Remove the paste operator, report use of the extension.
484  PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
485  }
486 
487  ResultToks.append(ArgToks, ArgToks+NumToks);
488 
489  // If the '##' came from expanding an argument, turn it into 'unknown'
490  // to avoid pasting.
491  for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
492  ResultToks.end())) {
493  if (Tok.is(tok::hashhash))
494  Tok.setKind(tok::unknown);
495  }
496 
497  if (ExpandLocStart.isValid()) {
498  updateLocForMacroArgTokens(CurTok.getLocation(),
499  ResultToks.end()-NumToks, ResultToks.end());
500  }
501 
502  // Transfer the leading whitespace information from the token
503  // (the macro argument) onto the first token of the
504  // expansion. Note that we don't do this for the GNU
505  // pseudo-paste extension ", ## __VA_ARGS__".
506  if (!VaArgsPseudoPaste) {
507  ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,
508  false);
509  ResultToks[ResultToks.size() - NumToks].setFlagValue(
510  Token::LeadingSpace, NextTokGetsSpace);
511  }
512 
513  NextTokGetsSpace = false;
514  continue;
515  }
516 
517  // If an empty argument is on the LHS or RHS of a paste, the standard (C99
518  // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We
519  // implement this by eating ## operators when a LHS or RHS expands to
520  // empty.
521  if (PasteAfter) {
522  // Discard the argument token and skip (don't copy to the expansion
523  // buffer) the paste operator after it.
524  ++I;
525  continue;
526  }
527 
528  // If this is on the RHS of a paste operator, we've already copied the
529  // paste operator to the ResultToks list, unless the LHS was empty too.
530  // Remove it.
531  assert(PasteBefore);
532  if (NonEmptyPasteBefore) {
533  assert(ResultToks.back().is(tok::hashhash));
534  // Do not remove the paste operator if it is the one before __VA_OPT__
535  // (and we are still processing tokens within VA_OPT). We handle the case
536  // of removing the paste operator if __VA_OPT__ reduces to the notional
537  // placemarker above when we encounter the closing paren of VA_OPT.
538  if (!VCtx.isInVAOpt() ||
539  ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt())
540  ResultToks.pop_back();
541  }
542 
543  // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
544  // and if the macro had at least one real argument, and if the token before
545  // the ## was a comma, remove the comma. This is a GCC extension which is
546  // disabled when using -std=c99.
547  if (ActualArgs->isVarargsElidedUse())
548  MaybeRemoveCommaBeforeVaArgs(ResultToks,
549  /*HasPasteOperator=*/true,
550  Macro, ArgNo, PP);
551  }
552 
553  // If anything changed, install this as the new Tokens list.
554  if (MadeChange) {
555  assert(!OwnsTokens && "This would leak if we already own the token list");
556  // This is deleted in the dtor.
557  NumTokens = ResultToks.size();
558  // The tokens will be added to Preprocessor's cache and will be removed
559  // when this TokenLexer finishes lexing them.
560  Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
561 
562  // The preprocessor cache of macro expanded tokens owns these tokens,not us.
563  OwnsTokens = false;
564  }
565 }
566 
567 /// \brief Checks if two tokens form wide string literal.
568 static bool isWideStringLiteralFromMacro(const Token &FirstTok,
569  const Token &SecondTok) {
570  return FirstTok.is(tok::identifier) &&
571  FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral() &&
572  SecondTok.stringifiedInMacro();
573 }
574 
575 /// Lex - Lex and return a token from this macro stream.
576 ///
578  // Lexing off the end of the macro, pop this macro off the expansion stack.
579  if (isAtEnd()) {
580  // If this is a macro (not a token stream), mark the macro enabled now
581  // that it is no longer being expanded.
582  if (Macro) Macro->EnableMacro();
583 
584  Tok.startToken();
585  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
586  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);
587  if (CurTokenIdx == 0)
589  return PP.HandleEndOfTokenLexer(Tok);
590  }
591 
593 
594  // If this is the first token of the expanded result, we inherit spacing
595  // properties later.
596  bool isFirstToken = CurTokenIdx == 0;
597 
598  // Get the next token to return.
599  Tok = Tokens[CurTokenIdx++];
600 
601  bool TokenIsFromPaste = false;
602 
603  // If this token is followed by a token paste (##) operator, paste the tokens!
604  // Note that ## is a normal token when not expanding a macro.
605  if (!isAtEnd() && Macro &&
606  (Tokens[CurTokenIdx].is(tok::hashhash) ||
607  // Special processing of L#x macros in -fms-compatibility mode.
608  // Microsoft compiler is able to form a wide string literal from
609  // 'L#macro_arg' construct in a function-like macro.
610  (PP.getLangOpts().MSVCCompat &&
611  isWideStringLiteralFromMacro(Tok, Tokens[CurTokenIdx])))) {
612  // When handling the microsoft /##/ extension, the final token is
613  // returned by pasteTokens, not the pasted token.
614  if (pasteTokens(Tok))
615  return true;
616 
617  TokenIsFromPaste = true;
618  }
619 
620  // The token's current location indicate where the token was lexed from. We
621  // need this information to compute the spelling of the token, but any
622  // diagnostics for the expanded token should appear as if they came from
623  // ExpansionLoc. Pull this information together into a new SourceLocation
624  // that captures all of this.
625  if (ExpandLocStart.isValid() && // Don't do this for token streams.
626  // Check that the token's location was not already set properly.
627  SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {
628  SourceLocation instLoc;
629  if (Tok.is(tok::comment)) {
630  instLoc = SM.createExpansionLoc(Tok.getLocation(),
631  ExpandLocStart,
632  ExpandLocEnd,
633  Tok.getLength());
634  } else {
635  instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());
636  }
637 
638  Tok.setLocation(instLoc);
639  }
640 
641  // If this is the first token, set the lexical properties of the token to
642  // match the lexical properties of the macro identifier.
643  if (isFirstToken) {
644  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
645  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
646  } else {
647  // If this is not the first token, we may still need to pass through
648  // leading whitespace if we've expanded a macro.
649  if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);
650  if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);
651  }
652  AtStartOfLine = false;
653  HasLeadingSpace = false;
654 
655  // Handle recursive expansion!
656  if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) {
657  // Change the kind of this identifier to the appropriate token kind, e.g.
658  // turning "for" into a keyword.
659  IdentifierInfo *II = Tok.getIdentifierInfo();
660  Tok.setKind(II->getTokenID());
661 
662  // If this identifier was poisoned and from a paste, emit an error. This
663  // won't be handled by Preprocessor::HandleIdentifier because this is coming
664  // from a macro expansion.
665  if (II->isPoisoned() && TokenIsFromPaste) {
666  PP.HandlePoisonedIdentifier(Tok);
667  }
668 
669  if (!DisableMacroExpansion && II->isHandleIdentifierCase())
670  return PP.HandleIdentifier(Tok);
671  }
672 
673  // Otherwise, return a normal token.
674  return true;
675 }
676 
677 bool TokenLexer::pasteTokens(Token &Tok) {
678  return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx);
679 }
680 /// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ##
681 /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there
682 /// are more ## after it, chomp them iteratively. Return the result as LHSTok.
683 /// If this returns true, the caller should immediately return the token.
684 bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
685  unsigned int &CurIdx) {
686  assert(CurIdx > 0 && "## can not be the first token within tokens");
687  assert((TokenStream[CurIdx].is(tok::hashhash) ||
688  (PP.getLangOpts().MSVCCompat &&
689  isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) &&
690  "Token at this Index must be ## or part of the MSVC 'L "
691  "#macro-arg' pasting pair");
692 
693  // MSVC: If previous token was pasted, this must be a recovery from an invalid
694  // paste operation. Ignore spaces before this token to mimic MSVC output.
695  // Required for generating valid UUID strings in some MS headers.
696  if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) &&
697  TokenStream[CurIdx - 2].is(tok::hashhash))
699 
700  SmallString<128> Buffer;
701  const char *ResultTokStrPtr = nullptr;
702  SourceLocation StartLoc = LHSTok.getLocation();
703  SourceLocation PasteOpLoc;
704 
705  auto IsAtEnd = [&TokenStream, &CurIdx] {
706  return TokenStream.size() == CurIdx;
707  };
708 
709  do {
710  // Consume the ## operator if any.
711  PasteOpLoc = TokenStream[CurIdx].getLocation();
712  if (TokenStream[CurIdx].is(tok::hashhash))
713  ++CurIdx;
714  assert(!IsAtEnd() && "No token on the RHS of a paste operator!");
715 
716  // Get the RHS token.
717  const Token &RHS = TokenStream[CurIdx];
718 
719  // Allocate space for the result token. This is guaranteed to be enough for
720  // the two tokens.
721  Buffer.resize(LHSTok.getLength() + RHS.getLength());
722 
723  // Get the spelling of the LHS token in Buffer.
724  const char *BufPtr = &Buffer[0];
725  bool Invalid = false;
726  unsigned LHSLen = PP.getSpelling(LHSTok, BufPtr, &Invalid);
727  if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer!
728  memcpy(&Buffer[0], BufPtr, LHSLen);
729  if (Invalid)
730  return true;
731 
732  BufPtr = Buffer.data() + LHSLen;
733  unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid);
734  if (Invalid)
735  return true;
736  if (RHSLen && BufPtr != &Buffer[LHSLen])
737  // Really, we want the chars in Buffer!
738  memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
739 
740  // Trim excess space.
741  Buffer.resize(LHSLen+RHSLen);
742 
743  // Plop the pasted result (including the trailing newline and null) into a
744  // scratch buffer where we can lex it.
745  Token ResultTokTmp;
746  ResultTokTmp.startToken();
747 
748  // Claim that the tmp token is a string_literal so that we can get the
749  // character pointer back from CreateString in getLiteralData().
750  ResultTokTmp.setKind(tok::string_literal);
751  PP.CreateString(Buffer, ResultTokTmp);
752  SourceLocation ResultTokLoc = ResultTokTmp.getLocation();
753  ResultTokStrPtr = ResultTokTmp.getLiteralData();
754 
755  // Lex the resultant pasted token into Result.
756  Token Result;
757 
758  if (LHSTok.isAnyIdentifier() && RHS.isAnyIdentifier()) {
759  // Common paste case: identifier+identifier = identifier. Avoid creating
760  // a lexer and other overhead.
761  PP.IncrementPasteCounter(true);
762  Result.startToken();
763  Result.setKind(tok::raw_identifier);
764  Result.setRawIdentifierData(ResultTokStrPtr);
765  Result.setLocation(ResultTokLoc);
766  Result.setLength(LHSLen+RHSLen);
767  } else {
768  PP.IncrementPasteCounter(false);
769 
770  assert(ResultTokLoc.isFileID() &&
771  "Should be a raw location into scratch buffer");
772  SourceManager &SourceMgr = PP.getSourceManager();
773  FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
774 
775  bool Invalid = false;
776  const char *ScratchBufStart
777  = SourceMgr.getBufferData(LocFileID, &Invalid).data();
778  if (Invalid)
779  return false;
780 
781  // Make a lexer to lex this string from. Lex just this one token.
782  // Make a lexer object so that we lex and expand the paste result.
783  Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
784  PP.getLangOpts(), ScratchBufStart,
785  ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
786 
787  // Lex a token in raw mode. This way it won't look up identifiers
788  // automatically, lexing off the end will return an eof token, and
789  // warnings are disabled. This returns true if the result token is the
790  // entire buffer.
791  bool isInvalid = !TL.LexFromRawLexer(Result);
792 
793  // If we got an EOF token, we didn't form even ONE token. For example, we
794  // did "/ ## /" to get "//".
795  isInvalid |= Result.is(tok::eof);
796 
797  // If pasting the two tokens didn't form a full new token, this is an
798  // error. This occurs with "x ## +" and other stuff. Return with LHSTok
799  // unmodified and with RHS as the next token to lex.
800  if (isInvalid) {
801  // Explicitly convert the token location to have proper expansion
802  // information so that the user knows where it came from.
804  SourceLocation Loc =
805  SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);
806 
807  // Test for the Microsoft extension of /##/ turning into // here on the
808  // error path.
809  if (PP.getLangOpts().MicrosoftExt && LHSTok.is(tok::slash) &&
810  RHS.is(tok::slash)) {
811  HandleMicrosoftCommentPaste(LHSTok, Loc);
812  return true;
813  }
814 
815  // Do not emit the error when preprocessing assembler code.
816  if (!PP.getLangOpts().AsmPreprocessor) {
817  // If we're in microsoft extensions mode, downgrade this from a hard
818  // error to an extension that defaults to an error. This allows
819  // disabling it.
820  PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms
821  : diag::err_pp_bad_paste)
822  << Buffer;
823  }
824 
825  // An error has occurred so exit loop.
826  break;
827  }
828 
829  // Turn ## into 'unknown' to avoid # ## # from looking like a paste
830  // operator.
831  if (Result.is(tok::hashhash))
832  Result.setKind(tok::unknown);
833  }
834 
835  // Transfer properties of the LHS over the Result.
838 
839  // Finally, replace LHS with the result, consume the RHS, and iterate.
840  ++CurIdx;
841  LHSTok = Result;
842  } while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));
843 
844  SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation();
845 
846  // The token's current location indicate where the token was lexed from. We
847  // need this information to compute the spelling of the token, but any
848  // diagnostics for the expanded token should appear as if the token was
849  // expanded from the full ## expression. Pull this information together into
850  // a new SourceLocation that captures all of this.
852  if (StartLoc.isFileID())
853  StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
854  if (EndLoc.isFileID())
855  EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
856  FileID MacroFID = SM.getFileID(MacroExpansionStart);
857  while (SM.getFileID(StartLoc) != MacroFID)
858  StartLoc = SM.getImmediateExpansionRange(StartLoc).first;
859  while (SM.getFileID(EndLoc) != MacroFID)
860  EndLoc = SM.getImmediateExpansionRange(EndLoc).second;
861 
862  LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc,
863  LHSTok.getLength()));
864 
865  // Now that we got the result token, it will be subject to expansion. Since
866  // token pasting re-lexes the result token in raw mode, identifier information
867  // isn't looked up. As such, if the result is an identifier, look up id info.
868  if (LHSTok.is(tok::raw_identifier)) {
869  // Look up the identifier info for the token. We disabled identifier lookup
870  // by saying we're skipping contents, so we need to do this manually.
871  PP.LookUpIdentifierInfo(LHSTok);
872  }
873  return false;
874 }
875 
876 /// isNextTokenLParen - If the next token lexed will pop this macro off the
877 /// expansion stack, return 2. If the next unexpanded token is a '(', return
878 /// 1, otherwise return 0.
880  // Out of tokens?
881  if (isAtEnd())
882  return 2;
883  return Tokens[CurTokenIdx].is(tok::l_paren);
884 }
885 
886 /// isParsingPreprocessorDirective - Return true if we are in the middle of a
887 /// preprocessor directive.
889  return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd();
890 }
891 
892 /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
893 /// together to form a comment that comments out everything in the current
894 /// macro, other active macros, and anything left on the current physical
895 /// source line of the expanded buffer. Handle this by returning the
896 /// first token on the next line.
897 void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {
898  PP.Diag(OpLoc, diag::ext_comment_paste_microsoft);
899 
900  // We 'comment out' the rest of this macro by just ignoring the rest of the
901  // tokens that have not been lexed yet, if any.
902 
903  // Since this must be a macro, mark the macro enabled now that it is no longer
904  // being expanded.
905  assert(Macro && "Token streams can't paste comments");
906  Macro->EnableMacro();
907 
909 }
910 
911 /// \brief If \arg loc is a file ID and points inside the current macro
912 /// definition, returns the appropriate source location pointing at the
913 /// macro expansion source location entry, otherwise it returns an invalid
914 /// SourceLocation.
916 TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {
917  assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&
918  "Not appropriate for token streams");
919  assert(loc.isValid() && loc.isFileID());
920 
922  assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
923  "Expected loc to come from the macro definition");
924 
925  unsigned relativeOffset = 0;
926  SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);
927  return MacroExpansionStart.getLocWithOffset(relativeOffset);
928 }
929 
930 /// \brief Finds the tokens that are consecutive (from the same FileID)
931 /// creates a single SLocEntry, and assigns SourceLocations to each token that
932 /// point to that SLocEntry. e.g for
933 /// assert(foo == bar);
934 /// There will be a single SLocEntry for the "foo == bar" chunk and locations
935 /// for the 'foo', '==', 'bar' tokens will point inside that chunk.
936 ///
937 /// \arg begin_tokens will be updated to a position past all the found
938 /// consecutive tokens.
940  SourceLocation InstLoc,
941  Token *&begin_tokens,
942  Token * end_tokens) {
943  assert(begin_tokens < end_tokens);
944 
945  SourceLocation FirstLoc = begin_tokens->getLocation();
946  SourceLocation CurLoc = FirstLoc;
947 
948  // Compare the source location offset of tokens and group together tokens that
949  // are close, even if their locations point to different FileIDs. e.g.
950  //
951  // |bar | foo | cake | (3 tokens from 3 consecutive FileIDs)
952  // ^ ^
953  // |bar foo cake| (one SLocEntry chunk for all tokens)
954  //
955  // we can perform this "merge" since the token's spelling location depends
956  // on the relative offset.
957 
958  Token *NextTok = begin_tokens + 1;
959  for (; NextTok < end_tokens; ++NextTok) {
960  SourceLocation NextLoc = NextTok->getLocation();
961  if (CurLoc.isFileID() != NextLoc.isFileID())
962  break; // Token from different kind of FileID.
963 
964  int RelOffs;
965  if (!SM.isInSameSLocAddrSpace(CurLoc, NextLoc, &RelOffs))
966  break; // Token from different local/loaded location.
967  // Check that token is not before the previous token or more than 50
968  // "characters" away.
969  if (RelOffs < 0 || RelOffs > 50)
970  break;
971 
972  if (CurLoc.isMacroID() && !SM.isWrittenInSameFile(CurLoc, NextLoc))
973  break; // Token from a different macro.
974 
975  CurLoc = NextLoc;
976  }
977 
978  // For the consecutive tokens, find the length of the SLocEntry to contain
979  // all of them.
980  Token &LastConsecutiveTok = *(NextTok-1);
981  int LastRelOffs = 0;
982  SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(),
983  &LastRelOffs);
984  unsigned FullLength = LastRelOffs + LastConsecutiveTok.getLength();
985 
986  // Create a macro expansion SLocEntry that will "contain" all of the tokens.
987  SourceLocation Expansion =
988  SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength);
989 
990  // Change the location of the tokens from the spelling location to the new
991  // expanded location.
992  for (; begin_tokens < NextTok; ++begin_tokens) {
993  Token &Tok = *begin_tokens;
994  int RelOffs = 0;
995  SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs);
996  Tok.setLocation(Expansion.getLocWithOffset(RelOffs));
997  }
998 }
999 
1000 /// \brief Creates SLocEntries and updates the locations of macro argument
1001 /// tokens to their new expanded locations.
1002 ///
1003 /// \param ArgIdSpellLoc the location of the macro argument id inside the macro
1004 /// definition.
1005 void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
1006  Token *begin_tokens,
1007  Token *end_tokens) {
1009 
1010  SourceLocation InstLoc =
1011  getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
1012 
1013  while (begin_tokens < end_tokens) {
1014  // If there's only one token just create a SLocEntry for it.
1015  if (end_tokens - begin_tokens == 1) {
1016  Token &Tok = *begin_tokens;
1018  InstLoc,
1019  Tok.getLength()));
1020  return;
1021  }
1022 
1023  updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens);
1024  }
1025 }
1026 
1027 void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
1028  AtStartOfLine = Result.isAtStartOfLine();
1029  HasLeadingSpace = Result.hasLeadingSpace();
1030 }
bool invokedWithVariadicArgument(const MacroInfo *const MI) const
Returns true if the macro was defined with a variadic (ellipsis) parameter AND was invoked with at le...
Definition: MacroArgs.cpp:142
bool isWrittenInSameFile(SourceLocation Loc1, SourceLocation Loc2) const
Returns true if the spelling locations for both SourceLocations are part of the same file buffer...
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:63
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void DisableMacro()
Definition: MacroInfo.h:259
void Init(Token &Tok, SourceLocation ILEnd, MacroInfo *MI, MacroArgs *ActualArgs)
Init - Initialize this TokenLexer to expand from the specified macro with the specified argument info...
Definition: TokenLexer.cpp:27
void sawOpeningParen(SourceLocation LParenLoc)
Call this function each time an lparen is seen.
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Definition: Token.h:257
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
ArgNeedsPreexpansion - If we can prove that the argument won&#39;t be affected by pre-expansion, return false.
Definition: MacroArgs.cpp:151
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
bool sawClosingParen()
Call this function each time an rparen is seen.
Defines the SourceManager interface.
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation InstLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry, and assigns SourceLocations to each token that point to that SLocEntry.
Definition: TokenLexer.cpp:939
A class for tracking whether we&#39;re inside a VA_OPT during a traversal of the tokens of a macro during...
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool isBeforeInSLocAddrSpace(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the "source location address space".
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:234
bool isVarargsElidedUse() const
isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...
Definition: MacroArgs.h:113
bool isVariadic() const
Definition: MacroInfo.h:201
bool isInSameSLocAddrSpace(SourceLocation LHS, SourceLocation RHS, int *RelativeOffset) const
Return true if both LHS and RHS are in the local source location address space or the loaded one...
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
Definition: Token.h:107
unsigned getNextLocalOffset() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:118
tokens_iterator tokens_begin() const
Definition: MacroInfo.h:236
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:113
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
One of these records is kept for each identifier that is lexed.
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:207
void sawVAOptFollowedByOpeningParens(const SourceLocation VAOptLoc, const unsigned int NumPriorTokens)
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
void setKind(tok::TokenKind K)
Definition: Token.h:91
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:728
bool Lex(Token &Tok)
Lex - Lex and return a token from this macro stream.
Definition: TokenLexer.cpp:577
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
const FormatToken & Tok
SourceLocation getExpansionLoc(SourceLocation Loc) const
Given a SourceLocation object Loc, return the expansion location referenced by the ID...
void destroy(Preprocessor &PP)
destroy - Destroy and deallocate the memory for this object.
Definition: MacroArgs.cpp:78
const Token & getStringifiedArgument(unsigned ArgNo, Preprocessor &PP, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
getStringifiedArgument - Compute, cache, and return the specified argument that has been &#39;stringified...
Definition: MacroArgs.cpp:316
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
Definition: TokenLexer.cpp:568
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...
Definition: MacroArgs.cpp:213
unsigned getNumParams() const
Definition: MacroInfo.h:176
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
bool isInSLocAddrSpace(SourceLocation Loc, SourceLocation Start, unsigned Length, unsigned *RelativeOffset=nullptr) const
Returns true if Loc is inside the [Start, +Length) chunk of the source location address space...
SourceLocation getVAOptLoc() const
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
SourceLocation createMacroArgExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLoc, unsigned TokLength)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
The result type of a method or function.
const SourceManager & SM
Definition: Format.cpp:1308
SourceManager & getSourceManager() const
Definition: Preprocessor.h:732
bool isVAOptToken(const Token &T) const
Encodes a location in the source.
std::pair< SourceLocation, SourceLocation > getImmediateExpansionRange(SourceLocation Loc) const
Return the start/end of the expansion information for an expansion location.
void setLength(unsigned Len)
Definition: Token.h:133
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:297
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:177
bool isInVAOpt() const
Returns true if we have seen the VA_OPT and &#39;(&#39; but before having seen the matching &#39;)&#39;...
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:266
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:96
static bool isInvalid(LocType Loc, bool *Invalid)
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
bool isFunctionLike() const
Definition: MacroInfo.h:193
const std::vector< Token > & getPreExpArgument(unsigned Arg, Preprocessor &PP)
getPreExpArgument - Return the pre-expanded form of the specified argument.
Definition: MacroArgs.cpp:166
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:97
tokens_iterator tokens_end() const
Definition: MacroInfo.h:237
static unsigned getArgLength(const Token *ArgPtr)
getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens...
Definition: MacroArgs.cpp:110
void sawHashOrHashAtBefore(const bool HasLeadingSpace, const bool IsHashAt)
unsigned getLength() const
Definition: Token.h:127
bool isMacroID() const
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:34
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:215
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
int getParameterNum(const IdentifierInfo *Arg) const
Return the parameter number of the specified identifier, or -1 if the identifier is not a formal para...
Definition: MacroInfo.h:183
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
unsigned int getNumberOfTokensPriorToVAOpt() const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token&#39;s location and length to ...
unsigned isNextTokenLParen() const
isNextTokenLParen - If the next token lexed will pop this macro off the expansion stack...
Definition: TokenLexer.cpp:879
void HandlePoisonedIdentifier(Token &Tok)
Display reason for poisoned identifier.
unsigned getDefinitionLength(const SourceManager &SM) const
Get length in characters of the macro definition.
Definition: MacroInfo.h:126
bool stringifiedInMacro() const
Returns true if this token is formed by macro by stringizing or charizing operator.
Definition: Token.h:298
void setLocation(SourceLocation L)
Definition: Token.h:132
void EnableMacro()
Definition: MacroInfo.h:254
void clearFlag(TokenFlags Flag)
Unset the specified flag.
Definition: Token.h:244
bool isParsingPreprocessorDirective() const
isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive...
Definition: TokenLexer.cpp:888
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:270
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
This class handles loading and caching of source files into memory.
const Token * getUnexpArgument(unsigned Arg) const
getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...
Definition: MacroArgs.cpp:120
void startToken()
Reset all flags to cleared.
Definition: Token.h:169
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:98