clang  6.0.0svn
TokenLexer.cpp
Go to the documentation of this file.
1 //===- TokenLexer.cpp - Lex from a token stream ---------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the TokenLexer interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "clang/Lex/TokenLexer.h"
15 #include "clang/Basic/Diagnostic.h"
20 #include "clang/Basic/TokenKinds.h"
22 #include "clang/Lex/Lexer.h"
23 #include "clang/Lex/MacroArgs.h"
24 #include "clang/Lex/MacroInfo.h"
25 #include "clang/Lex/Preprocessor.h"
26 #include "clang/Lex/Token.h"
28 #include "llvm/ADT/ArrayRef.h"
29 #include "llvm/ADT/SmallString.h"
30 #include "llvm/ADT/SmallVector.h"
31 #include "llvm/ADT/iterator_range.h"
32 #include <cassert>
33 #include <cstring>
34 
35 using namespace clang;
36 
37 /// Create a TokenLexer for the specified macro with the specified actual
38 /// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
40  MacroArgs *Actuals) {
41  // If the client is reusing a TokenLexer, make sure to free any memory
42  // associated with it.
43  destroy();
44 
45  Macro = MI;
46  ActualArgs = Actuals;
47  CurTokenIdx = 0;
48 
49  ExpandLocStart = Tok.getLocation();
50  ExpandLocEnd = ELEnd;
51  AtStartOfLine = Tok.isAtStartOfLine();
52  HasLeadingSpace = Tok.hasLeadingSpace();
53  NextTokGetsSpace = false;
54  Tokens = &*Macro->tokens_begin();
55  OwnsTokens = false;
56  DisableMacroExpansion = false;
57  NumTokens = Macro->tokens_end()-Macro->tokens_begin();
58  MacroExpansionStart = SourceLocation();
59 
61  MacroStartSLocOffset = SM.getNextLocalOffset();
62 
63  if (NumTokens > 0) {
64  assert(Tokens[0].getLocation().isValid());
65  assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
66  "Macro defined in macro?");
67  assert(ExpandLocStart.isValid());
68 
69  // Reserve a source location entry chunk for the length of the macro
70  // definition. Tokens that get lexed directly from the definition will
71  // have their locations pointing inside this chunk. This is to avoid
72  // creating separate source location entries for each token.
73  MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());
74  MacroDefLength = Macro->getDefinitionLength(SM);
75  MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,
76  ExpandLocStart,
77  ExpandLocEnd,
78  MacroDefLength);
79  }
80 
81  // If this is a function-like macro, expand the arguments and change
82  // Tokens to point to the expanded tokens.
83  if (Macro->isFunctionLike() && Macro->getNumParams())
84  ExpandFunctionArguments();
85 
86  // Mark the macro as currently disabled, so that it is not recursively
87  // expanded. The macro must be disabled only after argument pre-expansion of
88  // function-like macro arguments occurs.
89  Macro->DisableMacro();
90 }
91 
92 /// Create a TokenLexer for the specified token stream. This does not
93 /// take ownership of the specified token vector.
94 void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
95  bool disableMacroExpansion, bool ownsTokens) {
96  // If the client is reusing a TokenLexer, make sure to free any memory
97  // associated with it.
98  destroy();
99 
100  Macro = nullptr;
101  ActualArgs = nullptr;
102  Tokens = TokArray;
103  OwnsTokens = ownsTokens;
104  DisableMacroExpansion = disableMacroExpansion;
105  NumTokens = NumToks;
106  CurTokenIdx = 0;
107  ExpandLocStart = ExpandLocEnd = SourceLocation();
108  AtStartOfLine = false;
109  HasLeadingSpace = false;
110  NextTokGetsSpace = false;
111  MacroExpansionStart = SourceLocation();
112 
113  // Set HasLeadingSpace/AtStartOfLine so that the first token will be
114  // returned unmodified.
115  if (NumToks != 0) {
116  AtStartOfLine = TokArray[0].isAtStartOfLine();
117  HasLeadingSpace = TokArray[0].hasLeadingSpace();
118  }
119 }
120 
121 void TokenLexer::destroy() {
122  // If this was a function-like macro that actually uses its arguments, delete
123  // the expanded tokens.
124  if (OwnsTokens) {
125  delete [] Tokens;
126  Tokens = nullptr;
127  OwnsTokens = false;
128  }
129 
130  // TokenLexer owns its formal arguments.
131  if (ActualArgs) ActualArgs->destroy(PP);
132 }
133 
134 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
135  SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
136  unsigned MacroArgNo, Preprocessor &PP) {
137  // Is the macro argument __VA_ARGS__?
138  if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1)
139  return false;
140 
141  // In Microsoft-compatibility mode, a comma is removed in the expansion
142  // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is
143  // not supported by gcc.
144  if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)
145  return false;
146 
147  // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if
148  // __VA_ARGS__ is empty, but not in strict C99 mode where there are no
149  // named arguments, where it remains. In all other modes, including C99
150  // with GNU extensions, it is removed regardless of named arguments.
151  // Microsoft also appears to support this extension, unofficially.
152  if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode
153  && Macro->getNumParams() < 2)
154  return false;
155 
156  // Is a comma available to be removed?
157  if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
158  return false;
159 
160  // Issue an extension diagnostic for the paste operator.
161  if (HasPasteOperator)
162  PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
163 
164  // Remove the comma.
165  ResultToks.pop_back();
166 
167  if (!ResultToks.empty()) {
168  // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
169  // then removal of the comma should produce a placemarker token (in C99
170  // terms) which we model by popping off the previous ##, giving us a plain
171  // "X" when __VA_ARGS__ is empty.
172  if (ResultToks.back().is(tok::hashhash))
173  ResultToks.pop_back();
174 
175  // Remember that this comma was elided.
176  ResultToks.back().setFlag(Token::CommaAfterElided);
177  }
178 
179  // Never add a space, even if the comma, ##, or arg had a space.
180  NextTokGetsSpace = false;
181  return true;
182 }
183 
184 void TokenLexer::stringifyVAOPTContents(
185  SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,
186  const SourceLocation VAOPTClosingParenLoc) {
187  const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt();
188  const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
189  Token *const VAOPTTokens =
190  NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr;
191 
192  SmallVector<Token, 64> ConcatenatedVAOPTResultToks;
193  // FIXME: Should we keep track within VCtx that we did or didnot
194  // encounter pasting - and only then perform this loop.
195 
196  // Perform token pasting (concatenation) prior to stringization.
197  for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
198  ++CurTokenIdx) {
199  if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {
200  assert(CurTokenIdx != 0 &&
201  "Can not have __VAOPT__ contents begin with a ##");
202  Token &LHS = VAOPTTokens[CurTokenIdx - 1];
203  pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens),
204  CurTokenIdx);
205  // Replace the token prior to the first ## in this iteration.
206  ConcatenatedVAOPTResultToks.back() = LHS;
207  if (CurTokenIdx == NumVAOptTokens)
208  break;
209  }
210  ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);
211  }
212 
213  ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok());
214  // Get the SourceLocation that represents the start location within
215  // the macro definition that marks where this string is substituted
216  // into: i.e. the __VA_OPT__ and the ')' within the spelling of the
217  // macro definition, and use it to indicate that the stringified token
218  // was generated from that location.
219  const SourceLocation ExpansionLocStartWithinMacro =
220  getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc());
221  const SourceLocation ExpansionLocEndWithinMacro =
222  getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);
223 
224  Token StringifiedVAOPT = MacroArgs::StringifyArgument(
225  &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() /*Charify*/,
226  ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);
227 
229  StringifiedVAOPT.setFlag(Token::LeadingSpace);
230 
231  StringifiedVAOPT.setFlag(Token::StringifiedInMacro);
232  // Resize (shrink) the token stream to just capture this stringified token.
233  ResultToks.resize(NumToksPriorToVAOpt + 1);
234  ResultToks.back() = StringifiedVAOPT;
235 }
236 
237 /// Expand the arguments of a function-like macro so that we can quickly
238 /// return preexpanded tokens from Tokens.
239 void TokenLexer::ExpandFunctionArguments() {
240  SmallVector<Token, 128> ResultToks;
241 
242  // Loop through 'Tokens', expanding them into ResultToks. Keep
243  // track of whether we change anything. If not, no need to keep them. If so,
244  // we install the newly expanded sequence as the new 'Tokens' list.
245  bool MadeChange = false;
246 
247  const bool CalledWithVariadicArguments =
248  ActualArgs->invokedWithVariadicArgument(Macro);
249 
250  VAOptExpansionContext VCtx(PP);
251 
252  for (unsigned I = 0, E = NumTokens; I != E; ++I) {
253  const Token &CurTok = Tokens[I];
254  // We don't want a space for the next token after a paste
255  // operator. In valid code, the token will get smooshed onto the
256  // preceding one anyway. In assembler-with-cpp mode, invalid
257  // pastes are allowed through: in this case, we do not want the
258  // extra whitespace to be added. For example, we want ". ## foo"
259  // -> ".foo" not ". foo".
260  if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace())
261  NextTokGetsSpace = true;
262 
263  if (VCtx.isVAOptToken(CurTok)) {
264  MadeChange = true;
265  assert(Tokens[I + 1].is(tok::l_paren) &&
266  "__VA_OPT__ must be followed by '('");
267 
268  ++I; // Skip the l_paren
270  ResultToks.size());
271 
272  continue;
273  }
274 
275  // We have entered into the __VA_OPT__ context, so handle tokens
276  // appropriately.
277  if (VCtx.isInVAOpt()) {
278  // If we are about to process a token that is either an argument to
279  // __VA_OPT__ or its closing rparen, then:
280  // 1) If the token is the closing rparen that exits us out of __VA_OPT__,
281  // perform any necessary stringification or placemarker processing,
282  // and/or skip to the next token.
283  // 2) else if macro was invoked without variadic arguments skip this
284  // token.
285  // 3) else (macro was invoked with variadic arguments) process the token
286  // normally.
287 
288  if (Tokens[I].is(tok::l_paren))
289  VCtx.sawOpeningParen(Tokens[I].getLocation());
290  // Continue skipping tokens within __VA_OPT__ if the macro was not
291  // called with variadic arguments, else let the rest of the loop handle
292  // this token. Note sawClosingParen() returns true only if the r_paren matches
293  // the closing r_paren of the __VA_OPT__.
294  if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) {
295  if (!CalledWithVariadicArguments) {
296  // Skip this token.
297  continue;
298  }
299  // ... else the macro was called with variadic arguments, and we do not
300  // have a closing rparen - so process this token normally.
301  } else {
302  // Current token is the closing r_paren which marks the end of the
303  // __VA_OPT__ invocation, so handle any place-marker pasting (if
304  // empty) by removing hashhash either before (if exists) or after. And
305  // also stringify the entire contents if VAOPT was preceded by a hash,
306  // but do so only after any token concatenation that needs to occur
307  // within the contents of VAOPT.
308 
309  if (VCtx.hasStringifyOrCharifyBefore()) {
310  // Replace all the tokens just added from within VAOPT into a single
311  // stringified token. This requires token-pasting to eagerly occur
312  // within these tokens. If either the contents of VAOPT were empty
313  // or the macro wasn't called with any variadic arguments, the result
314  // is a token that represents an empty string.
315  stringifyVAOPTContents(ResultToks, VCtx,
316  /*ClosingParenLoc*/ Tokens[I].getLocation());
317 
318  } else if (/*No tokens within VAOPT*/ !(
319  ResultToks.size() - VCtx.getNumberOfTokensPriorToVAOpt())) {
320  // Treat VAOPT as a placemarker token. Eat either the '##' before the
321  // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that
322  // hashhash was not a placemarker) or the '##'
323  // after VAOPT, but not both.
324 
325  if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {
326  ResultToks.pop_back();
327  } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {
328  ++I; // Skip the following hashhash.
329  }
330  }
331  VCtx.reset();
332  // We processed __VA_OPT__'s closing paren (and the exit out of
333  // __VA_OPT__), so skip to the next token.
334  continue;
335  }
336  }
337 
338  // If we found the stringify operator, get the argument stringified. The
339  // preprocessor already verified that the following token is a macro
340  // parameter or __VA_OPT__ when the #define was lexed.
341 
342  if (CurTok.isOneOf(tok::hash, tok::hashat)) {
343  int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());
344  assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&
345  "Token following # is not an argument or __VA_OPT__!");
346 
347  if (ArgNo == -1) {
348  // Handle the __VA_OPT__ case.
349  VCtx.sawHashOrHashAtBefore(NextTokGetsSpace,
350  CurTok.is(tok::hashat));
351  continue;
352  }
353  // Else handle the simple argument case.
354  SourceLocation ExpansionLocStart =
355  getExpansionLocForMacroDefLoc(CurTok.getLocation());
356  SourceLocation ExpansionLocEnd =
357  getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());
358 
359  Token Res;
360  if (CurTok.is(tok::hash)) // Stringify
361  Res = ActualArgs->getStringifiedArgument(ArgNo, PP,
362  ExpansionLocStart,
363  ExpansionLocEnd);
364  else {
365  // 'charify': don't bother caching these.
366  Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo),
367  PP, true,
368  ExpansionLocStart,
369  ExpansionLocEnd);
370  }
372 
373  // The stringified/charified string leading space flag gets set to match
374  // the #/#@ operator.
375  if (NextTokGetsSpace)
377 
378  ResultToks.push_back(Res);
379  MadeChange = true;
380  ++I; // Skip arg name.
381  NextTokGetsSpace = false;
382  continue;
383  }
384 
385  // Find out if there is a paste (##) operator before or after the token.
386  bool NonEmptyPasteBefore =
387  !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
388  bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash);
389  bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash);
390 
391  assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&
392  "unexpected ## in ResultToks");
393 
394  // Otherwise, if this is not an argument token, just add the token to the
395  // output buffer.
396  IdentifierInfo *II = CurTok.getIdentifierInfo();
397  int ArgNo = II ? Macro->getParameterNum(II) : -1;
398  if (ArgNo == -1) {
399  // This isn't an argument, just add it.
400  ResultToks.push_back(CurTok);
401 
402  if (NextTokGetsSpace) {
403  ResultToks.back().setFlag(Token::LeadingSpace);
404  NextTokGetsSpace = false;
405  } else if (PasteBefore && !NonEmptyPasteBefore)
406  ResultToks.back().clearFlag(Token::LeadingSpace);
407 
408  continue;
409  }
410 
411  // An argument is expanded somehow, the result is different than the
412  // input.
413  MadeChange = true;
414 
415  // Otherwise, this is a use of the argument.
416 
417  // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
418  // are no trailing commas if __VA_ARGS__ is empty.
419  if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
420  MaybeRemoveCommaBeforeVaArgs(ResultToks,
421  /*HasPasteOperator=*/false,
422  Macro, ArgNo, PP))
423  continue;
424 
425  // If it is not the LHS/RHS of a ## operator, we must pre-expand the
426  // argument and substitute the expanded tokens into the result. This is
427  // C99 6.10.3.1p1.
428  if (!PasteBefore && !PasteAfter) {
429  const Token *ResultArgToks;
430 
431  // Only preexpand the argument if it could possibly need it. This
432  // avoids some work in common cases.
433  const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
434  if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
435  ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];
436  else
437  ResultArgToks = ArgTok; // Use non-preexpanded tokens.
438 
439  // If the arg token expanded into anything, append it.
440  if (ResultArgToks->isNot(tok::eof)) {
441  size_t FirstResult = ResultToks.size();
442  unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
443  ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
444 
445  // In Microsoft-compatibility mode, we follow MSVC's preprocessing
446  // behavior by not considering single commas from nested macro
447  // expansions as argument separators. Set a flag on the token so we can
448  // test for this later when the macro expansion is processed.
449  if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&
450  ResultToks.back().is(tok::comma))
451  ResultToks.back().setFlag(Token::IgnoredComma);
452 
453  // If the '##' came from expanding an argument, turn it into 'unknown'
454  // to avoid pasting.
455  for (Token &Tok : llvm::make_range(ResultToks.begin() + FirstResult,
456  ResultToks.end())) {
457  if (Tok.is(tok::hashhash))
458  Tok.setKind(tok::unknown);
459  }
460 
461  if(ExpandLocStart.isValid()) {
462  updateLocForMacroArgTokens(CurTok.getLocation(),
463  ResultToks.begin()+FirstResult,
464  ResultToks.end());
465  }
466 
467  // If any tokens were substituted from the argument, the whitespace
468  // before the first token should match the whitespace of the arg
469  // identifier.
470  ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
471  NextTokGetsSpace);
472  ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
473  NextTokGetsSpace = false;
474  }
475  continue;
476  }
477 
478  // Okay, we have a token that is either the LHS or RHS of a paste (##)
479  // argument. It gets substituted as its non-pre-expanded tokens.
480  const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
481  unsigned NumToks = MacroArgs::getArgLength(ArgToks);
482  if (NumToks) { // Not an empty argument?
483  bool VaArgsPseudoPaste = false;
484  // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned
485  // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when
486  // the expander trys to paste ',' with the first token of the __VA_ARGS__
487  // expansion.
488  if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
489  ResultToks[ResultToks.size()-2].is(tok::comma) &&
490  (unsigned)ArgNo == Macro->getNumParams()-1 &&
491  Macro->isVariadic()) {
492  VaArgsPseudoPaste = true;
493  // Remove the paste operator, report use of the extension.
494  PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
495  }
496 
497  ResultToks.append(ArgToks, ArgToks+NumToks);
498 
499  // If the '##' came from expanding an argument, turn it into 'unknown'
500  // to avoid pasting.
501  for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
502  ResultToks.end())) {
503  if (Tok.is(tok::hashhash))
504  Tok.setKind(tok::unknown);
505  }
506 
507  if (ExpandLocStart.isValid()) {
508  updateLocForMacroArgTokens(CurTok.getLocation(),
509  ResultToks.end()-NumToks, ResultToks.end());
510  }
511 
512  // Transfer the leading whitespace information from the token
513  // (the macro argument) onto the first token of the
514  // expansion. Note that we don't do this for the GNU
515  // pseudo-paste extension ", ## __VA_ARGS__".
516  if (!VaArgsPseudoPaste) {
517  ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,
518  false);
519  ResultToks[ResultToks.size() - NumToks].setFlagValue(
520  Token::LeadingSpace, NextTokGetsSpace);
521  }
522 
523  NextTokGetsSpace = false;
524  continue;
525  }
526 
527  // If an empty argument is on the LHS or RHS of a paste, the standard (C99
528  // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We
529  // implement this by eating ## operators when a LHS or RHS expands to
530  // empty.
531  if (PasteAfter) {
532  // Discard the argument token and skip (don't copy to the expansion
533  // buffer) the paste operator after it.
534  ++I;
535  continue;
536  }
537 
538  // If this is on the RHS of a paste operator, we've already copied the
539  // paste operator to the ResultToks list, unless the LHS was empty too.
540  // Remove it.
541  assert(PasteBefore);
542  if (NonEmptyPasteBefore) {
543  assert(ResultToks.back().is(tok::hashhash));
544  // Do not remove the paste operator if it is the one before __VA_OPT__
545  // (and we are still processing tokens within VA_OPT). We handle the case
546  // of removing the paste operator if __VA_OPT__ reduces to the notional
547  // placemarker above when we encounter the closing paren of VA_OPT.
548  if (!VCtx.isInVAOpt() ||
549  ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt())
550  ResultToks.pop_back();
551  }
552 
553  // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
554  // and if the macro had at least one real argument, and if the token before
555  // the ## was a comma, remove the comma. This is a GCC extension which is
556  // disabled when using -std=c99.
557  if (ActualArgs->isVarargsElidedUse())
558  MaybeRemoveCommaBeforeVaArgs(ResultToks,
559  /*HasPasteOperator=*/true,
560  Macro, ArgNo, PP);
561  }
562 
563  // If anything changed, install this as the new Tokens list.
564  if (MadeChange) {
565  assert(!OwnsTokens && "This would leak if we already own the token list");
566  // This is deleted in the dtor.
567  NumTokens = ResultToks.size();
568  // The tokens will be added to Preprocessor's cache and will be removed
569  // when this TokenLexer finishes lexing them.
570  Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
571 
572  // The preprocessor cache of macro expanded tokens owns these tokens,not us.
573  OwnsTokens = false;
574  }
575 }
576 
577 /// \brief Checks if two tokens form wide string literal.
578 static bool isWideStringLiteralFromMacro(const Token &FirstTok,
579  const Token &SecondTok) {
580  return FirstTok.is(tok::identifier) &&
581  FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral() &&
582  SecondTok.stringifiedInMacro();
583 }
584 
585 /// Lex - Lex and return a token from this macro stream.
587  // Lexing off the end of the macro, pop this macro off the expansion stack.
588  if (isAtEnd()) {
589  // If this is a macro (not a token stream), mark the macro enabled now
590  // that it is no longer being expanded.
591  if (Macro) Macro->EnableMacro();
592 
593  Tok.startToken();
594  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
595  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);
596  if (CurTokenIdx == 0)
598  return PP.HandleEndOfTokenLexer(Tok);
599  }
600 
602 
603  // If this is the first token of the expanded result, we inherit spacing
604  // properties later.
605  bool isFirstToken = CurTokenIdx == 0;
606 
607  // Get the next token to return.
608  Tok = Tokens[CurTokenIdx++];
609 
610  bool TokenIsFromPaste = false;
611 
612  // If this token is followed by a token paste (##) operator, paste the tokens!
613  // Note that ## is a normal token when not expanding a macro.
614  if (!isAtEnd() && Macro &&
615  (Tokens[CurTokenIdx].is(tok::hashhash) ||
616  // Special processing of L#x macros in -fms-compatibility mode.
617  // Microsoft compiler is able to form a wide string literal from
618  // 'L#macro_arg' construct in a function-like macro.
619  (PP.getLangOpts().MSVCCompat &&
620  isWideStringLiteralFromMacro(Tok, Tokens[CurTokenIdx])))) {
621  // When handling the microsoft /##/ extension, the final token is
622  // returned by pasteTokens, not the pasted token.
623  if (pasteTokens(Tok))
624  return true;
625 
626  TokenIsFromPaste = true;
627  }
628 
629  // The token's current location indicate where the token was lexed from. We
630  // need this information to compute the spelling of the token, but any
631  // diagnostics for the expanded token should appear as if they came from
632  // ExpansionLoc. Pull this information together into a new SourceLocation
633  // that captures all of this.
634  if (ExpandLocStart.isValid() && // Don't do this for token streams.
635  // Check that the token's location was not already set properly.
636  SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {
637  SourceLocation instLoc;
638  if (Tok.is(tok::comment)) {
639  instLoc = SM.createExpansionLoc(Tok.getLocation(),
640  ExpandLocStart,
641  ExpandLocEnd,
642  Tok.getLength());
643  } else {
644  instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());
645  }
646 
647  Tok.setLocation(instLoc);
648  }
649 
650  // If this is the first token, set the lexical properties of the token to
651  // match the lexical properties of the macro identifier.
652  if (isFirstToken) {
653  Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
654  Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
655  } else {
656  // If this is not the first token, we may still need to pass through
657  // leading whitespace if we've expanded a macro.
658  if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);
659  if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);
660  }
661  AtStartOfLine = false;
662  HasLeadingSpace = false;
663 
664  // Handle recursive expansion!
665  if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) {
666  // Change the kind of this identifier to the appropriate token kind, e.g.
667  // turning "for" into a keyword.
668  IdentifierInfo *II = Tok.getIdentifierInfo();
669  Tok.setKind(II->getTokenID());
670 
671  // If this identifier was poisoned and from a paste, emit an error. This
672  // won't be handled by Preprocessor::HandleIdentifier because this is coming
673  // from a macro expansion.
674  if (II->isPoisoned() && TokenIsFromPaste) {
675  PP.HandlePoisonedIdentifier(Tok);
676  }
677 
678  if (!DisableMacroExpansion && II->isHandleIdentifierCase())
679  return PP.HandleIdentifier(Tok);
680  }
681 
682  // Otherwise, return a normal token.
683  return true;
684 }
685 
686 bool TokenLexer::pasteTokens(Token &Tok) {
687  return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx);
688 }
689 
690 /// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ##
691 /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there
692 /// are more ## after it, chomp them iteratively. Return the result as LHSTok.
693 /// If this returns true, the caller should immediately return the token.
694 bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
695  unsigned int &CurIdx) {
696  assert(CurIdx > 0 && "## can not be the first token within tokens");
697  assert((TokenStream[CurIdx].is(tok::hashhash) ||
698  (PP.getLangOpts().MSVCCompat &&
699  isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) &&
700  "Token at this Index must be ## or part of the MSVC 'L "
701  "#macro-arg' pasting pair");
702 
703  // MSVC: If previous token was pasted, this must be a recovery from an invalid
704  // paste operation. Ignore spaces before this token to mimic MSVC output.
705  // Required for generating valid UUID strings in some MS headers.
706  if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) &&
707  TokenStream[CurIdx - 2].is(tok::hashhash))
709 
710  SmallString<128> Buffer;
711  const char *ResultTokStrPtr = nullptr;
712  SourceLocation StartLoc = LHSTok.getLocation();
713  SourceLocation PasteOpLoc;
714 
715  auto IsAtEnd = [&TokenStream, &CurIdx] {
716  return TokenStream.size() == CurIdx;
717  };
718 
719  do {
720  // Consume the ## operator if any.
721  PasteOpLoc = TokenStream[CurIdx].getLocation();
722  if (TokenStream[CurIdx].is(tok::hashhash))
723  ++CurIdx;
724  assert(!IsAtEnd() && "No token on the RHS of a paste operator!");
725 
726  // Get the RHS token.
727  const Token &RHS = TokenStream[CurIdx];
728 
729  // Allocate space for the result token. This is guaranteed to be enough for
730  // the two tokens.
731  Buffer.resize(LHSTok.getLength() + RHS.getLength());
732 
733  // Get the spelling of the LHS token in Buffer.
734  const char *BufPtr = &Buffer[0];
735  bool Invalid = false;
736  unsigned LHSLen = PP.getSpelling(LHSTok, BufPtr, &Invalid);
737  if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer!
738  memcpy(&Buffer[0], BufPtr, LHSLen);
739  if (Invalid)
740  return true;
741 
742  BufPtr = Buffer.data() + LHSLen;
743  unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid);
744  if (Invalid)
745  return true;
746  if (RHSLen && BufPtr != &Buffer[LHSLen])
747  // Really, we want the chars in Buffer!
748  memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
749 
750  // Trim excess space.
751  Buffer.resize(LHSLen+RHSLen);
752 
753  // Plop the pasted result (including the trailing newline and null) into a
754  // scratch buffer where we can lex it.
755  Token ResultTokTmp;
756  ResultTokTmp.startToken();
757 
758  // Claim that the tmp token is a string_literal so that we can get the
759  // character pointer back from CreateString in getLiteralData().
760  ResultTokTmp.setKind(tok::string_literal);
761  PP.CreateString(Buffer, ResultTokTmp);
762  SourceLocation ResultTokLoc = ResultTokTmp.getLocation();
763  ResultTokStrPtr = ResultTokTmp.getLiteralData();
764 
765  // Lex the resultant pasted token into Result.
766  Token Result;
767 
768  if (LHSTok.isAnyIdentifier() && RHS.isAnyIdentifier()) {
769  // Common paste case: identifier+identifier = identifier. Avoid creating
770  // a lexer and other overhead.
771  PP.IncrementPasteCounter(true);
772  Result.startToken();
773  Result.setKind(tok::raw_identifier);
774  Result.setRawIdentifierData(ResultTokStrPtr);
775  Result.setLocation(ResultTokLoc);
776  Result.setLength(LHSLen+RHSLen);
777  } else {
778  PP.IncrementPasteCounter(false);
779 
780  assert(ResultTokLoc.isFileID() &&
781  "Should be a raw location into scratch buffer");
782  SourceManager &SourceMgr = PP.getSourceManager();
783  FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
784 
785  bool Invalid = false;
786  const char *ScratchBufStart
787  = SourceMgr.getBufferData(LocFileID, &Invalid).data();
788  if (Invalid)
789  return false;
790 
791  // Make a lexer to lex this string from. Lex just this one token.
792  // Make a lexer object so that we lex and expand the paste result.
793  Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
794  PP.getLangOpts(), ScratchBufStart,
795  ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
796 
797  // Lex a token in raw mode. This way it won't look up identifiers
798  // automatically, lexing off the end will return an eof token, and
799  // warnings are disabled. This returns true if the result token is the
800  // entire buffer.
801  bool isInvalid = !TL.LexFromRawLexer(Result);
802 
803  // If we got an EOF token, we didn't form even ONE token. For example, we
804  // did "/ ## /" to get "//".
805  isInvalid |= Result.is(tok::eof);
806 
807  // If pasting the two tokens didn't form a full new token, this is an
808  // error. This occurs with "x ## +" and other stuff. Return with LHSTok
809  // unmodified and with RHS as the next token to lex.
810  if (isInvalid) {
811  // Explicitly convert the token location to have proper expansion
812  // information so that the user knows where it came from.
814  SourceLocation Loc =
815  SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);
816 
817  // Test for the Microsoft extension of /##/ turning into // here on the
818  // error path.
819  if (PP.getLangOpts().MicrosoftExt && LHSTok.is(tok::slash) &&
820  RHS.is(tok::slash)) {
821  HandleMicrosoftCommentPaste(LHSTok, Loc);
822  return true;
823  }
824 
825  // Do not emit the error when preprocessing assembler code.
826  if (!PP.getLangOpts().AsmPreprocessor) {
827  // If we're in microsoft extensions mode, downgrade this from a hard
828  // error to an extension that defaults to an error. This allows
829  // disabling it.
830  PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms
831  : diag::err_pp_bad_paste)
832  << Buffer;
833  }
834 
835  // An error has occurred so exit loop.
836  break;
837  }
838 
839  // Turn ## into 'unknown' to avoid # ## # from looking like a paste
840  // operator.
841  if (Result.is(tok::hashhash))
842  Result.setKind(tok::unknown);
843  }
844 
845  // Transfer properties of the LHS over the Result.
848 
849  // Finally, replace LHS with the result, consume the RHS, and iterate.
850  ++CurIdx;
851  LHSTok = Result;
852  } while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));
853 
854  SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation();
855 
856  // The token's current location indicate where the token was lexed from. We
857  // need this information to compute the spelling of the token, but any
858  // diagnostics for the expanded token should appear as if the token was
859  // expanded from the full ## expression. Pull this information together into
860  // a new SourceLocation that captures all of this.
862  if (StartLoc.isFileID())
863  StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
864  if (EndLoc.isFileID())
865  EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
866  FileID MacroFID = SM.getFileID(MacroExpansionStart);
867  while (SM.getFileID(StartLoc) != MacroFID)
868  StartLoc = SM.getImmediateExpansionRange(StartLoc).first;
869  while (SM.getFileID(EndLoc) != MacroFID)
870  EndLoc = SM.getImmediateExpansionRange(EndLoc).second;
871 
872  LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc,
873  LHSTok.getLength()));
874 
875  // Now that we got the result token, it will be subject to expansion. Since
876  // token pasting re-lexes the result token in raw mode, identifier information
877  // isn't looked up. As such, if the result is an identifier, look up id info.
878  if (LHSTok.is(tok::raw_identifier)) {
879  // Look up the identifier info for the token. We disabled identifier lookup
880  // by saying we're skipping contents, so we need to do this manually.
881  PP.LookUpIdentifierInfo(LHSTok);
882  }
883  return false;
884 }
885 
886 /// isNextTokenLParen - If the next token lexed will pop this macro off the
887 /// expansion stack, return 2. If the next unexpanded token is a '(', return
888 /// 1, otherwise return 0.
890  // Out of tokens?
891  if (isAtEnd())
892  return 2;
893  return Tokens[CurTokenIdx].is(tok::l_paren);
894 }
895 
896 /// isParsingPreprocessorDirective - Return true if we are in the middle of a
897 /// preprocessor directive.
899  return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd();
900 }
901 
902 /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
903 /// together to form a comment that comments out everything in the current
904 /// macro, other active macros, and anything left on the current physical
905 /// source line of the expanded buffer. Handle this by returning the
906 /// first token on the next line.
907 void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {
908  PP.Diag(OpLoc, diag::ext_comment_paste_microsoft);
909 
910  // We 'comment out' the rest of this macro by just ignoring the rest of the
911  // tokens that have not been lexed yet, if any.
912 
913  // Since this must be a macro, mark the macro enabled now that it is no longer
914  // being expanded.
915  assert(Macro && "Token streams can't paste comments");
916  Macro->EnableMacro();
917 
919 }
920 
921 /// \brief If \arg loc is a file ID and points inside the current macro
922 /// definition, returns the appropriate source location pointing at the
923 /// macro expansion source location entry, otherwise it returns an invalid
924 /// SourceLocation.
926 TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {
927  assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&
928  "Not appropriate for token streams");
929  assert(loc.isValid() && loc.isFileID());
930 
932  assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
933  "Expected loc to come from the macro definition");
934 
935  unsigned relativeOffset = 0;
936  SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);
937  return MacroExpansionStart.getLocWithOffset(relativeOffset);
938 }
939 
940 /// \brief Finds the tokens that are consecutive (from the same FileID)
941 /// creates a single SLocEntry, and assigns SourceLocations to each token that
942 /// point to that SLocEntry. e.g for
943 /// assert(foo == bar);
944 /// There will be a single SLocEntry for the "foo == bar" chunk and locations
945 /// for the 'foo', '==', 'bar' tokens will point inside that chunk.
946 ///
947 /// \arg begin_tokens will be updated to a position past all the found
948 /// consecutive tokens.
950  SourceLocation InstLoc,
951  Token *&begin_tokens,
952  Token * end_tokens) {
953  assert(begin_tokens < end_tokens);
954 
955  SourceLocation FirstLoc = begin_tokens->getLocation();
956  SourceLocation CurLoc = FirstLoc;
957 
958  // Compare the source location offset of tokens and group together tokens that
959  // are close, even if their locations point to different FileIDs. e.g.
960  //
961  // |bar | foo | cake | (3 tokens from 3 consecutive FileIDs)
962  // ^ ^
963  // |bar foo cake| (one SLocEntry chunk for all tokens)
964  //
965  // we can perform this "merge" since the token's spelling location depends
966  // on the relative offset.
967 
968  Token *NextTok = begin_tokens + 1;
969  for (; NextTok < end_tokens; ++NextTok) {
970  SourceLocation NextLoc = NextTok->getLocation();
971  if (CurLoc.isFileID() != NextLoc.isFileID())
972  break; // Token from different kind of FileID.
973 
974  int RelOffs;
975  if (!SM.isInSameSLocAddrSpace(CurLoc, NextLoc, &RelOffs))
976  break; // Token from different local/loaded location.
977  // Check that token is not before the previous token or more than 50
978  // "characters" away.
979  if (RelOffs < 0 || RelOffs > 50)
980  break;
981 
982  if (CurLoc.isMacroID() && !SM.isWrittenInSameFile(CurLoc, NextLoc))
983  break; // Token from a different macro.
984 
985  CurLoc = NextLoc;
986  }
987 
988  // For the consecutive tokens, find the length of the SLocEntry to contain
989  // all of them.
990  Token &LastConsecutiveTok = *(NextTok-1);
991  int LastRelOffs = 0;
992  SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(),
993  &LastRelOffs);
994  unsigned FullLength = LastRelOffs + LastConsecutiveTok.getLength();
995 
996  // Create a macro expansion SLocEntry that will "contain" all of the tokens.
997  SourceLocation Expansion =
998  SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength);
999 
1000  // Change the location of the tokens from the spelling location to the new
1001  // expanded location.
1002  for (; begin_tokens < NextTok; ++begin_tokens) {
1003  Token &Tok = *begin_tokens;
1004  int RelOffs = 0;
1005  SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs);
1006  Tok.setLocation(Expansion.getLocWithOffset(RelOffs));
1007  }
1008 }
1009 
1010 /// \brief Creates SLocEntries and updates the locations of macro argument
1011 /// tokens to their new expanded locations.
1012 ///
1013 /// \param ArgIdSpellLoc the location of the macro argument id inside the macro
1014 /// definition.
1015 void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
1016  Token *begin_tokens,
1017  Token *end_tokens) {
1019 
1020  SourceLocation InstLoc =
1021  getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
1022 
1023  while (begin_tokens < end_tokens) {
1024  // If there's only one token just create a SLocEntry for it.
1025  if (end_tokens - begin_tokens == 1) {
1026  Token &Tok = *begin_tokens;
1028  InstLoc,
1029  Tok.getLength()));
1030  return;
1031  }
1032 
1033  updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens);
1034  }
1035 }
1036 
1037 void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
1038  AtStartOfLine = Result.isAtStartOfLine();
1039  HasLeadingSpace = Result.hasLeadingSpace();
1040 }
bool invokedWithVariadicArgument(const MacroInfo *const MI) const
Returns true if the macro was defined with a variadic (ellipsis) parameter AND was invoked with at le...
Definition: MacroArgs.cpp:142
bool isWrittenInSameFile(SourceLocation Loc1, SourceLocation Loc2) const
Returns true if the spelling locations for both SourceLocations are part of the same file buffer...
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:77
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void DisableMacro()
Definition: MacroInfo.h:266
void Init(Token &Tok, SourceLocation ILEnd, MacroInfo *MI, MacroArgs *ActualArgs)
Init - Initialize this TokenLexer to expand from the specified macro with the specified argument info...
Definition: TokenLexer.cpp:39
void sawOpeningParen(SourceLocation LParenLoc)
Call this function each time an lparen is seen.
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Definition: Token.h:257
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
ArgNeedsPreexpansion - If we can prove that the argument won&#39;t be affected by pre-expansion, return false.
Definition: MacroArgs.cpp:151
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
bool sawClosingParen()
Call this function each time an rparen is seen.
Defines the SourceManager interface.
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation InstLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry, and assigns SourceLocations to each token that point to that SLocEntry.
Definition: TokenLexer.cpp:949
A class for tracking whether we&#39;re inside a VA_OPT during a traversal of the tokens of a macro during...
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool isBeforeInSLocAddrSpace(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the "source location address space".
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:234
bool isVarargsElidedUse() const
isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...
Definition: MacroArgs.h:113
bool isVariadic() const
Definition: MacroInfo.h:208
bool isInSameSLocAddrSpace(SourceLocation LHS, SourceLocation RHS, int *RelativeOffset) const
Return true if both LHS and RHS are in the local source location address space or the loaded one...
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
Definition: Token.h:107
unsigned getNextLocalOffset() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:118
tokens_iterator tokens_begin() const
Definition: MacroInfo.h:243
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:113
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
One of these records is kept for each identifier that is lexed.
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:207
void sawVAOptFollowedByOpeningParens(const SourceLocation VAOptLoc, const unsigned int NumPriorTokens)
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
void setKind(tok::TokenKind K)
Definition: Token.h:91
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:811
bool Lex(Token &Tok)
Lex - Lex and return a token from this macro stream.
Definition: TokenLexer.cpp:586
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
const FormatToken & Tok
SourceLocation getExpansionLoc(SourceLocation Loc) const
Given a SourceLocation object Loc, return the expansion location referenced by the ID...
Defines the Diagnostic-related interfaces.
void destroy(Preprocessor &PP)
destroy - Destroy and deallocate the memory for this object.
Definition: MacroArgs.cpp:78
const Token & getStringifiedArgument(unsigned ArgNo, Preprocessor &PP, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
getStringifiedArgument - Compute, cache, and return the specified argument that has been &#39;stringified...
Definition: MacroArgs.cpp:316
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
Definition: TokenLexer.cpp:578
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...
Definition: MacroArgs.cpp:213
unsigned getNumParams() const
Definition: MacroInfo.h:183
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Defines the clang::LangOptions interface.
bool isInSLocAddrSpace(SourceLocation Loc, SourceLocation Start, unsigned Length, unsigned *RelativeOffset=nullptr) const
Returns true if Loc is inside the [Start, +Length) chunk of the source location address space...
SourceLocation getVAOptLoc() const
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
SourceLocation createMacroArgExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLoc, unsigned TokLength)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
The result type of a method or function.
const SourceManager & SM
Definition: Format.cpp:1337
SourceManager & getSourceManager() const
Definition: Preprocessor.h:815
bool isVAOptToken(const Token &T) const
Encodes a location in the source.
std::pair< SourceLocation, SourceLocation > getImmediateExpansionRange(SourceLocation Loc) const
Return the start/end of the expansion information for an expansion location.
void setLength(unsigned Len)
Definition: Token.h:133
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:297
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:177
bool isInVAOpt() const
Returns true if we have seen the VA_OPT and &#39;(&#39; but before having seen the matching &#39;)&#39;...
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:266
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:96
static bool isInvalid(LocType Loc, bool *Invalid)
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
bool isFunctionLike() const
Definition: MacroInfo.h:200
const std::vector< Token > & getPreExpArgument(unsigned Arg, Preprocessor &PP)
getPreExpArgument - Return the pre-expanded form of the specified argument.
Definition: MacroArgs.cpp:166
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:97
tokens_iterator tokens_end() const
Definition: MacroInfo.h:244
static unsigned getArgLength(const Token *ArgPtr)
getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens...
Definition: MacroArgs.cpp:110
void sawHashOrHashAtBefore(const bool HasLeadingSpace, const bool IsHashAt)
unsigned getLength() const
Definition: Token.h:127
bool isMacroID() const
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:40
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:215
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
int getParameterNum(const IdentifierInfo *Arg) const
Return the parameter number of the specified identifier, or -1 if the identifier is not a formal para...
Definition: MacroInfo.h:190
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
unsigned int getNumberOfTokensPriorToVAOpt() const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token&#39;s location and length to ...
unsigned isNextTokenLParen() const
isNextTokenLParen - If the next token lexed will pop this macro off the expansion stack...
Definition: TokenLexer.cpp:889
Defines the clang::TokenKind enum and support functions.
void HandlePoisonedIdentifier(Token &Tok)
Display reason for poisoned identifier.
unsigned getDefinitionLength(const SourceManager &SM) const
Get length in characters of the macro definition.
Definition: MacroInfo.h:133
Defines the clang::SourceLocation class and associated facilities.
bool stringifiedInMacro() const
Returns true if this token is formed by macro by stringizing or charizing operator.
Definition: Token.h:298
void setLocation(SourceLocation L)
Definition: Token.h:132
void EnableMacro()
Definition: MacroInfo.h:261
void clearFlag(TokenFlags Flag)
Unset the specified flag.
Definition: Token.h:244
bool isParsingPreprocessorDirective() const
isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive...
Definition: TokenLexer.cpp:898
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:270
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
This class handles loading and caching of source files into memory.
const Token * getUnexpArgument(unsigned Arg) const
getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...
Definition: MacroArgs.cpp:120
void startToken()
Reset all flags to cleared.
Definition: Token.h:169
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:127