27 #include "llvm/ADT/ArrayRef.h"
28 #include "llvm/ADT/SmallString.h"
29 #include "llvm/ADT/SmallVector.h"
30 #include "llvm/ADT/iterator_range.h"
34 using namespace clang;
52 NextTokGetsSpace =
false;
55 DisableMacroExpansion =
false;
61 MacroStartSLocOffset =
SM.getNextLocalOffset();
64 assert(Tokens[0].getLocation().isValid());
65 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
66 "Macro defined in macro?");
67 assert(ExpandLocStart.
isValid());
73 MacroDefStart =
SM.getExpansionLoc(Tokens[0].getLocation());
75 MacroExpansionStart =
SM.createExpansionLoc(MacroDefStart,
84 ExpandFunctionArguments();
95 bool disableMacroExpansion,
bool ownsTokens,
97 assert(!isReinject || disableMacroExpansion);
103 ActualArgs =
nullptr;
105 OwnsTokens = ownsTokens;
106 DisableMacroExpansion = disableMacroExpansion;
107 IsReinject = isReinject;
111 AtStartOfLine =
false;
112 HasLeadingSpace =
false;
113 NextTokGetsSpace =
false;
124 void TokenLexer::destroy() {
134 if (ActualArgs) ActualArgs->
destroy(PP);
137 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
141 if (!
Macro->isVariadic() || MacroArgNo !=
Macro->getNumParams()-1)
147 if (!HasPasteOperator && !PP.
getLangOpts().MSVCCompat)
156 &&
Macro->getNumParams() < 2)
160 if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
164 if (HasPasteOperator)
165 PP.
Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
168 ResultToks.pop_back();
170 if (!ResultToks.empty()) {
175 if (ResultToks.back().is(tok::hashhash))
176 ResultToks.pop_back();
183 NextTokGetsSpace =
false;
187 void TokenLexer::stringifyVAOPTContents(
191 const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
192 Token *
const VAOPTTokens =
193 NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] :
nullptr;
200 for (
unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
202 if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {
203 assert(CurTokenIdx != 0 &&
204 "Can not have __VAOPT__ contents begin with a ##");
205 Token &LHS = VAOPTTokens[CurTokenIdx - 1];
206 pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens),
209 ConcatenatedVAOPTResultToks.back() = LHS;
210 if (CurTokenIdx == NumVAOptTokens)
213 ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);
216 ConcatenatedVAOPTResultToks.push_back(VCtx.
getEOFTok());
225 getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);
229 ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);
236 ResultToks.resize(NumToksPriorToVAOpt + 1);
237 ResultToks.back() = StringifiedVAOPT;
242 void TokenLexer::ExpandFunctionArguments() {
248 bool MadeChange =
false;
254 for (
unsigned I = 0, E = NumTokens; I != E; ++I) {
255 const Token &CurTok = Tokens[I];
262 if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.
hasLeadingSpace())
263 NextTokGetsSpace =
true;
267 assert(Tokens[I + 1].is(tok::l_paren) &&
268 "__VA_OPT__ must be followed by '('");
290 if (Tokens[I].is(tok::l_paren))
298 if (!CalledWithVariadicArguments.hasValue()) {
299 CalledWithVariadicArguments =
302 if (!*CalledWithVariadicArguments) {
322 stringifyVAOPTContents(ResultToks, VCtx,
323 Tokens[I].getLocation());
332 if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {
333 ResultToks.pop_back();
334 }
else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {
347 "no token paste before __VA_OPT__");
348 ResultToks.erase(ResultToks.begin() +
354 Tokens[I + 1].is(tok::hashhash)) {
369 if (CurTok.
isOneOf(tok::hash, tok::hashat)) {
370 int ArgNo =
Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());
371 assert((ArgNo != -1 || VCtx.
isVAOptToken(Tokens[I + 1])) &&
372 "Token following # is not an argument or __VA_OPT__!");
377 CurTok.
is(tok::hashat));
382 getExpansionLocForMacroDefLoc(CurTok.
getLocation());
384 getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());
386 bool Charify = CurTok.
is(tok::hashat);
389 UnexpArg, PP, Charify, ExpansionLocStart, ExpansionLocEnd);
394 if (NextTokGetsSpace)
397 ResultToks.push_back(Res);
400 NextTokGetsSpace =
false;
405 bool NonEmptyPasteBefore =
406 !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
407 bool PasteBefore = I != 0 && Tokens[I-1].
is(tok::hashhash);
408 bool PasteAfter = I+1 != E && Tokens[I+1].
is(tok::hashhash);
409 bool RParenAfter = I+1 != E && Tokens[I+1].
is(tok::r_paren);
411 assert((!NonEmptyPasteBefore || PasteBefore || VCtx.
isInVAOpt()) &&
412 "unexpected ## in ResultToks");
417 int ArgNo = II ?
Macro->getParameterNum(II) : -1;
420 ResultToks.push_back(CurTok);
422 if (NextTokGetsSpace) {
424 NextTokGetsSpace =
false;
425 }
else if (PasteBefore && !NonEmptyPasteBefore)
440 MaybeRemoveCommaBeforeVaArgs(ResultToks,
448 if (!PasteBefore && !PasteAfter) {
449 const Token *ResultArgToks;
457 ResultArgToks = ArgTok;
461 size_t FirstResult = ResultToks.size();
463 ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
470 ResultToks.back().is(tok::comma))
475 for (
Token &Tok : llvm::drop_begin(ResultToks, FirstResult))
476 if (Tok.is(tok::hashhash))
477 Tok.setKind(tok::unknown);
481 ResultToks.begin()+FirstResult,
491 NextTokGetsSpace =
false;
496 if (NonEmptyPasteBefore) {
499 assert(VCtx.
isInVAOpt() &&
"should only happen inside a __VA_OPT__");
513 bool VaArgsPseudoPaste =
false;
518 if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
519 ResultToks[ResultToks.size()-2].is(tok::comma) &&
520 (
unsigned)ArgNo ==
Macro->getNumParams()-1 &&
521 Macro->isVariadic()) {
522 VaArgsPseudoPaste =
true;
524 PP.
Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
527 ResultToks.append(ArgToks, ArgToks+NumToks);
531 for (
Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
533 if (Tok.is(tok::hashhash))
534 Tok.setKind(tok::unknown);
537 if (ExpandLocStart.
isValid()) {
539 ResultToks.end()-NumToks, ResultToks.end());
546 if (!VaArgsPseudoPaste) {
549 ResultToks[ResultToks.size() - NumToks].setFlagValue(
553 NextTokGetsSpace =
false;
575 if (NonEmptyPasteBefore) {
576 assert(ResultToks.back().is(tok::hashhash));
583 ResultToks.pop_back();
593 MaybeRemoveCommaBeforeVaArgs(ResultToks,
600 assert(!OwnsTokens &&
"This would leak if we already own the token list");
602 NumTokens = ResultToks.size();
605 Tokens = PP.cacheMacroExpandedTokens(
this, ResultToks);
614 const Token &SecondTok) {
615 return FirstTok.
is(tok::identifier) &&
626 if (Macro) Macro->EnableMacro();
631 if (CurTokenIdx == 0)
640 bool isFirstToken = CurTokenIdx == 0;
643 Tok = Tokens[CurTokenIdx++];
647 bool TokenIsFromPaste =
false;
651 if (!isAtEnd() && Macro &&
652 (Tokens[CurTokenIdx].is(tok::hashhash) ||
660 if (pasteTokens(Tok))
663 TokenIsFromPaste =
true;
671 if (ExpandLocStart.
isValid() &&
673 SM.isBeforeInSLocAddrSpace(Tok.
getLocation(), MacroStartSLocOffset)) {
675 if (Tok.
is(tok::comment)) {
681 instLoc = getExpansionLocForMacroDefLoc(Tok.
getLocation());
698 AtStartOfLine =
false;
699 HasLeadingSpace =
false;
723 bool TokenLexer::pasteTokens(
Token &Tok) {
724 return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx);
732 unsigned int &CurIdx) {
733 assert(CurIdx > 0 &&
"## can not be the first token within tokens");
734 assert((TokenStream[CurIdx].is(tok::hashhash) ||
737 "Token at this Index must be ## or part of the MSVC 'L "
738 "#macro-arg' pasting pair");
743 if (PP.
getLangOpts().MicrosoftExt && (CurIdx >= 2) &&
744 TokenStream[CurIdx - 2].is(tok::hashhash))
748 const char *ResultTokStrPtr =
nullptr;
752 auto IsAtEnd = [&TokenStream, &CurIdx] {
753 return TokenStream.size() == CurIdx;
758 PasteOpLoc = TokenStream[CurIdx].getLocation();
759 if (TokenStream[CurIdx].is(tok::hashhash))
761 assert(!IsAtEnd() &&
"No token on the RHS of a paste operator!");
764 const Token &RHS = TokenStream[CurIdx];
771 const char *BufPtr = &Buffer[0];
773 unsigned LHSLen = PP.
getSpelling(LHSTok, BufPtr, &Invalid);
774 if (BufPtr != &Buffer[0])
775 memcpy(&Buffer[0], BufPtr, LHSLen);
779 BufPtr = Buffer.data() + LHSLen;
780 unsigned RHSLen = PP.
getSpelling(RHS, BufPtr, &Invalid);
783 if (RHSLen && BufPtr != &Buffer[LHSLen])
785 memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
788 Buffer.resize(LHSLen+RHSLen);
797 ResultTokTmp.
setKind(tok::string_literal);
810 Result.setKind(tok::raw_identifier);
811 Result.setRawIdentifierData(ResultTokStrPtr);
813 Result.setLength(LHSLen+RHSLen);
818 "Should be a raw location into scratch buffer");
823 const char *ScratchBufStart
832 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
838 bool isInvalid = !TL.LexFromRawLexer(Result);
852 SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);
856 if (PP.
getLangOpts().MicrosoftExt && LHSTok.
is(tok::slash) &&
857 RHS.
is(tok::slash)) {
858 HandleMicrosoftCommentPaste(LHSTok, Loc);
868 : diag::err_pp_bad_paste)
878 if (Result.is(tok::hashhash))
879 Result.setKind(tok::unknown);
889 }
while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));
900 StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
902 EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
903 FileID MacroFID =
SM.getFileID(MacroExpansionStart);
904 while (
SM.getFileID(StartLoc) != MacroFID)
905 StartLoc =
SM.getImmediateExpansionRange(StartLoc).getBegin();
906 while (
SM.getFileID(EndLoc) != MacroFID)
907 EndLoc =
SM.getImmediateExpansionRange(EndLoc).getEnd();
915 if (LHSTok.
is(tok::raw_identifier)) {
930 return Tokens[CurTokenIdx].
is(tok::l_paren);
936 return Tokens[NumTokens-1].
is(tok::eod) && !isAtEnd();
945 PP.
Diag(OpLoc, diag::ext_comment_paste_microsoft);
952 assert(Macro &&
"Token streams can't paste comments");
953 Macro->EnableMacro();
963 TokenLexer::getExpansionLocForMacroDefLoc(
SourceLocation loc)
const {
964 assert(ExpandLocStart.
isValid() && MacroExpansionStart.
isValid() &&
965 "Not appropriate for token streams");
969 assert(
SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
970 "Expected loc to come from the macro definition");
973 SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);
988 Token *&begin_tokens,
989 Token * end_tokens) {
990 assert(begin_tokens < end_tokens);
1005 Token *NextTok = begin_tokens + 1;
1006 for (; NextTok < end_tokens; ++NextTok) {
1012 if (!
SM.isInSameSLocAddrSpace(CurLoc, NextLoc, &RelOffs))
1016 if (RelOffs < 0 || RelOffs > 50)
1019 if (CurLoc.
isMacroID() && !
SM.isWrittenInSameFile(CurLoc, NextLoc))
1027 Token &LastConsecutiveTok = *(NextTok-1);
1029 SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.
getLocation(),
1032 LastRelOffs + LastConsecutiveTok.
getLength();
1036 SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength);
1040 for (; begin_tokens < NextTok; ++begin_tokens) {
1041 Token &Tok = *begin_tokens;
1043 SM.isInSameSLocAddrSpace(FirstLoc, Tok.
getLocation(), &RelOffs);
1053 void TokenLexer::updateLocForMacroArgTokens(
SourceLocation ArgIdSpellLoc,
1054 Token *begin_tokens,
1055 Token *end_tokens) {
1059 getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
1061 while (begin_tokens < end_tokens) {
1063 if (end_tokens - begin_tokens == 1) {
1064 Token &Tok = *begin_tokens;
1075 void TokenLexer::PropagateLineStartLeadingSpaceInfo(
Token &Result) {
1076 AtStartOfLine = Result.isAtStartOfLine();
1077 HasLeadingSpace = Result.hasLeadingSpace();