clang API Documentation

ParseCXXInlineMethods.cpp
Go to the documentation of this file.
00001 //===--- ParseCXXInlineMethods.cpp - C++ class inline methods parsing------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 //  This file implements parsing for C++ class inline methods.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "clang/Parse/Parser.h"
00015 #include "RAIIObjectsForParser.h"
00016 #include "clang/AST/DeclTemplate.h"
00017 #include "clang/Parse/ParseDiagnostic.h"
00018 #include "clang/Sema/DeclSpec.h"
00019 #include "clang/Sema/Scope.h"
00020 using namespace clang;
00021 
00022 /// ParseCXXInlineMethodDef - We parsed and verified that the specified
00023 /// Declarator is a well formed C++ inline method definition. Now lex its body
00024 /// and store its tokens for parsing after the C++ class is complete.
00025 NamedDecl *Parser::ParseCXXInlineMethodDef(AccessSpecifier AS,
00026                                       AttributeList *AccessAttrs,
00027                                       ParsingDeclarator &D,
00028                                       const ParsedTemplateInfo &TemplateInfo,
00029                                       const VirtSpecifiers& VS, 
00030                                       FunctionDefinitionKind DefinitionKind,
00031                                       ExprResult& Init) {
00032   assert(D.isFunctionDeclarator() && "This isn't a function declarator!");
00033   assert((Tok.is(tok::l_brace) || Tok.is(tok::colon) || Tok.is(tok::kw_try) ||
00034           Tok.is(tok::equal)) &&
00035          "Current token not a '{', ':', '=', or 'try'!");
00036 
00037   MultiTemplateParamsArg TemplateParams(
00038       TemplateInfo.TemplateParams ? TemplateInfo.TemplateParams->data()
00039                                   : nullptr,
00040       TemplateInfo.TemplateParams ? TemplateInfo.TemplateParams->size() : 0);
00041 
00042   NamedDecl *FnD;
00043   D.setFunctionDefinitionKind(DefinitionKind);
00044   if (D.getDeclSpec().isFriendSpecified())
00045     FnD = Actions.ActOnFriendFunctionDecl(getCurScope(), D,
00046                                           TemplateParams);
00047   else {
00048     FnD = Actions.ActOnCXXMemberDeclarator(getCurScope(), AS, D,
00049                                            TemplateParams, nullptr,
00050                                            VS, ICIS_NoInit);
00051     if (FnD) {
00052       Actions.ProcessDeclAttributeList(getCurScope(), FnD, AccessAttrs);
00053       bool TypeSpecContainsAuto = D.getDeclSpec().containsPlaceholderType();
00054       if (Init.isUsable())
00055         Actions.AddInitializerToDecl(FnD, Init.get(), false,
00056                                      TypeSpecContainsAuto);
00057       else
00058         Actions.ActOnUninitializedDecl(FnD, TypeSpecContainsAuto);
00059     }
00060   }
00061 
00062   HandleMemberFunctionDeclDelays(D, FnD);
00063 
00064   D.complete(FnD);
00065 
00066   if (TryConsumeToken(tok::equal)) {
00067     if (!FnD) {
00068       SkipUntil(tok::semi);
00069       return nullptr;
00070     }
00071 
00072     bool Delete = false;
00073     SourceLocation KWLoc;
00074     if (TryConsumeToken(tok::kw_delete, KWLoc)) {
00075       Diag(KWLoc, getLangOpts().CPlusPlus11
00076                       ? diag::warn_cxx98_compat_deleted_function
00077                       : diag::ext_deleted_function);
00078       Actions.SetDeclDeleted(FnD, KWLoc);
00079       Delete = true;
00080     } else if (TryConsumeToken(tok::kw_default, KWLoc)) {
00081       Diag(KWLoc, getLangOpts().CPlusPlus11
00082                       ? diag::warn_cxx98_compat_defaulted_function
00083                       : diag::ext_defaulted_function);
00084       Actions.SetDeclDefaulted(FnD, KWLoc);
00085     } else {
00086       llvm_unreachable("function definition after = not 'delete' or 'default'");
00087     }
00088 
00089     if (Tok.is(tok::comma)) {
00090       Diag(KWLoc, diag::err_default_delete_in_multiple_declaration)
00091         << Delete;
00092       SkipUntil(tok::semi);
00093     } else if (ExpectAndConsume(tok::semi, diag::err_expected_after,
00094                                 Delete ? "delete" : "default")) {
00095       SkipUntil(tok::semi);
00096     }
00097 
00098     return FnD;
00099   }
00100   
00101   // In delayed template parsing mode, if we are within a class template
00102   // or if we are about to parse function member template then consume
00103   // the tokens and store them for parsing at the end of the translation unit.
00104   if (getLangOpts().DelayedTemplateParsing &&
00105       DefinitionKind == FDK_Definition &&
00106       !D.getDeclSpec().isConstexprSpecified() &&
00107       !(FnD && FnD->getAsFunction() &&
00108         FnD->getAsFunction()->getReturnType()->getContainedAutoType()) &&
00109       ((Actions.CurContext->isDependentContext() ||
00110         (TemplateInfo.Kind != ParsedTemplateInfo::NonTemplate &&
00111          TemplateInfo.Kind != ParsedTemplateInfo::ExplicitSpecialization)) &&
00112        !Actions.IsInsideALocalClassWithinATemplateFunction())) {
00113 
00114     CachedTokens Toks;
00115     LexTemplateFunctionForLateParsing(Toks);
00116 
00117     if (FnD) {
00118       FunctionDecl *FD = FnD->getAsFunction();
00119       Actions.CheckForFunctionRedefinition(FD);
00120       Actions.MarkAsLateParsedTemplate(FD, FnD, Toks);
00121     }
00122 
00123     return FnD;
00124   }
00125 
00126   // Consume the tokens and store them for later parsing.
00127 
00128   LexedMethod* LM = new LexedMethod(this, FnD);
00129   getCurrentClass().LateParsedDeclarations.push_back(LM);
00130   LM->TemplateScope = getCurScope()->isTemplateParamScope();
00131   CachedTokens &Toks = LM->Toks;
00132 
00133   tok::TokenKind kind = Tok.getKind();
00134   // Consume everything up to (and including) the left brace of the
00135   // function body.
00136   if (ConsumeAndStoreFunctionPrologue(Toks)) {
00137     // We didn't find the left-brace we expected after the
00138     // constructor initializer; we already printed an error, and it's likely
00139     // impossible to recover, so don't try to parse this method later.
00140     // Skip over the rest of the decl and back to somewhere that looks
00141     // reasonable.
00142     SkipMalformedDecl();
00143     delete getCurrentClass().LateParsedDeclarations.back();
00144     getCurrentClass().LateParsedDeclarations.pop_back();
00145     return FnD;
00146   } else {
00147     // Consume everything up to (and including) the matching right brace.
00148     ConsumeAndStoreUntil(tok::r_brace, Toks, /*StopAtSemi=*/false);
00149   }
00150 
00151   // If we're in a function-try-block, we need to store all the catch blocks.
00152   if (kind == tok::kw_try) {
00153     while (Tok.is(tok::kw_catch)) {
00154       ConsumeAndStoreUntil(tok::l_brace, Toks, /*StopAtSemi=*/false);
00155       ConsumeAndStoreUntil(tok::r_brace, Toks, /*StopAtSemi=*/false);
00156     }
00157   }
00158 
00159   if (FnD) {
00160     // If this is a friend function, mark that it's late-parsed so that
00161     // it's still known to be a definition even before we attach the
00162     // parsed body.  Sema needs to treat friend function definitions
00163     // differently during template instantiation, and it's possible for
00164     // the containing class to be instantiated before all its member
00165     // function definitions are parsed.
00166     //
00167     // If you remove this, you can remove the code that clears the flag
00168     // after parsing the member.
00169     if (D.getDeclSpec().isFriendSpecified()) {
00170       FunctionDecl *FD = FnD->getAsFunction();
00171       Actions.CheckForFunctionRedefinition(FD);
00172       FD->setLateTemplateParsed(true);
00173     }
00174   } else {
00175     // If semantic analysis could not build a function declaration,
00176     // just throw away the late-parsed declaration.
00177     delete getCurrentClass().LateParsedDeclarations.back();
00178     getCurrentClass().LateParsedDeclarations.pop_back();
00179   }
00180 
00181   return FnD;
00182 }
00183 
00184 /// ParseCXXNonStaticMemberInitializer - We parsed and verified that the
00185 /// specified Declarator is a well formed C++ non-static data member
00186 /// declaration. Now lex its initializer and store its tokens for parsing
00187 /// after the class is complete.
00188 void Parser::ParseCXXNonStaticMemberInitializer(Decl *VarD) {
00189   assert((Tok.is(tok::l_brace) || Tok.is(tok::equal)) &&
00190          "Current token not a '{' or '='!");
00191 
00192   LateParsedMemberInitializer *MI =
00193     new LateParsedMemberInitializer(this, VarD);
00194   getCurrentClass().LateParsedDeclarations.push_back(MI);
00195   CachedTokens &Toks = MI->Toks;
00196 
00197   tok::TokenKind kind = Tok.getKind();
00198   if (kind == tok::equal) {
00199     Toks.push_back(Tok);
00200     ConsumeToken();
00201   }
00202 
00203   if (kind == tok::l_brace) {
00204     // Begin by storing the '{' token.
00205     Toks.push_back(Tok);
00206     ConsumeBrace();
00207 
00208     // Consume everything up to (and including) the matching right brace.
00209     ConsumeAndStoreUntil(tok::r_brace, Toks, /*StopAtSemi=*/true);
00210   } else {
00211     // Consume everything up to (but excluding) the comma or semicolon.
00212     ConsumeAndStoreInitializer(Toks, CIK_DefaultInitializer);
00213   }
00214 
00215   // Store an artificial EOF token to ensure that we don't run off the end of
00216   // the initializer when we come to parse it.
00217   Token Eof;
00218   Eof.startToken();
00219   Eof.setKind(tok::eof);
00220   Eof.setLocation(Tok.getLocation());
00221   Eof.setEofData(VarD);
00222   Toks.push_back(Eof);
00223 }
00224 
00225 Parser::LateParsedDeclaration::~LateParsedDeclaration() {}
00226 void Parser::LateParsedDeclaration::ParseLexedMethodDeclarations() {}
00227 void Parser::LateParsedDeclaration::ParseLexedMemberInitializers() {}
00228 void Parser::LateParsedDeclaration::ParseLexedMethodDefs() {}
00229 
00230 Parser::LateParsedClass::LateParsedClass(Parser *P, ParsingClass *C)
00231   : Self(P), Class(C) {}
00232 
00233 Parser::LateParsedClass::~LateParsedClass() {
00234   Self->DeallocateParsedClasses(Class);
00235 }
00236 
00237 void Parser::LateParsedClass::ParseLexedMethodDeclarations() {
00238   Self->ParseLexedMethodDeclarations(*Class);
00239 }
00240 
00241 void Parser::LateParsedClass::ParseLexedMemberInitializers() {
00242   Self->ParseLexedMemberInitializers(*Class);
00243 }
00244 
00245 void Parser::LateParsedClass::ParseLexedMethodDefs() {
00246   Self->ParseLexedMethodDefs(*Class);
00247 }
00248 
00249 void Parser::LateParsedMethodDeclaration::ParseLexedMethodDeclarations() {
00250   Self->ParseLexedMethodDeclaration(*this);
00251 }
00252 
00253 void Parser::LexedMethod::ParseLexedMethodDefs() {
00254   Self->ParseLexedMethodDef(*this);
00255 }
00256 
00257 void Parser::LateParsedMemberInitializer::ParseLexedMemberInitializers() {
00258   Self->ParseLexedMemberInitializer(*this);
00259 }
00260 
00261 /// ParseLexedMethodDeclarations - We finished parsing the member
00262 /// specification of a top (non-nested) C++ class. Now go over the
00263 /// stack of method declarations with some parts for which parsing was
00264 /// delayed (such as default arguments) and parse them.
00265 void Parser::ParseLexedMethodDeclarations(ParsingClass &Class) {
00266   bool HasTemplateScope = !Class.TopLevelClass && Class.TemplateScope;
00267   ParseScope ClassTemplateScope(this, Scope::TemplateParamScope,
00268                                 HasTemplateScope);
00269   TemplateParameterDepthRAII CurTemplateDepthTracker(TemplateParameterDepth);
00270   if (HasTemplateScope) {
00271     Actions.ActOnReenterTemplateScope(getCurScope(), Class.TagOrTemplate);
00272     ++CurTemplateDepthTracker;
00273   }
00274 
00275   // The current scope is still active if we're the top-level class.
00276   // Otherwise we'll need to push and enter a new scope.
00277   bool HasClassScope = !Class.TopLevelClass;
00278   ParseScope ClassScope(this, Scope::ClassScope|Scope::DeclScope,
00279                         HasClassScope);
00280   if (HasClassScope)
00281     Actions.ActOnStartDelayedMemberDeclarations(getCurScope(),
00282                                                 Class.TagOrTemplate);
00283 
00284   for (size_t i = 0; i < Class.LateParsedDeclarations.size(); ++i) {
00285     Class.LateParsedDeclarations[i]->ParseLexedMethodDeclarations();
00286   }
00287 
00288   if (HasClassScope)
00289     Actions.ActOnFinishDelayedMemberDeclarations(getCurScope(),
00290                                                  Class.TagOrTemplate);
00291 }
00292 
00293 void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) {
00294   // If this is a member template, introduce the template parameter scope.
00295   ParseScope TemplateScope(this, Scope::TemplateParamScope, LM.TemplateScope);
00296   TemplateParameterDepthRAII CurTemplateDepthTracker(TemplateParameterDepth);
00297   if (LM.TemplateScope) {
00298     Actions.ActOnReenterTemplateScope(getCurScope(), LM.Method);
00299     ++CurTemplateDepthTracker;
00300   }
00301   // Start the delayed C++ method declaration
00302   Actions.ActOnStartDelayedCXXMethodDeclaration(getCurScope(), LM.Method);
00303 
00304   // Introduce the parameters into scope and parse their default
00305   // arguments.
00306   ParseScope PrototypeScope(this, Scope::FunctionPrototypeScope |
00307                             Scope::FunctionDeclarationScope | Scope::DeclScope);
00308   for (unsigned I = 0, N = LM.DefaultArgs.size(); I != N; ++I) {
00309     // Introduce the parameter into scope.
00310     Actions.ActOnDelayedCXXMethodParameter(getCurScope(),
00311                                            LM.DefaultArgs[I].Param);
00312 
00313     if (CachedTokens *Toks = LM.DefaultArgs[I].Toks) {
00314       // Save the current token position.
00315       SourceLocation origLoc = Tok.getLocation();
00316 
00317       // Parse the default argument from its saved token stream.
00318       Toks->push_back(Tok); // So that the current token doesn't get lost
00319       PP.EnterTokenStream(&Toks->front(), Toks->size(), true, false);
00320 
00321       // Consume the previously-pushed token.
00322       ConsumeAnyToken();
00323 
00324       // Consume the '='.
00325       assert(Tok.is(tok::equal) && "Default argument not starting with '='");
00326       SourceLocation EqualLoc = ConsumeToken();
00327 
00328       // The argument isn't actually potentially evaluated unless it is
00329       // used.
00330       EnterExpressionEvaluationContext Eval(Actions,
00331                                             Sema::PotentiallyEvaluatedIfUsed,
00332                                             LM.DefaultArgs[I].Param);
00333 
00334       ExprResult DefArgResult;
00335       if (getLangOpts().CPlusPlus11 && Tok.is(tok::l_brace)) {
00336         Diag(Tok, diag::warn_cxx98_compat_generalized_initializer_lists);
00337         DefArgResult = ParseBraceInitializer();
00338       } else
00339         DefArgResult = ParseAssignmentExpression();
00340       DefArgResult = Actions.CorrectDelayedTyposInExpr(DefArgResult);
00341       if (DefArgResult.isInvalid())
00342         Actions.ActOnParamDefaultArgumentError(LM.DefaultArgs[I].Param,
00343                                                EqualLoc);
00344       else {
00345         if (!TryConsumeToken(tok::cxx_defaultarg_end)) {
00346           // The last two tokens are the terminator and the saved value of
00347           // Tok; the last token in the default argument is the one before
00348           // those.
00349           assert(Toks->size() >= 3 && "expected a token in default arg");
00350           Diag(Tok.getLocation(), diag::err_default_arg_unparsed)
00351             << SourceRange(Tok.getLocation(),
00352                            (*Toks)[Toks->size() - 3].getLocation());
00353         }
00354         Actions.ActOnParamDefaultArgument(LM.DefaultArgs[I].Param, EqualLoc,
00355                                           DefArgResult.get());
00356       }
00357 
00358       assert(!PP.getSourceManager().isBeforeInTranslationUnit(origLoc,
00359                                                          Tok.getLocation()) &&
00360              "ParseAssignmentExpression went over the default arg tokens!");
00361       // There could be leftover tokens (e.g. because of an error).
00362       // Skip through until we reach the original token position.
00363       while (Tok.getLocation() != origLoc && Tok.isNot(tok::eof))
00364         ConsumeAnyToken();
00365 
00366       delete Toks;
00367       LM.DefaultArgs[I].Toks = nullptr;
00368     }
00369   }
00370 
00371   // Parse a delayed exception-specification, if there is one.
00372   if (CachedTokens *Toks = LM.ExceptionSpecTokens) {
00373     // Save the current token position.
00374     SourceLocation origLoc = Tok.getLocation();
00375 
00376     // Parse the default argument from its saved token stream.
00377     Toks->push_back(Tok); // So that the current token doesn't get lost
00378     PP.EnterTokenStream(&Toks->front(), Toks->size(), true, false);
00379 
00380     // Consume the previously-pushed token.
00381     ConsumeAnyToken();
00382 
00383     // C++11 [expr.prim.general]p3:
00384     //   If a declaration declares a member function or member function
00385     //   template of a class X, the expression this is a prvalue of type
00386     //   "pointer to cv-qualifier-seq X" between the optional cv-qualifer-seq
00387     //   and the end of the function-definition, member-declarator, or
00388     //   declarator.
00389     CXXMethodDecl *Method;
00390     if (FunctionTemplateDecl *FunTmpl
00391           = dyn_cast<FunctionTemplateDecl>(LM.Method))
00392       Method = cast<CXXMethodDecl>(FunTmpl->getTemplatedDecl());
00393     else
00394       Method = cast<CXXMethodDecl>(LM.Method);
00395 
00396     Sema::CXXThisScopeRAII ThisScope(Actions, Method->getParent(),
00397                                      Method->getTypeQualifiers(),
00398                                      getLangOpts().CPlusPlus11);
00399 
00400     // Parse the exception-specification.
00401     SourceRange SpecificationRange;
00402     SmallVector<ParsedType, 4> DynamicExceptions;
00403     SmallVector<SourceRange, 4> DynamicExceptionRanges;
00404     ExprResult NoexceptExpr;
00405     CachedTokens *ExceptionSpecTokens;
00406 
00407     ExceptionSpecificationType EST
00408       = tryParseExceptionSpecification(/*Delayed=*/false, SpecificationRange,
00409                                        DynamicExceptions,
00410                                        DynamicExceptionRanges, NoexceptExpr,
00411                                        ExceptionSpecTokens);
00412 
00413     // Clean up the remaining tokens.
00414     if (Tok.is(tok::cxx_exceptspec_end))
00415       ConsumeToken();
00416     else if (EST != EST_None)
00417       Diag(Tok.getLocation(), diag::err_except_spec_unparsed);
00418 
00419     // Attach the exception-specification to the method.
00420     Actions.actOnDelayedExceptionSpecification(LM.Method, EST,
00421                                                SpecificationRange,
00422                                                DynamicExceptions,
00423                                                DynamicExceptionRanges,
00424                                                NoexceptExpr.isUsable()?
00425                                                  NoexceptExpr.get() : nullptr);
00426 
00427     assert(!PP.getSourceManager().isBeforeInTranslationUnit(origLoc,
00428                                                             Tok.getLocation()) &&
00429            "tryParseExceptionSpecification went over the exception tokens!");
00430 
00431     // There could be leftover tokens (e.g. because of an error).
00432     // Skip through until we reach the original token position.
00433     while (Tok.getLocation() != origLoc && Tok.isNot(tok::eof))
00434       ConsumeAnyToken();
00435 
00436     delete Toks;
00437     LM.ExceptionSpecTokens = nullptr;
00438   }
00439 
00440   PrototypeScope.Exit();
00441 
00442   // Finish the delayed C++ method declaration.
00443   Actions.ActOnFinishDelayedCXXMethodDeclaration(getCurScope(), LM.Method);
00444 }
00445 
00446 /// ParseLexedMethodDefs - We finished parsing the member specification of a top
00447 /// (non-nested) C++ class. Now go over the stack of lexed methods that were
00448 /// collected during its parsing and parse them all.
00449 void Parser::ParseLexedMethodDefs(ParsingClass &Class) {
00450   bool HasTemplateScope = !Class.TopLevelClass && Class.TemplateScope;
00451   ParseScope ClassTemplateScope(this, Scope::TemplateParamScope, HasTemplateScope);
00452   TemplateParameterDepthRAII CurTemplateDepthTracker(TemplateParameterDepth);
00453   if (HasTemplateScope) {
00454     Actions.ActOnReenterTemplateScope(getCurScope(), Class.TagOrTemplate);
00455     ++CurTemplateDepthTracker;
00456   }
00457   bool HasClassScope = !Class.TopLevelClass;
00458   ParseScope ClassScope(this, Scope::ClassScope|Scope::DeclScope,
00459                         HasClassScope);
00460 
00461   for (size_t i = 0; i < Class.LateParsedDeclarations.size(); ++i) {
00462     Class.LateParsedDeclarations[i]->ParseLexedMethodDefs();
00463   }
00464 }
00465 
00466 void Parser::ParseLexedMethodDef(LexedMethod &LM) {
00467   // If this is a member template, introduce the template parameter scope.
00468   ParseScope TemplateScope(this, Scope::TemplateParamScope, LM.TemplateScope);
00469   TemplateParameterDepthRAII CurTemplateDepthTracker(TemplateParameterDepth);
00470   if (LM.TemplateScope) {
00471     Actions.ActOnReenterTemplateScope(getCurScope(), LM.D);
00472     ++CurTemplateDepthTracker;
00473   }
00474   // Save the current token position.
00475   SourceLocation origLoc = Tok.getLocation();
00476 
00477   assert(!LM.Toks.empty() && "Empty body!");
00478   // Append the current token at the end of the new token stream so that it
00479   // doesn't get lost.
00480   LM.Toks.push_back(Tok);
00481   PP.EnterTokenStream(LM.Toks.data(), LM.Toks.size(), true, false);
00482 
00483   // Consume the previously pushed token.
00484   ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
00485   assert((Tok.is(tok::l_brace) || Tok.is(tok::colon) || Tok.is(tok::kw_try))
00486          && "Inline method not starting with '{', ':' or 'try'");
00487 
00488   // Parse the method body. Function body parsing code is similar enough
00489   // to be re-used for method bodies as well.
00490   ParseScope FnScope(this, Scope::FnScope|Scope::DeclScope);
00491   Actions.ActOnStartOfFunctionDef(getCurScope(), LM.D);
00492 
00493   if (Tok.is(tok::kw_try)) {
00494     ParseFunctionTryBlock(LM.D, FnScope);
00495     assert(!PP.getSourceManager().isBeforeInTranslationUnit(origLoc,
00496                                                          Tok.getLocation()) &&
00497            "ParseFunctionTryBlock went over the cached tokens!");
00498     // There could be leftover tokens (e.g. because of an error).
00499     // Skip through until we reach the original token position.
00500     while (Tok.getLocation() != origLoc && Tok.isNot(tok::eof))
00501       ConsumeAnyToken();
00502     return;
00503   }
00504   if (Tok.is(tok::colon)) {
00505     ParseConstructorInitializer(LM.D);
00506 
00507     // Error recovery.
00508     if (!Tok.is(tok::l_brace)) {
00509       FnScope.Exit();
00510       Actions.ActOnFinishFunctionBody(LM.D, nullptr);
00511       while (Tok.getLocation() != origLoc && Tok.isNot(tok::eof))
00512         ConsumeAnyToken();
00513       return;
00514     }
00515   } else
00516     Actions.ActOnDefaultCtorInitializers(LM.D);
00517 
00518   assert((Actions.getDiagnostics().hasErrorOccurred() ||
00519           !isa<FunctionTemplateDecl>(LM.D) ||
00520           cast<FunctionTemplateDecl>(LM.D)->getTemplateParameters()->getDepth()
00521             < TemplateParameterDepth) &&
00522          "TemplateParameterDepth should be greater than the depth of "
00523          "current template being instantiated!");
00524 
00525   ParseFunctionStatementBody(LM.D, FnScope);
00526 
00527   // Clear the late-template-parsed bit if we set it before.
00528   if (LM.D)
00529     LM.D->getAsFunction()->setLateTemplateParsed(false);
00530 
00531   if (Tok.getLocation() != origLoc) {
00532     // Due to parsing error, we either went over the cached tokens or
00533     // there are still cached tokens left. If it's the latter case skip the
00534     // leftover tokens.
00535     // Since this is an uncommon situation that should be avoided, use the
00536     // expensive isBeforeInTranslationUnit call.
00537     if (PP.getSourceManager().isBeforeInTranslationUnit(Tok.getLocation(),
00538                                                         origLoc))
00539       while (Tok.getLocation() != origLoc && Tok.isNot(tok::eof))
00540         ConsumeAnyToken();
00541   }
00542 
00543   if (CXXMethodDecl *MD = dyn_cast_or_null<CXXMethodDecl>(LM.D))
00544     Actions.ActOnFinishInlineMethodDef(MD);
00545 }
00546 
00547 /// ParseLexedMemberInitializers - We finished parsing the member specification
00548 /// of a top (non-nested) C++ class. Now go over the stack of lexed data member
00549 /// initializers that were collected during its parsing and parse them all.
00550 void Parser::ParseLexedMemberInitializers(ParsingClass &Class) {
00551   bool HasTemplateScope = !Class.TopLevelClass && Class.TemplateScope;
00552   ParseScope ClassTemplateScope(this, Scope::TemplateParamScope,
00553                                 HasTemplateScope);
00554   TemplateParameterDepthRAII CurTemplateDepthTracker(TemplateParameterDepth);
00555   if (HasTemplateScope) {
00556     Actions.ActOnReenterTemplateScope(getCurScope(), Class.TagOrTemplate);
00557     ++CurTemplateDepthTracker;
00558   }
00559   // Set or update the scope flags.
00560   bool AlreadyHasClassScope = Class.TopLevelClass;
00561   unsigned ScopeFlags = Scope::ClassScope|Scope::DeclScope;
00562   ParseScope ClassScope(this, ScopeFlags, !AlreadyHasClassScope);
00563   ParseScopeFlags ClassScopeFlags(this, ScopeFlags, AlreadyHasClassScope);
00564 
00565   if (!AlreadyHasClassScope)
00566     Actions.ActOnStartDelayedMemberDeclarations(getCurScope(),
00567                                                 Class.TagOrTemplate);
00568 
00569   if (!Class.LateParsedDeclarations.empty()) {
00570     // C++11 [expr.prim.general]p4:
00571     //   Otherwise, if a member-declarator declares a non-static data member 
00572     //  (9.2) of a class X, the expression this is a prvalue of type "pointer
00573     //  to X" within the optional brace-or-equal-initializer. It shall not 
00574     //  appear elsewhere in the member-declarator.
00575     Sema::CXXThisScopeRAII ThisScope(Actions, Class.TagOrTemplate,
00576                                      /*TypeQuals=*/(unsigned)0);
00577 
00578     for (size_t i = 0; i < Class.LateParsedDeclarations.size(); ++i) {
00579       Class.LateParsedDeclarations[i]->ParseLexedMemberInitializers();
00580     }
00581   }
00582   
00583   if (!AlreadyHasClassScope)
00584     Actions.ActOnFinishDelayedMemberDeclarations(getCurScope(),
00585                                                  Class.TagOrTemplate);
00586 
00587   Actions.ActOnFinishDelayedMemberInitializers(Class.TagOrTemplate);
00588 }
00589 
00590 void Parser::ParseLexedMemberInitializer(LateParsedMemberInitializer &MI) {
00591   if (!MI.Field || MI.Field->isInvalidDecl())
00592     return;
00593 
00594   // Append the current token at the end of the new token stream so that it
00595   // doesn't get lost.
00596   MI.Toks.push_back(Tok);
00597   PP.EnterTokenStream(MI.Toks.data(), MI.Toks.size(), true, false);
00598 
00599   // Consume the previously pushed token.
00600   ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
00601 
00602   SourceLocation EqualLoc;
00603 
00604   Actions.ActOnStartCXXInClassMemberInitializer();
00605 
00606   ExprResult Init = ParseCXXMemberInitializer(MI.Field, /*IsFunction=*/false, 
00607                                               EqualLoc);
00608 
00609   Actions.ActOnFinishCXXInClassMemberInitializer(MI.Field, EqualLoc,
00610                                                  Init.get());
00611 
00612   // The next token should be our artificial terminating EOF token.
00613   if (Tok.isNot(tok::eof)) {
00614     if (!Init.isInvalid()) {
00615       SourceLocation EndLoc = PP.getLocForEndOfToken(PrevTokLocation);
00616       if (!EndLoc.isValid())
00617         EndLoc = Tok.getLocation();
00618       // No fixit; we can't recover as if there were a semicolon here.
00619       Diag(EndLoc, diag::err_expected_semi_decl_list);
00620     }
00621 
00622     // Consume tokens until we hit the artificial EOF.
00623     while (Tok.isNot(tok::eof))
00624       ConsumeAnyToken();
00625   }
00626   // Make sure this is *our* artificial EOF token.
00627   if (Tok.getEofData() == MI.Field)
00628     ConsumeAnyToken();
00629 }
00630 
00631 /// ConsumeAndStoreUntil - Consume and store the token at the passed token
00632 /// container until the token 'T' is reached (which gets
00633 /// consumed/stored too, if ConsumeFinalToken).
00634 /// If StopAtSemi is true, then we will stop early at a ';' character.
00635 /// Returns true if token 'T1' or 'T2' was found.
00636 /// NOTE: This is a specialized version of Parser::SkipUntil.
00637 bool Parser::ConsumeAndStoreUntil(tok::TokenKind T1, tok::TokenKind T2,
00638                                   CachedTokens &Toks,
00639                                   bool StopAtSemi, bool ConsumeFinalToken) {
00640   // We always want this function to consume at least one token if the first
00641   // token isn't T and if not at EOF.
00642   bool isFirstTokenConsumed = true;
00643   while (1) {
00644     // If we found one of the tokens, stop and return true.
00645     if (Tok.is(T1) || Tok.is(T2)) {
00646       if (ConsumeFinalToken) {
00647         Toks.push_back(Tok);
00648         ConsumeAnyToken();
00649       }
00650       return true;
00651     }
00652 
00653     switch (Tok.getKind()) {
00654     case tok::eof:
00655     case tok::annot_module_begin:
00656     case tok::annot_module_end:
00657     case tok::annot_module_include:
00658       // Ran out of tokens.
00659       return false;
00660 
00661     case tok::l_paren:
00662       // Recursively consume properly-nested parens.
00663       Toks.push_back(Tok);
00664       ConsumeParen();
00665       ConsumeAndStoreUntil(tok::r_paren, Toks, /*StopAtSemi=*/false);
00666       break;
00667     case tok::l_square:
00668       // Recursively consume properly-nested square brackets.
00669       Toks.push_back(Tok);
00670       ConsumeBracket();
00671       ConsumeAndStoreUntil(tok::r_square, Toks, /*StopAtSemi=*/false);
00672       break;
00673     case tok::l_brace:
00674       // Recursively consume properly-nested braces.
00675       Toks.push_back(Tok);
00676       ConsumeBrace();
00677       ConsumeAndStoreUntil(tok::r_brace, Toks, /*StopAtSemi=*/false);
00678       break;
00679 
00680     // Okay, we found a ']' or '}' or ')', which we think should be balanced.
00681     // Since the user wasn't looking for this token (if they were, it would
00682     // already be handled), this isn't balanced.  If there is a LHS token at a
00683     // higher level, we will assume that this matches the unbalanced token
00684     // and return it.  Otherwise, this is a spurious RHS token, which we skip.
00685     case tok::r_paren:
00686       if (ParenCount && !isFirstTokenConsumed)
00687         return false;  // Matches something.
00688       Toks.push_back(Tok);
00689       ConsumeParen();
00690       break;
00691     case tok::r_square:
00692       if (BracketCount && !isFirstTokenConsumed)
00693         return false;  // Matches something.
00694       Toks.push_back(Tok);
00695       ConsumeBracket();
00696       break;
00697     case tok::r_brace:
00698       if (BraceCount && !isFirstTokenConsumed)
00699         return false;  // Matches something.
00700       Toks.push_back(Tok);
00701       ConsumeBrace();
00702       break;
00703 
00704     case tok::code_completion:
00705       Toks.push_back(Tok);
00706       ConsumeCodeCompletionToken();
00707       break;
00708 
00709     case tok::string_literal:
00710     case tok::wide_string_literal:
00711     case tok::utf8_string_literal:
00712     case tok::utf16_string_literal:
00713     case tok::utf32_string_literal:
00714       Toks.push_back(Tok);
00715       ConsumeStringToken();
00716       break;
00717     case tok::semi:
00718       if (StopAtSemi)
00719         return false;
00720       // FALL THROUGH.
00721     default:
00722       // consume this token.
00723       Toks.push_back(Tok);
00724       ConsumeToken();
00725       break;
00726     }
00727     isFirstTokenConsumed = false;
00728   }
00729 }
00730 
00731 /// \brief Consume tokens and store them in the passed token container until
00732 /// we've passed the try keyword and constructor initializers and have consumed
00733 /// the opening brace of the function body. The opening brace will be consumed
00734 /// if and only if there was no error.
00735 ///
00736 /// \return True on error.
00737 bool Parser::ConsumeAndStoreFunctionPrologue(CachedTokens &Toks) {
00738   if (Tok.is(tok::kw_try)) {
00739     Toks.push_back(Tok);
00740     ConsumeToken();
00741   }
00742 
00743   if (Tok.isNot(tok::colon)) {
00744     // Easy case, just a function body.
00745 
00746     // Grab any remaining garbage to be diagnosed later. We stop when we reach a
00747     // brace: an opening one is the function body, while a closing one probably
00748     // means we've reached the end of the class.
00749     ConsumeAndStoreUntil(tok::l_brace, tok::r_brace, Toks,
00750                          /*StopAtSemi=*/true,
00751                          /*ConsumeFinalToken=*/false);
00752     if (Tok.isNot(tok::l_brace))
00753       return Diag(Tok.getLocation(), diag::err_expected) << tok::l_brace;
00754 
00755     Toks.push_back(Tok);
00756     ConsumeBrace();
00757     return false;
00758   }
00759 
00760   Toks.push_back(Tok);
00761   ConsumeToken();
00762 
00763   // We can't reliably skip over a mem-initializer-id, because it could be
00764   // a template-id involving not-yet-declared names. Given:
00765   //
00766   //   S ( ) : a < b < c > ( e )
00767   //
00768   // 'e' might be an initializer or part of a template argument, depending
00769   // on whether 'b' is a template.
00770 
00771   // Track whether we might be inside a template argument. We can give
00772   // significantly better diagnostics if we know that we're not.
00773   bool MightBeTemplateArgument = false;
00774 
00775   while (true) {
00776     // Skip over the mem-initializer-id, if possible.
00777     if (Tok.is(tok::kw_decltype)) {
00778       Toks.push_back(Tok);
00779       SourceLocation OpenLoc = ConsumeToken();
00780       if (Tok.isNot(tok::l_paren))
00781         return Diag(Tok.getLocation(), diag::err_expected_lparen_after)
00782                  << "decltype";
00783       Toks.push_back(Tok);
00784       ConsumeParen();
00785       if (!ConsumeAndStoreUntil(tok::r_paren, Toks, /*StopAtSemi=*/true)) {
00786         Diag(Tok.getLocation(), diag::err_expected) << tok::r_paren;
00787         Diag(OpenLoc, diag::note_matching) << tok::l_paren;
00788         return true;
00789       }
00790     }
00791     do {
00792       // Walk over a component of a nested-name-specifier.
00793       if (Tok.is(tok::coloncolon)) {
00794         Toks.push_back(Tok);
00795         ConsumeToken();
00796 
00797         if (Tok.is(tok::kw_template)) {
00798           Toks.push_back(Tok);
00799           ConsumeToken();
00800         }
00801       }
00802 
00803       if (Tok.is(tok::identifier) || Tok.is(tok::kw_template)) {
00804         Toks.push_back(Tok);
00805         ConsumeToken();
00806       } else if (Tok.is(tok::code_completion)) {
00807         Toks.push_back(Tok);
00808         ConsumeCodeCompletionToken();
00809         // Consume the rest of the initializers permissively.
00810         // FIXME: We should be able to perform code-completion here even if
00811         //        there isn't a subsequent '{' token.
00812         MightBeTemplateArgument = true;
00813         break;
00814       } else {
00815         break;
00816       }
00817     } while (Tok.is(tok::coloncolon));
00818 
00819     if (Tok.is(tok::less))
00820       MightBeTemplateArgument = true;
00821 
00822     if (MightBeTemplateArgument) {
00823       // We may be inside a template argument list. Grab up to the start of the
00824       // next parenthesized initializer or braced-init-list. This *might* be the
00825       // initializer, or it might be a subexpression in the template argument
00826       // list.
00827       // FIXME: Count angle brackets, and clear MightBeTemplateArgument
00828       //        if all angles are closed.
00829       if (!ConsumeAndStoreUntil(tok::l_paren, tok::l_brace, Toks,
00830                                 /*StopAtSemi=*/true,
00831                                 /*ConsumeFinalToken=*/false)) {
00832         // We're not just missing the initializer, we're also missing the
00833         // function body!
00834         return Diag(Tok.getLocation(), diag::err_expected) << tok::l_brace;
00835       }
00836     } else if (Tok.isNot(tok::l_paren) && Tok.isNot(tok::l_brace)) {
00837       // We found something weird in a mem-initializer-id.
00838       if (getLangOpts().CPlusPlus11)
00839         return Diag(Tok.getLocation(), diag::err_expected_either)
00840                << tok::l_paren << tok::l_brace;
00841       else
00842         return Diag(Tok.getLocation(), diag::err_expected) << tok::l_paren;
00843     }
00844 
00845     tok::TokenKind kind = Tok.getKind();
00846     Toks.push_back(Tok);
00847     bool IsLParen = (kind == tok::l_paren);
00848     SourceLocation OpenLoc = Tok.getLocation();
00849 
00850     if (IsLParen) {
00851       ConsumeParen();
00852     } else {
00853       assert(kind == tok::l_brace && "Must be left paren or brace here.");
00854       ConsumeBrace();
00855       // In C++03, this has to be the start of the function body, which
00856       // means the initializer is malformed; we'll diagnose it later.
00857       if (!getLangOpts().CPlusPlus11)
00858         return false;
00859     }
00860 
00861     // Grab the initializer (or the subexpression of the template argument).
00862     // FIXME: If we support lambdas here, we'll need to set StopAtSemi to false
00863     //        if we might be inside the braces of a lambda-expression.
00864     tok::TokenKind CloseKind = IsLParen ? tok::r_paren : tok::r_brace;
00865     if (!ConsumeAndStoreUntil(CloseKind, Toks, /*StopAtSemi=*/true)) {
00866       Diag(Tok, diag::err_expected) << CloseKind;
00867       Diag(OpenLoc, diag::note_matching) << kind;
00868       return true;
00869     }
00870 
00871     // Grab pack ellipsis, if present.
00872     if (Tok.is(tok::ellipsis)) {
00873       Toks.push_back(Tok);
00874       ConsumeToken();
00875     }
00876 
00877     // If we know we just consumed a mem-initializer, we must have ',' or '{'
00878     // next.
00879     if (Tok.is(tok::comma)) {
00880       Toks.push_back(Tok);
00881       ConsumeToken();
00882     } else if (Tok.is(tok::l_brace)) {
00883       // This is the function body if the ')' or '}' is immediately followed by
00884       // a '{'. That cannot happen within a template argument, apart from the
00885       // case where a template argument contains a compound literal:
00886       //
00887       //   S ( ) : a < b < c > ( d ) { }
00888       //   // End of declaration, or still inside the template argument?
00889       //
00890       // ... and the case where the template argument contains a lambda:
00891       //
00892       //   S ( ) : a < 0 && b < c > ( d ) + [ ] ( ) { return 0; }
00893       //     ( ) > ( ) { }
00894       //
00895       // FIXME: Disambiguate these cases. Note that the latter case is probably
00896       //        going to be made ill-formed by core issue 1607.
00897       Toks.push_back(Tok);
00898       ConsumeBrace();
00899       return false;
00900     } else if (!MightBeTemplateArgument) {
00901       return Diag(Tok.getLocation(), diag::err_expected_either) << tok::l_brace
00902                                                                 << tok::comma;
00903     }
00904   }
00905 }
00906 
00907 /// \brief Consume and store tokens from the '?' to the ':' in a conditional
00908 /// expression.
00909 bool Parser::ConsumeAndStoreConditional(CachedTokens &Toks) {
00910   // Consume '?'.
00911   assert(Tok.is(tok::question));
00912   Toks.push_back(Tok);
00913   ConsumeToken();
00914 
00915   while (Tok.isNot(tok::colon)) {
00916     if (!ConsumeAndStoreUntil(tok::question, tok::colon, Toks,
00917                               /*StopAtSemi=*/true,
00918                               /*ConsumeFinalToken=*/false))
00919       return false;
00920 
00921     // If we found a nested conditional, consume it.
00922     if (Tok.is(tok::question) && !ConsumeAndStoreConditional(Toks))
00923       return false;
00924   }
00925 
00926   // Consume ':'.
00927   Toks.push_back(Tok);
00928   ConsumeToken();
00929   return true;
00930 }
00931 
00932 /// \brief A tentative parsing action that can also revert token annotations.
00933 class Parser::UnannotatedTentativeParsingAction : public TentativeParsingAction {
00934 public:
00935   explicit UnannotatedTentativeParsingAction(Parser &Self,
00936                                              tok::TokenKind EndKind)
00937       : TentativeParsingAction(Self), Self(Self), EndKind(EndKind) {
00938     // Stash away the old token stream, so we can restore it once the
00939     // tentative parse is complete.
00940     TentativeParsingAction Inner(Self);
00941     Self.ConsumeAndStoreUntil(EndKind, Toks, true, /*ConsumeFinalToken*/false);
00942     Inner.Revert();
00943   }
00944 
00945   void RevertAnnotations() {
00946     Revert();
00947 
00948     // Put back the original tokens.
00949     Self.SkipUntil(EndKind, StopAtSemi | StopBeforeMatch);
00950     if (Toks.size()) {
00951       Token *Buffer = new Token[Toks.size()];
00952       std::copy(Toks.begin() + 1, Toks.end(), Buffer);
00953       Buffer[Toks.size() - 1] = Self.Tok;
00954       Self.PP.EnterTokenStream(Buffer, Toks.size(), true, /*Owned*/true);
00955 
00956       Self.Tok = Toks.front();
00957     }
00958   }
00959 
00960 private:
00961   Parser &Self;
00962   CachedTokens Toks;
00963   tok::TokenKind EndKind;
00964 };
00965 
00966 /// ConsumeAndStoreInitializer - Consume and store the token at the passed token
00967 /// container until the end of the current initializer expression (either a
00968 /// default argument or an in-class initializer for a non-static data member).
00969 ///
00970 /// Returns \c true if we reached the end of something initializer-shaped,
00971 /// \c false if we bailed out.
00972 bool Parser::ConsumeAndStoreInitializer(CachedTokens &Toks,
00973                                         CachedInitKind CIK) {
00974   // We always want this function to consume at least one token if not at EOF.
00975   bool IsFirstToken = true;
00976 
00977   // Number of possible unclosed <s we've seen so far. These might be templates,
00978   // and might not, but if there were none of them (or we know for sure that
00979   // we're within a template), we can avoid a tentative parse.
00980   unsigned AngleCount = 0;
00981   unsigned KnownTemplateCount = 0;
00982 
00983   while (1) {
00984     switch (Tok.getKind()) {
00985     case tok::comma:
00986       // If we might be in a template, perform a tentative parse to check.
00987       if (!AngleCount)
00988         // Not a template argument: this is the end of the initializer.
00989         return true;
00990       if (KnownTemplateCount)
00991         goto consume_token;
00992 
00993       // We hit a comma inside angle brackets. This is the hard case. The
00994       // rule we follow is:
00995       //  * For a default argument, if the tokens after the comma form a
00996       //    syntactically-valid parameter-declaration-clause, in which each
00997       //    parameter has an initializer, then this comma ends the default
00998       //    argument.
00999       //  * For a default initializer, if the tokens after the comma form a
01000       //    syntactically-valid init-declarator-list, then this comma ends
01001       //    the default initializer.
01002       {
01003         UnannotatedTentativeParsingAction PA(*this,
01004                                              CIK == CIK_DefaultInitializer
01005                                                ? tok::semi : tok::r_paren);
01006         Sema::TentativeAnalysisScope Scope(Actions);
01007 
01008         TPResult Result = TPResult::Error;
01009         ConsumeToken();
01010         switch (CIK) {
01011         case CIK_DefaultInitializer:
01012           Result = TryParseInitDeclaratorList();
01013           // If we parsed a complete, ambiguous init-declarator-list, this
01014           // is only syntactically-valid if it's followed by a semicolon.
01015           if (Result == TPResult::Ambiguous && Tok.isNot(tok::semi))
01016             Result = TPResult::False;
01017           break;
01018 
01019         case CIK_DefaultArgument:
01020           bool InvalidAsDeclaration = false;
01021           Result = TryParseParameterDeclarationClause(
01022               &InvalidAsDeclaration, /*VersusTemplateArgument*/true);
01023           // If this is an expression or a declaration with a missing
01024           // 'typename', assume it's not a declaration.
01025           if (Result == TPResult::Ambiguous && InvalidAsDeclaration)
01026             Result = TPResult::False;
01027           break;
01028         }
01029 
01030         // If what follows could be a declaration, it is a declaration.
01031         if (Result != TPResult::False && Result != TPResult::Error) {
01032           PA.Revert();
01033           return true;
01034         }
01035 
01036         // In the uncommon case that we decide the following tokens are part
01037         // of a template argument, revert any annotations we've performed in
01038         // those tokens. We're not going to look them up until we've parsed
01039         // the rest of the class, and that might add more declarations.
01040         PA.RevertAnnotations();
01041       }
01042 
01043       // Keep going. We know we're inside a template argument list now.
01044       ++KnownTemplateCount;
01045       goto consume_token;
01046 
01047     case tok::eof:
01048     case tok::annot_module_begin:
01049     case tok::annot_module_end:
01050     case tok::annot_module_include:
01051       // Ran out of tokens.
01052       return false;
01053 
01054     case tok::less:
01055       // FIXME: A '<' can only start a template-id if it's preceded by an
01056       // identifier, an operator-function-id, or a literal-operator-id.
01057       ++AngleCount;
01058       goto consume_token;
01059 
01060     case tok::question:
01061       // In 'a ? b : c', 'b' can contain an unparenthesized comma. If it does,
01062       // that is *never* the end of the initializer. Skip to the ':'.
01063       if (!ConsumeAndStoreConditional(Toks))
01064         return false;
01065       break;
01066 
01067     case tok::greatergreatergreater:
01068       if (!getLangOpts().CPlusPlus11)
01069         goto consume_token;
01070       if (AngleCount) --AngleCount;
01071       if (KnownTemplateCount) --KnownTemplateCount;
01072       // Fall through.
01073     case tok::greatergreater:
01074       if (!getLangOpts().CPlusPlus11)
01075         goto consume_token;
01076       if (AngleCount) --AngleCount;
01077       if (KnownTemplateCount) --KnownTemplateCount;
01078       // Fall through.
01079     case tok::greater:
01080       if (AngleCount) --AngleCount;
01081       if (KnownTemplateCount) --KnownTemplateCount;
01082       goto consume_token;
01083 
01084     case tok::kw_template:
01085       // 'template' identifier '<' is known to start a template argument list,
01086       // and can be used to disambiguate the parse.
01087       // FIXME: Support all forms of 'template' unqualified-id '<'.
01088       Toks.push_back(Tok);
01089       ConsumeToken();
01090       if (Tok.is(tok::identifier)) {
01091         Toks.push_back(Tok);
01092         ConsumeToken();
01093         if (Tok.is(tok::less)) {
01094           ++AngleCount;
01095           ++KnownTemplateCount;
01096           Toks.push_back(Tok);
01097           ConsumeToken();
01098         }
01099       }
01100       break;
01101 
01102     case tok::kw_operator:
01103       // If 'operator' precedes other punctuation, that punctuation loses
01104       // its special behavior.
01105       Toks.push_back(Tok);
01106       ConsumeToken();
01107       switch (Tok.getKind()) {
01108       case tok::comma:
01109       case tok::greatergreatergreater:
01110       case tok::greatergreater:
01111       case tok::greater:
01112       case tok::less:
01113         Toks.push_back(Tok);
01114         ConsumeToken();
01115         break;
01116       default:
01117         break;
01118       }
01119       break;
01120 
01121     case tok::l_paren:
01122       // Recursively consume properly-nested parens.
01123       Toks.push_back(Tok);
01124       ConsumeParen();
01125       ConsumeAndStoreUntil(tok::r_paren, Toks, /*StopAtSemi=*/false);
01126       break;
01127     case tok::l_square:
01128       // Recursively consume properly-nested square brackets.
01129       Toks.push_back(Tok);
01130       ConsumeBracket();
01131       ConsumeAndStoreUntil(tok::r_square, Toks, /*StopAtSemi=*/false);
01132       break;
01133     case tok::l_brace:
01134       // Recursively consume properly-nested braces.
01135       Toks.push_back(Tok);
01136       ConsumeBrace();
01137       ConsumeAndStoreUntil(tok::r_brace, Toks, /*StopAtSemi=*/false);
01138       break;
01139 
01140     // Okay, we found a ']' or '}' or ')', which we think should be balanced.
01141     // Since the user wasn't looking for this token (if they were, it would
01142     // already be handled), this isn't balanced.  If there is a LHS token at a
01143     // higher level, we will assume that this matches the unbalanced token
01144     // and return it.  Otherwise, this is a spurious RHS token, which we
01145     // consume and pass on to downstream code to diagnose.
01146     case tok::r_paren:
01147       if (CIK == CIK_DefaultArgument)
01148         return true; // End of the default argument.
01149       if (ParenCount && !IsFirstToken)
01150         return false;
01151       Toks.push_back(Tok);
01152       ConsumeParen();
01153       continue;
01154     case tok::r_square:
01155       if (BracketCount && !IsFirstToken)
01156         return false;
01157       Toks.push_back(Tok);
01158       ConsumeBracket();
01159       continue;
01160     case tok::r_brace:
01161       if (BraceCount && !IsFirstToken)
01162         return false;
01163       Toks.push_back(Tok);
01164       ConsumeBrace();
01165       continue;
01166 
01167     case tok::code_completion:
01168       Toks.push_back(Tok);
01169       ConsumeCodeCompletionToken();
01170       break;
01171 
01172     case tok::string_literal:
01173     case tok::wide_string_literal:
01174     case tok::utf8_string_literal:
01175     case tok::utf16_string_literal:
01176     case tok::utf32_string_literal:
01177       Toks.push_back(Tok);
01178       ConsumeStringToken();
01179       break;
01180     case tok::semi:
01181       if (CIK == CIK_DefaultInitializer)
01182         return true; // End of the default initializer.
01183       // FALL THROUGH.
01184     default:
01185     consume_token:
01186       Toks.push_back(Tok);
01187       ConsumeToken();
01188       break;
01189     }
01190     IsFirstToken = false;
01191   }
01192 }