clang  14.0.0git
TokenAnnotator.cpp
Go to the documentation of this file.
1 //===--- TokenAnnotator.cpp - Format C++ code -----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements a token annotator, i.e. creates
11 /// \c AnnotatedTokens out of \c FormatTokens with required extra information.
12 ///
13 //===----------------------------------------------------------------------===//
14 
15 #include "TokenAnnotator.h"
16 #include "FormatToken.h"
18 #include "clang/Basic/TokenKinds.h"
19 #include "llvm/ADT/SmallPtrSet.h"
20 #include "llvm/Support/Debug.h"
21 
22 #define DEBUG_TYPE "format-token-annotator"
23 
24 namespace clang {
25 namespace format {
26 
27 namespace {
28 
29 /// Returns \c true if the token can be used as an identifier in
30 /// an Objective-C \c \@selector, \c false otherwise.
31 ///
32 /// Because getFormattingLangOpts() always lexes source code as
33 /// Objective-C++, C++ keywords like \c new and \c delete are
34 /// lexed as tok::kw_*, not tok::identifier, even for Objective-C.
35 ///
36 /// For Objective-C and Objective-C++, both identifiers and keywords
37 /// are valid inside @selector(...) (or a macro which
38 /// invokes @selector(...)). So, we allow treat any identifier or
39 /// keyword as a potential Objective-C selector component.
40 static bool canBeObjCSelectorComponent(const FormatToken &Tok) {
41  return Tok.Tok.getIdentifierInfo() != nullptr;
42 }
43 
44 /// With `Left` being '(', check if we're at either `[...](` or
45 /// `[...]<...>(`, where the [ opens a lambda capture list.
46 static bool isLambdaParameterList(const FormatToken *Left) {
47  // Skip <...> if present.
48  if (Left->Previous && Left->Previous->is(tok::greater) &&
49  Left->Previous->MatchingParen &&
50  Left->Previous->MatchingParen->is(TT_TemplateOpener))
51  Left = Left->Previous->MatchingParen;
52 
53  // Check for `[...]`.
54  return Left->Previous && Left->Previous->is(tok::r_square) &&
55  Left->Previous->MatchingParen &&
56  Left->Previous->MatchingParen->is(TT_LambdaLSquare);
57 }
58 
59 /// Returns \c true if the token is followed by a boolean condition, \c false
60 /// otherwise.
61 static bool isKeywordWithCondition(const FormatToken &Tok) {
62  return Tok.isOneOf(tok::kw_if, tok::kw_for, tok::kw_while, tok::kw_switch,
63  tok::kw_constexpr, tok::kw_catch);
64 }
65 
66 /// A parser that gathers additional information about tokens.
67 ///
68 /// The \c TokenAnnotator tries to match parenthesis and square brakets and
69 /// store a parenthesis levels. It also tries to resolve matching "<" and ">"
70 /// into template parameter lists.
71 class AnnotatingParser {
72 public:
73  AnnotatingParser(const FormatStyle &Style, AnnotatedLine &Line,
74  const AdditionalKeywords &Keywords)
75  : Style(Style), Line(Line), CurrentToken(Line.First), AutoFound(false),
76  Keywords(Keywords) {
77  Contexts.push_back(Context(tok::unknown, 1, /*IsExpression=*/false));
78  resetTokenMetadata(CurrentToken);
79  }
80 
81 private:
82  bool parseAngle() {
83  if (!CurrentToken || !CurrentToken->Previous)
84  return false;
85  if (NonTemplateLess.count(CurrentToken->Previous))
86  return false;
87 
88  const FormatToken &Previous = *CurrentToken->Previous; // The '<'.
89  if (Previous.Previous) {
90  if (Previous.Previous->Tok.isLiteral())
91  return false;
92  if (Previous.Previous->is(tok::r_paren) && Contexts.size() > 1 &&
93  (!Previous.Previous->MatchingParen ||
94  !Previous.Previous->MatchingParen->is(TT_OverloadedOperatorLParen)))
95  return false;
96  }
97 
98  FormatToken *Left = CurrentToken->Previous;
99  Left->ParentBracket = Contexts.back().ContextKind;
100  ScopedContextCreator ContextCreator(*this, tok::less, 12);
101 
102  // If this angle is in the context of an expression, we need to be more
103  // hesitant to detect it as opening template parameters.
104  bool InExprContext = Contexts.back().IsExpression;
105 
106  Contexts.back().IsExpression = false;
107  // If there's a template keyword before the opening angle bracket, this is a
108  // template parameter, not an argument.
109  Contexts.back().InTemplateArgument =
110  Left->Previous && Left->Previous->Tok.isNot(tok::kw_template);
111 
112  if (Style.Language == FormatStyle::LK_Java &&
113  CurrentToken->is(tok::question))
114  next();
115 
116  while (CurrentToken) {
117  if (CurrentToken->is(tok::greater)) {
118  // Try to do a better job at looking for ">>" within the condition of
119  // a statement. Conservatively insert spaces between consecutive ">"
120  // tokens to prevent splitting right bitshift operators and potentially
121  // altering program semantics. This check is overly conservative and
122  // will prevent spaces from being inserted in select nested template
123  // parameter cases, but should not alter program semantics.
124  if (CurrentToken->Next && CurrentToken->Next->is(tok::greater) &&
125  Left->ParentBracket != tok::less &&
126  (isKeywordWithCondition(*Line.First) ||
127  CurrentToken->getStartOfNonWhitespace() ==
128  CurrentToken->Next->getStartOfNonWhitespace().getLocWithOffset(
129  -1)))
130  return false;
131  Left->MatchingParen = CurrentToken;
132  CurrentToken->MatchingParen = Left;
133  // In TT_Proto, we must distignuish between:
134  // map<key, value>
135  // msg < item: data >
136  // msg: < item: data >
137  // In TT_TextProto, map<key, value> does not occur.
138  if (Style.Language == FormatStyle::LK_TextProto ||
139  (Style.Language == FormatStyle::LK_Proto && Left->Previous &&
140  Left->Previous->isOneOf(TT_SelectorName, TT_DictLiteral)))
141  CurrentToken->setType(TT_DictLiteral);
142  else
143  CurrentToken->setType(TT_TemplateCloser);
144  next();
145  return true;
146  }
147  if (CurrentToken->is(tok::question) &&
148  Style.Language == FormatStyle::LK_Java) {
149  next();
150  continue;
151  }
152  if (CurrentToken->isOneOf(tok::r_paren, tok::r_square, tok::r_brace) ||
153  (CurrentToken->isOneOf(tok::colon, tok::question) && InExprContext &&
154  !Style.isCSharp() && Style.Language != FormatStyle::LK_Proto &&
155  Style.Language != FormatStyle::LK_TextProto))
156  return false;
157  // If a && or || is found and interpreted as a binary operator, this set
158  // of angles is likely part of something like "a < b && c > d". If the
159  // angles are inside an expression, the ||/&& might also be a binary
160  // operator that was misinterpreted because we are parsing template
161  // parameters.
162  // FIXME: This is getting out of hand, write a decent parser.
163  if (CurrentToken->Previous->isOneOf(tok::pipepipe, tok::ampamp) &&
164  CurrentToken->Previous->is(TT_BinaryOperator) &&
165  Contexts[Contexts.size() - 2].IsExpression &&
166  !Line.startsWith(tok::kw_template))
167  return false;
168  updateParameterCount(Left, CurrentToken);
169  if (Style.Language == FormatStyle::LK_Proto) {
170  if (FormatToken *Previous = CurrentToken->getPreviousNonComment()) {
171  if (CurrentToken->is(tok::colon) ||
172  (CurrentToken->isOneOf(tok::l_brace, tok::less) &&
173  Previous->isNot(tok::colon)))
174  Previous->setType(TT_SelectorName);
175  }
176  }
177  if (!consumeToken())
178  return false;
179  }
180  return false;
181  }
182 
183  bool parseUntouchableParens() {
184  while (CurrentToken) {
185  CurrentToken->Finalized = true;
186  switch (CurrentToken->Tok.getKind()) {
187  case tok::l_paren:
188  next();
189  if (!parseUntouchableParens())
190  return false;
191  continue;
192  case tok::r_paren:
193  next();
194  return true;
195  default:
196  // no-op
197  break;
198  }
199  next();
200  }
201  return false;
202  }
203 
204  bool parseParens(bool LookForDecls = false) {
205  if (!CurrentToken)
206  return false;
207  FormatToken *Left = CurrentToken->Previous;
208  assert(Left && "Unknown previous token");
209  FormatToken *PrevNonComment = Left->getPreviousNonComment();
210  Left->ParentBracket = Contexts.back().ContextKind;
211  ScopedContextCreator ContextCreator(*this, tok::l_paren, 1);
212 
213  // FIXME: This is a bit of a hack. Do better.
214  Contexts.back().ColonIsForRangeExpr =
215  Contexts.size() == 2 && Contexts[0].ColonIsForRangeExpr;
216 
217  if (Left->Previous && Left->Previous->is(TT_UntouchableMacroFunc)) {
218  Left->Finalized = true;
219  return parseUntouchableParens();
220  }
221 
222  bool StartsObjCMethodExpr = false;
223  if (FormatToken *MaybeSel = Left->Previous) {
224  // @selector( starts a selector.
225  if (MaybeSel->isObjCAtKeyword(tok::objc_selector) && MaybeSel->Previous &&
226  MaybeSel->Previous->is(tok::at)) {
227  StartsObjCMethodExpr = true;
228  }
229  }
230 
231  if (Left->is(TT_OverloadedOperatorLParen)) {
232  // Find the previous kw_operator token.
233  FormatToken *Prev = Left;
234  while (!Prev->is(tok::kw_operator)) {
235  Prev = Prev->Previous;
236  assert(Prev && "Expect a kw_operator prior to the OperatorLParen!");
237  }
238 
239  // If faced with "a.operator*(argument)" or "a->operator*(argument)",
240  // i.e. the operator is called as a member function,
241  // then the argument must be an expression.
242  bool OperatorCalledAsMemberFunction =
243  Prev->Previous && Prev->Previous->isOneOf(tok::period, tok::arrow);
244  Contexts.back().IsExpression = OperatorCalledAsMemberFunction;
245  } else if (Style.Language == FormatStyle::LK_JavaScript &&
246  (Line.startsWith(Keywords.kw_type, tok::identifier) ||
247  Line.startsWith(tok::kw_export, Keywords.kw_type,
248  tok::identifier))) {
249  // type X = (...);
250  // export type X = (...);
251  Contexts.back().IsExpression = false;
252  } else if (Left->Previous &&
253  (Left->Previous->isOneOf(tok::kw_static_assert, tok::kw_while,
254  tok::l_paren, tok::comma) ||
255  Left->Previous->isIf() ||
256  Left->Previous->is(TT_BinaryOperator))) {
257  // static_assert, if and while usually contain expressions.
258  Contexts.back().IsExpression = true;
259  } else if (Style.Language == FormatStyle::LK_JavaScript && Left->Previous &&
260  (Left->Previous->is(Keywords.kw_function) ||
261  (Left->Previous->endsSequence(tok::identifier,
262  Keywords.kw_function)))) {
263  // function(...) or function f(...)
264  Contexts.back().IsExpression = false;
265  } else if (Style.Language == FormatStyle::LK_JavaScript && Left->Previous &&
266  Left->Previous->is(TT_JsTypeColon)) {
267  // let x: (SomeType);
268  Contexts.back().IsExpression = false;
269  } else if (isLambdaParameterList(Left)) {
270  // This is a parameter list of a lambda expression.
271  Contexts.back().IsExpression = false;
272  } else if (Line.InPPDirective &&
273  (!Left->Previous || !Left->Previous->is(tok::identifier))) {
274  Contexts.back().IsExpression = true;
275  } else if (Contexts[Contexts.size() - 2].CaretFound) {
276  // This is the parameter list of an ObjC block.
277  Contexts.back().IsExpression = false;
278  } else if (Left->Previous && Left->Previous->is(TT_ForEachMacro)) {
279  // The first argument to a foreach macro is a declaration.
280  Contexts.back().IsForEachMacro = true;
281  Contexts.back().IsExpression = false;
282  } else if (Left->Previous && Left->Previous->MatchingParen &&
283  Left->Previous->MatchingParen->is(TT_ObjCBlockLParen)) {
284  Contexts.back().IsExpression = false;
285  } else if (!Line.MustBeDeclaration && !Line.InPPDirective) {
286  bool IsForOrCatch =
287  Left->Previous && Left->Previous->isOneOf(tok::kw_for, tok::kw_catch);
288  Contexts.back().IsExpression = !IsForOrCatch;
289  }
290 
291  // Infer the role of the l_paren based on the previous token if we haven't
292  // detected one one yet.
293  if (PrevNonComment && Left->is(TT_Unknown)) {
294  if (PrevNonComment->is(tok::kw___attribute)) {
295  Left->setType(TT_AttributeParen);
296  } else if (PrevNonComment->isOneOf(TT_TypenameMacro, tok::kw_decltype,
297  tok::kw_typeof, tok::kw__Atomic,
298  tok::kw___underlying_type)) {
299  Left->setType(TT_TypeDeclarationParen);
300  // decltype() and typeof() usually contain expressions.
301  if (PrevNonComment->isOneOf(tok::kw_decltype, tok::kw_typeof))
302  Contexts.back().IsExpression = true;
303  }
304  }
305 
306  if (StartsObjCMethodExpr) {
307  Contexts.back().ColonIsObjCMethodExpr = true;
308  Left->setType(TT_ObjCMethodExpr);
309  }
310 
311  // MightBeFunctionType and ProbablyFunctionType are used for
312  // function pointer and reference types as well as Objective-C
313  // block types:
314  //
315  // void (*FunctionPointer)(void);
316  // void (&FunctionReference)(void);
317  // void (^ObjCBlock)(void);
318  bool MightBeFunctionType = !Contexts[Contexts.size() - 2].IsExpression;
319  bool ProbablyFunctionType =
320  CurrentToken->isOneOf(tok::star, tok::amp, tok::caret);
321  bool HasMultipleLines = false;
322  bool HasMultipleParametersOnALine = false;
323  bool MightBeObjCForRangeLoop =
324  Left->Previous && Left->Previous->is(tok::kw_for);
325  FormatToken *PossibleObjCForInToken = nullptr;
326  while (CurrentToken) {
327  // LookForDecls is set when "if (" has been seen. Check for
328  // 'identifier' '*' 'identifier' followed by not '=' -- this
329  // '*' has to be a binary operator but determineStarAmpUsage() will
330  // categorize it as an unary operator, so set the right type here.
331  if (LookForDecls && CurrentToken->Next) {
332  FormatToken *Prev = CurrentToken->getPreviousNonComment();
333  if (Prev) {
334  FormatToken *PrevPrev = Prev->getPreviousNonComment();
335  FormatToken *Next = CurrentToken->Next;
336  if (PrevPrev && PrevPrev->is(tok::identifier) &&
337  Prev->isOneOf(tok::star, tok::amp, tok::ampamp) &&
338  CurrentToken->is(tok::identifier) && Next->isNot(tok::equal)) {
339  Prev->setType(TT_BinaryOperator);
340  LookForDecls = false;
341  }
342  }
343  }
344 
345  if (CurrentToken->Previous->is(TT_PointerOrReference) &&
346  CurrentToken->Previous->Previous->isOneOf(tok::l_paren,
347  tok::coloncolon))
348  ProbablyFunctionType = true;
349  if (CurrentToken->is(tok::comma))
350  MightBeFunctionType = false;
351  if (CurrentToken->Previous->is(TT_BinaryOperator))
352  Contexts.back().IsExpression = true;
353  if (CurrentToken->is(tok::r_paren)) {
354  if (MightBeFunctionType && ProbablyFunctionType && CurrentToken->Next &&
355  (CurrentToken->Next->is(tok::l_paren) ||
356  (CurrentToken->Next->is(tok::l_square) && Line.MustBeDeclaration)))
357  Left->setType(Left->Next->is(tok::caret) ? TT_ObjCBlockLParen
358  : TT_FunctionTypeLParen);
359  Left->MatchingParen = CurrentToken;
360  CurrentToken->MatchingParen = Left;
361 
362  if (CurrentToken->Next && CurrentToken->Next->is(tok::l_brace) &&
363  Left->Previous && Left->Previous->is(tok::l_paren)) {
364  // Detect the case where macros are used to generate lambdas or
365  // function bodies, e.g.:
366  // auto my_lambda = MACRO((Type *type, int i) { .. body .. });
367  for (FormatToken *Tok = Left; Tok != CurrentToken; Tok = Tok->Next) {
368  if (Tok->is(TT_BinaryOperator) &&
369  Tok->isOneOf(tok::star, tok::amp, tok::ampamp))
370  Tok->setType(TT_PointerOrReference);
371  }
372  }
373 
374  if (StartsObjCMethodExpr) {
375  CurrentToken->setType(TT_ObjCMethodExpr);
376  if (Contexts.back().FirstObjCSelectorName) {
377  Contexts.back().FirstObjCSelectorName->LongestObjCSelectorName =
378  Contexts.back().LongestObjCSelectorName;
379  }
380  }
381 
382  if (Left->is(TT_AttributeParen))
383  CurrentToken->setType(TT_AttributeParen);
384  if (Left->is(TT_TypeDeclarationParen))
385  CurrentToken->setType(TT_TypeDeclarationParen);
386  if (Left->Previous && Left->Previous->is(TT_JavaAnnotation))
387  CurrentToken->setType(TT_JavaAnnotation);
388  if (Left->Previous && Left->Previous->is(TT_LeadingJavaAnnotation))
389  CurrentToken->setType(TT_LeadingJavaAnnotation);
390  if (Left->Previous && Left->Previous->is(TT_AttributeSquare))
391  CurrentToken->setType(TT_AttributeSquare);
392 
393  if (!HasMultipleLines)
394  Left->setPackingKind(PPK_Inconclusive);
395  else if (HasMultipleParametersOnALine)
396  Left->setPackingKind(PPK_BinPacked);
397  else
398  Left->setPackingKind(PPK_OnePerLine);
399 
400  next();
401  return true;
402  }
403  if (CurrentToken->isOneOf(tok::r_square, tok::r_brace))
404  return false;
405 
406  if (CurrentToken->is(tok::l_brace))
407  Left->setType(TT_Unknown); // Not TT_ObjCBlockLParen
408  if (CurrentToken->is(tok::comma) && CurrentToken->Next &&
409  !CurrentToken->Next->HasUnescapedNewline &&
410  !CurrentToken->Next->isTrailingComment())
411  HasMultipleParametersOnALine = true;
412  bool ProbablyFunctionTypeLParen =
413  (CurrentToken->is(tok::l_paren) && CurrentToken->Next &&
414  CurrentToken->Next->isOneOf(tok::star, tok::amp, tok::caret));
415  if ((CurrentToken->Previous->isOneOf(tok::kw_const, tok::kw_auto) ||
416  CurrentToken->Previous->isSimpleTypeSpecifier()) &&
417  !(CurrentToken->is(tok::l_brace) ||
418  (CurrentToken->is(tok::l_paren) && !ProbablyFunctionTypeLParen)))
419  Contexts.back().IsExpression = false;
420  if (CurrentToken->isOneOf(tok::semi, tok::colon)) {
421  MightBeObjCForRangeLoop = false;
422  if (PossibleObjCForInToken) {
423  PossibleObjCForInToken->setType(TT_Unknown);
424  PossibleObjCForInToken = nullptr;
425  }
426  }
427  if (MightBeObjCForRangeLoop && CurrentToken->is(Keywords.kw_in)) {
428  PossibleObjCForInToken = CurrentToken;
429  PossibleObjCForInToken->setType(TT_ObjCForIn);
430  }
431  // When we discover a 'new', we set CanBeExpression to 'false' in order to
432  // parse the type correctly. Reset that after a comma.
433  if (CurrentToken->is(tok::comma))
434  Contexts.back().CanBeExpression = true;
435 
436  FormatToken *Tok = CurrentToken;
437  if (!consumeToken())
438  return false;
439  updateParameterCount(Left, Tok);
440  if (CurrentToken && CurrentToken->HasUnescapedNewline)
441  HasMultipleLines = true;
442  }
443  return false;
444  }
445 
446  bool isCSharpAttributeSpecifier(const FormatToken &Tok) {
447  if (!Style.isCSharp())
448  return false;
449 
450  // `identifier[i]` is not an attribute.
451  if (Tok.Previous && Tok.Previous->is(tok::identifier))
452  return false;
453 
454  // Chains of [] in `identifier[i][j][k]` are not attributes.
455  if (Tok.Previous && Tok.Previous->is(tok::r_square)) {
456  auto *MatchingParen = Tok.Previous->MatchingParen;
457  if (!MatchingParen || MatchingParen->is(TT_ArraySubscriptLSquare))
458  return false;
459  }
460 
461  const FormatToken *AttrTok = Tok.Next;
462  if (!AttrTok)
463  return false;
464 
465  // Just an empty declaration e.g. string [].
466  if (AttrTok->is(tok::r_square))
467  return false;
468 
469  // Move along the tokens inbetween the '[' and ']' e.g. [STAThread].
470  while (AttrTok && AttrTok->isNot(tok::r_square)) {
471  AttrTok = AttrTok->Next;
472  }
473 
474  if (!AttrTok)
475  return false;
476 
477  // Allow an attribute to be the only content of a file.
478  AttrTok = AttrTok->Next;
479  if (!AttrTok)
480  return true;
481 
482  // Limit this to being an access modifier that follows.
483  if (AttrTok->isOneOf(tok::kw_public, tok::kw_private, tok::kw_protected,
484  tok::comment, tok::kw_class, tok::kw_static,
485  tok::l_square, Keywords.kw_internal)) {
486  return true;
487  }
488 
489  // incase its a [XXX] retval func(....
490  if (AttrTok->Next &&
491  AttrTok->Next->startsSequence(tok::identifier, tok::l_paren))
492  return true;
493 
494  return false;
495  }
496 
497  bool isCpp11AttributeSpecifier(const FormatToken &Tok) {
498  if (!Style.isCpp() || !Tok.startsSequence(tok::l_square, tok::l_square))
499  return false;
500  // The first square bracket is part of an ObjC array literal
501  if (Tok.Previous && Tok.Previous->is(tok::at)) {
502  return false;
503  }
504  const FormatToken *AttrTok = Tok.Next->Next;
505  if (!AttrTok)
506  return false;
507  // C++17 '[[using ns: foo, bar(baz, blech)]]'
508  // We assume nobody will name an ObjC variable 'using'.
509  if (AttrTok->startsSequence(tok::kw_using, tok::identifier, tok::colon))
510  return true;
511  if (AttrTok->isNot(tok::identifier))
512  return false;
513  while (AttrTok && !AttrTok->startsSequence(tok::r_square, tok::r_square)) {
514  // ObjC message send. We assume nobody will use : in a C++11 attribute
515  // specifier parameter, although this is technically valid:
516  // [[foo(:)]].
517  if (AttrTok->is(tok::colon) ||
518  AttrTok->startsSequence(tok::identifier, tok::identifier) ||
519  AttrTok->startsSequence(tok::r_paren, tok::identifier))
520  return false;
521  if (AttrTok->is(tok::ellipsis))
522  return true;
523  AttrTok = AttrTok->Next;
524  }
525  return AttrTok && AttrTok->startsSequence(tok::r_square, tok::r_square);
526  }
527 
528  bool parseSquare() {
529  if (!CurrentToken)
530  return false;
531 
532  // A '[' could be an index subscript (after an identifier or after
533  // ')' or ']'), it could be the start of an Objective-C method
534  // expression, it could the start of an Objective-C array literal,
535  // or it could be a C++ attribute specifier [[foo::bar]].
536  FormatToken *Left = CurrentToken->Previous;
537  Left->ParentBracket = Contexts.back().ContextKind;
538  FormatToken *Parent = Left->getPreviousNonComment();
539 
540  // Cases where '>' is followed by '['.
541  // In C++, this can happen either in array of templates (foo<int>[10])
542  // or when array is a nested template type (unique_ptr<type1<type2>[]>).
543  bool CppArrayTemplates =
544  Style.isCpp() && Parent && Parent->is(TT_TemplateCloser) &&
545  (Contexts.back().CanBeExpression || Contexts.back().IsExpression ||
546  Contexts.back().InTemplateArgument);
547 
548  bool IsCpp11AttributeSpecifier = isCpp11AttributeSpecifier(*Left) ||
549  Contexts.back().InCpp11AttributeSpecifier;
550 
551  // Treat C# Attributes [STAThread] much like C++ attributes [[...]].
552  bool IsCSharpAttributeSpecifier =
553  isCSharpAttributeSpecifier(*Left) ||
554  Contexts.back().InCSharpAttributeSpecifier;
555 
556  bool InsideInlineASM = Line.startsWith(tok::kw_asm);
557  bool IsCppStructuredBinding = Left->isCppStructuredBinding(Style);
558  bool StartsObjCMethodExpr =
559  !IsCppStructuredBinding && !InsideInlineASM && !CppArrayTemplates &&
560  Style.isCpp() && !IsCpp11AttributeSpecifier &&
561  !IsCSharpAttributeSpecifier && Contexts.back().CanBeExpression &&
562  Left->isNot(TT_LambdaLSquare) &&
563  !CurrentToken->isOneOf(tok::l_brace, tok::r_square) &&
564  (!Parent ||
565  Parent->isOneOf(tok::colon, tok::l_square, tok::l_paren,
566  tok::kw_return, tok::kw_throw) ||
567  Parent->isUnaryOperator() ||
568  // FIXME(bug 36976): ObjC return types shouldn't use TT_CastRParen.
569  Parent->isOneOf(TT_ObjCForIn, TT_CastRParen) ||
570  (getBinOpPrecedence(Parent->Tok.getKind(), true, true) >
571  prec::Unknown));
572  bool ColonFound = false;
573 
574  unsigned BindingIncrease = 1;
575  if (IsCppStructuredBinding) {
576  Left->setType(TT_StructuredBindingLSquare);
577  } else if (Left->is(TT_Unknown)) {
578  if (StartsObjCMethodExpr) {
579  Left->setType(TT_ObjCMethodExpr);
580  } else if (InsideInlineASM) {
581  Left->setType(TT_InlineASMSymbolicNameLSquare);
582  } else if (IsCpp11AttributeSpecifier) {
583  Left->setType(TT_AttributeSquare);
584  } else if (Style.Language == FormatStyle::LK_JavaScript && Parent &&
585  Contexts.back().ContextKind == tok::l_brace &&
586  Parent->isOneOf(tok::l_brace, tok::comma)) {
587  Left->setType(TT_JsComputedPropertyName);
588  } else if (Style.isCpp() && Contexts.back().ContextKind == tok::l_brace &&
589  Parent && Parent->isOneOf(tok::l_brace, tok::comma)) {
590  Left->setType(TT_DesignatedInitializerLSquare);
591  } else if (IsCSharpAttributeSpecifier) {
592  Left->setType(TT_AttributeSquare);
593  } else if (CurrentToken->is(tok::r_square) && Parent &&
594  Parent->is(TT_TemplateCloser)) {
595  Left->setType(TT_ArraySubscriptLSquare);
596  } else if (Style.Language == FormatStyle::LK_Proto ||
597  Style.Language == FormatStyle::LK_TextProto) {
598  // Square braces in LK_Proto can either be message field attributes:
599  //
600  // optional Aaa aaa = 1 [
601  // (aaa) = aaa
602  // ];
603  //
604  // extensions 123 [
605  // (aaa) = aaa
606  // ];
607  //
608  // or text proto extensions (in options):
609  //
610  // option (Aaa.options) = {
611  // [type.type/type] {
612  // key: value
613  // }
614  // }
615  //
616  // or repeated fields (in options):
617  //
618  // option (Aaa.options) = {
619  // keys: [ 1, 2, 3 ]
620  // }
621  //
622  // In the first and the third case we want to spread the contents inside
623  // the square braces; in the second we want to keep them inline.
624  Left->setType(TT_ArrayInitializerLSquare);
625  if (!Left->endsSequence(tok::l_square, tok::numeric_constant,
626  tok::equal) &&
627  !Left->endsSequence(tok::l_square, tok::numeric_constant,
628  tok::identifier) &&
629  !Left->endsSequence(tok::l_square, tok::colon, TT_SelectorName)) {
630  Left->setType(TT_ProtoExtensionLSquare);
631  BindingIncrease = 10;
632  }
633  } else if (!CppArrayTemplates && Parent &&
634  Parent->isOneOf(TT_BinaryOperator, TT_TemplateCloser, tok::at,
635  tok::comma, tok::l_paren, tok::l_square,
636  tok::question, tok::colon, tok::kw_return,
637  // Should only be relevant to JavaScript:
638  tok::kw_default)) {
639  Left->setType(TT_ArrayInitializerLSquare);
640  } else {
641  BindingIncrease = 10;
642  Left->setType(TT_ArraySubscriptLSquare);
643  }
644  }
645 
646  ScopedContextCreator ContextCreator(*this, tok::l_square, BindingIncrease);
647  Contexts.back().IsExpression = true;
648  if (Style.Language == FormatStyle::LK_JavaScript && Parent &&
649  Parent->is(TT_JsTypeColon))
650  Contexts.back().IsExpression = false;
651 
652  Contexts.back().ColonIsObjCMethodExpr = StartsObjCMethodExpr;
653  Contexts.back().InCpp11AttributeSpecifier = IsCpp11AttributeSpecifier;
654  Contexts.back().InCSharpAttributeSpecifier = IsCSharpAttributeSpecifier;
655 
656  while (CurrentToken) {
657  if (CurrentToken->is(tok::r_square)) {
658  if (IsCpp11AttributeSpecifier)
659  CurrentToken->setType(TT_AttributeSquare);
660  if (IsCSharpAttributeSpecifier)
661  CurrentToken->setType(TT_AttributeSquare);
662  else if (((CurrentToken->Next &&
663  CurrentToken->Next->is(tok::l_paren)) ||
664  (CurrentToken->Previous &&
665  CurrentToken->Previous->Previous == Left)) &&
666  Left->is(TT_ObjCMethodExpr)) {
667  // An ObjC method call is rarely followed by an open parenthesis. It
668  // also can't be composed of just one token, unless it's a macro that
669  // will be expanded to more tokens.
670  // FIXME: Do we incorrectly label ":" with this?
671  StartsObjCMethodExpr = false;
672  Left->setType(TT_Unknown);
673  }
674  if (StartsObjCMethodExpr && CurrentToken->Previous != Left) {
675  CurrentToken->setType(TT_ObjCMethodExpr);
676  // If we haven't seen a colon yet, make sure the last identifier
677  // before the r_square is tagged as a selector name component.
678  if (!ColonFound && CurrentToken->Previous &&
679  CurrentToken->Previous->is(TT_Unknown) &&
680  canBeObjCSelectorComponent(*CurrentToken->Previous))
681  CurrentToken->Previous->setType(TT_SelectorName);
682  // determineStarAmpUsage() thinks that '*' '[' is allocating an
683  // array of pointers, but if '[' starts a selector then '*' is a
684  // binary operator.
685  if (Parent && Parent->is(TT_PointerOrReference))
686  Parent->setType(TT_BinaryOperator);
687  }
688  // An arrow after an ObjC method expression is not a lambda arrow.
689  if (CurrentToken->getType() == TT_ObjCMethodExpr &&
690  CurrentToken->Next && CurrentToken->Next->is(TT_LambdaArrow))
691  CurrentToken->Next->setType(TT_Unknown);
692  Left->MatchingParen = CurrentToken;
693  CurrentToken->MatchingParen = Left;
694  // FirstObjCSelectorName is set when a colon is found. This does
695  // not work, however, when the method has no parameters.
696  // Here, we set FirstObjCSelectorName when the end of the method call is
697  // reached, in case it was not set already.
698  if (!Contexts.back().FirstObjCSelectorName) {
699  FormatToken *Previous = CurrentToken->getPreviousNonComment();
700  if (Previous && Previous->is(TT_SelectorName)) {
701  Previous->ObjCSelectorNameParts = 1;
702  Contexts.back().FirstObjCSelectorName = Previous;
703  }
704  } else {
705  Left->ParameterCount =
706  Contexts.back().FirstObjCSelectorName->ObjCSelectorNameParts;
707  }
708  if (Contexts.back().FirstObjCSelectorName) {
709  Contexts.back().FirstObjCSelectorName->LongestObjCSelectorName =
710  Contexts.back().LongestObjCSelectorName;
711  if (Left->BlockParameterCount > 1)
712  Contexts.back().FirstObjCSelectorName->LongestObjCSelectorName = 0;
713  }
714  next();
715  return true;
716  }
717  if (CurrentToken->isOneOf(tok::r_paren, tok::r_brace))
718  return false;
719  if (CurrentToken->is(tok::colon)) {
720  if (IsCpp11AttributeSpecifier &&
721  CurrentToken->endsSequence(tok::colon, tok::identifier,
722  tok::kw_using)) {
723  // Remember that this is a [[using ns: foo]] C++ attribute, so we
724  // don't add a space before the colon (unlike other colons).
725  CurrentToken->setType(TT_AttributeColon);
726  } else if (Left->isOneOf(TT_ArraySubscriptLSquare,
727  TT_DesignatedInitializerLSquare)) {
728  Left->setType(TT_ObjCMethodExpr);
729  StartsObjCMethodExpr = true;
730  Contexts.back().ColonIsObjCMethodExpr = true;
731  if (Parent && Parent->is(tok::r_paren))
732  // FIXME(bug 36976): ObjC return types shouldn't use TT_CastRParen.
733  Parent->setType(TT_CastRParen);
734  }
735  ColonFound = true;
736  }
737  if (CurrentToken->is(tok::comma) && Left->is(TT_ObjCMethodExpr) &&
738  !ColonFound)
739  Left->setType(TT_ArrayInitializerLSquare);
740  FormatToken *Tok = CurrentToken;
741  if (!consumeToken())
742  return false;
743  updateParameterCount(Left, Tok);
744  }
745  return false;
746  }
747 
748  bool couldBeInStructArrayInitializer() const {
749  if (Contexts.size() < 2)
750  return false;
751  // We want to back up no more then 2 context levels i.e.
752  // . { { <-
753  const auto End = std::next(Contexts.rbegin(), 2);
754  auto Last = Contexts.rbegin();
755  unsigned Depth = 0;
756  for (; Last != End; ++Last) {
757  if (Last->ContextKind == tok::l_brace)
758  ++Depth;
759  }
760  return Depth == 2 && Last->ContextKind != tok::l_brace;
761  }
762 
763  bool parseBrace() {
764  if (CurrentToken) {
765  FormatToken *Left = CurrentToken->Previous;
766  Left->ParentBracket = Contexts.back().ContextKind;
767 
768  if (Contexts.back().CaretFound)
769  Left->setType(TT_ObjCBlockLBrace);
770  Contexts.back().CaretFound = false;
771 
772  ScopedContextCreator ContextCreator(*this, tok::l_brace, 1);
773  Contexts.back().ColonIsDictLiteral = true;
774  if (Left->is(BK_BracedInit))
775  Contexts.back().IsExpression = true;
776  if (Style.Language == FormatStyle::LK_JavaScript && Left->Previous &&
777  Left->Previous->is(TT_JsTypeColon))
778  Contexts.back().IsExpression = false;
779 
780  unsigned CommaCount = 0;
781  while (CurrentToken) {
782  if (CurrentToken->is(tok::r_brace)) {
783  Left->MatchingParen = CurrentToken;
784  CurrentToken->MatchingParen = Left;
785  if (Style.AlignArrayOfStructures != FormatStyle::AIAS_None) {
786  if (Left->ParentBracket == tok::l_brace &&
787  couldBeInStructArrayInitializer() && CommaCount > 0) {
788  Contexts.back().InStructArrayInitializer = true;
789  }
790  }
791  next();
792  return true;
793  }
794  if (CurrentToken->isOneOf(tok::r_paren, tok::r_square))
795  return false;
796  updateParameterCount(Left, CurrentToken);
797  if (CurrentToken->isOneOf(tok::colon, tok::l_brace, tok::less)) {
798  FormatToken *Previous = CurrentToken->getPreviousNonComment();
799  if (Previous->is(TT_JsTypeOptionalQuestion))
800  Previous = Previous->getPreviousNonComment();
801  if ((CurrentToken->is(tok::colon) &&
802  (!Contexts.back().ColonIsDictLiteral || !Style.isCpp())) ||
803  Style.Language == FormatStyle::LK_Proto ||
804  Style.Language == FormatStyle::LK_TextProto) {
805  Left->setType(TT_DictLiteral);
806  if (Previous->Tok.getIdentifierInfo() ||
807  Previous->is(tok::string_literal))
808  Previous->setType(TT_SelectorName);
809  }
810  if (CurrentToken->is(tok::colon) ||
811  Style.Language == FormatStyle::LK_JavaScript)
812  Left->setType(TT_DictLiteral);
813  }
814  if (CurrentToken->is(tok::comma)) {
815  if (Style.Language == FormatStyle::LK_JavaScript)
816  Left->setType(TT_DictLiteral);
817  ++CommaCount;
818  }
819  if (!consumeToken())
820  return false;
821  }
822  }
823  return true;
824  }
825 
826  void updateParameterCount(FormatToken *Left, FormatToken *Current) {
827  // For ObjC methods, the number of parameters is calculated differently as
828  // method declarations have a different structure (the parameters are not
829  // inside a bracket scope).
830  if (Current->is(tok::l_brace) && Current->is(BK_Block))
831  ++Left->BlockParameterCount;
832  if (Current->is(tok::comma)) {
833  ++Left->ParameterCount;
834  if (!Left->Role)
835  Left->Role.reset(new CommaSeparatedList(Style));
836  Left->Role->CommaFound(Current);
837  } else if (Left->ParameterCount == 0 && Current->isNot(tok::comment)) {
838  Left->ParameterCount = 1;
839  }
840  }
841 
842  bool parseConditional() {
843  while (CurrentToken) {
844  if (CurrentToken->is(tok::colon)) {
845  CurrentToken->setType(TT_ConditionalExpr);
846  next();
847  return true;
848  }
849  if (!consumeToken())
850  return false;
851  }
852  return false;
853  }
854 
855  bool parseTemplateDeclaration() {
856  if (CurrentToken && CurrentToken->is(tok::less)) {
857  CurrentToken->setType(TT_TemplateOpener);
858  next();
859  if (!parseAngle())
860  return false;
861  if (CurrentToken)
862  CurrentToken->Previous->ClosesTemplateDeclaration = true;
863  return true;
864  }
865  return false;
866  }
867 
868  bool consumeToken() {
869  FormatToken *Tok = CurrentToken;
870  next();
871  switch (Tok->Tok.getKind()) {
872  case tok::plus:
873  case tok::minus:
874  if (!Tok->Previous && Line.MustBeDeclaration)
875  Tok->setType(TT_ObjCMethodSpecifier);
876  break;
877  case tok::colon:
878  if (!Tok->Previous)
879  return false;
880  // Colons from ?: are handled in parseConditional().
881  if (Style.Language == FormatStyle::LK_JavaScript) {
882  if (Contexts.back().ColonIsForRangeExpr || // colon in for loop
883  (Contexts.size() == 1 && // switch/case labels
884  !Line.First->isOneOf(tok::kw_enum, tok::kw_case)) ||
885  Contexts.back().ContextKind == tok::l_paren || // function params
886  Contexts.back().ContextKind == tok::l_square || // array type
887  (!Contexts.back().IsExpression &&
888  Contexts.back().ContextKind == tok::l_brace) || // object type
889  (Contexts.size() == 1 &&
890  Line.MustBeDeclaration)) { // method/property declaration
891  Contexts.back().IsExpression = false;
892  Tok->setType(TT_JsTypeColon);
893  break;
894  }
895  } else if (Style.isCSharp()) {
896  if (Contexts.back().InCSharpAttributeSpecifier) {
897  Tok->setType(TT_AttributeColon);
898  break;
899  }
900  if (Contexts.back().ContextKind == tok::l_paren) {
901  Tok->setType(TT_CSharpNamedArgumentColon);
902  break;
903  }
904  }
905  if (Contexts.back().ColonIsDictLiteral ||
906  Style.Language == FormatStyle::LK_Proto ||
907  Style.Language == FormatStyle::LK_TextProto) {
908  Tok->setType(TT_DictLiteral);
909  if (Style.Language == FormatStyle::LK_TextProto) {
910  if (FormatToken *Previous = Tok->getPreviousNonComment())
911  Previous->setType(TT_SelectorName);
912  }
913  } else if (Contexts.back().ColonIsObjCMethodExpr ||
914  Line.startsWith(TT_ObjCMethodSpecifier)) {
915  Tok->setType(TT_ObjCMethodExpr);
916  const FormatToken *BeforePrevious = Tok->Previous->Previous;
917  // Ensure we tag all identifiers in method declarations as
918  // TT_SelectorName.
919  bool UnknownIdentifierInMethodDeclaration =
920  Line.startsWith(TT_ObjCMethodSpecifier) &&
921  Tok->Previous->is(tok::identifier) && Tok->Previous->is(TT_Unknown);
922  if (!BeforePrevious ||
923  // FIXME(bug 36976): ObjC return types shouldn't use TT_CastRParen.
924  !(BeforePrevious->is(TT_CastRParen) ||
925  (BeforePrevious->is(TT_ObjCMethodExpr) &&
926  BeforePrevious->is(tok::colon))) ||
927  BeforePrevious->is(tok::r_square) ||
928  Contexts.back().LongestObjCSelectorName == 0 ||
929  UnknownIdentifierInMethodDeclaration) {
930  Tok->Previous->setType(TT_SelectorName);
931  if (!Contexts.back().FirstObjCSelectorName)
932  Contexts.back().FirstObjCSelectorName = Tok->Previous;
933  else if (Tok->Previous->ColumnWidth >
934  Contexts.back().LongestObjCSelectorName)
935  Contexts.back().LongestObjCSelectorName =
936  Tok->Previous->ColumnWidth;
937  Tok->Previous->ParameterIndex =
938  Contexts.back().FirstObjCSelectorName->ObjCSelectorNameParts;
939  ++Contexts.back().FirstObjCSelectorName->ObjCSelectorNameParts;
940  }
941  } else if (Contexts.back().ColonIsForRangeExpr) {
942  Tok->setType(TT_RangeBasedForLoopColon);
943  } else if (CurrentToken && CurrentToken->is(tok::numeric_constant)) {
944  Tok->setType(TT_BitFieldColon);
945  } else if (Contexts.size() == 1 &&
946  !Line.First->isOneOf(tok::kw_enum, tok::kw_case,
947  tok::kw_default)) {
948  FormatToken *Prev = Tok->getPreviousNonComment();
949  if (Prev->isOneOf(tok::r_paren, tok::kw_noexcept))
950  Tok->setType(TT_CtorInitializerColon);
951  else if (Prev->is(tok::kw_try)) {
952  // Member initializer list within function try block.
953  FormatToken *PrevPrev = Prev->getPreviousNonComment();
954  if (PrevPrev && PrevPrev->isOneOf(tok::r_paren, tok::kw_noexcept))
955  Tok->setType(TT_CtorInitializerColon);
956  } else
957  Tok->setType(TT_InheritanceColon);
958  } else if (canBeObjCSelectorComponent(*Tok->Previous) && Tok->Next &&
959  (Tok->Next->isOneOf(tok::r_paren, tok::comma) ||
960  (canBeObjCSelectorComponent(*Tok->Next) && Tok->Next->Next &&
961  Tok->Next->Next->is(tok::colon)))) {
962  // This handles a special macro in ObjC code where selectors including
963  // the colon are passed as macro arguments.
964  Tok->setType(TT_ObjCMethodExpr);
965  } else if (Contexts.back().ContextKind == tok::l_paren) {
966  Tok->setType(TT_InlineASMColon);
967  }
968  break;
969  case tok::pipe:
970  case tok::amp:
971  // | and & in declarations/type expressions represent union and
972  // intersection types, respectively.
973  if (Style.Language == FormatStyle::LK_JavaScript &&
974  !Contexts.back().IsExpression)
975  Tok->setType(TT_JsTypeOperator);
976  break;
977  case tok::kw_if:
978  case tok::kw_while:
979  if (Tok->is(tok::kw_if) && CurrentToken &&
980  CurrentToken->isOneOf(tok::kw_constexpr, tok::identifier))
981  next();
982  if (CurrentToken && CurrentToken->is(tok::l_paren)) {
983  next();
984  if (!parseParens(/*LookForDecls=*/true))
985  return false;
986  }
987  break;
988  case tok::kw_for:
989  if (Style.Language == FormatStyle::LK_JavaScript) {
990  // x.for and {for: ...}
991  if ((Tok->Previous && Tok->Previous->is(tok::period)) ||
992  (Tok->Next && Tok->Next->is(tok::colon)))
993  break;
994  // JS' for await ( ...
995  if (CurrentToken && CurrentToken->is(Keywords.kw_await))
996  next();
997  }
998  Contexts.back().ColonIsForRangeExpr = true;
999  next();
1000  if (!parseParens())
1001  return false;
1002  break;
1003  case tok::l_paren:
1004  // When faced with 'operator()()', the kw_operator handler incorrectly
1005  // marks the first l_paren as a OverloadedOperatorLParen. Here, we make
1006  // the first two parens OverloadedOperators and the second l_paren an
1007  // OverloadedOperatorLParen.
1008  if (Tok->Previous && Tok->Previous->is(tok::r_paren) &&
1009  Tok->Previous->MatchingParen &&
1010  Tok->Previous->MatchingParen->is(TT_OverloadedOperatorLParen)) {
1011  Tok->Previous->setType(TT_OverloadedOperator);
1012  Tok->Previous->MatchingParen->setType(TT_OverloadedOperator);
1013  Tok->setType(TT_OverloadedOperatorLParen);
1014  }
1015 
1016  if (!parseParens())
1017  return false;
1018  if (Line.MustBeDeclaration && Contexts.size() == 1 &&
1019  !Contexts.back().IsExpression && !Line.startsWith(TT_ObjCProperty) &&
1020  !Tok->is(TT_TypeDeclarationParen) &&
1021  (!Tok->Previous || !Tok->Previous->isOneOf(tok::kw___attribute,
1022  TT_LeadingJavaAnnotation)))
1023  Line.MightBeFunctionDecl = true;
1024  break;
1025  case tok::l_square:
1026  if (!parseSquare())
1027  return false;
1028  break;
1029  case tok::l_brace:
1030  if (Style.Language == FormatStyle::LK_TextProto) {
1031  FormatToken *Previous = Tok->getPreviousNonComment();
1032  if (Previous && Previous->getType() != TT_DictLiteral)
1033  Previous->setType(TT_SelectorName);
1034  }
1035  if (!parseBrace())
1036  return false;
1037  break;
1038  case tok::less:
1039  if (parseAngle()) {
1040  Tok->setType(TT_TemplateOpener);
1041  // In TT_Proto, we must distignuish between:
1042  // map<key, value>
1043  // msg < item: data >
1044  // msg: < item: data >
1045  // In TT_TextProto, map<key, value> does not occur.
1046  if (Style.Language == FormatStyle::LK_TextProto ||
1047  (Style.Language == FormatStyle::LK_Proto && Tok->Previous &&
1048  Tok->Previous->isOneOf(TT_SelectorName, TT_DictLiteral))) {
1049  Tok->setType(TT_DictLiteral);
1050  FormatToken *Previous = Tok->getPreviousNonComment();
1051  if (Previous && Previous->getType() != TT_DictLiteral)
1052  Previous->setType(TT_SelectorName);
1053  }
1054  } else {
1055  Tok->setType(TT_BinaryOperator);
1056  NonTemplateLess.insert(Tok);
1057  CurrentToken = Tok;
1058  next();
1059  }
1060  break;
1061  case tok::r_paren:
1062  case tok::r_square:
1063  return false;
1064  case tok::r_brace:
1065  // Lines can start with '}'.
1066  if (Tok->Previous)
1067  return false;
1068  break;
1069  case tok::greater:
1070  if (Style.Language != FormatStyle::LK_TextProto)
1071  Tok->setType(TT_BinaryOperator);
1072  if (Tok->Previous && Tok->Previous->is(TT_TemplateCloser))
1073  Tok->SpacesRequiredBefore = 1;
1074  break;
1075  case tok::kw_operator:
1076  if (Style.Language == FormatStyle::LK_TextProto ||
1077  Style.Language == FormatStyle::LK_Proto)
1078  break;
1079  while (CurrentToken &&
1080  !CurrentToken->isOneOf(tok::l_paren, tok::semi, tok::r_paren)) {
1081  if (CurrentToken->isOneOf(tok::star, tok::amp))
1082  CurrentToken->setType(TT_PointerOrReference);
1083  consumeToken();
1084  if (CurrentToken && CurrentToken->is(tok::comma) &&
1085  CurrentToken->Previous->isNot(tok::kw_operator))
1086  break;
1087  if (CurrentToken && CurrentToken->Previous->isOneOf(
1088  TT_BinaryOperator, TT_UnaryOperator, tok::comma,
1089  tok::star, tok::arrow, tok::amp, tok::ampamp))
1090  CurrentToken->Previous->setType(TT_OverloadedOperator);
1091  }
1092  if (CurrentToken && CurrentToken->is(tok::l_paren))
1093  CurrentToken->setType(TT_OverloadedOperatorLParen);
1094  if (CurrentToken && CurrentToken->Previous->is(TT_BinaryOperator))
1095  CurrentToken->Previous->setType(TT_OverloadedOperator);
1096  break;
1097  case tok::question:
1098  if (Style.Language == FormatStyle::LK_JavaScript && Tok->Next &&
1099  Tok->Next->isOneOf(tok::semi, tok::comma, tok::colon, tok::r_paren,
1100  tok::r_brace)) {
1101  // Question marks before semicolons, colons, etc. indicate optional
1102  // types (fields, parameters), e.g.
1103  // function(x?: string, y?) {...}
1104  // class X { y?; }
1105  Tok->setType(TT_JsTypeOptionalQuestion);
1106  break;
1107  }
1108  // Declarations cannot be conditional expressions, this can only be part
1109  // of a type declaration.
1110  if (Line.MustBeDeclaration && !Contexts.back().IsExpression &&
1111  Style.Language == FormatStyle::LK_JavaScript)
1112  break;
1113  if (Style.isCSharp()) {
1114  // `Type?)`, `Type?>`, `Type? name;` and `Type? name =` can only be
1115  // nullable types.
1116  // Line.MustBeDeclaration will be true for `Type? name;`.
1117  if ((!Contexts.back().IsExpression && Line.MustBeDeclaration) ||
1118  (Tok->Next && Tok->Next->isOneOf(tok::r_paren, tok::greater)) ||
1119  (Tok->Next && Tok->Next->is(tok::identifier) && Tok->Next->Next &&
1120  Tok->Next->Next->is(tok::equal))) {
1121  Tok->setType(TT_CSharpNullable);
1122  break;
1123  }
1124  }
1125  parseConditional();
1126  break;
1127  case tok::kw_template:
1128  parseTemplateDeclaration();
1129  break;
1130  case tok::comma:
1131  if (Contexts.back().InCtorInitializer)
1132  Tok->setType(TT_CtorInitializerComma);
1133  else if (Contexts.back().InInheritanceList)
1134  Tok->setType(TT_InheritanceComma);
1135  else if (Contexts.back().FirstStartOfName &&
1136  (Contexts.size() == 1 || Line.startsWith(tok::kw_for))) {
1137  Contexts.back().FirstStartOfName->PartOfMultiVariableDeclStmt = true;
1138  Line.IsMultiVariableDeclStmt = true;
1139  }
1140  if (Contexts.back().IsForEachMacro)
1141  Contexts.back().IsExpression = true;
1142  break;
1143  case tok::identifier:
1144  if (Tok->isOneOf(Keywords.kw___has_include,
1145  Keywords.kw___has_include_next)) {
1146  parseHasInclude();
1147  }
1148  if (Style.isCSharp() && Tok->is(Keywords.kw_where) && Tok->Next &&
1149  Tok->Next->isNot(tok::l_paren)) {
1150  Tok->setType(TT_CSharpGenericTypeConstraint);
1151  parseCSharpGenericTypeConstraint();
1152  }
1153  break;
1154  default:
1155  break;
1156  }
1157  return true;
1158  }
1159 
1160  void parseCSharpGenericTypeConstraint() {
1161  int OpenAngleBracketsCount = 0;
1162  while (CurrentToken) {
1163  if (CurrentToken->is(tok::less)) {
1164  // parseAngle is too greedy and will consume the whole line.
1165  CurrentToken->setType(TT_TemplateOpener);
1166  ++OpenAngleBracketsCount;
1167  next();
1168  } else if (CurrentToken->is(tok::greater)) {
1169  CurrentToken->setType(TT_TemplateCloser);
1170  --OpenAngleBracketsCount;
1171  next();
1172  } else if (CurrentToken->is(tok::comma) && OpenAngleBracketsCount == 0) {
1173  // We allow line breaks after GenericTypeConstraintComma's
1174  // so do not flag commas in Generics as GenericTypeConstraintComma's.
1175  CurrentToken->setType(TT_CSharpGenericTypeConstraintComma);
1176  next();
1177  } else if (CurrentToken->is(Keywords.kw_where)) {
1178  CurrentToken->setType(TT_CSharpGenericTypeConstraint);
1179  next();
1180  } else if (CurrentToken->is(tok::colon)) {
1181  CurrentToken->setType(TT_CSharpGenericTypeConstraintColon);
1182  next();
1183  } else {
1184  next();
1185  }
1186  }
1187  }
1188 
1189  void parseIncludeDirective() {
1190  if (CurrentToken && CurrentToken->is(tok::less)) {
1191  next();
1192  while (CurrentToken) {
1193  // Mark tokens up to the trailing line comments as implicit string
1194  // literals.
1195  if (CurrentToken->isNot(tok::comment) &&
1196  !CurrentToken->TokenText.startswith("//"))
1197  CurrentToken->setType(TT_ImplicitStringLiteral);
1198  next();
1199  }
1200  }
1201  }
1202 
1203  void parseWarningOrError() {
1204  next();
1205  // We still want to format the whitespace left of the first token of the
1206  // warning or error.
1207  next();
1208  while (CurrentToken) {
1209  CurrentToken->setType(TT_ImplicitStringLiteral);
1210  next();
1211  }
1212  }
1213 
1214  void parsePragma() {
1215  next(); // Consume "pragma".
1216  if (CurrentToken &&
1217  CurrentToken->isOneOf(Keywords.kw_mark, Keywords.kw_option)) {
1218  bool IsMark = CurrentToken->is(Keywords.kw_mark);
1219  next(); // Consume "mark".
1220  next(); // Consume first token (so we fix leading whitespace).
1221  while (CurrentToken) {
1222  if (IsMark || CurrentToken->Previous->is(TT_BinaryOperator))
1223  CurrentToken->setType(TT_ImplicitStringLiteral);
1224  next();
1225  }
1226  }
1227  }
1228 
1229  void parseHasInclude() {
1230  if (!CurrentToken || !CurrentToken->is(tok::l_paren))
1231  return;
1232  next(); // '('
1233  parseIncludeDirective();
1234  next(); // ')'
1235  }
1236 
1237  LineType parsePreprocessorDirective() {
1238  bool IsFirstToken = CurrentToken->IsFirst;
1240  next();
1241  if (!CurrentToken)
1242  return Type;
1243 
1244  if (Style.Language == FormatStyle::LK_JavaScript && IsFirstToken) {
1245  // JavaScript files can contain shebang lines of the form:
1246  // #!/usr/bin/env node
1247  // Treat these like C++ #include directives.
1248  while (CurrentToken) {
1249  // Tokens cannot be comments here.
1250  CurrentToken->setType(TT_ImplicitStringLiteral);
1251  next();
1252  }
1253  return LT_ImportStatement;
1254  }
1255 
1256  if (CurrentToken->Tok.is(tok::numeric_constant)) {
1257  CurrentToken->SpacesRequiredBefore = 1;
1258  return Type;
1259  }
1260  // Hashes in the middle of a line can lead to any strange token
1261  // sequence.
1262  if (!CurrentToken->Tok.getIdentifierInfo())
1263  return Type;
1264  switch (CurrentToken->Tok.getIdentifierInfo()->getPPKeywordID()) {
1265  case tok::pp_include:
1266  case tok::pp_include_next:
1267  case tok::pp_import:
1268  next();
1269  parseIncludeDirective();
1271  break;
1272  case tok::pp_error:
1273  case tok::pp_warning:
1274  parseWarningOrError();
1275  break;
1276  case tok::pp_pragma:
1277  parsePragma();
1278  break;
1279  case tok::pp_if:
1280  case tok::pp_elif:
1281  Contexts.back().IsExpression = true;
1282  next();
1283  parseLine();
1284  break;
1285  default:
1286  break;
1287  }
1288  while (CurrentToken) {
1289  FormatToken *Tok = CurrentToken;
1290  next();
1291  if (Tok->is(tok::l_paren))
1292  parseParens();
1293  else if (Tok->isOneOf(Keywords.kw___has_include,
1294  Keywords.kw___has_include_next))
1295  parseHasInclude();
1296  }
1297  return Type;
1298  }
1299 
1300 public:
1301  LineType parseLine() {
1302  if (!CurrentToken)
1303  return LT_Invalid;
1304  NonTemplateLess.clear();
1305  if (CurrentToken->is(tok::hash))
1306  return parsePreprocessorDirective();
1307 
1308  // Directly allow to 'import <string-literal>' to support protocol buffer
1309  // definitions (github.com/google/protobuf) or missing "#" (either way we
1310  // should not break the line).
1311  IdentifierInfo *Info = CurrentToken->Tok.getIdentifierInfo();
1312  if ((Style.Language == FormatStyle::LK_Java &&
1313  CurrentToken->is(Keywords.kw_package)) ||
1314  (Info && Info->getPPKeywordID() == tok::pp_import &&
1315  CurrentToken->Next &&
1316  CurrentToken->Next->isOneOf(tok::string_literal, tok::identifier,
1317  tok::kw_static))) {
1318  next();
1319  parseIncludeDirective();
1320  return LT_ImportStatement;
1321  }
1322 
1323  // If this line starts and ends in '<' and '>', respectively, it is likely
1324  // part of "#define <a/b.h>".
1325  if (CurrentToken->is(tok::less) && Line.Last->is(tok::greater)) {
1326  parseIncludeDirective();
1327  return LT_ImportStatement;
1328  }
1329 
1330  // In .proto files, top-level options and package statements are very
1331  // similar to import statements and should not be line-wrapped.
1332  if (Style.Language == FormatStyle::LK_Proto && Line.Level == 0 &&
1333  CurrentToken->isOneOf(Keywords.kw_option, Keywords.kw_package)) {
1334  next();
1335  if (CurrentToken && CurrentToken->is(tok::identifier)) {
1336  while (CurrentToken)
1337  next();
1338  return LT_ImportStatement;
1339  }
1340  }
1341 
1342  bool KeywordVirtualFound = false;
1343  bool ImportStatement = false;
1344 
1345  // import {...} from '...';
1346  if (Style.Language == FormatStyle::LK_JavaScript &&
1347  CurrentToken->is(Keywords.kw_import))
1348  ImportStatement = true;
1349 
1350  while (CurrentToken) {
1351  if (CurrentToken->is(tok::kw_virtual))
1352  KeywordVirtualFound = true;
1353  if (Style.Language == FormatStyle::LK_JavaScript) {
1354  // export {...} from '...';
1355  // An export followed by "from 'some string';" is a re-export from
1356  // another module identified by a URI and is treated as a
1357  // LT_ImportStatement (i.e. prevent wraps on it for long URIs).
1358  // Just "export {...};" or "export class ..." should not be treated as
1359  // an import in this sense.
1360  if (Line.First->is(tok::kw_export) &&
1361  CurrentToken->is(Keywords.kw_from) && CurrentToken->Next &&
1362  CurrentToken->Next->isStringLiteral())
1363  ImportStatement = true;
1364  if (isClosureImportStatement(*CurrentToken))
1365  ImportStatement = true;
1366  }
1367  if (!consumeToken())
1368  return LT_Invalid;
1369  }
1370  if (KeywordVirtualFound)
1371  return LT_VirtualFunctionDecl;
1372  if (ImportStatement)
1373  return LT_ImportStatement;
1374 
1375  if (Line.startsWith(TT_ObjCMethodSpecifier)) {
1376  if (Contexts.back().FirstObjCSelectorName)
1377  Contexts.back().FirstObjCSelectorName->LongestObjCSelectorName =
1378  Contexts.back().LongestObjCSelectorName;
1379  return LT_ObjCMethodDecl;
1380  }
1381 
1382  for (const auto &ctx : Contexts) {
1383  if (ctx.InStructArrayInitializer) {
1385  }
1386  }
1387 
1388  return LT_Other;
1389  }
1390 
1391 private:
1392  bool isClosureImportStatement(const FormatToken &Tok) {
1393  // FIXME: Closure-library specific stuff should not be hard-coded but be
1394  // configurable.
1395  return Tok.TokenText == "goog" && Tok.Next && Tok.Next->is(tok::period) &&
1396  Tok.Next->Next &&
1397  (Tok.Next->Next->TokenText == "module" ||
1398  Tok.Next->Next->TokenText == "provide" ||
1399  Tok.Next->Next->TokenText == "require" ||
1400  Tok.Next->Next->TokenText == "requireType" ||
1401  Tok.Next->Next->TokenText == "forwardDeclare") &&
1402  Tok.Next->Next->Next && Tok.Next->Next->Next->is(tok::l_paren);
1403  }
1404 
1405  void resetTokenMetadata(FormatToken *Token) {
1406  if (!Token)
1407  return;
1408 
1409  // Reset token type in case we have already looked at it and then
1410  // recovered from an error (e.g. failure to find the matching >).
1411  if (!CurrentToken->isOneOf(
1412  TT_LambdaLSquare, TT_LambdaLBrace, TT_AttributeMacro, TT_IfMacro,
1413  TT_ForEachMacro, TT_TypenameMacro, TT_FunctionLBrace,
1414  TT_ImplicitStringLiteral, TT_InlineASMBrace, TT_FatArrow,
1415  TT_LambdaArrow, TT_NamespaceMacro, TT_OverloadedOperator,
1416  TT_RegexLiteral, TT_TemplateString, TT_ObjCStringLiteral,
1417  TT_UntouchableMacroFunc, TT_ConstraintJunctions,
1418  TT_StatementAttributeLikeMacro))
1419  CurrentToken->setType(TT_Unknown);
1420  CurrentToken->Role.reset();
1421  CurrentToken->MatchingParen = nullptr;
1422  CurrentToken->FakeLParens.clear();
1423  CurrentToken->FakeRParens = 0;
1424  }
1425 
1426  void next() {
1427  if (CurrentToken) {
1428  CurrentToken->NestingLevel = Contexts.size() - 1;
1429  CurrentToken->BindingStrength = Contexts.back().BindingStrength;
1430  modifyContext(*CurrentToken);
1431  determineTokenType(*CurrentToken);
1432  CurrentToken = CurrentToken->Next;
1433  }
1434 
1435  resetTokenMetadata(CurrentToken);
1436  }
1437 
1438  /// A struct to hold information valid in a specific context, e.g.
1439  /// a pair of parenthesis.
1440  struct Context {
1441  Context(tok::TokenKind ContextKind, unsigned BindingStrength,
1442  bool IsExpression)
1445 
1450  bool ColonIsForRangeExpr = false;
1451  bool ColonIsDictLiteral = false;
1453  FormatToken *FirstObjCSelectorName = nullptr;
1454  FormatToken *FirstStartOfName = nullptr;
1455  bool CanBeExpression = true;
1456  bool InTemplateArgument = false;
1457  bool InCtorInitializer = false;
1458  bool InInheritanceList = false;
1459  bool CaretFound = false;
1460  bool IsForEachMacro = false;
1464  };
1465 
1466  /// Puts a new \c Context onto the stack \c Contexts for the lifetime
1467  /// of each instance.
1468  struct ScopedContextCreator {
1469  AnnotatingParser &P;
1470 
1471  ScopedContextCreator(AnnotatingParser &P, tok::TokenKind ContextKind,
1472  unsigned Increase)
1473  : P(P) {
1474  P.Contexts.push_back(Context(ContextKind,
1475  P.Contexts.back().BindingStrength + Increase,
1476  P.Contexts.back().IsExpression));
1477  }
1478 
1479  ~ScopedContextCreator() {
1480  if (P.Style.AlignArrayOfStructures != FormatStyle::AIAS_None) {
1481  if (P.Contexts.back().InStructArrayInitializer) {
1482  P.Contexts.pop_back();
1483  P.Contexts.back().InStructArrayInitializer = true;
1484  return;
1485  }
1486  }
1487  P.Contexts.pop_back();
1488  }
1489  };
1490 
1491  void modifyContext(const FormatToken &Current) {
1492  if (Current.getPrecedence() == prec::Assignment &&
1493  !Line.First->isOneOf(tok::kw_template, tok::kw_using, tok::kw_return) &&
1494  // Type aliases use `type X = ...;` in TypeScript and can be exported
1495  // using `export type ...`.
1496  !(Style.Language == FormatStyle::LK_JavaScript &&
1497  (Line.startsWith(Keywords.kw_type, tok::identifier) ||
1498  Line.startsWith(tok::kw_export, Keywords.kw_type,
1499  tok::identifier))) &&
1500  (!Current.Previous || Current.Previous->isNot(tok::kw_operator))) {
1501  Contexts.back().IsExpression = true;
1502  if (!Line.startsWith(TT_UnaryOperator)) {
1503  for (FormatToken *Previous = Current.Previous;
1504  Previous && Previous->Previous &&
1505  !Previous->Previous->isOneOf(tok::comma, tok::semi);
1506  Previous = Previous->Previous) {
1507  if (Previous->isOneOf(tok::r_square, tok::r_paren)) {
1508  Previous = Previous->MatchingParen;
1509  if (!Previous)
1510  break;
1511  }
1512  if (Previous->opensScope())
1513  break;
1514  if (Previous->isOneOf(TT_BinaryOperator, TT_UnaryOperator) &&
1515  Previous->isOneOf(tok::star, tok::amp, tok::ampamp) &&
1516  Previous->Previous && Previous->Previous->isNot(tok::equal))
1517  Previous->setType(TT_PointerOrReference);
1518  }
1519  }
1520  } else if (Current.is(tok::lessless) &&
1521  (!Current.Previous || !Current.Previous->is(tok::kw_operator))) {
1522  Contexts.back().IsExpression = true;
1523  } else if (Current.isOneOf(tok::kw_return, tok::kw_throw)) {
1524  Contexts.back().IsExpression = true;
1525  } else if (Current.is(TT_TrailingReturnArrow)) {
1526  Contexts.back().IsExpression = false;
1527  } else if (Current.is(TT_LambdaArrow) || Current.is(Keywords.kw_assert)) {
1528  Contexts.back().IsExpression = Style.Language == FormatStyle::LK_Java;
1529  } else if (Current.Previous &&
1530  Current.Previous->is(TT_CtorInitializerColon)) {
1531  Contexts.back().IsExpression = true;
1532  Contexts.back().InCtorInitializer = true;
1533  } else if (Current.Previous && Current.Previous->is(TT_InheritanceColon)) {
1534  Contexts.back().InInheritanceList = true;
1535  } else if (Current.isOneOf(tok::r_paren, tok::greater, tok::comma)) {
1536  for (FormatToken *Previous = Current.Previous;
1537  Previous && Previous->isOneOf(tok::star, tok::amp);
1538  Previous = Previous->Previous)
1539  Previous->setType(TT_PointerOrReference);
1540  if (Line.MustBeDeclaration && !Contexts.front().InCtorInitializer)
1541  Contexts.back().IsExpression = false;
1542  } else if (Current.is(tok::kw_new)) {
1543  Contexts.back().CanBeExpression = false;
1544  } else if (Current.is(tok::semi) ||
1545  (Current.is(tok::exclaim) && Current.Previous &&
1546  !Current.Previous->is(tok::kw_operator))) {
1547  // This should be the condition or increment in a for-loop.
1548  // But not operator !() (can't use TT_OverloadedOperator here as its not
1549  // been annotated yet).
1550  Contexts.back().IsExpression = true;
1551  }
1552  }
1553 
1554  static FormatToken *untilMatchingParen(FormatToken *Current) {
1555  // Used when `MatchingParen` is not yet established.
1556  int ParenLevel = 0;
1557  while (Current) {
1558  if (Current->is(tok::l_paren))
1559  ParenLevel++;
1560  if (Current->is(tok::r_paren))
1561  ParenLevel--;
1562  if (ParenLevel < 1)
1563  break;
1564  Current = Current->Next;
1565  }
1566  return Current;
1567  }
1568 
1569  static bool isDeductionGuide(FormatToken &Current) {
1570  // Look for a deduction guide template<T> A(...) -> A<...>;
1571  if (Current.Previous && Current.Previous->is(tok::r_paren) &&
1572  Current.startsSequence(tok::arrow, tok::identifier, tok::less)) {
1573  // Find the TemplateCloser.
1574  FormatToken *TemplateCloser = Current.Next->Next;
1575  int NestingLevel = 0;
1576  while (TemplateCloser) {
1577  // Skip over an expressions in parens A<(3 < 2)>;
1578  if (TemplateCloser->is(tok::l_paren)) {
1579  // No Matching Paren yet so skip to matching paren
1580  TemplateCloser = untilMatchingParen(TemplateCloser);
1581  }
1582  if (TemplateCloser->is(tok::less))
1583  NestingLevel++;
1584  if (TemplateCloser->is(tok::greater))
1585  NestingLevel--;
1586  if (NestingLevel < 1)
1587  break;
1588  TemplateCloser = TemplateCloser->Next;
1589  }
1590  // Assuming we have found the end of the template ensure its followed
1591  // with a semi-colon.
1592  if (TemplateCloser && TemplateCloser->Next &&
1593  TemplateCloser->Next->is(tok::semi) &&
1594  Current.Previous->MatchingParen) {
1595  // Determine if the identifier `A` prior to the A<..>; is the same as
1596  // prior to the A(..)
1597  FormatToken *LeadingIdentifier =
1598  Current.Previous->MatchingParen->Previous;
1599 
1600  // Differentiate a deduction guide by seeing the
1601  // > of the template prior to the leading identifier.
1602  if (LeadingIdentifier) {
1603  FormatToken *PriorLeadingIdentifier = LeadingIdentifier->Previous;
1604  // Skip back past explicit decoration
1605  if (PriorLeadingIdentifier &&
1606  PriorLeadingIdentifier->is(tok::kw_explicit))
1607  PriorLeadingIdentifier = PriorLeadingIdentifier->Previous;
1608 
1609  return (PriorLeadingIdentifier &&
1610  PriorLeadingIdentifier->is(TT_TemplateCloser) &&
1611  LeadingIdentifier->TokenText == Current.Next->TokenText);
1612  }
1613  }
1614  }
1615  return false;
1616  }
1617 
1618  void determineTokenType(FormatToken &Current) {
1619  if (!Current.is(TT_Unknown))
1620  // The token type is already known.
1621  return;
1622 
1623  if ((Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp()) &&
1624  Current.is(tok::exclaim)) {
1625  if (Current.Previous) {
1626  bool IsIdentifier =
1627  Style.Language == FormatStyle::LK_JavaScript
1628  ? Keywords.IsJavaScriptIdentifier(
1629  *Current.Previous, /* AcceptIdentifierName= */ true)
1630  : Current.Previous->is(tok::identifier);
1631  if (IsIdentifier ||
1632  Current.Previous->isOneOf(
1633  tok::kw_namespace, tok::r_paren, tok::r_square, tok::r_brace,
1634  tok::kw_false, tok::kw_true, Keywords.kw_type, Keywords.kw_get,
1635  Keywords.kw_set) ||
1636  Current.Previous->Tok.isLiteral()) {
1637  Current.setType(TT_NonNullAssertion);
1638  return;
1639  }
1640  }
1641  if (Current.Next &&
1642  Current.Next->isOneOf(TT_BinaryOperator, Keywords.kw_as)) {
1643  Current.setType(TT_NonNullAssertion);
1644  return;
1645  }
1646  }
1647 
1648  // Line.MightBeFunctionDecl can only be true after the parentheses of a
1649  // function declaration have been found. In this case, 'Current' is a
1650  // trailing token of this declaration and thus cannot be a name.
1651  if (Current.is(Keywords.kw_instanceof)) {
1652  Current.setType(TT_BinaryOperator);
1653  } else if (isStartOfName(Current) &&
1654  (!Line.MightBeFunctionDecl || Current.NestingLevel != 0)) {
1655  Contexts.back().FirstStartOfName = &Current;
1656  Current.setType(TT_StartOfName);
1657  } else if (Current.is(tok::semi)) {
1658  // Reset FirstStartOfName after finding a semicolon so that a for loop
1659  // with multiple increment statements is not confused with a for loop
1660  // having multiple variable declarations.
1661  Contexts.back().FirstStartOfName = nullptr;
1662  } else if (Current.isOneOf(tok::kw_auto, tok::kw___auto_type)) {
1663  AutoFound = true;
1664  } else if (Current.is(tok::arrow) &&
1665  Style.Language == FormatStyle::LK_Java) {
1666  Current.setType(TT_LambdaArrow);
1667  } else if (Current.is(tok::arrow) && AutoFound && Line.MustBeDeclaration &&
1668  Current.NestingLevel == 0 &&
1669  !Current.Previous->is(tok::kw_operator)) {
1670  // not auto operator->() -> xxx;
1671  Current.setType(TT_TrailingReturnArrow);
1672  } else if (Current.is(tok::arrow) && Current.Previous &&
1673  Current.Previous->is(tok::r_brace)) {
1674  // Concept implicit conversion constraint needs to be treated like
1675  // a trailing return type ... } -> <type>.
1676  Current.setType(TT_TrailingReturnArrow);
1677  } else if (isDeductionGuide(Current)) {
1678  // Deduction guides trailing arrow " A(...) -> A<T>;".
1679  Current.setType(TT_TrailingReturnArrow);
1680  } else if (Current.isOneOf(tok::star, tok::amp, tok::ampamp)) {
1681  Current.setType(determineStarAmpUsage(
1682  Current,
1683  Contexts.back().CanBeExpression && Contexts.back().IsExpression,
1684  Contexts.back().InTemplateArgument));
1685  } else if (Current.isOneOf(tok::minus, tok::plus, tok::caret)) {
1686  Current.setType(determinePlusMinusCaretUsage(Current));
1687  if (Current.is(TT_UnaryOperator) && Current.is(tok::caret))
1688  Contexts.back().CaretFound = true;
1689  } else if (Current.isOneOf(tok::minusminus, tok::plusplus)) {
1690  Current.setType(determineIncrementUsage(Current));
1691  } else if (Current.isOneOf(tok::exclaim, tok::tilde)) {
1692  Current.setType(TT_UnaryOperator);
1693  } else if (Current.is(tok::question)) {
1694  if (Style.Language == FormatStyle::LK_JavaScript &&
1695  Line.MustBeDeclaration && !Contexts.back().IsExpression) {
1696  // In JavaScript, `interface X { foo?(): bar; }` is an optional method
1697  // on the interface, not a ternary expression.
1698  Current.setType(TT_JsTypeOptionalQuestion);
1699  } else {
1700  Current.setType(TT_ConditionalExpr);
1701  }
1702  } else if (Current.isBinaryOperator() &&
1703  (!Current.Previous || Current.Previous->isNot(tok::l_square)) &&
1704  (!Current.is(tok::greater) &&
1705  Style.Language != FormatStyle::LK_TextProto)) {
1706  Current.setType(TT_BinaryOperator);
1707  } else if (Current.is(tok::comment)) {
1708  if (Current.TokenText.startswith("/*")) {
1709  if (Current.TokenText.endswith("*/"))
1710  Current.setType(TT_BlockComment);
1711  else
1712  // The lexer has for some reason determined a comment here. But we
1713  // cannot really handle it, if it isn't properly terminated.
1714  Current.Tok.setKind(tok::unknown);
1715  } else {
1716  Current.setType(TT_LineComment);
1717  }
1718  } else if (Current.is(tok::r_paren)) {
1719  if (rParenEndsCast(Current))
1720  Current.setType(TT_CastRParen);
1721  if (Current.MatchingParen && Current.Next &&
1722  !Current.Next->isBinaryOperator() &&
1723  !Current.Next->isOneOf(tok::semi, tok::colon, tok::l_brace,
1724  tok::comma, tok::period, tok::arrow,
1725  tok::coloncolon))
1726  if (FormatToken *AfterParen = Current.MatchingParen->Next) {
1727  // Make sure this isn't the return type of an Obj-C block declaration
1728  if (AfterParen->Tok.isNot(tok::caret)) {
1729  if (FormatToken *BeforeParen = Current.MatchingParen->Previous)
1730  if (BeforeParen->is(tok::identifier) &&
1731  !BeforeParen->is(TT_TypenameMacro) &&
1732  BeforeParen->TokenText == BeforeParen->TokenText.upper() &&
1733  (!BeforeParen->Previous ||
1734  BeforeParen->Previous->ClosesTemplateDeclaration))
1735  Current.setType(TT_FunctionAnnotationRParen);
1736  }
1737  }
1738  } else if (Current.is(tok::at) && Current.Next &&
1739  Style.Language != FormatStyle::LK_JavaScript &&
1740  Style.Language != FormatStyle::LK_Java) {
1741  // In Java & JavaScript, "@..." is a decorator or annotation. In ObjC, it
1742  // marks declarations and properties that need special formatting.
1743  switch (Current.Next->Tok.getObjCKeywordID()) {
1744  case tok::objc_interface:
1745  case tok::objc_implementation:
1746  case tok::objc_protocol:
1747  Current.setType(TT_ObjCDecl);
1748  break;
1749  case tok::objc_property:
1750  Current.setType(TT_ObjCProperty);
1751  break;
1752  default:
1753  break;
1754  }
1755  } else if (Current.is(tok::period)) {
1756  FormatToken *PreviousNoComment = Current.getPreviousNonComment();
1757  if (PreviousNoComment &&
1758  PreviousNoComment->isOneOf(tok::comma, tok::l_brace))
1759  Current.setType(TT_DesignatedInitializerPeriod);
1760  else if (Style.Language == FormatStyle::LK_Java && Current.Previous &&
1761  Current.Previous->isOneOf(TT_JavaAnnotation,
1762  TT_LeadingJavaAnnotation)) {
1763  Current.setType(Current.Previous->getType());
1764  }
1765  } else if (canBeObjCSelectorComponent(Current) &&
1766  // FIXME(bug 36976): ObjC return types shouldn't use
1767  // TT_CastRParen.
1768  Current.Previous && Current.Previous->is(TT_CastRParen) &&
1769  Current.Previous->MatchingParen &&
1770  Current.Previous->MatchingParen->Previous &&
1771  Current.Previous->MatchingParen->Previous->is(
1772  TT_ObjCMethodSpecifier)) {
1773  // This is the first part of an Objective-C selector name. (If there's no
1774  // colon after this, this is the only place which annotates the identifier
1775  // as a selector.)
1776  Current.setType(TT_SelectorName);
1777  } else if (Current.isOneOf(tok::identifier, tok::kw_const, tok::kw_noexcept,
1778  tok::kw_requires) &&
1779  Current.Previous &&
1780  !Current.Previous->isOneOf(tok::equal, tok::at) &&
1781  Line.MightBeFunctionDecl && Contexts.size() == 1) {
1782  // Line.MightBeFunctionDecl can only be true after the parentheses of a
1783  // function declaration have been found.
1784  Current.setType(TT_TrailingAnnotation);
1785  } else if ((Style.Language == FormatStyle::LK_Java ||
1786  Style.Language == FormatStyle::LK_JavaScript) &&
1787  Current.Previous) {
1788  if (Current.Previous->is(tok::at) &&
1789  Current.isNot(Keywords.kw_interface)) {
1790  const FormatToken &AtToken = *Current.Previous;
1791  const FormatToken *Previous = AtToken.getPreviousNonComment();
1792  if (!Previous || Previous->is(TT_LeadingJavaAnnotation))
1793  Current.setType(TT_LeadingJavaAnnotation);
1794  else
1795  Current.setType(TT_JavaAnnotation);
1796  } else if (Current.Previous->is(tok::period) &&
1797  Current.Previous->isOneOf(TT_JavaAnnotation,
1798  TT_LeadingJavaAnnotation)) {
1799  Current.setType(Current.Previous->getType());
1800  }
1801  }
1802  }
1803 
1804  /// Take a guess at whether \p Tok starts a name of a function or
1805  /// variable declaration.
1806  ///
1807  /// This is a heuristic based on whether \p Tok is an identifier following
1808  /// something that is likely a type.
1809  bool isStartOfName(const FormatToken &Tok) {
1810  if (Tok.isNot(tok::identifier) || !Tok.Previous)
1811  return false;
1812 
1813  if (Tok.Previous->isOneOf(TT_LeadingJavaAnnotation, Keywords.kw_instanceof,
1814  Keywords.kw_as))
1815  return false;
1816  if (Style.Language == FormatStyle::LK_JavaScript &&
1817  Tok.Previous->is(Keywords.kw_in))
1818  return false;
1819 
1820  // Skip "const" as it does not have an influence on whether this is a name.
1821  FormatToken *PreviousNotConst = Tok.getPreviousNonComment();
1822  while (PreviousNotConst && PreviousNotConst->is(tok::kw_const))
1823  PreviousNotConst = PreviousNotConst->getPreviousNonComment();
1824 
1825  if (!PreviousNotConst)
1826  return false;
1827 
1828  bool IsPPKeyword = PreviousNotConst->is(tok::identifier) &&
1829  PreviousNotConst->Previous &&
1830  PreviousNotConst->Previous->is(tok::hash);
1831 
1832  if (PreviousNotConst->is(TT_TemplateCloser))
1833  return PreviousNotConst && PreviousNotConst->MatchingParen &&
1834  PreviousNotConst->MatchingParen->Previous &&
1835  PreviousNotConst->MatchingParen->Previous->isNot(tok::period) &&
1836  PreviousNotConst->MatchingParen->Previous->isNot(tok::kw_template);
1837 
1838  if (PreviousNotConst->is(tok::r_paren) &&
1839  PreviousNotConst->is(TT_TypeDeclarationParen))
1840  return true;
1841 
1842  return (!IsPPKeyword &&
1843  PreviousNotConst->isOneOf(tok::identifier, tok::kw_auto)) ||
1844  PreviousNotConst->is(TT_PointerOrReference) ||
1845  PreviousNotConst->isSimpleTypeSpecifier();
1846  }
1847 
1848  /// Determine whether ')' is ending a cast.
1849  bool rParenEndsCast(const FormatToken &Tok) {
1850  // C-style casts are only used in C++, C# and Java.
1851  if (!Style.isCSharp() && !Style.isCpp() &&
1852  Style.Language != FormatStyle::LK_Java)
1853  return false;
1854 
1855  // Empty parens aren't casts and there are no casts at the end of the line.
1856  if (Tok.Previous == Tok.MatchingParen || !Tok.Next || !Tok.MatchingParen)
1857  return false;
1858 
1859  FormatToken *LeftOfParens = Tok.MatchingParen->getPreviousNonComment();
1860  if (LeftOfParens) {
1861  // If there is a closing parenthesis left of the current parentheses,
1862  // look past it as these might be chained casts.
1863  if (LeftOfParens->is(tok::r_paren)) {
1864  if (!LeftOfParens->MatchingParen ||
1865  !LeftOfParens->MatchingParen->Previous)
1866  return false;
1867  LeftOfParens = LeftOfParens->MatchingParen->Previous;
1868  }
1869 
1870  // If there is an identifier (or with a few exceptions a keyword) right
1871  // before the parentheses, this is unlikely to be a cast.
1872  if (LeftOfParens->Tok.getIdentifierInfo() &&
1873  !LeftOfParens->isOneOf(Keywords.kw_in, tok::kw_return, tok::kw_case,
1874  tok::kw_delete))
1875  return false;
1876 
1877  // Certain other tokens right before the parentheses are also signals that
1878  // this cannot be a cast.
1879  if (LeftOfParens->isOneOf(tok::at, tok::r_square, TT_OverloadedOperator,
1880  TT_TemplateCloser, tok::ellipsis))
1881  return false;
1882  }
1883 
1884  if (Tok.Next->is(tok::question))
1885  return false;
1886 
1887  // `foreach((A a, B b) in someList)` should not be seen as a cast.
1888  if (Tok.Next->is(Keywords.kw_in) && Style.isCSharp())
1889  return false;
1890 
1891  // Functions which end with decorations like volatile, noexcept are unlikely
1892  // to be casts.
1893  if (Tok.Next->isOneOf(tok::kw_noexcept, tok::kw_volatile, tok::kw_const,
1894  tok::kw_requires, tok::kw_throw, tok::arrow,
1895  Keywords.kw_override, Keywords.kw_final) ||
1896  isCpp11AttributeSpecifier(*Tok.Next))
1897  return false;
1898 
1899  // As Java has no function types, a "(" after the ")" likely means that this
1900  // is a cast.
1901  if (Style.Language == FormatStyle::LK_Java && Tok.Next->is(tok::l_paren))
1902  return true;
1903 
1904  // If a (non-string) literal follows, this is likely a cast.
1905  if (Tok.Next->isNot(tok::string_literal) &&
1906  (Tok.Next->Tok.isLiteral() ||
1907  Tok.Next->isOneOf(tok::kw_sizeof, tok::kw_alignof)))
1908  return true;
1909 
1910  // Heuristically try to determine whether the parentheses contain a type.
1911  auto IsQualifiedPointerOrReference = [](FormatToken *T) {
1912  // This is used to handle cases such as x = (foo *const)&y;
1913  assert(!T->isSimpleTypeSpecifier() && "Should have already been checked");
1914  // Strip trailing qualifiers such as const or volatile when checking
1915  // whether the parens could be a cast to a pointer/reference type.
1916  while (T) {
1917  if (T->is(TT_AttributeParen)) {
1918  // Handle `x = (foo *__attribute__((foo)))&v;`:
1919  if (T->MatchingParen && T->MatchingParen->Previous &&
1920  T->MatchingParen->Previous->is(tok::kw___attribute)) {
1921  T = T->MatchingParen->Previous->Previous;
1922  continue;
1923  }
1924  } else if (T->is(TT_AttributeSquare)) {
1925  // Handle `x = (foo *[[clang::foo]])&v;`:
1926  if (T->MatchingParen && T->MatchingParen->Previous) {
1927  T = T->MatchingParen->Previous;
1928  continue;
1929  }
1930  } else if (T->canBePointerOrReferenceQualifier()) {
1931  T = T->Previous;
1932  continue;
1933  }
1934  break;
1935  }
1936  return T && T->is(TT_PointerOrReference);
1937  };
1938  bool ParensAreType =
1939  !Tok.Previous ||
1940  Tok.Previous->isOneOf(TT_TemplateCloser, TT_TypeDeclarationParen) ||
1941  Tok.Previous->isSimpleTypeSpecifier() ||
1942  IsQualifiedPointerOrReference(Tok.Previous);
1943  bool ParensCouldEndDecl =
1944  Tok.Next->isOneOf(tok::equal, tok::semi, tok::l_brace, tok::greater);
1945  if (ParensAreType && !ParensCouldEndDecl)
1946  return true;
1947 
1948  // At this point, we heuristically assume that there are no casts at the
1949  // start of the line. We assume that we have found most cases where there
1950  // are by the logic above, e.g. "(void)x;".
1951  if (!LeftOfParens)
1952  return false;
1953 
1954  // Certain token types inside the parentheses mean that this can't be a
1955  // cast.
1956  for (const FormatToken *Token = Tok.MatchingParen->Next; Token != &Tok;
1957  Token = Token->Next)
1958  if (Token->is(TT_BinaryOperator))
1959  return false;
1960 
1961  // If the following token is an identifier or 'this', this is a cast. All
1962  // cases where this can be something else are handled above.
1963  if (Tok.Next->isOneOf(tok::identifier, tok::kw_this))
1964  return true;
1965 
1966  // Look for a cast `( x ) (`.
1967  if (Tok.Next->is(tok::l_paren) && Tok.Previous && Tok.Previous->Previous) {
1968  if (Tok.Previous->is(tok::identifier) &&
1969  Tok.Previous->Previous->is(tok::l_paren))
1970  return true;
1971  }
1972 
1973  if (!Tok.Next->Next)
1974  return false;
1975 
1976  // If the next token after the parenthesis is a unary operator, assume
1977  // that this is cast, unless there are unexpected tokens inside the
1978  // parenthesis.
1979  bool NextIsUnary =
1980  Tok.Next->isUnaryOperator() || Tok.Next->isOneOf(tok::amp, tok::star);
1981  if (!NextIsUnary || Tok.Next->is(tok::plus) ||
1982  !Tok.Next->Next->isOneOf(tok::identifier, tok::numeric_constant))
1983  return false;
1984  // Search for unexpected tokens.
1985  for (FormatToken *Prev = Tok.Previous; Prev != Tok.MatchingParen;
1986  Prev = Prev->Previous) {
1987  if (!Prev->isOneOf(tok::kw_const, tok::identifier, tok::coloncolon))
1988  return false;
1989  }
1990  return true;
1991  }
1992 
1993  /// Return the type of the given token assuming it is * or &.
1994  TokenType determineStarAmpUsage(const FormatToken &Tok, bool IsExpression,
1995  bool InTemplateArgument) {
1996  if (Style.Language == FormatStyle::LK_JavaScript)
1997  return TT_BinaryOperator;
1998 
1999  // && in C# must be a binary operator.
2000  if (Style.isCSharp() && Tok.is(tok::ampamp))
2001  return TT_BinaryOperator;
2002 
2003  const FormatToken *PrevToken = Tok.getPreviousNonComment();
2004  if (!PrevToken)
2005  return TT_UnaryOperator;
2006 
2007  const FormatToken *NextToken = Tok.getNextNonComment();
2008  if (!NextToken ||
2009  NextToken->isOneOf(tok::arrow, tok::equal, tok::kw_noexcept) ||
2010  NextToken->canBePointerOrReferenceQualifier() ||
2011  (NextToken->is(tok::l_brace) && !NextToken->getNextNonComment()))
2012  return TT_PointerOrReference;
2013 
2014  if (PrevToken->is(tok::coloncolon))
2015  return TT_PointerOrReference;
2016 
2017  if (PrevToken->is(tok::r_paren) && PrevToken->is(TT_TypeDeclarationParen))
2018  return TT_PointerOrReference;
2019 
2020  if (PrevToken->isOneOf(tok::l_paren, tok::l_square, tok::l_brace,
2021  tok::comma, tok::semi, tok::kw_return, tok::colon,
2022  tok::kw_co_return, tok::kw_co_await,
2023  tok::kw_co_yield, tok::equal, tok::kw_delete,
2024  tok::kw_sizeof, tok::kw_throw) ||
2025  PrevToken->isOneOf(TT_BinaryOperator, TT_ConditionalExpr,
2026  TT_UnaryOperator, TT_CastRParen))
2027  return TT_UnaryOperator;
2028 
2029  if (NextToken->is(tok::l_square) && NextToken->isNot(TT_LambdaLSquare))
2030  return TT_PointerOrReference;
2031  if (NextToken->is(tok::kw_operator) && !IsExpression)
2032  return TT_PointerOrReference;
2033  if (NextToken->isOneOf(tok::comma, tok::semi))
2034  return TT_PointerOrReference;
2035 
2036  if (PrevToken->Tok.isLiteral() ||
2037  PrevToken->isOneOf(tok::r_paren, tok::r_square, tok::kw_true,
2038  tok::kw_false, tok::r_brace) ||
2039  NextToken->Tok.isLiteral() ||
2040  NextToken->isOneOf(tok::kw_true, tok::kw_false) ||
2041  NextToken->isUnaryOperator() ||
2042  // If we know we're in a template argument, there are no named
2043  // declarations. Thus, having an identifier on the right-hand side
2044  // indicates a binary operator.
2045  (InTemplateArgument && NextToken->Tok.isAnyIdentifier()))
2046  return TT_BinaryOperator;
2047 
2048  // "&&(" is quite unlikely to be two successive unary "&".
2049  if (Tok.is(tok::ampamp) && NextToken->is(tok::l_paren))
2050  return TT_BinaryOperator;
2051 
2052  // This catches some cases where evaluation order is used as control flow:
2053  // aaa && aaa->f();
2054  if (NextToken->Tok.isAnyIdentifier()) {
2055  const FormatToken *NextNextToken = NextToken->getNextNonComment();
2056  if (NextNextToken && NextNextToken->is(tok::arrow))
2057  return TT_BinaryOperator;
2058  }
2059 
2060  // It is very unlikely that we are going to find a pointer or reference type
2061  // definition on the RHS of an assignment.
2062  if (IsExpression && !Contexts.back().CaretFound)
2063  return TT_BinaryOperator;
2064 
2065  return TT_PointerOrReference;
2066  }
2067 
2068  TokenType determinePlusMinusCaretUsage(const FormatToken &Tok) {
2069  const FormatToken *PrevToken = Tok.getPreviousNonComment();
2070  if (!PrevToken)
2071  return TT_UnaryOperator;
2072 
2073  if (PrevToken->isOneOf(TT_CastRParen, TT_UnaryOperator))
2074  // This must be a sequence of leading unary operators.
2075  return TT_UnaryOperator;
2076 
2077  // Use heuristics to recognize unary operators.
2078  if (PrevToken->isOneOf(tok::equal, tok::l_paren, tok::comma, tok::l_square,
2079  tok::question, tok::colon, tok::kw_return,
2080  tok::kw_case, tok::at, tok::l_brace, tok::kw_throw,
2081  tok::kw_co_return, tok::kw_co_yield))
2082  return TT_UnaryOperator;
2083 
2084  // There can't be two consecutive binary operators.
2085  if (PrevToken->is(TT_BinaryOperator))
2086  return TT_UnaryOperator;
2087 
2088  // Fall back to marking the token as binary operator.
2089  return TT_BinaryOperator;
2090  }
2091 
2092  /// Determine whether ++/-- are pre- or post-increments/-decrements.
2093  TokenType determineIncrementUsage(const FormatToken &Tok) {
2094  const FormatToken *PrevToken = Tok.getPreviousNonComment();
2095  if (!PrevToken || PrevToken->is(TT_CastRParen))
2096  return TT_UnaryOperator;
2097  if (PrevToken->isOneOf(tok::r_paren, tok::r_square, tok::identifier))
2098  return TT_TrailingUnaryOperator;
2099 
2100  return TT_UnaryOperator;
2101  }
2102 
2103  SmallVector<Context, 8> Contexts;
2104 
2105  const FormatStyle &Style;
2106  AnnotatedLine &Line;
2107  FormatToken *CurrentToken;
2108  bool AutoFound;
2109  const AdditionalKeywords &Keywords;
2110 
2111  // Set of "<" tokens that do not open a template parameter list. If parseAngle
2112  // determines that a specific token can't be a template opener, it will make
2113  // same decision irrespective of the decisions for tokens leading up to it.
2114  // Store this information to prevent this from causing exponential runtime.
2115  llvm::SmallPtrSet<FormatToken *, 16> NonTemplateLess;
2116 };
2117 
2118 static const int PrecedenceUnaryOperator = prec::PointerToMember + 1;
2119 static const int PrecedenceArrowAndPeriod = prec::PointerToMember + 2;
2120 
2121 /// Parses binary expressions by inserting fake parenthesis based on
2122 /// operator precedence.
2123 class ExpressionParser {
2124 public:
2125  ExpressionParser(const FormatStyle &Style, const AdditionalKeywords &Keywords,
2126  AnnotatedLine &Line)
2127  : Style(Style), Keywords(Keywords), Current(Line.First) {}
2128 
2129  /// Parse expressions with the given operator precedence.
2130  void parse(int Precedence = 0) {
2131  // Skip 'return' and ObjC selector colons as they are not part of a binary
2132  // expression.
2133  while (Current && (Current->is(tok::kw_return) ||
2134  (Current->is(tok::colon) &&
2135  Current->isOneOf(TT_ObjCMethodExpr, TT_DictLiteral))))
2136  next();
2137 
2138  if (!Current || Precedence > PrecedenceArrowAndPeriod)
2139  return;
2140 
2141  // Conditional expressions need to be parsed separately for proper nesting.
2142  if (Precedence == prec::Conditional) {
2143  parseConditionalExpr();
2144  return;
2145  }
2146 
2147  // Parse unary operators, which all have a higher precedence than binary
2148  // operators.
2149  if (Precedence == PrecedenceUnaryOperator) {
2150  parseUnaryOperator();
2151  return;
2152  }
2153 
2154  FormatToken *Start = Current;
2155  FormatToken *LatestOperator = nullptr;
2156  unsigned OperatorIndex = 0;
2157 
2158  while (Current) {
2159  // Consume operators with higher precedence.
2160  parse(Precedence + 1);
2161 
2162  int CurrentPrecedence = getCurrentPrecedence();
2163 
2164  if (Current && Current->is(TT_SelectorName) &&
2165  Precedence == CurrentPrecedence) {
2166  if (LatestOperator)
2167  addFakeParenthesis(Start, prec::Level(Precedence));
2168  Start = Current;
2169  }
2170 
2171  // At the end of the line or when an operator with higher precedence is
2172  // found, insert fake parenthesis and return.
2173  if (!Current ||
2174  (Current->closesScope() &&
2175  (Current->MatchingParen || Current->is(TT_TemplateString))) ||
2176  (CurrentPrecedence != -1 && CurrentPrecedence < Precedence) ||
2177  (CurrentPrecedence == prec::Conditional &&
2178  Precedence == prec::Assignment && Current->is(tok::colon))) {
2179  break;
2180  }
2181 
2182  // Consume scopes: (), [], <> and {}
2183  if (Current->opensScope()) {
2184  // In fragment of a JavaScript template string can look like '}..${' and
2185  // thus close a scope and open a new one at the same time.
2186  while (Current && (!Current->closesScope() || Current->opensScope())) {
2187  next();
2188  parse();
2189  }
2190  next();
2191  } else {
2192  // Operator found.
2193  if (CurrentPrecedence == Precedence) {
2194  if (LatestOperator)
2195  LatestOperator->NextOperator = Current;
2196  LatestOperator = Current;
2197  Current->OperatorIndex = OperatorIndex;
2198  ++OperatorIndex;
2199  }
2200  next(/*SkipPastLeadingComments=*/Precedence > 0);
2201  }
2202  }
2203 
2204  if (LatestOperator && (Current || Precedence > 0)) {
2205  // LatestOperator->LastOperator = true;
2206  if (Precedence == PrecedenceArrowAndPeriod) {
2207  // Call expressions don't have a binary operator precedence.
2208  addFakeParenthesis(Start, prec::Unknown);
2209  } else {
2210  addFakeParenthesis(Start, prec::Level(Precedence));
2211  }
2212  }
2213  }
2214 
2215 private:
2216  /// Gets the precedence (+1) of the given token for binary operators
2217  /// and other tokens that we treat like binary operators.
2218  int getCurrentPrecedence() {
2219  if (Current) {
2220  const FormatToken *NextNonComment = Current->getNextNonComment();
2221  if (Current->is(TT_ConditionalExpr))
2222  return prec::Conditional;
2223  if (NextNonComment && Current->is(TT_SelectorName) &&
2224  (NextNonComment->isOneOf(TT_DictLiteral, TT_JsTypeColon) ||
2225  ((Style.Language == FormatStyle::LK_Proto ||
2226  Style.Language == FormatStyle::LK_TextProto) &&
2227  NextNonComment->is(tok::less))))
2228  return prec::Assignment;
2229  if (Current->is(TT_JsComputedPropertyName))
2230  return prec::Assignment;
2231  if (Current->is(TT_LambdaArrow))
2232  return prec::Comma;
2233  if (Current->is(TT_FatArrow))
2234  return prec::Assignment;
2235  if (Current->isOneOf(tok::semi, TT_InlineASMColon, TT_SelectorName) ||
2236  (Current->is(tok::comment) && NextNonComment &&
2237  NextNonComment->is(TT_SelectorName)))
2238  return 0;
2239  if (Current->is(TT_RangeBasedForLoopColon))
2240  return prec::Comma;
2241  if ((Style.Language == FormatStyle::LK_Java ||
2242  Style.Language == FormatStyle::LK_JavaScript) &&
2243  Current->is(Keywords.kw_instanceof))
2244  return prec::Relational;
2245  if (Style.Language == FormatStyle::LK_JavaScript &&
2246  Current->isOneOf(Keywords.kw_in, Keywords.kw_as))
2247  return prec::Relational;
2248  if (Current->is(TT_BinaryOperator) || Current->is(tok::comma))
2249  return Current->getPrecedence();
2250  if (Current->isOneOf(tok::period, tok::arrow))
2251  return PrecedenceArrowAndPeriod;
2252  if ((Style.Language == FormatStyle::LK_Java ||
2253  Style.Language == FormatStyle::LK_JavaScript) &&
2254  Current->isOneOf(Keywords.kw_extends, Keywords.kw_implements,
2255  Keywords.kw_throws))
2256  return 0;
2257  }
2258  return -1;
2259  }
2260 
2261  void addFakeParenthesis(FormatToken *Start, prec::Level Precedence) {
2262  Start->FakeLParens.push_back(Precedence);
2263  if (Precedence > prec::Unknown)
2264  Start->StartsBinaryExpression = true;
2265  if (Current) {
2266  FormatToken *Previous = Current->Previous;
2267  while (Previous->is(tok::comment) && Previous->Previous)
2268  Previous = Previous->Previous;
2269  ++Previous->FakeRParens;
2270  if (Precedence > prec::Unknown)
2271  Previous->EndsBinaryExpression = true;
2272  }
2273  }
2274 
2275  /// Parse unary operator expressions and surround them with fake
2276  /// parentheses if appropriate.
2277  void parseUnaryOperator() {
2279  while (Current && Current->is(TT_UnaryOperator)) {
2280  Tokens.push_back(Current);
2281  next();
2282  }
2283  parse(PrecedenceArrowAndPeriod);
2284  for (FormatToken *Token : llvm::reverse(Tokens))
2285  // The actual precedence doesn't matter.
2286  addFakeParenthesis(Token, prec::Unknown);
2287  }
2288 
2289  void parseConditionalExpr() {
2290  while (Current && Current->isTrailingComment()) {
2291  next();
2292  }
2293  FormatToken *Start = Current;
2294  parse(prec::LogicalOr);
2295  if (!Current || !Current->is(tok::question))
2296  return;
2297  next();
2298  parse(prec::Assignment);
2299  if (!Current || Current->isNot(TT_ConditionalExpr))
2300  return;
2301  next();
2302  parse(prec::Assignment);
2303  addFakeParenthesis(Start, prec::Conditional);
2304  }
2305 
2306  void next(bool SkipPastLeadingComments = true) {
2307  if (Current)
2308  Current = Current->Next;
2309  while (Current &&
2310  (Current->NewlinesBefore == 0 || SkipPastLeadingComments) &&
2311  Current->isTrailingComment())
2312  Current = Current->Next;
2313  }
2314 
2315  const FormatStyle &Style;
2316  const AdditionalKeywords &Keywords;
2317  FormatToken *Current;
2318 };
2319 
2320 } // end anonymous namespace
2321 
2324  const AnnotatedLine *NextNonCommentLine = nullptr;
2326  E = Lines.rend();
2327  I != E; ++I) {
2328  bool CommentLine = true;
2329  for (const FormatToken *Tok = (*I)->First; Tok; Tok = Tok->Next) {
2330  if (!Tok->is(tok::comment)) {
2331  CommentLine = false;
2332  break;
2333  }
2334  }
2335 
2336  // If the comment is currently aligned with the line immediately following
2337  // it, that's probably intentional and we should keep it.
2338  if (NextNonCommentLine && CommentLine &&
2339  NextNonCommentLine->First->NewlinesBefore <= 1 &&
2340  NextNonCommentLine->First->OriginalColumn ==
2341  (*I)->First->OriginalColumn) {
2342  // Align comments for preprocessor lines with the # in column 0 if
2343  // preprocessor lines are not indented. Otherwise, align with the next
2344  // line.
2345  (*I)->Level =
2346  (Style.IndentPPDirectives != FormatStyle::PPDIS_BeforeHash &&
2347  (NextNonCommentLine->Type == LT_PreprocessorDirective ||
2348  NextNonCommentLine->Type == LT_ImportStatement))
2349  ? 0
2350  : NextNonCommentLine->Level;
2351  } else {
2352  NextNonCommentLine = (*I)->First->isNot(tok::r_brace) ? (*I) : nullptr;
2353  }
2354 
2355  setCommentLineLevels((*I)->Children);
2356  }
2357 }
2358 
2359 static unsigned maxNestingDepth(const AnnotatedLine &Line) {
2360  unsigned Result = 0;
2361  for (const auto *Tok = Line.First; Tok != nullptr; Tok = Tok->Next)
2362  Result = std::max(Result, Tok->NestingLevel);
2363  return Result;
2364 }
2365 
2367  for (SmallVectorImpl<AnnotatedLine *>::iterator I = Line.Children.begin(),
2368  E = Line.Children.end();
2369  I != E; ++I) {
2370  annotate(**I);
2371  }
2372  AnnotatingParser Parser(Style, Line, Keywords);
2373  Line.Type = Parser.parseLine();
2374 
2375  // With very deep nesting, ExpressionParser uses lots of stack and the
2376  // formatting algorithm is very slow. We're not going to do a good job here
2377  // anyway - it's probably generated code being formatted by mistake.
2378  // Just skip the whole line.
2379  if (maxNestingDepth(Line) > 50)
2380  Line.Type = LT_Invalid;
2381 
2382  if (Line.Type == LT_Invalid)
2383  return;
2384 
2385  ExpressionParser ExprParser(Style, Keywords, Line);
2386  ExprParser.parse();
2387 
2388  if (Line.startsWith(TT_ObjCMethodSpecifier))
2389  Line.Type = LT_ObjCMethodDecl;
2390  else if (Line.startsWith(TT_ObjCDecl))
2391  Line.Type = LT_ObjCDecl;
2392  else if (Line.startsWith(TT_ObjCProperty))
2393  Line.Type = LT_ObjCProperty;
2394 
2395  Line.First->SpacesRequiredBefore = 1;
2396  Line.First->CanBreakBefore = Line.First->MustBreakBefore;
2397 }
2398 
2399 // This function heuristically determines whether 'Current' starts the name of a
2400 // function declaration.
2401 static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current,
2402  const AnnotatedLine &Line) {
2403  auto skipOperatorName = [](const FormatToken *Next) -> const FormatToken * {
2404  for (; Next; Next = Next->Next) {
2405  if (Next->is(TT_OverloadedOperatorLParen))
2406  return Next;
2407  if (Next->is(TT_OverloadedOperator))
2408  continue;
2409  if (Next->isOneOf(tok::kw_new, tok::kw_delete)) {
2410  // For 'new[]' and 'delete[]'.
2411  if (Next->Next &&
2412  Next->Next->startsSequence(tok::l_square, tok::r_square))
2413  Next = Next->Next->Next;
2414  continue;
2415  }
2416  if (Next->startsSequence(tok::l_square, tok::r_square)) {
2417  // For operator[]().
2418  Next = Next->Next;
2419  continue;
2420  }
2421  if ((Next->isSimpleTypeSpecifier() || Next->is(tok::identifier)) &&
2422  Next->Next && Next->Next->isOneOf(tok::star, tok::amp, tok::ampamp)) {
2423  // For operator void*(), operator char*(), operator Foo*().
2424  Next = Next->Next;
2425  continue;
2426  }
2427  if (Next->is(TT_TemplateOpener) && Next->MatchingParen) {
2428  Next = Next->MatchingParen;
2429  continue;
2430  }
2431 
2432  break;
2433  }
2434  return nullptr;
2435  };
2436 
2437  // Find parentheses of parameter list.
2438  const FormatToken *Next = Current.Next;
2439  if (Current.is(tok::kw_operator)) {
2440  if (Current.Previous && Current.Previous->is(tok::coloncolon))
2441  return false;
2442  Next = skipOperatorName(Next);
2443  } else {
2444  if (!Current.is(TT_StartOfName) || Current.NestingLevel != 0)
2445  return false;
2446  for (; Next; Next = Next->Next) {
2447  if (Next->is(TT_TemplateOpener)) {
2448  Next = Next->MatchingParen;
2449  } else if (Next->is(tok::coloncolon)) {
2450  Next = Next->Next;
2451  if (!Next)
2452  return false;
2453  if (Next->is(tok::kw_operator)) {
2454  Next = skipOperatorName(Next->Next);
2455  break;
2456  }
2457  if (!Next->is(tok::identifier))
2458  return false;
2459  } else if (Next->is(tok::l_paren)) {
2460  break;
2461  } else {
2462  return false;
2463  }
2464  }
2465  }
2466 
2467  // Check whether parameter list can belong to a function declaration.
2468  if (!Next || !Next->is(tok::l_paren) || !Next->MatchingParen)
2469  return false;
2470  // If the lines ends with "{", this is likely a function definition.
2471  if (Line.Last->is(tok::l_brace))
2472  return true;
2473  if (Next->Next == Next->MatchingParen)
2474  return true; // Empty parentheses.
2475  // If there is an &/&& after the r_paren, this is likely a function.
2476  if (Next->MatchingParen->Next &&
2477  Next->MatchingParen->Next->is(TT_PointerOrReference))
2478  return true;
2479 
2480  // Check for K&R C function definitions (and C++ function definitions with
2481  // unnamed parameters), e.g.:
2482  // int f(i)
2483  // {
2484  // return i + 1;
2485  // }
2486  // bool g(size_t = 0, bool b = false)
2487  // {
2488  // return !b;
2489  // }
2490  if (IsCpp && Next->Next && Next->Next->is(tok::identifier) &&
2491  !Line.endsWith(tok::semi))
2492  return true;
2493 
2494  for (const FormatToken *Tok = Next->Next; Tok && Tok != Next->MatchingParen;
2495  Tok = Tok->Next) {
2496  if (Tok->is(TT_TypeDeclarationParen))
2497  return true;
2498  if (Tok->isOneOf(tok::l_paren, TT_TemplateOpener) && Tok->MatchingParen) {
2499  Tok = Tok->MatchingParen;
2500  continue;
2501  }
2502  if (Tok->is(tok::kw_const) || Tok->isSimpleTypeSpecifier() ||
2503  Tok->isOneOf(TT_PointerOrReference, TT_StartOfName, tok::ellipsis))
2504  return true;
2505  if (Tok->isOneOf(tok::l_brace, tok::string_literal, TT_ObjCMethodExpr) ||
2506  Tok->Tok.isLiteral())
2507  return false;
2508  }
2509  return false;
2510 }
2511 
2512 bool TokenAnnotator::mustBreakForReturnType(const AnnotatedLine &Line) const {
2513  assert(Line.MightBeFunctionDecl);
2514 
2515  if ((Style.AlwaysBreakAfterReturnType == FormatStyle::RTBS_TopLevel ||
2516  Style.AlwaysBreakAfterReturnType ==
2518  Line.Level > 0)
2519  return false;
2520 
2521  switch (Style.AlwaysBreakAfterReturnType) {
2523  return false;
2524  case FormatStyle::RTBS_All:
2526  return true;
2529  return Line.mightBeFunctionDefinition();
2530  }
2531 
2532  return false;
2533 }
2534 
2536  for (SmallVectorImpl<AnnotatedLine *>::iterator I = Line.Children.begin(),
2537  E = Line.Children.end();
2538  I != E; ++I) {
2540  }
2541 
2542  Line.First->TotalLength =
2543  Line.First->IsMultiline ? Style.ColumnLimit
2544  : Line.FirstStartColumn + Line.First->ColumnWidth;
2545  FormatToken *Current = Line.First->Next;
2546  bool InFunctionDecl = Line.MightBeFunctionDecl;
2547  bool AlignArrayOfStructures =
2548  (Style.AlignArrayOfStructures != FormatStyle::AIAS_None &&
2550  if (AlignArrayOfStructures)
2551  calculateArrayInitializerColumnList(Line);
2552 
2553  while (Current) {
2554  if (isFunctionDeclarationName(Style.isCpp(), *Current, Line))
2555  Current->setType(TT_FunctionDeclarationName);
2556  if (Current->is(TT_LineComment)) {
2557  if (Current->Previous->is(BK_BracedInit) &&
2558  Current->Previous->opensScope())
2559  Current->SpacesRequiredBefore =
2560  (Style.Cpp11BracedListStyle && !Style.SpacesInParentheses) ? 0 : 1;
2561  else
2562  Current->SpacesRequiredBefore = Style.SpacesBeforeTrailingComments;
2563 
2564  // If we find a trailing comment, iterate backwards to determine whether
2565  // it seems to relate to a specific parameter. If so, break before that
2566  // parameter to avoid changing the comment's meaning. E.g. don't move 'b'
2567  // to the previous line in:
2568  // SomeFunction(a,
2569  // b, // comment
2570  // c);
2571  if (!Current->HasUnescapedNewline) {
2572  for (FormatToken *Parameter = Current->Previous; Parameter;
2573  Parameter = Parameter->Previous) {
2574  if (Parameter->isOneOf(tok::comment, tok::r_brace))
2575  break;
2576  if (Parameter->Previous && Parameter->Previous->is(tok::comma)) {
2577  if (!Parameter->Previous->is(TT_CtorInitializerComma) &&
2578  Parameter->HasUnescapedNewline)
2579  Parameter->MustBreakBefore = true;
2580  break;
2581  }
2582  }
2583  }
2584  } else if (Current->SpacesRequiredBefore == 0 &&
2585  spaceRequiredBefore(Line, *Current)) {
2586  Current->SpacesRequiredBefore = 1;
2587  }
2588 
2589  Current->MustBreakBefore =
2590  Current->MustBreakBefore || mustBreakBefore(Line, *Current);
2591 
2592  if (!Current->MustBreakBefore && InFunctionDecl &&
2593  Current->is(TT_FunctionDeclarationName))
2594  Current->MustBreakBefore = mustBreakForReturnType(Line);
2595 
2596  Current->CanBreakBefore =
2597  Current->MustBreakBefore || canBreakBefore(Line, *Current);
2598  unsigned ChildSize = 0;
2599  if (Current->Previous->Children.size() == 1) {
2600  FormatToken &LastOfChild = *Current->Previous->Children[0]->Last;
2601  ChildSize = LastOfChild.isTrailingComment() ? Style.ColumnLimit
2602  : LastOfChild.TotalLength + 1;
2603  }
2604  const FormatToken *Prev = Current->Previous;
2605  if (Current->MustBreakBefore || Prev->Children.size() > 1 ||
2606  (Prev->Children.size() == 1 &&
2607  Prev->Children[0]->First->MustBreakBefore) ||
2608  Current->IsMultiline)
2609  Current->TotalLength = Prev->TotalLength + Style.ColumnLimit;
2610  else
2611  Current->TotalLength = Prev->TotalLength + Current->ColumnWidth +
2612  ChildSize + Current->SpacesRequiredBefore;
2613 
2614  if (Current->is(TT_CtorInitializerColon))
2615  InFunctionDecl = false;
2616 
2617  // FIXME: Only calculate this if CanBreakBefore is true once static
2618  // initializers etc. are sorted out.
2619  // FIXME: Move magic numbers to a better place.
2620 
2621  // Reduce penalty for aligning ObjC method arguments using the colon
2622  // alignment as this is the canonical way (still prefer fitting everything
2623  // into one line if possible). Trying to fit a whole expression into one
2624  // line should not force other line breaks (e.g. when ObjC method
2625  // expression is a part of other expression).
2626  Current->SplitPenalty = splitPenalty(Line, *Current, InFunctionDecl);
2627  if (Style.Language == FormatStyle::LK_ObjC &&
2628  Current->is(TT_SelectorName) && Current->ParameterIndex > 0) {
2629  if (Current->ParameterIndex == 1)
2630  Current->SplitPenalty += 5 * Current->BindingStrength;
2631  } else {
2632  Current->SplitPenalty += 20 * Current->BindingStrength;
2633  }
2634 
2635  Current = Current->Next;
2636  }
2637 
2638  calculateUnbreakableTailLengths(Line);
2639  unsigned IndentLevel = Line.Level;
2640  for (Current = Line.First; Current != nullptr; Current = Current->Next) {
2641  if (Current->Role)
2642  Current->Role->precomputeFormattingInfos(Current);
2643  if (Current->MatchingParen &&
2644  Current->MatchingParen->opensBlockOrBlockTypeList(Style)) {
2645  assert(IndentLevel > 0);
2646  --IndentLevel;
2647  }
2648  Current->IndentLevel = IndentLevel;
2649  if (Current->opensBlockOrBlockTypeList(Style))
2650  ++IndentLevel;
2651  }
2652 
2653  LLVM_DEBUG({ printDebugInfo(Line); });
2654 }
2655 
2656 void TokenAnnotator::calculateUnbreakableTailLengths(AnnotatedLine &Line) {
2657  unsigned UnbreakableTailLength = 0;
2658  FormatToken *Current = Line.Last;
2659  while (Current) {
2660  Current->UnbreakableTailLength = UnbreakableTailLength;
2661  if (Current->CanBreakBefore ||
2662  Current->isOneOf(tok::comment, tok::string_literal)) {
2663  UnbreakableTailLength = 0;
2664  } else {
2665  UnbreakableTailLength +=
2666  Current->ColumnWidth + Current->SpacesRequiredBefore;
2667  }
2668  Current = Current->Previous;
2669  }
2670 }
2671 
2672 void TokenAnnotator::calculateArrayInitializerColumnList(AnnotatedLine &Line) {
2673  if (Line.First == Line.Last) {
2674  return;
2675  }
2676  auto *CurrentToken = Line.First;
2677  CurrentToken->ArrayInitializerLineStart = true;
2678  unsigned Depth = 0;
2679  while (CurrentToken != nullptr && CurrentToken != Line.Last) {
2680  if (CurrentToken->is(tok::l_brace)) {
2681  CurrentToken->IsArrayInitializer = true;
2682  if (CurrentToken->Next != nullptr)
2683  CurrentToken->Next->MustBreakBefore = true;
2684  CurrentToken =
2685  calculateInitializerColumnList(Line, CurrentToken->Next, Depth + 1);
2686  } else {
2687  CurrentToken = CurrentToken->Next;
2688  }
2689  }
2690 }
2691 
2692 FormatToken *TokenAnnotator::calculateInitializerColumnList(
2693  AnnotatedLine &Line, FormatToken *CurrentToken, unsigned Depth) {
2694  while (CurrentToken != nullptr && CurrentToken != Line.Last) {
2695  if (CurrentToken->is(tok::l_brace))
2696  ++Depth;
2697  else if (CurrentToken->is(tok::r_brace))
2698  --Depth;
2699  if (Depth == 2 && CurrentToken->isOneOf(tok::l_brace, tok::comma)) {
2700  CurrentToken = CurrentToken->Next;
2701  if (CurrentToken == nullptr)
2702  break;
2703  CurrentToken->StartsColumn = true;
2704  CurrentToken = CurrentToken->Previous;
2705  }
2706  CurrentToken = CurrentToken->Next;
2707  }
2708  return CurrentToken;
2709 }
2710 
2711 unsigned TokenAnnotator::splitPenalty(const AnnotatedLine &Line,
2712  const FormatToken &Tok,
2713  bool InFunctionDecl) {
2714  const FormatToken &Left = *Tok.Previous;
2715  const FormatToken &Right = Tok;
2716 
2717  if (Left.is(tok::semi))
2718  return 0;
2719 
2720  if (Style.Language == FormatStyle::LK_Java) {
2721  if (Right.isOneOf(Keywords.kw_extends, Keywords.kw_throws))
2722  return 1;
2723  if (Right.is(Keywords.kw_implements))
2724  return 2;
2725  if (Left.is(tok::comma) && Left.NestingLevel == 0)
2726  return 3;
2727  } else if (Style.Language == FormatStyle::LK_JavaScript) {
2728  if (Right.is(Keywords.kw_function) && Left.isNot(tok::comma))
2729  return 100;
2730  if (Left.is(TT_JsTypeColon))
2731  return 35;
2732  if ((Left.is(TT_TemplateString) && Left.TokenText.endswith("${")) ||
2733  (Right.is(TT_TemplateString) && Right.TokenText.startswith("}")))
2734  return 100;
2735  // Prefer breaking call chains (".foo") over empty "{}", "[]" or "()".
2736  if (Left.opensScope() && Right.closesScope())
2737  return 200;
2738  }
2739 
2740  if (Right.is(tok::identifier) && Right.Next && Right.Next->is(TT_DictLiteral))
2741  return 1;
2742  if (Right.is(tok::l_square)) {
2743  if (Style.Language == FormatStyle::LK_Proto)
2744  return 1;
2745  if (Left.is(tok::r_square))
2746  return 200;
2747  // Slightly prefer formatting local lambda definitions like functions.
2748  if (Right.is(TT_LambdaLSquare) && Left.is(tok::equal))
2749  return 35;
2750  if (!Right.isOneOf(TT_ObjCMethodExpr, TT_LambdaLSquare,
2751  TT_ArrayInitializerLSquare,
2752  TT_DesignatedInitializerLSquare, TT_AttributeSquare))
2753  return 500;
2754  }
2755 
2756  if (Left.is(tok::coloncolon) ||
2757  (Right.is(tok::period) && Style.Language == FormatStyle::LK_Proto))
2758  return 500;
2759  if (Right.isOneOf(TT_StartOfName, TT_FunctionDeclarationName) ||
2760  Right.is(tok::kw_operator)) {
2761  if (Line.startsWith(tok::kw_for) && Right.PartOfMultiVariableDeclStmt)
2762  return 3;
2763  if (Left.is(TT_StartOfName))
2764  return 110;
2765  if (InFunctionDecl && Right.NestingLevel == 0)
2766  return Style.PenaltyReturnTypeOnItsOwnLine;
2767  return 200;
2768  }
2769  if (Right.is(TT_PointerOrReference))
2770  return 190;
2771  if (Right.is(TT_LambdaArrow))
2772  return 110;
2773  if (Left.is(tok::equal) && Right.is(tok::l_brace))
2774  return 160;
2775  if (Left.is(TT_CastRParen))
2776  return 100;
2777  if (Left.isOneOf(tok::kw_class, tok::kw_struct))
2778  return 5000;
2779  if (Left.is(tok::comment))
2780  return 1000;
2781 
2782  if (Left.isOneOf(TT_RangeBasedForLoopColon, TT_InheritanceColon,
2783  TT_CtorInitializerColon))
2784  return 2;
2785 
2786  if (Right.isMemberAccess()) {
2787  // Breaking before the "./->" of a chained call/member access is reasonably
2788  // cheap, as formatting those with one call per line is generally
2789  // desirable. In particular, it should be cheaper to break before the call
2790  // than it is to break inside a call's parameters, which could lead to weird
2791  // "hanging" indents. The exception is the very last "./->" to support this
2792  // frequent pattern:
2793  //
2794  // aaaaaaaa.aaaaaaaa.bbbbbbb().ccccccccccccccccccccc(
2795  // dddddddd);
2796  //
2797  // which might otherwise be blown up onto many lines. Here, clang-format
2798  // won't produce "hanging" indents anyway as there is no other trailing
2799  // call.
2800  //
2801  // Also apply higher penalty is not a call as that might lead to a wrapping
2802  // like:
2803  //
2804  // aaaaaaa
2805  // .aaaaaaaaa.bbbbbbbb(cccccccc);
2806  return !Right.NextOperator || !Right.NextOperator->Previous->closesScope()
2807  ? 150
2808  : 35;
2809  }
2810 
2811  if (Right.is(TT_TrailingAnnotation) &&
2812  (!Right.Next || Right.Next->isNot(tok::l_paren))) {
2813  // Moving trailing annotations to the next line is fine for ObjC method
2814  // declarations.
2815  if (Line.startsWith(TT_ObjCMethodSpecifier))
2816  return 10;
2817  // Generally, breaking before a trailing annotation is bad unless it is
2818  // function-like. It seems to be especially preferable to keep standard
2819  // annotations (i.e. "const", "final" and "override") on the same line.
2820  // Use a slightly higher penalty after ")" so that annotations like
2821  // "const override" are kept together.
2822  bool is_short_annotation = Right.TokenText.size() < 10;
2823  return (Left.is(tok::r_paren) ? 100 : 120) + (is_short_annotation ? 50 : 0);
2824  }
2825 
2826  // In for-loops, prefer breaking at ',' and ';'.
2827  if (Line.startsWith(tok::kw_for) && Left.is(tok::equal))
2828  return 4;
2829 
2830  // In Objective-C method expressions, prefer breaking before "param:" over
2831  // breaking after it.
2832  if (Right.is(TT_SelectorName))
2833  return 0;
2834  if (Left.is(tok::colon) && Left.is(TT_ObjCMethodExpr))
2835  return Line.MightBeFunctionDecl ? 50 : 500;
2836 
2837  // In Objective-C type declarations, avoid breaking after the category's
2838  // open paren (we'll prefer breaking after the protocol list's opening
2839  // angle bracket, if present).
2840  if (Line.Type == LT_ObjCDecl && Left.is(tok::l_paren) && Left.Previous &&
2841  Left.Previous->isOneOf(tok::identifier, tok::greater))
2842  return 500;
2843 
2844  if (Left.is(tok::l_paren) && InFunctionDecl &&
2845  Style.AlignAfterOpenBracket != FormatStyle::BAS_DontAlign)
2846  return 100;
2847  if (Left.is(tok::l_paren) && Left.Previous &&
2848  (Left.Previous->is(tok::kw_for) || Left.Previous->isIf()))
2849  return 1000;
2850  if (Left.is(tok::equal) && InFunctionDecl)
2851  return 110;
2852  if (Right.is(tok::r_brace))
2853  return 1;
2854  if (Left.is(TT_TemplateOpener))
2855  return 100;
2856  if (Left.opensScope()) {
2857  // If we aren't aligning after opening parens/braces we can always break
2858  // here unless the style does not want us to place all arguments on the
2859  // next line.
2860  if (Style.AlignAfterOpenBracket == FormatStyle::BAS_DontAlign &&
2861  (Left.ParameterCount <= 1 || Style.AllowAllArgumentsOnNextLine))
2862  return 0;
2863  if (Left.is(tok::l_brace) && !Style.Cpp11BracedListStyle)
2864  return 19;
2865  return Left.ParameterCount > 1 ? Style.PenaltyBreakBeforeFirstCallParameter
2866  : 19;
2867  }
2868  if (Left.is(TT_JavaAnnotation))
2869  return 50;
2870 
2871  if (Left.is(TT_UnaryOperator))
2872  return 60;
2873  if (Left.isOneOf(tok::plus, tok::comma) && Left.Previous &&
2874  Left.Previous->isLabelString() &&
2875  (Left.NextOperator || Left.OperatorIndex != 0))
2876  return 50;
2877  if (Right.is(tok::plus) && Left.isLabelString() &&
2878  (Right.NextOperator || Right.OperatorIndex != 0))
2879  return 25;
2880  if (Left.is(tok::comma))
2881  return 1;
2882  if (Right.is(tok::lessless) && Left.isLabelString() &&
2883  (Right.NextOperator || Right.OperatorIndex != 1))
2884  return 25;
2885  if (Right.is(tok::lessless)) {
2886  // Breaking at a << is really cheap.
2887  if (!Left.is(tok::r_paren) || Right.OperatorIndex > 0)
2888  // Slightly prefer to break before the first one in log-like statements.
2889  return 2;
2890  return 1;
2891  }
2892  if (Left.ClosesTemplateDeclaration)
2893  return Style.PenaltyBreakTemplateDeclaration;
2894  if (Left.is(TT_ConditionalExpr))
2895  return prec::Conditional;
2896  prec::Level Level = Left.getPrecedence();
2897  if (Level == prec::Unknown)
2898  Level = Right.getPrecedence();
2899  if (Level == prec::Assignment)
2900  return Style.PenaltyBreakAssignment;
2901  if (Level != prec::Unknown)
2902  return Level;
2903 
2904  return 3;
2905 }
2906 
2907 bool TokenAnnotator::spaceRequiredBeforeParens(const FormatToken &Right) const {
2908  return Style.SpaceBeforeParens == FormatStyle::SBPO_Always ||
2909  (Style.SpaceBeforeParens == FormatStyle::SBPO_NonEmptyParentheses &&
2910  Right.ParameterCount > 0);
2911 }
2912 
2913 bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
2914  const FormatToken &Left,
2915  const FormatToken &Right) {
2916  if (Left.is(tok::kw_return) && Right.isNot(tok::semi))
2917  return true;
2918  if (Style.isJson() && Left.is(tok::string_literal) && Right.is(tok::colon))
2919  return false;
2920  if (Left.is(Keywords.kw_assert) && Style.Language == FormatStyle::LK_Java)
2921  return true;
2922  if (Style.ObjCSpaceAfterProperty && Line.Type == LT_ObjCProperty &&
2923  Left.Tok.getObjCKeywordID() == tok::objc_property)
2924  return true;
2925  if (Right.is(tok::hashhash))
2926  return Left.is(tok::hash);
2927  if (Left.isOneOf(tok::hashhash, tok::hash))
2928  return Right.is(tok::hash);
2929  if ((Left.is(tok::l_paren) && Right.is(tok::r_paren)) ||
2930  (Left.is(tok::l_brace) && Left.isNot(BK_Block) &&
2931  Right.is(tok::r_brace) && Right.isNot(BK_Block)))
2932  return Style.SpaceInEmptyParentheses;
2933  if (Style.SpacesInConditionalStatement) {
2934  if (Left.is(tok::l_paren) && Left.Previous &&
2935  isKeywordWithCondition(*Left.Previous))
2936  return true;
2937  if (Right.is(tok::r_paren) && Right.MatchingParen &&
2938  Right.MatchingParen->Previous &&
2939  isKeywordWithCondition(*Right.MatchingParen->Previous))
2940  return true;
2941  }
2942 
2943  // requires ( or requires(
2944  if (Right.is(tok::l_paren) && Left.is(tok::kw_requires))
2945  return spaceRequiredBeforeParens(Right);
2946  // requires clause Concept1<T> && Concept2<T>
2947  if (Left.is(TT_ConstraintJunctions) && Right.is(tok::identifier))
2948  return true;
2949 
2950  if (Left.is(tok::l_paren) || Right.is(tok::r_paren))
2951  return (Right.is(TT_CastRParen) ||
2952  (Left.MatchingParen && Left.MatchingParen->is(TT_CastRParen)))
2953  ? Style.SpacesInCStyleCastParentheses
2954  : Style.SpacesInParentheses;
2955  if (Right.isOneOf(tok::semi, tok::comma))
2956  return false;
2957  if (Right.is(tok::less) && Line.Type == LT_ObjCDecl) {
2958  bool IsLightweightGeneric = Right.MatchingParen &&
2959  Right.MatchingParen->Next &&
2960  Right.MatchingParen->Next->is(tok::colon);
2961  return !IsLightweightGeneric && Style.ObjCSpaceBeforeProtocolList;
2962  }
2963  if (Right.is(tok::less) && Left.is(tok::kw_template))
2964  return Style.SpaceAfterTemplateKeyword;
2965  if (Left.isOneOf(tok::exclaim, tok::tilde))
2966  return false;
2967  if (Left.is(tok::at) &&
2968  Right.isOneOf(tok::identifier, tok::string_literal, tok::char_constant,
2969  tok::numeric_constant, tok::l_paren, tok::l_brace,
2970  tok::kw_true, tok::kw_false))
2971  return false;
2972  if (Left.is(tok::colon))
2973  return !Left.is(TT_ObjCMethodExpr);
2974  if (Left.is(tok::coloncolon))
2975  return false;
2976  if (Left.is(tok::less) || Right.isOneOf(tok::greater, tok::less)) {
2977  if (Style.Language == FormatStyle::LK_TextProto ||
2978  (Style.Language == FormatStyle::LK_Proto &&
2979  (Left.is(TT_DictLiteral) || Right.is(TT_DictLiteral)))) {
2980  // Format empty list as `<>`.
2981  if (Left.is(tok::less) && Right.is(tok::greater))
2982  return false;
2983  return !Style.Cpp11BracedListStyle;
2984  }
2985  return false;
2986  }
2987  if (Right.is(tok::ellipsis))
2988  return Left.Tok.isLiteral() || (Left.is(tok::identifier) && Left.Previous &&
2989  Left.Previous->is(tok::kw_case));
2990  if (Left.is(tok::l_square) && Right.is(tok::amp))
2991  return Style.SpacesInSquareBrackets;
2992  if (Right.is(TT_PointerOrReference)) {
2993  if (Left.is(tok::r_paren) && Line.MightBeFunctionDecl) {
2994  if (!Left.MatchingParen)
2995  return true;
2996  FormatToken *TokenBeforeMatchingParen =
2997  Left.MatchingParen->getPreviousNonComment();
2998  if (!TokenBeforeMatchingParen || !Left.is(TT_TypeDeclarationParen))
2999  return true;
3000  }
3001  // Add a space if the previous token is a pointer qualifier or the closing
3002  // parenthesis of __attribute__(()) expression and the style requires spaces
3003  // after pointer qualifiers.
3004  if ((Style.SpaceAroundPointerQualifiers == FormatStyle::SAPQ_After ||
3005  Style.SpaceAroundPointerQualifiers == FormatStyle::SAPQ_Both) &&
3006  (Left.is(TT_AttributeParen) || Left.canBePointerOrReferenceQualifier()))
3007  return true;
3008  return (
3009  Left.Tok.isLiteral() ||
3010  (!Left.isOneOf(TT_PointerOrReference, tok::l_paren) &&
3011  (getTokenPointerOrReferenceAlignment(Right) != FormatStyle::PAS_Left ||
3012  (Line.IsMultiVariableDeclStmt &&
3013  (Left.NestingLevel == 0 ||
3014  (Left.NestingLevel == 1 && Line.First->is(tok::kw_for)))))));
3015  }
3016  if (Right.is(TT_FunctionTypeLParen) && Left.isNot(tok::l_paren) &&
3017  (!Left.is(TT_PointerOrReference) ||
3018  (getTokenPointerOrReferenceAlignment(Left) != FormatStyle::PAS_Right &&
3019  !Line.IsMultiVariableDeclStmt)))
3020  return true;
3021  if (Left.is(TT_PointerOrReference)) {
3022  // Add a space if the next token is a pointer qualifier and the style
3023  // requires spaces before pointer qualifiers.
3024  if ((Style.SpaceAroundPointerQualifiers == FormatStyle::SAPQ_Before ||
3025  Style.SpaceAroundPointerQualifiers == FormatStyle::SAPQ_Both) &&
3026  Right.canBePointerOrReferenceQualifier())
3027  return true;
3028  return Right.Tok.isLiteral() || Right.is(TT_BlockComment) ||
3029  (Right.isOneOf(Keywords.kw_override, Keywords.kw_final) &&
3030  !Right.is(TT_StartOfName)) ||
3031  (Right.is(tok::l_brace) && Right.is(BK_Block)) ||
3032  (!Right.isOneOf(TT_PointerOrReference, TT_ArraySubscriptLSquare,
3033  tok::l_paren) &&
3034  (getTokenPointerOrReferenceAlignment(Left) !=
3036  !Line.IsMultiVariableDeclStmt) &&
3037  Left.Previous &&
3038  !Left.Previous->isOneOf(tok::l_paren, tok::coloncolon,
3039  tok::l_square));
3040  }
3041  // Ensure right pointer alignment with ellipsis e.g. int *...P
3042  if (Left.is(tok::ellipsis) && Left.Previous &&
3043  Left.Previous->isOneOf(tok::star, tok::amp, tok::ampamp))
3044  return Style.PointerAlignment != FormatStyle::PAS_Right;
3045 
3046  if (Right.is(tok::star) && Left.is(tok::l_paren))
3047  return false;
3048  if (Left.is(tok::star) && Right.isOneOf(tok::star, tok::amp, tok::ampamp))
3049  return false;
3050  if (Right.isOneOf(tok::star, tok::amp, tok::ampamp)) {
3051  const FormatToken *Previous = &Left;
3052  while (Previous && !Previous->is(tok::kw_operator)) {
3053  if (Previous->is(tok::identifier) || Previous->isSimpleTypeSpecifier()) {
3054  Previous = Previous->getPreviousNonComment();
3055  continue;
3056  }
3057  if (Previous->is(TT_TemplateCloser) && Previous->MatchingParen) {
3058  Previous = Previous->MatchingParen->getPreviousNonComment();
3059  continue;
3060  }
3061  if (Previous->is(tok::coloncolon)) {
3062  Previous = Previous->getPreviousNonComment();
3063  continue;
3064  }
3065  break;
3066  }
3067  // Space between the type and the * in:
3068  // operator void*()
3069  // operator char*()
3070  // operator void const*()
3071  // operator void volatile*()
3072  // operator /*comment*/ const char*()
3073  // operator volatile /*comment*/ char*()
3074  // operator Foo*()
3075  // operator C<T>*()
3076  // operator std::Foo*()
3077  // operator C<T>::D<U>*()
3078  // dependent on PointerAlignment style.
3079  if (Previous) {
3080  if (Previous->endsSequence(tok::kw_operator))
3081  return (Style.PointerAlignment != FormatStyle::PAS_Left);
3082  if (Previous->is(tok::kw_const) || Previous->is(tok::kw_volatile))
3083  return (Style.PointerAlignment != FormatStyle::PAS_Left) ||
3084  (Style.SpaceAroundPointerQualifiers ==
3086  (Style.SpaceAroundPointerQualifiers == FormatStyle::SAPQ_Both);
3087  }
3088  }
3089  const auto SpaceRequiredForArrayInitializerLSquare =
3090  [](const FormatToken &LSquareTok, const FormatStyle &Style) {
3091  return Style.SpacesInContainerLiterals ||
3092  ((Style.Language == FormatStyle::LK_Proto ||
3093  Style.Language == FormatStyle::LK_TextProto) &&
3094  !Style.Cpp11BracedListStyle &&
3095  LSquareTok.endsSequence(tok::l_square, tok::colon,
3096  TT_SelectorName));
3097  };
3098  if (Left.is(tok::l_square))
3099  return (Left.is(TT_ArrayInitializerLSquare) && Right.isNot(tok::r_square) &&
3100  SpaceRequiredForArrayInitializerLSquare(Left, Style)) ||
3101  (Left.isOneOf(TT_ArraySubscriptLSquare, TT_StructuredBindingLSquare,
3102  TT_LambdaLSquare) &&
3103  Style.SpacesInSquareBrackets && Right.isNot(tok::r_square));
3104  if (Right.is(tok::r_square))
3105  return Right.MatchingParen &&
3106  ((Right.MatchingParen->is(TT_ArrayInitializerLSquare) &&
3107  SpaceRequiredForArrayInitializerLSquare(*Right.MatchingParen,
3108  Style)) ||
3109  (Style.SpacesInSquareBrackets &&
3110  Right.MatchingParen->isOneOf(TT_ArraySubscriptLSquare,
3111  TT_StructuredBindingLSquare,
3112  TT_LambdaLSquare)) ||
3113  Right.MatchingParen->is(TT_AttributeParen));
3114  if (Right.is(tok::l_square) &&
3115  !Right.isOneOf(TT_ObjCMethodExpr, TT_LambdaLSquare,
3116  TT_DesignatedInitializerLSquare,
3117  TT_StructuredBindingLSquare, TT_AttributeSquare) &&
3118  !Left.isOneOf(tok::numeric_constant, TT_DictLiteral) &&
3119  !(!Left.is(tok::r_square) && Style.SpaceBeforeSquareBrackets &&
3120  Right.is(TT_ArraySubscriptLSquare)))
3121  return false;
3122  if (Left.is(tok::l_brace) && Right.is(tok::r_brace))
3123  return !Left.Children.empty(); // No spaces in "{}".
3124  if ((Left.is(tok::l_brace) && Left.isNot(BK_Block)) ||
3125  (Right.is(tok::r_brace) && Right.MatchingParen &&
3126  Right.MatchingParen->isNot(BK_Block)))
3127  return Style.Cpp11BracedListStyle ? Style.SpacesInParentheses : true;
3128  if (Left.is(TT_BlockComment))
3129  // No whitespace in x(/*foo=*/1), except for JavaScript.
3130  return Style.Language == FormatStyle::LK_JavaScript ||
3131  !Left.TokenText.endswith("=*/");
3132 
3133  // Space between template and attribute.
3134  // e.g. template <typename T> [[nodiscard]] ...
3135  if (Left.is(TT_TemplateCloser) && Right.is(TT_AttributeSquare))
3136  return true;
3137  if (Right.is(tok::l_paren)) {
3138  if ((Left.is(tok::r_paren) && Left.is(TT_AttributeParen)) ||
3139  (Left.is(tok::r_square) && Left.is(TT_AttributeSquare)))
3140  return true;
3141  if (Style.SpaceBeforeParens ==
3143  Left.is(TT_ForEachMacro))
3144  return false;
3145  if (Style.SpaceBeforeParens ==
3147  Left.is(TT_IfMacro))
3148  return false;
3149  return Line.Type == LT_ObjCDecl || Left.is(tok::semi) ||
3150  (Style.SpaceBeforeParens != FormatStyle::SBPO_Never &&
3151  (Left.isOneOf(tok::pp_elif, tok::kw_for, tok::kw_while,
3152  tok::kw_switch, tok::kw_case, TT_ForEachMacro,
3153  TT_ObjCForIn) ||
3154  Left.isIf(Line.Type != LT_PreprocessorDirective) ||
3155  (Left.isOneOf(tok::kw_try, Keywords.kw___except, tok::kw_catch,
3156  tok::kw_new, tok::kw_delete) &&
3157  (!Left.Previous || Left.Previous->isNot(tok::period))))) ||
3158  (spaceRequiredBeforeParens(Right) &&
3159  (Left.is(tok::identifier) || Left.isFunctionLikeKeyword() ||
3160  Left.is(tok::r_paren) || Left.isSimpleTypeSpecifier() ||
3161  (Left.is(tok::r_square) && Left.MatchingParen &&
3162  Left.MatchingParen->is(TT_LambdaLSquare))) &&
3163  Line.Type != LT_PreprocessorDirective);
3164  }
3165  if (Left.is(tok::at) && Right.Tok.getObjCKeywordID() != tok::objc_not_keyword)
3166  return false;
3167  if (Right.is(TT_UnaryOperator))
3168  return !Left.isOneOf(tok::l_paren, tok::l_square, tok::at) &&
3169  (Left.isNot(tok::colon) || Left.isNot(TT_ObjCMethodExpr));
3170  if ((Left.isOneOf(tok::identifier, tok::greater, tok::r_square,
3171  tok::r_paren) ||
3172  Left.isSimpleTypeSpecifier()) &&
3173  Right.is(tok::l_brace) && Right.getNextNonComment() &&
3174  Right.isNot(BK_Block))
3175  return false;
3176  if (Left.is(tok::period) || Right.is(tok::period))
3177  return false;
3178  if (Right.is(tok::hash) && Left.is(tok::identifier) && Left.TokenText == "L")
3179  return false;
3180  if (Left.is(TT_TemplateCloser) && Left.MatchingParen &&
3181  Left.MatchingParen->Previous &&
3182  (Left.MatchingParen->Previous->is(tok::period) ||
3183  Left.MatchingParen->Previous->is(tok::coloncolon)))
3184  // Java call to generic function with explicit type:
3185  // A.<B<C<...>>>DoSomething();
3186  // A::<B<C<...>>>DoSomething(); // With a Java 8 method reference.
3187  return false;
3188  if (Left.is(TT_TemplateCloser) && Right.is(tok::l_square))
3189  return false;
3190  if (Left.is(tok::l_brace) && Left.endsSequence(TT_DictLiteral, tok::at))
3191  // Objective-C dictionary literal -> no space after opening brace.
3192  return false;
3193  if (Right.is(tok::r_brace) && Right.MatchingParen &&
3194  Right.MatchingParen->endsSequence(TT_DictLiteral, tok::at))
3195  // Objective-C dictionary literal -> no space before closing brace.
3196  return false;
3197  if (Right.getType() == TT_TrailingAnnotation &&
3198  Right.isOneOf(tok::amp, tok::ampamp) &&
3199  Left.isOneOf(tok::kw_const, tok::kw_volatile) &&
3200  (!Right.Next || Right.Next->is(tok::semi)))
3201  // Match const and volatile ref-qualifiers without any additional
3202  // qualifiers such as
3203  // void Fn() const &;
3204  return getTokenReferenceAlignment(Right) != FormatStyle::PAS_Left;
3205  return true;
3206 }
3207 
3208 bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
3209  const FormatToken &Right) {
3210  const FormatToken &Left = *Right.Previous;
3211  auto HasExistingWhitespace = [&Right]() {
3212  return Right.WhitespaceRange.getBegin() != Right.WhitespaceRange.getEnd();
3213  };
3214  if (Right.Tok.getIdentifierInfo() && Left.Tok.getIdentifierInfo())
3215  return true; // Never ever merge two identifiers.
3216  if (Style.isCpp()) {
3217  if (Left.is(tok::kw_operator))
3218  return Right.is(tok::coloncolon);
3219  if (Right.is(tok::l_brace) && Right.is(BK_BracedInit) &&
3220  !Left.opensScope() && Style.SpaceBeforeCpp11BracedList)
3221  return true;
3222  } else if (Style.Language == FormatStyle::LK_Proto ||
3223  Style.Language == FormatStyle::LK_TextProto) {
3224  if (Right.is(tok::period) &&
3225  Left.isOneOf(Keywords.kw_optional, Keywords.kw_required,
3226  Keywords.kw_repeated, Keywords.kw_extend))
3227  return true;
3228  if (Right.is(tok::l_paren) &&
3229  Left.isOneOf(Keywords.kw_returns, Keywords.kw_option))
3230  return true;
3231  if (Right.isOneOf(tok::l_brace, tok::less) && Left.is(TT_SelectorName))
3232  return true;
3233  // Slashes occur in text protocol extension syntax: [type/type] { ... }.
3234  if (Left.is(tok::slash) || Right.is(tok::slash))
3235  return false;
3236  if (Left.MatchingParen &&
3237  Left.MatchingParen->is(TT_ProtoExtensionLSquare) &&
3238  Right.isOneOf(tok::l_brace, tok::less))
3239  return !Style.Cpp11BracedListStyle;
3240  // A percent is probably part of a formatting specification, such as %lld.
3241  if (Left.is(tok::percent))
3242  return false;
3243  // Preserve the existence of a space before a percent for cases like 0x%04x
3244  // and "%d %d"
3245  if (Left.is(tok::numeric_constant) && Right.is(tok::percent))
3246  return HasExistingWhitespace();
3247  } else if (Style.isJson()) {
3248  if (Right.is(tok::colon))
3249  return false;
3250  } else if (Style.isCSharp()) {
3251  // Require spaces around '{' and before '}' unless they appear in
3252  // interpolated strings. Interpolated strings are merged into a single token
3253  // so cannot have spaces inserted by this function.
3254 
3255  // No space between 'this' and '['
3256  if (Left.is(tok::kw_this) && Right.is(tok::l_square))
3257  return false;
3258 
3259  // No space between 'new' and '('
3260  if (Left.is(tok::kw_new) && Right.is(tok::l_paren))
3261  return false;
3262 
3263  // Space before { (including space within '{ {').
3264  if (Right.is(tok::l_brace))
3265  return true;
3266 
3267  // Spaces inside braces.
3268  if (Left.is(tok::l_brace) && Right.isNot(tok::r_brace))
3269  return true;
3270 
3271  if (Left.isNot(tok::l_brace) && Right.is(tok::r_brace))
3272  return true;
3273 
3274  // Spaces around '=>'.
3275  if (Left.is(TT_FatArrow) || Right.is(TT_FatArrow))
3276  return true;
3277 
3278  // No spaces around attribute target colons
3279  if (Left.is(TT_AttributeColon) || Right.is(TT_AttributeColon))
3280  return false;
3281 
3282  // space between type and variable e.g. Dictionary<string,string> foo;
3283  if (Left.is(TT_TemplateCloser) && Right.is(TT_StartOfName))
3284  return true;
3285 
3286  // spaces inside square brackets.
3287  if (Left.is(tok::l_square) || Right.is(tok::r_square))
3288  return Style.SpacesInSquareBrackets;
3289 
3290  // No space before ? in nullable types.
3291  if (Right.is(TT_CSharpNullable))
3292  return false;
3293 
3294  // No space before null forgiving '!'.
3295  if (Right.is(TT_NonNullAssertion))
3296  return false;
3297 
3298  // No space between consecutive commas '[,,]'.
3299  if (Left.is(tok::comma) && Right.is(tok::comma))
3300  return false;
3301 
3302  // space after var in `var (key, value)`
3303  if (Left.is(Keywords.kw_var) && Right.is(tok::l_paren))
3304  return true;
3305 
3306  // space between keywords and paren e.g. "using ("
3307  if (Right.is(tok::l_paren))
3308  if (Left.isOneOf(tok::kw_using, Keywords.kw_async, Keywords.kw_when,
3309  Keywords.kw_lock))
3310  return Style.SpaceBeforeParens == FormatStyle::SBPO_ControlStatements ||
3311  spaceRequiredBeforeParens(Right);
3312 
3313  // space between method modifier and opening parenthesis of a tuple return
3314  // type
3315  if (Left.isOneOf(tok::kw_public, tok::kw_private, tok::kw_protected,
3316  tok::kw_virtual, tok::kw_extern, tok::kw_static,
3317  Keywords.kw_internal, Keywords.kw_abstract,
3318  Keywords.kw_sealed, Keywords.kw_override,
3319  Keywords.kw_async, Keywords.kw_unsafe) &&
3320  Right.is(tok::l_paren))
3321  return true;
3322  } else if (Style.Language == FormatStyle::LK_JavaScript) {
3323  if (Left.is(TT_FatArrow))
3324  return true;
3325  // for await ( ...
3326  if (Right.is(tok::l_paren) && Left.is(Keywords.kw_await) && Left.Previous &&
3327  Left.Previous->is(tok::kw_for))
3328  return true;
3329  if (Left.is(Keywords.kw_async) && Right.is(tok::l_paren) &&
3330  Right.MatchingParen) {
3331  const FormatToken *Next = Right.MatchingParen->getNextNonComment();
3332  // An async arrow function, for example: `x = async () => foo();`,
3333  // as opposed to calling a function called async: `x = async();`
3334  if (Next && Next->is(TT_FatArrow))
3335  return true;
3336  }
3337  if ((Left.is(TT_TemplateString) && Left.TokenText.endswith("${")) ||
3338  (Right.is(TT_TemplateString) && Right.TokenText.startswith("}")))
3339  return false;
3340  // In tagged template literals ("html`bar baz`"), there is no space between
3341  // the tag identifier and the template string.
3342  if (Keywords.IsJavaScriptIdentifier(Left,
3343  /* AcceptIdentifierName= */ false) &&
3344  Right.is(TT_TemplateString))
3345  return false;
3346  if (Right.is(tok::star) &&
3347  Left.isOneOf(Keywords.kw_function, Keywords.kw_yield))
3348  return false;
3349  if (Right.isOneOf(tok::l_brace, tok::l_square) &&
3350  Left.isOneOf(Keywords.kw_function, Keywords.kw_yield,
3351  Keywords.kw_extends, Keywords.kw_implements))
3352  return true;
3353  if (Right.is(tok::l_paren)) {
3354  // JS methods can use some keywords as names (e.g. `delete()`).
3355  if (Line.MustBeDeclaration && Left.Tok.getIdentifierInfo())
3356  return false;
3357  // Valid JS method names can include keywords, e.g. `foo.delete()` or
3358  // `bar.instanceof()`. Recognize call positions by preceding period.
3359  if (Left.Previous && Left.Previous->is(tok::period) &&
3360  Left.Tok.getIdentifierInfo())
3361  return false;
3362  // Additional unary JavaScript operators that need a space after.
3363  if (Left.isOneOf(tok::kw_throw, Keywords.kw_await, Keywords.kw_typeof,
3364  tok::kw_void))
3365  return true;
3366  }
3367  // `foo as const;` casts into a const type.
3368  if (Left.endsSequence(tok::kw_const, Keywords.kw_as)) {
3369  return false;
3370  }
3371  if ((Left.isOneOf(Keywords.kw_let, Keywords.kw_var, Keywords.kw_in,
3372  tok::kw_const) ||
3373  // "of" is only a keyword if it appears after another identifier
3374  // (e.g. as "const x of y" in a for loop), or after a destructuring
3375  // operation (const [x, y] of z, const {a, b} of c).
3376  (Left.is(Keywords.kw_of) && Left.Previous &&
3377  (Left.Previous->Tok.is(tok::identifier) ||
3378  Left.Previous->isOneOf(tok::r_square, tok::r_brace)))) &&
3379  (!Left.Previous || !Left.Previous->is(tok::period)))
3380  return true;
3381  if (Left.isOneOf(tok::kw_for, Keywords.kw_as) && Left.Previous &&
3382  Left.Previous->is(tok::period) && Right.is(tok::l_paren))
3383  return false;
3384  if (Left.is(Keywords.kw_as) &&
3385  Right.isOneOf(tok::l_square, tok::l_brace, tok::l_paren))
3386  return true;
3387  if (Left.is(tok::kw_default) && Left.Previous &&
3388  Left.Previous->is(tok::kw_export))
3389  return true;
3390  if (Left.is(Keywords.kw_is) && Right.is(tok::l_brace))
3391  return true;
3392  if (Right.isOneOf(TT_JsTypeColon, TT_JsTypeOptionalQuestion))
3393  return false;
3394  if (Left.is(TT_JsTypeOperator) || Right.is(TT_JsTypeOperator))
3395  return false;
3396  if ((Left.is(tok::l_brace) || Right.is(tok::r_brace)) &&
3397  Line.First->isOneOf(Keywords.kw_import, tok::kw_export))
3398  return false;
3399  if (Left.is(tok::ellipsis))
3400  return false;
3401  if (Left.is(TT_TemplateCloser) &&
3402  !Right.isOneOf(tok::equal, tok::l_brace, tok::comma, tok::l_square,
3403  Keywords.kw_implements, Keywords.kw_extends))
3404  // Type assertions ('<type>expr') are not followed by whitespace. Other
3405  // locations that should have whitespace following are identified by the
3406  // above set of follower tokens.
3407  return false;
3408  if (Right.is(TT_NonNullAssertion))
3409  return false;
3410  if (Left.is(TT_NonNullAssertion) &&
3411  Right.isOneOf(Keywords.kw_as, Keywords.kw_in))
3412  return true; // "x! as string", "x! in y"
3413  } else if (Style.Language == FormatStyle::LK_Java) {
3414  if (Left.is(tok::r_square) && Right.is(tok::l_brace))
3415  return true;
3416  if (Left.is(Keywords.kw_synchronized) && Right.is(tok::l_paren))
3417  return Style.SpaceBeforeParens != FormatStyle::SBPO_Never;
3418  if ((Left.isOneOf(tok::kw_static, tok::kw_public, tok::kw_private,
3419  tok::kw_protected) ||
3420  Left.isOneOf(Keywords.kw_final, Keywords.kw_abstract,
3421  Keywords.kw_native)) &&
3422  Right.is(TT_TemplateOpener))
3423  return true;
3424  }
3425  if (Left.is(TT_ImplicitStringLiteral))
3426  return HasExistingWhitespace();
3427  if (Line.Type == LT_ObjCMethodDecl) {
3428  if (Left.is(TT_ObjCMethodSpecifier))
3429  return true;
3430  if (Left.is(tok::r_paren) && canBeObjCSelectorComponent(Right))
3431  // Don't space between ')' and <id> or ')' and 'new'. 'new' is not a
3432  // keyword in Objective-C, and '+ (instancetype)new;' is a standard class
3433  // method declaration.
3434  return false;
3435  }
3436  if (Line.Type == LT_ObjCProperty &&
3437  (Right.is(tok::equal) || Left.is(tok::equal)))
3438  return false;
3439 
3440  if (Right.isOneOf(TT_TrailingReturnArrow, TT_LambdaArrow) ||
3441  Left.isOneOf(TT_TrailingReturnArrow, TT_LambdaArrow))
3442  return true;
3443  if (Right.is(TT_OverloadedOperatorLParen))
3444  return spaceRequiredBeforeParens(Right);
3445  if (Left.is(tok::comma))
3446  return true;
3447  if (Right.is(tok::comma))
3448  return false;
3449  if (Right.is(TT_ObjCBlockLParen))
3450  return true;
3451  if (Right.is(TT_CtorInitializerColon))
3452  return Style.SpaceBeforeCtorInitializerColon;
3453  if (Right.is(TT_InheritanceColon) && !Style.SpaceBeforeInheritanceColon)
3454  return false;
3455  if (Right.is(TT_RangeBasedForLoopColon) &&
3456  !Style.SpaceBeforeRangeBasedForLoopColon)
3457  return false;
3458  if (Left.is(TT_BitFieldColon))
3459  return Style.BitFieldColonSpacing == FormatStyle::BFCS_Both ||
3460  Style.BitFieldColonSpacing == FormatStyle::BFCS_After;
3461  if (Right.is(tok::colon)) {
3462  if (Line.First->isOneOf(tok::kw_default, tok::kw_case))
3463  return Style.SpaceBeforeCaseColon;
3464  if (!Right.getNextNonComment() || Right.getNextNonComment()->is(tok::semi))
3465  return false;
3466  if (Right.is(TT_ObjCMethodExpr))
3467  return false;
3468  if (Left.is(tok::question))
3469  return false;
3470  if (Right.is(TT_InlineASMColon) && Left.is(tok::coloncolon))
3471  return false;
3472  if (Right.is(TT_DictLiteral))
3473  return Style.SpacesInContainerLiterals;
3474  if (Right.is(TT_AttributeColon))
3475  return false;
3476  if (Right.is(TT_CSharpNamedArgumentColon))
3477  return false;
3478  if (Right.is(TT_BitFieldColon))
3479  return Style.BitFieldColonSpacing == FormatStyle::BFCS_Both ||
3480  Style.BitFieldColonSpacing == FormatStyle::BFCS_Before;
3481  return true;
3482  }
3483  // Do not merge "- -" into "--".
3484  if ((Left.isOneOf(tok::minus, tok::minusminus) &&
3485  Right.isOneOf(tok::minus, tok::minusminus)) ||
3486  (Left.isOneOf(tok::plus, tok::plusplus) &&
3487  Right.isOneOf(tok::plus, tok::plusplus)))
3488  return true;
3489  if (Left.is(TT_UnaryOperator)) {
3490  if (!Right.is(tok::l_paren)) {
3491  // The alternative operators for ~ and ! are "compl" and "not".
3492  // If they are used instead, we do not want to combine them with
3493  // the token to the right, unless that is a left paren.
3494  if (Left.is(tok::exclaim) && Left.TokenText == "not")
3495  return true;
3496  if (Left.is(tok::tilde) && Left.TokenText == "compl")
3497  return true;
3498  // Lambda captures allow for a lone &, so "&]" needs to be properly
3499  // handled.
3500  if (Left.is(tok::amp) && Right.is(tok::r_square))
3501  return Style.SpacesInSquareBrackets;
3502  }
3503  return (Style.SpaceAfterLogicalNot && Left.is(tok::exclaim)) ||
3504  Right.is(TT_BinaryOperator);
3505  }
3506 
3507  // If the next token is a binary operator or a selector name, we have
3508  // incorrectly classified the parenthesis as a cast. FIXME: Detect correctly.
3509  if (Left.is(TT_CastRParen))
3510  return Style.SpaceAfterCStyleCast ||
3511  Right.isOneOf(TT_BinaryOperator, TT_SelectorName);
3512 
3513  auto ShouldAddSpacesInAngles = [this, &HasExistingWhitespace]() {
3514  if (this->Style.SpacesInAngles == FormatStyle::SIAS_Always)
3515  return true;
3516  if (this->Style.SpacesInAngles == FormatStyle::SIAS_Leave)
3517  return HasExistingWhitespace();
3518  return false;
3519  };
3520 
3521  if (Left.is(tok::greater) && Right.is(tok::greater)) {
3522  if (Style.Language == FormatStyle::LK_TextProto ||
3523  (Style.Language == FormatStyle::LK_Proto && Left.is(TT_DictLiteral)))
3524  return !Style.Cpp11BracedListStyle;
3525  return Right.is(TT_TemplateCloser) && Left.is(TT_TemplateCloser) &&
3526  ((Style.Standard < FormatStyle::LS_Cpp11) ||
3527  ShouldAddSpacesInAngles());
3528  }
3529  if (Right.isOneOf(tok::arrow, tok::arrowstar, tok::periodstar) ||
3530  Left.isOneOf(tok::arrow, tok::period, tok::arrowstar, tok::periodstar) ||
3531  (Right.is(tok::period) && Right.isNot(TT_DesignatedInitializerPeriod)))
3532  return false;
3533  if (!Style.SpaceBeforeAssignmentOperators && Left.isNot(TT_TemplateCloser) &&
3534  Right.getPrecedence() == prec::Assignment)
3535  return false;
3536  if (Style.Language == FormatStyle::LK_Java && Right.is(tok::coloncolon) &&
3537  (Left.is(tok::identifier) || Left.is(tok::kw_this)))
3538  return false;
3539  if (Right.is(tok::coloncolon) && Left.is(tok::identifier))
3540  // Generally don't remove existing spaces between an identifier and "::".
3541  // The identifier might actually be a macro name such as ALWAYS_INLINE. If
3542  // this turns out to be too lenient, add analysis of the identifier itself.
3543  return HasExistingWhitespace();
3544  if (Right.is(tok::coloncolon) &&
3545  !Left.isOneOf(tok::l_brace, tok::comment, tok::l_paren))
3546  // Put a space between < and :: in vector< ::std::string >
3547  return (Left.is(TT_TemplateOpener) &&
3548  ((Style.Standard < FormatStyle::LS_Cpp11) ||
3549  ShouldAddSpacesInAngles())) ||
3550  !(Left.isOneOf(tok::l_paren, tok::r_paren, tok::l_square,
3551  tok::kw___super, TT_TemplateOpener,
3552  TT_TemplateCloser)) ||
3553  (Left.is(tok::l_paren) && Style.SpacesInParentheses);
3554  if ((Left.is(TT_TemplateOpener)) != (Right.is(TT_TemplateCloser)))
3555  return ShouldAddSpacesInAngles();
3556  // Space before TT_StructuredBindingLSquare.
3557  if (Right.is(TT_StructuredBindingLSquare))
3558  return !Left.isOneOf(tok::amp, tok::ampamp) ||
3559  getTokenReferenceAlignment(Left) != FormatStyle::PAS_Right;
3560  // Space before & or && following a TT_StructuredBindingLSquare.
3561  if (Right.Next && Right.Next->is(TT_StructuredBindingLSquare) &&
3562  Right.isOneOf(tok::amp, tok::ampamp))
3563  return getTokenReferenceAlignment(Right) != FormatStyle::PAS_Left;
3564  if ((Right.is(TT_BinaryOperator) && !Left.is(tok::l_paren)) ||
3565  (Left.isOneOf(TT_BinaryOperator, TT_ConditionalExpr) &&
3566  !Right.is(tok::r_paren)))
3567  return true;
3568  if (Left.is(TT_TemplateCloser) && Right.is(tok::l_paren) &&
3569  Right.isNot(TT_FunctionTypeLParen))
3570  return spaceRequiredBeforeParens(Right);
3571  if (Right.is(TT_TemplateOpener) && Left.is(tok::r_paren) &&
3572  Left.MatchingParen && Left.MatchingParen->is(TT_OverloadedOperatorLParen))
3573  return false;
3574  if (Right.is(tok::less) && Left.isNot(tok::l_paren) &&
3575  Line.startsWith(tok::hash))
3576  return true;
3577  if (Right.is(TT_TrailingUnaryOperator))
3578  return false;
3579  if (Left.is(TT_RegexLiteral))
3580  return false;
3581  return spaceRequiredBetween(Line, Left, Right);
3582 }
3583 
3584 // Returns 'true' if 'Tok' is a brace we'd want to break before in Allman style.
3585 static bool isAllmanBrace(const FormatToken &Tok) {
3586  return Tok.is(tok::l_brace) && Tok.is(BK_Block) &&
3587  !Tok.isOneOf(TT_ObjCBlockLBrace, TT_LambdaLBrace, TT_DictLiteral);
3588 }
3589 
3590 // Returns 'true' if 'Tok' is a function argument.
3591 static bool IsFunctionArgument(const FormatToken &Tok) {
3592  return Tok.MatchingParen && Tok.MatchingParen->Next &&
3593  Tok.MatchingParen->Next->isOneOf(tok::comma, tok::r_paren);
3594 }
3595 
3596 static bool
3598  FormatStyle::ShortLambdaStyle ShortLambdaOption) {
3599  return Tok.Children.empty() && ShortLambdaOption != FormatStyle::SLS_None;
3600 }
3601 
3602 static bool isAllmanLambdaBrace(const FormatToken &Tok) {
3603  return (Tok.is(tok::l_brace) && Tok.is(BK_Block) &&
3604  !Tok.isOneOf(TT_ObjCBlockLBrace, TT_DictLiteral));
3605 }
3606 
3607 // Returns the first token on the line that is not a comment.
3609  const FormatToken *Next = Line.First;
3610  if (!Next)
3611  return Next;
3612  if (Next->is(tok::comment))
3613  Next = Next->getNextNonComment();
3614  return Next;
3615 }
3616 
3617 bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
3618  const FormatToken &Right) {
3619  const FormatToken &Left = *Right.Previous;
3620  if (Right.NewlinesBefore > 1 && Style.MaxEmptyLinesToKeep > 0)
3621  return true;
3622 
3623  if (Style.isCSharp()) {
3624  if (Right.is(TT_CSharpNamedArgumentColon) ||
3625  Left.is(TT_CSharpNamedArgumentColon))
3626  return false;
3627  if (Right.is(TT_CSharpGenericTypeConstraint))
3628  return true;
3629 
3630  // Break after C# [...] and before public/protected/private/internal.
3631  if (Left.is(TT_AttributeSquare) && Left.is(tok::r_square) &&
3632  (Right.isAccessSpecifier(/*ColonRequired=*/false) ||
3633  Right.is(Keywords.kw_internal)))
3634  return true;
3635  // Break between ] and [ but only when there are really 2 attributes.
3636  if (Left.is(TT_AttributeSquare) && Right.is(TT_AttributeSquare) &&
3637  Left.is(tok::r_square) && Right.is(tok::l_square))
3638  return true;
3639 
3640  } else if (Style.Language == FormatStyle::LK_JavaScript) {
3641  // FIXME: This might apply to other languages and token kinds.
3642  if (Right.is(tok::string_literal) && Left.is(tok::plus) && Left.Previous &&
3643  Left.Previous->is(tok::string_literal))
3644  return true;
3645  if (Left.is(TT_DictLiteral) && Left.is(tok::l_brace) && Line.Level == 0 &&
3646  Left.Previous && Left.Previous->is(tok::equal) &&
3647  Line.First->isOneOf(tok::identifier, Keywords.kw_import, tok::kw_export,
3648  tok::kw_const) &&
3649  // kw_var/kw_let are pseudo-tokens that are tok::identifier, so match
3650  // above.
3651  !Line.First->isOneOf(Keywords.kw_var, Keywords.kw_let))
3652  // Object literals on the top level of a file are treated as "enum-style".
3653  // Each key/value pair is put on a separate line, instead of bin-packing.
3654  return true;
3655  if (Left.is(tok::l_brace) && Line.Level == 0 &&
3656  (Line.startsWith(tok::kw_enum) ||
3657  Line.startsWith(tok::kw_const, tok::kw_enum) ||
3658  Line.startsWith(tok::kw_export, tok::kw_enum) ||
3659  Line.startsWith(tok::kw_export, tok::kw_const, tok::kw_enum)))
3660  // JavaScript top-level enum key/value pairs are put on separate lines
3661  // instead of bin-packing.
3662  return true;
3663  if (Right.is(tok::r_brace) && Left.is(tok::l_brace) && Left.Previous &&
3664  Left.Previous->is(TT_FatArrow)) {
3665  // JS arrow function (=> {...}).
3666  switch (Style.AllowShortLambdasOnASingleLine) {
3667  case FormatStyle::SLS_All:
3668  return false;
3669  case FormatStyle::SLS_None:
3670  return true;
3672  return !Left.Children.empty();
3674  // allow one-lining inline (e.g. in function call args) and empty arrow
3675  // functions.
3676  return (Left.NestingLevel == 0 && Line.Level == 0) &&
3677  !Left.Children.empty();
3678  }
3679  llvm_unreachable("Unknown FormatStyle::ShortLambdaStyle enum");
3680  }
3681 
3682  if (Right.is(tok::r_brace) && Left.is(tok::l_brace) &&
3683  !Left.Children.empty())
3684  // Support AllowShortFunctionsOnASingleLine for JavaScript.
3685  return Style.AllowShortFunctionsOnASingleLine == FormatStyle::SFS_None ||
3686  Style.AllowShortFunctionsOnASingleLine == FormatStyle::SFS_Empty ||
3687  (Left.NestingLevel == 0 && Line.Level == 0 &&
3688  Style.AllowShortFunctionsOnASingleLine &
3690  } else if (Style.Language == FormatStyle::LK_Java) {
3691  if (Right.is(tok::plus) && Left.is(tok::string_literal) && Right.Next &&
3692  Right.Next->is(tok::string_literal))
3693  return true;
3694  } else if (Style.Language == FormatStyle::LK_Cpp ||
3695  Style.Language == FormatStyle::LK_ObjC ||
3696  Style.Language == FormatStyle::LK_Proto ||
3697  Style.Language == FormatStyle::LK_TableGen ||
3698  Style.Language == FormatStyle::LK_TextProto) {
3699  if (Left.isStringLiteral() && Right.isStringLiteral())
3700  return true;
3701  }
3702 
3703  // Basic JSON newline processing.
3704  if (Style.isJson()) {
3705  // Always break after a JSON record opener.
3706  // {
3707  // }
3708  if (Left.is(TT_DictLiteral) && Left.is(tok::l_brace))
3709  return true;
3710  // Always break after a JSON array opener.
3711  // [
3712  // ]
3713  if (Left.is(TT_ArrayInitializerLSquare) && Left.is(tok::l_square) &&
3714  !Right.is(tok::r_square))
3715  return true;
3716  // Always break after successive entries.
3717  // 1,
3718  // 2
3719  if (Left.is(tok::comma))
3720  return true;
3721  }
3722 
3723  // If the last token before a '}', ']', or ')' is a comma or a trailing
3724  // comment, the intention is to insert a line break after it in order to make
3725  // shuffling around entries easier. Import statements, especially in
3726  // JavaScript, can be an exception to this rule.
3727  if (Style.JavaScriptWrapImports || Line.Type != LT_ImportStatement) {
3728  const FormatToken *BeforeClosingBrace = nullptr;
3729  if ((Left.isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) ||
3730  (Style.Language == FormatStyle::LK_JavaScript &&
3731  Left.is(tok::l_paren))) &&
3732  Left.isNot(BK_Block) && Left.MatchingParen)
3733  BeforeClosingBrace = Left.MatchingParen->Previous;
3734  else if (Right.MatchingParen &&
3735  (Right.MatchingParen->isOneOf(tok::l_brace,
3736  TT_ArrayInitializerLSquare) ||
3737  (Style.Language == FormatStyle::LK_JavaScript &&
3738  Right.MatchingParen->is(tok::l_paren))))
3739  BeforeClosingBrace = &Left;
3740  if (BeforeClosingBrace && (BeforeClosingBrace->is(tok::comma) ||
3741  BeforeClosingBrace->isTrailingComment()))
3742  return true;
3743  }
3744 
3745  if (Right.is(tok::comment))
3746  return Left.isNot(BK_BracedInit) && Left.isNot(TT_CtorInitializerColon) &&
3747  (Right.NewlinesBefore > 0 && Right.HasUnescapedNewline);
3748  if (Left.isTrailingComment())
3749  return true;
3750  if (Right.Previous->IsUnterminatedLiteral)
3751  return true;
3752  if (Right.is(tok::lessless) && Right.Next &&
3753  Right.Previous->is(tok::string_literal) &&
3754  Right.Next->is(tok::string_literal))
3755  return true;
3756  // Can break after template<> declaration
3757  if (Right.Previous->ClosesTemplateDeclaration &&
3758  Right.Previous->MatchingParen &&
3759  Right.Previous->MatchingParen->NestingLevel == 0) {
3760  // Put concepts on the next line e.g.
3761  // template<typename T>
3762  // concept ...
3763  if (Right.is(tok::kw_concept))
3764  return Style.BreakBeforeConceptDeclarations;
3765  return (Style.AlwaysBreakTemplateDeclarations == FormatStyle::BTDS_Yes);
3766  }
3767  if (Style.PackConstructorInitializers == FormatStyle::PCIS_Never) {
3768  if (Style.BreakConstructorInitializers == FormatStyle::BCIS_BeforeColon &&
3769  (Left.is(TT_CtorInitializerComma) || Right.is(TT_CtorInitializerColon)))
3770  return true;
3771 
3772  if (Style.BreakConstructorInitializers == FormatStyle::BCIS_AfterColon &&
3773  Left.isOneOf(TT_CtorInitializerColon, TT_CtorInitializerComma))
3774  return true;
3775  }
3776  if (Style.PackConstructorInitializers < FormatStyle::PCIS_CurrentLine &&
3777  Style.BreakConstructorInitializers == FormatStyle::BCIS_BeforeComma &&
3778  Right.isOneOf(TT_CtorInitializerComma, TT_CtorInitializerColon))
3779  return true;
3780  // Break only if we have multiple inheritance.
3781  if (Style.BreakInheritanceList == FormatStyle::BILS_BeforeComma &&
3782  Right.is(TT_InheritanceComma))
3783  return true;
3784  if (Style.BreakInheritanceList == FormatStyle::BILS_AfterComma &&
3785  Left.is(TT_InheritanceComma))
3786  return true;
3787  if (Right.is(tok::string_literal) && Right.TokenText.startswith("R\""))
3788  // Multiline raw string literals are special wrt. line breaks. The author
3789  // has made a deliberate choice and might have aligned the contents of the
3790  // string literal accordingly. Thus, we try keep existing line breaks.
3791  return Right.IsMultiline && Right.NewlinesBefore > 0;
3792  if ((Right.Previous->is(tok::l_brace) ||
3793  (Right.Previous->is(tok::less) && Right.Previous->Previous &&
3794  Right.Previous->Previous->is(tok::equal))) &&
3795  Right.NestingLevel == 1 && Style.Language == FormatStyle::LK_Proto) {
3796  // Don't put enums or option definitions onto single lines in protocol
3797  // buffers.
3798  return true;
3799  }
3800  if (Right.is(TT_InlineASMBrace))
3801  return Right.HasUnescapedNewline;
3802 
3803  if (isAllmanBrace(Left) || isAllmanBrace(Right)) {
3804  auto FirstNonComment = getFirstNonComment(Line);
3805  bool AccessSpecifier =
3806  FirstNonComment &&
3807  FirstNonComment->isOneOf(Keywords.kw_internal, tok::kw_public,
3808  tok::kw_private, tok::kw_protected);
3809 
3810  if (Style.BraceWrapping.AfterEnum) {
3811  if (Line.startsWith(tok::kw_enum) ||
3812  Line.startsWith(tok::kw_typedef, tok::kw_enum))
3813  return true;
3814  // Ensure BraceWrapping for `public enum A {`.
3815  if (AccessSpecifier && FirstNonComment->Next &&
3816  FirstNonComment->Next->is(tok::kw_enum))
3817  return true;
3818  }
3819 
3820  // Ensure BraceWrapping for `public interface A {`.
3821  if (Style.BraceWrapping.AfterClass &&
3822  ((AccessSpecifier && FirstNonComment->Next &&
3823  FirstNonComment->Next->is(Keywords.kw_interface)) ||
3824  Line.startsWith(Keywords.kw_interface)))
3825  return true;
3826 
3827  return (Line.startsWith(tok::kw_class) && Style.BraceWrapping.AfterClass) ||
3828  (Line.startsWith(tok::kw_struct) && Style.BraceWrapping.AfterStruct);
3829  }
3830 
3831  if (Left.is(TT_ObjCBlockLBrace) &&
3832  Style.AllowShortBlocksOnASingleLine == FormatStyle::SBS_Never)
3833  return true;
3834 
3835  if (Left.is(TT_LambdaLBrace)) {
3836  if (IsFunctionArgument(Left) &&
3837  Style.AllowShortLambdasOnASingleLine == FormatStyle::SLS_Inline)
3838  return false;
3839 
3840  if (Style.AllowShortLambdasOnASingleLine == FormatStyle::SLS_None ||
3841  Style.AllowShortLambdasOnASingleLine == FormatStyle::SLS_Inline ||
3842  (!Left.Children.empty() &&
3843  Style.AllowShortLambdasOnASingleLine == FormatStyle::SLS_Empty))
3844  return true;
3845  }
3846 
3847  if (Style.BraceWrapping.BeforeLambdaBody && Right.is(TT_LambdaLBrace) &&
3848  Left.isOneOf(tok::star, tok::amp, tok::ampamp, TT_TemplateCloser)) {
3849  return true;
3850  }
3851 
3852  // Put multiple Java annotation on a new line.
3853  if ((Style.Language == FormatStyle::LK_Java ||
3854  Style.Language == FormatStyle::LK_JavaScript) &&
3855  Left.is(TT_LeadingJavaAnnotation) &&
3856  Right.isNot(TT_LeadingJavaAnnotation) && Right.isNot(tok::l_paren) &&
3857  (Line.Last->is(tok::l_brace) || Style.BreakAfterJavaFieldAnnotations))
3858  return true;
3859 
3860  if (Right.is(TT_ProtoExtensionLSquare))
3861  return true;
3862 
3863  // In text proto instances if a submessage contains at least 2 entries and at
3864  // least one of them is a submessage, like A { ... B { ... } ... },
3865  // put all of the entries of A on separate lines by forcing the selector of
3866  // the submessage B to be put on a newline.
3867  //
3868  // Example: these can stay on one line:
3869  // a { scalar_1: 1 scalar_2: 2 }
3870  // a { b { key: value } }
3871  //
3872  // and these entries need to be on a new line even if putting them all in one
3873  // line is under the column limit:
3874  // a {
3875  // scalar: 1
3876  // b { key: value }
3877  // }
3878  //
3879  // We enforce this by breaking before a submessage field that has previous
3880  // siblings, *and* breaking before a field that follows a submessage field.
3881  //
3882  // Be careful to exclude the case [proto.ext] { ... } since the `]` is
3883  // the TT_SelectorName there, but we don't want to break inside the brackets.
3884  //
3885  // Another edge case is @submessage { key: value }, which is a common
3886  // substitution placeholder. In this case we want to keep `@` and `submessage`
3887  // together.
3888  //
3889  // We ensure elsewhere that extensions are always on their own line.
3890  if ((Style.Language == FormatStyle::LK_Proto ||
3891  Style.Language == FormatStyle::LK_TextProto) &&
3892  Right.is(TT_SelectorName) && !Right.is(tok::r_square) && Right.Next) {
3893  // Keep `@submessage` together in:
3894  // @submessage { key: value }
3895  if (Right.Previous && Right.Previous->is(tok::at))
3896  return false;
3897  // Look for the scope opener after selector in cases like:
3898  // selector { ...
3899  // selector: { ...
3900  // selector: @base { ...
3901  FormatToken *LBrace = Right.Next;
3902  if (LBrace && LBrace->is(tok::colon)) {
3903  LBrace = LBrace->Next;
3904  if (LBrace && LBrace->is(tok::at)) {
3905  LBrace = LBrace->Next;
3906  if (LBrace)
3907  LBrace = LBrace->Next;
3908  }
3909  }
3910  if (LBrace &&
3911  // The scope opener is one of {, [, <:
3912  // selector { ... }
3913  // selector [ ... ]
3914  // selector < ... >
3915  //
3916  // In case of selector { ... }, the l_brace is TT_DictLiteral.
3917  // In case of an empty selector {}, the l_brace is not TT_DictLiteral,
3918  // so we check for immediately following r_brace.
3919  ((LBrace->is(tok::l_brace) &&
3920  (LBrace->is(TT_DictLiteral) ||
3921  (LBrace->Next && LBrace->Next->is(tok::r_brace)))) ||
3922  LBrace->is(TT_ArrayInitializerLSquare) || LBrace->is(tok::less))) {
3923  // If Left.ParameterCount is 0, then this submessage entry is not the
3924  // first in its parent submessage, and we want to break before this entry.
3925  // If Left.ParameterCount is greater than 0, then its parent submessage
3926  // might contain 1 or more entries and we want to break before this entry
3927  // if it contains at least 2 entries. We deal with this case later by
3928  // detecting and breaking before the next entry in the parent submessage.
3929  if (Left.ParameterCount == 0)
3930  return true;
3931  // However, if this submessage is the first entry in its parent
3932  // submessage, Left.ParameterCount might be 1 in some cases.
3933  // We deal with this case later by detecting an entry
3934  // following a closing paren of this submessage.
3935  }
3936 
3937  // If this is an entry immediately following a submessage, it will be
3938  // preceded by a closing paren of that submessage, like in:
3939  // left---. .---right
3940  // v v
3941  // sub: { ... } key: value
3942  // If there was a comment between `}` an `key` above, then `key` would be
3943  // put on a new line anyways.
3944  if (Left.isOneOf(tok::r_brace, tok::greater, tok::r_square))
3945  return true;
3946  }
3947 
3948  // Deal with lambda arguments in C++ - we want consistent line breaks whether
3949  // they happen to be at arg0, arg1 or argN. The selection is a bit nuanced
3950  // as aggressive line breaks are placed when the lambda is not the last arg.
3951  if ((Style.Language == FormatStyle::LK_Cpp ||
3952  Style.Language == FormatStyle::LK_ObjC) &&
3953  Left.is(tok::l_paren) && Left.BlockParameterCount > 0 &&
3954  !Right.isOneOf(tok::l_paren, TT_LambdaLSquare)) {
3955  // Multiple lambdas in the same function call force line breaks.
3956  if (Left.BlockParameterCount > 1)
3957  return true;
3958 
3959  // A lambda followed by another arg forces a line break.
3960  if (!Left.Role)
3961  return false;
3962  auto Comma = Left.Role->lastComma();
3963  if (!Comma)
3964  return false;
3965  auto Next = Comma->getNextNonComment();
3966  if (!Next)
3967  return false;
3968  if (!Next->isOneOf(TT_LambdaLSquare, tok::l_brace, tok::caret))
3969  return true;
3970  }
3971 
3972  return false;
3973 }
3974 
3975 bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line,
3976  const FormatToken &Right) {
3977  const FormatToken &Left = *Right.Previous;
3978  // Language-specific stuff.
3979  if (Style.isCSharp()) {
3980  if (Left.isOneOf(TT_CSharpNamedArgumentColon, TT_AttributeColon) ||
3981  Right.isOneOf(TT_CSharpNamedArgumentColon, TT_AttributeColon))
3982  return false;
3983  // Only break after commas for generic type constraints.
3984  if (Line.First->is(TT_CSharpGenericTypeConstraint))
3985  return Left.is(TT_CSharpGenericTypeConstraintComma);
3986  // Keep nullable operators attached to their identifiers.
3987  if (Right.is(TT_CSharpNullable)) {
3988  return false;
3989  }
3990  } else if (Style.Language == FormatStyle::LK_Java) {
3991  if (Left.isOneOf(Keywords.kw_throws, Keywords.kw_extends,
3992  Keywords.kw_implements))
3993  return false;
3994  if (Right.isOneOf(Keywords.kw_throws, Keywords.kw_extends,
3995  Keywords.kw_implements))
3996  return true;
3997  } else if (Style.Language == FormatStyle::LK_JavaScript) {
3998  const FormatToken *NonComment = Right.getPreviousNonComment();
3999  if (NonComment &&
4000  NonComment->isOneOf(
4001  tok::kw_return, Keywords.kw_yield, tok::kw_continue, tok::kw_break,
4002  tok::kw_throw, Keywords.kw_interface, Keywords.kw_type,
4003  tok::kw_static, tok::kw_public, tok::kw_private, tok::kw_protected,
4004  Keywords.kw_readonly, Keywords.kw_override, Keywords.kw_abstract,
4005  Keywords.kw_get, Keywords.kw_set, Keywords.kw_async,
4006  Keywords.kw_await))
4007  return false; // Otherwise automatic semicolon insertion would trigger.
4008  if (Right.NestingLevel == 0 &&
4009  (Left.Tok.getIdentifierInfo() ||
4010  Left.isOneOf(tok::r_square, tok::r_paren)) &&
4011  Right.isOneOf(tok::l_square, tok::l_paren))
4012  return false; // Otherwise automatic semicolon insertion would trigger.
4013  if (NonComment && NonComment->is(tok::identifier) &&
4014  NonComment->TokenText == "asserts")
4015  return false;
4016  if (Left.is(TT_FatArrow) && Right.is(tok::l_brace))
4017  return false;
4018  if (Left.is(TT_JsTypeColon))
4019  return true;
4020  // Don't wrap between ":" and "!" of a strict prop init ("field!: type;").
4021  if (Left.is(tok::exclaim) && Right.is(tok::colon))
4022  return false;
4023  // Look for is type annotations like:
4024  // function f(): a is B { ... }
4025  // Do not break before is in these cases.
4026  if (Right.is(Keywords.kw_is)) {
4027  const FormatToken *Next = Right.getNextNonComment();
4028  // If `is` is followed by a colon, it's likely that it's a dict key, so
4029  // ignore it for this check.
4030  // For example this is common in Polymer:
4031  // Polymer({
4032  // is: 'name',
4033  // ...
4034  // });
4035  if (!Next || !Next->is(tok::colon))
4036  return false;
4037  }
4038  if (Left.is(Keywords.kw_in))
4039  return Style.BreakBeforeBinaryOperators == FormatStyle::BOS_None;
4040  if (Right.is(Keywords.kw_in))
4041  return Style.BreakBeforeBinaryOperators != FormatStyle::BOS_None;
4042  if (Right.is(Keywords.kw_as))
4043  return false; // must not break before as in 'x as type' casts
4044  if (Right.isOneOf(Keywords.kw_extends, Keywords.kw_infer)) {
4045  // extends and infer can appear as keywords in conditional types:
4046  // https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-8.html#conditional-types
4047  // do not break before them, as the expressions are subject to ASI.
4048  return false;
4049  }
4050  if (Left.is(Keywords.kw_as))
4051  return true;
4052  if (Left.is(TT_NonNullAssertion))
4053  return true;
4054  if (Left.is(Keywords.kw_declare) &&
4055  Right.isOneOf(Keywords.kw_module, tok::kw_namespace,
4056  Keywords.kw_function, tok::kw_class, tok::kw_enum,
4057  Keywords.kw_interface, Keywords.kw_type, Keywords.kw_var,
4058  Keywords.kw_let, tok::kw_const))
4059  // See grammar for 'declare' statements at:
4060  // https://github.com/Microsoft/TypeScript/blob/master/doc/spec.md#A.10
4061  return false;
4062  if (Left.isOneOf(Keywords.kw_module, tok::kw_namespace) &&
4063  Right.isOneOf(tok::identifier, tok::string_literal))
4064  return false; // must not break in "module foo { ...}"
4065  if (Right.is(TT_TemplateString) && Right.closesScope())
4066  return false;
4067  // Don't split tagged template literal so there is a break between the tag
4068  // identifier and template string.
4069  if (Left.is(tok::identifier) && Right.is(TT_TemplateString)) {
4070  return false;
4071  }
4072  if (Left.is(TT_TemplateString) && Left.opensScope())
4073  return true;
4074  }
4075 
4076  if (Left.is(tok::at))
4077  return false;
4078  if (Left.Tok.getObjCKeywordID() == tok::objc_interface)
4079  return false;
4080  if (Left.isOneOf(TT_JavaAnnotation, TT_LeadingJavaAnnotation))
4081  return !Right.is(tok::l_paren);
4082  if (Right.is(TT_PointerOrReference))
4083  return Line.IsMultiVariableDeclStmt ||
4084  (getTokenPointerOrReferenceAlignment(Right) ==
4086  (!Right.Next || Right.Next->isNot(TT_FunctionDeclarationName)));
4087  if (Right.isOneOf(TT_StartOfName, TT_FunctionDeclarationName) ||
4088  Right.is(tok::kw_operator))
4089  return true;
4090  if (Left.is(TT_PointerOrReference))
4091  return false;
4092  if (Right.isTrailingComment())
4093  // We rely on MustBreakBefore being set correctly here as we should not
4094  // change the "binding" behavior of a comment.
4095  // The first comment in a braced lists is always interpreted as belonging to
4096  // the first list element. Otherwise, it should be placed outside of the
4097  // list.
4098  return Left.is(BK_BracedInit) ||
4099  (Left.is(TT_CtorInitializerColon) &&
4100  Style.BreakConstructorInitializers == FormatStyle::BCIS_AfterColon);
4101  if (Left.is(tok::question) && Right.is(tok::colon))
4102  return false;
4103  if (Right.is(TT_ConditionalExpr) || Right.is(tok::question))
4104  return Style.BreakBeforeTernaryOperators;
4105  if (Left.is(TT_ConditionalExpr) || Left.is(tok::question))
4106  return !Style.BreakBeforeTernaryOperators;
4107  if (Left.is(TT_InheritanceColon))
4108  return Style.BreakInheritanceList == FormatStyle::BILS_AfterColon;
4109  if (Right.is(TT_InheritanceColon))
4110  return Style.BreakInheritanceList != FormatStyle::BILS_AfterColon;
4111  if (Right.is(TT_ObjCMethodExpr) && !Right.is(tok::r_square) &&
4112  Left.isNot(TT_SelectorName))
4113  return true;
4114 
4115  if (Right.is(tok::colon) &&
4116  !Right.isOneOf(TT_CtorInitializerColon, TT_InlineASMColon))
4117  return false;
4118  if (Left.is(tok::colon) && Left.isOneOf(TT_DictLiteral, TT_ObjCMethodExpr)) {
4119  if (Style.Language == FormatStyle::LK_Proto ||
4120  Style.Language == FormatStyle::LK_TextProto) {
4121  if (!Style.AlwaysBreakBeforeMultilineStrings && Right.isStringLiteral())
4122  return false;
4123  // Prevent cases like:
4124  //
4125  // submessage:
4126  // { key: valueeeeeeeeeeee }
4127  //
4128  // when the snippet does not fit into one line.
4129  // Prefer:
4130  //
4131  // submessage: {
4132  // key: valueeeeeeeeeeee
4133  // }
4134  //
4135  // instead, even if it is longer by one line.
4136  //
4137  // Note that this allows allows the "{" to go over the column limit
4138  // when the column limit is just between ":" and "{", but that does
4139  // not happen too often and alternative formattings in this case are
4140  // not much better.
4141  //
4142  // The code covers the cases:
4143  //
4144  // submessage: { ... }
4145  // submessage: < ... >
4146  // repeated: [ ... ]
4147  if (((Right.is(tok::l_brace) || Right.is(tok::less)) &&
4148  Right.is(TT_DictLiteral)) ||
4149  Right.is(TT_ArrayInitializerLSquare))
4150  return false;
4151  }
4152  return true;
4153  }
4154  if (Right.is(tok::r_square) && Right.MatchingParen &&
4155  Right.MatchingParen->is(TT_ProtoExtensionLSquare))
4156  return false;
4157  if (Right.is(TT_SelectorName) || (Right.is(tok::identifier) && Right.Next &&
4158  Right.Next->is(TT_ObjCMethodExpr)))
4159  return Left.isNot(tok::period); // FIXME: Properly parse ObjC calls.
4160  if (Left.is(tok::r_paren) && Line.Type == LT_ObjCProperty)
4161  return true;
4162  if (Left.ClosesTemplateDeclaration || Left.is(TT_FunctionAnnotationRParen))
4163  return true;
4164  if (Right.isOneOf(TT_RangeBasedForLoopColon, TT_OverloadedOperatorLParen,
4165  TT_OverloadedOperator))
4166  return false;
4167  if (Left.is(TT_RangeBasedForLoopColon))
4168  return true;
4169  if (Right.is(TT_RangeBasedForLoopColon))
4170  return false;
4171  if (Left.is(TT_TemplateCloser) && Right.is(TT_TemplateOpener))
4172  return true;
4173  if (Left.isOneOf(TT_TemplateCloser, TT_UnaryOperator) ||
4174  Left.is(tok::kw_operator))
4175  return false;
4176  if (Left.is(tok::equal) && !Right.isOneOf(tok::kw_default, tok::kw_delete) &&
4177  Line.Type == LT_VirtualFunctionDecl && Left.NestingLevel == 0)
4178  return false;
4179  if (Left.is(tok::equal) && Right.is(tok::l_brace) &&
4180  !Style.Cpp11BracedListStyle)
4181  return false;
4182  if (Left.is(tok::l_paren) &&
4183  Left.isOneOf(TT_AttributeParen, TT_TypeDeclarationParen))
4184  return false;
4185  if (Left.is(tok::l_paren) && Left.Previous &&
4186  (Left.Previous->isOneOf(TT_BinaryOperator, TT_CastRParen)))
4187  return false;
4188  if (Right.is(TT_ImplicitStringLiteral))
4189  return false;
4190 
4191  if (Right.is(tok::r_paren) || Right.is(TT_TemplateCloser))
4192  return false;
4193  if (Right.is(tok::r_square) && Right.MatchingParen &&
4194  Right.MatchingParen->is(TT_LambdaLSquare))
4195  return false;
4196 
4197  // We only break before r_brace if there was a corresponding break before
4198  // the l_brace, which is tracked by BreakBeforeClosingBrace.
4199  if (Right.is(tok::r_brace))
4200  return Right.MatchingParen && Right.MatchingParen->is(BK_Block);
4201 
4202  // Allow breaking after a trailing annotation, e.g. after a method
4203  // declaration.
4204  if (Left.is(TT_TrailingAnnotation))
4205  return !Right.isOneOf(tok::l_brace, tok::semi, tok::equal, tok::l_paren,
4206  tok::less, tok::coloncolon);
4207 
4208  if (Right.is(tok::kw___attribute) ||
4209  (Right.is(tok::l_square) && Right.is(TT_AttributeSquare)))
4210  return !Left.is(TT_AttributeSquare);
4211 
4212  if (Left.is(tok::identifier) && Right.is(tok::string_literal))
4213  return true;
4214 
4215  if (Right.is(tok::identifier) && Right.Next && Right.Next->is(TT_DictLiteral))
4216  return true;
4217 
4218  if (Left.is(TT_CtorInitializerColon))
4219  return Style.BreakConstructorInitializers == FormatStyle::BCIS_AfterColon;
4220  if (Right.is(TT_CtorInitializerColon))
4221  return Style.BreakConstructorInitializers != FormatStyle::BCIS_AfterColon;
4222  if (Left.is(TT_CtorInitializerComma) &&
4223  Style.BreakConstructorInitializers == FormatStyle::BCIS_BeforeComma)
4224  return false;
4225  if (Right.is(TT_CtorInitializerComma) &&
4226  Style.BreakConstructorInitializers == FormatStyle::BCIS_BeforeComma)
4227  return true;
4228  if (Left.is(TT_InheritanceComma) &&
4229  Style.BreakInheritanceList == FormatStyle::BILS_BeforeComma)
4230  return false;
4231  if (Right.is(TT_InheritanceComma) &&
4232  Style.BreakInheritanceList == FormatStyle::BILS_BeforeComma)
4233  return true;
4234  if ((Left.is(tok::greater) && Right.is(tok::greater)) ||
4235  (Left.is(tok::less) && Right.is(tok::less)))
4236  return false;
4237  if (Right.is(TT_BinaryOperator) &&
4238  Style.BreakBeforeBinaryOperators != FormatStyle::BOS_None &&
4239  (Style.BreakBeforeBinaryOperators == FormatStyle::BOS_All ||
4240  Right.getPrecedence() != prec::Assignment))
4241  return true;
4242  if (Left.is(TT_ArrayInitializerLSquare))
4243  return true;
4244  if (Right.is(tok::kw_typename) && Left.isNot(tok::kw_const))
4245  return true;
4246  if ((Left.isBinaryOperator() || Left.is(TT_BinaryOperator)) &&
4247  !Left.isOneOf(tok::arrowstar, tok::lessless) &&
4248  Style.BreakBeforeBinaryOperators != FormatStyle::BOS_All &&
4249  (Style.BreakBeforeBinaryOperators == FormatStyle::BOS_None ||
4250  Left.getPrecedence() == prec::Assignment))
4251  return true;
4252  if ((Left.is(TT_AttributeSquare) && Right.is(tok::l_square)) ||
4253  (Left.is(tok::r_square) && Right.is(TT_AttributeSquare)))
4254  return false;
4255 
4256  auto ShortLambdaOption = Style.AllowShortLambdasOnASingleLine;
4257  if (Style.BraceWrapping.BeforeLambdaBody && Right.is(TT_LambdaLBrace)) {
4258  if (isAllmanLambdaBrace(Left))
4259  return !isItAnEmptyLambdaAllowed(Left, ShortLambdaOption);
4260  if (isAllmanLambdaBrace(Right))
4261  return !isItAnEmptyLambdaAllowed(Right, ShortLambdaOption);
4262  }
4263 
4264  return Left.isOneOf(tok::comma, tok::coloncolon, tok::semi, tok::l_brace,
4265  tok::kw_class, tok::kw_struct, tok::comment) ||
4266  Right.isMemberAccess() ||
4267  Right.isOneOf(TT_TrailingReturnArrow, TT_LambdaArrow, tok::lessless,
4268  tok::colon, tok::l_square, tok::at) ||
4269  (Left.is(tok::r_paren) &&
4270  Right.isOneOf(tok::identifier, tok::kw_const)) ||
4271  (Left.is(tok::l_paren) && !Right.is(tok::r_paren)) ||
4272  (Left.is(TT_TemplateOpener) && !Right.is(TT_TemplateCloser));
4273 }
4274 
4275 void TokenAnnotator::printDebugInfo(const AnnotatedLine &Line) {
4276  llvm::errs() << "AnnotatedTokens(L=" << Line.Level << "):\n";
4277  const FormatToken *Tok = Line.First;
4278  while (Tok) {
4279  llvm::errs() << " M=" << Tok->MustBreakBefore
4280  << " C=" << Tok->CanBreakBefore
4281  << " T=" << getTokenTypeName(Tok->getType())
4282  << " S=" << Tok->SpacesRequiredBefore
4283  << " F=" << Tok->Finalized << " B=" << Tok->BlockParameterCount
4284  << " BK=" << Tok->getBlockKind() << " P=" << Tok->SplitPenalty
4285  << " Name=" << Tok->Tok.getName() << " L=" << Tok->TotalLength
4286  << " PPK=" << Tok->getPackingKind() << " FakeLParens=";
4287  for (unsigned i = 0, e = Tok->FakeLParens.size(); i != e; ++i)
4288  llvm::errs() << Tok->FakeLParens[i] << "/";
4289  llvm::errs() << " FakeRParens=" << Tok->FakeRParens;
4290  llvm::errs() << " II=" << Tok->Tok.getIdentifierInfo();
4291  llvm::errs() << " Text='" << Tok->TokenText << "'\n";
4292  if (!Tok->Next)
4293  assert(Tok == Line.Last);
4294  Tok = Tok->Next;
4295  }
4296  llvm::errs() << "----\n";
4297 }
4298 
4300 TokenAnnotator::getTokenReferenceAlignment(const FormatToken &Reference) {
4301  assert(Reference.isOneOf(tok::amp, tok::ampamp));
4302  switch (Style.ReferenceAlignment) {
4304  return Style.PointerAlignment;
4305  case FormatStyle::RAS_Left:
4306  return FormatStyle::PAS_Left;
4308  return FormatStyle::PAS_Right;
4310  return FormatStyle::PAS_Middle;
4311  }
4312  assert(0); //"Unhandled value of ReferenceAlignment"
4313  return Style.PointerAlignment;
4314 }
4315 
4317 TokenAnnotator::getTokenPointerOrReferenceAlignment(
4318  const FormatToken &PointerOrReference) {
4319  if (PointerOrReference.isOneOf(tok::amp, tok::ampamp)) {
4320  switch (Style.ReferenceAlignment) {
4322  return Style.PointerAlignment;
4323  case FormatStyle::RAS_Left:
4324  return FormatStyle::PAS_Left;
4326  return FormatStyle::PAS_Right;
4328  return FormatStyle::PAS_Middle;
4329  }
4330  }
4331  assert(PointerOrReference.is(tok::star));
4332  return Style.PointerAlignment;
4333 }
4334 
4335 } // namespace format
4336 } // namespace clang
clang::format::FormatStyle::PAS_Left
@ PAS_Left
Align pointer to the left.
Definition: Format.h:2917
clang::prec::Comma
@ Comma
Definition: OperatorPrecedence.h:28
clang::prec::Conditional
@ Conditional
Definition: OperatorPrecedence.h:30
max
__DEVICE__ int max(int __a, int __b)
Definition: __clang_cuda_math.h:196
clang::format::TokenAnnotator::calculateFormattingInformation
void calculateFormattingInformation(AnnotatedLine &Line)
Definition: TokenAnnotator.cpp:2535
clang::format::isAllmanLambdaBrace
static bool isAllmanLambdaBrace(const FormatToken &Tok)
Definition: TokenAnnotator.cpp:3602
clang::format::FormatStyle::SBPO_Never
@ SBPO_Never
Never put a space before opening parentheses.
Definition: Format.h:3270
clang::format::FormatStyle::BILS_AfterColon
@ BILS_AfterColon
Break inheritance list after the colon and commas.
Definition: Format.h:1958
clang::format::FormatStyle::ShortLambdaStyle
ShortLambdaStyle
Different styles for merging short lambdas containing at most one statement.
Definition: Format.h:733
clang::ObjCSubstitutionContext::Parameter
@ Parameter
The parameter type of a method or function.
return
return(__x >> __y)|(__x<<(32 - __y))
clang::format::maxNestingDepth
static unsigned maxNestingDepth(const AnnotatedLine &Line)
Definition: TokenAnnotator.cpp:2359
clang::format::AnnotatedLine::First
FormatToken * First
Definition: TokenAnnotator.h:123
llvm::SmallVector
Definition: LLVM.h:38
clang::format::FormatStyle::BFCS_Before
@ BFCS_Before
Add space before the : only.
Definition: Format.h:3506
clang::if
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T -> getSizeExpr()))
Definition: RecursiveASTVisitor.h:1002
clang::format::FormatStyle::LK_Cpp
@ LK_Cpp
Should be used for C, C++.
Definition: Format.h:2649
clang::format::FormatStyle::SAPQ_After
@ SAPQ_After
Ensure that there is a space after pointer qualifiers.
Definition: Format.h:3199
clang::format::FormatStyle::SLS_Empty
@ SLS_Empty
Only merge empty lambdas.
Definition: Format.h:743
clang::format::FormatStyle::SAPQ_Both
@ SAPQ_Both
Ensure that there is a space both before and after pointer qualifiers.
Definition: Format.h:3205
clang::format::AnnotatedLine::Level
unsigned Level
Definition: TokenAnnotator.h:129
clang::Parser
Parser - This implements a parser for the C family of languages.
Definition: Parser.h:60
clang::format::FormatStyle::RTBS_All
@ RTBS_All
Always break after the return type.
Definition: Format.h:810
clang::format::FormatStyle::SLS_None
@ SLS_None
Never merge lambdas into a single line.
Definition: Format.h:735
clang::format::FormatToken
A wrapper around a Token storing information about the whitespace characters preceding it.
Definition: FormatToken.h:210
clang::ComparisonCategoryType::First
@ First
clang::minimize_source_to_dependency_directives::pp_import
@ pp_import
Definition: DependencyDirectivesSourceMinimizer.h:39
clang::format::FormatToken::isNot
bool isNot(T Kind) const
Definition: FormatToken.h:471
llvm::SmallPtrSet
Definition: ASTContext.h:82
InCSharpAttributeSpecifier
bool InCSharpAttributeSpecifier
Definition: TokenAnnotator.cpp:1462
clang::prec::Assignment
@ Assignment
Definition: OperatorPrecedence.h:29
SourceManager.h
clang::format::FormatStyle::BFCS_Both
@ BFCS_Both
Add one space on each side of the :
Definition: Format.h:3495
clang::index::SymbolRole::Reference
@ Reference
clang::format::FormatToken::OriginalColumn
unsigned OriginalColumn
The original 0-based column of this token, including expanded tabs.
Definition: FormatToken.h:371
clang::format::FormatStyle::RAS_Left
@ RAS_Left
Align reference to the left.
Definition: Format.h:3017
clang::format::FormatStyle::SBS_Never
@ SBS_Never
Never merge blocks into a single line.
Definition: Format.h:571
clang::format::FormatStyle::LK_TableGen
@ LK_TableGen
Should be used for TableGen code.
Definition: Format.h:2664
clang::format::BK_BracedInit
@ BK_BracedInit
Definition: FormatToken.h:136
End
SourceLocation End
Definition: USRLocFinder.cpp:167
clang::format::FormatStyle::SBPO_ControlStatements
@ SBPO_ControlStatements
Put a space before opening parentheses only after control statement keywords (for/if/while....
Definition: Format.h:3280
clang::format::FormatStyle::SBPO_ControlStatementsExceptControlMacros
@ SBPO_ControlStatementsExceptControlMacros
Same as SBPO_ControlStatements except this option doesn't apply to ForEach and If macros.
Definition: Format.h:3293
clang::format::PPK_BinPacked
@ PPK_BinPacked
Definition: FormatToken.h:139
clang::format::FormatToken::NewlinesBefore
unsigned NewlinesBefore
The number of newlines immediately before the Token.
Definition: FormatToken.h:336
clang::format::FormatStyle::BTDS_Yes
@ BTDS_Yes
Always break after template declaration.
Definition: Format.h:910
InCtorInitializer
bool InCtorInitializer
Definition: TokenAnnotator.cpp:1457
clang::format::FormatStyle::BCIS_AfterColon
@ BCIS_AfterColon
Break constructor initializers after the colon and commas.
Definition: Format.h:1815
InTemplateArgument
bool InTemplateArgument
Definition: TokenAnnotator.cpp:1456
TokenAnnotator.h
clang::minimize_source_to_dependency_directives::pp_if
@ pp_if
Definition: DependencyDirectivesSourceMinimizer.h:43
TokenKinds.h
clang::format::LT_PreprocessorDirective
@ LT_PreprocessorDirective
Definition: TokenAnnotator.h:33
clang::format::FormatStyle::LK_Proto
@ LK_Proto
Should be used for Protocol Buffers (https://developers.google.com/protocol-buffers/).
Definition: Format.h:2662
clang::format::FormatToken::NestingLevel
unsigned NestingLevel
The nesting level of this token, i.e.
Definition: FormatToken.h:384
clang::format::FormatToken::Previous
FormatToken * Previous
The previous token in the unwrapped line.
Definition: FormatToken.h:430
clang::format::FormatStyle::SIAS_Always
@ SIAS_Always
Add spaces after < and before >.
Definition: Format.h:3385
clang::prec::Unknown
@ Unknown
Definition: OperatorPrecedence.h:27
clang::format::FormatStyle::BAS_DontAlign
@ BAS_DontAlign
Don't align, instead use ContinuationIndentWidth, e.g.
Definition: Format.h:82
clang::format::FormatStyle::AIAS_None
@ AIAS_None
Don't align array initializer columns.
Definition: Format.h:122
InCpp11AttributeSpecifier
bool InCpp11AttributeSpecifier
Definition: TokenAnnotator.cpp:1461
clang::minimize_source_to_dependency_directives::pp_include_next
@ pp_include_next
Definition: DependencyDirectivesSourceMinimizer.h:42
clang::format::FormatStyle::BCIS_BeforeColon
@ BCIS_BeforeColon
Break constructor initializers before the colon and after the commas.
Definition: Format.h:1800
clang::format::FormatStyle::SIAS_Leave
@ SIAS_Leave
Keep a single space after < and before > if any spaces were present.
Definition: Format.h:3388
clang::format::FormatStyle::SFS_Empty
@ SFS_Empty
Only merge empty functions.
Definition: Format.h:633
clang::format::FormatStyle::LK_TextProto
@ LK_TextProto
Should be used for Protocol Buffer messages in text format (https://developers.google....
Definition: Format.h:2667
clang::format::AnnotatedLine
Definition: TokenAnnotator.h:38
ColonIsObjCMethodExpr
bool ColonIsObjCMethodExpr
Definition: TokenAnnotator.cpp:1452
Depth
int Depth
Definition: ASTDiff.cpp:191
clang::format::FormatStyle::RTBS_TopLevel
@ RTBS_TopLevel
Always break after the return types of top-level functions.
Definition: Format.h:823
P
AnnotatingParser & P
Definition: TokenAnnotator.cpp:1469
clang::format::LT_ImportStatement
@ LT_ImportStatement
Definition: TokenAnnotator.h:28
clang::format::IsFunctionArgument
static bool IsFunctionArgument(const FormatToken &Tok)
Definition: TokenAnnotator.cpp:3591
clang::format::FormatToken::isTrailingComment
bool isTrailingComment() const
Definition: FormatToken.h:582
clang::format::isAllmanBrace
static bool isAllmanBrace(const FormatToken &Tok)
Definition: TokenAnnotator.cpp:3585
clang::format::AnnotatedLine::Type
LineType Type
Definition: TokenAnnotator.h:128
clang::format::BK_Block
@ BK_Block
Definition: FormatToken.h:136
clang::minimize_source_to_dependency_directives::pp_include
@ pp_include
Definition: DependencyDirectivesSourceMinimizer.h:35
clang::format::FormatStyle::BOS_All
@ BOS_All
Break before operators.
Definition: Format.h:1048
clang::format::TokenAnnotator::annotate
void annotate(AnnotatedLine &Line)
Definition: TokenAnnotator.cpp:2366
clang::format::FormatStyle::PointerAlignmentStyle
PointerAlignmentStyle
The &, && and * alignment style.
Definition: Format.h:2912
Line
const AnnotatedLine * Line
Definition: UsingDeclarationsSorter.cpp:68
clang::format::LT_Invalid
@ LT_Invalid
Definition: TokenAnnotator.h:27
ColonIsForRangeExpr
bool ColonIsForRangeExpr
Definition: TokenAnnotator.cpp:1450
clang::prec::PointerToMember
@ PointerToMember
Definition: OperatorPrecedence.h:42
clang::format::FormatStyle::LK_JavaScript
@ LK_JavaScript
Should be used for JavaScript.
Definition: Format.h:2655
clang::format::PPK_OnePerLine
@ PPK_OnePerLine
Definition: FormatToken.h:139
clang::format::FormatStyle::LK_Java
@ LK_Java
Should be used for Java.
Definition: Format.h:2653
clang::format::getTokenTypeName
const char * getTokenTypeName(TokenType Type)
Determines the name of a token type.
Definition: FormatToken.cpp:24
CaretFound
bool CaretFound
Definition: TokenAnnotator.cpp:1459
clang::format::FormatStyle::SBPO_Always
@ SBPO_Always
Always put a space before opening parentheses, except when it's prohibited by the syntax rules (in fu...
Definition: Format.h:3316
IsExpression
bool IsExpression
Definition: TokenAnnotator.cpp:1448
clang::format::FormatStyle::RTBS_AllDefinitions
@ RTBS_AllDefinitions
Always break after the return type of function definitions.
Definition: Format.h:838
clang::format::FormatStyle::BILS_BeforeComma
@ BILS_BeforeComma
Break inheritance list before the colon and commas, and align the commas with the colon.
Definition: Format.h:1950
clang::AccessSpecifier
AccessSpecifier
A C++ access specifier (public, private, protected), plus the special value "none" which means differ...
Definition: Specifiers.h:108
clang::format::FormatStyle::PAS_Middle
@ PAS_Middle
Align pointer in the middle.
Definition: Format.h:2927
clang::format::FormatStyle::BILS_AfterComma
@ BILS_AfterComma
Break inheritance list only after the commas.
Definition: Format.h:1965
clang::tok::TokenKind
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
clang::format::FormatStyle::SFS_InlineOnly
@ SFS_InlineOnly
Only merge functions defined inside a class.
Definition: Format.h:625
clang::format::LineType
LineType
Definition: TokenAnnotator.h:26
clang::format::getFirstNonComment
static const FormatToken * getFirstNonComment(const AnnotatedLine &Line)
Definition: TokenAnnotator.cpp:3608
clang::format::FormatStyle::RAS_Right
@ RAS_Right
Align reference to the right.
Definition: Format.h:3022
LongestObjCSelectorName
unsigned LongestObjCSelectorName
Definition: TokenAnnotator.cpp:1449
clang::format::LT_ObjCDecl
@ LT_ObjCDecl
Definition: TokenAnnotator.h:29
FirstStartOfName
FormatToken * FirstStartOfName
Definition: TokenAnnotator.cpp:1454
clang::format::FormatStyle::RTBS_TopLevelDefinitions
@ RTBS_TopLevelDefinitions
Always break after the return type of top-level definitions.
Definition: Format.h:850
clang::format::AnnotatedLine::Last
FormatToken * Last
Definition: TokenAnnotator.h:124
clang::format::FormatStyle::RAS_Pointer
@ RAS_Pointer
Align reference like PointerAlignment.
Definition: Format.h:3012
clang::format::isFunctionDeclarationName
static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current, const AnnotatedLine &Line)
Definition: TokenAnnotator.cpp:2401
clang::format::FormatStyle::SLS_Inline
@ SLS_Inline
Merge lambda into a single line if argument of a function.
Definition: Format.h:751
IsForEachMacro
bool IsForEachMacro
Definition: TokenAnnotator.cpp:1460
clang::format::FormatToken::is
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:452
clang::format::FormatStyle::SAPQ_Before
@ SAPQ_Before
Ensure that there is a space before pointer qualifiers.
Definition: Format.h:3193
clang::format::FormatToken::isOneOf
bool isOneOf(A K1, B K2) const
Definition: FormatToken.h:464
clang::format::FormatStyle::PAS_Right
@ PAS_Right
Align pointer to the right.
Definition: Format.h:2922
clang::format::TokenAnnotator::setCommentLineLevels
void setCommentLineLevels(SmallVectorImpl< AnnotatedLine * > &Lines)
Adapts the indent levels of comment lines to the indent of the subsequent line.
Definition: TokenAnnotator.cpp:2322
clang::format::FormatStyle::BCIS_BeforeComma
@ BCIS_BeforeComma
Break constructor initializers before the colon and commas, and align the commas with the colon.
Definition: Format.h:1808
clang::format::isItAnEmptyLambdaAllowed
static bool isItAnEmptyLambdaAllowed(const FormatToken &Tok, FormatStyle::ShortLambdaStyle ShortLambdaOption)
Definition: TokenAnnotator.cpp:3597
clang::prec::Relational
@ Relational
Definition: OperatorPrecedence.h:37
BindingStrength
unsigned BindingStrength
Definition: TokenAnnotator.cpp:1447
clang::format::FormatStyle::PCIS_Never
@ PCIS_Never
Always put each constructor initializer on its own line.
Definition: Format.h:2192
clang
Definition: CalledOnceCheck.h:17
clang::format::FormatToken::Children
SmallVector< AnnotatedLine *, 1 > Children
If this token starts a block, this contains all the unwrapped lines in it.
Definition: FormatToken.h:446
clang::format::AnnotatedLine::endsWith
bool endsWith(Ts... Tokens) const
true if this line ends with the given tokens in reversed order, ignoring comments.
Definition: TokenAnnotator.h:95
clang::prec::LogicalOr
@ LogicalOr
Definition: OperatorPrecedence.h:31
clang::format::FormatStyle::LK_ObjC
@ LK_ObjC
Should be used for Objective-C, Objective-C++.
Definition: Format.h:2659
clang::format::LT_ArrayOfStructInitializer
@ LT_ArrayOfStructInitializer
Definition: TokenAnnotator.h:35
clang::prec::Level
Level
Definition: OperatorPrecedence.h:26
clang::format::FormatStyle::LS_Cpp11
@ LS_Cpp11
Parse and format as C++11.
Definition: Format.h:3531
clang::getBinOpPrecedence
prec::Level getBinOpPrecedence(tok::TokenKind Kind, bool GreaterThanIsOperator, bool CPlusPlus11)
Return the precedence of the specified binary operator token.
Definition: OperatorPrecedence.cpp:17
clang::format::FormatStyle::RAS_Middle
@ RAS_Middle
Align reference in the middle.
Definition: Format.h:3027
ColonIsDictLiteral
bool ColonIsDictLiteral
Definition: TokenAnnotator.cpp:1451
clang::format::FormatStyle::BFCS_After
@ BFCS_After
Add space after the : only (space may be added before if needed for AlignConsecutiveBitFields).
Definition: Format.h:3512
clang::format::FormatStyle::SBPO_NonEmptyParentheses
@ SBPO_NonEmptyParentheses
Put a space before opening parentheses only if the parentheses are not empty i.e.
Definition: Format.h:3304
FormatToken.h
Parent
NodeId Parent
Definition: ASTDiff.cpp:192
clang::format::LT_Other
@ LT_Other
Definition: TokenAnnotator.h:32
clang::format::FormatStyle::BOS_None
@ BOS_None
Break after operators.
Definition: Format.h:1024
llvm::SmallVectorImpl
Definition: LLVM.h:39
clang::format::FormatToken::TotalLength
unsigned TotalLength
The total length of the unwrapped line up to and including this token.
Definition: FormatToken.h:367
Previous
StateNode * Previous
Definition: UnwrappedLineFormatter.cpp:988
clang::format::PPK_Inconclusive
@ PPK_Inconclusive
Definition: FormatToken.h:139
clang::format::FormatToken::Next
FormatToken * Next
The next token in the unwrapped line.
Definition: FormatToken.h:433
clang::format::LT_ObjCMethodDecl
@ LT_ObjCMethodDecl
Definition: TokenAnnotator.h:30
CanBeExpression
bool CanBeExpression
Definition: TokenAnnotator.cpp:1455
clang::format::TokenType
TokenType
Determines the semantic type of a syntactic token, e.g.
Definition: FormatToken.h:125
InInheritanceList
bool InInheritanceList
Definition: TokenAnnotator.cpp:1458
clang::format::LT_VirtualFunctionDecl
@ LT_VirtualFunctionDecl
Definition: TokenAnnotator.h:34
clang::format::FormatStyle::SFS_None
@ SFS_None
Never merge functions into a single line.
Definition: Format.h:611
clang::format::FormatStyle::SLS_All
@ SLS_All
Merge all lambdas fitting on a single line.
Definition: Format.h:757
clang::format::FormatStyle::RTBS_None
@ RTBS_None
Break after return type automatically.
Definition: Format.h:794
clang::format::FormatStyle::PCIS_CurrentLine
@ PCIS_CurrentLine
Put all constructor initializers on the current line if they fit.
Definition: Format.h:2210
clang::format::FormatToken::MatchingParen
FormatToken * MatchingParen
If this is a bracket, this points to the matching one.
Definition: FormatToken.h:427
clang::minimize_source_to_dependency_directives::pp_elif
@ pp_elif
Definition: DependencyDirectivesSourceMinimizer.h:46
clang::format::LT_ObjCProperty
@ LT_ObjCProperty
Definition: TokenAnnotator.h:31
clang::format::FormatStyle::PPDIS_BeforeHash
@ PPDIS_BeforeHash
Indents directives before the hash.
Definition: Format.h:2457
ContextKind
tok::TokenKind ContextKind
Definition: TokenAnnotator.cpp:1446
FirstObjCSelectorName
FormatToken * FirstObjCSelectorName
Definition: TokenAnnotator.cpp:1453
Type
MatchType Type
Definition: ASTMatchFinder.cpp:70
InStructArrayInitializer
bool InStructArrayInitializer
Definition: TokenAnnotator.cpp:1463