clang  15.0.0git
FormatToken.cpp
Go to the documentation of this file.
1 //===--- FormatToken.cpp - Format C++ code --------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements specific functions of \c FormatTokens and their
11 /// roles.
12 ///
13 //===----------------------------------------------------------------------===//
14 
15 #include "FormatToken.h"
16 #include "ContinuationIndenter.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/Support/Debug.h"
19 #include <climits>
20 
21 namespace clang {
22 namespace format {
23 
25  static const char *const TokNames[] = {
26 #define TYPE(X) #X,
28 #undef TYPE
29  nullptr};
30 
31  if (Type < NUM_TOKEN_TYPES)
32  return TokNames[Type];
33  llvm_unreachable("unknown TokenType");
34  return nullptr;
35 }
36 
37 // FIXME: This is copy&pasted from Sema. Put it in a common place and remove
38 // duplication.
40  switch (Tok.getKind()) {
41  case tok::kw_short:
42  case tok::kw_long:
43  case tok::kw___int64:
44  case tok::kw___int128:
45  case tok::kw_signed:
46  case tok::kw_unsigned:
47  case tok::kw_void:
48  case tok::kw_char:
49  case tok::kw_int:
50  case tok::kw_half:
51  case tok::kw_float:
52  case tok::kw_double:
53  case tok::kw___bf16:
54  case tok::kw__Float16:
55  case tok::kw___float128:
56  case tok::kw___ibm128:
57  case tok::kw_wchar_t:
58  case tok::kw_bool:
59  case tok::kw___underlying_type:
60  case tok::annot_typename:
61  case tok::kw_char8_t:
62  case tok::kw_char16_t:
63  case tok::kw_char32_t:
64  case tok::kw_typeof:
65  case tok::kw_decltype:
66  case tok::kw__Atomic:
67  return true;
68  default:
69  return false;
70  }
71 }
72 
74  return isSimpleTypeSpecifier() || Tok.isOneOf(tok::kw_auto, tok::identifier);
75 }
76 
78  // C# Does not indent object initialisers as continuations.
79  if (is(tok::l_brace) && getBlockKind() == BK_BracedInit && Style.isCSharp())
80  return true;
81  if (is(TT_TemplateString) && opensScope())
82  return true;
83  return is(TT_ArrayInitializerLSquare) || is(TT_ProtoExtensionLSquare) ||
84  (is(tok::l_brace) &&
85  (getBlockKind() == BK_Block || is(TT_DictLiteral) ||
86  (!Style.Cpp11BracedListStyle && NestingLevel == 0))) ||
87  (is(tok::less) && (Style.Language == FormatStyle::LK_Proto ||
88  Style.Language == FormatStyle::LK_TextProto));
89 }
90 
92 
94 
97  bool DryRun) {
98  if (State.NextToken == nullptr || !State.NextToken->Previous)
99  return 0;
100 
101  if (Formats.size() == 1)
102  return 0; // Handled by formatFromToken
103 
104  // Ensure that we start on the opening brace.
105  const FormatToken *LBrace =
106  State.NextToken->Previous->getPreviousNonComment();
107  if (!LBrace || !LBrace->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) ||
108  LBrace->is(BK_Block) || LBrace->is(TT_DictLiteral) ||
109  LBrace->Next->is(TT_DesignatedInitializerPeriod))
110  return 0;
111 
112  // Calculate the number of code points we have to format this list. As the
113  // first token is already placed, we have to subtract it.
114  unsigned RemainingCodePoints =
115  Style.ColumnLimit - State.Column + State.NextToken->Previous->ColumnWidth;
116 
117  // Find the best ColumnFormat, i.e. the best number of columns to use.
118  const ColumnFormat *Format = getColumnFormat(RemainingCodePoints);
119 
120  // If no ColumnFormat can be used, the braced list would generally be
121  // bin-packed. Add a severe penalty to this so that column layouts are
122  // preferred if possible.
123  if (!Format)
124  return 10000;
125 
126  // Format the entire list.
127  unsigned Penalty = 0;
128  unsigned Column = 0;
129  unsigned Item = 0;
130  while (State.NextToken != LBrace->MatchingParen) {
131  bool NewLine = false;
132  unsigned ExtraSpaces = 0;
133 
134  // If the previous token was one of our commas, we are now on the next item.
135  if (Item < Commas.size() && State.NextToken->Previous == Commas[Item]) {
136  if (!State.NextToken->isTrailingComment()) {
137  ExtraSpaces += Format->ColumnSizes[Column] - ItemLengths[Item];
138  ++Column;
139  }
140  ++Item;
141  }
142 
143  if (Column == Format->Columns || State.NextToken->MustBreakBefore) {
144  Column = 0;
145  NewLine = true;
146  }
147 
148  // Place token using the continuation indenter and store the penalty.
149  Penalty += Indenter->addTokenToState(State, NewLine, DryRun, ExtraSpaces);
150  }
151  return Penalty;
152 }
153 
156  bool DryRun) {
157  // Formatting with 1 Column isn't really a column layout, so we don't need the
158  // special logic here. We can just avoid bin packing any of the parameters.
159  if (Formats.size() == 1 || HasNestedBracedList)
160  State.Stack.back().AvoidBinPacking = true;
161  return 0;
162 }
163 
164 // Returns the lengths in code points between Begin and End (both included),
165 // assuming that the entire sequence is put on a single line.
166 static unsigned CodePointsBetween(const FormatToken *Begin,
167  const FormatToken *End) {
168  assert(End->TotalLength >= Begin->TotalLength);
169  return End->TotalLength - Begin->TotalLength + Begin->ColumnWidth;
170 }
171 
173  // FIXME: At some point we might want to do this for other lists, too.
174  if (!Token->MatchingParen ||
175  !Token->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare))
176  return;
177 
178  // In C++11 braced list style, we should not format in columns unless they
179  // have many items (20 or more) or we allow bin-packing of function call
180  // arguments.
182  Commas.size() < 19)
183  return;
184 
185  // Limit column layout for JavaScript array initializers to 20 or more items
186  // for now to introduce it carefully. We can become more aggressive if this
187  // necessary.
188  if (Token->is(TT_ArrayInitializerLSquare) && Commas.size() < 19)
189  return;
190 
191  // Column format doesn't really make sense if we don't align after brackets.
193  return;
194 
195  FormatToken *ItemBegin = Token->Next;
196  while (ItemBegin->isTrailingComment())
197  ItemBegin = ItemBegin->Next;
198  SmallVector<bool, 8> MustBreakBeforeItem;
199 
200  // The lengths of an item if it is put at the end of the line. This includes
201  // trailing comments which are otherwise ignored for column alignment.
202  SmallVector<unsigned, 8> EndOfLineItemLength;
203  MustBreakBeforeItem.reserve(Commas.size() + 1);
204  EndOfLineItemLength.reserve(Commas.size() + 1);
205  ItemLengths.reserve(Commas.size() + 1);
206 
207  bool HasSeparatingComment = false;
208  for (unsigned i = 0, e = Commas.size() + 1; i != e; ++i) {
209  assert(ItemBegin);
210  // Skip comments on their own line.
211  while (ItemBegin->HasUnescapedNewline && ItemBegin->isTrailingComment()) {
212  ItemBegin = ItemBegin->Next;
213  HasSeparatingComment = i > 0;
214  }
215 
216  MustBreakBeforeItem.push_back(ItemBegin->MustBreakBefore);
217  if (ItemBegin->is(tok::l_brace))
218  HasNestedBracedList = true;
219  const FormatToken *ItemEnd = nullptr;
220  if (i == Commas.size()) {
221  ItemEnd = Token->MatchingParen;
222  const FormatToken *NonCommentEnd = ItemEnd->getPreviousNonComment();
223  ItemLengths.push_back(CodePointsBetween(ItemBegin, NonCommentEnd));
225  !ItemEnd->Previous->isTrailingComment()) {
226  // In Cpp11 braced list style, the } and possibly other subsequent
227  // tokens will need to stay on a line with the last element.
228  while (ItemEnd->Next && !ItemEnd->Next->CanBreakBefore)
229  ItemEnd = ItemEnd->Next;
230  } else {
231  // In other braced lists styles, the "}" can be wrapped to the new line.
232  ItemEnd = Token->MatchingParen->Previous;
233  }
234  } else {
235  ItemEnd = Commas[i];
236  // The comma is counted as part of the item when calculating the length.
237  ItemLengths.push_back(CodePointsBetween(ItemBegin, ItemEnd));
238 
239  // Consume trailing comments so the are included in EndOfLineItemLength.
240  if (ItemEnd->Next && !ItemEnd->Next->HasUnescapedNewline &&
241  ItemEnd->Next->isTrailingComment())
242  ItemEnd = ItemEnd->Next;
243  }
244  EndOfLineItemLength.push_back(CodePointsBetween(ItemBegin, ItemEnd));
245  // If there is a trailing comma in the list, the next item will start at the
246  // closing brace. Don't create an extra item for this.
247  if (ItemEnd->getNextNonComment() == Token->MatchingParen)
248  break;
249  ItemBegin = ItemEnd->Next;
250  }
251 
252  // Don't use column layout for lists with few elements and in presence of
253  // separating comments.
254  if (Commas.size() < 5 || HasSeparatingComment)
255  return;
256 
257  if (Token->NestingLevel != 0 && Token->is(tok::l_brace) && Commas.size() < 19)
258  return;
259 
260  // We can never place more than ColumnLimit / 3 items in a row (because of the
261  // spaces and the comma).
262  unsigned MaxItems = Style.ColumnLimit / 3;
263  std::vector<unsigned> MinSizeInColumn;
264  MinSizeInColumn.reserve(MaxItems);
265  for (unsigned Columns = 1; Columns <= MaxItems; ++Columns) {
266  ColumnFormat Format;
267  Format.Columns = Columns;
268  Format.ColumnSizes.resize(Columns);
269  MinSizeInColumn.assign(Columns, UINT_MAX);
270  Format.LineCount = 1;
271  bool HasRowWithSufficientColumns = false;
272  unsigned Column = 0;
273  for (unsigned i = 0, e = ItemLengths.size(); i != e; ++i) {
274  assert(i < MustBreakBeforeItem.size());
275  if (MustBreakBeforeItem[i] || Column == Columns) {
276  ++Format.LineCount;
277  Column = 0;
278  }
279  if (Column == Columns - 1)
280  HasRowWithSufficientColumns = true;
281  unsigned Length =
282  (Column == Columns - 1) ? EndOfLineItemLength[i] : ItemLengths[i];
283  Format.ColumnSizes[Column] = std::max(Format.ColumnSizes[Column], Length);
284  MinSizeInColumn[Column] = std::min(MinSizeInColumn[Column], Length);
285  ++Column;
286  }
287  // If all rows are terminated early (e.g. by trailing comments), we don't
288  // need to look further.
289  if (!HasRowWithSufficientColumns)
290  break;
291  Format.TotalWidth = Columns - 1; // Width of the N-1 spaces.
292 
293  for (unsigned i = 0; i < Columns; ++i)
294  Format.TotalWidth += Format.ColumnSizes[i];
295 
296  // Don't use this Format, if the difference between the longest and shortest
297  // element in a column exceeds a threshold to avoid excessive spaces.
298  if ([&] {
299  for (unsigned i = 0; i < Columns - 1; ++i)
300  if (Format.ColumnSizes[i] - MinSizeInColumn[i] > 10)
301  return true;
302  return false;
303  }())
304  continue;
305 
306  // Ignore layouts that are bound to violate the column limit.
307  if (Format.TotalWidth > Style.ColumnLimit && Columns > 1)
308  continue;
309 
310  Formats.push_back(Format);
311  }
312 }
313 
314 const CommaSeparatedList::ColumnFormat *
315 CommaSeparatedList::getColumnFormat(unsigned RemainingCharacters) const {
316  const ColumnFormat *BestFormat = nullptr;
317  for (const ColumnFormat &Format : llvm::reverse(Formats)) {
318  if (Format.TotalWidth <= RemainingCharacters || Format.Columns == 1) {
319  if (BestFormat && Format.LineCount > BestFormat->LineCount)
320  break;
321  BestFormat = &Format;
322  }
323  }
324  return BestFormat;
325 }
326 
327 } // namespace format
328 } // namespace clang
clang::format::FormatToken::MustBreakBefore
unsigned MustBreakBefore
Whether there must be a line break before this token.
Definition: FormatToken.h:270
max
__DEVICE__ int max(int __a, int __b)
Definition: __clang_cuda_math.h:196
clang::format::FormatToken::getNextNonComment
const LLVM_NODISCARD FormatToken * getNextNonComment() const
Returns the next token ignoring comments.
Definition: FormatToken.h:704
NewLine
bool NewLine
Definition: UnwrappedLineFormatter.cpp:1091
llvm::SmallVector< bool, 8 >
clang::if
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T -> getSizeExpr()))
Definition: RecursiveASTVisitor.h:1098
clang::format::TokenRole::~TokenRole
virtual ~TokenRole()
Definition: FormatToken.cpp:91
clang::format::FormatStyle
The FormatStyle is used to configure the formatting to follow specific guidelines.
Definition: Format.h:54
clang::format::FormatToken
A wrapper around a Token storing information about the whitespace characters preceding it.
Definition: FormatToken.h:227
clang::Token
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
clang::format::BK_BracedInit
@ BK_BracedInit
Definition: FormatToken.h:153
End
SourceLocation End
Definition: USRLocFinder.cpp:167
clang::Type
The base class of the type hierarchy.
Definition: Type.h:1500
clang::format::CommaSeparatedList::formatFromToken
unsigned formatFromToken(LineState &State, ContinuationIndenter *Indenter, bool DryRun) override
Apply the special formatting that the given role demands.
Definition: FormatToken.cpp:154
clang::format::FormatToken::CanBreakBefore
unsigned CanBreakBefore
true if it is allowed to break before this token.
Definition: FormatToken.h:276
clang::format::FormatStyle::Cpp11BracedListStyle
bool Cpp11BracedListStyle
If true, format braced lists as best suited for C++11 braced lists.
Definition: Format.h:1928
clang::format::FormatToken::NestingLevel
unsigned NestingLevel
The nesting level of this token, i.e.
Definition: FormatToken.h:423
LIST_TOKEN_TYPES
#define LIST_TOKEN_TYPES
Definition: FormatToken.h:28
clang::format::FormatToken::Previous
FormatToken * Previous
The previous token in the unwrapped line.
Definition: FormatToken.h:469
clang::format::LineState
The current state when indenting a unwrapped line.
Definition: ContinuationIndenter.h:415
min
__DEVICE__ int min(int __a, int __b)
Definition: __clang_cuda_math.h:197
clang::format::FormatToken::Tok
Token Tok
The Token.
Definition: FormatToken.h:239
clang::format::FormatStyle::LK_TextProto
@ LK_TextProto
Should be used for Protocol Buffer messages in text format (https://developers.google....
Definition: Format.h:2592
clang::Token::getKind
tok::TokenKind getKind() const
Definition: Token.h:92
clang::format::FormatStyle::LK_Proto
@ LK_Proto
Should be used for Protocol Buffers (https://developers.google.com/protocol-buffers/).
Definition: Format.h:2587
clang::format::FormatStyle::BinPackArguments
bool BinPackArguments
If false, a function call's arguments will either be all on the same line or will have one line each.
Definition: Format.h:828
clang::format::FormatToken::getPreviousNonComment
LLVM_NODISCARD FormatToken * getPreviousNonComment() const
Returns the previous token ignoring comments.
Definition: FormatToken.h:696
clang::Token::is
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:97
clang::format::FormatToken::isTrailingComment
bool isTrailingComment() const
Definition: FormatToken.h:632
clang::format::TokenRole::Style
const FormatStyle & Style
Definition: FormatToken.h:822
clang::format::FormatStyle::ColumnLimit
unsigned ColumnLimit
The column limit.
Definition: Format.h:1742
clang::format::BK_Block
@ BK_Block
Definition: FormatToken.h:153
clang::format::CommaSeparatedList::precomputeFormattingInfos
void precomputeFormattingInfos(const FormatToken *Token) override
After the TokenAnnotator has finished annotating all the tokens, this function precomputes required i...
Definition: FormatToken.cpp:172
clang::format::getTokenTypeName
const char * getTokenTypeName(TokenType Type)
Determines the name of a token type.
Definition: FormatToken.cpp:24
TokNames
static const char *const TokNames[]
Definition: TokenKinds.cpp:17
clang::format::NUM_TOKEN_TYPES
@ NUM_TOKEN_TYPES
Definition: FormatToken.h:146
UINT_MAX
#define UINT_MAX
Definition: limits.h:56
clang::format::CommaSeparatedList::formatAfterToken
unsigned formatAfterToken(LineState &State, ContinuationIndenter *Indenter, bool DryRun) override
Same as formatFromToken, but assumes that the first token has already been set thereby deciding on th...
Definition: FormatToken.cpp:95
clang::format::CodePointsBetween
static unsigned CodePointsBetween(const FormatToken *Begin, const FormatToken *End)
Definition: FormatToken.cpp:166
Begin
SourceLocation Begin
Definition: USRLocFinder.cpp:165
clang::format::FormatToken::getBlockKind
BraceBlockKind getBlockKind() const
Definition: FormatToken.h:310
clang::format::FormatToken::is
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:500
clang::format::FormatStyle::AlignAfterOpenBracket
BracketAlignmentStyle AlignAfterOpenBracket
If true, horizontally aligns arguments after an open bracket.
Definition: Format.h:105
State
LineState State
Definition: UnwrappedLineFormatter.cpp:1090
clang::format::FormatToken::isOneOf
bool isOneOf(A K1, B K2) const
Definition: FormatToken.h:512
clang::format::ContinuationIndenter
Definition: ContinuationIndenter.h:50
clang
Definition: CalledOnceCheck.h:17
clang::format::FormatToken::isTypeOrIdentifier
LLVM_NODISCARD bool isTypeOrIdentifier() const
Definition: FormatToken.cpp:73
clang::format::FormatToken::isSimpleTypeSpecifier
LLVM_NODISCARD bool isSimpleTypeSpecifier() const
Determine whether the token is a simple-type-specifier.
Definition: FormatToken.cpp:39
clang::format::FormatToken::opensBlockOrBlockTypeList
LLVM_NODISCARD bool opensBlockOrBlockTypeList(const FormatStyle &Style) const
Returns true if this tokens starts a block-type list, i.e.
Definition: FormatToken.cpp:77
clang::format::TokenRole::precomputeFormattingInfos
virtual void precomputeFormattingInfos(const FormatToken *Token)
After the TokenAnnotator has finished annotating all the tokens, this function precomputes required i...
Definition: FormatToken.cpp:93
clang::Token::isOneOf
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:99
FormatToken.h
clang::format::FormatStyle::BAS_DontAlign
@ BAS_DontAlign
Don't align, instead use ContinuationIndentWidth, e.g.
Definition: Format.h:77
ContinuationIndenter.h
Indenter
ContinuationIndenter * Indenter
Definition: UnwrappedLineFormatter.cpp:987
clang::format::FormatToken::Next
FormatToken * Next
The next token in the unwrapped line.
Definition: FormatToken.h:472
clang::format::FormatToken::HasUnescapedNewline
unsigned HasUnescapedNewline
Whether there is at least one unescaped newline before the Token.
Definition: FormatToken.h:258
clang::format::TokenType
TokenType
Determines the semantic type of a syntactic token, e.g.
Definition: FormatToken.h:142
clang::format::FormatToken::MatchingParen
FormatToken * MatchingParen
If this is a bracket, this points to the matching one.
Definition: FormatToken.h:466
clang::format::FormatToken::opensScope
bool opensScope() const
Returns whether Tok is ([{ or an opening < of a template or in protos.
Definition: FormatToken.h:585