clang 18.0.0git
FormatToken.cpp
Go to the documentation of this file.
1//===--- FormatToken.cpp - Format C++ code --------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements specific functions of \c FormatTokens and their
11/// roles.
12///
13//===----------------------------------------------------------------------===//
14
15#include "FormatToken.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/Support/Debug.h"
19#include <climits>
20
21namespace clang {
22namespace format {
23
25 static const char *const TokNames[] = {
26#define TYPE(X) #X,
28#undef TYPE
29 nullptr};
30
32 return TokNames[Type];
33 llvm_unreachable("unknown TokenType");
34 return nullptr;
35}
36
37// FIXME: This is copy&pasted from Sema. Put it in a common place and remove
38// duplication.
40 switch (Tok.getKind()) {
41 case tok::kw_short:
42 case tok::kw_long:
43 case tok::kw___int64:
44 case tok::kw___int128:
45 case tok::kw_signed:
46 case tok::kw_unsigned:
47 case tok::kw_void:
48 case tok::kw_char:
49 case tok::kw_int:
50 case tok::kw_half:
51 case tok::kw_float:
52 case tok::kw_double:
53 case tok::kw___bf16:
54 case tok::kw__Float16:
55 case tok::kw___float128:
56 case tok::kw___ibm128:
57 case tok::kw_wchar_t:
58 case tok::kw_bool:
59#define TRANSFORM_TYPE_TRAIT_DEF(_, Trait) case tok::kw___##Trait:
60#include "clang/Basic/TransformTypeTraits.def"
61 case tok::annot_typename:
62 case tok::kw_char8_t:
63 case tok::kw_char16_t:
64 case tok::kw_char32_t:
65 case tok::kw_typeof:
66 case tok::kw_decltype:
67 case tok::kw__Atomic:
68 return true;
69 default:
70 return false;
71 }
72}
73
75 return isSimpleTypeSpecifier() || Tok.isOneOf(tok::kw_auto, tok::identifier);
76}
77
79 assert(is(tok::r_brace));
80 if (!Style.Cpp11BracedListStyle ||
81 Style.AlignAfterOpenBracket != FormatStyle::BAS_BlockIndent) {
82 return false;
83 }
84 const auto *LBrace = MatchingParen;
85 assert(LBrace && LBrace->is(tok::l_brace));
86 if (LBrace->is(BK_BracedInit))
87 return true;
88 if (LBrace->Previous && LBrace->Previous->is(tok::equal))
89 return true;
90 return false;
91}
92
94 // C# Does not indent object initialisers as continuations.
95 if (is(tok::l_brace) && getBlockKind() == BK_BracedInit && Style.isCSharp())
96 return true;
97 if (is(TT_TemplateString) && opensScope())
98 return true;
99 return is(TT_ArrayInitializerLSquare) || is(TT_ProtoExtensionLSquare) ||
100 (is(tok::l_brace) &&
101 (getBlockKind() == BK_Block || is(TT_DictLiteral) ||
102 (!Style.Cpp11BracedListStyle && NestingLevel == 0))) ||
103 (is(tok::less) && (Style.Language == FormatStyle::LK_Proto ||
104 Style.Language == FormatStyle::LK_TextProto));
105}
106
108
110
113 bool DryRun) {
114 if (!State.NextToken || !State.NextToken->Previous)
115 return 0;
116
117 if (Formats.size() == 1)
118 return 0; // Handled by formatFromToken
119
120 // Ensure that we start on the opening brace.
121 const FormatToken *LBrace =
122 State.NextToken->Previous->getPreviousNonComment();
123 if (!LBrace || !LBrace->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) ||
124 LBrace->is(BK_Block) || LBrace->is(TT_DictLiteral) ||
125 LBrace->Next->is(TT_DesignatedInitializerPeriod)) {
126 return 0;
127 }
128
129 // Calculate the number of code points we have to format this list. As the
130 // first token is already placed, we have to subtract it.
131 unsigned RemainingCodePoints =
132 Style.ColumnLimit - State.Column + State.NextToken->Previous->ColumnWidth;
133
134 // Find the best ColumnFormat, i.e. the best number of columns to use.
135 const ColumnFormat *Format = getColumnFormat(RemainingCodePoints);
136
137 // If no ColumnFormat can be used, the braced list would generally be
138 // bin-packed. Add a severe penalty to this so that column layouts are
139 // preferred if possible.
140 if (!Format)
141 return 10000;
142
143 // Format the entire list.
144 unsigned Penalty = 0;
145 unsigned Column = 0;
146 unsigned Item = 0;
147 while (State.NextToken != LBrace->MatchingParen) {
148 bool NewLine = false;
149 unsigned ExtraSpaces = 0;
150
151 // If the previous token was one of our commas, we are now on the next item.
152 if (Item < Commas.size() && State.NextToken->Previous == Commas[Item]) {
153 if (!State.NextToken->isTrailingComment()) {
154 ExtraSpaces += Format->ColumnSizes[Column] - ItemLengths[Item];
155 ++Column;
156 }
157 ++Item;
158 }
159
160 if (Column == Format->Columns || State.NextToken->MustBreakBefore) {
161 Column = 0;
162 NewLine = true;
163 }
164
165 // Place token using the continuation indenter and store the penalty.
166 Penalty += Indenter->addTokenToState(State, NewLine, DryRun, ExtraSpaces);
167 }
168 return Penalty;
169}
170
173 bool DryRun) {
174 // Formatting with 1 Column isn't really a column layout, so we don't need the
175 // special logic here. We can just avoid bin packing any of the parameters.
176 if (Formats.size() == 1 || HasNestedBracedList)
177 State.Stack.back().AvoidBinPacking = true;
178 return 0;
179}
180
181// Returns the lengths in code points between Begin and End (both included),
182// assuming that the entire sequence is put on a single line.
183static unsigned CodePointsBetween(const FormatToken *Begin,
184 const FormatToken *End) {
185 assert(End->TotalLength >= Begin->TotalLength);
186 return End->TotalLength - Begin->TotalLength + Begin->ColumnWidth;
187}
188
190 // FIXME: At some point we might want to do this for other lists, too.
191 if (!Token->MatchingParen ||
192 !Token->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare)) {
193 return;
194 }
195
196 // In C++11 braced list style, we should not format in columns unless they
197 // have many items (20 or more) or we allow bin-packing of function call
198 // arguments.
200 Commas.size() < 19) {
201 return;
202 }
203
204 // Limit column layout for JavaScript array initializers to 20 or more items
205 // for now to introduce it carefully. We can become more aggressive if this
206 // necessary.
207 if (Token->is(TT_ArrayInitializerLSquare) && Commas.size() < 19)
208 return;
209
210 // Column format doesn't really make sense if we don't align after brackets.
212 return;
213
214 FormatToken *ItemBegin = Token->Next;
215 while (ItemBegin->isTrailingComment())
216 ItemBegin = ItemBegin->Next;
217 SmallVector<bool, 8> MustBreakBeforeItem;
218
219 // The lengths of an item if it is put at the end of the line. This includes
220 // trailing comments which are otherwise ignored for column alignment.
221 SmallVector<unsigned, 8> EndOfLineItemLength;
222 MustBreakBeforeItem.reserve(Commas.size() + 1);
223 EndOfLineItemLength.reserve(Commas.size() + 1);
224 ItemLengths.reserve(Commas.size() + 1);
225
226 bool HasSeparatingComment = false;
227 for (unsigned i = 0, e = Commas.size() + 1; i != e; ++i) {
228 assert(ItemBegin);
229 // Skip comments on their own line.
230 while (ItemBegin->HasUnescapedNewline && ItemBegin->isTrailingComment()) {
231 ItemBegin = ItemBegin->Next;
232 HasSeparatingComment = i > 0;
233 }
234
235 MustBreakBeforeItem.push_back(ItemBegin->MustBreakBefore);
236 if (ItemBegin->is(tok::l_brace))
237 HasNestedBracedList = true;
238 const FormatToken *ItemEnd = nullptr;
239 if (i == Commas.size()) {
240 ItemEnd = Token->MatchingParen;
241 const FormatToken *NonCommentEnd = ItemEnd->getPreviousNonComment();
242 ItemLengths.push_back(CodePointsBetween(ItemBegin, NonCommentEnd));
244 !ItemEnd->Previous->isTrailingComment()) {
245 // In Cpp11 braced list style, the } and possibly other subsequent
246 // tokens will need to stay on a line with the last element.
247 while (ItemEnd->Next && !ItemEnd->Next->CanBreakBefore)
248 ItemEnd = ItemEnd->Next;
249 } else {
250 // In other braced lists styles, the "}" can be wrapped to the new line.
251 ItemEnd = Token->MatchingParen->Previous;
252 }
253 } else {
254 ItemEnd = Commas[i];
255 // The comma is counted as part of the item when calculating the length.
256 ItemLengths.push_back(CodePointsBetween(ItemBegin, ItemEnd));
257
258 // Consume trailing comments so the are included in EndOfLineItemLength.
259 if (ItemEnd->Next && !ItemEnd->Next->HasUnescapedNewline &&
260 ItemEnd->Next->isTrailingComment()) {
261 ItemEnd = ItemEnd->Next;
262 }
263 }
264 EndOfLineItemLength.push_back(CodePointsBetween(ItemBegin, ItemEnd));
265 // If there is a trailing comma in the list, the next item will start at the
266 // closing brace. Don't create an extra item for this.
267 if (ItemEnd->getNextNonComment() == Token->MatchingParen)
268 break;
269 ItemBegin = ItemEnd->Next;
270 }
271
272 // Don't use column layout for lists with few elements and in presence of
273 // separating comments.
274 if (Commas.size() < 5 || HasSeparatingComment)
275 return;
276
277 if (Token->NestingLevel != 0 && Token->is(tok::l_brace) && Commas.size() < 19)
278 return;
279
280 // We can never place more than ColumnLimit / 3 items in a row (because of the
281 // spaces and the comma).
282 unsigned MaxItems = Style.ColumnLimit / 3;
283 SmallVector<unsigned> MinSizeInColumn;
284 MinSizeInColumn.reserve(MaxItems);
285 for (unsigned Columns = 1; Columns <= MaxItems; ++Columns) {
286 ColumnFormat Format;
287 Format.Columns = Columns;
288 Format.ColumnSizes.resize(Columns);
289 MinSizeInColumn.assign(Columns, UINT_MAX);
290 Format.LineCount = 1;
291 bool HasRowWithSufficientColumns = false;
292 unsigned Column = 0;
293 for (unsigned i = 0, e = ItemLengths.size(); i != e; ++i) {
294 assert(i < MustBreakBeforeItem.size());
295 if (MustBreakBeforeItem[i] || Column == Columns) {
296 ++Format.LineCount;
297 Column = 0;
298 }
299 if (Column == Columns - 1)
300 HasRowWithSufficientColumns = true;
301 unsigned Length =
302 (Column == Columns - 1) ? EndOfLineItemLength[i] : ItemLengths[i];
303 Format.ColumnSizes[Column] = std::max(Format.ColumnSizes[Column], Length);
304 MinSizeInColumn[Column] = std::min(MinSizeInColumn[Column], Length);
305 ++Column;
306 }
307 // If all rows are terminated early (e.g. by trailing comments), we don't
308 // need to look further.
309 if (!HasRowWithSufficientColumns)
310 break;
311 Format.TotalWidth = Columns - 1; // Width of the N-1 spaces.
312
313 for (unsigned i = 0; i < Columns; ++i)
314 Format.TotalWidth += Format.ColumnSizes[i];
315
316 // Don't use this Format, if the difference between the longest and shortest
317 // element in a column exceeds a threshold to avoid excessive spaces.
318 if ([&] {
319 for (unsigned i = 0; i < Columns - 1; ++i)
320 if (Format.ColumnSizes[i] - MinSizeInColumn[i] > 10)
321 return true;
322 return false;
323 }()) {
324 continue;
325 }
326
327 // Ignore layouts that are bound to violate the column limit.
328 if (Format.TotalWidth > Style.ColumnLimit && Columns > 1)
329 continue;
330
331 Formats.push_back(Format);
332 }
333}
334
335const CommaSeparatedList::ColumnFormat *
336CommaSeparatedList::getColumnFormat(unsigned RemainingCharacters) const {
337 const ColumnFormat *BestFormat = nullptr;
338 for (const ColumnFormat &Format : llvm::reverse(Formats)) {
339 if (Format.TotalWidth <= RemainingCharacters || Format.Columns == 1) {
340 if (BestFormat && Format.LineCount > BestFormat->LineCount)
341 break;
342 BestFormat = &Format;
343 }
344 }
345 return BestFormat;
346}
347
348} // namespace format
349} // namespace clang
This file implements an indenter that manages the indentation of continuations.
This file contains the declaration of the FormatToken, a wrapper around Token with additional informa...
#define LIST_TOKEN_TYPES
Definition: FormatToken.h:29
static const char *const TokNames[]
Definition: TokenKinds.cpp:17
SourceLocation Begin
ContinuationIndenter * Indenter
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:98
tok::TokenKind getKind() const
Definition: Token.h:93
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:100
The base class of the type hierarchy.
Definition: Type.h:1597
unsigned formatFromToken(LineState &State, ContinuationIndenter *Indenter, bool DryRun) override
Apply the special formatting that the given role demands.
unsigned formatAfterToken(LineState &State, ContinuationIndenter *Indenter, bool DryRun) override
Same as formatFromToken, but assumes that the first token has already been set thereby deciding on th...
void precomputeFormattingInfos(const FormatToken *Token) override
After the TokenAnnotator has finished annotating all the tokens, this function precomputes required i...
virtual void precomputeFormattingInfos(const FormatToken *Token)
After the TokenAnnotator has finished annotating all the tokens, this function precomputes required i...
const FormatStyle & Style
Definition: FormatToken.h:892
#define UINT_MAX
Definition: limits.h:60
const char * getTokenTypeName(TokenType Type)
Determines the name of a token type.
Definition: FormatToken.cpp:24
static unsigned CodePointsBetween(const FormatToken *Begin, const FormatToken *End)
TokenType
Determines the semantic type of a syntactic token, e.g.
Definition: FormatToken.h:175
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T -> getSizeExpr()))
The FormatStyle is used to configure the formatting to follow specific guidelines.
Definition: Format.h:55
@ LK_Proto
Should be used for Protocol Buffers (https://developers.google.com/protocol-buffers/).
Definition: Format.h:2928
@ LK_TextProto
Should be used for Protocol Buffer messages in text format (https://developers.google....
Definition: Format.h:2933
bool Cpp11BracedListStyle
If true, format braced lists as best suited for C++11 braced lists.
Definition: Format.h:2218
@ BAS_DontAlign
Don't align, instead use ContinuationIndentWidth, e.g.:
Definition: Format.h:78
@ BAS_BlockIndent
Always break after an open bracket, if the parameters don't fit on a single line.
Definition: Format.h:99
bool BinPackArguments
If false, a function call's arguments will either be all on the same line or will have one line each.
Definition: Format.h:1044
BracketAlignmentStyle AlignAfterOpenBracket
If true, horizontally aligns arguments after an open bracket.
Definition: Format.h:107
unsigned ColumnLimit
The column limit.
Definition: Format.h:2100
A wrapper around a Token storing information about the whitespace characters preceding it.
Definition: FormatToken.h:260
unsigned NestingLevel
The nesting level of this token, i.e.
Definition: FormatToken.h:474
unsigned CanBreakBefore
true if it is allowed to break before this token.
Definition: FormatToken.h:310
bool opensScope() const
Returns whether Tok is ([{ or an opening < of a template or in protos.
Definition: FormatToken.h:649
FormatToken * getNextNonComment() const
Returns the next token ignoring comments.
Definition: FormatToken.h:771
FormatToken * getPreviousNonComment() const
Returns the previous token ignoring comments.
Definition: FormatToken.h:763
bool isSimpleTypeSpecifier() const
Determine whether the token is a simple-type-specifier.
Definition: FormatToken.cpp:39
BraceBlockKind getBlockKind() const
Definition: FormatToken.h:347
FormatToken * Next
The next token in the unwrapped line.
Definition: FormatToken.h:523
unsigned MustBreakBefore
Whether there must be a line break before this token.
Definition: FormatToken.h:304
unsigned HasUnescapedNewline
Whether there is at least one unescaped newline before the Token.
Definition: FormatToken.h:292
bool isTypeOrIdentifier() const
Definition: FormatToken.cpp:74
bool isBlockIndentedInitRBrace(const FormatStyle &Style) const
Returns true if this token ends a block indented initializer list.
Definition: FormatToken.cpp:78
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:560
bool opensBlockOrBlockTypeList(const FormatStyle &Style) const
Returns true if this tokens starts a block-type list, i.e.
Definition: FormatToken.cpp:93
bool isOneOf(A K1, B K2) const
Definition: FormatToken.h:572
bool isTrailingComment() const
Definition: FormatToken.h:696
FormatToken * MatchingParen
If this is a bracket, this points to the matching one.
Definition: FormatToken.h:517
FormatToken * Previous
The previous token in the unwrapped line.
Definition: FormatToken.h:520
The current state when indenting a unwrapped line.