clang 19.0.0git
FormatToken.cpp
Go to the documentation of this file.
1//===--- FormatToken.cpp - Format C++ code --------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements specific functions of \c FormatTokens and their
11/// roles.
12///
13//===----------------------------------------------------------------------===//
14
15#include "FormatToken.h"
17
18namespace clang {
19namespace format {
20
21bool IsCpp = false;
22
24 static const char *const TokNames[] = {
25#define TYPE(X) #X,
27#undef TYPE
28 nullptr};
29
31 return TokNames[Type];
32 llvm_unreachable("unknown TokenType");
33 return nullptr;
34}
35
36// FIXME: This is copy&pasted from Sema. Put it in a common place and remove
37// duplication.
39 switch (Tok.getKind()) {
40 case tok::kw_short:
41 case tok::kw_long:
42 case tok::kw___int64:
43 case tok::kw___int128:
44 case tok::kw_signed:
45 case tok::kw_unsigned:
46 case tok::kw_void:
47 case tok::kw_char:
48 case tok::kw_int:
49 case tok::kw_half:
50 case tok::kw_float:
51 case tok::kw_double:
52 case tok::kw___bf16:
53 case tok::kw__Float16:
54 case tok::kw___float128:
55 case tok::kw___ibm128:
56 case tok::kw_wchar_t:
57 case tok::kw_bool:
58#define TRANSFORM_TYPE_TRAIT_DEF(_, Trait) case tok::kw___##Trait:
59#include "clang/Basic/TransformTypeTraits.def"
60 case tok::annot_typename:
61 case tok::kw_char8_t:
62 case tok::kw_char16_t:
63 case tok::kw_char32_t:
64 case tok::kw_typeof:
65 case tok::kw_decltype:
66 case tok::kw__Atomic:
67 return true;
68 default:
69 return false;
70 }
71}
72
73// Sorted common C++ non-keyword types.
75 "clock_t", "int16_t", "int32_t", "int64_t", "int8_t",
76 "intptr_t", "ptrdiff_t", "size_t", "time_t", "uint16_t",
77 "uint32_t", "uint64_t", "uint8_t", "uintptr_t",
78};
79
81 return is(TT_TypeName) || isSimpleTypeSpecifier() ||
82 (IsCpp && is(tok::identifier) &&
83 std::binary_search(CppNonKeywordTypes.begin(),
85}
86
88 return isTypeName() || isOneOf(tok::kw_auto, tok::identifier);
89}
90
92 assert(is(tok::r_brace));
93 if (!Style.Cpp11BracedListStyle ||
94 Style.AlignAfterOpenBracket != FormatStyle::BAS_BlockIndent) {
95 return false;
96 }
97 const auto *LBrace = MatchingParen;
98 assert(LBrace && LBrace->is(tok::l_brace));
99 if (LBrace->is(BK_BracedInit))
100 return true;
101 if (LBrace->Previous && LBrace->Previous->is(tok::equal))
102 return true;
103 return false;
104}
105
107 // C# Does not indent object initialisers as continuations.
108 if (is(tok::l_brace) && getBlockKind() == BK_BracedInit && Style.isCSharp())
109 return true;
110 if (is(TT_TemplateString) && opensScope())
111 return true;
112 return is(TT_ArrayInitializerLSquare) || is(TT_ProtoExtensionLSquare) ||
113 (is(tok::l_brace) &&
114 (getBlockKind() == BK_Block || is(TT_DictLiteral) ||
115 (!Style.Cpp11BracedListStyle && NestingLevel == 0))) ||
116 (is(tok::less) && Style.isProto());
117}
118
120
122
125 bool DryRun) {
126 if (!State.NextToken || !State.NextToken->Previous)
127 return 0;
128
129 if (Formats.size() <= 1)
130 return 0; // Handled by formatFromToken (1) or avoid severe penalty (0).
131
132 // Ensure that we start on the opening brace.
133 const FormatToken *LBrace =
134 State.NextToken->Previous->getPreviousNonComment();
135 if (!LBrace || !LBrace->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) ||
136 LBrace->is(BK_Block) || LBrace->is(TT_DictLiteral) ||
137 LBrace->Next->is(TT_DesignatedInitializerPeriod)) {
138 return 0;
139 }
140
141 // Calculate the number of code points we have to format this list. As the
142 // first token is already placed, we have to subtract it.
143 unsigned RemainingCodePoints =
144 Style.ColumnLimit - State.Column + State.NextToken->Previous->ColumnWidth;
145
146 // Find the best ColumnFormat, i.e. the best number of columns to use.
147 const ColumnFormat *Format = getColumnFormat(RemainingCodePoints);
148
149 // If no ColumnFormat can be used, the braced list would generally be
150 // bin-packed. Add a severe penalty to this so that column layouts are
151 // preferred if possible.
152 if (!Format)
153 return 10'000;
154
155 // Format the entire list.
156 unsigned Penalty = 0;
157 unsigned Column = 0;
158 unsigned Item = 0;
159 while (State.NextToken != LBrace->MatchingParen) {
160 bool NewLine = false;
161 unsigned ExtraSpaces = 0;
162
163 // If the previous token was one of our commas, we are now on the next item.
164 if (Item < Commas.size() && State.NextToken->Previous == Commas[Item]) {
165 if (!State.NextToken->isTrailingComment()) {
166 ExtraSpaces += Format->ColumnSizes[Column] - ItemLengths[Item];
167 ++Column;
168 }
169 ++Item;
170 }
171
172 if (Column == Format->Columns || State.NextToken->MustBreakBefore) {
173 Column = 0;
174 NewLine = true;
175 }
176
177 // Place token using the continuation indenter and store the penalty.
178 Penalty += Indenter->addTokenToState(State, NewLine, DryRun, ExtraSpaces);
179 }
180 return Penalty;
181}
182
185 bool DryRun) {
186 // Formatting with 1 Column isn't really a column layout, so we don't need the
187 // special logic here. We can just avoid bin packing any of the parameters.
188 if (Formats.size() == 1 || HasNestedBracedList)
189 State.Stack.back().AvoidBinPacking = true;
190 return 0;
191}
192
193// Returns the lengths in code points between Begin and End (both included),
194// assuming that the entire sequence is put on a single line.
195static unsigned CodePointsBetween(const FormatToken *Begin,
196 const FormatToken *End) {
197 assert(End->TotalLength >= Begin->TotalLength);
198 return End->TotalLength - Begin->TotalLength + Begin->ColumnWidth;
199}
200
202 // FIXME: At some point we might want to do this for other lists, too.
203 if (!Token->MatchingParen ||
204 !Token->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare)) {
205 return;
206 }
207
208 // In C++11 braced list style, we should not format in columns unless they
209 // have many items (20 or more) or we allow bin-packing of function call
210 // arguments.
212 Commas.size() < 19) {
213 return;
214 }
215
216 // Limit column layout for JavaScript array initializers to 20 or more items
217 // for now to introduce it carefully. We can become more aggressive if this
218 // necessary.
219 if (Token->is(TT_ArrayInitializerLSquare) && Commas.size() < 19)
220 return;
221
222 // Column format doesn't really make sense if we don't align after brackets.
224 return;
225
226 FormatToken *ItemBegin = Token->Next;
227 while (ItemBegin->isTrailingComment())
228 ItemBegin = ItemBegin->Next;
229 SmallVector<bool, 8> MustBreakBeforeItem;
230
231 // The lengths of an item if it is put at the end of the line. This includes
232 // trailing comments which are otherwise ignored for column alignment.
233 SmallVector<unsigned, 8> EndOfLineItemLength;
234 MustBreakBeforeItem.reserve(Commas.size() + 1);
235 EndOfLineItemLength.reserve(Commas.size() + 1);
236 ItemLengths.reserve(Commas.size() + 1);
237
238 bool HasSeparatingComment = false;
239 for (unsigned i = 0, e = Commas.size() + 1; i != e; ++i) {
240 assert(ItemBegin);
241 // Skip comments on their own line.
242 while (ItemBegin->HasUnescapedNewline && ItemBegin->isTrailingComment()) {
243 ItemBegin = ItemBegin->Next;
244 HasSeparatingComment = i > 0;
245 }
246
247 MustBreakBeforeItem.push_back(ItemBegin->MustBreakBefore);
248 if (ItemBegin->is(tok::l_brace))
249 HasNestedBracedList = true;
250 const FormatToken *ItemEnd = nullptr;
251 if (i == Commas.size()) {
252 ItemEnd = Token->MatchingParen;
253 const FormatToken *NonCommentEnd = ItemEnd->getPreviousNonComment();
254 ItemLengths.push_back(CodePointsBetween(ItemBegin, NonCommentEnd));
256 !ItemEnd->Previous->isTrailingComment()) {
257 // In Cpp11 braced list style, the } and possibly other subsequent
258 // tokens will need to stay on a line with the last element.
259 while (ItemEnd->Next && !ItemEnd->Next->CanBreakBefore)
260 ItemEnd = ItemEnd->Next;
261 } else {
262 // In other braced lists styles, the "}" can be wrapped to the new line.
263 ItemEnd = Token->MatchingParen->Previous;
264 }
265 } else {
266 ItemEnd = Commas[i];
267 // The comma is counted as part of the item when calculating the length.
268 ItemLengths.push_back(CodePointsBetween(ItemBegin, ItemEnd));
269
270 // Consume trailing comments so the are included in EndOfLineItemLength.
271 if (ItemEnd->Next && !ItemEnd->Next->HasUnescapedNewline &&
272 ItemEnd->Next->isTrailingComment()) {
273 ItemEnd = ItemEnd->Next;
274 }
275 }
276 EndOfLineItemLength.push_back(CodePointsBetween(ItemBegin, ItemEnd));
277 // If there is a trailing comma in the list, the next item will start at the
278 // closing brace. Don't create an extra item for this.
279 if (ItemEnd->getNextNonComment() == Token->MatchingParen)
280 break;
281 ItemBegin = ItemEnd->Next;
282 }
283
284 // Don't use column layout for lists with few elements and in presence of
285 // separating comments.
286 if (Commas.size() < 5 || HasSeparatingComment)
287 return;
288
289 if (Token->NestingLevel != 0 && Token->is(tok::l_brace) && Commas.size() < 19)
290 return;
291
292 // We can never place more than ColumnLimit / 3 items in a row (because of the
293 // spaces and the comma).
294 unsigned MaxItems = Style.ColumnLimit / 3;
295 SmallVector<unsigned> MinSizeInColumn;
296 MinSizeInColumn.reserve(MaxItems);
297 for (unsigned Columns = 1; Columns <= MaxItems; ++Columns) {
298 ColumnFormat Format;
299 Format.Columns = Columns;
300 Format.ColumnSizes.resize(Columns);
301 MinSizeInColumn.assign(Columns, UINT_MAX);
302 Format.LineCount = 1;
303 bool HasRowWithSufficientColumns = false;
304 unsigned Column = 0;
305 for (unsigned i = 0, e = ItemLengths.size(); i != e; ++i) {
306 assert(i < MustBreakBeforeItem.size());
307 if (MustBreakBeforeItem[i] || Column == Columns) {
308 ++Format.LineCount;
309 Column = 0;
310 }
311 if (Column == Columns - 1)
312 HasRowWithSufficientColumns = true;
313 unsigned Length =
314 (Column == Columns - 1) ? EndOfLineItemLength[i] : ItemLengths[i];
315 Format.ColumnSizes[Column] = std::max(Format.ColumnSizes[Column], Length);
316 MinSizeInColumn[Column] = std::min(MinSizeInColumn[Column], Length);
317 ++Column;
318 }
319 // If all rows are terminated early (e.g. by trailing comments), we don't
320 // need to look further.
321 if (!HasRowWithSufficientColumns)
322 break;
323 Format.TotalWidth = Columns - 1; // Width of the N-1 spaces.
324
325 for (unsigned i = 0; i < Columns; ++i)
326 Format.TotalWidth += Format.ColumnSizes[i];
327
328 // Don't use this Format, if the difference between the longest and shortest
329 // element in a column exceeds a threshold to avoid excessive spaces.
330 if ([&] {
331 for (unsigned i = 0; i < Columns - 1; ++i)
332 if (Format.ColumnSizes[i] - MinSizeInColumn[i] > 10)
333 return true;
334 return false;
335 }()) {
336 continue;
337 }
338
339 // Ignore layouts that are bound to violate the column limit.
340 if (Format.TotalWidth > Style.ColumnLimit && Columns > 1)
341 continue;
342
343 Formats.push_back(Format);
344 }
345}
346
347const CommaSeparatedList::ColumnFormat *
348CommaSeparatedList::getColumnFormat(unsigned RemainingCharacters) const {
349 const ColumnFormat *BestFormat = nullptr;
350 for (const ColumnFormat &Format : llvm::reverse(Formats)) {
351 if (Format.TotalWidth <= RemainingCharacters || Format.Columns == 1) {
352 if (BestFormat && Format.LineCount > BestFormat->LineCount)
353 break;
354 BestFormat = &Format;
355 }
356 }
357 return BestFormat;
358}
359
360} // namespace format
361} // namespace clang
This file implements an indenter that manages the indentation of continuations.
This file contains the declaration of the FormatToken, a wrapper around Token with additional informa...
#define LIST_TOKEN_TYPES
Definition: FormatToken.h:30
static constexpr bool isOneOf()
static const char *const TokNames[]
Definition: TokenKinds.cpp:17
SourceLocation Begin
ContinuationIndenter * Indenter
Token - This structure provides full information about a lexed token.
Definition: Token.h:36
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:99
tok::TokenKind getKind() const
Definition: Token.h:94
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:101
The base class of the type hierarchy.
Definition: Type.h:1606
unsigned formatFromToken(LineState &State, ContinuationIndenter *Indenter, bool DryRun) override
Apply the special formatting that the given role demands.
unsigned formatAfterToken(LineState &State, ContinuationIndenter *Indenter, bool DryRun) override
Same as formatFromToken, but assumes that the first token has already been set thereby deciding on th...
void precomputeFormattingInfos(const FormatToken *Token) override
After the TokenAnnotator has finished annotating all the tokens, this function precomputes required i...
virtual void precomputeFormattingInfos(const FormatToken *Token)
After the TokenAnnotator has finished annotating all the tokens, this function precomputes required i...
const FormatStyle & Style
Definition: FormatToken.h:926
#define UINT_MAX
Definition: limits.h:60
const char * getTokenTypeName(TokenType Type)
Determines the name of a token type.
Definition: FormatToken.cpp:23
static SmallVector< StringRef > CppNonKeywordTypes
Definition: FormatToken.cpp:74
bool IsCpp
Whether the language is C/C++/Objective-C/Objective-C++.
Definition: FormatToken.cpp:21
static unsigned CodePointsBetween(const FormatToken *Begin, const FormatToken *End)
TokenType
Determines the semantic type of a syntactic token, e.g.
Definition: FormatToken.h:198
The JSON file list parser is used to communicate input to InstallAPI.
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T -> getSizeExpr()))
The FormatStyle is used to configure the formatting to follow specific guidelines.
Definition: Format.h:55
bool Cpp11BracedListStyle
If true, format braced lists as best suited for C++11 braced lists.
Definition: Format.h:2392
@ BAS_DontAlign
Don't align, instead use ContinuationIndentWidth, e.g.:
Definition: Format.h:78
@ BAS_BlockIndent
Always break after an open bracket, if the parameters don't fit on a single line.
Definition: Format.h:99
bool BinPackArguments
If false, a function call's arguments will either be all on the same line or will have one line each.
Definition: Format.h:1140
BracketAlignmentStyle AlignAfterOpenBracket
If true, horizontally aligns arguments after an open bracket.
Definition: Format.h:107
unsigned ColumnLimit
The column limit.
Definition: Format.h:2270
A wrapper around a Token storing information about the whitespace characters preceding it.
Definition: FormatToken.h:283
unsigned NestingLevel
The nesting level of this token, i.e.
Definition: FormatToken.h:506
unsigned CanBreakBefore
true if it is allowed to break before this token.
Definition: FormatToken.h:338
StringRef TokenText
The raw text of the token.
Definition: FormatToken.h:303
bool opensScope() const
Returns whether Tok is ([{ or an opening < of a template or in protos.
Definition: FormatToken.h:688
FormatToken * getNextNonComment() const
Returns the next token ignoring comments.
Definition: FormatToken.h:805
FormatToken * getPreviousNonComment() const
Returns the previous token ignoring comments.
Definition: FormatToken.h:797
bool isSimpleTypeSpecifier() const
Determine whether the token is a simple-type-specifier.
Definition: FormatToken.cpp:38
BraceBlockKind getBlockKind() const
Definition: FormatToken.h:375
FormatToken * Next
The next token in the unwrapped line.
Definition: FormatToken.h:555
unsigned MustBreakBefore
Whether there must be a line break before this token.
Definition: FormatToken.h:328
unsigned HasUnescapedNewline
Whether there is at least one unescaped newline before the Token.
Definition: FormatToken.h:316
bool isTypeOrIdentifier() const
Definition: FormatToken.cpp:87
bool isBlockIndentedInitRBrace(const FormatStyle &Style) const
Returns true if this token ends a block indented initializer list.
Definition: FormatToken.cpp:91
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:592
bool opensBlockOrBlockTypeList(const FormatStyle &Style) const
Returns true if this tokens starts a block-type list, i.e.
bool isOneOf(A K1, B K2) const
Definition: FormatToken.h:604
bool isTrailingComment() const
Definition: FormatToken.h:739
FormatToken * MatchingParen
If this is a bracket, this points to the matching one.
Definition: FormatToken.h:549
FormatToken * Previous
The previous token in the unwrapped line.
Definition: FormatToken.h:552
The current state when indenting a unwrapped line.