clang 22.0.0git
FormatToken.cpp
Go to the documentation of this file.
1//===--- FormatToken.cpp - Format C++ code --------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements specific functions of \c FormatTokens and their
11/// roles.
12///
13//===----------------------------------------------------------------------===//
14
15#include "FormatToken.h"
17#include "llvm/ADT/SmallVector.h"
18#include <climits>
19
20namespace clang {
21namespace format {
22
24 static const char *const TokNames[] = {
25#define TYPE(X) #X,
27#undef TYPE
28 nullptr};
29
31 return TokNames[Type];
32 llvm_unreachable("unknown TokenType");
33 return nullptr;
34}
35
36// Sorted common C++ non-keyword types.
38 "clock_t", "int16_t", "int32_t", "int64_t", "int8_t",
39 "intptr_t", "ptrdiff_t", "size_t", "time_t", "uint16_t",
40 "uint32_t", "uint64_t", "uint8_t", "uintptr_t",
41};
42
43bool FormatToken::isTypeName(const LangOptions &LangOpts) const {
44 if (is(TT_TypeName) || Tok.isSimpleTypeSpecifier(LangOpts))
45 return true;
46 assert(llvm::is_sorted(CppNonKeywordTypes));
47 return (LangOpts.CXXOperatorNames || LangOpts.C11) && is(tok::identifier) &&
48 llvm::binary_search(CppNonKeywordTypes, TokenText);
49}
50
51bool FormatToken::isTypeOrIdentifier(const LangOptions &LangOpts) const {
52 return isTypeName(LangOpts) || isOneOf(tok::kw_auto, tok::identifier);
53}
54
55bool FormatToken::isBlockIndentedInitRBrace(const FormatStyle &Style) const {
56 assert(is(tok::r_brace));
57 if (!Style.Cpp11BracedListStyle ||
58 Style.AlignAfterOpenBracket != FormatStyle::BAS_BlockIndent) {
59 return false;
60 }
61 const auto *LBrace = MatchingParen;
62 assert(LBrace && LBrace->is(tok::l_brace));
63 if (LBrace->is(BK_BracedInit))
64 return true;
65 if (LBrace->Previous && LBrace->Previous->is(tok::equal))
66 return true;
67 return false;
68}
69
70bool FormatToken::opensBlockOrBlockTypeList(const FormatStyle &Style) const {
71 // C# Does not indent object initialisers as continuations.
72 if (is(tok::l_brace) && getBlockKind() == BK_BracedInit && Style.isCSharp())
73 return true;
74 if (is(TT_TemplateString) && opensScope())
75 return true;
76 return is(TT_ArrayInitializerLSquare) || is(TT_ProtoExtensionLSquare) ||
77 (is(tok::l_brace) &&
78 (getBlockKind() == BK_Block || is(TT_DictLiteral) ||
79 (!Style.Cpp11BracedListStyle && NestingLevel == 0))) ||
80 (is(tok::less) && Style.isProto());
81}
82
84
86
87unsigned CommaSeparatedList::formatAfterToken(LineState &State,
88 ContinuationIndenter *Indenter,
89 bool DryRun) {
90 if (!State.NextToken || !State.NextToken->Previous)
91 return 0;
92
93 if (Formats.size() <= 1)
94 return 0; // Handled by formatFromToken (1) or avoid severe penalty (0).
95
96 // Ensure that we start on the opening brace.
97 const FormatToken *LBrace =
98 State.NextToken->Previous->getPreviousNonComment();
99 if (!LBrace || !LBrace->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) ||
100 LBrace->is(BK_Block) || LBrace->is(TT_DictLiteral) ||
101 LBrace->Next->is(TT_DesignatedInitializerPeriod)) {
102 return 0;
103 }
104
105 // Calculate the number of code points we have to format this list. As the
106 // first token is already placed, we have to subtract it.
107 unsigned RemainingCodePoints =
108 Style.ColumnLimit - State.Column + State.NextToken->Previous->ColumnWidth;
109
110 // Find the best ColumnFormat, i.e. the best number of columns to use.
111 const ColumnFormat *Format = getColumnFormat(RemainingCodePoints);
112
113 // If no ColumnFormat can be used, the braced list would generally be
114 // bin-packed. Add a severe penalty to this so that column layouts are
115 // preferred if possible.
116 if (!Format)
117 return 10'000;
118
119 // Format the entire list.
120 unsigned Penalty = 0;
121 unsigned Column = 0;
122 unsigned Item = 0;
123 while (State.NextToken != LBrace->MatchingParen) {
124 bool NewLine = false;
125 unsigned ExtraSpaces = 0;
126
127 // If the previous token was one of our commas, we are now on the next item.
128 if (Item < Commas.size() && State.NextToken->Previous == Commas[Item]) {
129 if (!State.NextToken->isTrailingComment()) {
130 ExtraSpaces += Format->ColumnSizes[Column] - ItemLengths[Item];
131 ++Column;
132 }
133 ++Item;
134 }
135
136 if (Column == Format->Columns || State.NextToken->MustBreakBefore) {
137 Column = 0;
138 NewLine = true;
139 }
140
141 // Place token using the continuation indenter and store the penalty.
142 Penalty += Indenter->addTokenToState(State, NewLine, DryRun, ExtraSpaces);
143 }
144 return Penalty;
145}
146
147unsigned CommaSeparatedList::formatFromToken(LineState &State,
148 ContinuationIndenter *Indenter,
149 bool DryRun) {
150 // Formatting with 1 Column isn't really a column layout, so we don't need the
151 // special logic here. We can just avoid bin packing any of the parameters.
152 if (Formats.size() == 1 || HasNestedBracedList)
153 State.Stack.back().AvoidBinPacking = true;
154 return 0;
155}
156
157// Returns the lengths in code points between Begin and End (both included),
158// assuming that the entire sequence is put on a single line.
159static unsigned CodePointsBetween(const FormatToken *Begin,
160 const FormatToken *End) {
161 assert(End->TotalLength >= Begin->TotalLength);
162 return End->TotalLength - Begin->TotalLength + Begin->ColumnWidth;
163}
164
166 // FIXME: At some point we might want to do this for other lists, too.
167 if (!Token->MatchingParen ||
168 !Token->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare)) {
169 return;
170 }
171
172 // In C++11 braced list style, we should not format in columns unless they
173 // have many items (20 or more) or we allow bin-packing of function call
174 // arguments.
175 if (Style.Cpp11BracedListStyle && !Style.BinPackArguments &&
176 (Commas.size() < 19 || !Style.BinPackLongBracedList)) {
177 return;
178 }
179
180 // Limit column layout for JavaScript array initializers to 20 or more items
181 // for now to introduce it carefully. We can become more aggressive if this
182 // necessary.
183 if (Token->is(TT_ArrayInitializerLSquare) && Commas.size() < 19)
184 return;
185
186 // Column format doesn't really make sense if we don't align after brackets.
187 if (Style.AlignAfterOpenBracket == FormatStyle::BAS_DontAlign)
188 return;
189
190 FormatToken *ItemBegin = Token->Next;
191 while (ItemBegin->isTrailingComment())
192 ItemBegin = ItemBegin->Next;
193 SmallVector<bool, 8> MustBreakBeforeItem;
194
195 // The lengths of an item if it is put at the end of the line. This includes
196 // trailing comments which are otherwise ignored for column alignment.
197 SmallVector<unsigned, 8> EndOfLineItemLength;
198 MustBreakBeforeItem.reserve(Commas.size() + 1);
199 EndOfLineItemLength.reserve(Commas.size() + 1);
200 ItemLengths.reserve(Commas.size() + 1);
201
202 bool HasSeparatingComment = false;
203 for (unsigned i = 0, e = Commas.size() + 1; i != e; ++i) {
204 assert(ItemBegin);
205 // Skip comments on their own line.
206 while (ItemBegin->HasUnescapedNewline && ItemBegin->isTrailingComment()) {
207 ItemBegin = ItemBegin->Next;
208 HasSeparatingComment = i > 0;
209 }
210
211 MustBreakBeforeItem.push_back(ItemBegin->MustBreakBefore);
212 if (ItemBegin->is(tok::l_brace))
213 HasNestedBracedList = true;
214 const FormatToken *ItemEnd = nullptr;
215 if (i == Commas.size()) {
216 ItemEnd = Token->MatchingParen;
217 const FormatToken *NonCommentEnd = ItemEnd->getPreviousNonComment();
218 ItemLengths.push_back(CodePointsBetween(ItemBegin, NonCommentEnd));
219 if (Style.Cpp11BracedListStyle &&
220 !ItemEnd->Previous->isTrailingComment()) {
221 // In Cpp11 braced list style, the } and possibly other subsequent
222 // tokens will need to stay on a line with the last element.
223 while (ItemEnd->Next && !ItemEnd->Next->CanBreakBefore)
224 ItemEnd = ItemEnd->Next;
225 } else {
226 // In other braced lists styles, the "}" can be wrapped to the new line.
227 ItemEnd = Token->MatchingParen->Previous;
228 }
229 } else {
230 ItemEnd = Commas[i];
231 // The comma is counted as part of the item when calculating the length.
232 ItemLengths.push_back(CodePointsBetween(ItemBegin, ItemEnd));
233
234 // Consume trailing comments so the are included in EndOfLineItemLength.
235 if (ItemEnd->Next && !ItemEnd->Next->HasUnescapedNewline &&
236 ItemEnd->Next->isTrailingComment()) {
237 ItemEnd = ItemEnd->Next;
238 }
239 }
240 EndOfLineItemLength.push_back(CodePointsBetween(ItemBegin, ItemEnd));
241 // If there is a trailing comma in the list, the next item will start at the
242 // closing brace. Don't create an extra item for this.
243 if (ItemEnd->getNextNonComment() == Token->MatchingParen)
244 break;
245 ItemBegin = ItemEnd->Next;
246 }
247
248 // Don't use column layout for lists with few elements and in presence of
249 // separating comments.
250 if (Commas.size() < 5 || HasSeparatingComment)
251 return;
252
253 if (Token->NestingLevel != 0 && Token->is(tok::l_brace) && Commas.size() < 19)
254 return;
255
256 // We can never place more than ColumnLimit / 3 items in a row (because of the
257 // spaces and the comma).
258 unsigned MaxItems = Style.ColumnLimit / 3;
259 SmallVector<unsigned> MinSizeInColumn;
260 MinSizeInColumn.reserve(MaxItems);
261 for (unsigned Columns = 1; Columns <= MaxItems; ++Columns) {
262 ColumnFormat Format;
263 Format.Columns = Columns;
264 Format.ColumnSizes.resize(Columns);
265 MinSizeInColumn.assign(Columns, UINT_MAX);
266 Format.LineCount = 1;
267 bool HasRowWithSufficientColumns = false;
268 unsigned Column = 0;
269 for (unsigned i = 0, e = ItemLengths.size(); i != e; ++i) {
270 assert(i < MustBreakBeforeItem.size());
271 if (MustBreakBeforeItem[i] || Column == Columns) {
272 ++Format.LineCount;
273 Column = 0;
274 }
275 if (Column == Columns - 1)
276 HasRowWithSufficientColumns = true;
277 unsigned Length =
278 (Column == Columns - 1) ? EndOfLineItemLength[i] : ItemLengths[i];
279 Format.ColumnSizes[Column] = std::max(Format.ColumnSizes[Column], Length);
280 MinSizeInColumn[Column] = std::min(MinSizeInColumn[Column], Length);
281 ++Column;
282 }
283 // If all rows are terminated early (e.g. by trailing comments), we don't
284 // need to look further.
285 if (!HasRowWithSufficientColumns)
286 break;
287 Format.TotalWidth = Columns - 1; // Width of the N-1 spaces.
288
289 for (unsigned i = 0; i < Columns; ++i)
290 Format.TotalWidth += Format.ColumnSizes[i];
291
292 // Don't use this Format, if the difference between the longest and shortest
293 // element in a column exceeds a threshold to avoid excessive spaces.
294 if ([&] {
295 for (unsigned i = 0; i < Columns - 1; ++i)
296 if (Format.ColumnSizes[i] - MinSizeInColumn[i] > 10)
297 return true;
298 return false;
299 }()) {
300 continue;
301 }
302
303 // Ignore layouts that are bound to violate the column limit.
304 if (Format.TotalWidth > Style.ColumnLimit && Columns > 1)
305 continue;
306
307 Formats.push_back(Format);
308 }
309}
310
311const CommaSeparatedList::ColumnFormat *
312CommaSeparatedList::getColumnFormat(unsigned RemainingCharacters) const {
313 const ColumnFormat *BestFormat = nullptr;
314 for (const ColumnFormat &Format : llvm::reverse(Formats)) {
315 if (Format.TotalWidth <= RemainingCharacters || Format.Columns == 1) {
316 if (BestFormat && Format.LineCount > BestFormat->LineCount)
317 break;
318 BestFormat = &Format;
319 }
320 }
321 return BestFormat;
322}
323
324bool startsNextParameter(const FormatToken &Current, const FormatStyle &Style) {
325 assert(Current.Previous);
326 const auto &Previous = *Current.Previous;
327 if (Current.is(TT_CtorInitializerComma) &&
328 Style.BreakConstructorInitializers == FormatStyle::BCIS_BeforeComma) {
329 return true;
330 }
331 if (Style.Language == FormatStyle::LK_Proto && Current.is(TT_SelectorName))
332 return true;
333 return Previous.is(tok::comma) && !Current.isTrailingComment() &&
334 ((Previous.isNot(TT_CtorInitializerComma) ||
335 Style.BreakConstructorInitializers !=
336 FormatStyle::BCIS_BeforeComma) &&
337 (Previous.isNot(TT_InheritanceComma) ||
338 Style.BreakInheritanceList != FormatStyle::BILS_BeforeComma));
339}
340
341} // namespace format
342} // namespace clang
This file implements an indenter that manages the indentation of continuations.
This file contains the declaration of the FormatToken, a wrapper around Token with additional informa...
unsigned NestingLevel
The nesting level of this token, i.e.
#define LIST_TOKEN_TYPES
Definition FormatToken.h:27
StringRef TokenText
The raw text of the token.
FormatToken()
Token Tok
The Token.
FormatToken * MatchingParen
If this is a bracket, this points to the matching one.
FormatToken * Previous
The previous token in the unwrapped line.
static constexpr bool isOneOf()
static const char *const TokNames[]
unsigned formatFromToken(LineState &State, ContinuationIndenter *Indenter, bool DryRun) override
Apply the special formatting that the given role demands.
unsigned formatAfterToken(LineState &State, ContinuationIndenter *Indenter, bool DryRun) override
Same as formatFromToken, but assumes that the first token has already been set thereby deciding on th...
void precomputeFormattingInfos(const FormatToken *Token) override
After the TokenAnnotator has finished annotating all the tokens, this function precomputes required i...
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
virtual void precomputeFormattingInfos(const FormatToken *Token)
After the TokenAnnotator has finished annotating all the tokens, this function precomputes required i...
const FormatStyle & Style
virtual ~TokenRole()
Token - This structure provides full information about a lexed token.
Definition Token.h:36
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition Token.h:102
bool isOneOf(Ts... Ks) const
Definition Token.h:104
The base class of the type hierarchy.
Definition TypeBase.h:1833
#define UINT_MAX
Definition limits.h:64
const char * getTokenTypeName(TokenType Type)
Determines the name of a token type.
static SmallVector< StringRef > CppNonKeywordTypes
static unsigned CodePointsBetween(const FormatToken *Begin, const FormatToken *End)
TokenType
Determines the semantic type of a syntactic token, e.g.
bool startsNextParameter(const FormatToken &Current, const FormatStyle &Style)
The JSON file list parser is used to communicate input to InstallAPI.
A wrapper around a Token storing information about the whitespace characters preceding it.
BraceBlockKind getBlockKind() const
unsigned ColumnWidth
The width of the non-whitespace parts of the token (or its first line for multi-line tokens) in colum...
bool is(tok::TokenKind Kind) const
unsigned TotalLength
The total length of the unwrapped line up to and including this token.
FormatToken * Previous
The previous token in the unwrapped line.