clang  14.0.0git
FormatToken.cpp
Go to the documentation of this file.
1 //===--- FormatToken.cpp - Format C++ code --------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements specific functions of \c FormatTokens and their
11 /// roles.
12 ///
13 //===----------------------------------------------------------------------===//
14 
15 #include "FormatToken.h"
16 #include "ContinuationIndenter.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/Support/Debug.h"
19 #include <climits>
20 
21 namespace clang {
22 namespace format {
23 
25  static const char *const TokNames[] = {
26 #define TYPE(X) #X,
28 #undef TYPE
29  nullptr};
30 
31  if (Type < NUM_TOKEN_TYPES)
32  return TokNames[Type];
33  llvm_unreachable("unknown TokenType");
34  return nullptr;
35 }
36 
37 // FIXME: This is copy&pasted from Sema. Put it in a common place and remove
38 // duplication.
40  switch (Tok.getKind()) {
41  case tok::kw_short:
42  case tok::kw_long:
43  case tok::kw___int64:
44  case tok::kw___int128:
45  case tok::kw_signed:
46  case tok::kw_unsigned:
47  case tok::kw_void:
48  case tok::kw_char:
49  case tok::kw_int:
50  case tok::kw_half:
51  case tok::kw_float:
52  case tok::kw_double:
53  case tok::kw___bf16:
54  case tok::kw__Float16:
55  case tok::kw___float128:
56  case tok::kw___ibm128:
57  case tok::kw_wchar_t:
58  case tok::kw_bool:
59  case tok::kw___underlying_type:
60  case tok::annot_typename:
61  case tok::kw_char8_t:
62  case tok::kw_char16_t:
63  case tok::kw_char32_t:
64  case tok::kw_typeof:
65  case tok::kw_decltype:
66  case tok::kw__Atomic:
67  return true;
68  default:
69  return false;
70  }
71 }
72 
74 
76 
79  bool DryRun) {
80  if (State.NextToken == nullptr || !State.NextToken->Previous)
81  return 0;
82 
83  if (Formats.size() == 1)
84  return 0; // Handled by formatFromToken
85 
86  // Ensure that we start on the opening brace.
87  const FormatToken *LBrace =
88  State.NextToken->Previous->getPreviousNonComment();
89  if (!LBrace || !LBrace->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) ||
90  LBrace->is(BK_Block) || LBrace->is(TT_DictLiteral) ||
91  LBrace->Next->is(TT_DesignatedInitializerPeriod))
92  return 0;
93 
94  // Calculate the number of code points we have to format this list. As the
95  // first token is already placed, we have to subtract it.
96  unsigned RemainingCodePoints =
97  Style.ColumnLimit - State.Column + State.NextToken->Previous->ColumnWidth;
98 
99  // Find the best ColumnFormat, i.e. the best number of columns to use.
100  const ColumnFormat *Format = getColumnFormat(RemainingCodePoints);
101 
102  // If no ColumnFormat can be used, the braced list would generally be
103  // bin-packed. Add a severe penalty to this so that column layouts are
104  // preferred if possible.
105  if (!Format)
106  return 10000;
107 
108  // Format the entire list.
109  unsigned Penalty = 0;
110  unsigned Column = 0;
111  unsigned Item = 0;
112  while (State.NextToken != LBrace->MatchingParen) {
113  bool NewLine = false;
114  unsigned ExtraSpaces = 0;
115 
116  // If the previous token was one of our commas, we are now on the next item.
117  if (Item < Commas.size() && State.NextToken->Previous == Commas[Item]) {
118  if (!State.NextToken->isTrailingComment()) {
119  ExtraSpaces += Format->ColumnSizes[Column] - ItemLengths[Item];
120  ++Column;
121  }
122  ++Item;
123  }
124 
125  if (Column == Format->Columns || State.NextToken->MustBreakBefore) {
126  Column = 0;
127  NewLine = true;
128  }
129 
130  // Place token using the continuation indenter and store the penalty.
131  Penalty += Indenter->addTokenToState(State, NewLine, DryRun, ExtraSpaces);
132  }
133  return Penalty;
134 }
135 
138  bool DryRun) {
139  // Formatting with 1 Column isn't really a column layout, so we don't need the
140  // special logic here. We can just avoid bin packing any of the parameters.
141  if (Formats.size() == 1 || HasNestedBracedList)
142  State.Stack.back().AvoidBinPacking = true;
143  return 0;
144 }
145 
146 // Returns the lengths in code points between Begin and End (both included),
147 // assuming that the entire sequence is put on a single line.
148 static unsigned CodePointsBetween(const FormatToken *Begin,
149  const FormatToken *End) {
150  assert(End->TotalLength >= Begin->TotalLength);
151  return End->TotalLength - Begin->TotalLength + Begin->ColumnWidth;
152 }
153 
155  // FIXME: At some point we might want to do this for other lists, too.
156  if (!Token->MatchingParen ||
157  !Token->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare))
158  return;
159 
160  // In C++11 braced list style, we should not format in columns unless they
161  // have many items (20 or more) or we allow bin-packing of function call
162  // arguments.
164  Commas.size() < 19)
165  return;
166 
167  // Limit column layout for JavaScript array initializers to 20 or more items
168  // for now to introduce it carefully. We can become more aggressive if this
169  // necessary.
170  if (Token->is(TT_ArrayInitializerLSquare) && Commas.size() < 19)
171  return;
172 
173  // Column format doesn't really make sense if we don't align after brackets.
175  return;
176 
177  FormatToken *ItemBegin = Token->Next;
178  while (ItemBegin->isTrailingComment())
179  ItemBegin = ItemBegin->Next;
180  SmallVector<bool, 8> MustBreakBeforeItem;
181 
182  // The lengths of an item if it is put at the end of the line. This includes
183  // trailing comments which are otherwise ignored for column alignment.
184  SmallVector<unsigned, 8> EndOfLineItemLength;
185 
186  bool HasSeparatingComment = false;
187  for (unsigned i = 0, e = Commas.size() + 1; i != e; ++i) {
188  // Skip comments on their own line.
189  while (ItemBegin->HasUnescapedNewline && ItemBegin->isTrailingComment()) {
190  ItemBegin = ItemBegin->Next;
191  HasSeparatingComment = i > 0;
192  }
193 
194  MustBreakBeforeItem.push_back(ItemBegin->MustBreakBefore);
195  if (ItemBegin->is(tok::l_brace))
196  HasNestedBracedList = true;
197  const FormatToken *ItemEnd = nullptr;
198  if (i == Commas.size()) {
199  ItemEnd = Token->MatchingParen;
200  const FormatToken *NonCommentEnd = ItemEnd->getPreviousNonComment();
201  ItemLengths.push_back(CodePointsBetween(ItemBegin, NonCommentEnd));
203  !ItemEnd->Previous->isTrailingComment()) {
204  // In Cpp11 braced list style, the } and possibly other subsequent
205  // tokens will need to stay on a line with the last element.
206  while (ItemEnd->Next && !ItemEnd->Next->CanBreakBefore)
207  ItemEnd = ItemEnd->Next;
208  } else {
209  // In other braced lists styles, the "}" can be wrapped to the new line.
210  ItemEnd = Token->MatchingParen->Previous;
211  }
212  } else {
213  ItemEnd = Commas[i];
214  // The comma is counted as part of the item when calculating the length.
215  ItemLengths.push_back(CodePointsBetween(ItemBegin, ItemEnd));
216 
217  // Consume trailing comments so the are included in EndOfLineItemLength.
218  if (ItemEnd->Next && !ItemEnd->Next->HasUnescapedNewline &&
219  ItemEnd->Next->isTrailingComment())
220  ItemEnd = ItemEnd->Next;
221  }
222  EndOfLineItemLength.push_back(CodePointsBetween(ItemBegin, ItemEnd));
223  // If there is a trailing comma in the list, the next item will start at the
224  // closing brace. Don't create an extra item for this.
225  if (ItemEnd->getNextNonComment() == Token->MatchingParen)
226  break;
227  ItemBegin = ItemEnd->Next;
228  }
229 
230  // Don't use column layout for lists with few elements and in presence of
231  // separating comments.
232  if (Commas.size() < 5 || HasSeparatingComment)
233  return;
234 
235  if (Token->NestingLevel != 0 && Token->is(tok::l_brace) && Commas.size() < 19)
236  return;
237 
238  // We can never place more than ColumnLimit / 3 items in a row (because of the
239  // spaces and the comma).
240  unsigned MaxItems = Style.ColumnLimit / 3;
241  std::vector<unsigned> MinSizeInColumn;
242  MinSizeInColumn.reserve(MaxItems);
243  for (unsigned Columns = 1; Columns <= MaxItems; ++Columns) {
244  ColumnFormat Format;
245  Format.Columns = Columns;
246  Format.ColumnSizes.resize(Columns);
247  MinSizeInColumn.assign(Columns, UINT_MAX);
248  Format.LineCount = 1;
249  bool HasRowWithSufficientColumns = false;
250  unsigned Column = 0;
251  for (unsigned i = 0, e = ItemLengths.size(); i != e; ++i) {
252  assert(i < MustBreakBeforeItem.size());
253  if (MustBreakBeforeItem[i] || Column == Columns) {
254  ++Format.LineCount;
255  Column = 0;
256  }
257  if (Column == Columns - 1)
258  HasRowWithSufficientColumns = true;
259  unsigned Length =
260  (Column == Columns - 1) ? EndOfLineItemLength[i] : ItemLengths[i];
261  Format.ColumnSizes[Column] = std::max(Format.ColumnSizes[Column], Length);
262  MinSizeInColumn[Column] = std::min(MinSizeInColumn[Column], Length);
263  ++Column;
264  }
265  // If all rows are terminated early (e.g. by trailing comments), we don't
266  // need to look further.
267  if (!HasRowWithSufficientColumns)
268  break;
269  Format.TotalWidth = Columns - 1; // Width of the N-1 spaces.
270 
271  for (unsigned i = 0; i < Columns; ++i)
272  Format.TotalWidth += Format.ColumnSizes[i];
273 
274  // Don't use this Format, if the difference between the longest and shortest
275  // element in a column exceeds a threshold to avoid excessive spaces.
276  if ([&] {
277  for (unsigned i = 0; i < Columns - 1; ++i)
278  if (Format.ColumnSizes[i] - MinSizeInColumn[i] > 10)
279  return true;
280  return false;
281  }())
282  continue;
283 
284  // Ignore layouts that are bound to violate the column limit.
285  if (Format.TotalWidth > Style.ColumnLimit && Columns > 1)
286  continue;
287 
288  Formats.push_back(Format);
289  }
290 }
291 
292 const CommaSeparatedList::ColumnFormat *
293 CommaSeparatedList::getColumnFormat(unsigned RemainingCharacters) const {
294  const ColumnFormat *BestFormat = nullptr;
296  I = Formats.rbegin(),
297  E = Formats.rend();
298  I != E; ++I) {
299  if (I->TotalWidth <= RemainingCharacters || I->Columns == 1) {
300  if (BestFormat && I->LineCount > BestFormat->LineCount)
301  break;
302  BestFormat = &*I;
303  }
304  }
305  return BestFormat;
306 }
307 
308 } // namespace format
309 } // namespace clang
clang::format::FormatToken::getNextNonComment
const FormatToken * getNextNonComment() const
Returns the next token ignoring comments.
Definition: FormatToken.h:648
clang::format::FormatToken::MustBreakBefore
unsigned MustBreakBefore
Whether there must be a line break before this token.
Definition: FormatToken.h:252
max
__DEVICE__ int max(int __a, int __b)
Definition: __clang_cuda_math.h:196
NewLine
bool NewLine
Definition: UnwrappedLineFormatter.cpp:987
llvm::SmallVector
Definition: LLVM.h:38
clang::if
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T -> getSizeExpr()))
Definition: RecursiveASTVisitor.h:1002
clang::format::TokenRole::~TokenRole
virtual ~TokenRole()
Definition: FormatToken.cpp:73
clang::format::FormatToken
A wrapper around a Token storing information about the whitespace characters preceding it.
Definition: FormatToken.h:210
clang::Token
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
End
SourceLocation End
Definition: USRLocFinder.cpp:167
clang::Type
The base class of the type hierarchy.
Definition: Type.h:1490
clang::format::CommaSeparatedList::formatFromToken
unsigned formatFromToken(LineState &State, ContinuationIndenter *Indenter, bool DryRun) override
Apply the special formatting that the given role demands.
Definition: FormatToken.cpp:136
clang::format::FormatToken::CanBreakBefore
unsigned CanBreakBefore
true if it is allowed to break before this token.
Definition: FormatToken.h:258
clang::format::FormatStyle::Cpp11BracedListStyle
bool Cpp11BracedListStyle
If true, format braced lists as best suited for C++11 braced lists.
Definition: Format.h:2038
LIST_TOKEN_TYPES
#define LIST_TOKEN_TYPES
Definition: FormatToken.h:28
clang::format::FormatToken::Previous
FormatToken * Previous
The previous token in the unwrapped line.
Definition: FormatToken.h:430
clang::format::LineState
The current state when indenting a unwrapped line.
Definition: ContinuationIndenter.h:406
min
__DEVICE__ int min(int __a, int __b)
Definition: __clang_cuda_math.h:197
clang::format::FormatStyle::BAS_DontAlign
@ BAS_DontAlign
Don't align, instead use ContinuationIndentWidth, e.g.
Definition: Format.h:82
clang::format::FormatToken::Tok
Token Tok
The Token.
Definition: FormatToken.h:221
clang::Token::getKind
tok::TokenKind getKind() const
Definition: Token.h:92
clang::format::FormatStyle::BinPackArguments
bool BinPackArguments
If false, a function call's arguments will either be all on the same line or will have one line each.
Definition: Format.h:953
clang::Token::is
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:97
clang::format::FormatToken::isTrailingComment
bool isTrailingComment() const
Definition: FormatToken.h:582
clang::format::TokenRole::Style
const FormatStyle & Style
Definition: FormatToken.h:778
clang::format::FormatStyle::ColumnLimit
unsigned ColumnLimit
The column limit.
Definition: Format.h:1852
clang::format::BK_Block
@ BK_Block
Definition: FormatToken.h:136
clang::format::CommaSeparatedList::precomputeFormattingInfos
void precomputeFormattingInfos(const FormatToken *Token) override
After the TokenAnnotator has finished annotating all the tokens, this function precomputes required i...
Definition: FormatToken.cpp:154
clang::format::FormatToken::getPreviousNonComment
FormatToken * getPreviousNonComment() const
Returns the previous token ignoring comments.
Definition: FormatToken.h:640
clang::format::getTokenTypeName
const char * getTokenTypeName(TokenType Type)
Determines the name of a token type.
Definition: FormatToken.cpp:24
TokNames
static const char *const TokNames[]
Definition: TokenKinds.cpp:17
clang::format::NUM_TOKEN_TYPES
@ NUM_TOKEN_TYPES
Definition: FormatToken.h:129
UINT_MAX
#define UINT_MAX
Definition: limits.h:56
clang::format::CommaSeparatedList::formatAfterToken
unsigned formatAfterToken(LineState &State, ContinuationIndenter *Indenter, bool DryRun) override
Same as formatFromToken, but assumes that the first token has already been set thereby deciding on th...
Definition: FormatToken.cpp:77
clang::format::CodePointsBetween
static unsigned CodePointsBetween(const FormatToken *Begin, const FormatToken *End)
Definition: FormatToken.cpp:148
Begin
SourceLocation Begin
Definition: USRLocFinder.cpp:165
clang::format::FormatToken::is
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:452
clang::format::FormatStyle::AlignAfterOpenBracket
BracketAlignmentStyle AlignAfterOpenBracket
If true, horizontally aligns arguments after an open bracket.
Definition: Format.h:97
State
LineState State
Definition: UnwrappedLineFormatter.cpp:986
clang::format::FormatToken::isOneOf
bool isOneOf(A K1, B K2) const
Definition: FormatToken.h:464
clang::format::ContinuationIndenter
Definition: ContinuationIndenter.h:50
clang
Definition: CalledOnceCheck.h:17
clang::format::FormatToken::isSimpleTypeSpecifier
bool isSimpleTypeSpecifier() const
Determine whether the token is a simple-type-specifier.
Definition: FormatToken.cpp:39
clang::format::TokenRole::precomputeFormattingInfos
virtual void precomputeFormattingInfos(const FormatToken *Token)
After the TokenAnnotator has finished annotating all the tokens, this function precomputes required i...
Definition: FormatToken.cpp:75
clang::Token::isOneOf
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:99
FormatToken.h
ContinuationIndenter.h
Indenter
ContinuationIndenter * Indenter
Definition: UnwrappedLineFormatter.cpp:883
clang::format::FormatToken::Next
FormatToken * Next
The next token in the unwrapped line.
Definition: FormatToken.h:433
clang::format::FormatToken::HasUnescapedNewline
unsigned HasUnescapedNewline
Whether there is at least one unescaped newline before the Token.
Definition: FormatToken.h:240
clang::format::TokenType
TokenType
Determines the semantic type of a syntactic token, e.g.
Definition: FormatToken.h:125
clang::format::FormatToken::MatchingParen
FormatToken * MatchingParen
If this is a bracket, this points to the matching one.
Definition: FormatToken.h:427