clang  10.0.0svn
BreakableToken.h
Go to the documentation of this file.
1 //===--- BreakableToken.h - Format C++ code ---------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// Declares BreakableToken, BreakableStringLiteral, BreakableComment,
11 /// BreakableBlockComment and BreakableLineCommentSection classes, that contain
12 /// token type-specific logic to break long lines in tokens and reflow content
13 /// between tokens.
14 ///
15 //===----------------------------------------------------------------------===//
16 
17 #ifndef LLVM_CLANG_LIB_FORMAT_BREAKABLETOKEN_H
18 #define LLVM_CLANG_LIB_FORMAT_BREAKABLETOKEN_H
19 
20 #include "Encoding.h"
21 #include "TokenAnnotator.h"
22 #include "WhitespaceManager.h"
23 #include "llvm/ADT/StringSet.h"
24 #include "llvm/Support/Regex.h"
25 #include <utility>
26 
27 namespace clang {
28 namespace format {
29 
30 /// Checks if \p Token switches formatting, like /* clang-format off */.
31 /// \p Token must be a comment.
32 bool switchesFormatting(const FormatToken &Token);
33 
34 struct FormatStyle;
35 
36 /// Base class for tokens / ranges of tokens that can allow breaking
37 /// within the tokens - for example, to avoid whitespace beyond the column
38 /// limit, or to reflow text.
39 ///
40 /// Generally, a breakable token consists of logical lines, addressed by a line
41 /// index. For example, in a sequence of line comments, each line comment is its
42 /// own logical line; similarly, for a block comment, each line in the block
43 /// comment is on its own logical line.
44 ///
45 /// There are two methods to compute the layout of the token:
46 /// - getRangeLength measures the number of columns needed for a range of text
47 /// within a logical line, and
48 /// - getContentStartColumn returns the start column at which we want the
49 /// content of a logical line to start (potentially after introducing a line
50 /// break).
51 ///
52 /// The mechanism to adapt the layout of the breakable token is organised
53 /// around the concept of a \c Split, which is a whitespace range that signifies
54 /// a position of the content of a token where a reformatting might be done.
55 ///
56 /// Operating with splits is divided into two operations:
57 /// - getSplit, for finding a split starting at a position,
58 /// - insertBreak, for executing the split using a whitespace manager.
59 ///
60 /// There is a pair of operations that are used to compress a long whitespace
61 /// range with a single space if that will bring the line length under the
62 /// column limit:
63 /// - getLineLengthAfterCompression, for calculating the size in columns of the
64 /// line after a whitespace range has been compressed, and
65 /// - compressWhitespace, for executing the whitespace compression using a
66 /// whitespace manager; note that the compressed whitespace may be in the
67 /// middle of the original line and of the reformatted line.
68 ///
69 /// For tokens where the whitespace before each line needs to be also
70 /// reformatted, for example for tokens supporting reflow, there are analogous
71 /// operations that might be executed before the main line breaking occurs:
72 /// - getReflowSplit, for finding a split such that the content preceding it
73 /// needs to be specially reflown,
74 /// - reflow, for executing the split using a whitespace manager,
75 /// - introducesBreakBefore, for checking if reformatting the beginning
76 /// of the content introduces a line break before it,
77 /// - adaptStartOfLine, for executing the reflow using a whitespace
78 /// manager.
79 ///
80 /// For tokens that require the whitespace after the last line to be
81 /// reformatted, for example in multiline jsdoc comments that require the
82 /// trailing '*/' to be on a line of itself, there are analogous operations
83 /// that might be executed after the last line has been reformatted:
84 /// - getSplitAfterLastLine, for finding a split after the last line that needs
85 /// to be reflown,
86 /// - replaceWhitespaceAfterLastLine, for executing the reflow using a
87 /// whitespace manager.
88 ///
89 class BreakableToken {
90 public:
91  /// Contains starting character index and length of split.
92  typedef std::pair<StringRef::size_type, unsigned> Split;
93 
94  virtual ~BreakableToken() {}
95 
96  /// Returns the number of lines in this token in the original code.
97  virtual unsigned getLineCount() const = 0;
98 
99  /// Returns the number of columns required to format the text in the
100  /// byte range [\p Offset, \p Offset \c + \p Length).
101  ///
102  /// \p Offset is the byte offset from the start of the content of the line
103  /// at \p LineIndex.
104  ///
105  /// \p StartColumn is the column at which the text starts in the formatted
106  /// file, needed to compute tab stops correctly.
107  virtual unsigned getRangeLength(unsigned LineIndex, unsigned Offset,
108  StringRef::size_type Length,
109  unsigned StartColumn) const = 0;
110 
111  /// Returns the number of columns required to format the text following
112  /// the byte \p Offset in the line \p LineIndex, including potentially
113  /// unbreakable sequences of tokens following after the end of the token.
114  ///
115  /// \p Offset is the byte offset from the start of the content of the line
116  /// at \p LineIndex.
117  ///
118  /// \p StartColumn is the column at which the text starts in the formatted
119  /// file, needed to compute tab stops correctly.
120  ///
121  /// For breakable tokens that never use extra space at the end of a line, this
122  /// is equivalent to getRangeLength with a Length of StringRef::npos.
123  virtual unsigned getRemainingLength(unsigned LineIndex, unsigned Offset,
124  unsigned StartColumn) const {
125  return getRangeLength(LineIndex, Offset, StringRef::npos, StartColumn);
126  }
127 
128  /// Returns the column at which content in line \p LineIndex starts,
129  /// assuming no reflow.
130  ///
131  /// If \p Break is true, returns the column at which the line should start
132  /// after the line break.
133  /// If \p Break is false, returns the column at which the line itself will
134  /// start.
135  virtual unsigned getContentStartColumn(unsigned LineIndex,
136  bool Break) const = 0;
137 
138  /// Returns additional content indent required for the second line after the
139  /// content at line \p LineIndex is broken.
140  ///
141  // (Next lines do not start with `///` since otherwise -Wdocumentation picks
142  // up the example annotations and generates warnings for them)
143  // For example, Javadoc @param annotations require and indent of 4 spaces and
144  // in this example getContentIndex(1) returns 4.
145  // /**
146  // * @param loooooooooooooong line
147  // * continuation
148  // */
149  virtual unsigned getContentIndent(unsigned LineIndex) const { return 0; }
150 
151  /// Returns a range (offset, length) at which to break the line at
152  /// \p LineIndex, if previously broken at \p TailOffset. If possible, do not
153  /// violate \p ColumnLimit, assuming the text starting at \p TailOffset in
154  /// the token is formatted starting at ContentStartColumn in the reformatted
155  /// file.
156  virtual Split getSplit(unsigned LineIndex, unsigned TailOffset,
157  unsigned ColumnLimit, unsigned ContentStartColumn,
158  llvm::Regex &CommentPragmasRegex) const = 0;
159 
160  /// Emits the previously retrieved \p Split via \p Whitespaces.
161  virtual void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,
162  unsigned ContentIndent,
163  WhitespaceManager &Whitespaces) const = 0;
164 
165  /// Returns the number of columns needed to format
166  /// \p RemainingTokenColumns, assuming that Split is within the range measured
167  /// by \p RemainingTokenColumns, and that the whitespace in Split is reduced
168  /// to a single space.
169  unsigned getLengthAfterCompression(unsigned RemainingTokenColumns,
170  Split Split) const;
171 
172  /// Replaces the whitespace range described by \p Split with a single
173  /// space.
174  virtual void compressWhitespace(unsigned LineIndex, unsigned TailOffset,
175  Split Split,
176  WhitespaceManager &Whitespaces) const = 0;
177 
178  /// Returns whether the token supports reflowing text.
179  virtual bool supportsReflow() const { return false; }
180 
181  /// Returns a whitespace range (offset, length) of the content at \p
182  /// LineIndex such that the content of that line is reflown to the end of the
183  /// previous one.
184  ///
185  /// Returning (StringRef::npos, 0) indicates reflowing is not possible.
186  ///
187  /// The range will include any whitespace preceding the specified line's
188  /// content.
189  ///
190  /// If the split is not contained within one token, for example when reflowing
191  /// line comments, returns (0, <length>).
192  virtual Split getReflowSplit(unsigned LineIndex,
193  llvm::Regex &CommentPragmasRegex) const {
194  return Split(StringRef::npos, 0);
195  }
196 
197  /// Reflows the current line into the end of the previous one.
198  virtual void reflow(unsigned LineIndex,
199  WhitespaceManager &Whitespaces) const {}
200 
201  /// Returns whether there will be a line break at the start of the
202  /// token.
203  virtual bool introducesBreakBeforeToken() const { return false; }
204 
205  /// Replaces the whitespace between \p LineIndex-1 and \p LineIndex.
206  virtual void adaptStartOfLine(unsigned LineIndex,
207  WhitespaceManager &Whitespaces) const {}
208 
209  /// Returns a whitespace range (offset, length) of the content at
210  /// the last line that needs to be reformatted after the last line has been
211  /// reformatted.
212  ///
213  /// A result having offset == StringRef::npos means that no reformat is
214  /// necessary.
215  virtual Split getSplitAfterLastLine(unsigned TailOffset) const {
216  return Split(StringRef::npos, 0);
217  }
218 
219  /// Replaces the whitespace from \p SplitAfterLastLine on the last line
220  /// after the last line has been formatted by performing a reformatting.
221  void replaceWhitespaceAfterLastLine(unsigned TailOffset,
222  Split SplitAfterLastLine,
223  WhitespaceManager &Whitespaces) const {
224  insertBreak(getLineCount() - 1, TailOffset, SplitAfterLastLine,
225  /*ContentIndent=*/0, Whitespaces);
226  }
227 
228  /// Updates the next token of \p State to the next token after this
229  /// one. This can be used when this token manages a set of underlying tokens
230  /// as a unit and is responsible for the formatting of the them.
231  virtual void updateNextToken(LineState &State) const {}
232 
233 protected:
236  : Tok(Tok), InPPDirective(InPPDirective), Encoding(Encoding),
237  Style(Style) {}
238 
239  const FormatToken &Tok;
240  const bool InPPDirective;
243 };
244 
246 public:
247  /// Creates a breakable token for a single line string literal.
248  ///
249  /// \p StartColumn specifies the column in which the token will start
250  /// after formatting.
251  BreakableStringLiteral(const FormatToken &Tok, unsigned StartColumn,
252  StringRef Prefix, StringRef Postfix,
253  unsigned UnbreakableTailLength, bool InPPDirective,
255 
256  Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit,
257  unsigned ContentStartColumn,
258  llvm::Regex &CommentPragmasRegex) const override;
259  void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,
260  unsigned ContentIndent,
261  WhitespaceManager &Whitespaces) const override;
262  void compressWhitespace(unsigned LineIndex, unsigned TailOffset, Split Split,
263  WhitespaceManager &Whitespaces) const override {}
264  unsigned getLineCount() const override;
265  unsigned getRangeLength(unsigned LineIndex, unsigned Offset,
266  StringRef::size_type Length,
267  unsigned StartColumn) const override;
268  unsigned getRemainingLength(unsigned LineIndex, unsigned Offset,
269  unsigned StartColumn) const override;
270  unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override;
271 
272 protected:
273  // The column in which the token starts.
274  unsigned StartColumn;
275  // The prefix a line needs after a break in the token.
276  StringRef Prefix;
277  // The postfix a line needs before introducing a break.
278  StringRef Postfix;
279  // The token text excluding the prefix and postfix.
280  StringRef Line;
281  // Length of the sequence of tokens after this string literal that cannot
282  // contain line breaks.
284 };
285 
287 protected:
288  /// Creates a breakable token for a comment.
289  ///
290  /// \p StartColumn specifies the column in which the comment will start after
291  /// formatting.
292  BreakableComment(const FormatToken &Token, unsigned StartColumn,
294  const FormatStyle &Style);
295 
296 public:
297  bool supportsReflow() const override { return true; }
298  unsigned getLineCount() const override;
299  Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit,
300  unsigned ContentStartColumn,
301  llvm::Regex &CommentPragmasRegex) const override;
302  void compressWhitespace(unsigned LineIndex, unsigned TailOffset, Split Split,
303  WhitespaceManager &Whitespaces) const override;
304 
305 protected:
306  // Returns the token containing the line at LineIndex.
307  const FormatToken &tokenAt(unsigned LineIndex) const;
308 
309  // Checks if the content of line LineIndex may be reflown with the previous
310  // line.
311  virtual bool mayReflow(unsigned LineIndex,
312  llvm::Regex &CommentPragmasRegex) const = 0;
313 
314  // Contains the original text of the lines of the block comment.
315  //
316  // In case of a block comments, excludes the leading /* in the first line and
317  // trailing */ in the last line. In case of line comments, excludes the
318  // leading // and spaces.
320 
321  // Contains the text of the lines excluding all leading and trailing
322  // whitespace between the lines. Note that the decoration (if present) is also
323  // not considered part of the text.
325 
326  // Tokens[i] contains a reference to the token containing Lines[i] if the
327  // whitespace range before that token is managed by this block.
328  // Otherwise, Tokens[i] is a null pointer.
330 
331  // ContentColumn[i] is the target column at which Content[i] should be.
332  // Note that this excludes a leading "* " or "*" in case of block comments
333  // where all lines have a "*" prefix, or the leading "// " or "//" in case of
334  // line comments.
335  //
336  // In block comments, the first line's target column is always positive. The
337  // remaining lines' target columns are relative to the first line to allow
338  // correct indentation of comments in \c WhitespaceManager. Thus they can be
339  // negative as well (in case the first line needs to be unindented more than
340  // there's actual whitespace in another line).
342 
343  // The intended start column of the first line of text from this section.
344  unsigned StartColumn;
345 
346  // The prefix to use in front a line that has been reflown up.
347  // For example, when reflowing the second line after the first here:
348  // // comment 1
349  // // comment 2
350  // we expect:
351  // // comment 1 comment 2
352  // and not:
353  // // comment 1comment 2
354  StringRef ReflowPrefix = " ";
355 };
356 
358 public:
359  BreakableBlockComment(const FormatToken &Token, unsigned StartColumn,
360  unsigned OriginalStartColumn, bool FirstInLine,
362  const FormatStyle &Style, bool UseCRLF);
363 
364  Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit,
365  unsigned ContentStartColumn,
366  llvm::Regex &CommentPragmasRegex) const override;
367  unsigned getRangeLength(unsigned LineIndex, unsigned Offset,
368  StringRef::size_type Length,
369  unsigned StartColumn) const override;
370  unsigned getRemainingLength(unsigned LineIndex, unsigned Offset,
371  unsigned StartColumn) const override;
372  unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override;
373  unsigned getContentIndent(unsigned LineIndex) const override;
374  void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,
375  unsigned ContentIndent,
376  WhitespaceManager &Whitespaces) const override;
377  Split getReflowSplit(unsigned LineIndex,
378  llvm::Regex &CommentPragmasRegex) const override;
379  void reflow(unsigned LineIndex,
380  WhitespaceManager &Whitespaces) const override;
381  bool introducesBreakBeforeToken() const override;
382  void adaptStartOfLine(unsigned LineIndex,
383  WhitespaceManager &Whitespaces) const override;
384  Split getSplitAfterLastLine(unsigned TailOffset) const override;
385 
386  bool mayReflow(unsigned LineIndex,
387  llvm::Regex &CommentPragmasRegex) const override;
388 
389  // Contains Javadoc annotations that require additional indent when continued
390  // on multiple lines.
391  static const llvm::StringSet<> ContentIndentingJavadocAnnotations;
392 
393 private:
394  // Rearranges the whitespace between Lines[LineIndex-1] and Lines[LineIndex].
395  //
396  // Updates Content[LineIndex-1] and Content[LineIndex] by stripping off
397  // leading and trailing whitespace.
398  //
399  // Sets ContentColumn to the intended column in which the text at
400  // Lines[LineIndex] starts (note that the decoration, if present, is not
401  // considered part of the text).
402  void adjustWhitespace(unsigned LineIndex, int IndentDelta);
403 
404  // The column at which the text of a broken line should start.
405  // Note that an optional decoration would go before that column.
406  // IndentAtLineBreak is a uniform position for all lines in a block comment,
407  // regardless of their relative position.
408  // FIXME: Revisit the decision to do this; the main reason was to support
409  // patterns like
410  // /**************//**
411  // * Comment
412  // We could also support such patterns by special casing the first line
413  // instead.
414  unsigned IndentAtLineBreak;
415 
416  // This is to distinguish between the case when the last line was empty and
417  // the case when it started with a decoration ("*" or "* ").
418  bool LastLineNeedsDecoration;
419 
420  // Either "* " if all lines begin with a "*", or empty.
421  StringRef Decoration;
422 
423  // If this block comment has decorations, this is the column of the start of
424  // the decorations.
425  unsigned DecorationColumn;
426 
427  // If true, make sure that the opening '/**' and the closing '*/' ends on a
428  // line of itself. Styles like jsdoc require this for multiline comments.
429  bool DelimitersOnNewline;
430 
431  // Length of the sequence of tokens after this string literal that cannot
432  // contain line breaks.
433  unsigned UnbreakableTailLength;
434 };
435 
437 public:
438  BreakableLineCommentSection(const FormatToken &Token, unsigned StartColumn,
439  unsigned OriginalStartColumn, bool FirstInLine,
441  const FormatStyle &Style);
442 
443  unsigned getRangeLength(unsigned LineIndex, unsigned Offset,
444  StringRef::size_type Length,
445  unsigned StartColumn) const override;
446  unsigned getContentStartColumn(unsigned LineIndex, bool Break) const override;
447  void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,
448  unsigned ContentIndent,
449  WhitespaceManager &Whitespaces) const override;
450  Split getReflowSplit(unsigned LineIndex,
451  llvm::Regex &CommentPragmasRegex) const override;
452  void reflow(unsigned LineIndex,
453  WhitespaceManager &Whitespaces) const override;
454  void adaptStartOfLine(unsigned LineIndex,
455  WhitespaceManager &Whitespaces) const override;
456  void updateNextToken(LineState &State) const override;
457  bool mayReflow(unsigned LineIndex,
458  llvm::Regex &CommentPragmasRegex) const override;
459 
460 private:
461  // OriginalPrefix[i] contains the original prefix of line i, including
462  // trailing whitespace before the start of the content. The indentation
463  // preceding the prefix is not included.
464  // For example, if the line is:
465  // // content
466  // then the original prefix is "// ".
467  SmallVector<StringRef, 16> OriginalPrefix;
468 
469  // Prefix[i] contains the intended leading "//" with trailing spaces to
470  // account for the indentation of content within the comment at line i after
471  // formatting. It can be different than the original prefix when the original
472  // line starts like this:
473  // //content
474  // Then the original prefix is "//", but the prefix is "// ".
476 
477  SmallVector<unsigned, 16> OriginalContentColumn;
478 
479  /// The token to which the last line of this breakable token belongs
480  /// to; nullptr if that token is the initial token.
481  ///
482  /// The distinction is because if the token of the last line of this breakable
483  /// token is distinct from the initial token, this breakable token owns the
484  /// whitespace before the token of the last line, and the whitespace manager
485  /// must be able to modify it.
486  FormatToken *LastLineTok = nullptr;
487 };
488 } // namespace format
489 } // namespace clang
490 
491 #endif
virtual unsigned getRemainingLength(unsigned LineIndex, unsigned Offset, unsigned StartColumn) const
Returns the number of columns required to format the text following the byte Offset in the line LineI...
bool switchesFormatting(const FormatToken &Token)
Checks if Token switches formatting, like /* clang-format off.
virtual Split getSplitAfterLastLine(unsigned TailOffset) const
Returns a whitespace range (offset, length) of the content at the last line that needs to be reformat...
virtual unsigned getContentStartColumn(unsigned LineIndex, bool Break) const =0
Returns the column at which content in line LineIndex starts, assuming no reflow. ...
virtual ~BreakableToken()
static const llvm::StringSet ContentIndentingJavadocAnnotations
to be on a line of there are analogous operations *that might be executed after the last line has been for finding a split after the last line that needs *to be * replaceWhitespaceAfterLastLine
SmallVector< int, 16 > ContentColumn
LineState State
Contains functions for text encoding manipulation.
This file implements a token annotator, i.e.
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
virtual void updateNextToken(LineState &State) const
Updates the next token of State to the next token after this one.
Manages the whitespaces around tokens and their replacements.
const FormatToken & Tok
bool supportsReflow() const override
virtual void adaptStartOfLine(unsigned LineIndex, WhitespaceManager &Whitespaces) const
Replaces the whitespace between LineIndex-1 and LineIndex.
The current state when indenting a unwrapped line.
WhitespaceManager class manages whitespace around tokens and their replacements.
unsigned Offset
Definition: Format.cpp:1809
SmallVector< StringRef, 16 > Content
virtual unsigned getContentIndent(unsigned LineIndex) const
Returns additional content indent required for the second line after the content at line LineIndex is...
A wrapper around a Token storing information about the whitespace characters preceding it...
Definition: FormatToken.h:129
virtual unsigned getRangeLength(unsigned LineIndex, unsigned Offset, StringRef::size_type Length, unsigned StartColumn) const =0
Returns the number of columns required to format the text in the byte range [Offset, Offset + Length).
SmallVector< FormatToken *, 16 > Tokens
const bool InPPDirective
void compressWhitespace(unsigned LineIndex, unsigned TailOffset, Split Split, WhitespaceManager &Whitespaces) const override
virtual void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split, unsigned ContentIndent, WhitespaceManager &Whitespaces) const =0
Emits the previously retrieved Split via Whitespaces.
virtual bool supportsReflow() const
Returns whether the token supports reflowing text.
The FormatStyle is used to configure the formatting to follow specific guidelines.
Definition: Format.h:49
virtual void compressWhitespace(unsigned LineIndex, unsigned TailOffset, Split Split, WhitespaceManager &Whitespaces) const =0
Replaces the whitespace range described by Split with a single space.
Dataflow Directional Tag Classes.
unsigned getLengthAfterCompression(unsigned RemainingTokenColumns, Split Split) const
Returns the number of columns needed to format RemainingTokenColumns, assuming that Split is within t...
virtual unsigned getLineCount() const =0
Returns the number of lines in this token in the original code.
virtual bool introducesBreakBeforeToken() const
Returns whether there will be a line break at the start of the token.
virtual void reflow(unsigned LineIndex, WhitespaceManager &Whitespaces) const
Reflows the current line into the end of the previous one.
BreakableToken(const FormatToken &Tok, bool InPPDirective, encoding::Encoding Encoding, const FormatStyle &Style)
SmallVector< StringRef, 16 > Lines
virtual Split getReflowSplit(unsigned LineIndex, llvm::Regex &CommentPragmasRegex) const
Returns a whitespace range (offset, length) of the content at LineIndex such that the content of that...
const encoding::Encoding Encoding
const FormatStyle & Style
virtual Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit, unsigned ContentStartColumn, llvm::Regex &CommentPragmasRegex) const =0
Returns a range (offset, length) at which to break the line at LineIndex, if previously broken at Tai...