19#include "llvm/ADT/STLExtras.h"
20#include "llvm/Support/Debug.h"
23#define DEBUG_TYPE "format-token-breaker"
28static constexpr StringRef
Blanks =
" \t\v\f\r";
44 static constexpr StringRef KnownCStylePrefixes[] = {
"///<",
"//!<",
"///",
46 static constexpr StringRef KnownTextProtoPrefixes[] = {
"####",
"###",
"##",
50 KnownPrefixes = KnownTextProtoPrefixes;
53 llvm::is_sorted(KnownPrefixes, [](StringRef Lhs, StringRef Rhs)
noexcept {
54 return Lhs.size() > Rhs.size();
57 for (StringRef KnownPrefix : KnownPrefixes) {
58 if (Comment.startswith(KnownPrefix)) {
59 const auto PrefixLength =
60 Comment.find_first_not_of(
' ', KnownPrefix.size());
61 return Comment.substr(0, PrefixLength);
69 unsigned ColumnLimit,
unsigned TabWidth,
71 bool DecorationEndsWithStar =
false) {
72 LLVM_DEBUG(llvm::dbgs() <<
"Comment split: \"" <<
Text
73 <<
"\", Column limit: " << ColumnLimit
74 <<
", Content start: " << ContentStartColumn <<
"\n");
75 if (ColumnLimit <= ContentStartColumn + 1)
78 unsigned MaxSplit = ColumnLimit - ContentStartColumn + 1;
79 unsigned MaxSplitBytes = 0;
81 for (
unsigned NumChars = 0;
82 NumChars < MaxSplit && MaxSplitBytes <
Text.size();) {
83 unsigned BytesInChar =
87 ContentStartColumn, TabWidth, Encoding);
88 MaxSplitBytes += BytesInChar;
95 StringRef::size_type SpaceOffset =
97 if (SpaceOffset != StringRef::npos && SpaceOffset + 1 <
Text.size() &&
98 Text[SpaceOffset + 1] ==
'{') {
99 MaxSplitBytes = SpaceOffset + 1;
103 StringRef::size_type SpaceOffset =
Text.find_last_of(
Blanks, MaxSplitBytes);
105 static const auto kNumberedListRegexp = llvm::Regex(
"^[1-9][0-9]?\\.");
107 while (SpaceOffset != StringRef::npos) {
114 StringRef::size_type LastNonBlank =
116 if (LastNonBlank != StringRef::npos &&
Text[LastNonBlank] ==
'\\') {
117 SpaceOffset =
Text.find_last_of(
Blanks, LastNonBlank);
124 if (kNumberedListRegexp.match(
Text.substr(SpaceOffset).ltrim(
Blanks))) {
125 SpaceOffset =
Text.find_last_of(
Blanks, SpaceOffset);
131 (
Text[SpaceOffset + 1] ==
'{' ||
Text[SpaceOffset + 1] ==
'@')) {
132 SpaceOffset =
Text.find_last_of(
Blanks, SpaceOffset);
139 if (SpaceOffset == StringRef::npos ||
141 Text.find_last_not_of(
Blanks, SpaceOffset) == StringRef::npos) {
144 StringRef::size_type FirstNonWhitespace =
Text.find_first_not_of(
Blanks);
145 if (FirstNonWhitespace == StringRef::npos) {
149 SpaceOffset =
Text.find_first_of(
150 Blanks, std::max<unsigned>(MaxSplitBytes, FirstNonWhitespace));
152 if (SpaceOffset != StringRef::npos && SpaceOffset != 0) {
157 if (SpaceOffset == 1 &&
Text[SpaceOffset - 1] ==
'*')
159 StringRef BeforeCut =
Text.substr(0, SpaceOffset).rtrim(
Blanks);
160 StringRef AfterCut =
Text.substr(SpaceOffset);
162 if (!DecorationEndsWithStar || AfterCut.size() <= 1 || AfterCut[1] !=
'/')
163 AfterCut = AfterCut.ltrim(
Blanks);
165 AfterCut.begin() - BeforeCut.end());
176 if (ColumnLimit <= UsedColumns)
178 unsigned MaxSplit = ColumnLimit - UsedColumns;
179 StringRef::size_type SpaceOffset = 0;
180 StringRef::size_type SlashOffset = 0;
181 StringRef::size_type WordStartOffset = 0;
182 StringRef::size_type SplitPoint = 0;
183 for (
unsigned Chars = 0;;) {
185 if (
Text[0] ==
'\\') {
191 Text.substr(0, Advance), UsedColumns + Chars, TabWidth, Encoding);
194 if (Chars > MaxSplit ||
Text.size() <= Advance)
198 SpaceOffset = SplitPoint;
200 SlashOffset = SplitPoint;
202 WordStartOffset = SplitPoint;
204 SplitPoint += Advance;
208 if (SpaceOffset != 0)
210 if (SlashOffset != 0)
212 if (WordStartOffset != 0)
221 "formatting regions are switched by comment tokens");
222 StringRef Content =
Token.TokenText.substr(2).ltrim();
223 return Content.startswith(
"clang-format on") ||
224 Content.startswith(
"clang-format off");
241 return RemainingTokenColumns + 1 -
Split.second;
248 StringRef::size_type Length,
249 unsigned StartColumn)
const {
250 llvm_unreachable(
"Getting the length of a part of the string literal "
251 "indicates that the code tries to reflow it.");
256 unsigned StartColumn)
const {
268 const FormatToken &Tok,
unsigned StartColumn, StringRef Prefix,
269 StringRef Postfix,
unsigned UnbreakableTailLength,
bool InPPDirective,
272 StartColumn(StartColumn), Prefix(Prefix), Postfix(Postfix),
273 UnbreakableTailLength(UnbreakableTailLength) {
280 unsigned LineIndex,
unsigned TailOffset,
unsigned ColumnLimit,
281 unsigned ContentStartColumn,
const llvm::Regex &CommentPragmasRegex)
const {
288 unsigned ContentIndent,
290 Whitespaces.replaceWhitespaceInToken(
296 unsigned StartColumn,
bool InPPDirective,
300 StartColumn(StartColumn) {}
306 unsigned ColumnLimit,
unsigned ContentStartColumn,
307 const llvm::Regex &CommentPragmasRegex)
const {
309 if (CommentPragmasRegex.match(
Content[LineIndex]))
310 return Split(StringRef::npos, 0);
317 unsigned LineIndex,
unsigned TailOffset,
Split Split,
319 StringRef
Text =
Content[LineIndex].substr(TailOffset);
324 unsigned BreakOffsetInToken =
326 unsigned CharsToRemove =
Split.second;
327 Whitespaces.replaceWhitespaceInToken(
328 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"",
"",
337 Content = Content.trim(
Blanks);
340 bool hasSpecialMeaningPrefix =
false;
341 for (StringRef Prefix :
342 {
"@",
"TODO",
"FIXME",
"XXX",
"-# ",
"- ",
"+ ",
"* "}) {
343 if (Content.startswith(Prefix)) {
344 hasSpecialMeaningPrefix =
true;
352 static const auto kNumberedListRegexp = llvm::Regex(
"^[1-9][0-9]?\\. ");
353 hasSpecialMeaningPrefix =
354 hasSpecialMeaningPrefix || kNumberedListRegexp.match(Content);
359 return Content.size() >= 2 && !hasSpecialMeaningPrefix &&
360 !Content.endswith(
"\\") &&
368 unsigned OriginalStartColumn,
bool FirstInLine,
bool InPPDirective,
371 DelimitersOnNewline(
false),
372 UnbreakableTailLength(
Token.UnbreakableTailLength) {
373 assert(
Tok.
is(TT_BlockComment) &&
374 "block comment section must start with a block comment");
377 assert(TokenText.startswith(
"/*") && TokenText.endswith(
"*/"));
378 TokenText.substr(2, TokenText.size() - 4)
379 .split(
Lines, UseCRLF ?
"\r\n" :
"\n");
381 int IndentDelta =
StartColumn - OriginalStartColumn;
388 for (
size_t i = 1; i <
Lines.size(); ++i)
389 adjustWhitespace(i, IndentDelta);
406 if (
Lines.size() == 1 && !FirstInLine) {
414 for (
size_t i = 1, e =
Content.size(); i < e && !Decoration.empty(); ++i) {
420 }
else if (!
Text.empty() && Decoration.startswith(
Text)) {
423 while (!
Text.startswith(Decoration))
424 Decoration = Decoration.drop_back(1);
427 LastLineNeedsDecoration =
true;
429 for (
size_t i = 1, e =
Lines.size(); i < e; ++i) {
435 LastLineNeedsDecoration =
false;
437 if (e >= 2 && !Decoration.empty())
439 }
else if (Decoration.empty()) {
451 unsigned DecorationSize = Decoration.startswith(
Content[i])
457 if (!Decoration.startswith(
Content[i])) {
459 std::min<int>(IndentAtLineBreak, std::max(0,
ContentColumn[i]));
462 IndentAtLineBreak = std::max<unsigned>(IndentAtLineBreak, Decoration.size());
466 if ((
Lines[0] ==
"*" ||
Lines[0].startswith(
"* ")) &&
Lines.size() > 1) {
468 DelimitersOnNewline =
true;
469 }
else if (
Lines[0].startswith(
"* ") &&
Lines.size() == 1) {
483 llvm::dbgs() <<
"IndentAtLineBreak " << IndentAtLineBreak <<
"\n";
484 llvm::dbgs() <<
"DelimitersOnNewline " << DelimitersOnNewline <<
"\n";
485 for (
size_t i = 0; i <
Lines.size(); ++i) {
486 llvm::dbgs() << i <<
" |" <<
Content[i] <<
"| "
488 <<
"IN=" << (
Content[i].data() -
Lines[i].data()) <<
"\n";
494 unsigned LineIndex,
unsigned TailOffset,
unsigned ColumnLimit,
495 unsigned ContentStartColumn,
const llvm::Regex &CommentPragmasRegex)
const {
497 if (CommentPragmasRegex.match(
Content[LineIndex]))
498 return Split(StringRef::npos, 0);
504void BreakableBlockComment::adjustWhitespace(
unsigned LineIndex,
511 size_t EndOfPreviousLine =
Lines[LineIndex - 1].size();
517 Lines[LineIndex - 1].find_last_not_of(
Blanks, EndOfPreviousLine);
518 if (EndOfPreviousLine == StringRef::npos)
519 EndOfPreviousLine = 0;
523 size_t StartOfLine =
Lines[LineIndex].find_first_not_of(
Blanks);
524 if (StartOfLine == StringRef::npos)
525 StartOfLine =
Lines[LineIndex].size();
527 StringRef Whitespace =
Lines[LineIndex].substr(0, StartOfLine);
529 size_t PreviousContentOffset =
530 Content[LineIndex - 1].data() -
Lines[LineIndex - 1].data();
532 PreviousContentOffset, EndOfPreviousLine - PreviousContentOffset);
533 Content[LineIndex] =
Lines[LineIndex].substr(StartOfLine);
543 StringRef::size_type Length,
544 unsigned StartColumn)
const {
552 unsigned StartColumn)
const {
553 unsigned LineLength =
554 UnbreakableTailLength +
556 if (LineIndex + 1 ==
Lines.size()) {
560 if (!HasRemainingText) {
561 bool HasDecoration =
Lines[LineIndex].ltrim().startswith(Decoration);
563 LineLength -= Decoration.size();
572 return IndentAtLineBreak;
576const llvm::StringSet<>
578 "@param",
"@return",
"@returns",
"@throws",
"@type",
"@template",
579 "@see",
"@deprecated",
"@define",
"@exports",
"@mods",
"@private",
588 StringRef ContentWithNoDecoration =
Content[LineIndex];
589 if (LineIndex == 0 && ContentWithNoDecoration.startswith(
"*"))
590 ContentWithNoDecoration = ContentWithNoDecoration.substr(1).ltrim(
Blanks);
591 StringRef FirstWord = ContentWithNoDecoration.substr(
592 0, ContentWithNoDecoration.find_first_of(
Blanks));
601 StringRef
Text =
Content[LineIndex].substr(TailOffset);
602 StringRef Prefix = Decoration;
606 unsigned LocalIndentAtLineBreak = IndentAtLineBreak;
607 if (LineIndex + 1 ==
Lines.size() &&
611 if (LocalIndentAtLineBreak >= 2)
612 LocalIndentAtLineBreak -= 2;
616 unsigned BreakOffsetInToken =
618 unsigned CharsToRemove =
Split.second;
619 assert(LocalIndentAtLineBreak >= Prefix.size());
620 std::string PrefixWithTrailingIndent = std::string(Prefix);
621 PrefixWithTrailingIndent.append(ContentIndent,
' ');
622 Whitespaces.replaceWhitespaceInToken(
623 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"",
625 LocalIndentAtLineBreak + ContentIndent -
626 PrefixWithTrailingIndent.size());
630 unsigned LineIndex,
const llvm::Regex &CommentPragmasRegex)
const {
631 if (!
mayReflow(LineIndex, CommentPragmasRegex))
632 return Split(StringRef::npos, 0);
636 size_t Trimmed =
Content[LineIndex].find_first_not_of(
Blanks);
639 if (PreviousContentIndent && Trimmed != StringRef::npos &&
640 Trimmed != PreviousContentIndent) {
641 return Split(StringRef::npos, 0);
645 return Split(0, Trimmed != StringRef::npos ? Trimmed : 0);
650 return DelimitersOnNewline &&
651 Lines[0].substr(1).find_first_not_of(
Blanks) != StringRef::npos;
659 "Reflowing whitespace within a token");
662 unsigned WhitespaceOffsetInToken =
Content[LineIndex - 1].data() +
663 Content[LineIndex - 1].size() -
665 unsigned WhitespaceLength = TrimmedContent.data() -
667 WhitespaceOffsetInToken;
668 Whitespaces.replaceWhitespaceInToken(
669 tokenAt(LineIndex), WhitespaceOffsetInToken,
670 WhitespaceLength,
"",
677 if (LineIndex == 0) {
678 if (DelimitersOnNewline) {
683 size_t BreakLength =
Lines[0].substr(1).find_first_not_of(
Blanks);
684 if (BreakLength != StringRef::npos) {
693 StringRef Prefix = Decoration;
694 if (
Content[LineIndex].empty()) {
695 if (LineIndex + 1 ==
Lines.size()) {
696 if (!LastLineNeedsDecoration) {
701 }
else if (!Decoration.empty()) {
704 Prefix = Prefix.substr(0, 1);
708 Prefix = Prefix.substr(0, 1);
712 unsigned WhitespaceOffsetInToken =
Content[LineIndex - 1].data() +
713 Content[LineIndex - 1].size() -
715 unsigned WhitespaceLength =
Content[LineIndex].data() -
717 WhitespaceOffsetInToken;
718 Whitespaces.replaceWhitespaceInToken(
719 tokenAt(LineIndex), WhitespaceOffsetInToken, WhitespaceLength,
"", Prefix,
725 if (DelimitersOnNewline) {
729 StringRef Line =
Content.back().substr(TailOffset);
730 StringRef TrimmedLine = Line.rtrim(
Blanks);
731 if (!TrimmedLine.empty())
732 return Split(TrimmedLine.size(), Line.size() - TrimmedLine.size());
734 return Split(StringRef::npos, 0);
738 unsigned LineIndex,
const llvm::Regex &CommentPragmasRegex)
const {
741 StringRef IndentContent =
Content[LineIndex];
742 if (
Lines[LineIndex].ltrim(
Blanks).startswith(
"*"))
743 IndentContent =
Lines[LineIndex].ltrim(
Blanks).substr(1);
744 return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
753 assert(
Tok.
is(TT_LineComment) &&
754 "line comment section must start with a line comment");
759 int FirstLineSpaceChange = 0;
761 CurrentTok && CurrentTok->
is(TT_LineComment);
762 CurrentTok = CurrentTok->Next) {
763 LastLineTok = LineTok;
764 StringRef TokenText(CurrentTok->TokenText);
765 assert((TokenText.startswith(
"//") || TokenText.startswith(
"#")) &&
766 "unsupported line comment prefix, '//' and '#' are supported");
767 size_t FirstLineIndex =
Lines.size();
768 TokenText.split(
Lines,
"\n");
771 PrefixSpaceChange.resize(
Lines.size());
773 Prefix.resize(
Lines.size());
774 OriginalPrefix.resize(
Lines.size());
775 for (
size_t i = FirstLineIndex, e =
Lines.size(); i < e; ++i) {
778 OriginalPrefix[i] = IndentPrefix;
779 const int SpacesInPrefix = llvm::count(IndentPrefix,
' ');
783 const auto NoSpaceBeforeFirstCommentChar = [&]() {
784 assert(
Lines[i].size() > IndentPrefix.size());
785 const char FirstCommentChar =
Lines[i][IndentPrefix.size()];
786 const unsigned FirstCharByteSize =
789 Lines[i].substr(IndentPrefix.size(), FirstCharByteSize),
802 if (FirstCommentChar ==
'#' && !TokenText.startswith(
"#"))
804 return FirstCommentChar ==
'\\' ||
isPunctuation(FirstCommentChar) ||
813 if (i == 0 || OriginalPrefix[i].rtrim(
Blanks) !=
814 OriginalPrefix[i - 1].rtrim(
Blanks)) {
815 if (SpacesInPrefix < Minimum &&
Lines[i].size() > IndentPrefix.size() &&
816 !NoSpaceBeforeFirstCommentChar()) {
817 FirstLineSpaceChange = Minimum - SpacesInPrefix;
818 }
else if (
static_cast<unsigned>(SpacesInPrefix) >
820 FirstLineSpaceChange =
823 FirstLineSpaceChange = 0;
827 if (
Lines[i].size() != IndentPrefix.size()) {
828 PrefixSpaceChange[i] = FirstLineSpaceChange;
830 if (SpacesInPrefix + PrefixSpaceChange[i] < Minimum) {
831 PrefixSpaceChange[i] +=
832 Minimum - (SpacesInPrefix + PrefixSpaceChange[i]);
835 assert(
Lines[i].size() > IndentPrefix.size());
836 const auto FirstNonSpace =
Lines[i][IndentPrefix.size()];
838 const bool LineRequiresLeadingSpace =
839 !NoSpaceBeforeFirstCommentChar() ||
840 (FirstNonSpace ==
'}' && FirstLineSpaceChange != 0);
841 const bool AllowsSpaceChange =
843 (SpacesInPrefix != 0 || LineRequiresLeadingSpace);
845 if (PrefixSpaceChange[i] > 0 && AllowsSpaceChange) {
846 Prefix[i] = IndentPrefix.str();
847 Prefix[i].append(PrefixSpaceChange[i],
' ');
848 }
else if (PrefixSpaceChange[i] < 0 && AllowsSpaceChange) {
849 Prefix[i] = IndentPrefix
850 .drop_back(std::min<std::size_t>(
851 -PrefixSpaceChange[i], SpacesInPrefix))
854 Prefix[i] = IndentPrefix.str();
859 Prefix[i] = IndentPrefix.drop_back(SpacesInPrefix).str();
870 if (EndOfLine == StringRef::npos)
876 LineTok = CurrentTok->
Next;
877 if (CurrentTok->Next && !CurrentTok->Next->ContinuesLineCommentSection) {
899 StringRef::size_type Length,
900 unsigned StartColumn)
const {
913 unsigned LineIndex,
unsigned TailOffset,
Split Split,
915 StringRef
Text =
Content[LineIndex].substr(TailOffset);
918 unsigned BreakOffsetInToken =
920 unsigned CharsToRemove =
Split.second;
921 Whitespaces.replaceWhitespaceInToken(
922 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"",
928 unsigned LineIndex,
const llvm::Regex &CommentPragmasRegex)
const {
929 if (!
mayReflow(LineIndex, CommentPragmasRegex))
930 return Split(StringRef::npos, 0);
932 size_t Trimmed =
Content[LineIndex].find_first_not_of(
Blanks);
938 return Split(0, Trimmed != StringRef::npos ? Trimmed : 0);
943 if (LineIndex > 0 &&
Tokens[LineIndex] !=
Tokens[LineIndex - 1]) {
946 Whitespaces.replaceWhitespace(
950 }
else if (LineIndex > 0) {
962 Lines[LineIndex - 1].size() -
966 unsigned WhitespaceLength =
968 Whitespaces.replaceWhitespaceInToken(*
Tokens[LineIndex],
Offset,
979 unsigned WhitespaceLength =
981 Whitespaces.replaceWhitespaceInToken(*
Tokens[LineIndex],
Offset,
998 if (LineIndex > 0 &&
Tokens[LineIndex] !=
Tokens[LineIndex - 1]) {
1004 unsigned LineColumn =
1006 (
Content[LineIndex].data() -
Lines[LineIndex].data()) +
1007 (OriginalPrefix[LineIndex].size() - Prefix[LineIndex].size());
1013 Whitespaces.replaceWhitespace(*
Tokens[LineIndex],
1020 if (OriginalPrefix[LineIndex] != Prefix[LineIndex]) {
1022 const auto SpacesToRemove = -std::min(PrefixSpaceChange[LineIndex], 0);
1023 const auto SpacesToAdd = std::max(PrefixSpaceChange[LineIndex], 0);
1024 Whitespaces.replaceWhitespaceInToken(
1025 tokenAt(LineIndex), OriginalPrefix[LineIndex].size() - SpacesToRemove,
1026 SpacesToRemove,
"",
"",
false,
1033 State.NextToken = LastLineTok->
Next;
1037 unsigned LineIndex,
const llvm::Regex &CommentPragmasRegex)
const {
1040 StringRef IndentContent =
Content[LineIndex];
1041 if (
Lines[LineIndex].startswith(
"//"))
1042 IndentContent =
Lines[LineIndex].substr(2);
1049 return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
1052 OriginalPrefix[LineIndex] == OriginalPrefix[LineIndex - 1];
Declares BreakableToken, BreakableStringLiteral, BreakableComment, BreakableBlockComment and Breakabl...
This file implements an indenter that manages the indentation of continuations.
Token - This structure provides full information about a lexed token.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
LLVM_READONLY bool isAlphanumeric(unsigned char c)
Return true if this character is an ASCII letter or digit: [a-zA-Z0-9].
LLVM_READONLY bool isHorizontalWhitespace(unsigned char c)
Returns true if this character is horizontal ASCII whitespace: ' ', '\t', '\f', '\v'.
@ C
Languages that the frontend can parse and compile.
LLVM_READONLY bool isPunctuation(unsigned char c)
Return true if this character is an ASCII punctuation character.