19#define DEBUG_TYPE "format-token-breaker"
24static constexpr StringRef
Blanks =
" \t\v\f\r";
40 static constexpr StringRef KnownCStylePrefixes[] = {
"///<",
"//!<",
"///",
42 static constexpr StringRef KnownTextProtoPrefixes[] = {
"####",
"###",
"##",
46 KnownPrefixes = KnownTextProtoPrefixes;
49 llvm::is_sorted(KnownPrefixes, [](StringRef Lhs, StringRef Rhs)
noexcept {
50 return Lhs.size() > Rhs.size();
53 for (StringRef KnownPrefix : KnownPrefixes) {
54 if (Comment.starts_with(KnownPrefix)) {
55 const auto PrefixLength =
56 Comment.find_first_not_of(
' ', KnownPrefix.size());
57 return Comment.substr(0, PrefixLength);
65 unsigned ColumnLimit,
unsigned TabWidth,
67 bool DecorationEndsWithStar =
false) {
68 LLVM_DEBUG(llvm::dbgs() <<
"Comment split: \"" <<
Text
69 <<
"\", Column limit: " << ColumnLimit
70 <<
", Content start: " << ContentStartColumn <<
"\n");
71 if (ColumnLimit <= ContentStartColumn + 1)
74 unsigned MaxSplit = ColumnLimit - ContentStartColumn + 1;
75 unsigned MaxSplitBytes = 0;
77 for (
unsigned NumChars = 0;
78 NumChars < MaxSplit && MaxSplitBytes <
Text.size();) {
79 unsigned BytesInChar =
82 Text.substr(MaxSplitBytes, BytesInChar), ContentStartColumn + NumChars,
84 MaxSplitBytes += BytesInChar;
91 StringRef::size_type SpaceOffset =
93 if (SpaceOffset != StringRef::npos && SpaceOffset + 1 <
Text.size() &&
94 Text[SpaceOffset + 1] ==
'{') {
95 MaxSplitBytes = SpaceOffset + 1;
99 StringRef::size_type SpaceOffset =
Text.find_last_of(
Blanks, MaxSplitBytes);
101 static const auto kNumberedListRegexp = llvm::Regex(
"^[1-9][0-9]?\\.");
103 while (SpaceOffset != StringRef::npos) {
110 StringRef::size_type LastNonBlank =
112 if (LastNonBlank != StringRef::npos &&
Text[LastNonBlank] ==
'\\') {
113 SpaceOffset =
Text.find_last_of(
Blanks, LastNonBlank);
120 if (kNumberedListRegexp.match(
Text.substr(SpaceOffset).ltrim(
Blanks))) {
121 SpaceOffset =
Text.find_last_of(
Blanks, SpaceOffset);
127 (
Text[SpaceOffset + 1] ==
'{' ||
Text[SpaceOffset + 1] ==
'@')) {
128 SpaceOffset =
Text.find_last_of(
Blanks, SpaceOffset);
135 if (SpaceOffset == StringRef::npos ||
137 Text.find_last_not_of(
Blanks, SpaceOffset) == StringRef::npos) {
140 StringRef::size_type FirstNonWhitespace =
Text.find_first_not_of(
Blanks);
141 if (FirstNonWhitespace == StringRef::npos) {
145 SpaceOffset =
Text.find_first_of(
146 Blanks, std::max<unsigned>(MaxSplitBytes, FirstNonWhitespace));
148 if (SpaceOffset != StringRef::npos && SpaceOffset != 0) {
153 if (SpaceOffset == 1 &&
Text[SpaceOffset - 1] ==
'*')
155 StringRef BeforeCut =
Text.substr(0, SpaceOffset).rtrim(
Blanks);
156 StringRef AfterCut =
Text.substr(SpaceOffset);
158 if (!DecorationEndsWithStar || AfterCut.size() <= 1 || AfterCut[1] !=
'/')
159 AfterCut = AfterCut.ltrim(
Blanks);
161 AfterCut.begin() - BeforeCut.end());
172 if (ColumnLimit <= UsedColumns)
174 unsigned MaxSplit = ColumnLimit - UsedColumns;
175 StringRef::size_type SpaceOffset = 0;
176 StringRef::size_type SlashOffset = 0;
177 StringRef::size_type WordStartOffset = 0;
178 StringRef::size_type SplitPoint = 0;
179 for (
unsigned Chars = 0;;) {
181 if (
Text[0] ==
'\\') {
187 Text.substr(0, Advance), UsedColumns + Chars, TabWidth, Encoding);
190 if (Chars > MaxSplit ||
Text.size() <= Advance)
194 SpaceOffset = SplitPoint;
196 SlashOffset = SplitPoint;
198 WordStartOffset = SplitPoint;
200 SplitPoint += Advance;
204 if (SpaceOffset != 0)
206 if (SlashOffset != 0)
208 if (WordStartOffset != 0)
217 "formatting regions are switched by comment tokens");
218 StringRef Content =
Token.TokenText.substr(2).ltrim();
219 return Content.starts_with(
"clang-format on") ||
220 Content.starts_with(
"clang-format off");
237 return RemainingTokenColumns + 1 -
Split.second;
244 StringRef::size_type Length,
245 unsigned StartColumn)
const {
246 llvm_unreachable(
"Getting the length of a part of the string literal "
247 "indicates that the code tries to reflow it.");
252 unsigned StartColumn)
const {
264 const FormatToken &Tok,
unsigned StartColumn, StringRef Prefix,
265 StringRef Postfix,
unsigned UnbreakableTailLength,
bool InPPDirective,
268 StartColumn(StartColumn), Prefix(Prefix), Postfix(Postfix),
269 UnbreakableTailLength(UnbreakableTailLength) {
276 unsigned LineIndex,
unsigned TailOffset,
unsigned ColumnLimit,
277 unsigned ContentStartColumn,
const llvm::Regex &CommentPragmasRegex)
const {
284 unsigned ContentIndent,
286 Whitespaces.replaceWhitespaceInToken(
293 unsigned StartColumn,
unsigned UnbreakableTailLength,
bool InPPDirective,
296 Tok, StartColumn, QuoteStyle == SingleQuotes ?
"'"
297 : QuoteStyle == AtDoubleQuotes ?
"@\""
299 QuoteStyle == SingleQuotes ?
"'" :
"\"",
300 UnbreakableTailLength, InPPDirective, Encoding, Style),
301 BracesNeeded(Tok.isNot(TT_StringInConcatenation)),
302 QuoteStyle(QuoteStyle) {
328 Postfix = SignOnNewLine ?
"'" :
"' +";
329 Prefix = SignOnNewLine ?
"+ '" :
"'";
333 Prefix = SignOnNewLine ?
"+ @\"" :
"@\"";
336 Prefix = SignOnNewLine ?
"+ \"" :
"\"";
339 Postfix = SignOnNewLine ?
"\"" :
"\" +";
354 unsigned LineIndex,
unsigned Offset,
unsigned StartColumn)
const {
373 unsigned LineIndex,
unsigned TailOffset,
Split Split,
375 Whitespaces.replaceWhitespaceInToken(
395 Whitespaces.replaceWhitespaceInToken(
399 Whitespaces.replaceWhitespaceInToken(
406 unsigned StartColumn,
bool InPPDirective,
410 StartColumn(StartColumn) {}
416 unsigned ColumnLimit,
unsigned ContentStartColumn,
417 const llvm::Regex &CommentPragmasRegex)
const {
419 if (CommentPragmasRegex.match(
Content[LineIndex]))
420 return Split(StringRef::npos, 0);
427 unsigned LineIndex,
unsigned TailOffset,
Split Split,
429 StringRef
Text =
Content[LineIndex].substr(TailOffset);
434 unsigned BreakOffsetInToken =
436 unsigned CharsToRemove =
Split.second;
437 Whitespaces.replaceWhitespaceInToken(
438 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"",
"",
447 Content = Content.trim(
Blanks);
450 bool hasSpecialMeaningPrefix =
false;
451 for (StringRef Prefix :
452 {
"@",
"\\",
"TODO",
"FIXME",
"XXX",
"-# ",
"- ",
"+ ",
"* "}) {
453 if (Content.starts_with(Prefix)) {
454 hasSpecialMeaningPrefix =
true;
462 static const auto kNumberedListRegexp = llvm::Regex(
"^[1-9][0-9]?\\. ");
463 hasSpecialMeaningPrefix =
464 hasSpecialMeaningPrefix || kNumberedListRegexp.match(Content);
469 return Content.size() >= 2 && !hasSpecialMeaningPrefix &&
470 !Content.ends_with(
"\\") &&
478 unsigned OriginalStartColumn,
bool FirstInLine,
bool InPPDirective,
481 DelimitersOnNewline(
false),
482 UnbreakableTailLength(
Token.UnbreakableTailLength) {
483 assert(
Tok.
is(TT_BlockComment) &&
484 "block comment section must start with a block comment");
487 assert(TokenText.starts_with(
"/*") && TokenText.ends_with(
"*/"));
488 TokenText.substr(2, TokenText.size() - 4)
489 .split(
Lines, UseCRLF ?
"\r\n" :
"\n");
491 int IndentDelta =
StartColumn - OriginalStartColumn;
498 for (
size_t i = 1; i <
Lines.size(); ++i)
499 adjustWhitespace(i, IndentDelta);
510 if (
Lines.size() >= 2 &&
Content[1].starts_with(
"**") &&
516 if (
Lines.size() == 1 && !FirstInLine) {
524 for (
size_t i = 1, e =
Content.size(); i < e && !Decoration.empty(); ++i) {
530 }
else if (!
Text.empty() && Decoration.starts_with(
Text)) {
533 while (!
Text.starts_with(Decoration))
534 Decoration = Decoration.drop_back(1);
537 LastLineNeedsDecoration =
true;
539 for (
size_t i = 1, e =
Lines.size(); i < e; ++i) {
545 LastLineNeedsDecoration =
false;
547 if (e >= 2 && !Decoration.empty())
549 }
else if (Decoration.empty()) {
561 unsigned DecorationSize = Decoration.starts_with(
Content[i])
567 if (!Decoration.starts_with(
Content[i])) {
569 std::min<int>(IndentAtLineBreak, std::max(0,
ContentColumn[i]));
572 IndentAtLineBreak = std::max<unsigned>(IndentAtLineBreak, Decoration.size());
576 if ((
Lines[0] ==
"*" ||
Lines[0].starts_with(
"* ")) &&
Lines.size() > 1) {
578 DelimitersOnNewline =
true;
579 }
else if (
Lines[0].starts_with(
"* ") &&
Lines.size() == 1) {
593 llvm::dbgs() <<
"IndentAtLineBreak " << IndentAtLineBreak <<
"\n";
594 llvm::dbgs() <<
"DelimitersOnNewline " << DelimitersOnNewline <<
"\n";
595 for (
size_t i = 0; i <
Lines.size(); ++i) {
596 llvm::dbgs() << i <<
" |" <<
Content[i] <<
"| "
598 <<
"IN=" << (
Content[i].data() -
Lines[i].data()) <<
"\n";
604 unsigned LineIndex,
unsigned TailOffset,
unsigned ColumnLimit,
605 unsigned ContentStartColumn,
const llvm::Regex &CommentPragmasRegex)
const {
607 if (CommentPragmasRegex.match(
Content[LineIndex]))
608 return Split(StringRef::npos, 0);
614void BreakableBlockComment::adjustWhitespace(
unsigned LineIndex,
621 size_t EndOfPreviousLine =
Lines[LineIndex - 1].size();
627 Lines[LineIndex - 1].find_last_not_of(
Blanks, EndOfPreviousLine);
628 if (EndOfPreviousLine == StringRef::npos)
629 EndOfPreviousLine = 0;
633 size_t StartOfLine =
Lines[LineIndex].find_first_not_of(
Blanks);
634 if (StartOfLine == StringRef::npos)
635 StartOfLine =
Lines[LineIndex].size();
637 StringRef Whitespace =
Lines[LineIndex].substr(0, StartOfLine);
639 size_t PreviousContentOffset =
640 Content[LineIndex - 1].data() -
Lines[LineIndex - 1].data();
642 PreviousContentOffset, EndOfPreviousLine - PreviousContentOffset);
643 Content[LineIndex] =
Lines[LineIndex].substr(StartOfLine);
653 StringRef::size_type Length,
654 unsigned StartColumn)
const {
662 unsigned StartColumn)
const {
663 unsigned LineLength =
664 UnbreakableTailLength +
666 if (LineIndex + 1 ==
Lines.size()) {
669 bool HasRemainingText = Offset <
Content[LineIndex].size();
670 if (!HasRemainingText) {
671 bool HasDecoration =
Lines[LineIndex].ltrim().starts_with(Decoration);
673 LineLength -= Decoration.size();
682 return IndentAtLineBreak;
686const llvm::StringSet<>
688 "@param",
"@return",
"@returns",
"@throws",
"@type",
"@template",
689 "@see",
"@deprecated",
"@define",
"@exports",
"@mods",
"@private",
698 StringRef ContentWithNoDecoration =
Content[LineIndex];
699 if (LineIndex == 0 && ContentWithNoDecoration.starts_with(
"*"))
700 ContentWithNoDecoration = ContentWithNoDecoration.substr(1).ltrim(
Blanks);
701 StringRef FirstWord = ContentWithNoDecoration.substr(
702 0, ContentWithNoDecoration.find_first_of(
Blanks));
711 StringRef
Text =
Content[LineIndex].substr(TailOffset);
712 StringRef Prefix = Decoration;
716 unsigned LocalIndentAtLineBreak = IndentAtLineBreak;
717 if (LineIndex + 1 ==
Lines.size() &&
721 if (LocalIndentAtLineBreak >= 2)
722 LocalIndentAtLineBreak -= 2;
726 unsigned BreakOffsetInToken =
728 unsigned CharsToRemove =
Split.second;
729 assert(LocalIndentAtLineBreak >= Prefix.size());
730 std::string PrefixWithTrailingIndent = std::string(Prefix);
731 PrefixWithTrailingIndent.append(ContentIndent,
' ');
732 Whitespaces.replaceWhitespaceInToken(
733 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"",
735 LocalIndentAtLineBreak + ContentIndent -
736 PrefixWithTrailingIndent.size());
740 unsigned LineIndex,
const llvm::Regex &CommentPragmasRegex)
const {
741 if (!
mayReflow(LineIndex, CommentPragmasRegex))
742 return Split(StringRef::npos, 0);
746 size_t Trimmed =
Content[LineIndex].find_first_not_of(
Blanks);
749 if (PreviousContentIndent && Trimmed != StringRef::npos &&
750 Trimmed != PreviousContentIndent) {
751 return Split(StringRef::npos, 0);
755 return Split(0, Trimmed != StringRef::npos ? Trimmed : 0);
760 return DelimitersOnNewline &&
761 Lines[0].substr(1).find_first_not_of(
Blanks) != StringRef::npos;
769 "Reflowing whitespace within a token");
772 unsigned WhitespaceOffsetInToken =
Content[LineIndex - 1].data() +
773 Content[LineIndex - 1].size() -
775 unsigned WhitespaceLength = TrimmedContent.data() -
777 WhitespaceOffsetInToken;
778 Whitespaces.replaceWhitespaceInToken(
779 tokenAt(LineIndex), WhitespaceOffsetInToken,
780 WhitespaceLength,
"",
787 if (LineIndex == 0) {
788 if (DelimitersOnNewline) {
793 size_t BreakLength =
Lines[0].substr(1).find_first_not_of(
Blanks);
794 if (BreakLength != StringRef::npos) {
803 StringRef Prefix = Decoration;
804 if (
Content[LineIndex].empty()) {
805 if (LineIndex + 1 ==
Lines.size()) {
806 if (!LastLineNeedsDecoration) {
811 }
else if (!Decoration.empty()) {
814 Prefix = Prefix.substr(0, 1);
818 Prefix = Prefix.substr(0, 1);
822 unsigned WhitespaceOffsetInToken =
Content[LineIndex - 1].data() +
823 Content[LineIndex - 1].size() -
825 unsigned WhitespaceLength =
Content[LineIndex].data() -
827 WhitespaceOffsetInToken;
828 Whitespaces.replaceWhitespaceInToken(
829 tokenAt(LineIndex), WhitespaceOffsetInToken, WhitespaceLength,
"", Prefix,
835 if (DelimitersOnNewline) {
839 StringRef
Line =
Content.back().substr(TailOffset);
841 if (!TrimmedLine.empty())
842 return Split(TrimmedLine.size(),
Line.size() - TrimmedLine.size());
844 return Split(StringRef::npos, 0);
848 unsigned LineIndex,
const llvm::Regex &CommentPragmasRegex)
const {
851 StringRef IndentContent =
Content[LineIndex];
852 if (
Lines[LineIndex].ltrim(
Blanks).starts_with(
"*"))
853 IndentContent =
Lines[LineIndex].ltrim(
Blanks).substr(1);
854 return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
863 assert(
Tok.
is(TT_LineComment) &&
864 "line comment section must start with a line comment");
869 int FirstLineSpaceChange = 0;
871 CurrentTok && CurrentTok->
is(TT_LineComment);
872 CurrentTok = CurrentTok->Next) {
873 LastLineTok = LineTok;
874 StringRef TokenText(CurrentTok->TokenText);
875 assert((TokenText.starts_with(
"//") || TokenText.starts_with(
"#")) &&
876 "unsupported line comment prefix, '//' and '#' are supported");
877 size_t FirstLineIndex =
Lines.size();
878 TokenText.split(
Lines,
"\n");
881 PrefixSpaceChange.resize(
Lines.size());
883 Prefix.resize(
Lines.size());
884 OriginalPrefix.resize(
Lines.size());
885 for (
size_t i = FirstLineIndex, e =
Lines.size(); i < e; ++i) {
888 OriginalPrefix[i] = IndentPrefix;
889 const int SpacesInPrefix = llvm::count(IndentPrefix,
' ');
893 const auto NoSpaceBeforeFirstCommentChar = [&]() {
894 assert(
Lines[i].size() > IndentPrefix.size());
895 const char FirstCommentChar =
Lines[i][IndentPrefix.size()];
896 const unsigned FirstCharByteSize =
899 Lines[i].substr(IndentPrefix.size(), FirstCharByteSize),
912 if (FirstCommentChar ==
'#' && !TokenText.starts_with(
"#"))
914 return FirstCommentChar ==
'\\' ||
isPunctuation(FirstCommentChar) ||
923 if (i == 0 || OriginalPrefix[i].rtrim(
Blanks) !=
924 OriginalPrefix[i - 1].rtrim(
Blanks)) {
925 if (SpacesInPrefix < Minimum &&
Lines[i].size() > IndentPrefix.size() &&
926 !NoSpaceBeforeFirstCommentChar()) {
927 FirstLineSpaceChange = Minimum - SpacesInPrefix;
928 }
else if (
static_cast<unsigned>(SpacesInPrefix) >
930 FirstLineSpaceChange =
933 FirstLineSpaceChange = 0;
937 if (
Lines[i].size() != IndentPrefix.size()) {
938 PrefixSpaceChange[i] = FirstLineSpaceChange;
940 if (SpacesInPrefix + PrefixSpaceChange[i] < Minimum) {
941 PrefixSpaceChange[i] +=
942 Minimum - (SpacesInPrefix + PrefixSpaceChange[i]);
945 assert(
Lines[i].size() > IndentPrefix.size());
946 const auto FirstNonSpace =
Lines[i][IndentPrefix.size()];
948 const bool LineRequiresLeadingSpace =
949 !NoSpaceBeforeFirstCommentChar() ||
950 (FirstNonSpace ==
'}' && FirstLineSpaceChange != 0);
951 const bool AllowsSpaceChange =
953 (SpacesInPrefix != 0 || LineRequiresLeadingSpace);
955 if (PrefixSpaceChange[i] > 0 && AllowsSpaceChange) {
956 Prefix[i] = IndentPrefix.str();
957 Prefix[i].append(PrefixSpaceChange[i],
' ');
958 }
else if (PrefixSpaceChange[i] < 0 && AllowsSpaceChange) {
959 Prefix[i] = IndentPrefix
960 .drop_back(std::min<std::size_t>(
961 -PrefixSpaceChange[i], SpacesInPrefix))
964 Prefix[i] = IndentPrefix.str();
969 Prefix[i] = IndentPrefix.drop_back(SpacesInPrefix).str();
980 if (EndOfLine == StringRef::npos)
986 LineTok = CurrentTok->
Next;
987 if (CurrentTok->Next && !CurrentTok->Next->ContinuesLineCommentSection) {
1009 StringRef::size_type Length,
1010 unsigned StartColumn)
const {
1023 unsigned LineIndex,
unsigned TailOffset,
Split Split,
1025 StringRef
Text =
Content[LineIndex].substr(TailOffset);
1028 unsigned BreakOffsetInToken =
1030 unsigned CharsToRemove =
Split.second;
1031 Whitespaces.replaceWhitespaceInToken(
1032 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"",
1038 unsigned LineIndex,
const llvm::Regex &CommentPragmasRegex)
const {
1039 if (!
mayReflow(LineIndex, CommentPragmasRegex))
1040 return Split(StringRef::npos, 0);
1042 size_t Trimmed =
Content[LineIndex].find_first_not_of(
Blanks);
1048 return Split(0, Trimmed != StringRef::npos ? Trimmed : 0);
1053 if (LineIndex > 0 &&
Tokens[LineIndex] !=
Tokens[LineIndex - 1]) {
1056 Whitespaces.replaceWhitespace(
1057 *
Tokens[LineIndex], 0, 0,
1060 }
else if (LineIndex > 0) {
1071 unsigned Offset =
Lines[LineIndex - 1].data() +
1072 Lines[LineIndex - 1].size() -
1076 unsigned WhitespaceLength =
1078 Whitespaces.replaceWhitespaceInToken(*
Tokens[LineIndex], Offset,
1089 unsigned WhitespaceLength =
1091 Whitespaces.replaceWhitespaceInToken(*
Tokens[LineIndex], Offset,
1108 if (LineIndex > 0 &&
Tokens[LineIndex] !=
Tokens[LineIndex - 1]) {
1114 unsigned LineColumn =
1116 (
Content[LineIndex].data() -
Lines[LineIndex].data()) +
1117 (OriginalPrefix[LineIndex].size() - Prefix[LineIndex].size());
1123 Whitespaces.replaceWhitespace(*
Tokens[LineIndex],
1130 if (OriginalPrefix[LineIndex] != Prefix[LineIndex]) {
1132 const auto SpacesToRemove = -std::min(PrefixSpaceChange[LineIndex], 0);
1133 const auto SpacesToAdd = std::max(PrefixSpaceChange[LineIndex], 0);
1134 Whitespaces.replaceWhitespaceInToken(
1135 tokenAt(LineIndex), OriginalPrefix[LineIndex].size() - SpacesToRemove,
1136 SpacesToRemove,
"",
"",
false,
1143 State.NextToken = LastLineTok->
Next;
1147 unsigned LineIndex,
const llvm::Regex &CommentPragmasRegex)
const {
1150 StringRef IndentContent =
Content[LineIndex];
1151 if (
Lines[LineIndex].starts_with(
"//"))
1152 IndentContent =
Lines[LineIndex].substr(2);
1159 return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
1162 OriginalPrefix[LineIndex] == OriginalPrefix[LineIndex - 1];
Declares BreakableToken, BreakableStringLiteral, BreakableComment, BreakableBlockComment and Breakabl...
This file implements an indenter that manages the indentation of continuations.
Token - This structure provides full information about a lexed token.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
The JSON file list parser is used to communicate input to InstallAPI.
LLVM_READONLY bool isAlphanumeric(unsigned char c)
Return true if this character is an ASCII letter or digit: [a-zA-Z0-9].
LLVM_READONLY bool isHorizontalWhitespace(unsigned char c)
Returns true if this character is horizontal ASCII whitespace: ' ', '\t', '\f', '\v'.
LLVM_READONLY bool isPunctuation(unsigned char c)
Return true if this character is an ASCII punctuation character.