19 #include "llvm/ADT/STLExtras.h" 20 #include "llvm/Support/Debug.h" 23 #define DEBUG_TYPE "format-token-breaker" 28 static const char *
const Blanks =
" \t\v\f\r";
44 static const char *
const KnownCStylePrefixes[] = {
"///<",
"//!<",
"///",
"//",
46 static const char *
const KnownTextProtoPrefixes[] = {
"//",
"#",
"##",
"###",
49 if (Style.Language == FormatStyle::LK_TextProto)
50 KnownPrefixes = KnownTextProtoPrefixes;
52 StringRef LongestPrefix;
53 for (StringRef KnownPrefix : KnownPrefixes) {
54 if (Comment.startswith(KnownPrefix)) {
55 size_t PrefixLength = KnownPrefix.size();
56 while (PrefixLength < Comment.size() && Comment[PrefixLength] ==
' ')
58 if (PrefixLength > LongestPrefix.size())
59 LongestPrefix = Comment.substr(0, PrefixLength);
65 static BreakableToken::Split
67 unsigned ColumnLimit,
unsigned TabWidth,
69 bool DecorationEndsWithStar =
false) {
70 LLVM_DEBUG(llvm::dbgs() <<
"Comment split: \"" << Text
71 <<
"\", Column limit: " << ColumnLimit
72 <<
", Content start: " << ContentStartColumn <<
"\n");
73 if (ColumnLimit <= ContentStartColumn + 1)
74 return BreakableToken::Split(StringRef::npos, 0);
76 unsigned MaxSplit = ColumnLimit - ContentStartColumn + 1;
77 unsigned MaxSplitBytes = 0;
79 for (
unsigned NumChars = 0;
80 NumChars < MaxSplit && MaxSplitBytes < Text.size();) {
81 unsigned BytesInChar =
85 ContentStartColumn, TabWidth,
Encoding);
86 MaxSplitBytes += BytesInChar;
89 StringRef::size_type SpaceOffset = Text.find_last_of(Blanks, MaxSplitBytes);
91 static auto *
const kNumberedListRegexp =
new llvm::Regex(
"^[1-9][0-9]?\\.");
92 while (SpaceOffset != StringRef::npos) {
95 if (kNumberedListRegexp->match(Text.substr(SpaceOffset).ltrim(Blanks)))
96 SpaceOffset = Text.find_last_of(Blanks, SpaceOffset);
100 else if (Style.Language == FormatStyle::LK_JavaScript &&
101 SpaceOffset + 1 < Text.size() && Text[SpaceOffset + 1] ==
'{')
102 SpaceOffset = Text.find_last_of(Blanks, SpaceOffset);
107 if (SpaceOffset == StringRef::npos ||
109 Text.find_last_not_of(Blanks, SpaceOffset) == StringRef::npos) {
112 StringRef::size_type FirstNonWhitespace = Text.find_first_not_of(Blanks);
113 if (FirstNonWhitespace == StringRef::npos)
115 return BreakableToken::Split(StringRef::npos, 0);
116 SpaceOffset = Text.find_first_of(
117 Blanks, std::max<unsigned>(MaxSplitBytes, FirstNonWhitespace));
119 if (SpaceOffset != StringRef::npos && SpaceOffset != 0) {
124 if (SpaceOffset == 1 && Text[SpaceOffset - 1] ==
'*')
125 return BreakableToken::Split(StringRef::npos, 0);
126 StringRef BeforeCut = Text.substr(0, SpaceOffset).rtrim(Blanks);
127 StringRef AfterCut = Text.substr(SpaceOffset);
129 if (!DecorationEndsWithStar || AfterCut.size() <= 1 || AfterCut[1] !=
'/')
130 AfterCut = AfterCut.ltrim(Blanks);
131 return BreakableToken::Split(BeforeCut.size(),
132 AfterCut.begin() - BeforeCut.end());
134 return BreakableToken::Split(StringRef::npos, 0);
137 static BreakableToken::Split
142 return BreakableToken::Split(StringRef::npos, 0);
143 if (ColumnLimit <= UsedColumns)
144 return BreakableToken::Split(StringRef::npos, 0);
145 unsigned MaxSplit = ColumnLimit - UsedColumns;
146 StringRef::size_type SpaceOffset = 0;
147 StringRef::size_type SlashOffset = 0;
148 StringRef::size_type WordStartOffset = 0;
149 StringRef::size_type SplitPoint = 0;
150 for (
unsigned Chars = 0;;) {
152 if (Text[0] ==
'\\') {
158 Text.substr(0, Advance), UsedColumns + Chars, TabWidth,
Encoding);
161 if (Chars > MaxSplit || Text.size() <= Advance)
165 SpaceOffset = SplitPoint;
167 SlashOffset = SplitPoint;
169 WordStartOffset = SplitPoint;
171 SplitPoint += Advance;
172 Text = Text.substr(Advance);
175 if (SpaceOffset != 0)
176 return BreakableToken::Split(SpaceOffset + 1, 0);
177 if (SlashOffset != 0)
178 return BreakableToken::Split(SlashOffset + 1, 0);
179 if (WordStartOffset != 0)
180 return BreakableToken::Split(WordStartOffset + 1, 0);
182 return BreakableToken::Split(SplitPoint, 0);
183 return BreakableToken::Split(StringRef::npos, 0);
187 assert((Token.
is(TT_BlockComment) || Token.
is(TT_LineComment)) &&
188 "formatting regions are switched by comment tokens");
189 StringRef Content = Token.
TokenText.substr(2).ltrim();
190 return Content.startswith(
"clang-format on") ||
191 Content.startswith(
"clang-format off");
208 return RemainingTokenColumns + 1 - Split.second;
215 StringRef::size_type Length,
217 llvm_unreachable(
"Getting the length of a part of the string literal " 218 "indicates that the code tries to reflow it.");
239 StartColumn(StartColumn), Prefix(Prefix), Postfix(Postfix),
240 UnbreakableTailLength(UnbreakableTailLength) {
243 Prefix.size(), Tok.
TokenText.size() - Prefix.size() - Postfix.size());
247 unsigned LineIndex,
unsigned TailOffset,
unsigned ColumnLimit,
248 unsigned ContentStartColumn, llvm::Regex &CommentPragmasRegex)
const {
254 unsigned TailOffset, Split Split,
255 unsigned ContentIndent,
267 StartColumn(StartColumn) {}
271 BreakableToken::Split
273 unsigned ColumnLimit,
unsigned ContentStartColumn,
274 llvm::Regex &CommentPragmasRegex)
const {
276 if (CommentPragmasRegex.match(
Content[LineIndex]))
277 return Split(StringRef::npos, 0);
279 ContentStartColumn, ColumnLimit,
Style.TabWidth,
284 unsigned LineIndex,
unsigned TailOffset, Split Split,
286 StringRef
Text =
Content[LineIndex].substr(TailOffset);
291 unsigned BreakOffsetInToken =
293 unsigned CharsToRemove = Split.second;
295 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"",
"",
304 Content = Content.trim(Blanks);
307 bool hasSpecialMeaningPrefix =
false;
308 for (StringRef Prefix :
309 {
"@",
"TODO",
"FIXME",
"XXX",
"-# ",
"- ",
"+ ",
"* "}) {
310 if (Content.startswith(Prefix)) {
311 hasSpecialMeaningPrefix =
true;
319 static auto *
const kNumberedListRegexp =
new llvm::Regex(
"^[1-9][0-9]?\\. ");
320 hasSpecialMeaningPrefix =
321 hasSpecialMeaningPrefix || kNumberedListRegexp->match(Content);
326 return Content.size() >= 2 && !hasSpecialMeaningPrefix &&
327 !Content.endswith(
"\\") &&
335 unsigned OriginalStartColumn,
bool FirstInLine,
bool InPPDirective,
338 DelimitersOnNewline(
false),
339 UnbreakableTailLength(Token.UnbreakableTailLength) {
340 assert(
Tok.
is(TT_BlockComment) &&
341 "block comment section must start with a block comment");
344 assert(TokenText.startswith(
"/*") && TokenText.endswith(
"*/"));
345 TokenText.substr(2, TokenText.size() - 4)
346 .split(
Lines, UseCRLF ?
"\r\n" :
"\n");
348 int IndentDelta = StartColumn - OriginalStartColumn;
355 for (
size_t i = 1; i <
Lines.size(); ++i)
356 adjustWhitespace(i, IndentDelta);
360 DecorationColumn = StartColumn + 1;
373 if (
Lines.size() == 1 && !FirstInLine) {
381 for (
size_t i = 1, e =
Lines.size(); i < e && !Decoration.empty(); ++i) {
383 if (i + 1 == e &&
Content[i].empty())
385 if (!
Content[i].empty() && i + 1 != e && Decoration.startswith(
Content[i]))
387 while (!
Content[i].startswith(Decoration))
388 Decoration = Decoration.substr(0, Decoration.size() - 1);
391 LastLineNeedsDecoration =
true;
393 for (
size_t i = 1, e =
Lines.size(); i < e; ++i) {
399 LastLineNeedsDecoration =
false;
401 if (e >= 2 && !Decoration.empty()) {
404 }
else if (Decoration.empty()) {
416 unsigned DecorationSize = Decoration.startswith(
Content[i])
419 if (DecorationSize) {
423 if (!Decoration.startswith(
Content[i]))
427 IndentAtLineBreak = std::max<unsigned>(IndentAtLineBreak, Decoration.size());
430 if (Style.Language == FormatStyle::LK_JavaScript ||
431 Style.Language == FormatStyle::LK_Java) {
432 if ((
Lines[0] ==
"*" ||
Lines[0].startswith(
"* ")) &&
Lines.size() > 1) {
434 DelimitersOnNewline =
true;
435 }
else if (
Lines[0].startswith(
"* ") &&
Lines.size() == 1) {
442 Style.TabWidth, Encoding) +
444 DelimitersOnNewline = EndColumn > Style.ColumnLimit;
449 llvm::dbgs() <<
"IndentAtLineBreak " << IndentAtLineBreak <<
"\n";
450 llvm::dbgs() <<
"DelimitersOnNewline " << DelimitersOnNewline <<
"\n";
451 for (
size_t i = 0; i <
Lines.size(); ++i) {
452 llvm::dbgs() << i <<
" |" <<
Content[i] <<
"| " 454 <<
"IN=" << (
Content[i].data() -
Lines[i].data()) <<
"\n";
460 unsigned LineIndex,
unsigned TailOffset,
unsigned ColumnLimit,
461 unsigned ContentStartColumn, llvm::Regex &CommentPragmasRegex)
const {
463 if (CommentPragmasRegex.match(
Content[LineIndex]))
464 return Split(StringRef::npos, 0);
466 ContentStartColumn, ColumnLimit,
Style.TabWidth,
470 void BreakableBlockComment::adjustWhitespace(
unsigned LineIndex,
477 size_t EndOfPreviousLine =
Lines[LineIndex - 1].size();
483 Lines[LineIndex - 1].find_last_not_of(Blanks, EndOfPreviousLine);
484 if (EndOfPreviousLine == StringRef::npos)
485 EndOfPreviousLine = 0;
489 size_t StartOfLine =
Lines[LineIndex].find_first_not_of(Blanks);
490 if (StartOfLine == StringRef::npos)
491 StartOfLine =
Lines[LineIndex].size();
493 StringRef Whitespace =
Lines[LineIndex].substr(0, StartOfLine);
495 size_t PreviousContentOffset =
496 Content[LineIndex - 1].data() -
Lines[LineIndex - 1].data();
498 PreviousContentOffset, EndOfPreviousLine - PreviousContentOffset);
499 Content[LineIndex] =
Lines[LineIndex].substr(StartOfLine);
509 StringRef::size_type Length,
511 unsigned LineLength =
517 if (LineIndex + 1 ==
Lines.size()) {
522 if (
Content[LineIndex].substr(Offset, StringRef::npos).empty()) {
523 LineLength -= Decoration.size();
532 return UnbreakableTailLength +
539 return IndentAtLineBreak;
543 const llvm::StringSet<>
545 "@param",
"@return",
"@returns",
"@throws",
"@type",
"@template",
546 "@see",
"@deprecated",
"@define",
"@exports",
"@mods",
"@private",
550 if (
Style.Language != FormatStyle::LK_Java &&
551 Style.Language != FormatStyle::LK_JavaScript)
556 StringRef ContentWithNoDecoration =
Content[LineIndex];
557 if (LineIndex == 0 && ContentWithNoDecoration.startswith(
"*")) {
558 ContentWithNoDecoration = ContentWithNoDecoration.substr(1).ltrim(Blanks);
560 StringRef FirstWord = ContentWithNoDecoration.substr(
561 0, ContentWithNoDecoration.find_first_of(Blanks));
564 return Style.ContinuationIndentWidth;
569 Split Split,
unsigned ContentIndent,
571 StringRef
Text =
Content[LineIndex].substr(TailOffset);
572 StringRef Prefix = Decoration;
576 unsigned LocalIndentAtLineBreak = IndentAtLineBreak;
577 if (LineIndex + 1 ==
Lines.size() &&
578 Text.size() == Split.first + Split.second) {
581 if (LocalIndentAtLineBreak >= 2)
582 LocalIndentAtLineBreak -= 2;
586 unsigned BreakOffsetInToken =
588 unsigned CharsToRemove = Split.second;
589 assert(LocalIndentAtLineBreak >= Prefix.size());
590 std::string PrefixWithTrailingIndent = Prefix;
591 for (
unsigned I = 0; I < ContentIndent; ++I)
592 PrefixWithTrailingIndent +=
" ";
594 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"",
596 LocalIndentAtLineBreak + ContentIndent -
597 PrefixWithTrailingIndent.size());
600 BreakableToken::Split
602 llvm::Regex &CommentPragmasRegex)
const {
603 if (!
mayReflow(LineIndex, CommentPragmasRegex))
604 return Split(StringRef::npos, 0);
608 size_t Trimmed =
Content[LineIndex].find_first_not_of(Blanks);
611 if (PreviousContentIndent && Trimmed != StringRef::npos &&
612 Trimmed != PreviousContentIndent)
613 return Split(StringRef::npos, 0);
616 return Split(0, Trimmed != StringRef::npos ? Trimmed : 0);
621 return DelimitersOnNewline &&
622 Lines[0].substr(1).find_first_not_of(Blanks) != StringRef::npos;
627 StringRef TrimmedContent =
Content[LineIndex].ltrim(Blanks);
630 "Reflowing whitespace within a token");
633 unsigned WhitespaceOffsetInToken =
Content[LineIndex - 1].data() +
634 Content[LineIndex - 1].size() -
636 unsigned WhitespaceLength = TrimmedContent.data() -
638 WhitespaceOffsetInToken;
640 tokenAt(LineIndex), WhitespaceOffsetInToken,
641 WhitespaceLength,
"",
648 if (LineIndex == 0) {
649 if (DelimitersOnNewline) {
654 size_t BreakLength =
Lines[0].substr(1).find_first_not_of(Blanks);
655 if (BreakLength != StringRef::npos)
663 StringRef Prefix = Decoration;
664 if (
Content[LineIndex].empty()) {
665 if (LineIndex + 1 ==
Lines.size()) {
666 if (!LastLineNeedsDecoration) {
671 }
else if (!Decoration.empty()) {
674 Prefix = Prefix.substr(0, 1);
679 Prefix = Prefix.substr(0, 1);
684 unsigned WhitespaceOffsetInToken =
Content[LineIndex - 1].data() +
685 Content[LineIndex - 1].size() -
687 unsigned WhitespaceLength =
Content[LineIndex].data() -
689 WhitespaceOffsetInToken;
691 tokenAt(LineIndex), WhitespaceOffsetInToken, WhitespaceLength,
"", Prefix,
695 BreakableToken::Split
697 if (DelimitersOnNewline) {
701 StringRef
Line =
Content.back().substr(TailOffset);
702 StringRef TrimmedLine = Line.rtrim(Blanks);
703 if (!TrimmedLine.empty())
704 return Split(TrimmedLine.size(), Line.size() - TrimmedLine.size());
706 return Split(StringRef::npos, 0);
710 llvm::Regex &CommentPragmasRegex)
const {
713 StringRef IndentContent =
Content[LineIndex];
714 if (
Lines[LineIndex].ltrim(Blanks).startswith(
"*")) {
715 IndentContent =
Lines[LineIndex].ltrim(Blanks).substr(1);
717 return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
724 unsigned OriginalStartColumn,
bool FirstInLine,
bool InPPDirective,
727 assert(
Tok.
is(TT_LineComment) &&
728 "line comment section must start with a line comment");
731 CurrentTok && CurrentTok->
is(TT_LineComment);
732 CurrentTok = CurrentTok->Next) {
733 LastLineTok = LineTok;
734 StringRef TokenText(CurrentTok->TokenText);
735 assert((TokenText.startswith(
"//") || TokenText.startswith(
"#")) &&
736 "unsupported line comment prefix, '//' and '#' are supported");
737 size_t FirstLineIndex =
Lines.size();
738 TokenText.split(
Lines,
"\n");
741 OriginalContentColumn.resize(
Lines.size());
743 Prefix.resize(
Lines.size());
744 OriginalPrefix.resize(
Lines.size());
745 for (
size_t i = FirstLineIndex, e =
Lines.size(); i < e; ++i) {
749 StringRef IndentPrefix =
751 assert((TokenText.startswith(
"//") || TokenText.startswith(
"#")) &&
752 "unsupported line comment prefix, '//' and '#' are supported");
753 OriginalPrefix[i] = Prefix[i] = IndentPrefix;
754 if (
Lines[i].size() > Prefix[i].size() &&
756 if (Prefix[i] ==
"//")
758 else if (Prefix[i] ==
"///")
760 else if (Prefix[i] ==
"//!")
762 else if (Prefix[i] ==
"///<")
764 else if (Prefix[i] ==
"//!<")
766 else if (Prefix[i] ==
"#" &&
767 Style.Language == FormatStyle::LK_TextProto)
773 OriginalContentColumn[i] =
776 Style.TabWidth, Encoding);
779 Style.TabWidth, Encoding);
782 size_t EndOfLine =
Content[i].find_last_not_of(Blanks);
783 if (EndOfLine == StringRef::npos)
789 LineTok = CurrentTok->Next;
790 if (CurrentTok->Next && !CurrentTok->Next->ContinuesLineCommentSection) {
812 StringRef::size_type Length,
815 Content[LineIndex].substr(Offset, Length), StartColumn,
Style.TabWidth,
822 return OriginalContentColumn[LineIndex];
827 unsigned LineIndex,
unsigned TailOffset, Split Split,
829 StringRef
Text =
Content[LineIndex].substr(TailOffset);
832 unsigned BreakOffsetInToken =
834 unsigned CharsToRemove = Split.second;
837 unsigned IndentAtLineBreak = OriginalContentColumn[LineIndex] +
838 Prefix[LineIndex].size() -
839 OriginalPrefix[LineIndex].size();
840 assert(IndentAtLineBreak >= Prefix[LineIndex].size());
842 tokenAt(LineIndex), BreakOffsetInToken, CharsToRemove,
"",
844 IndentAtLineBreak - Prefix[LineIndex].size());
848 unsigned LineIndex, llvm::Regex &CommentPragmasRegex)
const {
849 if (!
mayReflow(LineIndex, CommentPragmasRegex))
850 return Split(StringRef::npos, 0);
852 size_t Trimmed =
Content[LineIndex].find_first_not_of(Blanks);
858 return Split(0, Trimmed != StringRef::npos ? Trimmed : 0);
863 if (LineIndex > 0 &&
Tokens[LineIndex] !=
Tokens[LineIndex - 1]) {
869 }
else if (LineIndex > 0) {
881 Lines[LineIndex - 1].size() -
885 unsigned WhitespaceLength =
898 unsigned WhitespaceLength =
917 if (LineIndex > 0 &&
Tokens[LineIndex] !=
Tokens[LineIndex - 1]) {
923 unsigned LineColumn =
926 (OriginalPrefix[LineIndex].size() - Prefix[LineIndex].size());
938 if (OriginalPrefix[LineIndex] != Prefix[LineIndex]) {
942 assert(Prefix[LineIndex] == (OriginalPrefix[LineIndex] +
" ").str() &&
943 "Expecting a line comment prefix to differ from original by at most " 946 tokenAt(LineIndex), OriginalPrefix[LineIndex].size(), 0,
"",
"",
958 unsigned LineIndex, llvm::Regex &CommentPragmasRegex)
const {
961 StringRef IndentContent =
Content[LineIndex];
962 if (
Lines[LineIndex].startswith(
"//")) {
963 IndentContent =
Lines[LineIndex].substr(2);
971 return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
974 OriginalPrefix[LineIndex] == OriginalPrefix[LineIndex - 1];
Declares BreakableToken, BreakableStringLiteral, BreakableComment, BreakableBlockComment and Breakabl...
Token - This structure provides full information about a lexed token.
__DEVICE__ int max(int __a, int __b)
const AnnotatedLine * Line
LLVM_READONLY bool isAlphanumeric(unsigned char c)
Return true if this character is an ASCII letter or digit: [a-zA-Z0-9].
Dataflow Directional Tag Classes.
LLVM_READONLY bool isPunctuation(unsigned char c)
Return true if this character is an ASCII punctuation character.
This file implements an indenter that manages the indentation of continuations.