clang  6.0.0svn
RawCommentList.cpp
Go to the documentation of this file.
1 //===--- RawCommentList.cpp - Processing raw comments -----------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
11 #include "clang/AST/ASTContext.h"
12 #include "clang/AST/Comment.h"
15 #include "clang/AST/CommentLexer.h"
17 #include "clang/AST/CommentSema.h"
18 #include "clang/Basic/CharInfo.h"
19 #include "llvm/ADT/STLExtras.h"
20 
21 using namespace clang;
22 
23 namespace {
24 /// Get comment kind and bool describing if it is a trailing comment.
25 std::pair<RawComment::CommentKind, bool> getCommentKind(StringRef Comment,
26  bool ParseAllComments) {
27  const size_t MinCommentLength = ParseAllComments ? 2 : 3;
28  if ((Comment.size() < MinCommentLength) || Comment[0] != '/')
29  return std::make_pair(RawComment::RCK_Invalid, false);
30 
32  if (Comment[1] == '/') {
33  if (Comment.size() < 3)
34  return std::make_pair(RawComment::RCK_OrdinaryBCPL, false);
35 
36  if (Comment[2] == '/')
38  else if (Comment[2] == '!')
40  else
41  return std::make_pair(RawComment::RCK_OrdinaryBCPL, false);
42  } else {
43  assert(Comment.size() >= 4);
44 
45  // Comment lexer does not understand escapes in comment markers, so pretend
46  // that this is not a comment.
47  if (Comment[1] != '*' ||
48  Comment[Comment.size() - 2] != '*' ||
49  Comment[Comment.size() - 1] != '/')
50  return std::make_pair(RawComment::RCK_Invalid, false);
51 
52  if (Comment[2] == '*')
54  else if (Comment[2] == '!')
56  else
57  return std::make_pair(RawComment::RCK_OrdinaryC, false);
58  }
59  const bool TrailingComment = (Comment.size() > 3) && (Comment[3] == '<');
60  return std::make_pair(K, TrailingComment);
61 }
62 
63 bool mergedCommentIsTrailingComment(StringRef Comment) {
64  return (Comment.size() > 3) && (Comment[3] == '<');
65 }
66 
67 /// Returns true if R1 and R2 both have valid locations that start on the same
68 /// column.
69 bool commentsStartOnSameColumn(const SourceManager &SM, const RawComment &R1,
70  const RawComment &R2) {
71  SourceLocation L1 = R1.getLocStart();
72  SourceLocation L2 = R2.getLocStart();
73  bool Invalid = false;
74  unsigned C1 = SM.getPresumedColumnNumber(L1, &Invalid);
75  if (!Invalid) {
76  unsigned C2 = SM.getPresumedColumnNumber(L2, &Invalid);
77  return !Invalid && (C1 == C2);
78  }
79  return false;
80 }
81 } // unnamed namespace
82 
83 /// \brief Determines whether there is only whitespace in `Buffer` between `P`
84 /// and the previous line.
85 /// \param Buffer The buffer to search in.
86 /// \param P The offset from the beginning of `Buffer` to start from.
87 /// \return true if all of the characters in `Buffer` ranging from the closest
88 /// line-ending character before `P` (or the beginning of `Buffer`) to `P - 1`
89 /// are whitespace.
90 static bool onlyWhitespaceOnLineBefore(const char *Buffer, unsigned P) {
91  // Search backwards until we see linefeed or carriage return.
92  for (unsigned I = P; I != 0; --I) {
93  char C = Buffer[I - 1];
94  if (isVerticalWhitespace(C))
95  return true;
96  if (!isHorizontalWhitespace(C))
97  return false;
98  }
99  // We hit the beginning of the buffer.
100  return true;
101 }
102 
103 /// Returns whether `K` is an ordinary comment kind.
105  return (K == RawComment::RCK_OrdinaryBCPL) ||
107 }
108 
110  bool Merged, bool ParseAllComments) :
111  Range(SR), RawTextValid(false), BriefTextValid(false),
112  IsAttached(false), IsTrailingComment(false), IsAlmostTrailingComment(false),
113  ParseAllComments(ParseAllComments) {
114  // Extract raw comment text, if possible.
115  if (SR.getBegin() == SR.getEnd() || getRawText(SourceMgr).empty()) {
116  Kind = RCK_Invalid;
117  return;
118  }
119 
120  // Guess comment kind.
121  std::pair<CommentKind, bool> K = getCommentKind(RawText, ParseAllComments);
122 
123  // Guess whether an ordinary comment is trailing.
124  if (ParseAllComments && isOrdinaryKind(K.first)) {
125  FileID BeginFileID;
126  unsigned BeginOffset;
127  std::tie(BeginFileID, BeginOffset) =
128  SourceMgr.getDecomposedLoc(Range.getBegin());
129  if (BeginOffset != 0) {
130  bool Invalid = false;
131  const char *Buffer =
132  SourceMgr.getBufferData(BeginFileID, &Invalid).data();
133  IsTrailingComment |=
134  (!Invalid && !onlyWhitespaceOnLineBefore(Buffer, BeginOffset));
135  }
136  }
137 
138  if (!Merged) {
139  Kind = K.first;
140  IsTrailingComment |= K.second;
141 
142  IsAlmostTrailingComment = RawText.startswith("//<") ||
143  RawText.startswith("/*<");
144  } else {
145  Kind = RCK_Merged;
146  IsTrailingComment =
147  IsTrailingComment || mergedCommentIsTrailingComment(RawText);
148  }
149 }
150 
151 StringRef RawComment::getRawTextSlow(const SourceManager &SourceMgr) const {
152  FileID BeginFileID;
153  FileID EndFileID;
154  unsigned BeginOffset;
155  unsigned EndOffset;
156 
157  std::tie(BeginFileID, BeginOffset) =
158  SourceMgr.getDecomposedLoc(Range.getBegin());
159  std::tie(EndFileID, EndOffset) = SourceMgr.getDecomposedLoc(Range.getEnd());
160 
161  const unsigned Length = EndOffset - BeginOffset;
162  if (Length < 2)
163  return StringRef();
164 
165  // The comment can't begin in one file and end in another.
166  assert(BeginFileID == EndFileID);
167 
168  bool Invalid = false;
169  const char *BufferStart = SourceMgr.getBufferData(BeginFileID,
170  &Invalid).data();
171  if (Invalid)
172  return StringRef();
173 
174  return StringRef(BufferStart + BeginOffset, Length);
175 }
176 
177 const char *RawComment::extractBriefText(const ASTContext &Context) const {
178  // Lazily initialize RawText using the accessor before using it.
179  (void)getRawText(Context.getSourceManager());
180 
181  // Since we will be copying the resulting text, all allocations made during
182  // parsing are garbage after resulting string is formed. Thus we can use
183  // a separate allocator for all temporary stuff.
184  llvm::BumpPtrAllocator Allocator;
185 
186  comments::Lexer L(Allocator, Context.getDiagnostics(),
187  Context.getCommentCommandTraits(),
188  Range.getBegin(),
189  RawText.begin(), RawText.end());
191 
192  const std::string Result = P.Parse();
193  const unsigned BriefTextLength = Result.size();
194  char *BriefTextPtr = new (Context) char[BriefTextLength + 1];
195  memcpy(BriefTextPtr, Result.c_str(), BriefTextLength + 1);
196  BriefText = BriefTextPtr;
197  BriefTextValid = true;
198 
199  return BriefTextPtr;
200 }
201 
203  const Preprocessor *PP,
204  const Decl *D) const {
205  // Lazily initialize RawText using the accessor before using it.
206  (void)getRawText(Context.getSourceManager());
207 
208  comments::Lexer L(Context.getAllocator(), Context.getDiagnostics(),
209  Context.getCommentCommandTraits(),
211  RawText.begin(), RawText.end());
212  comments::Sema S(Context.getAllocator(), Context.getSourceManager(),
213  Context.getDiagnostics(),
214  Context.getCommentCommandTraits(),
215  PP);
216  S.setDecl(D);
217  comments::Parser P(L, S, Context.getAllocator(), Context.getSourceManager(),
218  Context.getDiagnostics(),
219  Context.getCommentCommandTraits());
220 
221  return P.parseFullComment();
222 }
223 
225  SourceLocation Loc1, SourceLocation Loc2,
226  unsigned MaxNewlinesAllowed) {
227  std::pair<FileID, unsigned> Loc1Info = SM.getDecomposedLoc(Loc1);
228  std::pair<FileID, unsigned> Loc2Info = SM.getDecomposedLoc(Loc2);
229 
230  // Question does not make sense if locations are in different files.
231  if (Loc1Info.first != Loc2Info.first)
232  return false;
233 
234  bool Invalid = false;
235  const char *Buffer = SM.getBufferData(Loc1Info.first, &Invalid).data();
236  if (Invalid)
237  return false;
238 
239  unsigned NumNewlines = 0;
240  assert(Loc1Info.second <= Loc2Info.second && "Loc1 after Loc2!");
241  // Look for non-whitespace characters and remember any newlines seen.
242  for (unsigned I = Loc1Info.second; I != Loc2Info.second; ++I) {
243  switch (Buffer[I]) {
244  default:
245  return false;
246  case ' ':
247  case '\t':
248  case '\f':
249  case '\v':
250  break;
251  case '\r':
252  case '\n':
253  ++NumNewlines;
254 
255  // Check if we have found more than the maximum allowed number of
256  // newlines.
257  if (NumNewlines > MaxNewlinesAllowed)
258  return false;
259 
260  // Collapse \r\n and \n\r into a single newline.
261  if (I + 1 != Loc2Info.second &&
262  (Buffer[I + 1] == '\n' || Buffer[I + 1] == '\r') &&
263  Buffer[I] != Buffer[I + 1])
264  ++I;
265  break;
266  }
267  }
268 
269  return true;
270 }
271 
273  llvm::BumpPtrAllocator &Allocator) {
274  if (RC.isInvalid())
275  return;
276 
277  // Check if the comments are not in source order.
278  while (!Comments.empty() &&
279  !SourceMgr.isBeforeInTranslationUnit(Comments.back()->getLocStart(),
280  RC.getLocStart())) {
281  // If they are, just pop a few last comments that don't fit.
282  // This happens if an \#include directive contains comments.
283  Comments.pop_back();
284  }
285 
286  // Ordinary comments are not interesting for us.
287  if (RC.isOrdinary())
288  return;
289 
290  // If this is the first Doxygen comment, save it (because there isn't
291  // anything to merge it with).
292  if (Comments.empty()) {
293  Comments.push_back(new (Allocator) RawComment(RC));
294  return;
295  }
296 
297  const RawComment &C1 = *Comments.back();
298  const RawComment &C2 = RC;
299 
300  // Merge comments only if there is only whitespace between them.
301  // Can't merge trailing and non-trailing comments unless the second is
302  // non-trailing ordinary in the same column, as in the case:
303  // int x; // documents x
304  // // more text
305  // versus:
306  // int x; // documents x
307  // int y; // documents y
308  // or:
309  // int x; // documents x
310  // // documents y
311  // int y;
312  // Merge comments if they are on same or consecutive lines.
313  if ((C1.isTrailingComment() == C2.isTrailingComment() ||
314  (C1.isTrailingComment() && !C2.isTrailingComment() &&
315  isOrdinaryKind(C2.getKind()) &&
316  commentsStartOnSameColumn(SourceMgr, C1, C2))) &&
317  onlyWhitespaceBetween(SourceMgr, C1.getLocEnd(), C2.getLocStart(),
318  /*MaxNewlinesAllowed=*/1)) {
319  SourceRange MergedRange(C1.getLocStart(), C2.getLocEnd());
320  *Comments.back() = RawComment(SourceMgr, MergedRange, true,
321  RC.isParseAllComments());
322  } else {
323  Comments.push_back(new (Allocator) RawComment(RC));
324  }
325 }
326 
327 void RawCommentList::addDeserializedComments(ArrayRef<RawComment *> DeserializedComments) {
328  std::vector<RawComment *> MergedComments;
329  MergedComments.reserve(Comments.size() + DeserializedComments.size());
330 
331  std::merge(Comments.begin(), Comments.end(),
332  DeserializedComments.begin(), DeserializedComments.end(),
333  std::back_inserter(MergedComments),
334  BeforeThanCompare<RawComment>(SourceMgr));
335  std::swap(Comments, MergedComments);
336 }
Defines the clang::ASTContext interface.
bool isInvalid() const LLVM_READONLY
SourceLocation getLocEnd() const LLVM_READONLY
void addComment(const RawComment &RC, llvm::BumpPtrAllocator &Allocator)
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:81
unsigned getPresumedColumnNumber(SourceLocation Loc, bool *Invalid=nullptr) const
StringRef P
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
DiagnosticsEngine & getDiagnostics() const
Any normal BCPL comments.
SourceRange getSourceRange() const LLVM_READONLY
StringRef getBufferData(FileID FID, bool *Invalid=nullptr) const
Return a StringRef to the source buffer data for the specified FileID.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:128
static LLVM_READONLY bool isHorizontalWhitespace(unsigned char c)
Returns true if this character is horizontal ASCII whitespace: &#39; &#39;, &#39;\t&#39;, &#39;\f&#39;, &#39;\v&#39;.
Definition: CharInfo.h:71
static bool onlyWhitespaceBetween(SourceManager &SM, SourceLocation Loc1, SourceLocation Loc2, unsigned MaxNewlinesAllowed)
Any normal C comment.
comments::CommandTraits & getCommentCommandTraits() const
Definition: ASTContext.h:829
SourceLocation getEnd() const
The result type of a method or function.
const SourceManager & SM
Definition: Format.cpp:1308
comments::FullComment * parse(const ASTContext &Context, const Preprocessor *PP, const Decl *D) const
Parse the comment, assuming it is attached to decl D.
bool isOrdinary() const LLVM_READONLY
Returns true if this comment is not a documentation comment.
Compare comments&#39; source locations.
#define false
Definition: stdbool.h:33
Kind
Two or more documentation comments merged together.
Encodes a location in the source.
static bool isOrdinaryKind(RawComment::CommentKind K)
Returns whether K is an ordinary comment kind.
Comment lexer.
Definition: CommentLexer.h:221
bool isParseAllComments() const LLVM_READONLY
Returns whether we are parsing all comments.
static bool onlyWhitespaceOnLineBefore(const char *Buffer, unsigned P)
Determines whether there is only whitespace in Buffer between P and the previous line.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Dataflow Directional Tag Classes.
bool isTrailingComment() const LLVM_READONLY
Returns true if it is a comment that should be put after a member:
SourceLocation getLocStart() const LLVM_READONLY
llvm::BumpPtrAllocator & getAllocator() const
Definition: ASTContext.h:621
SourceManager & getSourceManager()
Definition: ASTContext.h:618
Doxygen comment parser.
Definition: CommentParser.h:30
StringRef getRawText(const SourceManager &SourceMgr) const
Returns raw comment text with comment markers.
CommentKind getKind() const LLVM_READONLY
static LLVM_READONLY bool isVerticalWhitespace(unsigned char c)
Returns true if this character is vertical ASCII whitespace: &#39;\n&#39;, &#39;\r&#39;.
Definition: CharInfo.h:79
A very simple comment parser that extracts "a brief description".
A trivial tuple used to represent a source range.
SourceLocation getBegin() const
This class handles loading and caching of source files into memory.
A full comment attached to a declaration, contains block content.
Definition: Comment.h:1097
std::pair< FileID, unsigned > getDecomposedLoc(SourceLocation Loc) const
Decompose the specified location into a raw FileID + Offset pair.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:98