clang 19.0.0git
UnwrappedLineParser.h
Go to the documentation of this file.
1//===--- UnwrappedLineParser.h - Format C++ code ----------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file contains the declaration of the UnwrappedLineParser,
11/// which turns a stream of tokens into UnwrappedLines.
12///
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_CLANG_LIB_FORMAT_UNWRAPPEDLINEPARSER_H
16#define LLVM_CLANG_LIB_FORMAT_UNWRAPPEDLINEPARSER_H
17
18#include "Macros.h"
19#include <stack>
20
21namespace clang {
22namespace format {
23
24struct UnwrappedLineNode;
25
26/// An unwrapped line is a sequence of \c Token, that we would like to
27/// put on a single line if there was no column limit.
28///
29/// This is used as a main interface between the \c UnwrappedLineParser and the
30/// \c UnwrappedLineFormatter. The key property is that changing the formatting
31/// within an unwrapped line does not affect any other unwrapped lines.
33 UnwrappedLine() = default;
34
35 /// The \c Tokens comprising this \c UnwrappedLine.
36 std::list<UnwrappedLineNode> Tokens;
37
38 /// The indent level of the \c UnwrappedLine.
39 unsigned Level = 0;
40
41 /// The \c PPBranchLevel (adjusted for header guards) if this line is a
42 /// \c InMacroBody line, and 0 otherwise.
43 unsigned PPLevel = 0;
44
45 /// Whether this \c UnwrappedLine is part of a preprocessor directive.
46 bool InPPDirective = false;
47 /// Whether this \c UnwrappedLine is part of a pramga directive.
48 bool InPragmaDirective = false;
49 /// Whether it is part of a macro body.
50 bool InMacroBody = false;
51
52 /// Nesting level of unbraced body of a control statement.
53 unsigned UnbracedBodyLevel = 0;
54
55 bool MustBeDeclaration = false;
56
57 /// Whether the parser has seen \c decltype(auto) in this line.
58 bool SeenDecltypeAuto = false;
59
60 /// \c True if this line should be indented by ContinuationIndent in
61 /// addition to the normal indention level.
62 bool IsContinuation = false;
63
64 /// If this \c UnwrappedLine closes a block in a sequence of lines,
65 /// \c MatchingOpeningBlockLineIndex stores the index of the corresponding
66 /// opening line. Otherwise, \c MatchingOpeningBlockLineIndex must be
67 /// \c kInvalidIndex.
69
70 /// If this \c UnwrappedLine opens a block, stores the index of the
71 /// line with the corresponding closing brace.
73
74 static const size_t kInvalidIndex = -1;
75
76 unsigned FirstStartColumn = 0;
77};
78
79/// Interface for users of the UnwrappedLineParser to receive the parsed lines.
80/// Parsing a single snippet of code can lead to multiple runs, where each
81/// run is a coherent view of the file.
82///
83/// For example, different runs are generated:
84/// - for different combinations of #if blocks
85/// - when macros are involved, for the expanded code and the as-written code
86///
87/// Some tokens will only be visible in a subset of the runs.
88/// For each run, \c UnwrappedLineParser will call \c consumeUnwrappedLine
89/// for each parsed unwrapped line, and then \c finishRun to indicate
90/// that the set of unwrapped lines before is one coherent view of the
91/// code snippet to be formatted.
93public:
95 virtual void consumeUnwrappedLine(const UnwrappedLine &Line) = 0;
96 virtual void finishRun() = 0;
97};
98
100
102public:
103 UnwrappedLineParser(SourceManager &SourceMgr, const FormatStyle &Style,
104 const AdditionalKeywords &Keywords,
105 unsigned FirstStartColumn, ArrayRef<FormatToken *> Tokens,
106 UnwrappedLineConsumer &Callback,
107 llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator,
108 IdentifierTable &IdentTable);
109
110 void parse();
111
112private:
113 enum class IfStmtKind {
114 NotIf, // Not an if statement.
115 IfOnly, // An if statement without the else clause.
116 IfElse, // An if statement followed by else but not else if.
117 IfElseIf // An if statement followed by else if.
118 };
119
120 void reset();
121 void parseFile();
122 bool precededByCommentOrPPDirective() const;
123 bool parseLevel(const FormatToken *OpeningBrace = nullptr,
124 IfStmtKind *IfKind = nullptr,
125 FormatToken **IfLeftBrace = nullptr);
126 bool mightFitOnOneLine(UnwrappedLine &Line,
127 const FormatToken *OpeningBrace = nullptr) const;
128 FormatToken *parseBlock(bool MustBeDeclaration = false,
129 unsigned AddLevels = 1u, bool MunchSemi = true,
130 bool KeepBraces = true, IfStmtKind *IfKind = nullptr,
131 bool UnindentWhitesmithsBraces = false);
132 void parseChildBlock();
133 void parsePPDirective();
134 void parsePPDefine();
135 void parsePPIf(bool IfDef);
136 void parsePPElse();
137 void parsePPEndIf();
138 void parsePPPragma();
139 void parsePPUnknown();
140 void readTokenWithJavaScriptASI();
141 void parseStructuralElement(const FormatToken *OpeningBrace = nullptr,
142 IfStmtKind *IfKind = nullptr,
143 FormatToken **IfLeftBrace = nullptr,
144 bool *HasDoWhile = nullptr,
145 bool *HasLabel = nullptr);
146 bool tryToParseBracedList();
147 bool parseBracedList(bool IsAngleBracket = false, bool IsEnum = false);
148 bool parseParens(TokenType AmpAmpTokenType = TT_Unknown);
149 void parseSquare(bool LambdaIntroducer = false);
150 void keepAncestorBraces();
151 void parseUnbracedBody(bool CheckEOF = false);
152 void handleAttributes();
153 bool handleCppAttributes();
154 bool isBlockBegin(const FormatToken &Tok) const;
155 FormatToken *parseIfThenElse(IfStmtKind *IfKind, bool KeepBraces = false,
156 bool IsVerilogAssert = false);
157 void parseTryCatch();
158 void parseLoopBody(bool KeepBraces, bool WrapRightBrace);
159 void parseForOrWhileLoop(bool HasParens = true);
160 void parseDoWhile();
161 void parseLabel(bool LeftAlignLabel = false);
162 void parseCaseLabel();
163 void parseSwitch(bool IsExpr);
164 void parseNamespace();
165 bool parseModuleImport();
166 void parseNew();
167 void parseAccessSpecifier();
168 bool parseEnum();
169 bool parseStructLike();
170 bool parseRequires();
171 void parseRequiresClause(FormatToken *RequiresToken);
172 void parseRequiresExpression(FormatToken *RequiresToken);
173 void parseConstraintExpression();
174 void parseJavaEnumBody();
175 // Parses a record (aka class) as a top level element. If ParseAsExpr is true,
176 // parses the record as a child block, i.e. if the class declaration is an
177 // expression.
178 void parseRecord(bool ParseAsExpr = false);
179 void parseObjCLightweightGenerics();
180 void parseObjCMethod();
181 void parseObjCProtocolList();
182 void parseObjCUntilAtEnd();
183 void parseObjCInterfaceOrImplementation();
184 bool parseObjCProtocol();
185 void parseJavaScriptEs6ImportExport();
186 void parseStatementMacro();
187 void parseCSharpAttribute();
188 // Parse a C# generic type constraint: `where T : IComparable<T>`.
189 // See:
190 // https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/keywords/where-generic-type-constraint
191 void parseCSharpGenericTypeConstraint();
192 bool tryToParseLambda();
193 bool tryToParseChildBlock();
194 bool tryToParseLambdaIntroducer();
195 bool tryToParsePropertyAccessor();
196 void tryToParseJSFunction();
197 bool tryToParseSimpleAttribute();
198 void parseVerilogHierarchyIdentifier();
199 void parseVerilogSensitivityList();
200 // Returns the number of levels of indentation in addition to the normal 1
201 // level for a block, used for indenting case labels.
202 unsigned parseVerilogHierarchyHeader();
203 void parseVerilogTable();
204 void parseVerilogCaseLabel();
205 std::optional<llvm::SmallVector<llvm::SmallVector<FormatToken *, 8>, 1>>
206 parseMacroCall();
207
208 // Used by addUnwrappedLine to denote whether to keep or remove a level
209 // when resetting the line state.
210 enum class LineLevel { Remove, Keep };
211
212 void addUnwrappedLine(LineLevel AdjustLevel = LineLevel::Remove);
213 bool eof() const;
214 // LevelDifference is the difference of levels after and before the current
215 // token. For example:
216 // - if the token is '{' and opens a block, LevelDifference is 1.
217 // - if the token is '}' and closes a block, LevelDifference is -1.
218 void nextToken(int LevelDifference = 0);
219 void readToken(int LevelDifference = 0);
220
221 // Decides which comment tokens should be added to the current line and which
222 // should be added as comments before the next token.
223 //
224 // Comments specifies the sequence of comment tokens to analyze. They get
225 // either pushed to the current line or added to the comments before the next
226 // token.
227 //
228 // NextTok specifies the next token. A null pointer NextTok is supported, and
229 // signifies either the absence of a next token, or that the next token
230 // shouldn't be taken into account for the analysis.
231 void distributeComments(const SmallVectorImpl<FormatToken *> &Comments,
232 const FormatToken *NextTok);
233
234 // Adds the comment preceding the next token to unwrapped lines.
235 void flushComments(bool NewlineBeforeNext);
236 void pushToken(FormatToken *Tok);
237 void calculateBraceTypes(bool ExpectClassBody = false);
238 void setPreviousRBraceType(TokenType Type);
239
240 // Marks a conditional compilation edge (for example, an '#if', '#ifdef',
241 // '#else' or merge conflict marker). If 'Unreachable' is true, assumes
242 // this branch either cannot be taken (for example '#if false'), or should
243 // not be taken in this round.
244 void conditionalCompilationCondition(bool Unreachable);
245 void conditionalCompilationStart(bool Unreachable);
246 void conditionalCompilationAlternative();
247 void conditionalCompilationEnd();
248
249 bool isOnNewLine(const FormatToken &FormatTok);
250
251 // Returns whether there is a macro expansion in the line, i.e. a token that
252 // was expanded from a macro call.
253 bool containsExpansion(const UnwrappedLine &Line) const;
254
255 // Compute hash of the current preprocessor branch.
256 // This is used to identify the different branches, and thus track if block
257 // open and close in the same branch.
258 size_t computePPHash() const;
259
260 bool parsingPPDirective() const { return CurrentLines != &Lines; }
261
262 // FIXME: We are constantly running into bugs where Line.Level is incorrectly
263 // subtracted from beyond 0. Introduce a method to subtract from Line.Level
264 // and use that everywhere in the Parser.
265 std::unique_ptr<UnwrappedLine> Line;
266
267 // Lines that are created by macro expansion.
268 // When formatting code containing macro calls, we first format the expanded
269 // lines to set the token types correctly. Afterwards, we format the
270 // reconstructed macro calls, re-using the token types determined in the first
271 // step.
272 // ExpandedLines will be reset every time we create a new LineAndExpansion
273 // instance once a line containing macro calls has been parsed.
274 SmallVector<UnwrappedLine, 8> CurrentExpandedLines;
275
276 // Maps from the first token of a top-level UnwrappedLine that contains
277 // a macro call to the replacement UnwrappedLines expanded from the macro
278 // call.
279 llvm::DenseMap<FormatToken *, SmallVector<UnwrappedLine, 8>> ExpandedLines;
280
281 // Map from the macro identifier to a line containing the full unexpanded
282 // macro call.
283 llvm::DenseMap<FormatToken *, std::unique_ptr<UnwrappedLine>> Unexpanded;
284
285 // For recursive macro expansions, trigger reconstruction only on the
286 // outermost expansion.
287 bool InExpansion = false;
288
289 // Set while we reconstruct a macro call.
290 // For reconstruction, we feed the expanded lines into the reconstructor
291 // until it is finished.
292 std::optional<MacroCallReconstructor> Reconstruct;
293
294 // Comments are sorted into unwrapped lines by whether they are in the same
295 // line as the previous token, or not. If not, they belong to the next token.
296 // Since the next token might already be in a new unwrapped line, we need to
297 // store the comments belonging to that token.
298 SmallVector<FormatToken *, 1> CommentsBeforeNextToken;
299 FormatToken *FormatTok = nullptr;
300 bool MustBreakBeforeNextToken;
301
302 // The parsed lines. Only added to through \c CurrentLines.
304
305 // Preprocessor directives are parsed out-of-order from other unwrapped lines.
306 // Thus, we need to keep a list of preprocessor directives to be reported
307 // after an unwrapped line that has been started was finished.
308 SmallVector<UnwrappedLine, 4> PreprocessorDirectives;
309
310 // New unwrapped lines are added via CurrentLines.
311 // Usually points to \c &Lines. While parsing a preprocessor directive when
312 // there is an unfinished previous unwrapped line, will point to
313 // \c &PreprocessorDirectives.
314 SmallVectorImpl<UnwrappedLine> *CurrentLines;
315
316 // We store for each line whether it must be a declaration depending on
317 // whether we are in a compound statement or not.
318 llvm::BitVector DeclarationScopeStack;
319
320 const FormatStyle &Style;
321 bool IsCpp;
322 LangOptions LangOpts;
323 const AdditionalKeywords &Keywords;
324
325 llvm::Regex CommentPragmasRegex;
326
327 FormatTokenSource *Tokens;
328 UnwrappedLineConsumer &Callback;
329
330 ArrayRef<FormatToken *> AllTokens;
331
332 // Keeps a stack of the states of nested control statements (true if the
333 // statement contains more than some predefined number of nested statements).
334 SmallVector<bool, 8> NestedTooDeep;
335
336 // Keeps a stack of the states of nested lambdas (true if the return type of
337 // the lambda is `decltype(auto)`).
338 SmallVector<bool, 4> NestedLambdas;
339
340 // Whether the parser is parsing the body of a function whose return type is
341 // `decltype(auto)`.
342 bool IsDecltypeAutoFunction = false;
343
344 // Represents preprocessor branch type, so we can find matching
345 // #if/#else/#endif directives.
346 enum PPBranchKind {
347 PP_Conditional, // Any #if, #ifdef, #ifndef, #elif, block outside #if 0
348 PP_Unreachable // #if 0 or a conditional preprocessor block inside #if 0
349 };
350
351 struct PPBranch {
352 PPBranch(PPBranchKind Kind, size_t Line) : Kind(Kind), Line(Line) {}
353 PPBranchKind Kind;
354 size_t Line;
355 };
356
357 // Keeps a stack of currently active preprocessor branching directives.
359
360 // The \c UnwrappedLineParser re-parses the code for each combination
361 // of preprocessor branches that can be taken.
362 // To that end, we take the same branch (#if, #else, or one of the #elif
363 // branches) for each nesting level of preprocessor branches.
364 // \c PPBranchLevel stores the current nesting level of preprocessor
365 // branches during one pass over the code.
366 int PPBranchLevel;
367
368 // Contains the current branch (#if, #else or one of the #elif branches)
369 // for each nesting level.
370 SmallVector<int, 8> PPLevelBranchIndex;
371
372 // Contains the maximum number of branches at each nesting level.
373 SmallVector<int, 8> PPLevelBranchCount;
374
375 // Contains the number of branches per nesting level we are currently
376 // in while parsing a preprocessor branch sequence.
377 // This is used to update PPLevelBranchCount at the end of a branch
378 // sequence.
379 std::stack<int> PPChainBranchIndex;
380
381 // Include guard search state. Used to fixup preprocessor indent levels
382 // so that include guards do not participate in indentation.
383 enum IncludeGuardState {
384 IG_Inited, // Search started, looking for #ifndef.
385 IG_IfNdefed, // #ifndef found, IncludeGuardToken points to condition.
386 IG_Defined, // Matching #define found, checking other requirements.
387 IG_Found, // All requirements met, need to fix indents.
388 IG_Rejected, // Search failed or never started.
389 };
390
391 // Current state of include guard search.
392 IncludeGuardState IncludeGuard;
393
394 // Points to the #ifndef condition for a potential include guard. Null unless
395 // IncludeGuardState == IG_IfNdefed.
396 FormatToken *IncludeGuardToken;
397
398 // Contains the first start column where the source begins. This is zero for
399 // normal source code and may be nonzero when formatting a code fragment that
400 // does not start at the beginning of the file.
401 unsigned FirstStartColumn;
402
403 MacroExpander Macros;
404
405 friend class ScopedLineState;
407};
408
410 UnwrappedLineNode() : Tok(nullptr) {}
413 : Tok(Tok), Children(Children.begin(), Children.end()) {}
414
417};
418
419std::ostream &operator<<(std::ostream &Stream, const UnwrappedLine &Line);
420
421} // end namespace format
422} // end namespace clang
423
424#endif
This file contains the main building blocks of macro support in clang-format.
Implements an efficient mapping from strings to IdentifierInfo nodes.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:461
This class handles loading and caching of source files into memory.
The base class of the type hierarchy.
Definition: Type.h:1826
Takes a set of macro definitions as strings and allows expanding calls to those macros.
Definition: Macros.h:80
Interface for users of the UnwrappedLineParser to receive the parsed lines.
virtual void consumeUnwrappedLine(const UnwrappedLine &Line)=0
std::ostream & operator<<(std::ostream &Stream, const UnwrappedLine &Line)
TokenType
Determines the semantic type of a syntactic token, e.g.
Definition: FormatToken.h:206
The JSON file list parser is used to communicate input to InstallAPI.
Represents a complete lambda introducer.
Definition: DeclSpec.h:2840
Encapsulates keywords that are context sensitive or for languages not properly supported by Clang's l...
Definition: FormatToken.h:1023
The FormatStyle is used to configure the formatting to follow specific guidelines.
Definition: Format.h:55
A wrapper around a Token storing information about the whitespace characters preceding it.
Definition: FormatToken.h:291
SmallVector< UnwrappedLine, 0 > Children
UnwrappedLineNode(FormatToken *Tok, llvm::ArrayRef< UnwrappedLine > Children={})
An unwrapped line is a sequence of Token, that we would like to put on a single line if there was no ...
unsigned PPLevel
The PPBranchLevel (adjusted for header guards) if this line is a InMacroBody line,...
bool InMacroBody
Whether it is part of a macro body.
std::list< UnwrappedLineNode > Tokens
The Tokens comprising this UnwrappedLine.
bool IsContinuation
True if this line should be indented by ContinuationIndent in addition to the normal indention level.
unsigned Level
The indent level of the UnwrappedLine.
unsigned UnbracedBodyLevel
Nesting level of unbraced body of a control statement.
bool InPragmaDirective
Whether this UnwrappedLine is part of a pramga directive.
bool InPPDirective
Whether this UnwrappedLine is part of a preprocessor directive.
bool SeenDecltypeAuto
Whether the parser has seen decltype(auto) in this line.
size_t MatchingClosingBlockLineIndex
If this UnwrappedLine opens a block, stores the index of the line with the corresponding closing brac...
static const size_t kInvalidIndex
size_t MatchingOpeningBlockLineIndex
If this UnwrappedLine closes a block in a sequence of lines, MatchingOpeningBlockLineIndex stores the...