clang 17.0.0git
FormatTokenLexer.h
Go to the documentation of this file.
1//===--- FormatTokenLexer.h - Format C++ code ----------------*- C++ ----*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file contains FormatTokenLexer, which tokenizes a source file
11/// into a token stream suitable for ClangFormat.
12///
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_CLANG_LIB_FORMAT_FORMATTOKENLEXER_H
16#define LLVM_CLANG_LIB_FORMAT_FORMATTOKENLEXER_H
17
18#include "Encoding.h"
19#include "FormatToken.h"
23#include "clang/Format/Format.h"
24#include "llvm/ADT/MapVector.h"
25#include "llvm/ADT/StringSet.h"
26#include "llvm/Support/Regex.h"
27
28#include <stack>
29
30namespace clang {
31namespace format {
32
37};
38
40public:
41 FormatTokenLexer(const SourceManager &SourceMgr, FileID ID, unsigned Column,
42 const FormatStyle &Style, encoding::Encoding Encoding,
43 llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator,
44 IdentifierTable &IdentTable);
45
47
48 const AdditionalKeywords &getKeywords() { return Keywords; }
49
50private:
51 void tryMergePreviousTokens();
52
53 bool tryMergeLessLess();
54 bool tryMergeGreaterGreater();
55 bool tryMergeNSStringLiteral();
56 bool tryMergeJSPrivateIdentifier();
57 bool tryMergeCSharpStringLiteral();
58 bool tryMergeCSharpKeywordVariables();
59 bool tryMergeNullishCoalescingEqual();
60 bool tryTransformCSharpForEach();
61 bool tryMergeForEach();
62 bool tryTransformTryUsageForC();
63
64 // Merge the most recently lexed tokens into a single token if their kinds are
65 // correct.
66 bool tryMergeTokens(ArrayRef<tok::TokenKind> Kinds, TokenType NewType);
67 // Merge without checking their kinds.
68 bool tryMergeTokens(size_t Count, TokenType NewType);
69 // Merge if their kinds match any one of Kinds.
70 bool tryMergeTokensAny(ArrayRef<ArrayRef<tok::TokenKind>> Kinds,
71 TokenType NewType);
72
73 // Returns \c true if \p Tok can only be followed by an operand in JavaScript.
74 bool precedesOperand(FormatToken *Tok);
75
76 bool canPrecedeRegexLiteral(FormatToken *Prev);
77
78 // Tries to parse a JavaScript Regex literal starting at the current token,
79 // if that begins with a slash and is in a location where JavaScript allows
80 // regex literals. Changes the current token to a regex literal and updates
81 // its text if successful.
82 void tryParseJSRegexLiteral();
83
84 // Handles JavaScript template strings.
85 //
86 // JavaScript template strings use backticks ('`') as delimiters, and allow
87 // embedding expressions nested in ${expr-here}. Template strings can be
88 // nested recursively, i.e. expressions can contain template strings in turn.
89 //
90 // The code below parses starting from a backtick, up to a closing backtick or
91 // an opening ${. It also maintains a stack of lexing contexts to handle
92 // nested template parts by balancing curly braces.
93 void handleTemplateStrings();
94
95 void handleCSharpVerbatimAndInterpolatedStrings();
96
97 void tryParsePythonComment();
98
99 bool tryMerge_TMacro();
100
101 bool tryMergeConflictMarkers();
102
103 void truncateToken(size_t NewLen);
104
105 FormatToken *getStashedToken();
106
107 FormatToken *getNextToken();
108
109 FormatToken *FormatTok;
110 bool IsFirstToken;
111 std::stack<LexerState> StateStack;
112 unsigned Column;
113 unsigned TrailingWhitespace;
114 std::unique_ptr<Lexer> Lex;
115 LangOptions LangOpts;
116 const SourceManager &SourceMgr;
117 FileID ID;
118 const FormatStyle &Style;
119 IdentifierTable &IdentTable;
120 AdditionalKeywords Keywords;
121 encoding::Encoding Encoding;
122 llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator;
123 // Index (in 'Tokens') of the last token that starts a new line.
124 unsigned FirstInLineIndex;
126
127 llvm::SmallMapVector<IdentifierInfo *, TokenType, 8> Macros;
128
129 bool FormattingDisabled;
130
131 llvm::Regex MacroBlockBeginRegex;
132 llvm::Regex MacroBlockEndRegex;
133
134 // Targets that may appear inside a C# attribute.
135 static const llvm::StringSet<> CSharpAttributeTargets;
136
137 /// Handle Verilog-specific tokens.
138 bool readRawTokenVerilogSpecific(Token &Tok);
139
140 void readRawToken(FormatToken &Tok);
141
142 void resetLexer(unsigned Offset);
143};
144
145} // namespace format
146} // namespace clang
147
148#endif
Contains functions for text encoding manipulation.
This file contains the declaration of the FormatToken, a wrapper around Token with additional informa...
unsigned Offset
Definition: Format.cpp:2797
Various functions to configurably format source code.
Defines the clang::LangOptions interface.
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Implements an efficient mapping from strings to IdentifierInfo nodes.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:82
This class handles loading and caching of source files into memory.
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
const AdditionalKeywords & getKeywords()
ArrayRef< FormatToken * > lex()
TokenType
Determines the semantic type of a syntactic token, e.g.
Definition: FormatToken.h:171
Encapsulates keywords that are context sensitive or for languages not properly supported by Clang's l...
Definition: FormatToken.h:942
The FormatStyle is used to configure the formatting to follow specific guidelines.
Definition: Format.h:55
A wrapper around a Token storing information about the whitespace characters preceding it.
Definition: FormatToken.h:256