clang  6.0.0svn
FormatTokenLexer.h
Go to the documentation of this file.
1 //===--- FormatTokenLexer.h - Format C++ code ----------------*- C++ ----*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains FormatTokenLexer, which tokenizes a source file
12 /// into a token stream suitable for ClangFormat.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #ifndef LLVM_CLANG_LIB_FORMAT_FORMATTOKENLEXER_H
17 #define LLVM_CLANG_LIB_FORMAT_FORMATTOKENLEXER_H
18 
19 #include "Encoding.h"
20 #include "FormatToken.h"
23 #include "clang/Format/Format.h"
24 #include "llvm/Support/Regex.h"
25 
26 #include <stack>
27 
28 namespace clang {
29 namespace format {
30 
31 enum LexerState {
35 };
36 
38 public:
39  FormatTokenLexer(const SourceManager &SourceMgr, FileID ID,
40  const FormatStyle &Style, encoding::Encoding Encoding);
41 
43 
44  const AdditionalKeywords &getKeywords() { return Keywords; }
45 
46 private:
47  void tryMergePreviousTokens();
48 
49  bool tryMergeLessLess();
50  bool tryMergeNSStringLiteral();
51 
52  bool tryMergeTokens(ArrayRef<tok::TokenKind> Kinds, TokenType NewType);
53 
54  // Returns \c true if \p Tok can only be followed by an operand in JavaScript.
55  bool precedesOperand(FormatToken *Tok);
56 
57  bool canPrecedeRegexLiteral(FormatToken *Prev);
58 
59  // Tries to parse a JavaScript Regex literal starting at the current token,
60  // if that begins with a slash and is in a location where JavaScript allows
61  // regex literals. Changes the current token to a regex literal and updates
62  // its text if successful.
63  void tryParseJSRegexLiteral();
64 
65  // Handles JavaScript template strings.
66  //
67  // JavaScript template strings use backticks ('`') as delimiters, and allow
68  // embedding expressions nested in ${expr-here}. Template strings can be
69  // nested recursively, i.e. expressions can contain template strings in turn.
70  //
71  // The code below parses starting from a backtick, up to a closing backtick or
72  // an opening ${. It also maintains a stack of lexing contexts to handle
73  // nested template parts by balancing curly braces.
74  void handleTemplateStrings();
75 
76  bool tryMerge_TMacro();
77 
78  bool tryMergeConflictMarkers();
79 
80  FormatToken *getStashedToken();
81 
82  FormatToken *getNextToken();
83 
84  FormatToken *FormatTok;
85  bool IsFirstToken;
86  std::stack<LexerState> StateStack;
87  unsigned Column;
88  unsigned TrailingWhitespace;
89  std::unique_ptr<Lexer> Lex;
90  const SourceManager &SourceMgr;
91  FileID ID;
92  const FormatStyle &Style;
93  IdentifierTable IdentTable;
94  AdditionalKeywords Keywords;
95  encoding::Encoding Encoding;
96  llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
97  // Index (in 'Tokens') of the last token that starts a new line.
98  unsigned FirstInLineIndex;
100  SmallVector<IdentifierInfo *, 8> ForEachMacros;
101 
102  bool FormattingDisabled;
103 
104  llvm::Regex MacroBlockBeginRegex;
105  llvm::Regex MacroBlockEndRegex;
106 
107  void readRawToken(FormatToken &Tok);
108 
109  void resetLexer(unsigned Offset);
110 };
111 
112 } // namespace format
113 } // namespace clang
114 
115 #endif
Defines the SourceManager interface.
const AdditionalKeywords & getKeywords()
Contains functions for text encoding manipulation.
uint32_t Offset
Definition: CacheTokens.cpp:43
const FormatToken & Tok
Implements an efficient mapping from strings to IdentifierInfo nodes.
A wrapper around a Token storing information about the whitespace characters preceding it...
Definition: FormatToken.h:120
Various functions to configurably format source code.
Encapsulates keywords that are context sensitive or for languages not properly supported by Clang&#39;s l...
Definition: FormatToken.h:634
FormatTokenLexer(const SourceManager &SourceMgr, FileID ID, const FormatStyle &Style, encoding::Encoding Encoding)
The FormatStyle is used to configure the formatting to follow specific guidelines.
Definition: Format.h:46
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Dataflow Directional Tag Classes.
Defines the clang::SourceLocation class and associated facilities.
This file contains the declaration of the FormatToken, a wrapper around Token with additional informa...
This class handles loading and caching of source files into memory.
ArrayRef< FormatToken * > lex()