clang  8.0.0svn
FormatTokenLexer.h
Go to the documentation of this file.
1 //===--- FormatTokenLexer.h - Format C++ code ----------------*- C++ ----*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// This file contains FormatTokenLexer, which tokenizes a source file
12 /// into a token stream suitable for ClangFormat.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #ifndef LLVM_CLANG_LIB_FORMAT_FORMATTOKENLEXER_H
17 #define LLVM_CLANG_LIB_FORMAT_FORMATTOKENLEXER_H
18 
19 #include "Encoding.h"
20 #include "FormatToken.h"
23 #include "clang/Format/Format.h"
24 #include "llvm/Support/Regex.h"
25 #include "llvm/ADT/MapVector.h"
26 
27 #include <stack>
28 
29 namespace clang {
30 namespace format {
31 
32 enum LexerState {
36 };
37 
39 public:
40  FormatTokenLexer(const SourceManager &SourceMgr, FileID ID, unsigned Column,
41  const FormatStyle &Style, encoding::Encoding Encoding);
42 
44 
45  const AdditionalKeywords &getKeywords() { return Keywords; }
46 
47 private:
48  void tryMergePreviousTokens();
49 
50  bool tryMergeLessLess();
51  bool tryMergeNSStringLiteral();
52 
53  bool tryMergeTokens(ArrayRef<tok::TokenKind> Kinds, TokenType NewType);
54 
55  // Returns \c true if \p Tok can only be followed by an operand in JavaScript.
56  bool precedesOperand(FormatToken *Tok);
57 
58  bool canPrecedeRegexLiteral(FormatToken *Prev);
59 
60  // Tries to parse a JavaScript Regex literal starting at the current token,
61  // if that begins with a slash and is in a location where JavaScript allows
62  // regex literals. Changes the current token to a regex literal and updates
63  // its text if successful.
64  void tryParseJSRegexLiteral();
65 
66  // Handles JavaScript template strings.
67  //
68  // JavaScript template strings use backticks ('`') as delimiters, and allow
69  // embedding expressions nested in ${expr-here}. Template strings can be
70  // nested recursively, i.e. expressions can contain template strings in turn.
71  //
72  // The code below parses starting from a backtick, up to a closing backtick or
73  // an opening ${. It also maintains a stack of lexing contexts to handle
74  // nested template parts by balancing curly braces.
75  void handleTemplateStrings();
76 
77  void tryParsePythonComment();
78 
79  bool tryMerge_TMacro();
80 
81  bool tryMergeConflictMarkers();
82 
83  FormatToken *getStashedToken();
84 
85  FormatToken *getNextToken();
86 
87  FormatToken *FormatTok;
88  bool IsFirstToken;
89  std::stack<LexerState> StateStack;
90  unsigned Column;
91  unsigned TrailingWhitespace;
92  std::unique_ptr<Lexer> Lex;
93  const SourceManager &SourceMgr;
94  FileID ID;
95  const FormatStyle &Style;
96  IdentifierTable IdentTable;
97  AdditionalKeywords Keywords;
98  encoding::Encoding Encoding;
99  llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
100  // Index (in 'Tokens') of the last token that starts a new line.
101  unsigned FirstInLineIndex;
103 
104  llvm::SmallMapVector<IdentifierInfo *, TokenType, 8> Macros;
105 
106  bool FormattingDisabled;
107 
108  llvm::Regex MacroBlockBeginRegex;
109  llvm::Regex MacroBlockEndRegex;
110 
111  void readRawToken(FormatToken &Tok);
112 
113  void resetLexer(unsigned Offset);
114 };
115 
116 } // namespace format
117 } // namespace clang
118 
119 #endif
Defines the SourceManager interface.
const AdditionalKeywords & getKeywords()
Contains functions for text encoding manipulation.
const FormatToken & Tok
unsigned Offset
Definition: Format.cpp:1631
FormatTokenLexer(const SourceManager &SourceMgr, FileID ID, unsigned Column, const FormatStyle &Style, encoding::Encoding Encoding)
Implements an efficient mapping from strings to IdentifierInfo nodes.
A wrapper around a Token storing information about the whitespace characters preceding it...
Definition: FormatToken.h:124
Various functions to configurably format source code.
Encapsulates keywords that are context sensitive or for languages not properly supported by Clang&#39;s l...
Definition: FormatToken.h:668
The FormatStyle is used to configure the formatting to follow specific guidelines.
Definition: Format.h:50
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Dataflow Directional Tag Classes.
Defines the clang::SourceLocation class and associated facilities.
This file contains the declaration of the FormatToken, a wrapper around Token with additional informa...
This class handles loading and caching of source files into memory.
ArrayRef< FormatToken * > lex()