clang-tools  14.0.0git
SemanticHighlightingTests.cpp
Go to the documentation of this file.
1 //==- SemanticHighlightingTests.cpp - SemanticHighlighting tests-*- C++ -* -==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "Annotations.h"
10 #include "ClangdServer.h"
11 #include "Protocol.h"
12 #include "SemanticHighlighting.h"
13 #include "SourceCode.h"
14 #include "TestFS.h"
15 #include "TestTU.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/Support/Error.h"
20 #include "llvm/Support/ScopedPrinter.h"
21 #include "llvm/Support/raw_ostream.h"
22 #include "gmock/gmock.h"
23 #include <algorithm>
24 
25 namespace clang {
26 namespace clangd {
27 namespace {
28 
29 using testing::IsEmpty;
30 using testing::SizeIs;
31 
32 /// Annotates the input code with provided semantic highlightings. Results look
33 /// something like:
34 /// class $Class[[X]] {
35 /// $Primitive[[int]] $Field[[a]] = 0;
36 /// };
37 std::string annotate(llvm::StringRef Input,
38  llvm::ArrayRef<HighlightingToken> Tokens) {
39  assert(std::is_sorted(
40  Tokens.begin(), Tokens.end(),
41  [](const HighlightingToken &L, const HighlightingToken &R) {
42  return L.R.start < R.R.start;
43  }));
44 
45  std::string Buf;
46  llvm::raw_string_ostream OS(Buf);
47  unsigned NextChar = 0;
48  for (auto &T : Tokens) {
49  unsigned StartOffset = llvm::cantFail(positionToOffset(Input, T.R.start));
50  unsigned EndOffset = llvm::cantFail(positionToOffset(Input, T.R.end));
51  assert(StartOffset <= EndOffset);
52  assert(NextChar <= StartOffset);
53 
54  OS << Input.substr(NextChar, StartOffset - NextChar);
55  OS << '$' << T.Kind;
56  for (unsigned I = 0;
57  I <= static_cast<uint32_t>(HighlightingModifier::LastModifier); ++I) {
58  if (T.Modifiers & (1 << I))
59  OS << '_' << static_cast<HighlightingModifier>(I);
60  }
61  OS << "[[" << Input.substr(StartOffset, EndOffset - StartOffset) << "]]";
62  NextChar = EndOffset;
63  }
64  OS << Input.substr(NextChar);
65  return std::move(OS.str());
66 }
67 
68 void checkHighlightings(llvm::StringRef Code,
69  std::vector<std::pair</*FileName*/ llvm::StringRef,
70  /*FileContent*/ llvm::StringRef>>
71  AdditionalFiles = {},
72  uint32_t ModifierMask = -1,
73  std::vector<std::string> AdditionalArgs = {}) {
74  Annotations Test(Code);
75  TestTU TU;
76  TU.Code = std::string(Test.code());
77 
78  TU.ExtraArgs.push_back("-std=c++20");
79  TU.ExtraArgs.push_back("-xobjective-c++");
80  TU.ExtraArgs.insert(std::end(TU.ExtraArgs), std::begin(AdditionalArgs),
81  std::end(AdditionalArgs));
82 
83  for (auto File : AdditionalFiles)
84  TU.AdditionalFiles.insert({File.first, std::string(File.second)});
85  auto AST = TU.build();
86  auto Actual = getSemanticHighlightings(AST);
87  for (auto &Token : Actual)
88  Token.Modifiers &= ModifierMask;
89 
90  EXPECT_EQ(Code, annotate(Test.code(), Actual));
91 }
92 
93 constexpr static uint32_t ScopeModifierMask =
95  1 << unsigned(HighlightingModifier::ClassScope) |
96  1 << unsigned(HighlightingModifier::FileScope) |
97  1 << unsigned(HighlightingModifier::GlobalScope);
98 
99 TEST(SemanticHighlighting, GetsCorrectTokens) {
100  const char *TestCases[] = {
101  R"cpp(
102  struct $Class_decl[[AS]] {
103  double $Field_decl[[SomeMember]];
104  };
105  struct {
106  } $Variable_decl[[S]];
107  void $Function_decl[[foo]](int $Parameter_decl[[A]], $Class[[AS]] $Parameter_decl[[As]]) {
108  $Primitive_deduced_defaultLibrary[[auto]] $LocalVariable_decl[[VeryLongVariableName]] = 12312;
109  $Class[[AS]] $LocalVariable_decl[[AA]];
110  $Primitive_deduced_defaultLibrary[[auto]] $LocalVariable_decl[[L]] = $LocalVariable[[AA]].$Field[[SomeMember]] + $Parameter[[A]];
111  auto $LocalVariable_decl[[FN]] = [ $LocalVariable[[AA]]](int $Parameter_decl[[A]]) -> void {};
112  $LocalVariable[[FN]](12312);
113  }
114  )cpp",
115  R"cpp(
116  void $Function_decl[[foo]](int);
117  void $Function_decl[[Gah]]();
118  void $Function_decl[[foo]]() {
119  auto $LocalVariable_decl[[Bou]] = $Function[[Gah]];
120  }
121  struct $Class_decl[[A]] {
122  void $Method_decl[[abc]]();
123  };
124  )cpp",
125  R"cpp(
126  namespace $Namespace_decl[[abc]] {
127  template<typename $TemplateParameter_decl[[T]]>
128  struct $Class_decl[[A]] {
129  $TemplateParameter[[T]] $Field_decl[[t]];
130  };
131  }
132  template<typename $TemplateParameter_decl[[T]]>
133  struct $Class_decl[[C]] : $Namespace[[abc]]::$Class[[A]]<$TemplateParameter[[T]]> {
134  typename $TemplateParameter[[T]]::$Type_dependentName[[A]]* $Field_decl[[D]];
135  };
136  $Namespace[[abc]]::$Class[[A]]<int> $Variable_decl[[AA]];
137  typedef $Namespace[[abc]]::$Class[[A]]<int> $Class_decl[[AAA]];
138  struct $Class_decl[[B]] {
139  $Class_decl[[B]]();
140  ~$Class[[B]](); // FIXME: inconsistent with constructor
141  void operator<<($Class[[B]]);
142  $Class[[AAA]] $Field_decl[[AA]];
143  };
144  $Class[[B]]::$Class_decl[[B]]() {}
145  $Class[[B]]::~$Class[[B]]() {} // FIXME: inconsistent with constructor
146  void $Function_decl[[f]] () {
147  $Class[[B]] $LocalVariable_decl[[BB]] = $Class[[B]]();
148  $LocalVariable[[BB]].~$Class[[B]]();
149  $Class[[B]]();
150  }
151  )cpp",
152  R"cpp(
153  enum class $Enum_decl[[E]] {
154  $EnumConstant_decl_readonly[[A]],
155  $EnumConstant_decl_readonly[[B]],
156  };
157  enum $Enum_decl[[EE]] {
158  $EnumConstant_decl_readonly[[Hi]],
159  };
160  struct $Class_decl[[A]] {
161  $Enum[[E]] $Field_decl[[EEE]];
162  $Enum[[EE]] $Field_decl[[EEEE]];
163  };
164  int $Variable_decl[[I]] = $EnumConstant_readonly[[Hi]];
165  $Enum[[E]] $Variable_decl[[L]] = $Enum[[E]]::$EnumConstant_readonly[[B]];
166  )cpp",
167  R"cpp(
168  namespace $Namespace_decl[[abc]] {
169  namespace {}
170  namespace $Namespace_decl[[bcd]] {
171  struct $Class_decl[[A]] {};
172  namespace $Namespace_decl[[cde]] {
173  struct $Class_decl[[A]] {
174  enum class $Enum_decl[[B]] {
175  $EnumConstant_decl_readonly[[Hi]],
176  };
177  };
178  }
179  }
180  }
181  using namespace $Namespace[[abc]]::$Namespace[[bcd]];
182  namespace $Namespace_decl[[vwz]] =
183  $Namespace[[abc]]::$Namespace[[bcd]]::$Namespace[[cde]];
184  $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable_decl[[AA]];
185  $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable_decl[[AAA]] =
186  $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant_readonly[[Hi]];
187  ::$Namespace[[vwz]]::$Class[[A]] $Variable_decl[[B]];
188  ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable_decl[[BB]];
189  )cpp",
190  R"cpp(
191  struct $Class_decl[[D]] {
192  double $Field_decl[[C]];
193  };
194  struct $Class_decl[[A]] {
195  double $Field_decl[[B]];
196  $Class[[D]] $Field_decl[[E]];
197  static double $StaticField_decl_static[[S]];
198  static void $StaticMethod_decl_static[[bar]]() {}
199  void $Method_decl[[foo]]() {
200  $Field[[B]] = 123;
201  this->$Field[[B]] = 156;
202  this->$Method[[foo]]();
203  $Method[[foo]]();
204  $StaticMethod_static[[bar]]();
205  $StaticField_static[[S]] = 90.1;
206  }
207  };
208  void $Function_decl[[foo]]() {
209  $Class[[A]] $LocalVariable_decl[[AA]];
210  $LocalVariable[[AA]].$Field[[B]] += 2;
211  $LocalVariable[[AA]].$Method[[foo]]();
212  $LocalVariable[[AA]].$Field[[E]].$Field[[C]];
213  $Class[[A]]::$StaticField_static[[S]] = 90;
214  }
215  )cpp",
216  R"cpp(
217  struct $Class_decl[[AA]] {
218  int $Field_decl[[A]];
219  };
220  int $Variable_decl[[B]];
221  $Class[[AA]] $Variable_decl[[A]]{$Variable[[B]]};
222  )cpp",
223  R"cpp(
224  namespace $Namespace_decl[[a]] {
225  struct $Class_decl[[A]] {};
226  typedef char $Primitive_decl[[C]];
227  }
228  typedef $Namespace[[a]]::$Class[[A]] $Class_decl[[B]];
229  using $Class_decl[[BB]] = $Namespace[[a]]::$Class[[A]];
230  enum class $Enum_decl[[E]] {};
231  typedef $Enum[[E]] $Enum_decl[[C]];
232  typedef $Enum[[C]] $Enum_decl[[CC]];
233  using $Enum_decl[[CD]] = $Enum[[CC]];
234  $Enum[[CC]] $Function_decl[[f]]($Class[[B]]);
235  $Enum[[CD]] $Function_decl[[f]]($Class[[BB]]);
236  typedef $Namespace[[a]]::$Primitive[[C]] $Primitive_decl[[PC]];
237  typedef float $Primitive_decl[[F]];
238  )cpp",
239  R"cpp(
240  template<typename $TemplateParameter_decl[[T]], typename = void>
241  class $Class_decl[[A]] {
242  $TemplateParameter[[T]] $Field_decl[[AA]];
243  $TemplateParameter[[T]] $Method_decl[[foo]]();
244  };
245  template<class $TemplateParameter_decl[[TT]]>
246  class $Class_decl[[B]] {
247  $Class[[A]]<$TemplateParameter[[TT]]> $Field_decl[[AA]];
248  };
249  template<class $TemplateParameter_decl[[TT]], class $TemplateParameter_decl[[GG]]>
250  class $Class_decl[[BB]] {};
251  template<class $TemplateParameter_decl[[T]]>
252  class $Class_decl[[BB]]<$TemplateParameter[[T]], int> {};
253  template<class $TemplateParameter_decl[[T]]>
254  class $Class_decl[[BB]]<$TemplateParameter[[T]], $TemplateParameter[[T]]*> {};
255 
256  template<template<class> class $TemplateParameter_decl[[T]], class $TemplateParameter_decl[[C]]>
257  $TemplateParameter[[T]]<$TemplateParameter[[C]]> $Function_decl[[f]]();
258 
259  template<typename>
260  class $Class_decl[[Foo]] {};
261 
262  template<typename $TemplateParameter_decl[[T]]>
263  void $Function_decl[[foo]]($TemplateParameter[[T]] ...);
264  )cpp",
265  R"cpp(
266  template <class $TemplateParameter_decl[[T]]>
267  struct $Class_decl[[Tmpl]] {$TemplateParameter[[T]] $Field_decl[[x]] = 0;};
268  extern template struct $Class_decl[[Tmpl]]<float>;
269  template struct $Class_decl[[Tmpl]]<double>;
270  )cpp",
271  // This test is to guard against highlightings disappearing when using
272  // conversion operators as their behaviour in the clang AST differ from
273  // other CXXMethodDecls.
274  R"cpp(
275  class $Class_decl[[Foo]] {};
276  struct $Class_decl[[Bar]] {
277  explicit operator $Class[[Foo]]*() const;
278  explicit operator int() const;
279  operator $Class[[Foo]]();
280  };
281  void $Function_decl[[f]]() {
282  $Class[[Bar]] $LocalVariable_decl[[B]];
283  $Class[[Foo]] $LocalVariable_decl[[F]] = $LocalVariable[[B]];
284  $Class[[Foo]] *$LocalVariable_decl[[FP]] = ($Class[[Foo]]*)$LocalVariable[[B]];
285  int $LocalVariable_decl[[I]] = (int)$LocalVariable[[B]];
286  }
287  )cpp",
288  R"cpp(
289  struct $Class_decl[[B]] {};
290  struct $Class_decl[[A]] {
291  $Class[[B]] $Field_decl[[BB]];
292  $Class[[A]] &operator=($Class[[A]] &&$Parameter_decl[[O]]);
293  };
294 
295  $Class[[A]] &$Class[[A]]::operator=($Class[[A]] &&$Parameter_decl[[O]]) = default;
296  )cpp",
297  R"cpp(
298  enum $Enum_decl[[En]] {
299  $EnumConstant_decl_readonly[[EC]],
300  };
301  class $Class_decl[[Foo]] {};
302  class $Class_decl[[Bar]] {
303  public:
304  $Class[[Foo]] $Field_decl[[Fo]];
305  $Enum[[En]] $Field_decl[[E]];
306  int $Field_decl[[I]];
307  $Class_decl[[Bar]] ($Class[[Foo]] $Parameter_decl[[F]],
308  $Enum[[En]] $Parameter_decl[[E]])
309  : $Field[[Fo]] ($Parameter[[F]]), $Field[[E]] ($Parameter[[E]]),
310  $Field[[I]] (123) {}
311  };
312  class $Class_decl[[Bar2]] : public $Class[[Bar]] {
313  $Class_decl[[Bar2]]() : $Class[[Bar]]($Class[[Foo]](), $EnumConstant_readonly[[EC]]) {}
314  };
315  )cpp",
316  R"cpp(
317  enum $Enum_decl[[E]] {
318  $EnumConstant_decl_readonly[[E]],
319  };
320  class $Class_decl[[Foo]] {};
321  $Enum_deduced[[auto]] $Variable_decl[[AE]] = $Enum[[E]]::$EnumConstant_readonly[[E]];
322  $Class_deduced[[auto]] $Variable_decl[[AF]] = $Class[[Foo]]();
323  $Class_deduced[[decltype]](auto) $Variable_decl[[AF2]] = $Class[[Foo]]();
324  $Class_deduced[[auto]] *$Variable_decl[[AFP]] = &$Variable[[AF]];
325  $Enum_deduced[[auto]] &$Variable_decl[[AER]] = $Variable[[AE]];
326  $Primitive_deduced_defaultLibrary[[auto]] $Variable_decl[[Form]] = 10.2 + 2 * 4;
327  $Primitive_deduced_defaultLibrary[[decltype]]($Variable[[Form]]) $Variable_decl[[F]] = 10;
328  auto $Variable_decl[[Fun]] = []()->void{};
329  )cpp",
330  R"cpp(
331  class $Class_decl[[G]] {};
332  template<$Class[[G]] *$TemplateParameter_decl_readonly[[U]]>
333  class $Class_decl[[GP]] {};
334  template<$Class[[G]] &$TemplateParameter_decl_readonly[[U]]>
335  class $Class_decl[[GR]] {};
336  template<int *$TemplateParameter_decl_readonly[[U]]>
337  class $Class_decl[[IP]] {
338  void $Method_decl[[f]]() {
339  *$TemplateParameter_readonly[[U]] += 5;
340  }
341  };
342  template<unsigned $TemplateParameter_decl_readonly[[U]] = 2>
343  class $Class_decl[[Foo]] {
344  void $Method_decl[[f]]() {
345  for(int $LocalVariable_decl[[I]] = 0;
346  $LocalVariable[[I]] < $TemplateParameter_readonly[[U]];) {}
347  }
348  };
349 
350  $Class[[G]] $Variable_decl[[L]];
351  void $Function_decl[[f]]() {
352  $Class[[Foo]]<123> $LocalVariable_decl[[F]];
353  $Class[[GP]]<&$Variable[[L]]> $LocalVariable_decl[[LL]];
354  $Class[[GR]]<$Variable[[L]]> $LocalVariable_decl[[LLL]];
355  }
356  )cpp",
357  R"cpp(
358  template<typename $TemplateParameter_decl[[T]],
359  void ($TemplateParameter[[T]]::*$TemplateParameter_decl_readonly[[method]])(int)>
360  struct $Class_decl[[G]] {
361  void $Method_decl[[foo]](
362  $TemplateParameter[[T]] *$Parameter_decl[[O]]) {
363  ($Parameter[[O]]->*$TemplateParameter_readonly[[method]])(10);
364  }
365  };
366  struct $Class_decl[[F]] {
367  void $Method_decl[[f]](int);
368  };
369  template<void (*$TemplateParameter_decl_readonly[[Func]])()>
370  struct $Class_decl[[A]] {
371  void $Method_decl[[f]]() {
372  (*$TemplateParameter_readonly[[Func]])();
373  }
374  };
375 
376  void $Function_decl[[foo]]() {
377  $Class[[F]] $LocalVariable_decl[[FF]];
378  $Class[[G]]<$Class[[F]], &$Class[[F]]::$Method[[f]]> $LocalVariable_decl[[GG]];
379  $LocalVariable[[GG]].$Method[[foo]](&$LocalVariable[[FF]]);
380  $Class[[A]]<$Function[[foo]]> $LocalVariable_decl[[AA]];
381  }
382  )cpp",
383  // Tokens that share a source range but have conflicting Kinds are not
384  // highlighted.
385  R"cpp(
386  #define $Macro_decl[[DEF_MULTIPLE]](X) namespace X { class X { int X; }; }
387  #define $Macro_decl[[DEF_CLASS]](T) class T {};
388  // Preamble ends.
389  $Macro[[DEF_MULTIPLE]](XYZ);
390  $Macro[[DEF_MULTIPLE]](XYZW);
391  $Macro[[DEF_CLASS]]($Class_decl[[A]])
392  #define $Macro_decl[[MACRO_CONCAT]](X, V, T) T foo##X = V
393  #define $Macro_decl[[DEF_VAR]](X, V) int X = V
394  #define $Macro_decl[[DEF_VAR_T]](T, X, V) T X = V
395  #define $Macro_decl[[DEF_VAR_REV]](V, X) DEF_VAR(X, V)
396  #define $Macro_decl[[CPY]](X) X
397  #define $Macro_decl[[DEF_VAR_TYPE]](X, Y) X Y
398  #define $Macro_decl[[SOME_NAME]] variable
399  #define $Macro_decl[[SOME_NAME_SET]] variable2 = 123
400  #define $Macro_decl[[INC_VAR]](X) X += 2
401  void $Function_decl[[foo]]() {
402  $Macro[[DEF_VAR]]($LocalVariable_decl[[X]], 123);
403  $Macro[[DEF_VAR_REV]](908, $LocalVariable_decl[[XY]]);
404  int $Macro[[CPY]]( $LocalVariable_decl[[XX]] );
405  $Macro[[DEF_VAR_TYPE]]($Class[[A]], $LocalVariable_decl[[AA]]);
406  double $Macro[[SOME_NAME]];
407  int $Macro[[SOME_NAME_SET]];
408  $LocalVariable[[variable]] = 20.1;
409  $Macro[[MACRO_CONCAT]](var, 2, float);
410  $Macro[[DEF_VAR_T]]($Class[[A]], $Macro[[CPY]](
411  $Macro[[CPY]]($LocalVariable_decl[[Nested]])),
412  $Macro[[CPY]]($Class[[A]]()));
413  $Macro[[INC_VAR]]($LocalVariable[[variable]]);
414  }
415  void $Macro[[SOME_NAME]]();
416  $Macro[[DEF_VAR]]($Variable_decl[[MMMMM]], 567);
417  $Macro[[DEF_VAR_REV]](756, $Variable_decl[[AB]]);
418 
419  #define $Macro_decl[[CALL_FN]](F) F();
420  #define $Macro_decl[[DEF_FN]](F) void F ()
421  $Macro[[DEF_FN]]($Function_decl[[g]]) {
422  $Macro[[CALL_FN]]($Function[[foo]]);
423  }
424  )cpp",
425  R"cpp(
426  #define $Macro_decl[[fail]](expr) expr
427  #define $Macro_decl[[assert]](COND) if (!(COND)) { fail("assertion failed" #COND); }
428  // Preamble ends.
429  int $Variable_decl[[x]];
430  int $Variable_decl[[y]];
431  int $Function_decl[[f]]();
432  void $Function_decl[[foo]]() {
433  $Macro[[assert]]($Variable[[x]] != $Variable[[y]]);
434  $Macro[[assert]]($Variable[[x]] != $Function[[f]]());
435  }
436  )cpp",
437  // highlighting all macro references
438  R"cpp(
439  #ifndef $Macro[[name]]
440  #define $Macro_decl[[name]]
441  #endif
442 
443  #define $Macro_decl[[test]]
444  #undef $Macro[[test]]
445 $InactiveCode[[#ifdef test]]
446 $InactiveCode[[#endif]]
447 
448 $InactiveCode[[#if defined(test)]]
449 $InactiveCode[[#endif]]
450  )cpp",
451  R"cpp(
452  struct $Class_decl[[S]] {
453  float $Field_decl[[Value]];
454  $Class[[S]] *$Field_decl[[Next]];
455  };
456  $Class[[S]] $Variable_decl[[Global]][2] = {$Class[[S]](), $Class[[S]]()};
457  auto [$Variable_decl[[G1]], $Variable_decl[[G2]]] = $Variable[[Global]];
458  void $Function_decl[[f]]($Class[[S]] $Parameter_decl[[P]]) {
459  int $LocalVariable_decl[[A]][2] = {1,2};
460  auto [$LocalVariable_decl[[B1]], $LocalVariable_decl[[B2]]] = $LocalVariable[[A]];
461  auto [$LocalVariable_decl[[G1]], $LocalVariable_decl[[G2]]] = $Variable[[Global]];
462  $Class_deduced[[auto]] [$LocalVariable_decl[[P1]], $LocalVariable_decl[[P2]]] = $Parameter[[P]];
463  // Highlights references to BindingDecls.
464  $LocalVariable[[B1]]++;
465  }
466  )cpp",
467  R"cpp(
468  template<class $TemplateParameter_decl[[T]]>
469  class $Class_decl[[A]] {
470  using $TemplateParameter_decl[[TemplateParam1]] = $TemplateParameter[[T]];
471  typedef $TemplateParameter[[T]] $TemplateParameter_decl[[TemplateParam2]];
472  using $Primitive_decl[[IntType]] = int;
473 
474  using $Typedef_decl[[Pointer]] = $TemplateParameter[[T]] *;
475  using $Typedef_decl[[LVReference]] = $TemplateParameter[[T]] &;
476  using $Typedef_decl[[RVReference]] = $TemplateParameter[[T]]&&;
477  using $Typedef_decl[[Array]] = $TemplateParameter[[T]]*[3];
478  using $Typedef_decl[[MemberPointer]] = int ($Class[[A]]::*)(int);
479 
480  // Use various previously defined typedefs in a function type.
481  void $Method_decl[[func]](
482  $Typedef[[Pointer]], $Typedef[[LVReference]], $Typedef[[RVReference]],
483  $Typedef[[Array]], $Typedef[[MemberPointer]]);
484  };
485  )cpp",
486  R"cpp(
487  template <class $TemplateParameter_decl[[T]]>
488  void $Function_decl[[phase1]]($TemplateParameter[[T]]);
489  template <class $TemplateParameter_decl[[T]]>
490  void $Function_decl[[foo]]($TemplateParameter[[T]] $Parameter_decl[[P]]) {
491  $Function[[phase1]]($Parameter[[P]]);
492  $Unknown_dependentName[[phase2]]($Parameter[[P]]);
493  }
494  )cpp",
495  R"cpp(
496  class $Class_decl[[A]] {
497  template <class $TemplateParameter_decl[[T]]>
498  void $Method_decl[[bar]]($TemplateParameter[[T]]);
499  };
500 
501  template <class $TemplateParameter_decl[[U]]>
502  void $Function_decl[[foo]]($TemplateParameter[[U]] $Parameter_decl[[P]]) {
503  $Class[[A]]().$Method[[bar]]($Parameter[[P]]);
504  }
505  )cpp",
506  R"cpp(
507  struct $Class_decl[[A]] {
508  template <class $TemplateParameter_decl[[T]]>
509  static void $StaticMethod_decl_static[[foo]]($TemplateParameter[[T]]);
510  };
511 
512  template <class $TemplateParameter_decl[[T]]>
513  struct $Class_decl[[B]] {
514  void $Method_decl[[bar]]() {
515  $Class[[A]]::$StaticMethod_static[[foo]]($TemplateParameter[[T]]());
516  }
517  };
518  )cpp",
519  R"cpp(
520  template <class $TemplateParameter_decl[[T]]>
521  void $Function_decl[[foo]](typename $TemplateParameter[[T]]::$Type_dependentName[[Type]]
522  = $TemplateParameter[[T]]::$Unknown_dependentName[[val]]);
523  )cpp",
524  R"cpp(
525  template <class $TemplateParameter_decl[[T]]>
526  void $Function_decl[[foo]]($TemplateParameter[[T]] $Parameter_decl[[P]]) {
527  $Parameter[[P]].$Unknown_dependentName[[Field]];
528  }
529  )cpp",
530  R"cpp(
531  template <class $TemplateParameter_decl[[T]]>
532  class $Class_decl[[A]] {
533  int $Method_decl[[foo]]() {
534  return $TemplateParameter[[T]]::$Unknown_dependentName[[Field]];
535  }
536  };
537  )cpp",
538  // Highlighting the using decl as the underlying using shadow decl.
539  R"cpp(
540  void $Function_decl[[foo]]();
541  using ::$Function[[foo]];
542  )cpp",
543  // Highlighting of template template arguments.
544  R"cpp(
545  template <template <class> class $TemplateParameter_decl[[TT]],
546  template <class> class ...$TemplateParameter_decl[[TTs]]>
547  struct $Class_decl[[Foo]] {
548  $Class[[Foo]]<$TemplateParameter[[TT]], $TemplateParameter[[TTs]]...>
549  *$Field_decl[[t]];
550  };
551  )cpp",
552  // Inactive code highlighting
553  R"cpp(
554  // Code in the preamble.
555  // Inactive lines get an empty InactiveCode token at the beginning.
556 $InactiveCode[[#ifdef test]]
557 $InactiveCode[[#endif]]
558 
559  // A declaration to cause the preamble to end.
560  int $Variable_decl[[EndPreamble]];
561 
562  // Code after the preamble.
563  // Code inside inactive blocks does not get regular highlightings
564  // because it's not part of the AST.
565  #define $Macro_decl[[test2]]
566 $InactiveCode[[#if defined(test)]]
567 $InactiveCode[[int Inactive2;]]
568 $InactiveCode[[#elif defined(test2)]]
569  int $Variable_decl[[Active1]];
570 $InactiveCode[[#else]]
571 $InactiveCode[[int Inactive3;]]
572 $InactiveCode[[#endif]]
573 
574  #ifndef $Macro[[test]]
575  int $Variable_decl[[Active2]];
576  #endif
577 
578 $InactiveCode[[#ifdef test]]
579 $InactiveCode[[int Inactive4;]]
580 $InactiveCode[[#else]]
581  int $Variable_decl[[Active3]];
582  #endif
583  )cpp",
584  // Argument to 'sizeof...'
585  R"cpp(
586  template <typename... $TemplateParameter_decl[[Elements]]>
587  struct $Class_decl[[TupleSize]] {
588  static const int $StaticField_decl_readonly_static[[size]] =
589 sizeof...($TemplateParameter[[Elements]]);
590  };
591  )cpp",
592  // More dependent types
593  R"cpp(
594  template <typename $TemplateParameter_decl[[T]]>
595  struct $Class_decl[[Waldo]] {
596  using $Typedef_decl[[Location1]] = typename $TemplateParameter[[T]]
597  ::$Type_dependentName[[Resolver]]::$Type_dependentName[[Location]];
598  using $Typedef_decl[[Location2]] = typename $TemplateParameter[[T]]
599  ::template $Type_dependentName[[Resolver]]<$TemplateParameter[[T]]>
600  ::$Type_dependentName[[Location]];
601  using $Typedef_decl[[Location3]] = typename $TemplateParameter[[T]]
602  ::$Type_dependentName[[Resolver]]
603  ::template $Type_dependentName[[Location]]<$TemplateParameter[[T]]>;
604  static const int $StaticField_decl_readonly_static[[Value]] = $TemplateParameter[[T]]
605  ::$Type_dependentName[[Resolver]]::$Unknown_dependentName[[Value]];
606  };
607  )cpp",
608  // Dependent name with heuristic target
609  R"cpp(
610  template <typename>
611  struct $Class_decl[[Foo]] {
612  int $Field_decl[[Waldo]];
613  void $Method_decl[[bar]]() {
614  $Class[[Foo]]().$Field_dependentName[[Waldo]];
615  }
616  template <typename $TemplateParameter_decl[[U]]>
617  void $Method_decl[[bar1]]() {
618  $Class[[Foo]]<$TemplateParameter[[U]]>().$Field_dependentName[[Waldo]];
619  }
620  };
621  )cpp",
622  // Concepts
623  R"cpp(
624  template <typename $TemplateParameter_decl[[T]]>
625  concept $Concept_decl[[Fooable]] =
626  requires($TemplateParameter[[T]] $Parameter_decl[[F]]) {
627  $Parameter[[F]].$Unknown_dependentName[[foo]]();
628  };
629  template <typename $TemplateParameter_decl[[T]]>
630  requires $Concept[[Fooable]]<$TemplateParameter[[T]]>
631  void $Function_decl[[bar]]($TemplateParameter[[T]] $Parameter_decl[[F]]) {
632  $Parameter[[F]].$Unknown_dependentName[[foo]]();
633  }
634  )cpp",
635  // Dependent template name
636  R"cpp(
637  template <template <typename> class> struct $Class_decl[[A]] {};
638  template <typename $TemplateParameter_decl[[T]]>
639  using $Typedef_decl[[W]] = $Class[[A]]<
640  $TemplateParameter[[T]]::template $Class_dependentName[[Waldo]]
641  >;
642  )cpp",
643  R"cpp(
644  class $Class_decl_abstract[[Abstract]] {
645  public:
646  virtual void $Method_decl_abstract_virtual[[pure]]() = 0;
647  virtual void $Method_decl_virtual[[impl]]();
648  };
649  void $Function_decl[[foo]]($Class_abstract[[Abstract]]* $Parameter_decl[[A]]) {
650  $Parameter[[A]]->$Method_abstract_virtual[[pure]]();
651  $Parameter[[A]]->$Method_virtual[[impl]]();
652  }
653  )cpp",
654  R"cpp(
655  <:[deprecated]:> int $Variable_decl_deprecated[[x]];
656  )cpp",
657  R"cpp(
658  // ObjC: Classes and methods
659  @class $Class_decl[[Forward]];
660 
661  @interface $Class_decl[[Foo]]
662  @end
663  @interface $Class_decl[[Bar]] : $Class[[Foo]]
664  -(id) $Method_decl[[x]]:(int)$Parameter_decl[[a]] $Method_decl[[y]]:(int)$Parameter_decl[[b]];
665  +(instancetype)$StaticMethod_decl_static[[sharedInstance]];
666  +(void) $StaticMethod_decl_static[[explode]];
667  @end
668  @implementation $Class_decl[[Bar]]
669  -(id) $Method_decl[[x]]:(int)$Parameter_decl[[a]] $Method_decl[[y]]:(int)$Parameter_decl[[b]] {
670  return self;
671  }
672  +(instancetype)$StaticMethod_decl_static[[sharedInstance]] { return 0; }
673  +(void) $StaticMethod_decl_static[[explode]] {}
674  @end
675 
676  void $Function_decl[[m]]($Class[[Bar]] *$Parameter_decl[[b]]) {
677  [$Parameter[[b]] $Method[[x]]:1 $Method[[y]]:2];
678  [$Class[[Bar]] $StaticMethod_static[[explode]]];
679  }
680  )cpp",
681  R"cpp(
682  // ObjC: Protocols
683  @protocol $Interface_decl[[Protocol]]
684  @end
685  @protocol $Interface_decl[[Protocol2]] <$Interface[[Protocol]]>
686  @end
687  @interface $Class_decl[[Klass]] <$Interface[[Protocol]]>
688  @end
689  id<$Interface[[Protocol]]> $Variable_decl[[x]];
690  )cpp",
691  R"cpp(
692  // ObjC: Categories
693  @interface $Class_decl[[Foo]]
694  @end
695  @interface $Class[[Foo]]($Namespace_decl[[Bar]])
696  @end
697  @implementation $Class[[Foo]]($Namespace_decl[[Bar]])
698  @end
699  )cpp",
700  R"cpp(
701  // ObjC: Properties and Ivars.
702  @interface $Class_decl[[Foo]] {
703  int $Field_decl[[_someProperty]];
704  }
705  @property(nonatomic, assign) int $Field_decl[[someProperty]];
706  @property(readonly, class) $Class[[Foo]] *$Field_decl_readonly_static[[sharedInstance]];
707  @end
708  @implementation $Class_decl[[Foo]]
709  @synthesize someProperty = _someProperty;
710  - (int)$Method_decl[[otherMethod]] {
711  return 0;
712  }
713  - (int)$Method_decl[[doSomething]] {
714  $Class[[Foo]].$Field_static[[sharedInstance]].$Field[[someProperty]] = 1;
715  self.$Field[[someProperty]] = self.$Field[[someProperty]] + self.$Field[[otherMethod]] + 1;
716  self->$Field[[_someProperty]] = $Field[[_someProperty]] + 1;
717  }
718  @end
719  )cpp",
720  // Member imported from dependent base
721  R"cpp(
722  template <typename> struct $Class_decl[[Base]] {
723  int $Field_decl[[member]];
724  };
725  template <typename $TemplateParameter_decl[[T]]>
726  struct $Class_decl[[Derived]] : $Class[[Base]]<$TemplateParameter[[T]]> {
727  using $Class[[Base]]<$TemplateParameter[[T]]>::$Field_dependentName[[member]];
728 
729  void $Method_decl[[method]]() {
730  (void)$Field_dependentName[[member]];
731  }
732  };
733  )cpp",
734  // Modifier for variables passed as non-const references
735  R"cpp(
736  void $Function_decl[[fun]](int, const int,
737  int*, const int*,
738  int&, const int&,
739  int*&, const int*&, const int* const &,
740  int**, int**&, int** const &,
741  int = 123) {
742  int $LocalVariable_decl[[val]];
743  int* $LocalVariable_decl[[ptr]];
744  const int* $LocalVariable_decl_readonly[[constPtr]];
745  int** $LocalVariable_decl[[array]];
746  $Function[[fun]]($LocalVariable[[val]], $LocalVariable[[val]],
747  $LocalVariable[[ptr]], $LocalVariable_readonly[[constPtr]],
748  $LocalVariable_usedAsMutableReference[[val]], $LocalVariable[[val]],
749 
750  $LocalVariable_usedAsMutableReference[[ptr]],
751  $LocalVariable_readonly_usedAsMutableReference[[constPtr]],
752  $LocalVariable_readonly[[constPtr]],
753 
754  $LocalVariable[[array]], $LocalVariable_usedAsMutableReference[[array]],
755  $LocalVariable[[array]]
756  );
757  }
758  struct $Class_decl[[S]] {
759  $Class_decl[[S]](int&) {
760  $Class[[S]] $LocalVariable_decl[[s1]]($Field_usedAsMutableReference[[field]]);
761  $Class[[S]] $LocalVariable_decl[[s2]]($LocalVariable[[s1]].$Field_usedAsMutableReference[[field]]);
762 
763  $Class[[S]] $LocalVariable_decl[[s3]]($StaticField_static_usedAsMutableReference[[staticField]]);
764  $Class[[S]] $LocalVariable_decl[[s4]]($Class[[S]]::$StaticField_static_usedAsMutableReference[[staticField]]);
765  }
766  int $Field_decl[[field]];
767  static int $StaticField_decl_static[[staticField]];
768  };
769  template <typename $TemplateParameter_decl[[X]]>
770  void $Function_decl[[foo]]($TemplateParameter[[X]]& $Parameter_decl[[x]]) {
771  // We do not support dependent types, so this one should *not* get the modifier.
772  $Function[[foo]]($Parameter[[x]]);
773  }
774  )cpp",
775  // init-captures
776  R"cpp(
777  void $Function_decl[[foo]]() {
778  int $LocalVariable_decl[[a]], $LocalVariable_decl[[b]];
779  [ $LocalVariable_decl[[c]] = $LocalVariable[[a]],
780  $LocalVariable_decl[[d]]($LocalVariable[[b]]) ]() {}();
781  }
782  )cpp",
783  };
784  for (const auto &TestCase : TestCases)
785  // Mask off scope modifiers to keep the tests manageable.
786  // They're tested separately.
787  checkHighlightings(TestCase, {}, ~ScopeModifierMask);
788 
789  checkHighlightings(R"cpp(
790  class $Class_decl[[A]] {
791  #include "imp.h"
792  };
793  )cpp",
794  {{"imp.h", R"cpp(
795  int someMethod();
796  void otherMethod();
797  )cpp"}},
798  ~ScopeModifierMask);
799 
800  // A separate test for macros in headers.
801  checkHighlightings(R"cpp(
802  #include "imp.h"
803  $Macro[[DEFINE_Y]]
804  $Macro[[DXYZ_Y]](A);
805  )cpp",
806  {{"imp.h", R"cpp(
807  #define DXYZ(X) class X {};
808  #define DXYZ_Y(Y) DXYZ(x##Y)
809  #define DEFINE(X) int X;
810  #define DEFINE_Y DEFINE(Y)
811  )cpp"}},
812  ~ScopeModifierMask);
813 
814  checkHighlightings(R"cpp(
815  #include "SYSObject.h"
816  @interface $Class_defaultLibrary[[SYSObject]] ($Namespace_decl[[UserCategory]])
817  @property(nonatomic, readonly) int $Field_decl_readonly[[user_property]];
818  @end
819  int $Function_decl[[somethingUsingSystemSymbols]]() {
820  $Class_defaultLibrary[[SYSObject]] *$LocalVariable_decl[[obj]] = [$Class_defaultLibrary[[SYSObject]] $StaticMethod_static_defaultLibrary[[new]]];
821  return $LocalVariable[[obj]].$Field_defaultLibrary[[value]] + $LocalVariable[[obj]].$Field_readonly[[user_property]];
822  }
823  )cpp",
824  {{"SystemSDK/SYSObject.h", R"cpp(
825  @interface SYSObject
826  @property(nonatomic, assign) int value;
827  + (instancetype)new;
828  @end
829  )cpp"}},
830  ~ScopeModifierMask, {"-isystemSystemSDK/"});
831 }
832 
833 TEST(SemanticHighlighting, ScopeModifiers) {
834  const char *TestCases[] = {
835  R"cpp(
836  static int $Variable_fileScope[[x]];
837  namespace $Namespace_globalScope[[ns]] {
838  class $Class_globalScope[[x]];
839  }
840  namespace {
841  void $Function_fileScope[[foo]]();
842  }
843  )cpp",
844  R"cpp(
845  void $Function_globalScope[[foo]](int $Parameter_functionScope[[y]]) {
846  int $LocalVariable_functionScope[[z]];
847  }
848  )cpp",
849  R"cpp(
850  // Lambdas are considered functions, not classes.
851  auto $Variable_fileScope[[x]] = [$LocalVariable_functionScope[[m]](42)] {
852  return $LocalVariable_functionScope[[m]];
853  };
854  )cpp",
855  R"cpp(
856  // Classes in functions are classes.
857  void $Function_globalScope[[foo]]() {
858  class $Class_functionScope[[X]] {
859  int $Field_classScope[[x]];
860  };
861  };
862  )cpp",
863  R"cpp(
864  template <int $TemplateParameter_classScope[[T]]>
865  class $Class_globalScope[[X]] {
866  };
867  )cpp",
868  R"cpp(
869  // No useful scope for template parameters of variable templates.
870  template <typename $TemplateParameter[[A]]>
871  unsigned $Variable_globalScope[[X]] =
872  $TemplateParameter[[A]]::$Unknown_classScope[[x]];
873  )cpp",
874  R"cpp(
875  #define $Macro_globalScope[[X]] 1
876  int $Variable_globalScope[[Y]] = $Macro_globalScope[[X]];
877  )cpp",
878  };
879 
880  for (const char *Test : TestCases)
881  checkHighlightings(Test, {}, ScopeModifierMask);
882 }
883 
884 // Ranges are highlighted as variables, unless highlighted as $Function etc.
885 std::vector<HighlightingToken> tokens(llvm::StringRef MarkedText) {
886  Annotations A(MarkedText);
887  std::vector<HighlightingToken> Results;
888  for (const Range& R : A.ranges())
889  Results.push_back({HighlightingKind::Variable, 0, R});
890  for (unsigned I = 0; I < static_cast<unsigned>(HighlightingKind::LastKind); ++I) {
891  HighlightingKind Kind = static_cast<HighlightingKind>(I);
892  for (const Range& R : A.ranges(llvm::to_string(Kind)))
893  Results.push_back({Kind, 0, R});
894  }
895  llvm::sort(Results);
896  return Results;
897 }
898 
899 TEST(SemanticHighlighting, toSemanticTokens) {
900  auto Tokens = tokens(R"(
901  [[blah]]
902 
903  $Function[[big]] [[bang]]
904  )");
905  Tokens.front().Modifiers |= unsigned(HighlightingModifier::Declaration);
906  Tokens.front().Modifiers |= unsigned(HighlightingModifier::Readonly);
907  auto Results = toSemanticTokens(Tokens);
908 
909  ASSERT_THAT(Results, SizeIs(3));
910  EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable));
911  EXPECT_EQ(Results[0].tokenModifiers,
914  EXPECT_EQ(Results[0].deltaLine, 1u);
915  EXPECT_EQ(Results[0].deltaStart, 1u);
916  EXPECT_EQ(Results[0].length, 4u);
917 
918  EXPECT_EQ(Results[1].tokenType, unsigned(HighlightingKind::Function));
919  EXPECT_EQ(Results[1].tokenModifiers, 0u);
920  EXPECT_EQ(Results[1].deltaLine, 2u);
921  EXPECT_EQ(Results[1].deltaStart, 4u);
922  EXPECT_EQ(Results[1].length, 3u);
923 
924  EXPECT_EQ(Results[2].tokenType, unsigned(HighlightingKind::Variable));
925  EXPECT_EQ(Results[1].tokenModifiers, 0u);
926  EXPECT_EQ(Results[2].deltaLine, 0u);
927  EXPECT_EQ(Results[2].deltaStart, 4u);
928  EXPECT_EQ(Results[2].length, 4u);
929 }
930 
931 TEST(SemanticHighlighting, diffSemanticTokens) {
932  auto Before = toSemanticTokens(tokens(R"(
933  [[foo]] [[bar]] [[baz]]
934  [[one]] [[two]] [[three]]
935  )"));
936  EXPECT_THAT(diffTokens(Before, Before), IsEmpty());
937 
938  auto After = toSemanticTokens(tokens(R"(
939  [[foo]] [[hello]] [[world]] [[baz]]
940  [[one]] [[two]] [[three]]
941  )"));
942 
943  // Replace [bar, baz] with [hello, world, baz]
944  auto Diff = diffTokens(Before, After);
945  ASSERT_THAT(Diff, SizeIs(1));
946  EXPECT_EQ(1u, Diff.front().startToken);
947  EXPECT_EQ(2u, Diff.front().deleteTokens);
948  ASSERT_THAT(Diff.front().tokens, SizeIs(3));
949  // hello
950  EXPECT_EQ(0u, Diff.front().tokens[0].deltaLine);
951  EXPECT_EQ(4u, Diff.front().tokens[0].deltaStart);
952  EXPECT_EQ(5u, Diff.front().tokens[0].length);
953  // world
954  EXPECT_EQ(0u, Diff.front().tokens[1].deltaLine);
955  EXPECT_EQ(6u, Diff.front().tokens[1].deltaStart);
956  EXPECT_EQ(5u, Diff.front().tokens[1].length);
957  // baz
958  EXPECT_EQ(0u, Diff.front().tokens[2].deltaLine);
959  EXPECT_EQ(6u, Diff.front().tokens[2].deltaStart);
960  EXPECT_EQ(3u, Diff.front().tokens[2].length);
961 }
962 
963 } // namespace
964 } // namespace clangd
965 } // namespace clang
Range
CharSourceRange Range
SourceRange for the file name.
Definition: IncludeOrderCheck.cpp:38
clang::clangd::HighlightingModifier::FunctionScope
@ FunctionScope
clang::clangd::HighlightingModifier::FileScope
@ FileScope
clang::clangd::TEST
TEST(BackgroundQueueTest, Priority)
Definition: BackgroundIndexTests.cpp:751
clang::clangd::HighlightingModifier
HighlightingModifier
Definition: SemanticHighlighting.h:61
clang::clangd::HighlightingModifier::LastModifier
@ LastModifier
TestTU.h
Kind
BindArgumentKind Kind
Definition: AvoidBindCheck.cpp:59
clang::clangd::HighlightingKind
HighlightingKind
Definition: SemanticHighlighting.h:30
clang::clangd::HighlightingKind::LastKind
@ LastKind
clang::clangd::HighlightingKind::Variable
@ Variable
Protocol.h
clang::clangd::ParsedAST::build
static llvm::Optional< ParsedAST > build(llvm::StringRef Filename, const ParseInputs &Inputs, std::unique_ptr< clang::CompilerInvocation > CI, llvm::ArrayRef< Diag > CompilerInvocationDiags, std::shared_ptr< const PreambleData > Preamble)
Attempts to run Clang and store the parsed AST.
Definition: ParsedAST.cpp:248
ns1::ns2::A
@ A
Definition: CategoricalFeature.h:3
Code
std::string Code
Definition: FindTargetTests.cpp:67
clang::clangd::HighlightingKind::Function
@ Function
clang::clangd::diffTokens
std::vector< SemanticTokensEdit > diffTokens(llvm::ArrayRef< SemanticToken > Old, llvm::ArrayRef< SemanticToken > New)
Definition: SemanticHighlighting.cpp:1027
TestFS.h
clang::clangd::positionToOffset
llvm::Expected< size_t > positionToOffset(llvm::StringRef Code, Position P, bool AllowColumnsBeyondLineLength)
Turn a [line, column] pair into an offset in Code.
Definition: SourceCode.cpp:175
Results
std::vector< CodeCompletionResult > Results
Definition: CodeComplete.cpp:771
clang::clangd::HighlightingModifier::GlobalScope
@ GlobalScope
Annotations.h
clang::clangd::getSemanticHighlightings
std::vector< HighlightingToken > getSemanticHighlightings(ParsedAST &AST)
Definition: SemanticHighlighting.cpp:787
SourceCode.h
clang::clangd::toSemanticTokens
std::vector< SemanticToken > toSemanticTokens(llvm::ArrayRef< HighlightingToken > Tokens)
Definition: SemanticHighlighting.cpp:916
SemanticHighlighting.h
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
OS
llvm::raw_string_ostream OS
Definition: TraceTests.cpp:163
ClangdServer.h
clang::clangd::HighlightingModifier::Readonly
@ Readonly
clang::clangd::HighlightingModifier::ClassScope
@ ClassScope
clang::clangd::HighlightingModifier::Declaration
@ Declaration