clang-tools  13.0.0git
SemanticHighlighting.cpp
Go to the documentation of this file.
1 //===--- SemanticHighlighting.cpp - ------------------------- ---*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "SemanticHighlighting.h"
10 #include "FindTarget.h"
11 #include "HeuristicResolver.h"
12 #include "ParsedAST.h"
13 #include "Protocol.h"
14 #include "SourceCode.h"
15 #include "support/Logger.h"
16 #include "clang/AST/ASTContext.h"
17 #include "clang/AST/Decl.h"
18 #include "clang/AST/DeclCXX.h"
19 #include "clang/AST/DeclObjC.h"
20 #include "clang/AST/DeclTemplate.h"
21 #include "clang/AST/DeclarationName.h"
22 #include "clang/AST/ExprCXX.h"
23 #include "clang/AST/RecursiveASTVisitor.h"
24 #include "clang/AST/Type.h"
25 #include "clang/AST/TypeLoc.h"
26 #include "clang/Basic/LangOptions.h"
27 #include "clang/Basic/SourceLocation.h"
28 #include "clang/Basic/SourceManager.h"
29 #include "clang/Tooling/Syntax/Tokens.h"
30 #include "llvm/ADT/None.h"
31 #include "llvm/ADT/Optional.h"
32 #include "llvm/ADT/STLExtras.h"
33 #include "llvm/Support/Base64.h"
34 #include "llvm/Support/Casting.h"
35 #include <algorithm>
36 
37 namespace clang {
38 namespace clangd {
39 namespace {
40 
41 /// Some names are not written in the source code and cannot be highlighted,
42 /// e.g. anonymous classes. This function detects those cases.
43 bool canHighlightName(DeclarationName Name) {
44  switch (Name.getNameKind()) {
45  case DeclarationName::Identifier: {
46  auto *II = Name.getAsIdentifierInfo();
47  return II && !II->getName().empty();
48  }
49  case DeclarationName::CXXConstructorName:
50  case DeclarationName::CXXDestructorName:
51  return true;
52  case DeclarationName::ObjCZeroArgSelector:
53  case DeclarationName::ObjCOneArgSelector:
54  case DeclarationName::ObjCMultiArgSelector:
55  // Multi-arg selectors need special handling, and we handle 0/1 arg
56  // selectors there too.
57  return false;
58  case DeclarationName::CXXConversionFunctionName:
59  case DeclarationName::CXXOperatorName:
60  case DeclarationName::CXXDeductionGuideName:
61  case DeclarationName::CXXLiteralOperatorName:
62  case DeclarationName::CXXUsingDirective:
63  return false;
64  }
65  llvm_unreachable("invalid name kind");
66 }
67 
68 llvm::Optional<HighlightingKind> kindForType(const Type *TP,
69  const HeuristicResolver *Resolver);
70 llvm::Optional<HighlightingKind>
71 kindForDecl(const NamedDecl *D, const HeuristicResolver *Resolver) {
72  if (auto *USD = dyn_cast<UsingShadowDecl>(D)) {
73  if (auto *Target = USD->getTargetDecl())
74  D = Target;
75  }
76  if (auto *TD = dyn_cast<TemplateDecl>(D)) {
77  if (auto *Templated = TD->getTemplatedDecl())
78  D = Templated;
79  }
80  if (auto *TD = dyn_cast<TypedefNameDecl>(D)) {
81  // We try to highlight typedefs as their underlying type.
82  if (auto K =
83  kindForType(TD->getUnderlyingType().getTypePtrOrNull(), Resolver))
84  return K;
85  // And fallback to a generic kind if this fails.
87  }
88  // We highlight class decls, constructor decls and destructor decls as
89  // `Class` type. The destructor decls are handled in `VisitTagTypeLoc` (we
90  // will visit a TypeLoc where the underlying Type is a CXXRecordDecl).
91  if (auto *RD = llvm::dyn_cast<RecordDecl>(D)) {
92  // We don't want to highlight lambdas like classes.
93  if (RD->isLambda())
94  return llvm::None;
96  }
97  if (isa<ClassTemplateDecl, RecordDecl, CXXConstructorDecl, ObjCInterfaceDecl,
98  ObjCImplementationDecl>(D))
100  if (isa<ObjCProtocolDecl>(D))
102  if (isa<ObjCCategoryDecl>(D))
104  if (auto *MD = dyn_cast<CXXMethodDecl>(D))
105  return MD->isStatic() ? HighlightingKind::StaticMethod
107  if (auto *OMD = dyn_cast<ObjCMethodDecl>(D))
108  return OMD->isClassMethod() ? HighlightingKind::StaticMethod
110  if (isa<FieldDecl, ObjCPropertyDecl>(D))
112  if (isa<EnumDecl>(D))
113  return HighlightingKind::Enum;
114  if (isa<EnumConstantDecl>(D))
116  if (isa<ParmVarDecl>(D))
118  if (auto *VD = dyn_cast<VarDecl>(D)) {
119  if (isa<ImplicitParamDecl>(VD)) // e.g. ObjC Self
120  return llvm::None;
121  return VD->isStaticDataMember()
123  : VD->isLocalVarDecl() ? HighlightingKind::LocalVariable
125  }
126  if (const auto *BD = dyn_cast<BindingDecl>(D))
127  return BD->getDeclContext()->isFunctionOrMethod()
130  if (isa<FunctionDecl>(D))
132  if (isa<NamespaceDecl>(D) || isa<NamespaceAliasDecl>(D) ||
133  isa<UsingDirectiveDecl>(D))
135  if (isa<TemplateTemplateParmDecl>(D) || isa<TemplateTypeParmDecl>(D) ||
136  isa<NonTypeTemplateParmDecl>(D))
138  if (isa<ConceptDecl>(D))
140  if (const auto *UUVD = dyn_cast<UnresolvedUsingValueDecl>(D)) {
141  auto Targets = Resolver->resolveUsingValueDecl(UUVD);
142  if (!Targets.empty()) {
143  return kindForDecl(Targets[0], Resolver);
144  }
146  }
147  return llvm::None;
148 }
149 llvm::Optional<HighlightingKind>
150 kindForType(const Type *TP, const HeuristicResolver *Resolver) {
151  if (!TP)
152  return llvm::None;
153  if (TP->isBuiltinType()) // Builtins are special, they do not have decls.
155  if (auto *TD = dyn_cast<TemplateTypeParmType>(TP))
156  return kindForDecl(TD->getDecl(), Resolver);
157  if (isa<ObjCObjectPointerType>(TP))
159  if (auto *TD = TP->getAsTagDecl())
160  return kindForDecl(TD, Resolver);
161  return llvm::None;
162 }
163 
164 // Whether T is const in a loose sense - is a variable with this type readonly?
165 bool isConst(QualType T) {
166  if (T.isNull() || T->isDependentType())
167  return false;
168  T = T.getNonReferenceType();
169  if (T.isConstQualified())
170  return true;
171  if (const auto *AT = T->getAsArrayTypeUnsafe())
172  return isConst(AT->getElementType());
173  if (isConst(T->getPointeeType()))
174  return true;
175  return false;
176 }
177 
178 // Whether D is const in a loose sense (should it be highlighted as such?)
179 // FIXME: This is separate from whether *a particular usage* can mutate D.
180 // We may want V in V.size() to be readonly even if V is mutable.
181 bool isConst(const Decl *D) {
182  if (llvm::isa<EnumConstantDecl>(D) || llvm::isa<NonTypeTemplateParmDecl>(D))
183  return true;
184  if (llvm::isa<FieldDecl>(D) || llvm::isa<VarDecl>(D) ||
185  llvm::isa<MSPropertyDecl>(D) || llvm::isa<BindingDecl>(D)) {
186  if (isConst(llvm::cast<ValueDecl>(D)->getType()))
187  return true;
188  }
189  if (const auto *OCPD = llvm::dyn_cast<ObjCPropertyDecl>(D)) {
190  if (OCPD->isReadOnly())
191  return true;
192  }
193  if (const auto *MPD = llvm::dyn_cast<MSPropertyDecl>(D)) {
194  if (!MPD->hasSetter())
195  return true;
196  }
197  if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D)) {
198  if (CMD->isConst())
199  return true;
200  }
201  return false;
202 }
203 
204 // "Static" means many things in C++, only some get the "static" modifier.
205 //
206 // Meanings that do:
207 // - Members associated with the class rather than the instance.
208 // This is what 'static' most often means across languages.
209 // - static local variables
210 // These are similarly "detached from their context" by the static keyword.
211 // In practice, these are rarely used inside classes, reducing confusion.
212 //
213 // Meanings that don't:
214 // - Namespace-scoped variables, which have static storage class.
215 // This is implicit, so the keyword "static" isn't so strongly associated.
216 // If we want a modifier for these, "global scope" is probably the concept.
217 // - Namespace-scoped variables/functions explicitly marked "static".
218 // There the keyword changes *linkage* , which is a totally different concept.
219 // If we want to model this, "file scope" would be a nice modifier.
220 //
221 // This is confusing, and maybe we should use another name, but because "static"
222 // is a standard LSP modifier, having one with that name has advantages.
223 bool isStatic(const Decl *D) {
224  if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
225  return CMD->isStatic();
226  if (const VarDecl *VD = llvm::dyn_cast<VarDecl>(D))
227  return VD->isStaticDataMember() || VD->isStaticLocal();
228  if (const auto *OPD = llvm::dyn_cast<ObjCPropertyDecl>(D))
229  return OPD->isClassProperty();
230  if (const auto *OMD = llvm::dyn_cast<ObjCMethodDecl>(D))
231  return OMD->isClassMethod();
232  return false;
233 }
234 
235 bool isAbstract(const Decl *D) {
236  if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
237  return CMD->isPure();
238  if (const auto *CRD = llvm::dyn_cast<CXXRecordDecl>(D))
239  return CRD->hasDefinition() && CRD->isAbstract();
240  return false;
241 }
242 
243 bool isDependent(const Decl *D) {
244  if (isa<UnresolvedUsingValueDecl>(D))
245  return true;
246  return false;
247 }
248 
249 /// Returns true if `Decl` is considered to be from a default/system library.
250 /// This currently checks the systemness of the file by include type, although
251 /// different heuristics may be used in the future (e.g. sysroot paths).
252 bool isDefaultLibrary(const Decl *D) {
253  SourceLocation Loc = D->getLocation();
254  if (!Loc.isValid())
255  return false;
256  return D->getASTContext().getSourceManager().isInSystemHeader(Loc);
257 }
258 
259 bool isDefaultLibrary(const Type *T) {
260  if (!T)
261  return false;
262  const Type *Underlying = T->getPointeeOrArrayElementType();
263  if (Underlying->isBuiltinType())
264  return true;
265  if (auto *TD = dyn_cast<TemplateTypeParmType>(Underlying))
266  return isDefaultLibrary(TD->getDecl());
267  if (auto *TD = Underlying->getAsTagDecl())
268  return isDefaultLibrary(TD);
269  return false;
270 }
271 
272 // For a macro usage `DUMP(foo)`, we want:
273 // - DUMP --> "macro"
274 // - foo --> "variable".
275 SourceLocation getHighlightableSpellingToken(SourceLocation L,
276  const SourceManager &SM) {
277  if (L.isFileID())
278  return SM.isWrittenInMainFile(L) ? L : SourceLocation{};
279  // Tokens expanded from the macro body contribute no highlightings.
280  if (!SM.isMacroArgExpansion(L))
281  return {};
282  // Tokens expanded from macro args are potentially highlightable.
283  return getHighlightableSpellingToken(SM.getImmediateSpellingLoc(L), SM);
284 }
285 
286 unsigned evaluateHighlightPriority(const HighlightingToken &Tok) {
287  enum HighlightPriority { Dependent = 0, Resolved = 1 };
288  return (Tok.Modifiers & (1 << uint32_t(HighlightingModifier::DependentName)))
289  ? Dependent
290  : Resolved;
291 }
292 
293 // Sometimes we get multiple tokens at the same location:
294 //
295 // - findExplicitReferences() returns a heuristic result for a dependent name
296 // (e.g. Method) and CollectExtraHighlighting returning a fallback dependent
297 // highlighting (e.g. Unknown+Dependent).
298 // - macro arguments are expanded multiple times and have different roles
299 // - broken code recovery produces several AST nodes at the same location
300 //
301 // We should either resolve these to a single token, or drop them all.
302 // Our heuristics are:
303 //
304 // - token kinds that come with "dependent-name" modifiers are less reliable
305 // (these tend to be vague, like Type or Unknown)
306 // - if we have multiple equally reliable kinds, drop token rather than guess
307 // - take the union of modifiers from all tokens
308 //
309 // In particular, heuristically resolved dependent names get their heuristic
310 // kind, plus the dependent modifier.
311 llvm::Optional<HighlightingToken>
312 resolveConflict(ArrayRef<HighlightingToken> Tokens) {
313  if (Tokens.size() == 1)
314  return Tokens[0];
315 
316  if (Tokens.size() != 2)
317  return llvm::None;
318 
319  unsigned Priority1 = evaluateHighlightPriority(Tokens[0]);
320  unsigned Priority2 = evaluateHighlightPriority(Tokens[1]);
321  if (Priority1 == Priority2 && Tokens[0].Kind != Tokens[1].Kind)
322  return llvm::None;
323  auto Result = Priority1 > Priority2 ? Tokens[0] : Tokens[1];
324  Result.Modifiers = Tokens[0].Modifiers | Tokens[1].Modifiers;
325  return Result;
326 }
327 
328 /// Consumes source locations and maps them to text ranges for highlightings.
329 class HighlightingsBuilder {
330 public:
331  HighlightingsBuilder(const ParsedAST &AST)
332  : TB(AST.getTokens()), SourceMgr(AST.getSourceManager()),
333  LangOpts(AST.getLangOpts()) {}
334 
335  HighlightingToken &addToken(SourceLocation Loc, HighlightingKind Kind) {
336  Loc = getHighlightableSpellingToken(Loc, SourceMgr);
337  if (Loc.isInvalid())
338  return InvalidHighlightingToken;
339  const auto *Tok = TB.spelledTokenAt(Loc);
340  assert(Tok);
341  return addToken(
343  Tok->range(SourceMgr).toCharRange(SourceMgr)),
344  Kind);
345  }
346 
347  HighlightingToken &addToken(Range R, HighlightingKind Kind) {
348  HighlightingToken HT;
349  HT.R = std::move(R);
350  HT.Kind = Kind;
351  Tokens.push_back(std::move(HT));
352  return Tokens.back();
353  }
354 
355  std::vector<HighlightingToken> collect(ParsedAST &AST) && {
356  // Initializer lists can give duplicates of tokens, therefore all tokens
357  // must be deduplicated.
358  llvm::sort(Tokens);
359  auto Last = std::unique(Tokens.begin(), Tokens.end());
360  Tokens.erase(Last, Tokens.end());
361 
362  // Macros can give tokens that have the same source range but conflicting
363  // kinds. In this case all tokens sharing this source range should be
364  // removed.
365  std::vector<HighlightingToken> NonConflicting;
366  NonConflicting.reserve(Tokens.size());
367  for (ArrayRef<HighlightingToken> TokRef = Tokens; !TokRef.empty();) {
368  ArrayRef<HighlightingToken> Conflicting =
369  TokRef.take_while([&](const HighlightingToken &T) {
370  // TokRef is guaranteed at least one element here because otherwise
371  // this predicate would never fire.
372  return T.R == TokRef.front().R;
373  });
374  if (auto Resolved = resolveConflict(Conflicting))
375  NonConflicting.push_back(*Resolved);
376  // TokRef[Conflicting.size()] is the next token with a different range (or
377  // the end of the Tokens).
378  TokRef = TokRef.drop_front(Conflicting.size());
379  }
380  const auto &SM = AST.getSourceManager();
381  StringRef MainCode = SM.getBufferOrFake(SM.getMainFileID()).getBuffer();
382 
383  // Merge token stream with "inactive line" markers.
384  std::vector<HighlightingToken> WithInactiveLines;
385  auto SortedSkippedRanges = AST.getMacros().SkippedRanges;
386  llvm::sort(SortedSkippedRanges);
387  auto It = NonConflicting.begin();
388  for (const Range &R : SortedSkippedRanges) {
389  // Create one token for each line in the skipped range, so it works
390  // with line-based diffing.
391  assert(R.start.line <= R.end.line);
392  for (int Line = R.start.line; Line <= R.end.line; ++Line) {
393  // If the end of the inactive range is at the beginning
394  // of a line, that line is not inactive.
395  if (Line == R.end.line && R.end.character == 0)
396  continue;
397  // Copy tokens before the inactive line
398  for (; It != NonConflicting.end() && It->R.start.line < Line; ++It)
399  WithInactiveLines.push_back(std::move(*It));
400  // Add a token for the inactive line itself.
401  auto StartOfLine = positionToOffset(MainCode, Position{Line, 0});
402  if (StartOfLine) {
403  StringRef LineText =
404  MainCode.drop_front(*StartOfLine).take_until([](char C) {
405  return C == '\n';
406  });
407  HighlightingToken HT;
408  WithInactiveLines.emplace_back();
409  WithInactiveLines.back().Kind = HighlightingKind::InactiveCode;
410  WithInactiveLines.back().R.start.line = Line;
411  WithInactiveLines.back().R.end.line = Line;
412  WithInactiveLines.back().R.end.character =
413  static_cast<int>(lspLength(LineText));
414  } else {
415  elog("Failed to convert position to offset: {0}",
416  StartOfLine.takeError());
417  }
418 
419  // Skip any other tokens on the inactive line. e.g.
420  // `#ifndef Foo` is considered as part of an inactive region when Foo is
421  // defined, and there is a Foo macro token.
422  // FIXME: we should reduce the scope of the inactive region to not
423  // include the directive itself.
424  while (It != NonConflicting.end() && It->R.start.line == Line)
425  ++It;
426  }
427  }
428  // Copy tokens after the last inactive line
429  for (; It != NonConflicting.end(); ++It)
430  WithInactiveLines.push_back(std::move(*It));
431  return WithInactiveLines;
432  }
433 
434  const HeuristicResolver *getResolver() const { return Resolver; }
435 
436 private:
437  const syntax::TokenBuffer &TB;
438  const SourceManager &SourceMgr;
439  const LangOptions &LangOpts;
440  std::vector<HighlightingToken> Tokens;
441  const HeuristicResolver *Resolver;
442  // returned from addToken(InvalidLoc)
443  HighlightingToken InvalidHighlightingToken;
444 };
445 
446 llvm::Optional<HighlightingModifier> scopeModifier(const NamedDecl *D) {
447  const DeclContext *DC = D->getDeclContext();
448  // Injected "Foo" within the class "Foo" has file scope, not class scope.
449  if (auto *R = dyn_cast_or_null<RecordDecl>(D))
450  if (R->isInjectedClassName())
451  DC = DC->getParent();
452  // Lambda captures are considered function scope, not class scope.
453  if (llvm::isa<FieldDecl>(D))
454  if (const auto *RD = llvm::dyn_cast<RecordDecl>(DC))
455  if (RD->isLambda())
457  // Walk up the DeclContext hierarchy until we find something interesting.
458  for (; !DC->isFileContext(); DC = DC->getParent()) {
459  if (DC->isFunctionOrMethod())
461  if (DC->isRecord())
463  }
464  // Some template parameters (e.g. those for variable templates) don't have
465  // meaningful DeclContexts. That doesn't mean they're global!
466  if (DC->isTranslationUnit() && D->isTemplateParameter())
467  return llvm::None;
468  // ExternalLinkage threshold could be tweaked, e.g. module-visible as global.
469  if (D->getLinkageInternal() < ExternalLinkage)
472 }
473 
474 llvm::Optional<HighlightingModifier> scopeModifier(const Type *T) {
475  if (!T)
476  return llvm::None;
477  if (T->isBuiltinType())
479  if (auto *TD = dyn_cast<TemplateTypeParmType>(T))
480  return scopeModifier(TD->getDecl());
481  if (auto *TD = T->getAsTagDecl())
482  return scopeModifier(TD);
483  return llvm::None;
484 }
485 
486 /// Produces highlightings, which are not captured by findExplicitReferences,
487 /// e.g. highlights dependent names and 'auto' as the underlying type.
488 class CollectExtraHighlightings
489  : public RecursiveASTVisitor<CollectExtraHighlightings> {
490 public:
491  CollectExtraHighlightings(HighlightingsBuilder &H) : H(H) {}
492 
493  bool VisitDecltypeTypeLoc(DecltypeTypeLoc L) {
494  if (auto K = kindForType(L.getTypePtr(), H.getResolver())) {
495  auto &Tok = H.addToken(L.getBeginLoc(), *K)
496  .addModifier(HighlightingModifier::Deduced);
497  if (auto Mod = scopeModifier(L.getTypePtr()))
498  Tok.addModifier(*Mod);
499  if (isDefaultLibrary(L.getTypePtr()))
500  Tok.addModifier(HighlightingModifier::DefaultLibrary);
501  }
502  return true;
503  }
504 
505  bool VisitDeclaratorDecl(DeclaratorDecl *D) {
506  auto *AT = D->getType()->getContainedAutoType();
507  if (!AT)
508  return true;
509  if (auto K = kindForType(AT->getDeducedType().getTypePtrOrNull(),
510  H.getResolver())) {
511  auto &Tok = H.addToken(D->getTypeSpecStartLoc(), *K)
512  .addModifier(HighlightingModifier::Deduced);
513  const Type *Deduced = AT->getDeducedType().getTypePtrOrNull();
514  if (auto Mod = scopeModifier(Deduced))
515  Tok.addModifier(*Mod);
516  if (isDefaultLibrary(Deduced))
517  Tok.addModifier(HighlightingModifier::DefaultLibrary);
518  }
519  return true;
520  }
521 
522  // We handle objective-C selectors specially, because one reference can
523  // cover several non-contiguous tokens.
524  void highlightObjCSelector(const ArrayRef<SourceLocation> &Locs, bool Decl,
525  bool Class, bool DefaultLibrary) {
528  for (SourceLocation Part : Locs) {
529  auto &Tok =
530  H.addToken(Part, Kind).addModifier(HighlightingModifier::ClassScope);
531  if (Decl)
532  Tok.addModifier(HighlightingModifier::Declaration);
533  if (Class)
534  Tok.addModifier(HighlightingModifier::Static);
535  if (DefaultLibrary)
536  Tok.addModifier(HighlightingModifier::DefaultLibrary);
537  }
538  }
539 
540  bool VisitObjCMethodDecl(ObjCMethodDecl *OMD) {
541  llvm::SmallVector<SourceLocation> Locs;
542  OMD->getSelectorLocs(Locs);
543  highlightObjCSelector(Locs, /*Decl=*/true, OMD->isClassMethod(),
544  isDefaultLibrary(OMD));
545  return true;
546  }
547 
548  bool VisitObjCMessageExpr(ObjCMessageExpr *OME) {
549  llvm::SmallVector<SourceLocation> Locs;
550  OME->getSelectorLocs(Locs);
551  bool DefaultLibrary = false;
552  if (ObjCMethodDecl *OMD = OME->getMethodDecl())
553  DefaultLibrary = isDefaultLibrary(OMD);
554  highlightObjCSelector(Locs, /*Decl=*/false, OME->isClassMessage(),
555  DefaultLibrary);
556  return true;
557  }
558 
559  // Objective-C allows you to use property syntax `self.prop` as sugar for
560  // `[self prop]` and `[self setProp:]` when there's no explicit `@property`
561  // for `prop` as well as for class properties. We treat this like a property
562  // even though semantically it's equivalent to a method expression.
563  void highlightObjCImplicitPropertyRef(const ObjCMethodDecl *OMD,
564  SourceLocation Loc) {
565  auto &Tok = H.addToken(Loc, HighlightingKind::Field)
566  .addModifier(HighlightingModifier::ClassScope);
567  if (OMD->isClassMethod())
568  Tok.addModifier(HighlightingModifier::Static);
569  if (isDefaultLibrary(OMD))
570  Tok.addModifier(HighlightingModifier::DefaultLibrary);
571  }
572 
573  bool VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *OPRE) {
574  // We need to handle implicit properties here since they will appear to
575  // reference `ObjCMethodDecl` via an implicit `ObjCMessageExpr`, so normal
576  // highlighting will not work.
577  if (!OPRE->isImplicitProperty())
578  return true;
579  // A single property expr can reference both a getter and setter, but we can
580  // only provide a single semantic token, so prefer the getter. In most cases
581  // the end result should be the same, although it's technically possible
582  // that the user defines a setter for a system SDK.
583  if (OPRE->isMessagingGetter()) {
584  highlightObjCImplicitPropertyRef(OPRE->getImplicitPropertyGetter(),
585  OPRE->getLocation());
586  return true;
587  }
588  if (OPRE->isMessagingSetter()) {
589  highlightObjCImplicitPropertyRef(OPRE->getImplicitPropertySetter(),
590  OPRE->getLocation());
591  }
592  return true;
593  }
594 
595  bool VisitOverloadExpr(OverloadExpr *E) {
596  if (!E->decls().empty())
597  return true; // handled by findExplicitReferences.
598  auto &Tok = H.addToken(E->getNameLoc(), HighlightingKind::Unknown)
600  if (llvm::isa<UnresolvedMemberExpr>(E))
601  Tok.addModifier(HighlightingModifier::ClassScope);
602  // other case is UnresolvedLookupExpr, scope is unknown.
603  return true;
604  }
605 
606  bool VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E) {
607  H.addToken(E->getMemberNameInfo().getLoc(), HighlightingKind::Unknown)
609  .addModifier(HighlightingModifier::ClassScope);
610  return true;
611  }
612 
613  bool VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
614  H.addToken(E->getNameInfo().getLoc(), HighlightingKind::Unknown)
616  .addModifier(HighlightingModifier::ClassScope);
617  return true;
618  }
619 
620  bool VisitDependentNameTypeLoc(DependentNameTypeLoc L) {
621  H.addToken(L.getNameLoc(), HighlightingKind::Type)
623  .addModifier(HighlightingModifier::ClassScope);
624  return true;
625  }
626 
627  bool VisitDependentTemplateSpecializationTypeLoc(
628  DependentTemplateSpecializationTypeLoc L) {
629  H.addToken(L.getTemplateNameLoc(), HighlightingKind::Type)
631  .addModifier(HighlightingModifier::ClassScope);
632  return true;
633  }
634 
635  bool TraverseTemplateArgumentLoc(TemplateArgumentLoc L) {
636  // Handle template template arguments only (other arguments are handled by
637  // their Expr, TypeLoc etc values).
638  if (L.getArgument().getKind() != TemplateArgument::Template &&
639  L.getArgument().getKind() != TemplateArgument::TemplateExpansion)
640  return RecursiveASTVisitor::TraverseTemplateArgumentLoc(L);
641 
642  TemplateName N = L.getArgument().getAsTemplateOrTemplatePattern();
643  switch (N.getKind()) {
644  case TemplateName::OverloadedTemplate:
645  // Template template params must always be class templates.
646  // Don't bother to try to work out the scope here.
647  H.addToken(L.getTemplateNameLoc(), HighlightingKind::Class);
648  break;
649  case TemplateName::DependentTemplate:
650  case TemplateName::AssumedTemplate:
651  H.addToken(L.getTemplateNameLoc(), HighlightingKind::Class)
653  break;
654  case TemplateName::Template:
655  case TemplateName::QualifiedTemplate:
656  case TemplateName::SubstTemplateTemplateParm:
657  case TemplateName::SubstTemplateTemplateParmPack:
658  // Names that could be resolved to a TemplateDecl are handled elsewhere.
659  break;
660  }
661  return RecursiveASTVisitor::TraverseTemplateArgumentLoc(L);
662  }
663 
664  // findExplicitReferences will walk nested-name-specifiers and
665  // find anything that can be resolved to a Decl. However, non-leaf
666  // components of nested-name-specifiers which are dependent names
667  // (kind "Identifier") cannot be resolved to a decl, so we visit
668  // them here.
669  bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc Q) {
670  if (NestedNameSpecifier *NNS = Q.getNestedNameSpecifier()) {
671  if (NNS->getKind() == NestedNameSpecifier::Identifier)
672  H.addToken(Q.getLocalBeginLoc(), HighlightingKind::Type)
674  .addModifier(HighlightingModifier::ClassScope);
675  }
676  return RecursiveASTVisitor::TraverseNestedNameSpecifierLoc(Q);
677  }
678 
679 private:
680  HighlightingsBuilder &H;
681 };
682 } // namespace
683 
684 std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST) {
685  auto &C = AST.getASTContext();
686  // Add highlightings for AST nodes.
687  HighlightingsBuilder Builder(AST);
688  // Highlight 'decltype' and 'auto' as their underlying types.
689  CollectExtraHighlightings(Builder).TraverseAST(C);
690  // Highlight all decls and references coming from the AST.
692  C,
693  [&](ReferenceLoc R) {
694  for (const NamedDecl *Decl : R.Targets) {
695  if (!canHighlightName(Decl->getDeclName()))
696  continue;
697  auto Kind = kindForDecl(Decl, AST.getHeuristicResolver());
698  if (!Kind)
699  continue;
700  auto &Tok = Builder.addToken(R.NameLoc, *Kind);
701 
702  // The attribute tests don't want to look at the template.
703  if (auto *TD = dyn_cast<TemplateDecl>(Decl)) {
704  if (auto *Templated = TD->getTemplatedDecl())
705  Decl = Templated;
706  }
707  if (auto Mod = scopeModifier(Decl))
708  Tok.addModifier(*Mod);
709  if (isConst(Decl))
710  Tok.addModifier(HighlightingModifier::Readonly);
711  if (isStatic(Decl))
712  Tok.addModifier(HighlightingModifier::Static);
713  if (isAbstract(Decl))
714  Tok.addModifier(HighlightingModifier::Abstract);
715  if (isDependent(Decl))
716  Tok.addModifier(HighlightingModifier::DependentName);
717  if (isDefaultLibrary(Decl))
718  Tok.addModifier(HighlightingModifier::DefaultLibrary);
719  if (Decl->isDeprecated())
720  Tok.addModifier(HighlightingModifier::Deprecated);
721  // Do not treat an UnresolvedUsingValueDecl as a declaration.
722  // It's more common to think of it as a reference to the
723  // underlying declaration.
724  if (R.IsDecl && !isa<UnresolvedUsingValueDecl>(Decl))
725  Tok.addModifier(HighlightingModifier::Declaration);
726  }
727  },
728  AST.getHeuristicResolver());
729  // Add highlightings for macro references.
730  auto AddMacro = [&](const MacroOccurrence &M) {
731  auto &T = Builder.addToken(M.Rng, HighlightingKind::Macro);
732  T.addModifier(HighlightingModifier::GlobalScope);
733  if (M.IsDefinition)
734  T.addModifier(HighlightingModifier::Declaration);
735  };
736  for (const auto &SIDToRefs : AST.getMacros().MacroRefs)
737  for (const auto &M : SIDToRefs.second)
738  AddMacro(M);
739  for (const auto &M : AST.getMacros().UnknownMacros)
740  AddMacro(M);
741 
742  return std::move(Builder).collect(AST);
743 }
744 
745 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K) {
746  switch (K) {
747  case HighlightingKind::Variable:
748  return OS << "Variable";
749  case HighlightingKind::LocalVariable:
750  return OS << "LocalVariable";
751  case HighlightingKind::Parameter:
752  return OS << "Parameter";
753  case HighlightingKind::Function:
754  return OS << "Function";
755  case HighlightingKind::Method:
756  return OS << "Method";
757  case HighlightingKind::StaticMethod:
758  return OS << "StaticMethod";
759  case HighlightingKind::Field:
760  return OS << "Field";
761  case HighlightingKind::StaticField:
762  return OS << "StaticField";
763  case HighlightingKind::Class:
764  return OS << "Class";
765  case HighlightingKind::Interface:
766  return OS << "Interface";
767  case HighlightingKind::Enum:
768  return OS << "Enum";
769  case HighlightingKind::EnumConstant:
770  return OS << "EnumConstant";
771  case HighlightingKind::Typedef:
772  return OS << "Typedef";
774  return OS << "Type";
776  return OS << "Unknown";
777  case HighlightingKind::Namespace:
778  return OS << "Namespace";
779  case HighlightingKind::TemplateParameter:
780  return OS << "TemplateParameter";
781  case HighlightingKind::Concept:
782  return OS << "Concept";
783  case HighlightingKind::Primitive:
784  return OS << "Primitive";
785  case HighlightingKind::Macro:
786  return OS << "Macro";
787  case HighlightingKind::InactiveCode:
788  return OS << "InactiveCode";
789  }
790  llvm_unreachable("invalid HighlightingKind");
791 }
792 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K) {
793  switch (K) {
794  case HighlightingModifier::Declaration:
795  return OS << "decl"; // abbrevation for common case
796  default:
797  return OS << toSemanticTokenModifier(K);
798  }
799 }
800 
802  return std::tie(L.R, L.Kind, L.Modifiers) ==
803  std::tie(R.R, R.Kind, R.Modifiers);
804 }
806  return std::tie(L.R, L.Kind, R.Modifiers) <
807  std::tie(R.R, R.Kind, R.Modifiers);
808 }
809 
810 std::vector<SemanticToken>
811 toSemanticTokens(llvm::ArrayRef<HighlightingToken> Tokens) {
812  assert(std::is_sorted(Tokens.begin(), Tokens.end()));
813  std::vector<SemanticToken> Result;
814  const HighlightingToken *Last = nullptr;
815  for (const HighlightingToken &Tok : Tokens) {
816  Result.emplace_back();
817  SemanticToken &Out = Result.back();
818  // deltaStart/deltaLine are relative if possible.
819  if (Last) {
820  assert(Tok.R.start.line >= Last->R.start.line);
821  Out.deltaLine = Tok.R.start.line - Last->R.start.line;
822  if (Out.deltaLine == 0) {
823  assert(Tok.R.start.character >= Last->R.start.character);
824  Out.deltaStart = Tok.R.start.character - Last->R.start.character;
825  } else {
826  Out.deltaStart = Tok.R.start.character;
827  }
828  } else {
829  Out.deltaLine = Tok.R.start.line;
830  Out.deltaStart = Tok.R.start.character;
831  }
832  assert(Tok.R.end.line == Tok.R.start.line);
833  Out.length = Tok.R.end.character - Tok.R.start.character;
834  Out.tokenType = static_cast<unsigned>(Tok.Kind);
835  Out.tokenModifiers = Tok.Modifiers;
836 
837  Last = &Tok;
838  }
839  return Result;
840 }
842  switch (Kind) {
843  case HighlightingKind::Variable:
844  case HighlightingKind::LocalVariable:
845  case HighlightingKind::StaticField:
846  return "variable";
847  case HighlightingKind::Parameter:
848  return "parameter";
849  case HighlightingKind::Function:
850  return "function";
851  case HighlightingKind::Method:
852  return "method";
853  case HighlightingKind::StaticMethod:
854  // FIXME: better method with static modifier?
855  return "function";
856  case HighlightingKind::Field:
857  return "property";
858  case HighlightingKind::Class:
859  return "class";
860  case HighlightingKind::Interface:
861  return "interface";
862  case HighlightingKind::Enum:
863  return "enum";
864  case HighlightingKind::EnumConstant:
865  return "enumMember";
866  case HighlightingKind::Typedef:
868  return "type";
870  return "unknown"; // nonstandard
871  case HighlightingKind::Namespace:
872  return "namespace";
873  case HighlightingKind::TemplateParameter:
874  return "typeParameter";
875  case HighlightingKind::Concept:
876  return "concept"; // nonstandard
877  case HighlightingKind::Primitive:
878  return "type";
879  case HighlightingKind::Macro:
880  return "macro";
881  case HighlightingKind::InactiveCode:
882  return "comment";
883  }
884  llvm_unreachable("unhandled HighlightingKind");
885 }
886 
888  switch (Modifier) {
889  case HighlightingModifier::Declaration:
890  return "declaration";
891  case HighlightingModifier::Deprecated:
892  return "deprecated";
893  case HighlightingModifier::Readonly:
894  return "readonly";
895  case HighlightingModifier::Static:
896  return "static";
897  case HighlightingModifier::Deduced:
898  return "deduced"; // nonstandard
899  case HighlightingModifier::Abstract:
900  return "abstract";
901  case HighlightingModifier::DependentName:
902  return "dependentName"; // nonstandard
903  case HighlightingModifier::DefaultLibrary:
904  return "defaultLibrary";
905  case HighlightingModifier::FunctionScope:
906  return "functionScope"; // nonstandard
907  case HighlightingModifier::ClassScope:
908  return "classScope"; // nonstandard
909  case HighlightingModifier::FileScope:
910  return "fileScope"; // nonstandard
911  case HighlightingModifier::GlobalScope:
912  return "globalScope"; // nonstandard
913  }
914  llvm_unreachable("unhandled HighlightingModifier");
915 }
916 
917 std::vector<SemanticTokensEdit>
918 diffTokens(llvm::ArrayRef<SemanticToken> Old,
919  llvm::ArrayRef<SemanticToken> New) {
920  // For now, just replace everything from the first-last modification.
921  // FIXME: use a real diff instead, this is bad with include-insertion.
922 
923  unsigned Offset = 0;
924  while (!Old.empty() && !New.empty() && Old.front() == New.front()) {
925  ++Offset;
926  Old = Old.drop_front();
927  New = New.drop_front();
928  }
929  while (!Old.empty() && !New.empty() && Old.back() == New.back()) {
930  Old = Old.drop_back();
931  New = New.drop_back();
932  }
933 
934  if (Old.empty() && New.empty())
935  return {};
937  Edit.startToken = Offset;
938  Edit.deleteTokens = Old.size();
939  Edit.tokens = New;
940  return {std::move(Edit)};
941 }
942 
943 } // namespace clangd
944 } // namespace clang
Range
CharSourceRange Range
SourceRange for the file name.
Definition: IncludeOrderCheck.cpp:38
clang::clangd::HighlightingModifier::FunctionScope
@ FunctionScope
Loc
SourceLocation Loc
Definition: KernelNameRestrictionCheck.cpp:45
clang::clangd::HighlightingModifier::FileScope
@ FileScope
RecursiveASTVisitor
clang::clangd::HighlightingModifier
HighlightingModifier
Definition: SemanticHighlighting.h:61
clang::clangd::Edit
A set of edits generated for a single file.
Definition: SourceCode.h:180
clang::clangd::HighlightingToken::Modifiers
uint32_t Modifiers
Definition: SemanticHighlighting.h:86
E
const Expr * E
Definition: AvoidBindCheck.cpp:88
Type
NodeType Type
Definition: HTMLGenerator.cpp:73
clang::clangd::CompletionItemKind::Class
@ Class
clang::clangd::HighlightingKind::Primitive
@ Primitive
clang::doc::MD
static GeneratorRegistry::Add< MDGenerator > MD(MDGenerator::Format, "Generator for MD output.")
clang::clangd::toSemanticTokenModifier
llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier)
Definition: SemanticHighlighting.cpp:887
clang::clangd::HighlightingModifier::Deduced
@ Deduced
clang::clangd::ParsedAST::getASTContext
ASTContext & getASTContext()
Note that the returned ast will not contain decls from the preamble that were not deserialized during...
Definition: ParsedAST.cpp:519
clang::clangd::SemanticTokensEdit
Describes a a replacement of a contiguous range of semanticTokens.
Definition: Protocol.h:1608
clang::clangd::lspLength
size_t lspLength(llvm::StringRef Code)
Definition: SourceCode.cpp:151
clang::clangd::HighlightingToken::R
Range R
Definition: SemanticHighlighting.h:87
Kind
BindArgumentKind Kind
Definition: AvoidBindCheck.cpp:59
clang::clangd::Range::start
Position start
The range's start position.
Definition: Protocol.h:178
clang::clangd::HighlightingKind
HighlightingKind
Definition: SemanticHighlighting.h:30
clang::clangd::HighlightingKind::Typedef
@ Typedef
clang::clangd::MainFileMacros::MacroRefs
llvm::DenseMap< SymbolID, std::vector< MacroOccurrence > > MacroRefs
Definition: CollectMacros.h:33
clang::clangd::HighlightingToken::Kind
HighlightingKind Kind
Definition: SemanticHighlighting.h:85
SourceMgr
llvm::SourceMgr * SourceMgr
Definition: ConfigCompile.cpp:102
clang::clangd::HighlightingKind::TemplateParameter
@ TemplateParameter
clang::clangd::operator<
bool operator<(const HighlightingToken &L, const HighlightingToken &R)
Definition: SemanticHighlighting.cpp:805
FindTarget.h
Target
std::string Target
Definition: QueryDriverDatabase.cpp:64
clang::clangd::HighlightingKind::Macro
@ Macro
clang::clangd::Position::line
int line
Line position in a document (zero-based).
Definition: Protocol.h:149
clang::clangd::HighlightingKind::Variable
@ Variable
clang::clangd::HighlightingKind::InactiveCode
@ InactiveCode
Protocol.h
M
const google::protobuf::Message & M
Definition: Server.cpp:309
Offset
size_t Offset
Definition: CodeComplete.cpp:1095
clang::clangd::halfOpenToRange
Range halfOpenToRange(const SourceManager &SM, CharSourceRange R)
Definition: SourceCode.cpp:466
ns1::ns2::D
@ D
Definition: CategoricalFeature.h:3
clang::clangd::HighlightingKind::Function
@ Function
Decl
const FunctionDecl * Decl
Definition: AvoidBindCheck.cpp:100
clang::clangd::Unknown
@ Unknown
Definition: FuzzyMatch.h:56
Line
int Line
Definition: PreprocessorTracker.cpp:514
clang::clangd::SemanticToken
Specifies a single semantic token in the document.
Definition: Protocol.h:1560
clang::clangd::diffTokens
std::vector< SemanticTokensEdit > diffTokens(llvm::ArrayRef< SemanticToken > Old, llvm::ArrayRef< SemanticToken > New)
Definition: SemanticHighlighting.cpp:918
clang::clangd::HighlightingKind::Unknown
@ Unknown
Builder
CodeCompletionBuilder Builder
Definition: CodeCompletionStringsTests.cpp:36
Logger.h
clang::clangd::positionToOffset
llvm::Expected< size_t > positionToOffset(llvm::StringRef Code, Position P, bool AllowColumnsBeyondLineLength)
Turn a [line, column] pair into an offset in Code.
Definition: SourceCode.cpp:175
if
if(CLANGD_ENABLE_REMOTE) generate_protos(RemoteIndexProto "Index.proto") generate_protos(MonitoringServiceProto "MonitoringService.proto" GRPC) generate_protos(RemoteIndexServiceProto "Service.proto" DEPENDS "Index.proto" GRPC) target_link_libraries(RemoteIndexServiceProto PRIVATE RemoteIndexProto MonitoringServiceProto) include_directories($
Definition: clangd/index/remote/CMakeLists.txt:1
clang::clangd::HighlightingKind::Namespace
@ Namespace
Name
static constexpr llvm::StringLiteral Name
Definition: UppercaseLiteralSuffixCheck.cpp:27
clang::clangd::HighlightingKind::Type
@ Type
clang::clangd::HighlightingKind::Concept
@ Concept
clang::clangd::HighlightingKind::LocalVariable
@ LocalVariable
clang::clangd::HighlightingModifier::Deprecated
@ Deprecated
clang::clangd::Position::character
int character
Character offset on a line in a document (zero-based).
Definition: Protocol.h:154
clang::clangd::HeuristicResolver::resolveUsingValueDecl
std::vector< const NamedDecl * > resolveUsingValueDecl(const UnresolvedUsingValueDecl *UUVD) const
Definition: HeuristicResolver.cpp:165
clang::clangd::HighlightingModifier::Abstract
@ Abstract
clang::clangd::HighlightingModifier::GlobalScope
@ GlobalScope
clang::clangd::ParsedAST::getHeuristicResolver
const HeuristicResolver * getHeuristicResolver() const
Definition: ParsedAST.h:115
clang::clangd::DeclRelation::Underlying
@ Underlying
This is the underlying declaration for a renaming-alias, decltype etc.
clang::clangd::HighlightingKind::Enum
@ Enum
clang::clangd::getSemanticHighlightings
std::vector< HighlightingToken > getSemanticHighlightings(ParsedAST &AST)
Definition: SemanticHighlighting.cpp:684
clang::clangd::HighlightingKind::Method
@ Method
SourceCode.h
HeuristicResolver.h
clang::clangd::operator==
bool operator==(const HighlightingToken &L, const HighlightingToken &R)
Definition: SemanticHighlighting.cpp:801
clang::clangd::HighlightingModifier::Static
@ Static
clang::clangd::HighlightingKind::Parameter
@ Parameter
clang::clangd::ReferenceLoc
Information about a reference written in the source code, independent of the actual AST node that thi...
Definition: FindTarget.h:129
clang::clangd::HighlightingToken
Definition: SemanticHighlighting.h:84
clang::clangd::toSemanticTokens
std::vector< SemanticToken > toSemanticTokens(llvm::ArrayRef< HighlightingToken > Tokens)
Definition: SemanticHighlighting.cpp:811
clang::clangd::HighlightingKind::StaticMethod
@ StaticMethod
C
const Criteria C
Definition: FunctionCognitiveComplexityCheck.cpp:93
clang::clangd::toSemanticTokenType
llvm::StringRef toSemanticTokenType(HighlightingKind Kind)
Definition: SemanticHighlighting.cpp:841
SemanticHighlighting.h
clang::clangd::HighlightingModifier::DefaultLibrary
@ DefaultLibrary
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
OS
llvm::raw_string_ostream OS
Definition: TraceTests.cpp:163
clang::clangd::HighlightingKind::Interface
@ Interface
clang::clangd::HighlightingKind::StaticField
@ StaticField
clang::clangd::operator<<
llvm::raw_ostream & operator<<(llvm::raw_ostream &OS, HighlightingModifier K)
Definition: SemanticHighlighting.cpp:792
clang::clangd::ReferenceLoc::IsDecl
bool IsDecl
True if the reference is a declaration or definition;.
Definition: FindTarget.h:135
clang::clangd::ParsedAST
Stores and provides access to parsed AST.
Definition: ParsedAST.h:49
clang::clangd::HighlightingKind::EnumConstant
@ EnumConstant
clang::clangd::MainFileMacros::UnknownMacros
std::vector< MacroOccurrence > UnknownMacros
Definition: CollectMacros.h:37
Out
CompiledFragmentImpl & Out
Definition: ConfigCompile.cpp:100
clang::clangd::elog
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:62
clang::clangd::findExplicitReferences
void findExplicitReferences(const Stmt *S, llvm::function_ref< void(ReferenceLoc)> Out, const HeuristicResolver *Resolver)
Recursively traverse S and report all references explicitly written in the code.
Definition: FindTarget.cpp:1100
clang::clangd::HighlightingKind::Field
@ Field
clang::clangd::HighlightingKind::Class
@ Class
clang::clangd::ReferenceLoc::Targets
llvm::SmallVector< const NamedDecl *, 1 > Targets
A list of targets referenced by this name.
Definition: FindTarget.h:141
clang::clangd::HighlightingModifier::Readonly
@ Readonly
K
Kind K
Definition: Rename.cpp:442
clang::clangd::HighlightingModifier::DependentName
@ DependentName
clang::clangd::HighlightingModifier::ClassScope
@ ClassScope
ParsedAST.h
clang::clangd::HighlightingModifier::Declaration
@ Declaration
clang::clangd::ParsedAST::getMacros
const MainFileMacros & getMacros() const
Gets all macro references (definition, expansions) present in the main file, including those in the p...
Definition: ParsedAST.cpp:539