clang 17.0.0git
SortJavaScriptImports.cpp
Go to the documentation of this file.
1//===--- SortJavaScriptImports.cpp - Sort ES6 Imports -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements a sort operation for JavaScript ES6 imports.
11///
12//===----------------------------------------------------------------------===//
13
15#include "TokenAnalyzer.h"
16#include "TokenAnnotator.h"
19#include "clang/Basic/LLVM.h"
23#include "clang/Format/Format.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/Support/Debug.h"
27#include <algorithm>
28#include <string>
29
30#define DEBUG_TYPE "format-formatter"
31
32namespace clang {
33namespace format {
34
35class FormatTokenLexer;
36
38
39// An imported symbol in a JavaScript ES6 import/export, possibly aliased.
41 StringRef Symbol;
42 StringRef Alias;
44
45 bool operator==(const JsImportedSymbol &RHS) const {
46 // Ignore Range for comparison, it is only used to stitch code together,
47 // but imports at different code locations are still conceptually the same.
48 return Symbol == RHS.Symbol && Alias == RHS.Alias;
49 }
50};
51
52// An ES6 module reference.
53//
54// ES6 implements a module system, where individual modules (~= source files)
55// can reference other modules, either importing symbols from them, or exporting
56// symbols from them:
57// import {foo} from 'foo';
58// export {foo};
59// export {bar} from 'bar';
60//
61// `export`s with URLs are syntactic sugar for an import of the symbol from the
62// URL, followed by an export of the symbol, allowing this code to treat both
63// statements more or less identically, with the exception being that `export`s
64// are sorted last.
65//
66// imports and exports support individual symbols, but also a wildcard syntax:
67// import * as prefix from 'foo';
68// export * from 'bar';
69//
70// This struct represents both exports and imports to build up the information
71// required for sorting module references.
73 bool FormattingOff = false;
74 bool IsExport = false;
75 // Module references are sorted into these categories, in order.
77 SIDE_EFFECT, // "import 'something';"
78 ABSOLUTE, // from 'something'
79 RELATIVE_PARENT, // from '../*'
80 RELATIVE, // from './*'
81 ALIAS, // import X = A.B;
82 };
84 // The URL imported, e.g. `import .. from 'url';`. Empty for `export {a, b};`.
85 StringRef URL;
86 // Prefix from "import * as prefix". Empty for symbol imports and `export *`.
87 // Implies an empty names list.
88 StringRef Prefix;
89 // Default import from "import DefaultName from '...';".
90 StringRef DefaultImport;
91 // Symbols from `import {SymbolA, SymbolB, ...} from ...;`.
93 // Whether some symbols were merged into this one. Controls if the module
94 // reference needs re-formatting.
95 bool SymbolsMerged = false;
96 // The source location just after { and just before } in the import.
97 // Extracted eagerly to allow modification of Symbols later on.
99 // Textual position of the import/export, including preceding and trailing
100 // comments.
102};
103
104bool operator<(const JsModuleReference &LHS, const JsModuleReference &RHS) {
105 if (LHS.IsExport != RHS.IsExport)
106 return LHS.IsExport < RHS.IsExport;
107 if (LHS.Category != RHS.Category)
108 return LHS.Category < RHS.Category;
111 // Side effect imports and aliases might be ordering sensitive. Consider
112 // them equal so that they maintain their relative order in the stable sort
113 // below. This retains transitivity because LHS.Category == RHS.Category
114 // here.
115 return false;
116 }
117 // Empty URLs sort *last* (for export {...};).
118 if (LHS.URL.empty() != RHS.URL.empty())
119 return LHS.URL.empty() < RHS.URL.empty();
120 if (int Res = LHS.URL.compare_insensitive(RHS.URL))
121 return Res < 0;
122 // '*' imports (with prefix) sort before {a, b, ...} imports.
123 if (LHS.Prefix.empty() != RHS.Prefix.empty())
124 return LHS.Prefix.empty() < RHS.Prefix.empty();
125 if (LHS.Prefix != RHS.Prefix)
126 return LHS.Prefix > RHS.Prefix;
127 return false;
128}
129
130// JavaScriptImportSorter sorts JavaScript ES6 imports and exports. It is
131// implemented as a TokenAnalyzer because ES6 imports have substantial syntactic
132// structure, making it messy to sort them using regular expressions.
134public:
137 FileContents(Env.getSourceManager().getBufferData(Env.getFileID())) {
138 // FormatToken.Tok starts out in an uninitialized state.
139 invalidToken.Tok.startToken();
140 }
141
142 std::pair<tooling::Replacements, unsigned>
144 SmallVectorImpl<AnnotatedLine *> &AnnotatedLines,
145 FormatTokenLexer &Tokens) override {
148
149 const AdditionalKeywords &Keywords = Tokens.getKeywords();
151 AnnotatedLine *FirstNonImportLine;
152 std::tie(References, FirstNonImportLine) =
153 parseModuleReferences(Keywords, AnnotatedLines);
154
155 if (References.empty())
156 return {Result, 0};
157
158 // The text range of all parsed imports, to be replaced later.
159 SourceRange InsertionPoint = References[0].Range;
160 InsertionPoint.setEnd(References[References.size() - 1].Range.getEnd());
161
162 References = sortModuleReferences(References);
163
164 std::string ReferencesText;
165 for (unsigned I = 0, E = References.size(); I != E; ++I) {
166 JsModuleReference Reference = References[I];
167 appendReference(ReferencesText, Reference);
168 if (I + 1 < E) {
169 // Insert breaks between imports and exports.
170 ReferencesText += "\n";
171 // Separate imports groups with two line breaks, but keep all exports
172 // in a single group.
173 if (!Reference.IsExport &&
174 (Reference.IsExport != References[I + 1].IsExport ||
175 Reference.Category != References[I + 1].Category)) {
176 ReferencesText += "\n";
177 }
178 }
179 }
180 llvm::StringRef PreviousText = getSourceText(InsertionPoint);
181 if (ReferencesText == PreviousText)
182 return {Result, 0};
183
184 // The loop above might collapse previously existing line breaks between
185 // import blocks, and thus shrink the file. SortIncludes must not shrink
186 // overall source length as there is currently no re-calculation of ranges
187 // after applying source sorting.
188 // This loop just backfills trailing spaces after the imports, which are
189 // harmless and will be stripped by the subsequent formatting pass.
190 // FIXME: A better long term fix is to re-calculate Ranges after sorting.
191 unsigned PreviousSize = PreviousText.size();
192 while (ReferencesText.size() < PreviousSize)
193 ReferencesText += " ";
194
195 // Separate references from the main code body of the file.
196 if (FirstNonImportLine && FirstNonImportLine->First->NewlinesBefore < 2 &&
197 !(FirstNonImportLine->First->is(tok::comment) &&
198 isClangFormatOn(FirstNonImportLine->First->TokenText.trim()))) {
199 ReferencesText += "\n";
200 }
201
202 LLVM_DEBUG(llvm::dbgs() << "Replacing imports:\n"
203 << PreviousText << "\nwith:\n"
204 << ReferencesText << "\n");
205 auto Err = Result.add(tooling::Replacement(
207 ReferencesText));
208 // FIXME: better error handling. For now, just print error message and skip
209 // the replacement for the release version.
210 if (Err) {
211 llvm::errs() << llvm::toString(std::move(Err)) << "\n";
212 assert(false);
213 }
214
215 return {Result, 0};
216 }
217
218private:
219 FormatToken *Current;
220 FormatToken *LineEnd;
221
222 FormatToken invalidToken;
223
224 StringRef FileContents;
225
226 void skipComments() { Current = skipComments(Current); }
227
228 FormatToken *skipComments(FormatToken *Tok) {
229 while (Tok && Tok->is(tok::comment))
230 Tok = Tok->Next;
231 return Tok;
232 }
233
234 void nextToken() {
235 Current = Current->Next;
236 skipComments();
237 if (!Current || Current == LineEnd->Next) {
238 // Set the current token to an invalid token, so that further parsing on
239 // this line fails.
240 Current = &invalidToken;
241 }
242 }
243
244 StringRef getSourceText(SourceRange Range) {
245 return getSourceText(Range.getBegin(), Range.getEnd());
246 }
247
248 StringRef getSourceText(SourceLocation Begin, SourceLocation End) {
249 const SourceManager &SM = Env.getSourceManager();
250 return FileContents.substr(SM.getFileOffset(Begin),
251 SM.getFileOffset(End) - SM.getFileOffset(Begin));
252 }
253
254 // Sorts the given module references.
255 // Imports can have formatting disabled (FormattingOff), so the code below
256 // skips runs of "no-formatting" module references, and sorts/merges the
257 // references that have formatting enabled in individual chunks.
258 SmallVector<JsModuleReference, 16>
259 sortModuleReferences(const SmallVector<JsModuleReference, 16> &References) {
260 // Sort module references.
261 // Imports can have formatting disabled (FormattingOff), so the code below
262 // skips runs of "no-formatting" module references, and sorts other
263 // references per group.
264 const auto *Start = References.begin();
265 SmallVector<JsModuleReference, 16> ReferencesSorted;
266 while (Start != References.end()) {
267 while (Start != References.end() && Start->FormattingOff) {
268 // Skip over all imports w/ disabled formatting.
269 ReferencesSorted.push_back(*Start);
270 ++Start;
271 }
272 SmallVector<JsModuleReference, 16> SortChunk;
273 while (Start != References.end() && !Start->FormattingOff) {
274 // Skip over all imports w/ disabled formatting.
275 SortChunk.push_back(*Start);
276 ++Start;
277 }
278 llvm::stable_sort(SortChunk);
279 mergeModuleReferences(SortChunk);
280 ReferencesSorted.insert(ReferencesSorted.end(), SortChunk.begin(),
281 SortChunk.end());
282 }
283 return ReferencesSorted;
284 }
285
286 // Merge module references.
287 // After sorting, find all references that import named symbols from the
288 // same URL and merge their names. E.g.
289 // import {X} from 'a';
290 // import {Y} from 'a';
291 // should be rewritten to:
292 // import {X, Y} from 'a';
293 // Note: this modifies the passed in ``References`` vector (by removing no
294 // longer needed references).
295 void mergeModuleReferences(SmallVector<JsModuleReference, 16> &References) {
296 if (References.empty())
297 return;
298 JsModuleReference *PreviousReference = References.begin();
299 auto *Reference = std::next(References.begin());
300 while (Reference != References.end()) {
301 // Skip:
302 // import 'foo';
303 // import * as foo from 'foo'; on either previous or this.
304 // import Default from 'foo'; on either previous or this.
305 // mismatching
306 if (Reference->Category == JsModuleReference::SIDE_EFFECT ||
307 PreviousReference->Category == JsModuleReference::SIDE_EFFECT ||
308 Reference->IsExport != PreviousReference->IsExport ||
309 !PreviousReference->Prefix.empty() || !Reference->Prefix.empty() ||
310 !PreviousReference->DefaultImport.empty() ||
311 !Reference->DefaultImport.empty() || Reference->Symbols.empty() ||
312 PreviousReference->URL != Reference->URL) {
313 PreviousReference = Reference;
314 ++Reference;
315 continue;
316 }
317 // Merge symbols from identical imports.
318 PreviousReference->Symbols.append(Reference->Symbols);
319 PreviousReference->SymbolsMerged = true;
320 // Remove the merged import.
321 Reference = References.erase(Reference);
322 }
323 }
324
325 // Appends ``Reference`` to ``Buffer``.
326 void appendReference(std::string &Buffer, JsModuleReference &Reference) {
327 if (Reference.FormattingOff) {
328 Buffer +=
329 getSourceText(Reference.Range.getBegin(), Reference.Range.getEnd());
330 return;
331 }
332 // Sort the individual symbols within the import.
333 // E.g. `import {b, a} from 'x';` -> `import {a, b} from 'x';`
334 SmallVector<JsImportedSymbol, 1> Symbols = Reference.Symbols;
335 llvm::stable_sort(
336 Symbols, [&](const JsImportedSymbol &LHS, const JsImportedSymbol &RHS) {
337 return LHS.Symbol.compare_insensitive(RHS.Symbol) < 0;
338 });
339 if (!Reference.SymbolsMerged && Symbols == Reference.Symbols) {
340 // Symbols didn't change, just emit the entire module reference.
341 StringRef ReferenceStmt = getSourceText(Reference.Range);
342 Buffer += ReferenceStmt;
343 return;
344 }
345 // Stitch together the module reference start...
346 Buffer += getSourceText(Reference.Range.getBegin(), Reference.SymbolsStart);
347 // ... then the references in order ...
348 if (!Symbols.empty()) {
349 Buffer += getSourceText(Symbols.front().Range);
350 for (const JsImportedSymbol &Symbol : llvm::drop_begin(Symbols)) {
351 Buffer += ",";
352 Buffer += getSourceText(Symbol.Range);
353 }
354 }
355 // ... followed by the module reference end.
356 Buffer += getSourceText(Reference.SymbolsEnd, Reference.Range.getEnd());
357 }
358
359 // Parses module references in the given lines. Returns the module references,
360 // and a pointer to the first "main code" line if that is adjacent to the
361 // affected lines of module references, nullptr otherwise.
362 std::pair<SmallVector<JsModuleReference, 16>, AnnotatedLine *>
363 parseModuleReferences(const AdditionalKeywords &Keywords,
364 SmallVectorImpl<AnnotatedLine *> &AnnotatedLines) {
365 SmallVector<JsModuleReference, 16> References;
366 SourceLocation Start;
367 AnnotatedLine *FirstNonImportLine = nullptr;
368 bool AnyImportAffected = false;
369 bool FormattingOff = false;
370 for (auto *Line : AnnotatedLines) {
371 assert(Line->First);
372 Current = Line->First;
373 LineEnd = Line->Last;
374 // clang-format comments toggle formatting on/off.
375 // This is tracked in FormattingOff here and on JsModuleReference.
376 while (Current && Current->is(tok::comment)) {
377 StringRef CommentText = Current->TokenText.trim();
378 if (isClangFormatOff(CommentText)) {
379 FormattingOff = true;
380 } else if (isClangFormatOn(CommentText)) {
381 FormattingOff = false;
382 // Special case: consider a trailing "clang-format on" line to be part
383 // of the module reference, so that it gets moved around together with
384 // it (as opposed to the next module reference, which might get sorted
385 // around).
386 if (!References.empty()) {
387 References.back().Range.setEnd(Current->Tok.getEndLoc());
388 Start = Current->Tok.getEndLoc().getLocWithOffset(1);
389 }
390 }
391 // Handle all clang-format comments on a line, e.g. for an empty block.
392 Current = Current->Next;
393 }
394 skipComments();
395 if (Start.isInvalid() || References.empty()) {
396 // After the first file level comment, consider line comments to be part
397 // of the import that immediately follows them by using the previously
398 // set Start.
399 Start = Line->First->Tok.getLocation();
400 }
401 if (!Current) {
402 // Only comments on this line. Could be the first non-import line.
403 FirstNonImportLine = Line;
404 continue;
405 }
406 JsModuleReference Reference;
407 Reference.FormattingOff = FormattingOff;
408 Reference.Range.setBegin(Start);
409 // References w/o a URL, e.g. export {A}, groups with RELATIVE.
411 if (!parseModuleReference(Keywords, Reference)) {
412 if (!FirstNonImportLine)
413 FirstNonImportLine = Line; // if no comment before.
414 break;
415 }
416 FirstNonImportLine = nullptr;
417 AnyImportAffected = AnyImportAffected || Line->Affected;
418 Reference.Range.setEnd(LineEnd->Tok.getEndLoc());
419 LLVM_DEBUG({
420 llvm::dbgs() << "JsModuleReference: {"
421 << "formatting_off: " << Reference.FormattingOff
422 << ", is_export: " << Reference.IsExport
423 << ", cat: " << Reference.Category
424 << ", url: " << Reference.URL
425 << ", prefix: " << Reference.Prefix;
426 for (const JsImportedSymbol &Symbol : Reference.Symbols)
427 llvm::dbgs() << ", " << Symbol.Symbol << " as " << Symbol.Alias;
428 llvm::dbgs() << ", text: " << getSourceText(Reference.Range);
429 llvm::dbgs() << "}\n";
430 });
431 References.push_back(Reference);
432 Start = SourceLocation();
433 }
434 // Sort imports if any import line was affected.
435 if (!AnyImportAffected)
436 References.clear();
437 return std::make_pair(References, FirstNonImportLine);
438 }
439
440 // Parses a JavaScript/ECMAScript 6 module reference.
441 // See http://www.ecma-international.org/ecma-262/6.0/#sec-scripts-and-modules
442 // for grammar EBNF (production ModuleItem).
443 bool parseModuleReference(const AdditionalKeywords &Keywords,
444 JsModuleReference &Reference) {
445 if (!Current || !Current->isOneOf(Keywords.kw_import, tok::kw_export))
446 return false;
447 Reference.IsExport = Current->is(tok::kw_export);
448
449 nextToken();
450 if (Current->isStringLiteral() && !Reference.IsExport) {
451 // "import 'side-effect';"
453 Reference.URL =
454 Current->TokenText.substr(1, Current->TokenText.size() - 2);
455 return true;
456 }
457
458 if (!parseModuleBindings(Keywords, Reference))
459 return false;
460
461 if (Current->is(Keywords.kw_from)) {
462 // imports have a 'from' clause, exports might not.
463 nextToken();
464 if (!Current->isStringLiteral())
465 return false;
466 // URL = TokenText without the quotes.
467 Reference.URL =
468 Current->TokenText.substr(1, Current->TokenText.size() - 2);
469 if (Reference.URL.startswith("..")) {
470 Reference.Category =
472 } else if (Reference.URL.startswith(".")) {
474 } else {
476 }
477 }
478 return true;
479 }
480
481 bool parseModuleBindings(const AdditionalKeywords &Keywords,
482 JsModuleReference &Reference) {
483 if (parseStarBinding(Keywords, Reference))
484 return true;
485 return parseNamedBindings(Keywords, Reference);
486 }
487
488 bool parseStarBinding(const AdditionalKeywords &Keywords,
489 JsModuleReference &Reference) {
490 // * as prefix from '...';
491 if (Current->isNot(tok::star))
492 return false;
493 nextToken();
494 if (Current->isNot(Keywords.kw_as))
495 return false;
496 nextToken();
497 if (Current->isNot(tok::identifier))
498 return false;
499 Reference.Prefix = Current->TokenText;
500 nextToken();
501 return true;
502 }
503
504 bool parseNamedBindings(const AdditionalKeywords &Keywords,
505 JsModuleReference &Reference) {
506 // eat a potential "import X, " prefix.
507 if (Current->is(tok::identifier)) {
508 Reference.DefaultImport = Current->TokenText;
509 nextToken();
510 if (Current->is(Keywords.kw_from))
511 return true;
512 // import X = A.B.C;
513 if (Current->is(tok::equal)) {
515 nextToken();
516 while (Current->is(tok::identifier)) {
517 nextToken();
518 if (Current->is(tok::semi))
519 return true;
520 if (!Current->is(tok::period))
521 return false;
522 nextToken();
523 }
524 }
525 if (Current->isNot(tok::comma))
526 return false;
527 nextToken(); // eat comma.
528 }
529 if (Current->isNot(tok::l_brace))
530 return false;
531
532 // {sym as alias, sym2 as ...} from '...';
533 Reference.SymbolsStart = Current->Tok.getEndLoc();
534 while (Current->isNot(tok::r_brace)) {
535 nextToken();
536 if (Current->is(tok::r_brace))
537 break;
538 if (!Current->isOneOf(tok::identifier, tok::kw_default))
539 return false;
540
541 JsImportedSymbol Symbol;
542 Symbol.Symbol = Current->TokenText;
543 // Make sure to include any preceding comments.
544 Symbol.Range.setBegin(
545 Current->getPreviousNonComment()->Next->WhitespaceRange.getBegin());
546 nextToken();
547
548 if (Current->is(Keywords.kw_as)) {
549 nextToken();
550 if (!Current->isOneOf(tok::identifier, tok::kw_default))
551 return false;
552 Symbol.Alias = Current->TokenText;
553 nextToken();
554 }
555 Symbol.Range.setEnd(Current->Tok.getLocation());
556 Reference.Symbols.push_back(Symbol);
557
558 if (!Current->isOneOf(tok::r_brace, tok::comma))
559 return false;
560 }
561 Reference.SymbolsEnd = Current->Tok.getLocation();
562 // For named imports with a trailing comma ("import {X,}"), consider the
563 // comma to be the end of the import list, so that it doesn't get removed.
564 if (Current->Previous->is(tok::comma))
565 Reference.SymbolsEnd = Current->Previous->Tok.getLocation();
566 nextToken(); // consume r_brace
567 return true;
568 }
569};
570
572 StringRef Code,
574 StringRef FileName) {
575 // FIXME: Cursor support.
576 auto Env = Environment::make(Code, FileName, Ranges);
577 if (!Env)
578 return {};
579 return JavaScriptImportSorter(*Env, Style).process().first;
580}
581
582} // end namespace format
583} // end namespace clang
#define SM(sm)
Definition: Cuda.cpp:78
Defines the Diagnostic-related interfaces.
Various functions to configurably format source code.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
This file implements a sorter for JavaScript ES6 imports.
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
This file declares an abstract TokenAnalyzer, and associated helper classes.
This file implements a token annotator, i.e.
Defines the clang::TokenKind enum and support functions.
SourceLocation Begin
static CharSourceRange getCharRange(SourceRange R)
Encodes a location in the source.
A trivial tuple used to represent a source range.
void setEnd(SourceLocation e)
SourceLocation getEndLoc() const
Definition: Token.h:153
void startToken()
Reset all flags to cleared.
Definition: Token.h:171
bool computeAffectedLines(SmallVectorImpl< AnnotatedLine * > &Lines)
SourceManager & getSourceManager() const
Definition: TokenAnalyzer.h:49
static std::unique_ptr< Environment > make(StringRef Code, StringRef FileName, ArrayRef< tooling::Range > Ranges, unsigned FirstStartColumn=0, unsigned NextStartColumn=0, unsigned LastStartColumn=0)
std::pair< tooling::Replacements, unsigned > analyze(TokenAnnotator &Annotator, SmallVectorImpl< AnnotatedLine * > &AnnotatedLines, FormatTokenLexer &Tokens) override
JavaScriptImportSorter(const Environment &Env, const FormatStyle &Style)
AffectedRangeManager AffectedRangeMgr
const Environment & Env
std::pair< tooling::Replacements, unsigned > process(bool SkipAnnotation=false)
Determines extra information about the tokens comprising an UnwrappedLine.
A text replacement.
Definition: Replacement.h:83
Maintains a set of replacements that are conflict-free.
Definition: Replacement.h:212
@ Reference
An optional reference should be skipped past.
bool operator<(const JsModuleReference &LHS, const JsModuleReference &RHS)
bool isClangFormatOff(StringRef Comment)
Definition: Format.cpp:3911
tooling::Replacements sortJavaScriptImports(const FormatStyle &Style, StringRef Code, ArrayRef< tooling::Range > Ranges, StringRef FileName)
bool isClangFormatOn(StringRef Comment)
Definition: Format.cpp:3907
@ Result
The result type of a method or function.
Encapsulates keywords that are context sensitive or for languages not properly supported by Clang's l...
Definition: FormatToken.h:933
The FormatStyle is used to configure the formatting to follow specific guidelines.
Definition: Format.h:55
A wrapper around a Token storing information about the whitespace characters preceding it.
Definition: FormatToken.h:247
StringRef TokenText
The raw text of the token.
Definition: FormatToken.h:266
FormatToken * Next
The next token in the unwrapped line.
Definition: FormatToken.h:504
unsigned NewlinesBefore
The number of newlines immediately before the Token.
Definition: FormatToken.h:407
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:541
bool operator==(const JsImportedSymbol &RHS) const
SmallVector< JsImportedSymbol, 1 > Symbols