clang  9.0.0svn
JSONCompilationDatabase.cpp
Go to the documentation of this file.
1 //===- JSONCompilationDatabase.cpp ----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the implementation of the JSONCompilationDatabase.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "clang/Basic/LLVM.h"
17 #include "llvm/ADT/Optional.h"
18 #include "llvm/ADT/SmallString.h"
19 #include "llvm/ADT/SmallVector.h"
20 #include "llvm/ADT/StringRef.h"
21 #include "llvm/ADT/Triple.h"
22 #include "llvm/Support/Allocator.h"
23 #include "llvm/Support/Casting.h"
24 #include "llvm/Support/CommandLine.h"
25 #include "llvm/Support/ErrorOr.h"
26 #include "llvm/Support/Host.h"
27 #include "llvm/Support/MemoryBuffer.h"
28 #include "llvm/Support/Path.h"
29 #include "llvm/Support/StringSaver.h"
30 #include "llvm/Support/YAMLParser.h"
31 #include "llvm/Support/raw_ostream.h"
32 #include <cassert>
33 #include <memory>
34 #include <string>
35 #include <system_error>
36 #include <tuple>
37 #include <utility>
38 #include <vector>
39 
40 using namespace clang;
41 using namespace tooling;
42 
43 namespace {
44 
45 /// A parser for escaped strings of command line arguments.
46 ///
47 /// Assumes \-escaping for quoted arguments (see the documentation of
48 /// unescapeCommandLine(...)).
49 class CommandLineArgumentParser {
50  public:
51  CommandLineArgumentParser(StringRef CommandLine)
52  : Input(CommandLine), Position(Input.begin()-1) {}
53 
54  std::vector<std::string> parse() {
55  bool HasMoreInput = true;
56  while (HasMoreInput && nextNonWhitespace()) {
57  std::string Argument;
58  HasMoreInput = parseStringInto(Argument);
59  CommandLine.push_back(Argument);
60  }
61  return CommandLine;
62  }
63 
64  private:
65  // All private methods return true if there is more input available.
66 
67  bool parseStringInto(std::string &String) {
68  do {
69  if (*Position == '"') {
70  if (!parseDoubleQuotedStringInto(String)) return false;
71  } else if (*Position == '\'') {
72  if (!parseSingleQuotedStringInto(String)) return false;
73  } else {
74  if (!parseFreeStringInto(String)) return false;
75  }
76  } while (*Position != ' ');
77  return true;
78  }
79 
80  bool parseDoubleQuotedStringInto(std::string &String) {
81  if (!next()) return false;
82  while (*Position != '"') {
83  if (!skipEscapeCharacter()) return false;
84  String.push_back(*Position);
85  if (!next()) return false;
86  }
87  return next();
88  }
89 
90  bool parseSingleQuotedStringInto(std::string &String) {
91  if (!next()) return false;
92  while (*Position != '\'') {
93  String.push_back(*Position);
94  if (!next()) return false;
95  }
96  return next();
97  }
98 
99  bool parseFreeStringInto(std::string &String) {
100  do {
101  if (!skipEscapeCharacter()) return false;
102  String.push_back(*Position);
103  if (!next()) return false;
104  } while (*Position != ' ' && *Position != '"' && *Position != '\'');
105  return true;
106  }
107 
108  bool skipEscapeCharacter() {
109  if (*Position == '\\') {
110  return next();
111  }
112  return true;
113  }
114 
115  bool nextNonWhitespace() {
116  do {
117  if (!next()) return false;
118  } while (*Position == ' ');
119  return true;
120  }
121 
122  bool next() {
123  ++Position;
124  return Position != Input.end();
125  }
126 
127  const StringRef Input;
128  StringRef::iterator Position;
129  std::vector<std::string> CommandLine;
130 };
131 
132 std::vector<std::string> unescapeCommandLine(JSONCommandLineSyntax Syntax,
133  StringRef EscapedCommandLine) {
134  if (Syntax == JSONCommandLineSyntax::AutoDetect) {
136  llvm::Triple Triple(llvm::sys::getProcessTriple());
137  if (Triple.getOS() == llvm::Triple::OSType::Win32) {
138  // Assume Windows command line parsing on Win32 unless the triple
139  // explicitly tells us otherwise.
140  if (!Triple.hasEnvironment() ||
141  Triple.getEnvironment() == llvm::Triple::EnvironmentType::MSVC)
143  }
144  }
145 
146  if (Syntax == JSONCommandLineSyntax::Windows) {
147  llvm::BumpPtrAllocator Alloc;
148  llvm::StringSaver Saver(Alloc);
150  llvm::cl::TokenizeWindowsCommandLine(EscapedCommandLine, Saver, T);
151  std::vector<std::string> Result(T.begin(), T.end());
152  return Result;
153  }
154  assert(Syntax == JSONCommandLineSyntax::Gnu);
155  CommandLineArgumentParser parser(EscapedCommandLine);
156  return parser.parse();
157 }
158 
159 // This plugin locates a nearby compile_command.json file, and also infers
160 // compile commands for files not present in the database.
161 class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
162  std::unique_ptr<CompilationDatabase>
163  loadFromDirectory(StringRef Directory, std::string &ErrorMessage) override {
164  SmallString<1024> JSONDatabasePath(Directory);
165  llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
167  JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect);
168  return Base ? inferMissingCompileCommands(std::move(Base)) : nullptr;
169  }
170 };
171 
172 } // namespace
173 
174 // Register the JSONCompilationDatabasePlugin with the
175 // CompilationDatabasePluginRegistry using this statically initialized variable.
176 static CompilationDatabasePluginRegistry::Add<JSONCompilationDatabasePlugin>
177 X("json-compilation-database", "Reads JSON formatted compilation databases");
178 
179 namespace clang {
180 namespace tooling {
181 
182 // This anchor is used to force the linker to link in the generated object file
183 // and thus register the JSONCompilationDatabasePlugin.
184 volatile int JSONAnchorSource = 0;
185 
186 } // namespace tooling
187 } // namespace clang
188 
189 std::unique_ptr<JSONCompilationDatabase>
191  std::string &ErrorMessage,
192  JSONCommandLineSyntax Syntax) {
193  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> DatabaseBuffer =
194  llvm::MemoryBuffer::getFile(FilePath);
195  if (std::error_code Result = DatabaseBuffer.getError()) {
196  ErrorMessage = "Error while opening JSON database: " + Result.message();
197  return nullptr;
198  }
199  std::unique_ptr<JSONCompilationDatabase> Database(
200  new JSONCompilationDatabase(std::move(*DatabaseBuffer), Syntax));
201  if (!Database->parse(ErrorMessage))
202  return nullptr;
203  return Database;
204 }
205 
206 std::unique_ptr<JSONCompilationDatabase>
208  std::string &ErrorMessage,
209  JSONCommandLineSyntax Syntax) {
210  std::unique_ptr<llvm::MemoryBuffer> DatabaseBuffer(
211  llvm::MemoryBuffer::getMemBuffer(DatabaseString));
212  std::unique_ptr<JSONCompilationDatabase> Database(
213  new JSONCompilationDatabase(std::move(DatabaseBuffer), Syntax));
214  if (!Database->parse(ErrorMessage))
215  return nullptr;
216  return Database;
217 }
218 
219 std::vector<CompileCommand>
221  SmallString<128> NativeFilePath;
222  llvm::sys::path::native(FilePath, NativeFilePath);
223 
224  std::string Error;
225  llvm::raw_string_ostream ES(Error);
226  StringRef Match = MatchTrie.findEquivalent(NativeFilePath, ES);
227  if (Match.empty())
228  return {};
229  const auto CommandsRefI = IndexByFile.find(Match);
230  if (CommandsRefI == IndexByFile.end())
231  return {};
232  std::vector<CompileCommand> Commands;
233  getCommands(CommandsRefI->getValue(), Commands);
234  return Commands;
235 }
236 
237 std::vector<std::string>
239  std::vector<std::string> Result;
240  for (const auto &CommandRef : IndexByFile)
241  Result.push_back(CommandRef.first().str());
242  return Result;
243 }
244 
245 std::vector<CompileCommand>
247  std::vector<CompileCommand> Commands;
248  getCommands(AllCommands, Commands);
249  return Commands;
250 }
251 
252 static std::vector<std::string>
254  const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
255  SmallString<1024> Storage;
256  if (Nodes.size() == 1)
257  return unescapeCommandLine(Syntax, Nodes[0]->getValue(Storage));
258  std::vector<std::string> Arguments;
259  for (const auto *Node : Nodes)
260  Arguments.push_back(Node->getValue(Storage));
261  return Arguments;
262 }
263 
264 void JSONCompilationDatabase::getCommands(
265  ArrayRef<CompileCommandRef> CommandsRef,
266  std::vector<CompileCommand> &Commands) const {
267  for (const auto &CommandRef : CommandsRef) {
268  SmallString<8> DirectoryStorage;
269  SmallString<32> FilenameStorage;
270  SmallString<32> OutputStorage;
271  auto Output = std::get<3>(CommandRef);
272  Commands.emplace_back(
273  std::get<0>(CommandRef)->getValue(DirectoryStorage),
274  std::get<1>(CommandRef)->getValue(FilenameStorage),
275  nodeToCommandLine(Syntax, std::get<2>(CommandRef)),
276  Output ? Output->getValue(OutputStorage) : "");
277  }
278 }
279 
280 bool JSONCompilationDatabase::parse(std::string &ErrorMessage) {
281  llvm::yaml::document_iterator I = YAMLStream.begin();
282  if (I == YAMLStream.end()) {
283  ErrorMessage = "Error while parsing YAML.";
284  return false;
285  }
286  llvm::yaml::Node *Root = I->getRoot();
287  if (!Root) {
288  ErrorMessage = "Error while parsing YAML.";
289  return false;
290  }
291  auto *Array = dyn_cast<llvm::yaml::SequenceNode>(Root);
292  if (!Array) {
293  ErrorMessage = "Expected array.";
294  return false;
295  }
296  for (auto &NextObject : *Array) {
297  auto *Object = dyn_cast<llvm::yaml::MappingNode>(&NextObject);
298  if (!Object) {
299  ErrorMessage = "Expected object.";
300  return false;
301  }
302  llvm::yaml::ScalarNode *Directory = nullptr;
304  llvm::yaml::ScalarNode *File = nullptr;
305  llvm::yaml::ScalarNode *Output = nullptr;
306  for (auto& NextKeyValue : *Object) {
307  auto *KeyString = dyn_cast<llvm::yaml::ScalarNode>(NextKeyValue.getKey());
308  if (!KeyString) {
309  ErrorMessage = "Expected strings as key.";
310  return false;
311  }
312  SmallString<10> KeyStorage;
313  StringRef KeyValue = KeyString->getValue(KeyStorage);
314  llvm::yaml::Node *Value = NextKeyValue.getValue();
315  if (!Value) {
316  ErrorMessage = "Expected value.";
317  return false;
318  }
319  auto *ValueString = dyn_cast<llvm::yaml::ScalarNode>(Value);
320  auto *SequenceString = dyn_cast<llvm::yaml::SequenceNode>(Value);
321  if (KeyValue == "arguments" && !SequenceString) {
322  ErrorMessage = "Expected sequence as value.";
323  return false;
324  } else if (KeyValue != "arguments" && !ValueString) {
325  ErrorMessage = "Expected string as value.";
326  return false;
327  }
328  if (KeyValue == "directory") {
329  Directory = ValueString;
330  } else if (KeyValue == "arguments") {
331  Command = std::vector<llvm::yaml::ScalarNode *>();
332  for (auto &Argument : *SequenceString) {
333  auto *Scalar = dyn_cast<llvm::yaml::ScalarNode>(&Argument);
334  if (!Scalar) {
335  ErrorMessage = "Only strings are allowed in 'arguments'.";
336  return false;
337  }
338  Command->push_back(Scalar);
339  }
340  } else if (KeyValue == "command") {
341  if (!Command)
342  Command = std::vector<llvm::yaml::ScalarNode *>(1, ValueString);
343  } else if (KeyValue == "file") {
344  File = ValueString;
345  } else if (KeyValue == "output") {
346  Output = ValueString;
347  } else {
348  ErrorMessage = ("Unknown key: \"" +
349  KeyString->getRawValue() + "\"").str();
350  return false;
351  }
352  }
353  if (!File) {
354  ErrorMessage = "Missing key: \"file\".";
355  return false;
356  }
357  if (!Command) {
358  ErrorMessage = "Missing key: \"command\" or \"arguments\".";
359  return false;
360  }
361  if (!Directory) {
362  ErrorMessage = "Missing key: \"directory\".";
363  return false;
364  }
365  SmallString<8> FileStorage;
366  StringRef FileName = File->getValue(FileStorage);
367  SmallString<128> NativeFilePath;
368  if (llvm::sys::path::is_relative(FileName)) {
369  SmallString<8> DirectoryStorage;
370  SmallString<128> AbsolutePath(
371  Directory->getValue(DirectoryStorage));
372  llvm::sys::path::append(AbsolutePath, FileName);
373  llvm::sys::path::remove_dots(AbsolutePath, /*remove_dot_dot=*/ true);
374  llvm::sys::path::native(AbsolutePath, NativeFilePath);
375  } else {
376  llvm::sys::path::native(FileName, NativeFilePath);
377  }
378  auto Cmd = CompileCommandRef(Directory, File, *Command, Output);
379  IndexByFile[NativeFilePath].push_back(Cmd);
380  AllCommands.push_back(Cmd);
381  MatchTrie.insert(NativeFilePath);
382  }
383  return true;
384 }
static std::unique_ptr< JSONCompilationDatabase > loadFromFile(StringRef FilePath, std::string &ErrorMessage, JSONCommandLineSyntax Syntax)
Loads a JSON compilation database from the specified file.
Interface for compilation database plugins.
std::vector< std::string > getAllFiles() const override
Returns the list of all files available in the compilation database.
BoundNodesTreeBuilder Nodes
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
CompileCommand Cmd
static std::unique_ptr< JSONCompilationDatabase > loadFromBuffer(StringRef DatabaseString, std::string &ErrorMessage, JSONCommandLineSyntax Syntax)
Loads a JSON compilation database from a data buffer.
return Out str()
static SVal getValue(SVal val, SValBuilder &svalBuilder)
std::vector< CompileCommand > getCompileCommands(StringRef FilePath) const override
Returns all compile commands in which the specified file was compiled.
std::unique_ptr< CompilationDatabase > inferMissingCompileCommands(std::unique_ptr< CompilationDatabase >)
Returns a wrapped CompilationDatabase that defers to the provided one, but getCompileCommands() will ...
Dataflow Directional Tag Classes.
std::vector< CompileCommand > getAllCompileCommands() const override
Returns all compile commands for all the files in the compilation database.
static ToolExecutorPluginRegistry::Add< AllTUsToolExecutorPlugin > X("all-TUs", "Runs FrontendActions on all TUs in the compilation database. " "Tool results are stored in memory.")
JSONCommandLineSyntax
A JSON based compilation database.
static std::vector< std::string > nodeToCommandLine(JSONCommandLineSyntax Syntax, const std::vector< llvm::yaml::ScalarNode *> &Nodes)