clang  8.0.0svn
JSONCompilationDatabase.cpp
Go to the documentation of this file.
1 //===- JSONCompilationDatabase.cpp ----------------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the implementation of the JSONCompilationDatabase.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "clang/Basic/LLVM.h"
18 #include "llvm/ADT/Optional.h"
19 #include "llvm/ADT/SmallString.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/ADT/StringRef.h"
22 #include "llvm/ADT/Triple.h"
23 #include "llvm/Support/Allocator.h"
24 #include "llvm/Support/Casting.h"
25 #include "llvm/Support/CommandLine.h"
26 #include "llvm/Support/ErrorOr.h"
27 #include "llvm/Support/Host.h"
28 #include "llvm/Support/MemoryBuffer.h"
29 #include "llvm/Support/Path.h"
30 #include "llvm/Support/StringSaver.h"
31 #include "llvm/Support/YAMLParser.h"
32 #include "llvm/Support/raw_ostream.h"
33 #include <cassert>
34 #include <memory>
35 #include <string>
36 #include <system_error>
37 #include <tuple>
38 #include <utility>
39 #include <vector>
40 
41 using namespace clang;
42 using namespace tooling;
43 
44 namespace {
45 
46 /// A parser for escaped strings of command line arguments.
47 ///
48 /// Assumes \-escaping for quoted arguments (see the documentation of
49 /// unescapeCommandLine(...)).
50 class CommandLineArgumentParser {
51  public:
52  CommandLineArgumentParser(StringRef CommandLine)
53  : Input(CommandLine), Position(Input.begin()-1) {}
54 
55  std::vector<std::string> parse() {
56  bool HasMoreInput = true;
57  while (HasMoreInput && nextNonWhitespace()) {
58  std::string Argument;
59  HasMoreInput = parseStringInto(Argument);
60  CommandLine.push_back(Argument);
61  }
62  return CommandLine;
63  }
64 
65  private:
66  // All private methods return true if there is more input available.
67 
68  bool parseStringInto(std::string &String) {
69  do {
70  if (*Position == '"') {
71  if (!parseDoubleQuotedStringInto(String)) return false;
72  } else if (*Position == '\'') {
73  if (!parseSingleQuotedStringInto(String)) return false;
74  } else {
75  if (!parseFreeStringInto(String)) return false;
76  }
77  } while (*Position != ' ');
78  return true;
79  }
80 
81  bool parseDoubleQuotedStringInto(std::string &String) {
82  if (!next()) return false;
83  while (*Position != '"') {
84  if (!skipEscapeCharacter()) return false;
85  String.push_back(*Position);
86  if (!next()) return false;
87  }
88  return next();
89  }
90 
91  bool parseSingleQuotedStringInto(std::string &String) {
92  if (!next()) return false;
93  while (*Position != '\'') {
94  String.push_back(*Position);
95  if (!next()) return false;
96  }
97  return next();
98  }
99 
100  bool parseFreeStringInto(std::string &String) {
101  do {
102  if (!skipEscapeCharacter()) return false;
103  String.push_back(*Position);
104  if (!next()) return false;
105  } while (*Position != ' ' && *Position != '"' && *Position != '\'');
106  return true;
107  }
108 
109  bool skipEscapeCharacter() {
110  if (*Position == '\\') {
111  return next();
112  }
113  return true;
114  }
115 
116  bool nextNonWhitespace() {
117  do {
118  if (!next()) return false;
119  } while (*Position == ' ');
120  return true;
121  }
122 
123  bool next() {
124  ++Position;
125  return Position != Input.end();
126  }
127 
128  const StringRef Input;
129  StringRef::iterator Position;
130  std::vector<std::string> CommandLine;
131 };
132 
133 std::vector<std::string> unescapeCommandLine(JSONCommandLineSyntax Syntax,
134  StringRef EscapedCommandLine) {
135  if (Syntax == JSONCommandLineSyntax::AutoDetect) {
137  llvm::Triple Triple(llvm::sys::getProcessTriple());
138  if (Triple.getOS() == llvm::Triple::OSType::Win32) {
139  // Assume Windows command line parsing on Win32 unless the triple
140  // explicitly tells us otherwise.
141  if (!Triple.hasEnvironment() ||
142  Triple.getEnvironment() == llvm::Triple::EnvironmentType::MSVC)
144  }
145  }
146 
147  if (Syntax == JSONCommandLineSyntax::Windows) {
148  llvm::BumpPtrAllocator Alloc;
149  llvm::StringSaver Saver(Alloc);
151  llvm::cl::TokenizeWindowsCommandLine(EscapedCommandLine, Saver, T);
152  std::vector<std::string> Result(T.begin(), T.end());
153  return Result;
154  }
155  assert(Syntax == JSONCommandLineSyntax::Gnu);
156  CommandLineArgumentParser parser(EscapedCommandLine);
157  return parser.parse();
158 }
159 
160 // This plugin locates a nearby compile_command.json file, and also infers
161 // compile commands for files not present in the database.
162 class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
163  std::unique_ptr<CompilationDatabase>
164  loadFromDirectory(StringRef Directory, std::string &ErrorMessage) override {
165  SmallString<1024> JSONDatabasePath(Directory);
166  llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
168  JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect);
169  return Base ? inferMissingCompileCommands(std::move(Base)) : nullptr;
170  }
171 };
172 
173 } // namespace
174 
175 // Register the JSONCompilationDatabasePlugin with the
176 // CompilationDatabasePluginRegistry using this statically initialized variable.
177 static CompilationDatabasePluginRegistry::Add<JSONCompilationDatabasePlugin>
178 X("json-compilation-database", "Reads JSON formatted compilation databases");
179 
180 namespace clang {
181 namespace tooling {
182 
183 // This anchor is used to force the linker to link in the generated object file
184 // and thus register the JSONCompilationDatabasePlugin.
185 volatile int JSONAnchorSource = 0;
186 
187 } // namespace tooling
188 } // namespace clang
189 
190 std::unique_ptr<JSONCompilationDatabase>
192  std::string &ErrorMessage,
193  JSONCommandLineSyntax Syntax) {
194  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> DatabaseBuffer =
195  llvm::MemoryBuffer::getFile(FilePath);
196  if (std::error_code Result = DatabaseBuffer.getError()) {
197  ErrorMessage = "Error while opening JSON database: " + Result.message();
198  return nullptr;
199  }
200  std::unique_ptr<JSONCompilationDatabase> Database(
201  new JSONCompilationDatabase(std::move(*DatabaseBuffer), Syntax));
202  if (!Database->parse(ErrorMessage))
203  return nullptr;
204  return Database;
205 }
206 
207 std::unique_ptr<JSONCompilationDatabase>
209  std::string &ErrorMessage,
210  JSONCommandLineSyntax Syntax) {
211  std::unique_ptr<llvm::MemoryBuffer> DatabaseBuffer(
212  llvm::MemoryBuffer::getMemBuffer(DatabaseString));
213  std::unique_ptr<JSONCompilationDatabase> Database(
214  new JSONCompilationDatabase(std::move(DatabaseBuffer), Syntax));
215  if (!Database->parse(ErrorMessage))
216  return nullptr;
217  return Database;
218 }
219 
220 std::vector<CompileCommand>
222  SmallString<128> NativeFilePath;
223  llvm::sys::path::native(FilePath, NativeFilePath);
224 
225  std::string Error;
226  llvm::raw_string_ostream ES(Error);
227  StringRef Match = MatchTrie.findEquivalent(NativeFilePath, ES);
228  if (Match.empty())
229  return {};
230  const auto CommandsRefI = IndexByFile.find(Match);
231  if (CommandsRefI == IndexByFile.end())
232  return {};
233  std::vector<CompileCommand> Commands;
234  getCommands(CommandsRefI->getValue(), Commands);
235  return Commands;
236 }
237 
238 std::vector<std::string>
240  std::vector<std::string> Result;
241  for (const auto &CommandRef : IndexByFile)
242  Result.push_back(CommandRef.first().str());
243  return Result;
244 }
245 
246 std::vector<CompileCommand>
248  std::vector<CompileCommand> Commands;
249  getCommands(AllCommands, Commands);
250  return Commands;
251 }
252 
253 static std::vector<std::string>
255  const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
256  SmallString<1024> Storage;
257  if (Nodes.size() == 1)
258  return unescapeCommandLine(Syntax, Nodes[0]->getValue(Storage));
259  std::vector<std::string> Arguments;
260  for (const auto *Node : Nodes)
261  Arguments.push_back(Node->getValue(Storage));
262  return Arguments;
263 }
264 
265 void JSONCompilationDatabase::getCommands(
266  ArrayRef<CompileCommandRef> CommandsRef,
267  std::vector<CompileCommand> &Commands) const {
268  for (const auto &CommandRef : CommandsRef) {
269  SmallString<8> DirectoryStorage;
270  SmallString<32> FilenameStorage;
271  SmallString<32> OutputStorage;
272  auto Output = std::get<3>(CommandRef);
273  Commands.emplace_back(
274  std::get<0>(CommandRef)->getValue(DirectoryStorage),
275  std::get<1>(CommandRef)->getValue(FilenameStorage),
276  nodeToCommandLine(Syntax, std::get<2>(CommandRef)),
277  Output ? Output->getValue(OutputStorage) : "");
278  }
279 }
280 
281 bool JSONCompilationDatabase::parse(std::string &ErrorMessage) {
282  llvm::yaml::document_iterator I = YAMLStream.begin();
283  if (I == YAMLStream.end()) {
284  ErrorMessage = "Error while parsing YAML.";
285  return false;
286  }
287  llvm::yaml::Node *Root = I->getRoot();
288  if (!Root) {
289  ErrorMessage = "Error while parsing YAML.";
290  return false;
291  }
292  auto *Array = dyn_cast<llvm::yaml::SequenceNode>(Root);
293  if (!Array) {
294  ErrorMessage = "Expected array.";
295  return false;
296  }
297  for (auto &NextObject : *Array) {
298  auto *Object = dyn_cast<llvm::yaml::MappingNode>(&NextObject);
299  if (!Object) {
300  ErrorMessage = "Expected object.";
301  return false;
302  }
303  llvm::yaml::ScalarNode *Directory = nullptr;
305  llvm::yaml::ScalarNode *File = nullptr;
306  llvm::yaml::ScalarNode *Output = nullptr;
307  for (auto& NextKeyValue : *Object) {
308  auto *KeyString = dyn_cast<llvm::yaml::ScalarNode>(NextKeyValue.getKey());
309  if (!KeyString) {
310  ErrorMessage = "Expected strings as key.";
311  return false;
312  }
313  SmallString<10> KeyStorage;
314  StringRef KeyValue = KeyString->getValue(KeyStorage);
315  llvm::yaml::Node *Value = NextKeyValue.getValue();
316  if (!Value) {
317  ErrorMessage = "Expected value.";
318  return false;
319  }
320  auto *ValueString = dyn_cast<llvm::yaml::ScalarNode>(Value);
321  auto *SequenceString = dyn_cast<llvm::yaml::SequenceNode>(Value);
322  if (KeyValue == "arguments" && !SequenceString) {
323  ErrorMessage = "Expected sequence as value.";
324  return false;
325  } else if (KeyValue != "arguments" && !ValueString) {
326  ErrorMessage = "Expected string as value.";
327  return false;
328  }
329  if (KeyValue == "directory") {
330  Directory = ValueString;
331  } else if (KeyValue == "arguments") {
332  Command = std::vector<llvm::yaml::ScalarNode *>();
333  for (auto &Argument : *SequenceString) {
334  auto *Scalar = dyn_cast<llvm::yaml::ScalarNode>(&Argument);
335  if (!Scalar) {
336  ErrorMessage = "Only strings are allowed in 'arguments'.";
337  return false;
338  }
339  Command->push_back(Scalar);
340  }
341  } else if (KeyValue == "command") {
342  if (!Command)
343  Command = std::vector<llvm::yaml::ScalarNode *>(1, ValueString);
344  } else if (KeyValue == "file") {
345  File = ValueString;
346  } else if (KeyValue == "output") {
347  Output = ValueString;
348  } else {
349  ErrorMessage = ("Unknown key: \"" +
350  KeyString->getRawValue() + "\"").str();
351  return false;
352  }
353  }
354  if (!File) {
355  ErrorMessage = "Missing key: \"file\".";
356  return false;
357  }
358  if (!Command) {
359  ErrorMessage = "Missing key: \"command\" or \"arguments\".";
360  return false;
361  }
362  if (!Directory) {
363  ErrorMessage = "Missing key: \"directory\".";
364  return false;
365  }
366  SmallString<8> FileStorage;
367  StringRef FileName = File->getValue(FileStorage);
368  SmallString<128> NativeFilePath;
369  if (llvm::sys::path::is_relative(FileName)) {
370  SmallString<8> DirectoryStorage;
371  SmallString<128> AbsolutePath(
372  Directory->getValue(DirectoryStorage));
373  llvm::sys::path::append(AbsolutePath, FileName);
374  llvm::sys::path::native(AbsolutePath, NativeFilePath);
375  } else {
376  llvm::sys::path::native(FileName, NativeFilePath);
377  }
378  auto Cmd = CompileCommandRef(Directory, File, *Command, Output);
379  IndexByFile[NativeFilePath].push_back(Cmd);
380  AllCommands.push_back(Cmd);
381  MatchTrie.insert(NativeFilePath);
382  }
383  return true;
384 }
static std::unique_ptr< JSONCompilationDatabase > loadFromFile(StringRef FilePath, std::string &ErrorMessage, JSONCommandLineSyntax Syntax)
Loads a JSON compilation database from the specified file.
Interface for compilation database plugins.
std::vector< std::string > getAllFiles() const override
Returns the list of all files available in the compilation database.
BoundNodesTreeBuilder Nodes
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
CompileCommand Cmd
static std::unique_ptr< JSONCompilationDatabase > loadFromBuffer(StringRef DatabaseString, std::string &ErrorMessage, JSONCommandLineSyntax Syntax)
Loads a JSON compilation database from a data buffer.
static SVal getValue(SVal val, SValBuilder &svalBuilder)
std::vector< CompileCommand > getCompileCommands(StringRef FilePath) const override
Returns all compile commands in which the specified file was compiled.
std::unique_ptr< CompilationDatabase > inferMissingCompileCommands(std::unique_ptr< CompilationDatabase >)
Returns a wrapped CompilationDatabase that defers to the provided one, but getCompileCommands() will ...
ast_type_traits::DynTypedNode Node
Dataflow Directional Tag Classes.
std::vector< CompileCommand > getAllCompileCommands() const override
Returns all compile commands for all the files in the compilation database.
static ToolExecutorPluginRegistry::Add< AllTUsToolExecutorPlugin > X("all-TUs", "Runs FrontendActions on all TUs in the compilation database. " "Tool results are stored in memory.")
JSONCommandLineSyntax
A JSON based compilation database.
static std::vector< std::string > nodeToCommandLine(JSONCommandLineSyntax Syntax, const std::vector< llvm::yaml::ScalarNode *> &Nodes)