clang  7.0.0svn
JSONCompilationDatabase.cpp
Go to the documentation of this file.
1 //===- JSONCompilationDatabase.cpp ----------------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the implementation of the JSONCompilationDatabase.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "clang/Basic/LLVM.h"
18 #include "llvm/ADT/Optional.h"
19 #include "llvm/ADT/SmallString.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/ADT/StringRef.h"
22 #include "llvm/ADT/Triple.h"
23 #include "llvm/Support/Allocator.h"
24 #include "llvm/Support/Casting.h"
25 #include "llvm/Support/CommandLine.h"
26 #include "llvm/Support/ErrorOr.h"
27 #include "llvm/Support/Host.h"
28 #include "llvm/Support/MemoryBuffer.h"
29 #include "llvm/Support/Path.h"
30 #include "llvm/Support/StringSaver.h"
31 #include "llvm/Support/YAMLParser.h"
32 #include "llvm/Support/raw_ostream.h"
33 #include <cassert>
34 #include <memory>
35 #include <string>
36 #include <system_error>
37 #include <tuple>
38 #include <utility>
39 #include <vector>
40 
41 using namespace clang;
42 using namespace tooling;
43 
44 namespace {
45 
46 /// A parser for escaped strings of command line arguments.
47 ///
48 /// Assumes \-escaping for quoted arguments (see the documentation of
49 /// unescapeCommandLine(...)).
50 class CommandLineArgumentParser {
51  public:
52  CommandLineArgumentParser(StringRef CommandLine)
53  : Input(CommandLine), Position(Input.begin()-1) {}
54 
55  std::vector<std::string> parse() {
56  bool HasMoreInput = true;
57  while (HasMoreInput && nextNonWhitespace()) {
58  std::string Argument;
59  HasMoreInput = parseStringInto(Argument);
60  CommandLine.push_back(Argument);
61  }
62  return CommandLine;
63  }
64 
65  private:
66  // All private methods return true if there is more input available.
67 
68  bool parseStringInto(std::string &String) {
69  do {
70  if (*Position == '"') {
71  if (!parseDoubleQuotedStringInto(String)) return false;
72  } else if (*Position == '\'') {
73  if (!parseSingleQuotedStringInto(String)) return false;
74  } else {
75  if (!parseFreeStringInto(String)) return false;
76  }
77  } while (*Position != ' ');
78  return true;
79  }
80 
81  bool parseDoubleQuotedStringInto(std::string &String) {
82  if (!next()) return false;
83  while (*Position != '"') {
84  if (!skipEscapeCharacter()) return false;
85  String.push_back(*Position);
86  if (!next()) return false;
87  }
88  return next();
89  }
90 
91  bool parseSingleQuotedStringInto(std::string &String) {
92  if (!next()) return false;
93  while (*Position != '\'') {
94  String.push_back(*Position);
95  if (!next()) return false;
96  }
97  return next();
98  }
99 
100  bool parseFreeStringInto(std::string &String) {
101  do {
102  if (!skipEscapeCharacter()) return false;
103  String.push_back(*Position);
104  if (!next()) return false;
105  } while (*Position != ' ' && *Position != '"' && *Position != '\'');
106  return true;
107  }
108 
109  bool skipEscapeCharacter() {
110  if (*Position == '\\') {
111  return next();
112  }
113  return true;
114  }
115 
116  bool nextNonWhitespace() {
117  do {
118  if (!next()) return false;
119  } while (*Position == ' ');
120  return true;
121  }
122 
123  bool next() {
124  ++Position;
125  return Position != Input.end();
126  }
127 
128  const StringRef Input;
129  StringRef::iterator Position;
130  std::vector<std::string> CommandLine;
131 };
132 
133 std::vector<std::string> unescapeCommandLine(JSONCommandLineSyntax Syntax,
134  StringRef EscapedCommandLine) {
135  if (Syntax == JSONCommandLineSyntax::AutoDetect) {
137  llvm::Triple Triple(llvm::sys::getProcessTriple());
138  if (Triple.getOS() == llvm::Triple::OSType::Win32) {
139  // Assume Windows command line parsing on Win32 unless the triple
140  // explicitly tells us otherwise.
141  if (!Triple.hasEnvironment() ||
142  Triple.getEnvironment() == llvm::Triple::EnvironmentType::MSVC)
144  }
145  }
146 
147  if (Syntax == JSONCommandLineSyntax::Windows) {
148  llvm::BumpPtrAllocator Alloc;
149  llvm::StringSaver Saver(Alloc);
151  llvm::cl::TokenizeWindowsCommandLine(EscapedCommandLine, Saver, T);
152  std::vector<std::string> Result(T.begin(), T.end());
153  return Result;
154  }
155  assert(Syntax == JSONCommandLineSyntax::Gnu);
156  CommandLineArgumentParser parser(EscapedCommandLine);
157  return parser.parse();
158 }
159 
160 class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
161  std::unique_ptr<CompilationDatabase>
162  loadFromDirectory(StringRef Directory, std::string &ErrorMessage) override {
163  SmallString<1024> JSONDatabasePath(Directory);
164  llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
166  JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect);
167  }
168 };
169 
170 } // namespace
171 
172 // Register the JSONCompilationDatabasePlugin with the
173 // CompilationDatabasePluginRegistry using this statically initialized variable.
174 static CompilationDatabasePluginRegistry::Add<JSONCompilationDatabasePlugin>
175 X("json-compilation-database", "Reads JSON formatted compilation databases");
176 
177 namespace clang {
178 namespace tooling {
179 
180 // This anchor is used to force the linker to link in the generated object file
181 // and thus register the JSONCompilationDatabasePlugin.
182 volatile int JSONAnchorSource = 0;
183 
184 } // namespace tooling
185 } // namespace clang
186 
187 std::unique_ptr<JSONCompilationDatabase>
189  std::string &ErrorMessage,
190  JSONCommandLineSyntax Syntax) {
191  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> DatabaseBuffer =
192  llvm::MemoryBuffer::getFile(FilePath);
193  if (std::error_code Result = DatabaseBuffer.getError()) {
194  ErrorMessage = "Error while opening JSON database: " + Result.message();
195  return nullptr;
196  }
197  std::unique_ptr<JSONCompilationDatabase> Database(
198  new JSONCompilationDatabase(std::move(*DatabaseBuffer), Syntax));
199  if (!Database->parse(ErrorMessage))
200  return nullptr;
201  return Database;
202 }
203 
204 std::unique_ptr<JSONCompilationDatabase>
206  std::string &ErrorMessage,
207  JSONCommandLineSyntax Syntax) {
208  std::unique_ptr<llvm::MemoryBuffer> DatabaseBuffer(
209  llvm::MemoryBuffer::getMemBuffer(DatabaseString));
210  std::unique_ptr<JSONCompilationDatabase> Database(
211  new JSONCompilationDatabase(std::move(DatabaseBuffer), Syntax));
212  if (!Database->parse(ErrorMessage))
213  return nullptr;
214  return Database;
215 }
216 
217 std::vector<CompileCommand>
219  SmallString<128> NativeFilePath;
220  llvm::sys::path::native(FilePath, NativeFilePath);
221 
222  std::string Error;
223  llvm::raw_string_ostream ES(Error);
224  StringRef Match = MatchTrie.findEquivalent(NativeFilePath, ES);
225  if (Match.empty())
226  return {};
227  const auto CommandsRefI = IndexByFile.find(Match);
228  if (CommandsRefI == IndexByFile.end())
229  return {};
230  std::vector<CompileCommand> Commands;
231  getCommands(CommandsRefI->getValue(), Commands);
232  return Commands;
233 }
234 
235 std::vector<std::string>
237  std::vector<std::string> Result;
238  for (const auto &CommandRef : IndexByFile)
239  Result.push_back(CommandRef.first().str());
240  return Result;
241 }
242 
243 std::vector<CompileCommand>
245  std::vector<CompileCommand> Commands;
246  getCommands(AllCommands, Commands);
247  return Commands;
248 }
249 
250 static std::vector<std::string>
252  const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
253  SmallString<1024> Storage;
254  if (Nodes.size() == 1)
255  return unescapeCommandLine(Syntax, Nodes[0]->getValue(Storage));
256  std::vector<std::string> Arguments;
257  for (const auto *Node : Nodes)
258  Arguments.push_back(Node->getValue(Storage));
259  return Arguments;
260 }
261 
262 void JSONCompilationDatabase::getCommands(
263  ArrayRef<CompileCommandRef> CommandsRef,
264  std::vector<CompileCommand> &Commands) const {
265  for (const auto &CommandRef : CommandsRef) {
266  SmallString<8> DirectoryStorage;
267  SmallString<32> FilenameStorage;
268  SmallString<32> OutputStorage;
269  auto Output = std::get<3>(CommandRef);
270  Commands.emplace_back(
271  std::get<0>(CommandRef)->getValue(DirectoryStorage),
272  std::get<1>(CommandRef)->getValue(FilenameStorage),
273  nodeToCommandLine(Syntax, std::get<2>(CommandRef)),
274  Output ? Output->getValue(OutputStorage) : "");
275  }
276 }
277 
278 bool JSONCompilationDatabase::parse(std::string &ErrorMessage) {
279  llvm::yaml::document_iterator I = YAMLStream.begin();
280  if (I == YAMLStream.end()) {
281  ErrorMessage = "Error while parsing YAML.";
282  return false;
283  }
284  llvm::yaml::Node *Root = I->getRoot();
285  if (!Root) {
286  ErrorMessage = "Error while parsing YAML.";
287  return false;
288  }
289  auto *Array = dyn_cast<llvm::yaml::SequenceNode>(Root);
290  if (!Array) {
291  ErrorMessage = "Expected array.";
292  return false;
293  }
294  for (auto &NextObject : *Array) {
295  auto *Object = dyn_cast<llvm::yaml::MappingNode>(&NextObject);
296  if (!Object) {
297  ErrorMessage = "Expected object.";
298  return false;
299  }
300  llvm::yaml::ScalarNode *Directory = nullptr;
302  llvm::yaml::ScalarNode *File = nullptr;
303  llvm::yaml::ScalarNode *Output = nullptr;
304  for (auto& NextKeyValue : *Object) {
305  auto *KeyString = dyn_cast<llvm::yaml::ScalarNode>(NextKeyValue.getKey());
306  if (!KeyString) {
307  ErrorMessage = "Expected strings as key.";
308  return false;
309  }
310  SmallString<10> KeyStorage;
311  StringRef KeyValue = KeyString->getValue(KeyStorage);
312  llvm::yaml::Node *Value = NextKeyValue.getValue();
313  if (!Value) {
314  ErrorMessage = "Expected value.";
315  return false;
316  }
317  auto *ValueString = dyn_cast<llvm::yaml::ScalarNode>(Value);
318  auto *SequenceString = dyn_cast<llvm::yaml::SequenceNode>(Value);
319  if (KeyValue == "arguments" && !SequenceString) {
320  ErrorMessage = "Expected sequence as value.";
321  return false;
322  } else if (KeyValue != "arguments" && !ValueString) {
323  ErrorMessage = "Expected string as value.";
324  return false;
325  }
326  if (KeyValue == "directory") {
327  Directory = ValueString;
328  } else if (KeyValue == "arguments") {
329  Command = std::vector<llvm::yaml::ScalarNode *>();
330  for (auto &Argument : *SequenceString) {
331  auto *Scalar = dyn_cast<llvm::yaml::ScalarNode>(&Argument);
332  if (!Scalar) {
333  ErrorMessage = "Only strings are allowed in 'arguments'.";
334  return false;
335  }
336  Command->push_back(Scalar);
337  }
338  } else if (KeyValue == "command") {
339  if (!Command)
340  Command = std::vector<llvm::yaml::ScalarNode *>(1, ValueString);
341  } else if (KeyValue == "file") {
342  File = ValueString;
343  } else if (KeyValue == "output") {
344  Output = ValueString;
345  } else {
346  ErrorMessage = ("Unknown key: \"" +
347  KeyString->getRawValue() + "\"").str();
348  return false;
349  }
350  }
351  if (!File) {
352  ErrorMessage = "Missing key: \"file\".";
353  return false;
354  }
355  if (!Command) {
356  ErrorMessage = "Missing key: \"command\" or \"arguments\".";
357  return false;
358  }
359  if (!Directory) {
360  ErrorMessage = "Missing key: \"directory\".";
361  return false;
362  }
363  SmallString<8> FileStorage;
364  StringRef FileName = File->getValue(FileStorage);
365  SmallString<128> NativeFilePath;
366  if (llvm::sys::path::is_relative(FileName)) {
367  SmallString<8> DirectoryStorage;
368  SmallString<128> AbsolutePath(
369  Directory->getValue(DirectoryStorage));
370  llvm::sys::path::append(AbsolutePath, FileName);
371  llvm::sys::path::native(AbsolutePath, NativeFilePath);
372  } else {
373  llvm::sys::path::native(FileName, NativeFilePath);
374  }
375  auto Cmd = CompileCommandRef(Directory, File, *Command, Output);
376  IndexByFile[NativeFilePath].push_back(Cmd);
377  AllCommands.push_back(Cmd);
378  MatchTrie.insert(NativeFilePath);
379  }
380  return true;
381 }
static std::unique_ptr< JSONCompilationDatabase > loadFromFile(StringRef FilePath, std::string &ErrorMessage, JSONCommandLineSyntax Syntax)
Loads a JSON compilation database from the specified file.
Interface for compilation database plugins.
std::vector< std::string > getAllFiles() const override
Returns the list of all files available in the compilation database.
BoundNodesTreeBuilder Nodes
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
CompileCommand Cmd
static std::unique_ptr< JSONCompilationDatabase > loadFromBuffer(StringRef DatabaseString, std::string &ErrorMessage, JSONCommandLineSyntax Syntax)
Loads a JSON compilation database from a data buffer.
const FunctionProtoType * T
static SVal getValue(SVal val, SValBuilder &svalBuilder)
std::vector< CompileCommand > getCompileCommands(StringRef FilePath) const override
Returns all compile commands in which the specified file was compiled.
ast_type_traits::DynTypedNode Node
Dataflow Directional Tag Classes.
std::vector< CompileCommand > getAllCompileCommands() const override
Returns all compile commands for all the files in the compilation database.
static ToolExecutorPluginRegistry::Add< AllTUsToolExecutorPlugin > X("all-TUs", "Runs FrontendActions on all TUs in the compilation database. " "Tool results are stored in memory.")
JSONCommandLineSyntax
A JSON based compilation database.
static std::vector< std::string > nodeToCommandLine(JSONCommandLineSyntax Syntax, const std::vector< llvm::yaml::ScalarNode *> &Nodes)