clang  6.0.0svn
JSONCompilationDatabase.cpp
Go to the documentation of this file.
1 //===--- JSONCompilationDatabase.cpp - ------------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the implementation of the JSONCompilationDatabase.
11 //
12 //===----------------------------------------------------------------------===//
13 
17 #include "clang/Tooling/Tooling.h"
18 #include "llvm/ADT/SmallString.h"
19 #include "llvm/Support/Allocator.h"
20 #include "llvm/Support/CommandLine.h"
21 #include "llvm/Support/Path.h"
22 #include "llvm/Support/StringSaver.h"
23 #include <system_error>
24 
25 namespace clang {
26 namespace tooling {
27 
28 namespace {
29 
30 /// \brief A parser for escaped strings of command line arguments.
31 ///
32 /// Assumes \-escaping for quoted arguments (see the documentation of
33 /// unescapeCommandLine(...)).
34 class CommandLineArgumentParser {
35  public:
36  CommandLineArgumentParser(StringRef CommandLine)
37  : Input(CommandLine), Position(Input.begin()-1) {}
38 
39  std::vector<std::string> parse() {
40  bool HasMoreInput = true;
41  while (HasMoreInput && nextNonWhitespace()) {
42  std::string Argument;
43  HasMoreInput = parseStringInto(Argument);
44  CommandLine.push_back(Argument);
45  }
46  return CommandLine;
47  }
48 
49  private:
50  // All private methods return true if there is more input available.
51 
52  bool parseStringInto(std::string &String) {
53  do {
54  if (*Position == '"') {
55  if (!parseDoubleQuotedStringInto(String)) return false;
56  } else if (*Position == '\'') {
57  if (!parseSingleQuotedStringInto(String)) return false;
58  } else {
59  if (!parseFreeStringInto(String)) return false;
60  }
61  } while (*Position != ' ');
62  return true;
63  }
64 
65  bool parseDoubleQuotedStringInto(std::string &String) {
66  if (!next()) return false;
67  while (*Position != '"') {
68  if (!skipEscapeCharacter()) return false;
69  String.push_back(*Position);
70  if (!next()) return false;
71  }
72  return next();
73  }
74 
75  bool parseSingleQuotedStringInto(std::string &String) {
76  if (!next()) return false;
77  while (*Position != '\'') {
78  String.push_back(*Position);
79  if (!next()) return false;
80  }
81  return next();
82  }
83 
84  bool parseFreeStringInto(std::string &String) {
85  do {
86  if (!skipEscapeCharacter()) return false;
87  String.push_back(*Position);
88  if (!next()) return false;
89  } while (*Position != ' ' && *Position != '"' && *Position != '\'');
90  return true;
91  }
92 
93  bool skipEscapeCharacter() {
94  if (*Position == '\\') {
95  return next();
96  }
97  return true;
98  }
99 
100  bool nextNonWhitespace() {
101  do {
102  if (!next()) return false;
103  } while (*Position == ' ');
104  return true;
105  }
106 
107  bool next() {
108  ++Position;
109  return Position != Input.end();
110  }
111 
112  const StringRef Input;
113  StringRef::iterator Position;
114  std::vector<std::string> CommandLine;
115 };
116 
117 std::vector<std::string> unescapeCommandLine(JSONCommandLineSyntax Syntax,
118  StringRef EscapedCommandLine) {
119  if (Syntax == JSONCommandLineSyntax::AutoDetect) {
121  llvm::Triple Triple(llvm::sys::getProcessTriple());
122  if (Triple.getOS() == llvm::Triple::OSType::Win32) {
123  // Assume Windows command line parsing on Win32 unless the triple
124  // explicitly tells us otherwise.
125  if (!Triple.hasEnvironment() ||
126  Triple.getEnvironment() == llvm::Triple::EnvironmentType::MSVC)
128  }
129  }
130 
131  if (Syntax == JSONCommandLineSyntax::Windows) {
132  llvm::BumpPtrAllocator Alloc;
133  llvm::StringSaver Saver(Alloc);
135  llvm::cl::TokenizeWindowsCommandLine(EscapedCommandLine, Saver, T);
136  std::vector<std::string> Result(T.begin(), T.end());
137  return Result;
138  }
139  assert(Syntax == JSONCommandLineSyntax::Gnu);
140  CommandLineArgumentParser parser(EscapedCommandLine);
141  return parser.parse();
142 }
143 
144 class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
145  std::unique_ptr<CompilationDatabase>
146  loadFromDirectory(StringRef Directory, std::string &ErrorMessage) override {
147  SmallString<1024> JSONDatabasePath(Directory);
148  llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
150  JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect);
151  }
152 };
153 
154 } // end namespace
155 
156 // Register the JSONCompilationDatabasePlugin with the
157 // CompilationDatabasePluginRegistry using this statically initialized variable.
158 static CompilationDatabasePluginRegistry::Add<JSONCompilationDatabasePlugin>
159 X("json-compilation-database", "Reads JSON formatted compilation databases");
160 
161 // This anchor is used to force the linker to link in the generated object file
162 // and thus register the JSONCompilationDatabasePlugin.
163 volatile int JSONAnchorSource = 0;
164 
165 std::unique_ptr<JSONCompilationDatabase>
167  std::string &ErrorMessage,
168  JSONCommandLineSyntax Syntax) {
169  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> DatabaseBuffer =
170  llvm::MemoryBuffer::getFile(FilePath);
171  if (std::error_code Result = DatabaseBuffer.getError()) {
172  ErrorMessage = "Error while opening JSON database: " + Result.message();
173  return nullptr;
174  }
175  std::unique_ptr<JSONCompilationDatabase> Database(
176  new JSONCompilationDatabase(std::move(*DatabaseBuffer), Syntax));
177  if (!Database->parse(ErrorMessage))
178  return nullptr;
179  return Database;
180 }
181 
182 std::unique_ptr<JSONCompilationDatabase>
184  std::string &ErrorMessage,
185  JSONCommandLineSyntax Syntax) {
186  std::unique_ptr<llvm::MemoryBuffer> DatabaseBuffer(
187  llvm::MemoryBuffer::getMemBuffer(DatabaseString));
188  std::unique_ptr<JSONCompilationDatabase> Database(
189  new JSONCompilationDatabase(std::move(DatabaseBuffer), Syntax));
190  if (!Database->parse(ErrorMessage))
191  return nullptr;
192  return Database;
193 }
194 
195 std::vector<CompileCommand>
197  SmallString<128> NativeFilePath;
198  llvm::sys::path::native(FilePath, NativeFilePath);
199 
200  std::string Error;
201  llvm::raw_string_ostream ES(Error);
202  StringRef Match = MatchTrie.findEquivalent(NativeFilePath, ES);
203  if (Match.empty())
204  return std::vector<CompileCommand>();
205  llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator
206  CommandsRefI = IndexByFile.find(Match);
207  if (CommandsRefI == IndexByFile.end())
208  return std::vector<CompileCommand>();
209  std::vector<CompileCommand> Commands;
210  getCommands(CommandsRefI->getValue(), Commands);
211  return Commands;
212 }
213 
214 std::vector<std::string>
216  std::vector<std::string> Result;
217 
218  llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator
219  CommandsRefI = IndexByFile.begin();
220  const llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator
221  CommandsRefEnd = IndexByFile.end();
222  for (; CommandsRefI != CommandsRefEnd; ++CommandsRefI) {
223  Result.push_back(CommandsRefI->first().str());
224  }
225 
226  return Result;
227 }
228 
229 std::vector<CompileCommand>
231  std::vector<CompileCommand> Commands;
232  getCommands(AllCommands, Commands);
233  return Commands;
234 }
235 
236 static std::vector<std::string>
238  const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
239  SmallString<1024> Storage;
240  if (Nodes.size() == 1) {
241  return unescapeCommandLine(Syntax, Nodes[0]->getValue(Storage));
242  }
243  std::vector<std::string> Arguments;
244  for (auto *Node : Nodes) {
245  Arguments.push_back(Node->getValue(Storage));
246  }
247  return Arguments;
248 }
249 
250 void JSONCompilationDatabase::getCommands(
251  ArrayRef<CompileCommandRef> CommandsRef,
252  std::vector<CompileCommand> &Commands) const {
253  for (int I = 0, E = CommandsRef.size(); I != E; ++I) {
254  SmallString<8> DirectoryStorage;
255  SmallString<32> FilenameStorage;
256  SmallString<32> OutputStorage;
257  auto Output = std::get<3>(CommandsRef[I]);
258  Commands.emplace_back(
259  std::get<0>(CommandsRef[I])->getValue(DirectoryStorage),
260  std::get<1>(CommandsRef[I])->getValue(FilenameStorage),
261  nodeToCommandLine(Syntax, std::get<2>(CommandsRef[I])),
262  Output ? Output->getValue(OutputStorage) : "");
263  }
264 }
265 
266 bool JSONCompilationDatabase::parse(std::string &ErrorMessage) {
267  llvm::yaml::document_iterator I = YAMLStream.begin();
268  if (I == YAMLStream.end()) {
269  ErrorMessage = "Error while parsing YAML.";
270  return false;
271  }
272  llvm::yaml::Node *Root = I->getRoot();
273  if (!Root) {
274  ErrorMessage = "Error while parsing YAML.";
275  return false;
276  }
277  llvm::yaml::SequenceNode *Array = dyn_cast<llvm::yaml::SequenceNode>(Root);
278  if (!Array) {
279  ErrorMessage = "Expected array.";
280  return false;
281  }
282  for (auto& NextObject : *Array) {
283  llvm::yaml::MappingNode *Object = dyn_cast<llvm::yaml::MappingNode>(&NextObject);
284  if (!Object) {
285  ErrorMessage = "Expected object.";
286  return false;
287  }
288  llvm::yaml::ScalarNode *Directory = nullptr;
290  llvm::yaml::ScalarNode *File = nullptr;
291  llvm::yaml::ScalarNode *Output = nullptr;
292  for (auto& NextKeyValue : *Object) {
293  llvm::yaml::ScalarNode *KeyString =
294  dyn_cast<llvm::yaml::ScalarNode>(NextKeyValue.getKey());
295  if (!KeyString) {
296  ErrorMessage = "Expected strings as key.";
297  return false;
298  }
299  SmallString<10> KeyStorage;
300  StringRef KeyValue = KeyString->getValue(KeyStorage);
301  llvm::yaml::Node *Value = NextKeyValue.getValue();
302  if (!Value) {
303  ErrorMessage = "Expected value.";
304  return false;
305  }
306  llvm::yaml::ScalarNode *ValueString =
307  dyn_cast<llvm::yaml::ScalarNode>(Value);
308  llvm::yaml::SequenceNode *SequenceString =
309  dyn_cast<llvm::yaml::SequenceNode>(Value);
310  if (KeyValue == "arguments" && !SequenceString) {
311  ErrorMessage = "Expected sequence as value.";
312  return false;
313  } else if (KeyValue != "arguments" && !ValueString) {
314  ErrorMessage = "Expected string as value.";
315  return false;
316  }
317  if (KeyValue == "directory") {
318  Directory = ValueString;
319  } else if (KeyValue == "arguments") {
320  Command = std::vector<llvm::yaml::ScalarNode *>();
321  for (auto &Argument : *SequenceString) {
322  auto Scalar = dyn_cast<llvm::yaml::ScalarNode>(&Argument);
323  if (!Scalar) {
324  ErrorMessage = "Only strings are allowed in 'arguments'.";
325  return false;
326  }
327  Command->push_back(Scalar);
328  }
329  } else if (KeyValue == "command") {
330  if (!Command)
331  Command = std::vector<llvm::yaml::ScalarNode *>(1, ValueString);
332  } else if (KeyValue == "file") {
333  File = ValueString;
334  } else if (KeyValue == "output") {
335  Output = ValueString;
336  } else {
337  ErrorMessage = ("Unknown key: \"" +
338  KeyString->getRawValue() + "\"").str();
339  return false;
340  }
341  }
342  if (!File) {
343  ErrorMessage = "Missing key: \"file\".";
344  return false;
345  }
346  if (!Command) {
347  ErrorMessage = "Missing key: \"command\" or \"arguments\".";
348  return false;
349  }
350  if (!Directory) {
351  ErrorMessage = "Missing key: \"directory\".";
352  return false;
353  }
354  SmallString<8> FileStorage;
355  StringRef FileName = File->getValue(FileStorage);
356  SmallString<128> NativeFilePath;
357  if (llvm::sys::path::is_relative(FileName)) {
358  SmallString<8> DirectoryStorage;
359  SmallString<128> AbsolutePath(
360  Directory->getValue(DirectoryStorage));
361  llvm::sys::path::append(AbsolutePath, FileName);
362  llvm::sys::path::native(AbsolutePath, NativeFilePath);
363  } else {
364  llvm::sys::path::native(FileName, NativeFilePath);
365  }
366  auto Cmd = CompileCommandRef(Directory, File, *Command, Output);
367  IndexByFile[NativeFilePath].push_back(Cmd);
368  AllCommands.push_back(Cmd);
369  MatchTrie.insert(NativeFilePath);
370  }
371  return true;
372 }
373 
374 } // end namespace tooling
375 } // end namespace clang
BoundNodesTreeBuilder Nodes
static std::unique_ptr< JSONCompilationDatabase > loadFromBuffer(StringRef DatabaseString, std::string &ErrorMessage, JSONCommandLineSyntax Syntax)
Loads a JSON compilation database from a data buffer.
static std::unique_ptr< JSONCompilationDatabase > loadFromFile(StringRef FilePath, std::string &ErrorMessage, JSONCommandLineSyntax Syntax)
Loads a JSON compilation database from the specified file.
const FunctionProtoType * T
std::vector< std::string > getAllFiles() const override
Returns the list of all files available in the compilation database.
static SVal getValue(SVal val, SValBuilder &svalBuilder)
static std::vector< std::string > nodeToCommandLine(JSONCommandLineSyntax Syntax, const std::vector< llvm::yaml::ScalarNode *> &Nodes)
static CompilationDatabasePluginRegistry::Add< JSONCompilationDatabasePlugin > X("json-compilation-database", "Reads JSON formatted compilation databases")
std::vector< CompileCommand > getCompileCommands(StringRef FilePath) const override
Returns all compile commands in which the specified file was compiled.
ast_type_traits::DynTypedNode Node
Dataflow Directional Tag Classes.
std::vector< CompileCommand > getAllCompileCommands() const override
Returns all compile commands for all the files in the compilation database.
JSONCommandLineSyntax
A JSON based compilation database.