clang-tools  14.0.0git
QueryDriverDatabase.cpp
Go to the documentation of this file.
1 //===--- QueryDriverDatabase.cpp ---------------------------------*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // Some compiler drivers have implicit search mechanism for system headers.
9 // This compilation database implementation tries to extract that information by
10 // executing the driver in verbose mode. gcc-compatible drivers print something
11 // like:
12 // ....
13 // ....
14 // #include <...> search starts here:
15 // /usr/lib/gcc/x86_64-linux-gnu/7/include
16 // /usr/local/include
17 // /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed
18 // /usr/include/x86_64-linux-gnu
19 // /usr/include
20 // End of search list.
21 // ....
22 // ....
23 // This component parses that output and adds each path to command line args
24 // provided by Base, after prepending them with -isystem. Therefore current
25 // implementation would not work with a driver that is not gcc-compatible.
26 //
27 // First argument of the command line received from underlying compilation
28 // database is used as compiler driver path. Due to this arbitrary binary
29 // execution, this mechanism is not used by default and only executes binaries
30 // in the paths that are explicitly included by the user.
31 
33 #include "support/Logger.h"
34 #include "support/Path.h"
35 #include "support/Trace.h"
36 #include "clang/Basic/Diagnostic.h"
37 #include "clang/Basic/TargetInfo.h"
38 #include "clang/Basic/TargetOptions.h"
39 #include "clang/Driver/Types.h"
40 #include "clang/Tooling/CompilationDatabase.h"
41 #include "llvm/ADT/DenseMap.h"
42 #include "llvm/ADT/ScopeExit.h"
43 #include "llvm/ADT/SmallString.h"
44 #include "llvm/ADT/StringExtras.h"
45 #include "llvm/ADT/StringRef.h"
46 #include "llvm/ADT/iterator_range.h"
47 #include "llvm/Support/FileSystem.h"
48 #include "llvm/Support/MemoryBuffer.h"
49 #include "llvm/Support/Path.h"
50 #include "llvm/Support/Program.h"
51 #include "llvm/Support/Regex.h"
52 #include "llvm/Support/ScopedPrinter.h"
53 #include <algorithm>
54 #include <map>
55 #include <string>
56 #include <vector>
57 
58 namespace clang {
59 namespace clangd {
60 namespace {
61 
62 struct DriverInfo {
63  std::vector<std::string> SystemIncludes;
64  std::string Target;
65 };
66 
67 bool isValidTarget(llvm::StringRef Triple) {
68  std::shared_ptr<TargetOptions> TargetOpts(new TargetOptions);
69  TargetOpts->Triple = Triple.str();
70  DiagnosticsEngine Diags(new DiagnosticIDs, new DiagnosticOptions,
71  new IgnoringDiagConsumer);
72  IntrusiveRefCntPtr<TargetInfo> Target =
73  TargetInfo::CreateTargetInfo(Diags, TargetOpts);
74  return bool(Target);
75 }
76 
77 llvm::Optional<DriverInfo> parseDriverOutput(llvm::StringRef Output) {
78  DriverInfo Info;
79  const char SIS[] = "#include <...> search starts here:";
80  const char SIE[] = "End of search list.";
81  const char TS[] = "Target: ";
82  llvm::SmallVector<llvm::StringRef> Lines;
83  Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
84 
85  enum {
86  Initial, // Initial state: searching for target or includes list.
87  IncludesExtracting, // Includes extracting.
88  Done // Includes and target extraction done.
89  } State = Initial;
90  bool SeenIncludes = false;
91  bool SeenTarget = false;
92  for (auto *It = Lines.begin(); State != Done && It != Lines.end(); ++It) {
93  auto Line = *It;
94  switch (State) {
95  case Initial:
96  if (!SeenIncludes && Line.trim() == SIS) {
97  SeenIncludes = true;
98  State = IncludesExtracting;
99  } else if (!SeenTarget && Line.trim().startswith(TS)) {
100  SeenTarget = true;
101  llvm::StringRef TargetLine = Line.trim();
102  TargetLine.consume_front(TS);
103  // Only detect targets that clang understands
104  if (!isValidTarget(TargetLine)) {
105  elog("System include extraction: invalid target \"{0}\", ignoring",
106  TargetLine);
107  } else {
108  Info.Target = TargetLine.str();
109  vlog("System include extraction: target extracted: \"{0}\"",
110  TargetLine);
111  }
112  }
113  break;
114  case IncludesExtracting:
115  if (Line.trim() == SIE) {
116  State = SeenTarget ? Done : Initial;
117  } else {
118  Info.SystemIncludes.push_back(Line.trim().str());
119  vlog("System include extraction: adding {0}", Line);
120  }
121  break;
122  default:
123  llvm_unreachable("Impossible state of the driver output parser");
124  break;
125  }
126  }
127  if (!SeenIncludes) {
128  elog("System include extraction: start marker not found: {0}", Output);
129  return llvm::None;
130  }
131  if (State == IncludesExtracting) {
132  elog("System include extraction: end marker missing: {0}", Output);
133  return llvm::None;
134  }
135  return std::move(Info);
136 }
137 
138 llvm::Optional<DriverInfo>
139 extractSystemIncludesAndTarget(llvm::SmallString<128> Driver,
140  llvm::StringRef Lang,
141  llvm::ArrayRef<std::string> CommandLine,
142  const llvm::Regex &QueryDriverRegex) {
143  trace::Span Tracer("Extract system includes and target");
144 
145  if (!llvm::sys::path::is_absolute(Driver)) {
146  assert(llvm::none_of(
147  Driver, [](char C) { return llvm::sys::path::is_separator(C); }));
148  auto DriverProgram = llvm::sys::findProgramByName(Driver);
149  if (DriverProgram) {
150  vlog("System include extraction: driver {0} expanded to {1}", Driver,
151  *DriverProgram);
152  Driver = *DriverProgram;
153  } else {
154  elog("System include extraction: driver {0} not found in PATH", Driver);
155  return llvm::None;
156  }
157  }
158 
159  SPAN_ATTACH(Tracer, "driver", Driver);
160  SPAN_ATTACH(Tracer, "lang", Lang);
161 
162  if (!QueryDriverRegex.match(Driver)) {
163  vlog("System include extraction: not allowed driver {0}", Driver);
164  return llvm::None;
165  }
166 
167  llvm::SmallString<128> StdErrPath;
168  if (auto EC = llvm::sys::fs::createTemporaryFile("system-includes", "clangd",
169  StdErrPath)) {
170  elog("System include extraction: failed to create temporary file with "
171  "error {0}",
172  EC.message());
173  return llvm::None;
174  }
175  auto CleanUp = llvm::make_scope_exit(
176  [&StdErrPath]() { llvm::sys::fs::remove(StdErrPath); });
177 
178  llvm::Optional<llvm::StringRef> Redirects[] = {{""}, {""}, StdErrPath.str()};
179 
180  llvm::SmallVector<llvm::StringRef> Args = {Driver, "-E", "-x",
181  Lang, "-", "-v"};
182 
183  // These flags will be preserved
184  const llvm::StringRef FlagsToPreserve[] = {
185  "-nostdinc", "--no-standard-includes", "-nostdinc++", "-nobuiltininc"};
186  // Preserves these flags and their values, either as separate args or with an
187  // equalsbetween them
188  const llvm::StringRef ArgsToPreserve[] = {"--sysroot", "-isysroot"};
189 
190  for (size_t I = 0, E = CommandLine.size(); I < E; ++I) {
191  llvm::StringRef Arg = CommandLine[I];
192  if (llvm::any_of(FlagsToPreserve,
193  [&Arg](llvm::StringRef S) { return S == Arg; })) {
194  Args.push_back(Arg);
195  } else {
196  const auto *Found =
197  llvm::find_if(ArgsToPreserve, [&Arg](llvm::StringRef S) {
198  return Arg.startswith(S);
199  });
200  if (Found == std::end(ArgsToPreserve))
201  continue;
202  Arg = Arg.drop_front(Found->size());
203  if (Arg.empty() && I + 1 < E) {
204  Args.push_back(CommandLine[I]);
205  Args.push_back(CommandLine[++I]);
206  } else if (Arg.startswith("=")) {
207  Args.push_back(CommandLine[I]);
208  }
209  }
210  }
211 
212  std::string ErrMsg;
213  if (int RC = llvm::sys::ExecuteAndWait(Driver, Args, /*Env=*/llvm::None,
214  Redirects, /*SecondsToWait=*/0,
215  /*MemoryLimit=*/0, &ErrMsg)) {
216  elog("System include extraction: driver execution failed with return code: "
217  "{0} - '{1}'. Args: [{2}]",
218  llvm::to_string(RC), ErrMsg, printArgv(Args));
219  return llvm::None;
220  }
221 
222  auto BufOrError = llvm::MemoryBuffer::getFile(StdErrPath);
223  if (!BufOrError) {
224  elog("System include extraction: failed to read {0} with error {1}",
225  StdErrPath, BufOrError.getError().message());
226  return llvm::None;
227  }
228 
229  llvm::Optional<DriverInfo> Info =
230  parseDriverOutput(BufOrError->get()->getBuffer());
231  if (!Info)
232  return llvm::None;
233  log("System includes extractor: successfully executed {0}\n\tgot includes: "
234  "\"{1}\"\n\tgot target: \"{2}\"",
235  Driver, llvm::join(Info->SystemIncludes, ", "), Info->Target);
236  return Info;
237 }
238 
239 tooling::CompileCommand &
240 addSystemIncludes(tooling::CompileCommand &Cmd,
241  llvm::ArrayRef<std::string> SystemIncludes) {
242  for (llvm::StringRef Include : SystemIncludes) {
243  // FIXME(kadircet): This doesn't work when we have "--driver-mode=cl"
244  Cmd.CommandLine.push_back("-isystem");
245  Cmd.CommandLine.push_back(Include.str());
246  }
247  return Cmd;
248 }
249 
250 tooling::CompileCommand &setTarget(tooling::CompileCommand &Cmd,
251  const std::string &Target) {
252  if (!Target.empty()) {
253  // We do not want to override existing target with extracted one.
254  for (llvm::StringRef Arg : Cmd.CommandLine) {
255  if (Arg == "-target" || Arg.startswith("--target="))
256  return Cmd;
257  }
258  Cmd.CommandLine.push_back("--target=" + Target);
259  }
260  return Cmd;
261 }
262 
263 /// Converts a glob containing only ** or * into a regex.
264 std::string convertGlobToRegex(llvm::StringRef Glob) {
265  std::string RegText;
266  llvm::raw_string_ostream RegStream(RegText);
267  RegStream << '^';
268  for (size_t I = 0, E = Glob.size(); I < E; ++I) {
269  if (Glob[I] == '*') {
270  if (I + 1 < E && Glob[I + 1] == '*') {
271  // Double star, accept any sequence.
272  RegStream << ".*";
273  // Also skip the second star.
274  ++I;
275  } else {
276  // Single star, accept any sequence without a slash.
277  RegStream << "[^/]*";
278  }
279  } else {
280  RegStream << llvm::Regex::escape(Glob.substr(I, 1));
281  }
282  }
283  RegStream << '$';
284  RegStream.flush();
285  return RegText;
286 }
287 
288 /// Converts a glob containing only ** or * into a regex.
289 llvm::Regex convertGlobsToRegex(llvm::ArrayRef<std::string> Globs) {
290  assert(!Globs.empty() && "Globs cannot be empty!");
291  std::vector<std::string> RegTexts;
292  RegTexts.reserve(Globs.size());
293  for (llvm::StringRef Glob : Globs)
294  RegTexts.push_back(convertGlobToRegex(Glob));
295 
296  llvm::Regex Reg(llvm::join(RegTexts, "|"));
297  assert(Reg.isValid(RegTexts.front()) &&
298  "Created an invalid regex from globs");
299  return Reg;
300 }
301 
302 /// Extracts system includes from a trusted driver by parsing the output of
303 /// include search path and appends them to the commands coming from underlying
304 /// compilation database.
305 class QueryDriverDatabase : public DelegatingCDB {
306 public:
307  QueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
308  std::unique_ptr<GlobalCompilationDatabase> Base)
309  : DelegatingCDB(std::move(Base)),
310  QueryDriverRegex(convertGlobsToRegex(QueryDriverGlobs)) {}
311 
312  llvm::Optional<tooling::CompileCommand>
313  getCompileCommand(PathRef File) const override {
314  auto Cmd = DelegatingCDB::getCompileCommand(File);
315  if (!Cmd || Cmd->CommandLine.empty())
316  return Cmd;
317 
318  llvm::StringRef Lang;
319  for (size_t I = 0, E = Cmd->CommandLine.size(); I < E; ++I) {
320  llvm::StringRef Arg = Cmd->CommandLine[I];
321  if (Arg == "-x" && I + 1 < E)
322  Lang = Cmd->CommandLine[I + 1];
323  else if (Arg.startswith("-x"))
324  Lang = Arg.drop_front(2).trim();
325  }
326  if (Lang.empty()) {
327  llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.');
328  auto Type = driver::types::lookupTypeForExtension(Ext);
329  if (Type == driver::types::TY_INVALID) {
330  elog("System include extraction: invalid file type for {0}", Ext);
331  return {};
332  }
333  Lang = driver::types::getTypeName(Type);
334  }
335 
336  llvm::SmallString<128> Driver(Cmd->CommandLine.front());
337  if (llvm::any_of(Driver,
338  [](char C) { return llvm::sys::path::is_separator(C); }))
339  // Driver is a not a single executable name but instead a path (either
340  // relative or absolute).
341  llvm::sys::fs::make_absolute(Cmd->Directory, Driver);
342 
343  if (auto Info =
344  QueriedDrivers.get(/*Key=*/(Driver + ":" + Lang).str(), [&] {
345  return extractSystemIncludesAndTarget(
346  Driver, Lang, Cmd->CommandLine, QueryDriverRegex);
347  })) {
348  setTarget(addSystemIncludes(*Cmd, Info->SystemIncludes), Info->Target);
349  }
350  return Cmd;
351  }
352 
353 private:
354  // Caches includes extracted from a driver. Key is driver:lang.
355  Memoize<llvm::StringMap<llvm::Optional<DriverInfo>>> QueriedDrivers;
356  llvm::Regex QueryDriverRegex;
357 };
358 } // namespace
359 
360 std::unique_ptr<GlobalCompilationDatabase>
361 getQueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
362  std::unique_ptr<GlobalCompilationDatabase> Base) {
363  assert(Base && "Null base to SystemIncludeExtractor");
364  if (QueryDriverGlobs.empty())
365  return Base;
366  return std::make_unique<QueryDriverDatabase>(QueryDriverGlobs,
367  std::move(Base));
368 }
369 
370 } // namespace clangd
371 } // namespace clang
Base
std::unique_ptr< GlobalCompilationDatabase > Base
Definition: GlobalCompilationDatabaseTests.cpp:90
E
const Expr * E
Definition: AvoidBindCheck.cpp:88
Type
NodeType Type
Definition: HTMLGenerator.cpp:73
clang::clangd::printArgv
std::string printArgv(llvm::ArrayRef< llvm::StringRef > Args)
Definition: CompileCommands.cpp:578
Tracer
std::unique_ptr< trace::EventTracer > Tracer
Definition: TraceTests.cpp:164
Path.h
Trace.h
clang::tidy::cppcoreguidelines::join
static std::string join(ArrayRef< SpecialMemberFunctionsCheck::SpecialMemberFunctionKind > SMFS, llvm::StringRef AndOr)
Definition: SpecialMemberFunctionsCheck.cpp:78
Target
std::string Target
Definition: QueryDriverDatabase.cpp:64
run-clang-tidy.make_absolute
def make_absolute(f, directory)
Definition: run-clang-tidy.py:76
clang::clangd::DelegatingCDB::getCompileCommand
llvm::Optional< tooling::CompileCommand > getCompileCommand(PathRef File) const override
If there are any known-good commands for building this file, returns one.
Definition: GlobalCompilationDatabase.cpp:810
SystemIncludes
std::vector< std::string > SystemIncludes
Definition: QueryDriverDatabase.cpp:63
Line
int Line
Definition: PreprocessorTracker.cpp:514
GlobalCompilationDatabase.h
Logger.h
CommandLine
std::vector< llvm::StringRef > CommandLine
Definition: Serialization.cpp:420
Args
llvm::json::Object Args
Definition: Trace.cpp:139
SPAN_ATTACH
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:164
clang::clangd::vlog
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:73
Output
std::string Output
Definition: TraceTests.cpp:162
clang::clangd::log
void log(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:68
clang::clangd::getQueryDriverDatabase
std::unique_ptr< GlobalCompilationDatabase > getQueryDriverDatabase(llvm::ArrayRef< std::string > QueryDriverGlobs, std::unique_ptr< GlobalCompilationDatabase > Base)
Extracts system include search path from drivers matching QueryDriverGlobs and adds them to the compi...
Definition: QueryDriverDatabase.cpp:361
Info
FunctionInfo Info
Definition: FunctionSizeCheck.cpp:120
clang::clangd::PathRef
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:29
C
const Criteria C
Definition: FunctionCognitiveComplexityCheck.cpp:93
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
Diags
CapturedDiags Diags
Definition: ConfigCompileTests.cpp:38
clang::clangd::elog
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:62
Lines
unsigned Lines
Definition: FunctionSizeCheck.cpp:113