clang-tools 17.0.0git
SystemIncludeExtractor.cpp
Go to the documentation of this file.
1//===--- SystemIncludeExtractor.cpp ------------------------------*- C++-*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// Some compiler drivers have implicit search mechanism for system headers.
9// This compilation database implementation tries to extract that information by
10// executing the driver in verbose mode. gcc-compatible drivers print something
11// like:
12// ....
13// ....
14// #include <...> search starts here:
15// /usr/lib/gcc/x86_64-linux-gnu/7/include
16// /usr/local/include
17// /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed
18// /usr/include/x86_64-linux-gnu
19// /usr/include
20// End of search list.
21// ....
22// ....
23// This component parses that output and adds each path to command line args
24// provided by Base, after prepending them with -isystem. Therefore current
25// implementation would not work with a driver that is not gcc-compatible.
26//
27// First argument of the command line received from underlying compilation
28// database is used as compiler driver path. Due to this arbitrary binary
29// execution, this mechanism is not used by default and only executes binaries
30// in the paths that are explicitly included by the user.
31
32#include "CompileCommands.h"
34#include "support/Logger.h"
35#include "support/Threading.h"
36#include "support/Trace.h"
37#include "clang/Basic/Diagnostic.h"
38#include "clang/Basic/DiagnosticIDs.h"
39#include "clang/Basic/DiagnosticOptions.h"
40#include "clang/Basic/TargetInfo.h"
41#include "clang/Basic/TargetOptions.h"
42#include "clang/Driver/Types.h"
43#include "clang/Tooling/CompilationDatabase.h"
44#include "llvm/ADT/ArrayRef.h"
45#include "llvm/ADT/DenseMap.h"
46#include "llvm/ADT/Hashing.h"
47#include "llvm/ADT/IntrusiveRefCntPtr.h"
48#include "llvm/ADT/STLExtras.h"
49#include "llvm/ADT/ScopeExit.h"
50#include "llvm/ADT/SmallString.h"
51#include "llvm/ADT/SmallVector.h"
52#include "llvm/ADT/StringExtras.h"
53#include "llvm/ADT/StringRef.h"
54#include "llvm/Support/ErrorHandling.h"
55#include "llvm/Support/FileSystem.h"
56#include "llvm/Support/MemoryBuffer.h"
57#include "llvm/Support/Path.h"
58#include "llvm/Support/Program.h"
59#include "llvm/Support/Regex.h"
60#include "llvm/Support/ScopedPrinter.h"
61#include "llvm/Support/raw_ostream.h"
62#include <cassert>
63#include <cstddef>
64#include <iterator>
65#include <memory>
66#include <optional>
67#include <string>
68#include <tuple>
69#include <utility>
70#include <vector>
71
72namespace clang::clangd {
73namespace {
74
75struct DriverInfo {
76 std::vector<std::string> SystemIncludes;
77 std::string Target;
78};
79
80struct DriverArgs {
81 // Name of the driver program to execute or absolute path to it.
82 std::string Driver;
83 // Whether certain includes should be part of query.
84 bool StandardIncludes = true;
86 bool BuiltinIncludes = true;
87 // Language to use while querying.
88 std::string Lang;
89 std::string Sysroot;
90 std::string ISysroot;
91
92 bool operator==(const DriverArgs &RHS) const {
95 std::tie(RHS.Driver, RHS.StandardIncludes, RHS.StandardCXXIncludes,
96 RHS.BuiltinIncludes, RHS.Lang, RHS.Sysroot, ISysroot);
97 }
98
99 DriverArgs(const tooling::CompileCommand &Cmd, llvm::StringRef File) {
100 llvm::SmallString<128> Driver(Cmd.CommandLine.front());
101 // Driver is a not a single executable name but instead a path (either
102 // relative or absolute).
103 if (llvm::any_of(Driver,
104 [](char C) { return llvm::sys::path::is_separator(C); })) {
105 llvm::sys::fs::make_absolute(Cmd.Directory, Driver);
106 }
107 this->Driver = Driver.str().str();
108 for (size_t I = 0, E = Cmd.CommandLine.size(); I < E; ++I) {
109 llvm::StringRef Arg = Cmd.CommandLine[I];
110
111 // Look for Language related flags.
112 if (Arg.consume_front("-x")) {
113 if (Arg.empty() && I + 1 < E)
114 Lang = Cmd.CommandLine[I + 1];
115 else
116 Lang = Arg.str();
117 }
118 // Look for standard/builtin includes.
119 else if (Arg == "-nostdinc" || Arg == "--no-standard-includes")
120 StandardIncludes = false;
121 else if (Arg == "-nostdinc++")
122 StandardCXXIncludes = false;
123 else if (Arg == "-nobuiltininc")
124 BuiltinIncludes = false;
125 // Figure out sysroot
126 else if (Arg.consume_front("--sysroot")) {
127 if (Arg.consume_front("="))
128 Sysroot = Arg.str();
129 else if (Arg.empty() && I + 1 < E)
130 Sysroot = Cmd.CommandLine[I + 1];
131 } else if (Arg.consume_front("-isysroot")) {
132 if (Arg.empty() && I + 1 < E)
133 ISysroot = Cmd.CommandLine[I + 1];
134 else
135 ISysroot = Arg.str();
136 }
137 }
138
139 // If language is not explicit in the flags, infer from the file.
140 // This is important as we want to cache each language separately.
141 if (Lang.empty()) {
142 llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.');
143 auto Type = driver::types::lookupTypeForExtension(Ext);
144 if (Type == driver::types::TY_INVALID) {
145 elog("System include extraction: invalid file type for {0}", Ext);
146 } else {
147 Lang = driver::types::getTypeName(Type);
148 }
149 }
150 }
151 llvm::SmallVector<llvm::StringRef> render() const {
152 // FIXME: Don't treat lang specially?
153 assert(!Lang.empty());
154 llvm::SmallVector<llvm::StringRef> Args = {"-x", Lang};
155 if (!StandardIncludes)
156 Args.push_back("-nostdinc");
157 if (!StandardCXXIncludes)
158 Args.push_back("-nostdinc++");
159 if (!BuiltinIncludes)
160 Args.push_back("-nobuiltininc++");
161 if (!Sysroot.empty())
162 Args.append({"--sysroot", Sysroot});
163 if (!ISysroot.empty())
164 Args.append({"-isysroot", ISysroot});
165 return Args;
166 }
167
168 static DriverArgs getEmpty() { return {}; }
169
170private:
171 DriverArgs() = default;
172};
173} // namespace
174} // namespace clang::clangd
175namespace llvm {
176using DriverArgs = clang::clangd::DriverArgs;
177template <> struct DenseMapInfo<DriverArgs> {
179 auto Driver = DriverArgs::getEmpty();
180 Driver.Driver = "EMPTY_KEY";
181 return Driver;
182 }
184 auto Driver = DriverArgs::getEmpty();
185 Driver.Driver = "TOMBSTONE_KEY";
186 return Driver;
187 }
188 static unsigned getHashValue(const DriverArgs &Val) {
189 return llvm::hash_value(std::tuple{
190 Val.Driver,
191 Val.StandardIncludes,
192 Val.StandardCXXIncludes,
193 Val.BuiltinIncludes,
194 Val.Lang,
195 Val.Sysroot,
196 Val.ISysroot,
197 });
198 }
199 static bool isEqual(const DriverArgs &LHS, const DriverArgs &RHS) {
200 return LHS == RHS;
201 }
202};
203} // namespace llvm
204namespace clang::clangd {
205namespace {
206bool isValidTarget(llvm::StringRef Triple) {
207 std::shared_ptr<TargetOptions> TargetOpts(new TargetOptions);
208 TargetOpts->Triple = Triple.str();
209 DiagnosticsEngine Diags(new DiagnosticIDs, new DiagnosticOptions,
210 new IgnoringDiagConsumer);
211 llvm::IntrusiveRefCntPtr<TargetInfo> Target =
212 TargetInfo::CreateTargetInfo(Diags, TargetOpts);
213 return bool(Target);
214}
215
216std::optional<DriverInfo> parseDriverOutput(llvm::StringRef Output) {
217 DriverInfo Info;
218 const char SIS[] = "#include <...> search starts here:";
219 const char SIE[] = "End of search list.";
220 const char TS[] = "Target: ";
221 llvm::SmallVector<llvm::StringRef> Lines;
222 Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
223
224 enum {
225 Initial, // Initial state: searching for target or includes list.
226 IncludesExtracting, // Includes extracting.
227 Done // Includes and target extraction done.
228 } State = Initial;
229 bool SeenIncludes = false;
230 bool SeenTarget = false;
231 for (auto *It = Lines.begin(); State != Done && It != Lines.end(); ++It) {
232 auto Line = *It;
233 switch (State) {
234 case Initial:
235 if (!SeenIncludes && Line.trim() == SIS) {
236 SeenIncludes = true;
237 State = IncludesExtracting;
238 } else if (!SeenTarget && Line.trim().startswith(TS)) {
239 SeenTarget = true;
240 llvm::StringRef TargetLine = Line.trim();
241 TargetLine.consume_front(TS);
242 // Only detect targets that clang understands
243 if (!isValidTarget(TargetLine)) {
244 elog("System include extraction: invalid target \"{0}\", ignoring",
245 TargetLine);
246 } else {
247 Info.Target = TargetLine.str();
248 vlog("System include extraction: target extracted: \"{0}\"",
249 TargetLine);
250 }
251 }
252 break;
253 case IncludesExtracting:
254 if (Line.trim() == SIE) {
255 State = SeenTarget ? Done : Initial;
256 } else {
257 Info.SystemIncludes.push_back(Line.trim().str());
258 vlog("System include extraction: adding {0}", Line);
259 }
260 break;
261 default:
262 llvm_unreachable("Impossible state of the driver output parser");
263 break;
264 }
265 }
266 if (!SeenIncludes) {
267 elog("System include extraction: start marker not found: {0}", Output);
268 return std::nullopt;
269 }
270 if (State == IncludesExtracting) {
271 elog("System include extraction: end marker missing: {0}", Output);
272 return std::nullopt;
273 }
274 return std::move(Info);
275}
276
277std::optional<DriverInfo>
278extractSystemIncludesAndTarget(const DriverArgs &InputArgs,
279 const llvm::Regex &QueryDriverRegex) {
280 trace::Span Tracer("Extract system includes and target");
281
282 std::string Driver = InputArgs.Driver;
283 if (!llvm::sys::path::is_absolute(Driver)) {
284 auto DriverProgram = llvm::sys::findProgramByName(Driver);
285 if (DriverProgram) {
286 vlog("System include extraction: driver {0} expanded to {1}", Driver,
287 *DriverProgram);
288 Driver = *DriverProgram;
289 } else {
290 elog("System include extraction: driver {0} not found in PATH", Driver);
291 return std::nullopt;
292 }
293 }
294
295 SPAN_ATTACH(Tracer, "driver", Driver);
296 SPAN_ATTACH(Tracer, "lang", InputArgs.Lang);
297
298 if (!QueryDriverRegex.match(Driver)) {
299 vlog("System include extraction: not allowed driver {0}", Driver);
300 return std::nullopt;
301 }
302
303 llvm::SmallString<128> StdErrPath;
304 if (auto EC = llvm::sys::fs::createTemporaryFile("system-includes", "clangd",
305 StdErrPath)) {
306 elog("System include extraction: failed to create temporary file with "
307 "error {0}",
308 EC.message());
309 return std::nullopt;
310 }
311 auto CleanUp = llvm::make_scope_exit(
312 [&StdErrPath]() { llvm::sys::fs::remove(StdErrPath); });
313
314 std::optional<llvm::StringRef> Redirects[] = {{""}, {""}, StdErrPath.str()};
315
316 llvm::SmallVector<llvm::StringRef> Args = {Driver, "-E", "-v"};
317 Args.append(InputArgs.render());
318 // Input needs to go after Lang flags.
319 Args.push_back("-");
320
321 std::string ErrMsg;
322 if (int RC = llvm::sys::ExecuteAndWait(Driver, Args, /*Env=*/std::nullopt,
323 Redirects, /*SecondsToWait=*/0,
324 /*MemoryLimit=*/0, &ErrMsg)) {
325 elog("System include extraction: driver execution failed with return code: "
326 "{0} - '{1}'. Args: [{2}]",
327 llvm::to_string(RC), ErrMsg, printArgv(Args));
328 return std::nullopt;
329 }
330
331 auto BufOrError = llvm::MemoryBuffer::getFile(StdErrPath);
332 if (!BufOrError) {
333 elog("System include extraction: failed to read {0} with error {1}",
334 StdErrPath, BufOrError.getError().message());
335 return std::nullopt;
336 }
337
338 std::optional<DriverInfo> Info =
339 parseDriverOutput(BufOrError->get()->getBuffer());
340 if (!Info)
341 return std::nullopt;
342 log("System includes extractor: successfully executed {0}\n\tgot includes: "
343 "\"{1}\"\n\tgot target: \"{2}\"",
344 Driver, llvm::join(Info->SystemIncludes, ", "), Info->Target);
345 return Info;
346}
347
348tooling::CompileCommand &
349addSystemIncludes(tooling::CompileCommand &Cmd,
350 llvm::ArrayRef<std::string> SystemIncludes) {
351 std::vector<std::string> ToAppend;
352 for (llvm::StringRef Include : SystemIncludes) {
353 // FIXME(kadircet): This doesn't work when we have "--driver-mode=cl"
354 ToAppend.push_back("-isystem");
355 ToAppend.push_back(Include.str());
356 }
357 if (!ToAppend.empty()) {
358 // Just append when `--` isn't present.
359 auto InsertAt = llvm::find(Cmd.CommandLine, "--");
360 Cmd.CommandLine.insert(InsertAt, std::make_move_iterator(ToAppend.begin()),
361 std::make_move_iterator(ToAppend.end()));
362 }
363 return Cmd;
364}
365
366tooling::CompileCommand &setTarget(tooling::CompileCommand &Cmd,
367 const std::string &Target) {
368 if (!Target.empty()) {
369 // We do not want to override existing target with extracted one.
370 for (llvm::StringRef Arg : Cmd.CommandLine) {
371 if (Arg == "-target" || Arg.startswith("--target="))
372 return Cmd;
373 }
374 // Just append when `--` isn't present.
375 auto InsertAt = llvm::find(Cmd.CommandLine, "--");
376 Cmd.CommandLine.insert(InsertAt, "--target=" + Target);
377 }
378 return Cmd;
379}
380
381/// Converts a glob containing only ** or * into a regex.
382std::string convertGlobToRegex(llvm::StringRef Glob) {
383 std::string RegText;
384 llvm::raw_string_ostream RegStream(RegText);
385 RegStream << '^';
386 for (size_t I = 0, E = Glob.size(); I < E; ++I) {
387 if (Glob[I] == '*') {
388 if (I + 1 < E && Glob[I + 1] == '*') {
389 // Double star, accept any sequence.
390 RegStream << ".*";
391 // Also skip the second star.
392 ++I;
393 } else {
394 // Single star, accept any sequence without a slash.
395 RegStream << "[^/]*";
396 }
397 } else if (llvm::sys::path::is_separator(Glob[I]) &&
398 llvm::sys::path::is_separator('/') &&
399 llvm::sys::path::is_separator('\\')) {
400 RegStream << R"([/\\])"; // Accept either slash on windows.
401 } else {
402 RegStream << llvm::Regex::escape(Glob.substr(I, 1));
403 }
404 }
405 RegStream << '$';
406 RegStream.flush();
407 return RegText;
408}
409
410/// Converts a glob containing only ** or * into a regex.
411llvm::Regex convertGlobsToRegex(llvm::ArrayRef<std::string> Globs) {
412 assert(!Globs.empty() && "Globs cannot be empty!");
413 std::vector<std::string> RegTexts;
414 RegTexts.reserve(Globs.size());
415 for (llvm::StringRef Glob : Globs)
416 RegTexts.push_back(convertGlobToRegex(Glob));
417
418 // Tempting to pass IgnoreCase, but we don't know the FS sensitivity.
419 llvm::Regex Reg(llvm::join(RegTexts, "|"));
420 assert(Reg.isValid(RegTexts.front()) &&
421 "Created an invalid regex from globs");
422 return Reg;
423}
424
425/// Extracts system includes from a trusted driver by parsing the output of
426/// include search path and appends them to the commands coming from underlying
427/// compilation database.
428class SystemIncludeExtractor {
429public:
430 SystemIncludeExtractor(llvm::ArrayRef<std::string> QueryDriverGlobs)
431 : QueryDriverRegex(convertGlobsToRegex(QueryDriverGlobs)) {}
432
433 void operator()(tooling::CompileCommand &Cmd, llvm::StringRef File) const {
434 if (Cmd.CommandLine.empty())
435 return;
436
437 DriverArgs Args(Cmd, File);
438 if (Args.Lang.empty())
439 return;
440 if (auto Info = QueriedDrivers.get(Args, [&] {
441 return extractSystemIncludesAndTarget(Args, QueryDriverRegex);
442 })) {
443 setTarget(addSystemIncludes(Cmd, Info->SystemIncludes), Info->Target);
444 }
445 }
446
447private:
448 // Caches includes extracted from a driver. Key is driver:lang.
449 Memoize<llvm::DenseMap<DriverArgs, std::optional<DriverInfo>>> QueriedDrivers;
450 llvm::Regex QueryDriverRegex;
451};
452} // namespace
453
455getSystemIncludeExtractor(llvm::ArrayRef<std::string> QueryDriverGlobs) {
456 if (QueryDriverGlobs.empty())
457 return nullptr;
458 return SystemIncludeExtractor(QueryDriverGlobs);
459}
460
461} // namespace clang::clangd
const Expr * E
const Criteria C
unsigned Lines
bool StandardCXXIncludes
std::string Driver
bool BuiltinIncludes
bool StandardIncludes
std::string ISysroot
std::string Sysroot
std::vector< std::string > SystemIncludes
std::string Output
Definition: TraceTests.cpp:159
llvm::json::Object Args
Definition: Trace.cpp:138
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:164
@ Info
An information message.
SystemIncludeExtractorFn getSystemIncludeExtractor(llvm::ArrayRef< std::string > QueryDriverGlobs)
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:72
bool operator==(const Inclusion &LHS, const Inclusion &RHS)
Definition: Headers.cpp:317
void log(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:67
std::string printArgv(llvm::ArrayRef< llvm::StringRef > Args)
@ Type
An inlay hint that for a type annotation.
llvm::unique_function< void(tooling::CompileCommand &, llvm::StringRef) const > SystemIncludeExtractorFn
Extracts system include search path from drivers matching QueryDriverGlobs and adds them to the compi...
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:61
Some operations such as code completion produce a set of candidates.
clang::clangd::DriverArgs DriverArgs
static unsigned getHashValue(const DriverArgs &Val)
static bool isEqual(const DriverArgs &LHS, const DriverArgs &RHS)