clang-tools 17.0.0git
SystemIncludeExtractor.cpp
Go to the documentation of this file.
1//===--- SystemIncludeExtractor.cpp ------------------------------*- C++-*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// Some compiler drivers have implicit search mechanism for system headers.
9// This compilation database implementation tries to extract that information by
10// executing the driver in verbose mode. gcc-compatible drivers print something
11// like:
12// ....
13// ....
14// #include <...> search starts here:
15// /usr/lib/gcc/x86_64-linux-gnu/7/include
16// /usr/local/include
17// /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed
18// /usr/include/x86_64-linux-gnu
19// /usr/include
20// End of search list.
21// ....
22// ....
23// This component parses that output and adds each path to command line args
24// provided by Base, after prepending them with -isystem. Therefore current
25// implementation would not work with a driver that is not gcc-compatible.
26//
27// First argument of the command line received from underlying compilation
28// database is used as compiler driver path. Due to this arbitrary binary
29// execution, this mechanism is not used by default and only executes binaries
30// in the paths that are explicitly included by the user.
31
32#include "CompileCommands.h"
34#include "support/Logger.h"
35#include "support/Path.h"
36#include "support/Trace.h"
37#include "clang/Basic/Diagnostic.h"
38#include "clang/Basic/TargetInfo.h"
39#include "clang/Basic/TargetOptions.h"
40#include "clang/Driver/Types.h"
41#include "clang/Tooling/CompilationDatabase.h"
42#include "llvm/ADT/ScopeExit.h"
43#include "llvm/ADT/SmallString.h"
44#include "llvm/ADT/StringExtras.h"
45#include "llvm/ADT/StringRef.h"
46#include "llvm/Support/FileSystem.h"
47#include "llvm/Support/MemoryBuffer.h"
48#include "llvm/Support/Path.h"
49#include "llvm/Support/Program.h"
50#include "llvm/Support/Regex.h"
51#include "llvm/Support/ScopedPrinter.h"
52#include <algorithm>
53#include <iterator>
54#include <map>
55#include <optional>
56#include <string>
57#include <vector>
58
59namespace clang {
60namespace clangd {
61namespace {
62
63struct DriverInfo {
64 std::vector<std::string> SystemIncludes;
65 std::string Target;
66};
67
68bool isValidTarget(llvm::StringRef Triple) {
69 std::shared_ptr<TargetOptions> TargetOpts(new TargetOptions);
70 TargetOpts->Triple = Triple.str();
71 DiagnosticsEngine Diags(new DiagnosticIDs, new DiagnosticOptions,
72 new IgnoringDiagConsumer);
73 IntrusiveRefCntPtr<TargetInfo> Target =
74 TargetInfo::CreateTargetInfo(Diags, TargetOpts);
75 return bool(Target);
76}
77
78std::optional<DriverInfo> parseDriverOutput(llvm::StringRef Output) {
79 DriverInfo Info;
80 const char SIS[] = "#include <...> search starts here:";
81 const char SIE[] = "End of search list.";
82 const char TS[] = "Target: ";
83 llvm::SmallVector<llvm::StringRef> Lines;
84 Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
85
86 enum {
87 Initial, // Initial state: searching for target or includes list.
88 IncludesExtracting, // Includes extracting.
89 Done // Includes and target extraction done.
90 } State = Initial;
91 bool SeenIncludes = false;
92 bool SeenTarget = false;
93 for (auto *It = Lines.begin(); State != Done && It != Lines.end(); ++It) {
94 auto Line = *It;
95 switch (State) {
96 case Initial:
97 if (!SeenIncludes && Line.trim() == SIS) {
98 SeenIncludes = true;
99 State = IncludesExtracting;
100 } else if (!SeenTarget && Line.trim().startswith(TS)) {
101 SeenTarget = true;
102 llvm::StringRef TargetLine = Line.trim();
103 TargetLine.consume_front(TS);
104 // Only detect targets that clang understands
105 if (!isValidTarget(TargetLine)) {
106 elog("System include extraction: invalid target \"{0}\", ignoring",
107 TargetLine);
108 } else {
109 Info.Target = TargetLine.str();
110 vlog("System include extraction: target extracted: \"{0}\"",
111 TargetLine);
112 }
113 }
114 break;
115 case IncludesExtracting:
116 if (Line.trim() == SIE) {
117 State = SeenTarget ? Done : Initial;
118 } else {
119 Info.SystemIncludes.push_back(Line.trim().str());
120 vlog("System include extraction: adding {0}", Line);
121 }
122 break;
123 default:
124 llvm_unreachable("Impossible state of the driver output parser");
125 break;
126 }
127 }
128 if (!SeenIncludes) {
129 elog("System include extraction: start marker not found: {0}", Output);
130 return std::nullopt;
131 }
132 if (State == IncludesExtracting) {
133 elog("System include extraction: end marker missing: {0}", Output);
134 return std::nullopt;
135 }
136 return std::move(Info);
137}
138
139std::optional<DriverInfo>
140extractSystemIncludesAndTarget(llvm::SmallString<128> Driver,
141 llvm::StringRef Lang,
142 llvm::ArrayRef<std::string> CommandLine,
143 const llvm::Regex &QueryDriverRegex) {
144 trace::Span Tracer("Extract system includes and target");
145
146 if (!llvm::sys::path::is_absolute(Driver)) {
147 assert(llvm::none_of(
148 Driver, [](char C) { return llvm::sys::path::is_separator(C); }));
149 auto DriverProgram = llvm::sys::findProgramByName(Driver);
150 if (DriverProgram) {
151 vlog("System include extraction: driver {0} expanded to {1}", Driver,
152 *DriverProgram);
153 Driver = *DriverProgram;
154 } else {
155 elog("System include extraction: driver {0} not found in PATH", Driver);
156 return std::nullopt;
157 }
158 }
159
160 SPAN_ATTACH(Tracer, "driver", Driver);
161 SPAN_ATTACH(Tracer, "lang", Lang);
162
163 if (!QueryDriverRegex.match(Driver)) {
164 vlog("System include extraction: not allowed driver {0}", Driver);
165 return std::nullopt;
166 }
167
168 llvm::SmallString<128> StdErrPath;
169 if (auto EC = llvm::sys::fs::createTemporaryFile("system-includes", "clangd",
170 StdErrPath)) {
171 elog("System include extraction: failed to create temporary file with "
172 "error {0}",
173 EC.message());
174 return std::nullopt;
175 }
176 auto CleanUp = llvm::make_scope_exit(
177 [&StdErrPath]() { llvm::sys::fs::remove(StdErrPath); });
178
179 std::optional<llvm::StringRef> Redirects[] = {{""}, {""}, StdErrPath.str()};
180
181 llvm::SmallVector<llvm::StringRef> Args = {Driver, "-E", "-x",
182 Lang, "-", "-v"};
183
184 // These flags will be preserved
185 const llvm::StringRef FlagsToPreserve[] = {
186 "-nostdinc", "--no-standard-includes", "-nostdinc++", "-nobuiltininc"};
187 // Preserves these flags and their values, either as separate args or with an
188 // equalsbetween them
189 const llvm::StringRef ArgsToPreserve[] = {"--sysroot", "-isysroot"};
190
191 for (size_t I = 0, E = CommandLine.size(); I < E; ++I) {
192 llvm::StringRef Arg = CommandLine[I];
193 if (llvm::is_contained(FlagsToPreserve, Arg)) {
194 Args.push_back(Arg);
195 } else {
196 const auto *Found =
197 llvm::find_if(ArgsToPreserve, [&Arg](llvm::StringRef S) {
198 return Arg.startswith(S);
199 });
200 if (Found == std::end(ArgsToPreserve))
201 continue;
202 Arg = Arg.drop_front(Found->size());
203 if (Arg.empty() && I + 1 < E) {
204 Args.push_back(CommandLine[I]);
205 Args.push_back(CommandLine[++I]);
206 } else if (Arg.startswith("=")) {
207 Args.push_back(CommandLine[I]);
208 }
209 }
210 }
211
212 std::string ErrMsg;
213 if (int RC = llvm::sys::ExecuteAndWait(Driver, Args, /*Env=*/std::nullopt,
214 Redirects, /*SecondsToWait=*/0,
215 /*MemoryLimit=*/0, &ErrMsg)) {
216 elog("System include extraction: driver execution failed with return code: "
217 "{0} - '{1}'. Args: [{2}]",
218 llvm::to_string(RC), ErrMsg, printArgv(Args));
219 return std::nullopt;
220 }
221
222 auto BufOrError = llvm::MemoryBuffer::getFile(StdErrPath);
223 if (!BufOrError) {
224 elog("System include extraction: failed to read {0} with error {1}",
225 StdErrPath, BufOrError.getError().message());
226 return std::nullopt;
227 }
228
229 std::optional<DriverInfo> Info =
230 parseDriverOutput(BufOrError->get()->getBuffer());
231 if (!Info)
232 return std::nullopt;
233 log("System includes extractor: successfully executed {0}\n\tgot includes: "
234 "\"{1}\"\n\tgot target: \"{2}\"",
235 Driver, llvm::join(Info->SystemIncludes, ", "), Info->Target);
236 return Info;
237}
238
239tooling::CompileCommand &
240addSystemIncludes(tooling::CompileCommand &Cmd,
241 llvm::ArrayRef<std::string> SystemIncludes) {
242 std::vector<std::string> ToAppend;
243 for (llvm::StringRef Include : SystemIncludes) {
244 // FIXME(kadircet): This doesn't work when we have "--driver-mode=cl"
245 ToAppend.push_back("-isystem");
246 ToAppend.push_back(Include.str());
247 }
248 if (!ToAppend.empty()) {
249 // Just append when `--` isn't present.
250 auto InsertAt = llvm::find(Cmd.CommandLine, "--");
251 Cmd.CommandLine.insert(InsertAt, std::make_move_iterator(ToAppend.begin()),
252 std::make_move_iterator(ToAppend.end()));
253 }
254 return Cmd;
255}
256
257tooling::CompileCommand &setTarget(tooling::CompileCommand &Cmd,
258 const std::string &Target) {
259 if (!Target.empty()) {
260 // We do not want to override existing target with extracted one.
261 for (llvm::StringRef Arg : Cmd.CommandLine) {
262 if (Arg == "-target" || Arg.startswith("--target="))
263 return Cmd;
264 }
265 // Just append when `--` isn't present.
266 auto InsertAt = llvm::find(Cmd.CommandLine, "--");
267 Cmd.CommandLine.insert(InsertAt, "--target=" + Target);
268 }
269 return Cmd;
270}
271
272/// Converts a glob containing only ** or * into a regex.
273std::string convertGlobToRegex(llvm::StringRef Glob) {
274 std::string RegText;
275 llvm::raw_string_ostream RegStream(RegText);
276 RegStream << '^';
277 for (size_t I = 0, E = Glob.size(); I < E; ++I) {
278 if (Glob[I] == '*') {
279 if (I + 1 < E && Glob[I + 1] == '*') {
280 // Double star, accept any sequence.
281 RegStream << ".*";
282 // Also skip the second star.
283 ++I;
284 } else {
285 // Single star, accept any sequence without a slash.
286 RegStream << "[^/]*";
287 }
288 } else if (llvm::sys::path::is_separator(Glob[I]) &&
289 llvm::sys::path::is_separator('/') &&
290 llvm::sys::path::is_separator('\\')) {
291 RegStream << R"([/\\])"; // Accept either slash on windows.
292 } else {
293 RegStream << llvm::Regex::escape(Glob.substr(I, 1));
294 }
295 }
296 RegStream << '$';
297 RegStream.flush();
298 return RegText;
299}
300
301/// Converts a glob containing only ** or * into a regex.
302llvm::Regex convertGlobsToRegex(llvm::ArrayRef<std::string> Globs) {
303 assert(!Globs.empty() && "Globs cannot be empty!");
304 std::vector<std::string> RegTexts;
305 RegTexts.reserve(Globs.size());
306 for (llvm::StringRef Glob : Globs)
307 RegTexts.push_back(convertGlobToRegex(Glob));
308
309 // Tempting to pass IgnoreCase, but we don't know the FS sensitivity.
310 llvm::Regex Reg(llvm::join(RegTexts, "|"));
311 assert(Reg.isValid(RegTexts.front()) &&
312 "Created an invalid regex from globs");
313 return Reg;
314}
315
316/// Extracts system includes from a trusted driver by parsing the output of
317/// include search path and appends them to the commands coming from underlying
318/// compilation database.
319class SystemIncludeExtractor {
320public:
321 SystemIncludeExtractor(llvm::ArrayRef<std::string> QueryDriverGlobs)
322 : QueryDriverRegex(convertGlobsToRegex(QueryDriverGlobs)) {}
323
324 void operator()(tooling::CompileCommand &Cmd, llvm::StringRef File) const {
325 if (Cmd.CommandLine.empty())
326 return;
327
328 llvm::StringRef Lang;
329 for (size_t I = 0, E = Cmd.CommandLine.size(); I < E; ++I) {
330 llvm::StringRef Arg = Cmd.CommandLine[I];
331 if (Arg == "-x" && I + 1 < E)
332 Lang = Cmd.CommandLine[I + 1];
333 else if (Arg.startswith("-x"))
334 Lang = Arg.drop_front(2).trim();
335 }
336 if (Lang.empty()) {
337 llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.');
338 auto Type = driver::types::lookupTypeForExtension(Ext);
339 if (Type == driver::types::TY_INVALID) {
340 elog("System include extraction: invalid file type for {0}", Ext);
341 return;
342 }
343 Lang = driver::types::getTypeName(Type);
344 }
345
346 llvm::SmallString<128> Driver(Cmd.CommandLine.front());
347 if (llvm::any_of(Driver,
348 [](char C) { return llvm::sys::path::is_separator(C); }))
349 // Driver is a not a single executable name but instead a path (either
350 // relative or absolute).
351 llvm::sys::fs::make_absolute(Cmd.Directory, Driver);
352
353 if (auto Info =
354 QueriedDrivers.get(/*Key=*/(Driver + ":" + Lang).str(), [&] {
355 return extractSystemIncludesAndTarget(
356 Driver, Lang, Cmd.CommandLine, QueryDriverRegex);
357 })) {
358 setTarget(addSystemIncludes(Cmd, Info->SystemIncludes), Info->Target);
359 }
360 }
361
362private:
363 // Caches includes extracted from a driver. Key is driver:lang.
364 Memoize<llvm::StringMap<std::optional<DriverInfo>>> QueriedDrivers;
365 llvm::Regex QueryDriverRegex;
366};
367} // namespace
368
370getSystemIncludeExtractor(llvm::ArrayRef<std::string> QueryDriverGlobs) {
371 if (QueryDriverGlobs.empty())
372 return nullptr;
373 return SystemIncludeExtractor(QueryDriverGlobs);
374}
375
376} // namespace clangd
377} // namespace clang
const Expr * E
const Criteria C
unsigned Lines
std::vector< llvm::StringRef > CommandLine
std::vector< std::string > SystemIncludes
std::string Output
Definition: TraceTests.cpp:159
llvm::json::Object Args
Definition: Trace.cpp:138
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:164
@ Info
An information message.
SystemIncludeExtractorFn getSystemIncludeExtractor(llvm::ArrayRef< std::string > QueryDriverGlobs)
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:72
void log(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:67
std::string printArgv(llvm::ArrayRef< llvm::StringRef > Args)
@ Type
An inlay hint that for a type annotation.
llvm::unique_function< void(tooling::CompileCommand &, llvm::StringRef) const > SystemIncludeExtractorFn
Extracts system include search path from drivers matching QueryDriverGlobs and adds them to the compi...
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:61
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//