clang 17.0.0git
Job.cpp
Go to the documentation of this file.
1//===- Job.cpp - Command to Execute ---------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/Driver/Job.h"
10#include "clang/Basic/LLVM.h"
11#include "clang/Driver/Driver.h"
14#include "clang/Driver/Tool.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SmallString.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/StringSet.h"
21#include "llvm/ADT/StringSwitch.h"
22#include "llvm/Support/CrashRecoveryContext.h"
23#include "llvm/Support/FileSystem.h"
24#include "llvm/Support/Path.h"
25#include "llvm/Support/PrettyStackTrace.h"
26#include "llvm/Support/Program.h"
27#include "llvm/Support/raw_ostream.h"
28#include <algorithm>
29#include <cassert>
30#include <cstddef>
31#include <string>
32#include <system_error>
33#include <utility>
34
35using namespace clang;
36using namespace driver;
37
38Command::Command(const Action &Source, const Tool &Creator,
39 ResponseFileSupport ResponseSupport, const char *Executable,
40 const llvm::opt::ArgStringList &Arguments,
42 const char *PrependArg)
43 : Source(Source), Creator(Creator), ResponseSupport(ResponseSupport),
44 Executable(Executable), PrependArg(PrependArg), Arguments(Arguments) {
45 for (const auto &II : Inputs)
46 if (II.isFilename())
47 InputInfoList.push_back(II);
48 for (const auto &II : Outputs)
49 if (II.isFilename())
50 OutputFilenames.push_back(II.getFilename());
51}
52
53/// Check if the compiler flag in question should be skipped when
54/// emitting a reproducer. Also track how many arguments it has and if the
55/// option is some kind of include path.
56static bool skipArgs(const char *Flag, bool HaveCrashVFS, int &SkipNum,
57 bool &IsInclude) {
58 SkipNum = 2;
59 // These flags are all of the form -Flag <Arg> and are treated as two
60 // arguments. Therefore, we need to skip the flag and the next argument.
61 bool ShouldSkip = llvm::StringSwitch<bool>(Flag)
62 .Cases("-MF", "-MT", "-MQ", "-serialize-diagnostic-file", true)
63 .Cases("-o", "-dependency-file", true)
64 .Cases("-fdebug-compilation-dir", "-diagnostic-log-file", true)
65 .Cases("-dwarf-debug-flags", "-ivfsoverlay", true)
66 .Default(false);
67 if (ShouldSkip)
68 return true;
69
70 // Some include flags shouldn't be skipped if we have a crash VFS
71 IsInclude = llvm::StringSwitch<bool>(Flag)
72 .Cases("-include", "-header-include-file", true)
73 .Cases("-idirafter", "-internal-isystem", "-iwithprefix", true)
74 .Cases("-internal-externc-isystem", "-iprefix", true)
75 .Cases("-iwithprefixbefore", "-isystem", "-iquote", true)
76 .Cases("-isysroot", "-I", "-F", "-resource-dir", true)
77 .Cases("-iframework", "-include-pch", true)
78 .Default(false);
79 if (IsInclude)
80 return !HaveCrashVFS;
81
82 // The remaining flags are treated as a single argument.
83
84 // These flags are all of the form -Flag and have no second argument.
85 ShouldSkip = llvm::StringSwitch<bool>(Flag)
86 .Cases("-M", "-MM", "-MG", "-MP", "-MD", true)
87 .Case("-MMD", true)
88 .Default(false);
89
90 // Match found.
91 SkipNum = 1;
92 if (ShouldSkip)
93 return true;
94
95 // These flags are treated as a single argument (e.g., -F<Dir>).
96 StringRef FlagRef(Flag);
97 IsInclude = FlagRef.startswith("-F") || FlagRef.startswith("-I");
98 if (IsInclude)
99 return !HaveCrashVFS;
100 if (FlagRef.startswith("-fmodules-cache-path="))
101 return true;
102
103 SkipNum = 0;
104 return false;
105}
106
107void Command::writeResponseFile(raw_ostream &OS) const {
108 // In a file list, we only write the set of inputs to the response file
109 if (ResponseSupport.ResponseKind == ResponseFileSupport::RF_FileList) {
110 for (const auto *Arg : InputFileList) {
111 OS << Arg << '\n';
112 }
113 return;
114 }
115
116 // In regular response files, we send all arguments to the response file.
117 // Wrapping all arguments in double quotes ensures that both Unix tools and
118 // Windows tools understand the response file.
119 for (const auto *Arg : Arguments) {
120 OS << '"';
121
122 for (; *Arg != '\0'; Arg++) {
123 if (*Arg == '\"' || *Arg == '\\') {
124 OS << '\\';
125 }
126 OS << *Arg;
127 }
128
129 OS << "\" ";
130 }
131}
132
133void Command::buildArgvForResponseFile(
135 // When not a file list, all arguments are sent to the response file.
136 // This leaves us to set the argv to a single parameter, requesting the tool
137 // to read the response file.
138 if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList) {
139 Out.push_back(Executable);
140 Out.push_back(ResponseFileFlag.c_str());
141 return;
142 }
143
144 llvm::StringSet<> Inputs;
145 for (const auto *InputName : InputFileList)
146 Inputs.insert(InputName);
147 Out.push_back(Executable);
148
149 if (PrependArg)
150 Out.push_back(PrependArg);
151
152 // In a file list, build args vector ignoring parameters that will go in the
153 // response file (elements of the InputFileList vector)
154 bool FirstInput = true;
155 for (const auto *Arg : Arguments) {
156 if (Inputs.count(Arg) == 0) {
157 Out.push_back(Arg);
158 } else if (FirstInput) {
159 FirstInput = false;
160 Out.push_back(ResponseSupport.ResponseFlag);
161 Out.push_back(ResponseFile);
162 }
163 }
164}
165
166/// Rewrite relative include-like flag paths to absolute ones.
167static void
169 size_t NumArgs,
171 using namespace llvm;
172 using namespace sys;
173
174 auto getAbsPath = [](StringRef InInc, SmallVectorImpl<char> &OutInc) -> bool {
175 if (path::is_absolute(InInc)) // Nothing to do here...
176 return false;
177 std::error_code EC = fs::current_path(OutInc);
178 if (EC)
179 return false;
180 path::append(OutInc, InInc);
181 return true;
182 };
183
184 SmallString<128> NewInc;
185 if (NumArgs == 1) {
186 StringRef FlagRef(Args[Idx + NumArgs - 1]);
187 assert((FlagRef.startswith("-F") || FlagRef.startswith("-I")) &&
188 "Expecting -I or -F");
189 StringRef Inc = FlagRef.slice(2, StringRef::npos);
190 if (getAbsPath(Inc, NewInc)) {
191 SmallString<128> NewArg(FlagRef.slice(0, 2));
192 NewArg += NewInc;
193 IncFlags.push_back(std::move(NewArg));
194 }
195 return;
196 }
197
198 assert(NumArgs == 2 && "Not expecting more than two arguments");
199 StringRef Inc(Args[Idx + NumArgs - 1]);
200 if (!getAbsPath(Inc, NewInc))
201 return;
202 IncFlags.push_back(SmallString<128>(Args[Idx]));
203 IncFlags.push_back(std::move(NewInc));
204}
205
206void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,
207 CrashReportInfo *CrashInfo) const {
208 // Always quote the exe.
209 OS << ' ';
210 llvm::sys::printArg(OS, Executable, /*Quote=*/true);
211
212 ArrayRef<const char *> Args = Arguments;
214 if (ResponseFile != nullptr) {
215 buildArgvForResponseFile(ArgsRespFile);
216 Args = ArrayRef<const char *>(ArgsRespFile).slice(1); // no executable name
217 } else if (PrependArg) {
218 OS << ' ';
219 llvm::sys::printArg(OS, PrependArg, /*Quote=*/true);
220 }
221
222 bool HaveCrashVFS = CrashInfo && !CrashInfo->VFSPath.empty();
223 for (size_t i = 0, e = Args.size(); i < e; ++i) {
224 const char *const Arg = Args[i];
225
226 if (CrashInfo) {
227 int NumArgs = 0;
228 bool IsInclude = false;
229 if (skipArgs(Arg, HaveCrashVFS, NumArgs, IsInclude)) {
230 i += NumArgs - 1;
231 continue;
232 }
233
234 // Relative includes need to be expanded to absolute paths.
235 if (HaveCrashVFS && IsInclude) {
236 SmallVector<SmallString<128>, 2> NewIncFlags;
237 rewriteIncludes(Args, i, NumArgs, NewIncFlags);
238 if (!NewIncFlags.empty()) {
239 for (auto &F : NewIncFlags) {
240 OS << ' ';
241 llvm::sys::printArg(OS, F.c_str(), Quote);
242 }
243 i += NumArgs - 1;
244 continue;
245 }
246 }
247
248 auto Found = llvm::find_if(InputInfoList, [&Arg](const InputInfo &II) {
249 return II.getFilename() == Arg;
250 });
251 if (Found != InputInfoList.end() &&
252 (i == 0 || StringRef(Args[i - 1]) != "-main-file-name")) {
253 // Replace the input file name with the crashinfo's file name.
254 OS << ' ';
255 StringRef ShortName = llvm::sys::path::filename(CrashInfo->Filename);
256 llvm::sys::printArg(OS, ShortName.str(), Quote);
257 continue;
258 }
259 }
260
261 OS << ' ';
262 llvm::sys::printArg(OS, Arg, Quote);
263 }
264
265 if (CrashInfo && HaveCrashVFS) {
266 OS << ' ';
267 llvm::sys::printArg(OS, "-ivfsoverlay", Quote);
268 OS << ' ';
269 llvm::sys::printArg(OS, CrashInfo->VFSPath.str(), Quote);
270
271 // The leftover modules from the crash are stored in
272 // <name>.cache/vfs/modules
273 // Leave it untouched for pcm inspection and provide a clean/empty dir
274 // path to contain the future generated module cache:
275 // <name>.cache/vfs/repro-modules
276 SmallString<128> RelModCacheDir = llvm::sys::path::parent_path(
277 llvm::sys::path::parent_path(CrashInfo->VFSPath));
278 llvm::sys::path::append(RelModCacheDir, "repro-modules");
279
280 std::string ModCachePath = "-fmodules-cache-path=";
281 ModCachePath.append(RelModCacheDir.c_str());
282
283 OS << ' ';
284 llvm::sys::printArg(OS, ModCachePath, Quote);
285 }
286
287 if (ResponseFile != nullptr) {
288 OS << "\n Arguments passed via response file:\n";
289 writeResponseFile(OS);
290 // Avoiding duplicated newline terminator, since FileLists are
291 // newline-separated.
292 if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList)
293 OS << "\n";
294 OS << " (end of response file)";
295 }
296
297 OS << Terminator;
298}
299
300void Command::setResponseFile(const char *FileName) {
301 ResponseFile = FileName;
302 ResponseFileFlag = ResponseSupport.ResponseFlag;
303 ResponseFileFlag += FileName;
304}
305
307 Environment.reserve(NewEnvironment.size() + 1);
308 Environment.assign(NewEnvironment.begin(), NewEnvironment.end());
309 Environment.push_back(nullptr);
310}
311
313 const std::vector<std::optional<std::string>> &Redirects) {
314 RedirectFiles = Redirects;
315}
316
319 for (const auto &Arg : InputInfoList)
320 llvm::outs() << llvm::sys::path::filename(Arg.getFilename()) << "\n";
321 llvm::outs().flush();
322 }
323}
324
325int Command::Execute(ArrayRef<std::optional<StringRef>> Redirects,
326 std::string *ErrMsg, bool *ExecutionFailed) const {
328
330 if (ResponseFile == nullptr) {
331 Argv.push_back(Executable);
332 if (PrependArg)
333 Argv.push_back(PrependArg);
334 Argv.append(Arguments.begin(), Arguments.end());
335 Argv.push_back(nullptr);
336 } else {
337 // If the command is too large, we need to put arguments in a response file.
338 std::string RespContents;
339 llvm::raw_string_ostream SS(RespContents);
340
341 // Write file contents and build the Argv vector
342 writeResponseFile(SS);
343 buildArgvForResponseFile(Argv);
344 Argv.push_back(nullptr);
345 SS.flush();
346
347 // Save the response file in the appropriate encoding
348 if (std::error_code EC = writeFileWithEncoding(
349 ResponseFile, RespContents, ResponseSupport.ResponseEncoding)) {
350 if (ErrMsg)
351 *ErrMsg = EC.message();
352 if (ExecutionFailed)
353 *ExecutionFailed = true;
354 // Return -1 by convention (see llvm/include/llvm/Support/Program.h) to
355 // indicate the requested executable cannot be started.
356 return -1;
357 }
358 }
359
360 std::optional<ArrayRef<StringRef>> Env;
361 std::vector<StringRef> ArgvVectorStorage;
362 if (!Environment.empty()) {
363 assert(Environment.back() == nullptr &&
364 "Environment vector should be null-terminated by now");
365 ArgvVectorStorage = llvm::toStringRefArray(Environment.data());
366 Env = ArrayRef(ArgvVectorStorage);
367 }
368
369 auto Args = llvm::toStringRefArray(Argv.data());
370
371 // Use Job-specific redirect files if they are present.
372 if (!RedirectFiles.empty()) {
373 std::vector<std::optional<StringRef>> RedirectFilesOptional;
374 for (const auto &Ele : RedirectFiles)
375 if (Ele)
376 RedirectFilesOptional.push_back(std::optional<StringRef>(*Ele));
377 else
378 RedirectFilesOptional.push_back(std::nullopt);
379
380 return llvm::sys::ExecuteAndWait(Executable, Args, Env,
381 ArrayRef(RedirectFilesOptional),
382 /*secondsToWait=*/0, /*memoryLimit=*/0,
383 ErrMsg, ExecutionFailed, &ProcStat);
384 }
385
386 return llvm::sys::ExecuteAndWait(Executable, Args, Env, Redirects,
387 /*secondsToWait*/ 0, /*memoryLimit*/ 0,
388 ErrMsg, ExecutionFailed, &ProcStat);
389}
390
391CC1Command::CC1Command(const Action &Source, const Tool &Creator,
392 ResponseFileSupport ResponseSupport,
393 const char *Executable,
394 const llvm::opt::ArgStringList &Arguments,
396 const char *PrependArg)
397 : Command(Source, Creator, ResponseSupport, Executable, Arguments, Inputs,
398 Outputs, PrependArg) {
399 InProcess = true;
400}
401
402void CC1Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,
403 CrashReportInfo *CrashInfo) const {
404 if (InProcess)
405 OS << " (in-process)\n";
406 Command::Print(OS, Terminator, Quote, CrashInfo);
407}
408
409int CC1Command::Execute(ArrayRef<std::optional<StringRef>> Redirects,
410 std::string *ErrMsg, bool *ExecutionFailed) const {
411 // FIXME: Currently, if there're more than one job, we disable
412 // -fintegrate-cc1. If we're no longer a integrated-cc1 job, fallback to
413 // out-of-process execution. See discussion in https://reviews.llvm.org/D74447
414 if (!InProcess)
415 return Command::Execute(Redirects, ErrMsg, ExecutionFailed);
416
418
420 Argv.push_back(getExecutable());
421 Argv.append(getArguments().begin(), getArguments().end());
422 Argv.push_back(nullptr);
423 Argv.pop_back(); // The terminating null element shall not be part of the
424 // slice (main() behavior).
425
426 // This flag simply indicates that the program couldn't start, which isn't
427 // applicable here.
428 if (ExecutionFailed)
429 *ExecutionFailed = false;
430
431 llvm::CrashRecoveryContext CRC;
432 CRC.DumpStackAndCleanupOnFailure = true;
433
434 const void *PrettyState = llvm::SavePrettyStackState();
435 const Driver &D = getCreator().getToolChain().getDriver();
436
437 int R = 0;
438 // Enter ExecuteCC1Tool() instead of starting up a new process
439 if (!CRC.RunSafely([&]() { R = D.CC1Main(Argv); })) {
440 llvm::RestorePrettyStackState(PrettyState);
441 return CRC.RetCode;
442 }
443 return R;
444}
445
447 // We don't support set a new environment when calling into ExecuteCC1Tool()
448 llvm_unreachable(
449 "The CC1Command doesn't support changing the environment vars!");
450}
451
453 const Action &Source_, const Tool &Creator_,
454 ResponseFileSupport ResponseSupport, const char *Executable_,
455 const llvm::opt::ArgStringList &Arguments_, ArrayRef<InputInfo> Inputs,
456 ArrayRef<InputInfo> Outputs)
457 : Command(Source_, Creator_, ResponseSupport, Executable_, Arguments_,
458 Inputs, Outputs) {}
459
460void ForceSuccessCommand::Print(raw_ostream &OS, const char *Terminator,
461 bool Quote, CrashReportInfo *CrashInfo) const {
462 Command::Print(OS, "", Quote, CrashInfo);
463 OS << " || (exit 0)" << Terminator;
464}
465
466int ForceSuccessCommand::Execute(ArrayRef<std::optional<StringRef>> Redirects,
467 std::string *ErrMsg,
468 bool *ExecutionFailed) const {
469 int Status = Command::Execute(Redirects, ErrMsg, ExecutionFailed);
470 (void)Status;
471 if (ExecutionFailed)
472 *ExecutionFailed = false;
473 return 0;
474}
475
476void JobList::Print(raw_ostream &OS, const char *Terminator, bool Quote,
477 CrashReportInfo *CrashInfo) const {
478 for (const auto &Job : *this)
479 Job.Print(OS, Terminator, Quote, CrashInfo);
480}
481
482void JobList::clear() { Jobs.clear(); }
static void rewriteIncludes(const llvm::ArrayRef< const char * > &Args, size_t Idx, size_t NumArgs, llvm::SmallVectorImpl< llvm::SmallString< 128 > > &IncFlags)
Rewrite relative include-like flag paths to absolute ones.
Definition: Job.cpp:168
static bool skipArgs(const char *Flag, bool HaveCrashVFS, int &SkipNum, bool &IsInclude)
Check if the compiler flag in question should be skipped when emitting a reproducer.
Definition: Job.cpp:56
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
llvm::raw_ostream & OS
Definition: Logger.cpp:24
Action - Represent an abstract compilation step to perform.
Definition: Action.h:47
void setEnvironment(llvm::ArrayRef< const char * > NewEnvironment) override
Sets the environment to be used by the new process.
Definition: Job.cpp:446
void Print(llvm::raw_ostream &OS, const char *Terminator, bool Quote, CrashReportInfo *CrashInfo=nullptr) const override
Definition: Job.cpp:402
CC1Command(const Action &Source, const Tool &Creator, ResponseFileSupport ResponseSupport, const char *Executable, const llvm::opt::ArgStringList &Arguments, ArrayRef< InputInfo > Inputs, ArrayRef< InputInfo > Outputs=std::nullopt, const char *PrependArg=nullptr)
Definition: Job.cpp:391
int Execute(ArrayRef< std::optional< StringRef > > Redirects, std::string *ErrMsg, bool *ExecutionFailed) const override
Definition: Job.cpp:409
Command - An executable path/name and argument vector to execute.
Definition: Job.h:106
const Tool & getCreator() const
getCreator - Return the Tool which caused the creation of this job.
Definition: Job.h:192
const llvm::opt::ArgStringList & getArguments() const
Definition: Job.h:225
void setResponseFile(const char *FileName)
Set to pass arguments via a response file when launching the command.
Definition: Job.cpp:300
void setRedirectFiles(const std::vector< std::optional< std::string > > &Redirects)
Definition: Job.cpp:312
bool PrintInputFilenames
Whether to print the input filenames when executing.
Definition: Job.h:167
const char * getExecutable() const
Definition: Job.h:223
virtual void Print(llvm::raw_ostream &OS, const char *Terminator, bool Quote, CrashReportInfo *CrashInfo=nullptr) const
Definition: Job.cpp:206
bool InProcess
Whether the command will be executed in this process or not.
Definition: Job.h:170
virtual void setEnvironment(llvm::ArrayRef< const char * > NewEnvironment)
Sets the environment to be used by the new process.
Definition: Job.cpp:306
Command(const Action &Source, const Tool &Creator, ResponseFileSupport ResponseSupport, const char *Executable, const llvm::opt::ArgStringList &Arguments, ArrayRef< InputInfo > Inputs, ArrayRef< InputInfo > Outputs=std::nullopt, const char *PrependArg=nullptr)
Definition: Job.cpp:38
void PrintFileNames() const
Optionally print the filenames to be compiled.
Definition: Job.cpp:317
virtual int Execute(ArrayRef< std::optional< StringRef > > Redirects, std::string *ErrMsg, bool *ExecutionFailed) const
Definition: Job.cpp:325
Driver - Encapsulate logic for constructing compilation processes from a set of gcc-driver-like comma...
Definition: Driver.h:77
int Execute(ArrayRef< std::optional< StringRef > > Redirects, std::string *ErrMsg, bool *ExecutionFailed) const override
Definition: Job.cpp:466
void Print(llvm::raw_ostream &OS, const char *Terminator, bool Quote, CrashReportInfo *CrashInfo=nullptr) const override
Definition: Job.cpp:460
ForceSuccessCommand(const Action &Source_, const Tool &Creator_, ResponseFileSupport ResponseSupport, const char *Executable_, const llvm::opt::ArgStringList &Arguments_, ArrayRef< InputInfo > Inputs, ArrayRef< InputInfo > Outputs=std::nullopt)
Definition: Job.cpp:452
InputInfo - Wrapper for information about an input source.
Definition: InputInfo.h:22
const char * getFilename() const
Definition: InputInfo.h:83
void clear()
Clear the job list.
Definition: Job.cpp:482
void Print(llvm::raw_ostream &OS, const char *Terminator, bool Quote, CrashReportInfo *CrashInfo=nullptr) const
Definition: Job.cpp:476
const Driver & getDriver() const
Definition: ToolChain.h:236
Tool - Information on a specific compilation tool.
Definition: Tool.h:32
const ToolChain & getToolChain() const
Definition: Tool.h:52
YAML serialization mapping.
Definition: Dominators.h:30
ResponseFileKind ResponseKind
The level of support for response files.
Definition: Job.h:57
llvm::sys::WindowsEncodingMethod ResponseEncoding
The encoding to use when writing response files on Windows.
Definition: Job.h:70
const char * ResponseFlag
What prefix to use for the command-line argument when passing a response file.
Definition: Job.h:74