clang 19.0.0git
OffloadBundler.cpp
Go to the documentation of this file.
1//===- OffloadBundler.cpp - File Bundling and Unbundling ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements an offload bundling API that bundles different files
11/// that relate with the same source code but different targets into a single
12/// one. Also the implements the opposite functionality, i.e. unbundle files
13/// previous created by this API.
14///
15//===----------------------------------------------------------------------===//
16
18#include "clang/Basic/Cuda.h"
20#include "clang/Basic/Version.h"
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringExtras.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringRef.h"
27#include "llvm/BinaryFormat/Magic.h"
28#include "llvm/Object/Archive.h"
29#include "llvm/Object/ArchiveWriter.h"
30#include "llvm/Object/Binary.h"
31#include "llvm/Object/ObjectFile.h"
32#include "llvm/Support/Casting.h"
33#include "llvm/Support/Compression.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/EndianStream.h"
36#include "llvm/Support/Errc.h"
37#include "llvm/Support/Error.h"
38#include "llvm/Support/ErrorOr.h"
39#include "llvm/Support/FileSystem.h"
40#include "llvm/Support/MD5.h"
41#include "llvm/Support/MemoryBuffer.h"
42#include "llvm/Support/Path.h"
43#include "llvm/Support/Program.h"
44#include "llvm/Support/Signals.h"
45#include "llvm/Support/StringSaver.h"
46#include "llvm/Support/Timer.h"
47#include "llvm/Support/WithColor.h"
48#include "llvm/Support/raw_ostream.h"
49#include "llvm/TargetParser/Host.h"
50#include "llvm/TargetParser/Triple.h"
51#include <algorithm>
52#include <cassert>
53#include <cstddef>
54#include <cstdint>
55#include <forward_list>
56#include <llvm/Support/Process.h>
57#include <memory>
58#include <set>
59#include <string>
60#include <system_error>
61#include <utility>
62
63using namespace llvm;
64using namespace llvm::object;
65using namespace clang;
66
67static llvm::TimerGroup
68 ClangOffloadBundlerTimerGroup("Clang Offload Bundler Timer Group",
69 "Timer group for clang offload bundler");
70
71/// Magic string that marks the existence of offloading data.
72#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
73
75 const OffloadBundlerConfig &BC)
76 : BundlerConfig(BC) {
77
78 // TODO: Add error checking from ClangOffloadBundler.cpp
79 auto TargetFeatures = Target.split(':');
80 auto TripleOrGPU = TargetFeatures.first.rsplit('-');
81
82 if (clang::StringToCudaArch(TripleOrGPU.second) != clang::CudaArch::UNKNOWN) {
83 auto KindTriple = TripleOrGPU.first.split('-');
84 this->OffloadKind = KindTriple.first;
85
86 // Enforce optional env field to standardize bundles
87 llvm::Triple t = llvm::Triple(KindTriple.second);
88 this->Triple = llvm::Triple(t.getArchName(), t.getVendorName(),
89 t.getOSName(), t.getEnvironmentName());
90
91 this->TargetID = Target.substr(Target.find(TripleOrGPU.second));
92 } else {
93 auto KindTriple = TargetFeatures.first.split('-');
94 this->OffloadKind = KindTriple.first;
95
96 // Enforce optional env field to standardize bundles
97 llvm::Triple t = llvm::Triple(KindTriple.second);
98 this->Triple = llvm::Triple(t.getArchName(), t.getVendorName(),
99 t.getOSName(), t.getEnvironmentName());
100
101 this->TargetID = "";
102 }
103}
104
106 return this->OffloadKind == "host";
107}
108
110 return OffloadKind == "host" || OffloadKind == "openmp" ||
111 OffloadKind == "hip" || OffloadKind == "hipv4";
112}
113
115 const StringRef TargetOffloadKind) const {
116 if (OffloadKind == TargetOffloadKind)
117 return true;
119 bool HIPCompatibleWithOpenMP = OffloadKind.starts_with_insensitive("hip") &&
120 TargetOffloadKind == "openmp";
121 bool OpenMPCompatibleWithHIP =
122 OffloadKind == "openmp" &&
123 TargetOffloadKind.starts_with_insensitive("hip");
124 return HIPCompatibleWithOpenMP || OpenMPCompatibleWithHIP;
125 }
126 return false;
127}
128
130 return !Triple.str().empty() && Triple.getArch() != Triple::UnknownArch;
131}
132
134 return OffloadKind == Target.OffloadKind &&
135 Triple.isCompatibleWith(Target.Triple) && TargetID == Target.TargetID;
136}
137
138std::string OffloadTargetInfo::str() const {
139 return Twine(OffloadKind + "-" + Triple.str() + "-" + TargetID).str();
140}
141
142static StringRef getDeviceFileExtension(StringRef Device,
143 StringRef BundleFileName) {
144 if (Device.contains("gfx"))
145 return ".bc";
146 if (Device.contains("sm_"))
147 return ".cubin";
148 return sys::path::extension(BundleFileName);
149}
150
151static std::string getDeviceLibraryFileName(StringRef BundleFileName,
152 StringRef Device) {
153 StringRef LibName = sys::path::stem(BundleFileName);
154 StringRef Extension = getDeviceFileExtension(Device, BundleFileName);
155
156 std::string Result;
157 Result += LibName;
158 Result += Extension;
159 return Result;
160}
161
162namespace {
163/// Generic file handler interface.
164class FileHandler {
165public:
166 struct BundleInfo {
167 StringRef BundleID;
168 };
169
170 FileHandler() {}
171
172 virtual ~FileHandler() {}
173
174 /// Update the file handler with information from the header of the bundled
175 /// file.
176 virtual Error ReadHeader(MemoryBuffer &Input) = 0;
177
178 /// Read the marker of the next bundled to be read in the file. The bundle
179 /// name is returned if there is one in the file, or `std::nullopt` if there
180 /// are no more bundles to be read.
182 ReadBundleStart(MemoryBuffer &Input) = 0;
183
184 /// Read the marker that closes the current bundle.
185 virtual Error ReadBundleEnd(MemoryBuffer &Input) = 0;
186
187 /// Read the current bundle and write the result into the stream \a OS.
188 virtual Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) = 0;
189
190 /// Write the header of the bundled file to \a OS based on the information
191 /// gathered from \a Inputs.
192 virtual Error WriteHeader(raw_ostream &OS,
193 ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) = 0;
194
195 /// Write the marker that initiates a bundle for the triple \a TargetTriple to
196 /// \a OS.
197 virtual Error WriteBundleStart(raw_ostream &OS, StringRef TargetTriple) = 0;
198
199 /// Write the marker that closes a bundle for the triple \a TargetTriple to \a
200 /// OS.
201 virtual Error WriteBundleEnd(raw_ostream &OS, StringRef TargetTriple) = 0;
202
203 /// Write the bundle from \a Input into \a OS.
204 virtual Error WriteBundle(raw_ostream &OS, MemoryBuffer &Input) = 0;
205
206 /// Finalize output file.
207 virtual Error finalizeOutputFile() { return Error::success(); }
208
209 /// List bundle IDs in \a Input.
210 virtual Error listBundleIDs(MemoryBuffer &Input) {
211 if (Error Err = ReadHeader(Input))
212 return Err;
213 return forEachBundle(Input, [&](const BundleInfo &Info) -> Error {
214 llvm::outs() << Info.BundleID << '\n';
215 Error Err = listBundleIDsCallback(Input, Info);
216 if (Err)
217 return Err;
218 return Error::success();
219 });
220 }
221
222 /// Get bundle IDs in \a Input in \a BundleIds.
223 virtual Error getBundleIDs(MemoryBuffer &Input,
224 std::set<StringRef> &BundleIds) {
225 if (Error Err = ReadHeader(Input))
226 return Err;
227 return forEachBundle(Input, [&](const BundleInfo &Info) -> Error {
228 BundleIds.insert(Info.BundleID);
229 Error Err = listBundleIDsCallback(Input, Info);
230 if (Err)
231 return Err;
232 return Error::success();
233 });
234 }
235
236 /// For each bundle in \a Input, do \a Func.
237 Error forEachBundle(MemoryBuffer &Input,
238 std::function<Error(const BundleInfo &)> Func) {
239 while (true) {
240 Expected<std::optional<StringRef>> CurTripleOrErr =
241 ReadBundleStart(Input);
242 if (!CurTripleOrErr)
243 return CurTripleOrErr.takeError();
244
245 // No more bundles.
246 if (!*CurTripleOrErr)
247 break;
248
249 StringRef CurTriple = **CurTripleOrErr;
250 assert(!CurTriple.empty());
251
252 BundleInfo Info{CurTriple};
253 if (Error Err = Func(Info))
254 return Err;
255 }
256 return Error::success();
257 }
258
259protected:
260 virtual Error listBundleIDsCallback(MemoryBuffer &Input,
261 const BundleInfo &Info) {
262 return Error::success();
263 }
264};
265
266/// Handler for binary files. The bundled file will have the following format
267/// (all integers are stored in little-endian format):
268///
269/// "OFFLOAD_BUNDLER_MAGIC_STR" (ASCII encoding of the string)
270///
271/// NumberOfOffloadBundles (8-byte integer)
272///
273/// OffsetOfBundle1 (8-byte integer)
274/// SizeOfBundle1 (8-byte integer)
275/// NumberOfBytesInTripleOfBundle1 (8-byte integer)
276/// TripleOfBundle1 (byte length defined before)
277///
278/// ...
279///
280/// OffsetOfBundleN (8-byte integer)
281/// SizeOfBundleN (8-byte integer)
282/// NumberOfBytesInTripleOfBundleN (8-byte integer)
283/// TripleOfBundleN (byte length defined before)
284///
285/// Bundle1
286/// ...
287/// BundleN
288
289/// Read 8-byte integers from a buffer in little-endian format.
290static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) {
291 return llvm::support::endian::read64le(Buffer.data() + pos);
292}
293
294/// Write 8-byte integers to a buffer in little-endian format.
295static void Write8byteIntegerToBuffer(raw_ostream &OS, uint64_t Val) {
296 llvm::support::endian::write(OS, Val, llvm::endianness::little);
297}
298
299class BinaryFileHandler final : public FileHandler {
300 /// Information about the bundles extracted from the header.
301 struct BinaryBundleInfo final : public BundleInfo {
302 /// Size of the bundle.
303 uint64_t Size = 0u;
304 /// Offset at which the bundle starts in the bundled file.
305 uint64_t Offset = 0u;
306
307 BinaryBundleInfo() {}
308 BinaryBundleInfo(uint64_t Size, uint64_t Offset)
309 : Size(Size), Offset(Offset) {}
310 };
311
312 /// Map between a triple and the corresponding bundle information.
313 StringMap<BinaryBundleInfo> BundlesInfo;
314
315 /// Iterator for the bundle information that is being read.
316 StringMap<BinaryBundleInfo>::iterator CurBundleInfo;
317 StringMap<BinaryBundleInfo>::iterator NextBundleInfo;
318
319 /// Current bundle target to be written.
320 std::string CurWriteBundleTarget;
321
322 /// Configuration options and arrays for this bundler job
323 const OffloadBundlerConfig &BundlerConfig;
324
325public:
326 // TODO: Add error checking from ClangOffloadBundler.cpp
327 BinaryFileHandler(const OffloadBundlerConfig &BC) : BundlerConfig(BC) {}
328
329 ~BinaryFileHandler() final {}
330
331 Error ReadHeader(MemoryBuffer &Input) final {
332 StringRef FC = Input.getBuffer();
333
334 // Initialize the current bundle with the end of the container.
335 CurBundleInfo = BundlesInfo.end();
336
337 // Check if buffer is smaller than magic string.
338 size_t ReadChars = sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1;
339 if (ReadChars > FC.size())
340 return Error::success();
341
342 // Check if no magic was found.
343 if (llvm::identify_magic(FC) != llvm::file_magic::offload_bundle)
344 return Error::success();
345
346 // Read number of bundles.
347 if (ReadChars + 8 > FC.size())
348 return Error::success();
349
350 uint64_t NumberOfBundles = Read8byteIntegerFromBuffer(FC, ReadChars);
351 ReadChars += 8;
352
353 // Read bundle offsets, sizes and triples.
354 for (uint64_t i = 0; i < NumberOfBundles; ++i) {
355
356 // Read offset.
357 if (ReadChars + 8 > FC.size())
358 return Error::success();
359
360 uint64_t Offset = Read8byteIntegerFromBuffer(FC, ReadChars);
361 ReadChars += 8;
362
363 // Read size.
364 if (ReadChars + 8 > FC.size())
365 return Error::success();
366
367 uint64_t Size = Read8byteIntegerFromBuffer(FC, ReadChars);
368 ReadChars += 8;
369
370 // Read triple size.
371 if (ReadChars + 8 > FC.size())
372 return Error::success();
373
374 uint64_t TripleSize = Read8byteIntegerFromBuffer(FC, ReadChars);
375 ReadChars += 8;
376
377 // Read triple.
378 if (ReadChars + TripleSize > FC.size())
379 return Error::success();
380
381 StringRef Triple(&FC.data()[ReadChars], TripleSize);
382 ReadChars += TripleSize;
383
384 // Check if the offset and size make sense.
385 if (!Offset || Offset + Size > FC.size())
386 return Error::success();
387
388 assert(!BundlesInfo.contains(Triple) && "Triple is duplicated??");
389 BundlesInfo[Triple] = BinaryBundleInfo(Size, Offset);
390 }
391 // Set the iterator to where we will start to read.
392 CurBundleInfo = BundlesInfo.end();
393 NextBundleInfo = BundlesInfo.begin();
394 return Error::success();
395 }
396
398 ReadBundleStart(MemoryBuffer &Input) final {
399 if (NextBundleInfo == BundlesInfo.end())
400 return std::nullopt;
401 CurBundleInfo = NextBundleInfo++;
402 return CurBundleInfo->first();
403 }
404
405 Error ReadBundleEnd(MemoryBuffer &Input) final {
406 assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
407 return Error::success();
408 }
409
410 Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
411 assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
412 StringRef FC = Input.getBuffer();
413 OS.write(FC.data() + CurBundleInfo->second.Offset,
414 CurBundleInfo->second.Size);
415 return Error::success();
416 }
417
418 Error WriteHeader(raw_ostream &OS,
419 ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
420
421 // Compute size of the header.
422 uint64_t HeaderSize = 0;
423
424 HeaderSize += sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1;
425 HeaderSize += 8; // Number of Bundles
426
427 for (auto &T : BundlerConfig.TargetNames) {
428 HeaderSize += 3 * 8; // Bundle offset, Size of bundle and size of triple.
429 HeaderSize += T.size(); // The triple.
430 }
431
432 // Write to the buffer the header.
434
435 Write8byteIntegerToBuffer(OS, BundlerConfig.TargetNames.size());
436
437 unsigned Idx = 0;
438 for (auto &T : BundlerConfig.TargetNames) {
439 MemoryBuffer &MB = *Inputs[Idx++];
440 HeaderSize = alignTo(HeaderSize, BundlerConfig.BundleAlignment);
441 // Bundle offset.
442 Write8byteIntegerToBuffer(OS, HeaderSize);
443 // Size of the bundle (adds to the next bundle's offset)
444 Write8byteIntegerToBuffer(OS, MB.getBufferSize());
445 BundlesInfo[T] = BinaryBundleInfo(MB.getBufferSize(), HeaderSize);
446 HeaderSize += MB.getBufferSize();
447 // Size of the triple
448 Write8byteIntegerToBuffer(OS, T.size());
449 // Triple
450 OS << T;
451 }
452 return Error::success();
453 }
454
455 Error WriteBundleStart(raw_ostream &OS, StringRef TargetTriple) final {
456 CurWriteBundleTarget = TargetTriple.str();
457 return Error::success();
458 }
459
460 Error WriteBundleEnd(raw_ostream &OS, StringRef TargetTriple) final {
461 return Error::success();
462 }
463
464 Error WriteBundle(raw_ostream &OS, MemoryBuffer &Input) final {
465 auto BI = BundlesInfo[CurWriteBundleTarget];
466
467 // Pad with 0 to reach specified offset.
468 size_t CurrentPos = OS.tell();
469 size_t PaddingSize = BI.Offset > CurrentPos ? BI.Offset - CurrentPos : 0;
470 for (size_t I = 0; I < PaddingSize; ++I)
471 OS.write('\0');
472 assert(OS.tell() == BI.Offset);
473
474 OS.write(Input.getBufferStart(), Input.getBufferSize());
475
476 return Error::success();
477 }
478};
479
480// This class implements a list of temporary files that are removed upon
481// object destruction.
482class TempFileHandlerRAII {
483public:
484 ~TempFileHandlerRAII() {
485 for (const auto &File : Files)
486 sys::fs::remove(File);
487 }
488
489 // Creates temporary file with given contents.
490 Expected<StringRef> Create(std::optional<ArrayRef<char>> Contents) {
492 if (std::error_code EC =
493 sys::fs::createTemporaryFile("clang-offload-bundler", "tmp", File))
494 return createFileError(File, EC);
495 Files.push_front(File);
496
497 if (Contents) {
498 std::error_code EC;
499 raw_fd_ostream OS(File, EC);
500 if (EC)
501 return createFileError(File, EC);
502 OS.write(Contents->data(), Contents->size());
503 }
504 return Files.front().str();
505 }
506
507private:
508 std::forward_list<SmallString<128u>> Files;
509};
510
511/// Handler for object files. The bundles are organized by sections with a
512/// designated name.
513///
514/// To unbundle, we just copy the contents of the designated section.
515class ObjectFileHandler final : public FileHandler {
516
517 /// The object file we are currently dealing with.
518 std::unique_ptr<ObjectFile> Obj;
519
520 /// Return the input file contents.
521 StringRef getInputFileContents() const { return Obj->getData(); }
522
523 /// Return bundle name (<kind>-<triple>) if the provided section is an offload
524 /// section.
526 IsOffloadSection(SectionRef CurSection) {
527 Expected<StringRef> NameOrErr = CurSection.getName();
528 if (!NameOrErr)
529 return NameOrErr.takeError();
530
531 // If it does not start with the reserved suffix, just skip this section.
532 if (llvm::identify_magic(*NameOrErr) != llvm::file_magic::offload_bundle)
533 return std::nullopt;
534
535 // Return the triple that is right after the reserved prefix.
536 return NameOrErr->substr(sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1);
537 }
538
539 /// Total number of inputs.
540 unsigned NumberOfInputs = 0;
541
542 /// Total number of processed inputs, i.e, inputs that were already
543 /// read from the buffers.
544 unsigned NumberOfProcessedInputs = 0;
545
546 /// Iterator of the current and next section.
547 section_iterator CurrentSection;
548 section_iterator NextSection;
549
550 /// Configuration options and arrays for this bundler job
551 const OffloadBundlerConfig &BundlerConfig;
552
553public:
554 // TODO: Add error checking from ClangOffloadBundler.cpp
555 ObjectFileHandler(std::unique_ptr<ObjectFile> ObjIn,
556 const OffloadBundlerConfig &BC)
557 : Obj(std::move(ObjIn)), CurrentSection(Obj->section_begin()),
558 NextSection(Obj->section_begin()), BundlerConfig(BC) {}
559
560 ~ObjectFileHandler() final {}
561
562 Error ReadHeader(MemoryBuffer &Input) final { return Error::success(); }
563
565 ReadBundleStart(MemoryBuffer &Input) final {
566 while (NextSection != Obj->section_end()) {
567 CurrentSection = NextSection;
568 ++NextSection;
569
570 // Check if the current section name starts with the reserved prefix. If
571 // so, return the triple.
573 IsOffloadSection(*CurrentSection);
574 if (!TripleOrErr)
575 return TripleOrErr.takeError();
576 if (*TripleOrErr)
577 return **TripleOrErr;
578 }
579 return std::nullopt;
580 }
581
582 Error ReadBundleEnd(MemoryBuffer &Input) final { return Error::success(); }
583
584 Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
585 Expected<StringRef> ContentOrErr = CurrentSection->getContents();
586 if (!ContentOrErr)
587 return ContentOrErr.takeError();
588 StringRef Content = *ContentOrErr;
589
590 // Copy fat object contents to the output when extracting host bundle.
591 std::string ModifiedContent;
592 if (Content.size() == 1u && Content.front() == 0) {
593 auto HostBundleOrErr = getHostBundle(
594 StringRef(Input.getBufferStart(), Input.getBufferSize()));
595 if (!HostBundleOrErr)
596 return HostBundleOrErr.takeError();
597
598 ModifiedContent = std::move(*HostBundleOrErr);
599 Content = ModifiedContent;
600 }
601
602 OS.write(Content.data(), Content.size());
603 return Error::success();
604 }
605
606 Error WriteHeader(raw_ostream &OS,
607 ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
608 assert(BundlerConfig.HostInputIndex != ~0u &&
609 "Host input index not defined.");
610
611 // Record number of inputs.
612 NumberOfInputs = Inputs.size();
613 return Error::success();
614 }
615
616 Error WriteBundleStart(raw_ostream &OS, StringRef TargetTriple) final {
617 ++NumberOfProcessedInputs;
618 return Error::success();
619 }
620
621 Error WriteBundleEnd(raw_ostream &OS, StringRef TargetTriple) final {
622 return Error::success();
623 }
624
625 Error finalizeOutputFile() final {
626 assert(NumberOfProcessedInputs <= NumberOfInputs &&
627 "Processing more inputs that actually exist!");
628 assert(BundlerConfig.HostInputIndex != ~0u &&
629 "Host input index not defined.");
630
631 // If this is not the last output, we don't have to do anything.
632 if (NumberOfProcessedInputs != NumberOfInputs)
633 return Error::success();
634
635 // We will use llvm-objcopy to add target objects sections to the output
636 // fat object. These sections should have 'exclude' flag set which tells
637 // link editor to remove them from linker inputs when linking executable or
638 // shared library.
639
640 assert(BundlerConfig.ObjcopyPath != "" &&
641 "llvm-objcopy path not specified");
642
643 // Temporary files that need to be removed.
644 TempFileHandlerRAII TempFiles;
645
646 // Compose llvm-objcopy command line for add target objects' sections with
647 // appropriate flags.
648 BumpPtrAllocator Alloc;
649 StringSaver SS{Alloc};
650 SmallVector<StringRef, 8u> ObjcopyArgs{"llvm-objcopy"};
651
652 for (unsigned I = 0; I < NumberOfInputs; ++I) {
653 StringRef InputFile = BundlerConfig.InputFileNames[I];
654 if (I == BundlerConfig.HostInputIndex) {
655 // Special handling for the host bundle. We do not need to add a
656 // standard bundle for the host object since we are going to use fat
657 // object as a host object. Therefore use dummy contents (one zero byte)
658 // when creating section for the host bundle.
659 Expected<StringRef> TempFileOrErr = TempFiles.Create(ArrayRef<char>(0));
660 if (!TempFileOrErr)
661 return TempFileOrErr.takeError();
662 InputFile = *TempFileOrErr;
663 }
664
665 ObjcopyArgs.push_back(
666 SS.save(Twine("--add-section=") + OFFLOAD_BUNDLER_MAGIC_STR +
667 BundlerConfig.TargetNames[I] + "=" + InputFile));
668 ObjcopyArgs.push_back(
669 SS.save(Twine("--set-section-flags=") + OFFLOAD_BUNDLER_MAGIC_STR +
670 BundlerConfig.TargetNames[I] + "=readonly,exclude"));
671 }
672 ObjcopyArgs.push_back("--");
673 ObjcopyArgs.push_back(
674 BundlerConfig.InputFileNames[BundlerConfig.HostInputIndex]);
675 ObjcopyArgs.push_back(BundlerConfig.OutputFileNames.front());
676
677 if (Error Err = executeObjcopy(BundlerConfig.ObjcopyPath, ObjcopyArgs))
678 return Err;
679
680 return Error::success();
681 }
682
683 Error WriteBundle(raw_ostream &OS, MemoryBuffer &Input) final {
684 return Error::success();
685 }
686
687private:
688 Error executeObjcopy(StringRef Objcopy, ArrayRef<StringRef> Args) {
689 // If the user asked for the commands to be printed out, we do that
690 // instead of executing it.
691 if (BundlerConfig.PrintExternalCommands) {
692 errs() << "\"" << Objcopy << "\"";
693 for (StringRef Arg : drop_begin(Args, 1))
694 errs() << " \"" << Arg << "\"";
695 errs() << "\n";
696 } else {
697 if (sys::ExecuteAndWait(Objcopy, Args))
698 return createStringError(inconvertibleErrorCode(),
699 "'llvm-objcopy' tool failed");
700 }
701 return Error::success();
702 }
703
704 Expected<std::string> getHostBundle(StringRef Input) {
705 TempFileHandlerRAII TempFiles;
706
707 auto ModifiedObjPathOrErr = TempFiles.Create(std::nullopt);
708 if (!ModifiedObjPathOrErr)
709 return ModifiedObjPathOrErr.takeError();
710 StringRef ModifiedObjPath = *ModifiedObjPathOrErr;
711
712 BumpPtrAllocator Alloc;
713 StringSaver SS{Alloc};
714 SmallVector<StringRef, 16> ObjcopyArgs{"llvm-objcopy"};
715
716 ObjcopyArgs.push_back("--regex");
717 ObjcopyArgs.push_back("--remove-section=__CLANG_OFFLOAD_BUNDLE__.*");
718 ObjcopyArgs.push_back("--");
719
720 StringRef ObjcopyInputFileName;
721 // When unbundling an archive, the content of each object file in the
722 // archive is passed to this function by parameter Input, which is different
723 // from the content of the original input archive file, therefore it needs
724 // to be saved to a temporary file before passed to llvm-objcopy. Otherwise,
725 // Input is the same as the content of the original input file, therefore
726 // temporary file is not needed.
727 if (StringRef(BundlerConfig.FilesType).starts_with("a")) {
728 auto InputFileOrErr =
729 TempFiles.Create(ArrayRef<char>(Input.data(), Input.size()));
730 if (!InputFileOrErr)
731 return InputFileOrErr.takeError();
732 ObjcopyInputFileName = *InputFileOrErr;
733 } else
734 ObjcopyInputFileName = BundlerConfig.InputFileNames.front();
735
736 ObjcopyArgs.push_back(ObjcopyInputFileName);
737 ObjcopyArgs.push_back(ModifiedObjPath);
738
739 if (Error Err = executeObjcopy(BundlerConfig.ObjcopyPath, ObjcopyArgs))
740 return std::move(Err);
741
742 auto BufOrErr = MemoryBuffer::getFile(ModifiedObjPath);
743 if (!BufOrErr)
744 return createStringError(BufOrErr.getError(),
745 "Failed to read back the modified object file");
746
747 return BufOrErr->get()->getBuffer().str();
748 }
749};
750
751/// Handler for text files. The bundled file will have the following format.
752///
753/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
754/// Bundle 1
755/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
756/// ...
757/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
758/// Bundle N
759/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
760class TextFileHandler final : public FileHandler {
761 /// String that begins a line comment.
762 StringRef Comment;
763
764 /// String that initiates a bundle.
765 std::string BundleStartString;
766
767 /// String that closes a bundle.
768 std::string BundleEndString;
769
770 /// Number of chars read from input.
771 size_t ReadChars = 0u;
772
773protected:
774 Error ReadHeader(MemoryBuffer &Input) final { return Error::success(); }
775
777 ReadBundleStart(MemoryBuffer &Input) final {
778 StringRef FC = Input.getBuffer();
779
780 // Find start of the bundle.
781 ReadChars = FC.find(BundleStartString, ReadChars);
782 if (ReadChars == FC.npos)
783 return std::nullopt;
784
785 // Get position of the triple.
786 size_t TripleStart = ReadChars = ReadChars + BundleStartString.size();
787
788 // Get position that closes the triple.
789 size_t TripleEnd = ReadChars = FC.find("\n", ReadChars);
790 if (TripleEnd == FC.npos)
791 return std::nullopt;
792
793 // Next time we read after the new line.
794 ++ReadChars;
795
796 return StringRef(&FC.data()[TripleStart], TripleEnd - TripleStart);
797 }
798
799 Error ReadBundleEnd(MemoryBuffer &Input) final {
800 StringRef FC = Input.getBuffer();
801
802 // Read up to the next new line.
803 assert(FC[ReadChars] == '\n' && "The bundle should end with a new line.");
804
805 size_t TripleEnd = ReadChars = FC.find("\n", ReadChars + 1);
806 if (TripleEnd != FC.npos)
807 // Next time we read after the new line.
808 ++ReadChars;
809
810 return Error::success();
811 }
812
813 Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
814 StringRef FC = Input.getBuffer();
815 size_t BundleStart = ReadChars;
816
817 // Find end of the bundle.
818 size_t BundleEnd = ReadChars = FC.find(BundleEndString, ReadChars);
819
820 StringRef Bundle(&FC.data()[BundleStart], BundleEnd - BundleStart);
821 OS << Bundle;
822
823 return Error::success();
824 }
825
826 Error WriteHeader(raw_ostream &OS,
827 ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
828 return Error::success();
829 }
830
831 Error WriteBundleStart(raw_ostream &OS, StringRef TargetTriple) final {
832 OS << BundleStartString << TargetTriple << "\n";
833 return Error::success();
834 }
835
836 Error WriteBundleEnd(raw_ostream &OS, StringRef TargetTriple) final {
837 OS << BundleEndString << TargetTriple << "\n";
838 return Error::success();
839 }
840
841 Error WriteBundle(raw_ostream &OS, MemoryBuffer &Input) final {
842 OS << Input.getBuffer();
843 return Error::success();
844 }
845
846public:
847 TextFileHandler(StringRef Comment) : Comment(Comment), ReadChars(0) {
848 BundleStartString =
849 "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__START__ ";
850 BundleEndString =
851 "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__END__ ";
852 }
853
854 Error listBundleIDsCallback(MemoryBuffer &Input,
855 const BundleInfo &Info) final {
856 // TODO: To list bundle IDs in a bundled text file we need to go through
857 // all bundles. The format of bundled text file may need to include a
858 // header if the performance of listing bundle IDs of bundled text file is
859 // important.
860 ReadChars = Input.getBuffer().find(BundleEndString, ReadChars);
861 if (Error Err = ReadBundleEnd(Input))
862 return Err;
863 return Error::success();
864 }
865};
866} // namespace
867
868/// Return an appropriate object file handler. We use the specific object
869/// handler if we know how to deal with that format, otherwise we use a default
870/// binary file handler.
871static std::unique_ptr<FileHandler>
872CreateObjectFileHandler(MemoryBuffer &FirstInput,
873 const OffloadBundlerConfig &BundlerConfig) {
874 // Check if the input file format is one that we know how to deal with.
875 Expected<std::unique_ptr<Binary>> BinaryOrErr = createBinary(FirstInput);
876
877 // We only support regular object files. If failed to open the input as a
878 // known binary or this is not an object file use the default binary handler.
879 if (errorToBool(BinaryOrErr.takeError()) || !isa<ObjectFile>(*BinaryOrErr))
880 return std::make_unique<BinaryFileHandler>(BundlerConfig);
881
882 // Otherwise create an object file handler. The handler will be owned by the
883 // client of this function.
884 return std::make_unique<ObjectFileHandler>(
885 std::unique_ptr<ObjectFile>(cast<ObjectFile>(BinaryOrErr->release())),
886 BundlerConfig);
887}
888
889/// Return an appropriate handler given the input files and options.
891CreateFileHandler(MemoryBuffer &FirstInput,
892 const OffloadBundlerConfig &BundlerConfig) {
893 std::string FilesType = BundlerConfig.FilesType;
894
895 if (FilesType == "i")
896 return std::make_unique<TextFileHandler>(/*Comment=*/"//");
897 if (FilesType == "ii")
898 return std::make_unique<TextFileHandler>(/*Comment=*/"//");
899 if (FilesType == "cui")
900 return std::make_unique<TextFileHandler>(/*Comment=*/"//");
901 if (FilesType == "hipi")
902 return std::make_unique<TextFileHandler>(/*Comment=*/"//");
903 // TODO: `.d` should be eventually removed once `-M` and its variants are
904 // handled properly in offload compilation.
905 if (FilesType == "d")
906 return std::make_unique<TextFileHandler>(/*Comment=*/"#");
907 if (FilesType == "ll")
908 return std::make_unique<TextFileHandler>(/*Comment=*/";");
909 if (FilesType == "bc")
910 return std::make_unique<BinaryFileHandler>(BundlerConfig);
911 if (FilesType == "s")
912 return std::make_unique<TextFileHandler>(/*Comment=*/"#");
913 if (FilesType == "o")
914 return CreateObjectFileHandler(FirstInput, BundlerConfig);
915 if (FilesType == "a")
916 return CreateObjectFileHandler(FirstInput, BundlerConfig);
917 if (FilesType == "gch")
918 return std::make_unique<BinaryFileHandler>(BundlerConfig);
919 if (FilesType == "ast")
920 return std::make_unique<BinaryFileHandler>(BundlerConfig);
921
922 return createStringError(errc::invalid_argument,
923 "'" + FilesType + "': invalid file type specified");
924}
925
927 if (llvm::compression::zstd::isAvailable()) {
928 CompressionFormat = llvm::compression::Format::Zstd;
929 // Compression level 3 is usually sufficient for zstd since long distance
930 // matching is enabled.
932 } else if (llvm::compression::zlib::isAvailable()) {
933 CompressionFormat = llvm::compression::Format::Zlib;
934 // Use default level for zlib since higher level does not have significant
935 // improvement.
936 CompressionLevel = llvm::compression::zlib::DefaultCompression;
937 }
938 auto IgnoreEnvVarOpt =
939 llvm::sys::Process::GetEnv("OFFLOAD_BUNDLER_IGNORE_ENV_VAR");
940 if (IgnoreEnvVarOpt.has_value() && IgnoreEnvVarOpt.value() == "1")
941 return;
942
943 auto VerboseEnvVarOpt = llvm::sys::Process::GetEnv("OFFLOAD_BUNDLER_VERBOSE");
944 if (VerboseEnvVarOpt.has_value())
945 Verbose = VerboseEnvVarOpt.value() == "1";
946
947 auto CompressEnvVarOpt =
948 llvm::sys::Process::GetEnv("OFFLOAD_BUNDLER_COMPRESS");
949 if (CompressEnvVarOpt.has_value())
950 Compress = CompressEnvVarOpt.value() == "1";
951
952 auto CompressionLevelEnvVarOpt =
953 llvm::sys::Process::GetEnv("OFFLOAD_BUNDLER_COMPRESSION_LEVEL");
954 if (CompressionLevelEnvVarOpt.has_value()) {
955 llvm::StringRef CompressionLevelStr = CompressionLevelEnvVarOpt.value();
956 int Level;
957 if (!CompressionLevelStr.getAsInteger(10, Level))
958 CompressionLevel = Level;
959 else
960 llvm::errs()
961 << "Warning: Invalid value for OFFLOAD_BUNDLER_COMPRESSION_LEVEL: "
962 << CompressionLevelStr.str() << ". Ignoring it.\n";
963 }
964}
965
966// Utility function to format numbers with commas
967static std::string formatWithCommas(unsigned long long Value) {
968 std::string Num = std::to_string(Value);
969 int InsertPosition = Num.length() - 3;
970 while (InsertPosition > 0) {
971 Num.insert(InsertPosition, ",");
972 InsertPosition -= 3;
973 }
974 return Num;
975}
976
978CompressedOffloadBundle::compress(llvm::compression::Params P,
979 const llvm::MemoryBuffer &Input,
980 bool Verbose) {
981 if (!llvm::compression::zstd::isAvailable() &&
982 !llvm::compression::zlib::isAvailable())
983 return createStringError(llvm::inconvertibleErrorCode(),
984 "Compression not supported");
985
986 llvm::Timer HashTimer("Hash Calculation Timer", "Hash calculation time",
988 if (Verbose)
989 HashTimer.startTimer();
990 llvm::MD5 Hash;
991 llvm::MD5::MD5Result Result;
992 Hash.update(Input.getBuffer());
993 Hash.final(Result);
994 uint64_t TruncatedHash = Result.low();
995 if (Verbose)
996 HashTimer.stopTimer();
997
998 SmallVector<uint8_t, 0> CompressedBuffer;
999 auto BufferUint8 = llvm::ArrayRef<uint8_t>(
1000 reinterpret_cast<const uint8_t *>(Input.getBuffer().data()),
1001 Input.getBuffer().size());
1002
1003 llvm::Timer CompressTimer("Compression Timer", "Compression time",
1005 if (Verbose)
1006 CompressTimer.startTimer();
1007 llvm::compression::compress(P, BufferUint8, CompressedBuffer);
1008 if (Verbose)
1009 CompressTimer.stopTimer();
1010
1011 uint16_t CompressionMethod = static_cast<uint16_t>(P.format);
1012 uint32_t UncompressedSize = Input.getBuffer().size();
1013 uint32_t TotalFileSize = MagicNumber.size() + sizeof(TotalFileSize) +
1014 sizeof(Version) + sizeof(CompressionMethod) +
1015 sizeof(UncompressedSize) + sizeof(TruncatedHash) +
1016 CompressedBuffer.size();
1017
1018 SmallVector<char, 0> FinalBuffer;
1019 llvm::raw_svector_ostream OS(FinalBuffer);
1020 OS << MagicNumber;
1021 OS.write(reinterpret_cast<const char *>(&Version), sizeof(Version));
1022 OS.write(reinterpret_cast<const char *>(&CompressionMethod),
1023 sizeof(CompressionMethod));
1024 OS.write(reinterpret_cast<const char *>(&TotalFileSize),
1025 sizeof(TotalFileSize));
1026 OS.write(reinterpret_cast<const char *>(&UncompressedSize),
1027 sizeof(UncompressedSize));
1028 OS.write(reinterpret_cast<const char *>(&TruncatedHash),
1029 sizeof(TruncatedHash));
1030 OS.write(reinterpret_cast<const char *>(CompressedBuffer.data()),
1031 CompressedBuffer.size());
1032
1033 if (Verbose) {
1034 auto MethodUsed =
1035 P.format == llvm::compression::Format::Zstd ? "zstd" : "zlib";
1036 double CompressionRate =
1037 static_cast<double>(UncompressedSize) / CompressedBuffer.size();
1038 double CompressionTimeSeconds = CompressTimer.getTotalTime().getWallTime();
1039 double CompressionSpeedMBs =
1040 (UncompressedSize / (1024.0 * 1024.0)) / CompressionTimeSeconds;
1041
1042 llvm::errs() << "Compressed bundle format version: " << Version << "\n"
1043 << "Total file size (including headers): "
1044 << formatWithCommas(TotalFileSize) << " bytes\n"
1045 << "Compression method used: " << MethodUsed << "\n"
1046 << "Compression level: " << P.level << "\n"
1047 << "Binary size before compression: "
1048 << formatWithCommas(UncompressedSize) << " bytes\n"
1049 << "Binary size after compression: "
1050 << formatWithCommas(CompressedBuffer.size()) << " bytes\n"
1051 << "Compression rate: "
1052 << llvm::format("%.2lf", CompressionRate) << "\n"
1053 << "Compression ratio: "
1054 << llvm::format("%.2lf%%", 100.0 / CompressionRate) << "\n"
1055 << "Compression speed: "
1056 << llvm::format("%.2lf MB/s", CompressionSpeedMBs) << "\n"
1057 << "Truncated MD5 hash: "
1058 << llvm::format_hex(TruncatedHash, 16) << "\n";
1059 }
1060 return llvm::MemoryBuffer::getMemBufferCopy(
1061 llvm::StringRef(FinalBuffer.data(), FinalBuffer.size()));
1062}
1063
1065CompressedOffloadBundle::decompress(const llvm::MemoryBuffer &Input,
1066 bool Verbose) {
1067
1068 StringRef Blob = Input.getBuffer();
1069
1070 if (Blob.size() < V1HeaderSize)
1071 return llvm::MemoryBuffer::getMemBufferCopy(Blob);
1072
1073 if (llvm::identify_magic(Blob) !=
1074 llvm::file_magic::offload_bundle_compressed) {
1075 if (Verbose)
1076 llvm::errs() << "Uncompressed bundle.\n";
1077 return llvm::MemoryBuffer::getMemBufferCopy(Blob);
1078 }
1079
1080 size_t CurrentOffset = MagicSize;
1081
1082 uint16_t ThisVersion;
1083 memcpy(&ThisVersion, Blob.data() + CurrentOffset, sizeof(uint16_t));
1084 CurrentOffset += VersionFieldSize;
1085
1086 uint16_t CompressionMethod;
1087 memcpy(&CompressionMethod, Blob.data() + CurrentOffset, sizeof(uint16_t));
1088 CurrentOffset += MethodFieldSize;
1089
1090 uint32_t TotalFileSize;
1091 if (ThisVersion >= 2) {
1092 if (Blob.size() < V2HeaderSize)
1093 return createStringError(inconvertibleErrorCode(),
1094 "Compressed bundle header size too small");
1095 memcpy(&TotalFileSize, Blob.data() + CurrentOffset, sizeof(uint32_t));
1096 CurrentOffset += FileSizeFieldSize;
1097 }
1098
1099 uint32_t UncompressedSize;
1100 memcpy(&UncompressedSize, Blob.data() + CurrentOffset, sizeof(uint32_t));
1101 CurrentOffset += UncompressedSizeFieldSize;
1102
1103 uint64_t StoredHash;
1104 memcpy(&StoredHash, Blob.data() + CurrentOffset, sizeof(uint64_t));
1105 CurrentOffset += HashFieldSize;
1106
1107 llvm::compression::Format CompressionFormat;
1108 if (CompressionMethod ==
1109 static_cast<uint16_t>(llvm::compression::Format::Zlib))
1110 CompressionFormat = llvm::compression::Format::Zlib;
1111 else if (CompressionMethod ==
1112 static_cast<uint16_t>(llvm::compression::Format::Zstd))
1113 CompressionFormat = llvm::compression::Format::Zstd;
1114 else
1115 return createStringError(inconvertibleErrorCode(),
1116 "Unknown compressing method");
1117
1118 llvm::Timer DecompressTimer("Decompression Timer", "Decompression time",
1120 if (Verbose)
1121 DecompressTimer.startTimer();
1122
1123 SmallVector<uint8_t, 0> DecompressedData;
1124 StringRef CompressedData = Blob.substr(CurrentOffset);
1125 if (llvm::Error DecompressionError = llvm::compression::decompress(
1126 CompressionFormat, llvm::arrayRefFromStringRef(CompressedData),
1127 DecompressedData, UncompressedSize))
1128 return createStringError(inconvertibleErrorCode(),
1129 "Could not decompress embedded file contents: " +
1130 llvm::toString(std::move(DecompressionError)));
1131
1132 if (Verbose) {
1133 DecompressTimer.stopTimer();
1134
1135 double DecompressionTimeSeconds =
1136 DecompressTimer.getTotalTime().getWallTime();
1137
1138 // Recalculate MD5 hash for integrity check
1139 llvm::Timer HashRecalcTimer("Hash Recalculation Timer",
1140 "Hash recalculation time",
1142 HashRecalcTimer.startTimer();
1143 llvm::MD5 Hash;
1144 llvm::MD5::MD5Result Result;
1145 Hash.update(llvm::ArrayRef<uint8_t>(DecompressedData.data(),
1146 DecompressedData.size()));
1147 Hash.final(Result);
1148 uint64_t RecalculatedHash = Result.low();
1149 HashRecalcTimer.stopTimer();
1150 bool HashMatch = (StoredHash == RecalculatedHash);
1151
1152 double CompressionRate =
1153 static_cast<double>(UncompressedSize) / CompressedData.size();
1154 double DecompressionSpeedMBs =
1155 (UncompressedSize / (1024.0 * 1024.0)) / DecompressionTimeSeconds;
1156
1157 llvm::errs() << "Compressed bundle format version: " << ThisVersion << "\n";
1158 if (ThisVersion >= 2)
1159 llvm::errs() << "Total file size (from header): "
1160 << formatWithCommas(TotalFileSize) << " bytes\n";
1161 llvm::errs() << "Decompression method: "
1162 << (CompressionFormat == llvm::compression::Format::Zlib
1163 ? "zlib"
1164 : "zstd")
1165 << "\n"
1166 << "Size before decompression: "
1167 << formatWithCommas(CompressedData.size()) << " bytes\n"
1168 << "Size after decompression: "
1169 << formatWithCommas(UncompressedSize) << " bytes\n"
1170 << "Compression rate: "
1171 << llvm::format("%.2lf", CompressionRate) << "\n"
1172 << "Compression ratio: "
1173 << llvm::format("%.2lf%%", 100.0 / CompressionRate) << "\n"
1174 << "Decompression speed: "
1175 << llvm::format("%.2lf MB/s", DecompressionSpeedMBs) << "\n"
1176 << "Stored hash: " << llvm::format_hex(StoredHash, 16) << "\n"
1177 << "Recalculated hash: "
1178 << llvm::format_hex(RecalculatedHash, 16) << "\n"
1179 << "Hashes match: " << (HashMatch ? "Yes" : "No") << "\n";
1180 }
1181
1182 return llvm::MemoryBuffer::getMemBufferCopy(
1183 llvm::toStringRef(DecompressedData));
1184}
1185
1186// List bundle IDs. Return true if an error was found.
1188 StringRef InputFileName, const OffloadBundlerConfig &BundlerConfig) {
1189 // Open Input file.
1190 ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
1191 MemoryBuffer::getFileOrSTDIN(InputFileName);
1192 if (std::error_code EC = CodeOrErr.getError())
1193 return createFileError(InputFileName, EC);
1194
1195 // Decompress the input if necessary.
1196 Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr =
1198 if (!DecompressedBufferOrErr)
1199 return createStringError(
1200 inconvertibleErrorCode(),
1201 "Failed to decompress input: " +
1202 llvm::toString(DecompressedBufferOrErr.takeError()));
1203
1204 MemoryBuffer &DecompressedInput = **DecompressedBufferOrErr;
1205
1206 // Select the right files handler.
1207 Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
1208 CreateFileHandler(DecompressedInput, BundlerConfig);
1209 if (!FileHandlerOrErr)
1210 return FileHandlerOrErr.takeError();
1211
1212 std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr;
1213 assert(FH);
1214 return FH->listBundleIDs(DecompressedInput);
1215}
1216
1217/// @brief Checks if a code object \p CodeObjectInfo is compatible with a given
1218/// target \p TargetInfo.
1219/// @link https://clang.llvm.org/docs/ClangOffloadBundler.html#bundle-entry-id
1222
1223 // Compatible in case of exact match.
1224 if (CodeObjectInfo == TargetInfo) {
1225 DEBUG_WITH_TYPE("CodeObjectCompatibility",
1226 dbgs() << "Compatible: Exact match: \t[CodeObject: "
1227 << CodeObjectInfo.str()
1228 << "]\t:\t[Target: " << TargetInfo.str() << "]\n");
1229 return true;
1230 }
1231
1232 // Incompatible if Kinds or Triples mismatch.
1233 if (!CodeObjectInfo.isOffloadKindCompatible(TargetInfo.OffloadKind) ||
1234 !CodeObjectInfo.Triple.isCompatibleWith(TargetInfo.Triple)) {
1235 DEBUG_WITH_TYPE(
1236 "CodeObjectCompatibility",
1237 dbgs() << "Incompatible: Kind/Triple mismatch \t[CodeObject: "
1238 << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str()
1239 << "]\n");
1240 return false;
1241 }
1242
1243 // Incompatible if Processors mismatch.
1244 llvm::StringMap<bool> CodeObjectFeatureMap, TargetFeatureMap;
1245 std::optional<StringRef> CodeObjectProc = clang::parseTargetID(
1246 CodeObjectInfo.Triple, CodeObjectInfo.TargetID, &CodeObjectFeatureMap);
1247 std::optional<StringRef> TargetProc = clang::parseTargetID(
1248 TargetInfo.Triple, TargetInfo.TargetID, &TargetFeatureMap);
1249
1250 // Both TargetProc and CodeObjectProc can't be empty here.
1251 if (!TargetProc || !CodeObjectProc ||
1252 CodeObjectProc.value() != TargetProc.value()) {
1253 DEBUG_WITH_TYPE("CodeObjectCompatibility",
1254 dbgs() << "Incompatible: Processor mismatch \t[CodeObject: "
1255 << CodeObjectInfo.str()
1256 << "]\t:\t[Target: " << TargetInfo.str() << "]\n");
1257 return false;
1258 }
1259
1260 // Incompatible if CodeObject has more features than Target, irrespective of
1261 // type or sign of features.
1262 if (CodeObjectFeatureMap.getNumItems() > TargetFeatureMap.getNumItems()) {
1263 DEBUG_WITH_TYPE("CodeObjectCompatibility",
1264 dbgs() << "Incompatible: CodeObject has more features "
1265 "than target \t[CodeObject: "
1266 << CodeObjectInfo.str()
1267 << "]\t:\t[Target: " << TargetInfo.str() << "]\n");
1268 return false;
1269 }
1270
1271 // Compatible if each target feature specified by target is compatible with
1272 // target feature of code object. The target feature is compatible if the
1273 // code object does not specify it (meaning Any), or if it specifies it
1274 // with the same value (meaning On or Off).
1275 for (const auto &CodeObjectFeature : CodeObjectFeatureMap) {
1276 auto TargetFeature = TargetFeatureMap.find(CodeObjectFeature.getKey());
1277 if (TargetFeature == TargetFeatureMap.end()) {
1278 DEBUG_WITH_TYPE(
1279 "CodeObjectCompatibility",
1280 dbgs()
1281 << "Incompatible: Value of CodeObject's non-ANY feature is "
1282 "not matching with Target feature's ANY value \t[CodeObject: "
1283 << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str()
1284 << "]\n");
1285 return false;
1286 } else if (TargetFeature->getValue() != CodeObjectFeature.getValue()) {
1287 DEBUG_WITH_TYPE(
1288 "CodeObjectCompatibility",
1289 dbgs() << "Incompatible: Value of CodeObject's non-ANY feature is "
1290 "not matching with Target feature's non-ANY value "
1291 "\t[CodeObject: "
1292 << CodeObjectInfo.str()
1293 << "]\t:\t[Target: " << TargetInfo.str() << "]\n");
1294 return false;
1295 }
1296 }
1297
1298 // CodeObject is compatible if all features of Target are:
1299 // - either, present in the Code Object's features map with the same sign,
1300 // - or, the feature is missing from CodeObjects's features map i.e. it is
1301 // set to ANY
1302 DEBUG_WITH_TYPE(
1303 "CodeObjectCompatibility",
1304 dbgs() << "Compatible: Target IDs are compatible \t[CodeObject: "
1305 << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str()
1306 << "]\n");
1307 return true;
1308}
1309
1310/// Bundle the files. Return true if an error was found.
1312 std::error_code EC;
1313
1314 // Create a buffer to hold the content before compressing.
1315 SmallVector<char, 0> Buffer;
1316 llvm::raw_svector_ostream BufferStream(Buffer);
1317
1318 // Open input files.
1320 InputBuffers.reserve(BundlerConfig.InputFileNames.size());
1321 for (auto &I : BundlerConfig.InputFileNames) {
1322 ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
1323 MemoryBuffer::getFileOrSTDIN(I);
1324 if (std::error_code EC = CodeOrErr.getError())
1325 return createFileError(I, EC);
1326 InputBuffers.emplace_back(std::move(*CodeOrErr));
1327 }
1328
1329 // Get the file handler. We use the host buffer as reference.
1331 "Host input index undefined??");
1333 *InputBuffers[BundlerConfig.AllowNoHost ? 0
1336 if (!FileHandlerOrErr)
1337 return FileHandlerOrErr.takeError();
1338
1339 std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr;
1340 assert(FH);
1341
1342 // Write header.
1343 if (Error Err = FH->WriteHeader(BufferStream, InputBuffers))
1344 return Err;
1345
1346 // Write all bundles along with the start/end markers. If an error was found
1347 // writing the end of the bundle component, abort the bundle writing.
1348 auto Input = InputBuffers.begin();
1349 for (auto &Triple : BundlerConfig.TargetNames) {
1350 if (Error Err = FH->WriteBundleStart(BufferStream, Triple))
1351 return Err;
1352 if (Error Err = FH->WriteBundle(BufferStream, **Input))
1353 return Err;
1354 if (Error Err = FH->WriteBundleEnd(BufferStream, Triple))
1355 return Err;
1356 ++Input;
1357 }
1358
1359 raw_fd_ostream OutputFile(BundlerConfig.OutputFileNames.front(), EC,
1360 sys::fs::OF_None);
1361 if (EC)
1362 return createFileError(BundlerConfig.OutputFileNames.front(), EC);
1363
1364 SmallVector<char, 0> CompressedBuffer;
1365 if (BundlerConfig.Compress) {
1366 std::unique_ptr<llvm::MemoryBuffer> BufferMemory =
1367 llvm::MemoryBuffer::getMemBufferCopy(
1368 llvm::StringRef(Buffer.data(), Buffer.size()));
1369 auto CompressionResult = CompressedOffloadBundle::compress(
1371 /*zstdEnableLdm=*/true},
1372 *BufferMemory, BundlerConfig.Verbose);
1373 if (auto Error = CompressionResult.takeError())
1374 return Error;
1375
1376 auto CompressedMemBuffer = std::move(CompressionResult.get());
1377 CompressedBuffer.assign(CompressedMemBuffer->getBufferStart(),
1378 CompressedMemBuffer->getBufferEnd());
1379 } else
1380 CompressedBuffer = Buffer;
1381
1382 OutputFile.write(CompressedBuffer.data(), CompressedBuffer.size());
1383
1384 return FH->finalizeOutputFile();
1385}
1386
1387// Unbundle the files. Return true if an error was found.
1389 // Open Input file.
1390 ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
1391 MemoryBuffer::getFileOrSTDIN(BundlerConfig.InputFileNames.front());
1392 if (std::error_code EC = CodeOrErr.getError())
1393 return createFileError(BundlerConfig.InputFileNames.front(), EC);
1394
1395 // Decompress the input if necessary.
1396 Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr =
1398 if (!DecompressedBufferOrErr)
1399 return createStringError(
1400 inconvertibleErrorCode(),
1401 "Failed to decompress input: " +
1402 llvm::toString(DecompressedBufferOrErr.takeError()));
1403
1404 MemoryBuffer &Input = **DecompressedBufferOrErr;
1405
1406 // Select the right files handler.
1407 Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
1409 if (!FileHandlerOrErr)
1410 return FileHandlerOrErr.takeError();
1411
1412 std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr;
1413 assert(FH);
1414
1415 // Read the header of the bundled file.
1416 if (Error Err = FH->ReadHeader(Input))
1417 return Err;
1418
1419 // Create a work list that consist of the map triple/output file.
1420 StringMap<StringRef> Worklist;
1421 auto Output = BundlerConfig.OutputFileNames.begin();
1422 for (auto &Triple : BundlerConfig.TargetNames) {
1423 Worklist[Triple] = *Output;
1424 ++Output;
1425 }
1426
1427 // Read all the bundles that are in the work list. If we find no bundles we
1428 // assume the file is meant for the host target.
1429 bool FoundHostBundle = false;
1430 while (!Worklist.empty()) {
1431 Expected<std::optional<StringRef>> CurTripleOrErr =
1432 FH->ReadBundleStart(Input);
1433 if (!CurTripleOrErr)
1434 return CurTripleOrErr.takeError();
1435
1436 // We don't have more bundles.
1437 if (!*CurTripleOrErr)
1438 break;
1439
1440 StringRef CurTriple = **CurTripleOrErr;
1441 assert(!CurTriple.empty());
1442
1443 auto Output = Worklist.begin();
1444 for (auto E = Worklist.end(); Output != E; Output++) {
1446 OffloadTargetInfo(CurTriple, BundlerConfig),
1447 OffloadTargetInfo((*Output).first(), BundlerConfig))) {
1448 break;
1449 }
1450 }
1451
1452 if (Output == Worklist.end())
1453 continue;
1454 // Check if the output file can be opened and copy the bundle to it.
1455 std::error_code EC;
1456 raw_fd_ostream OutputFile((*Output).second, EC, sys::fs::OF_None);
1457 if (EC)
1458 return createFileError((*Output).second, EC);
1459 if (Error Err = FH->ReadBundle(OutputFile, Input))
1460 return Err;
1461 if (Error Err = FH->ReadBundleEnd(Input))
1462 return Err;
1463 Worklist.erase(Output);
1464
1465 // Record if we found the host bundle.
1466 auto OffloadInfo = OffloadTargetInfo(CurTriple, BundlerConfig);
1467 if (OffloadInfo.hasHostKind())
1468 FoundHostBundle = true;
1469 }
1470
1471 if (!BundlerConfig.AllowMissingBundles && !Worklist.empty()) {
1472 std::string ErrMsg = "Can't find bundles for";
1473 std::set<StringRef> Sorted;
1474 for (auto &E : Worklist)
1475 Sorted.insert(E.first());
1476 unsigned I = 0;
1477 unsigned Last = Sorted.size() - 1;
1478 for (auto &E : Sorted) {
1479 if (I != 0 && Last > 1)
1480 ErrMsg += ",";
1481 ErrMsg += " ";
1482 if (I == Last && I != 0)
1483 ErrMsg += "and ";
1484 ErrMsg += E.str();
1485 ++I;
1486 }
1487 return createStringError(inconvertibleErrorCode(), ErrMsg);
1488 }
1489
1490 // If no bundles were found, assume the input file is the host bundle and
1491 // create empty files for the remaining targets.
1492 if (Worklist.size() == BundlerConfig.TargetNames.size()) {
1493 for (auto &E : Worklist) {
1494 std::error_code EC;
1495 raw_fd_ostream OutputFile(E.second, EC, sys::fs::OF_None);
1496 if (EC)
1497 return createFileError(E.second, EC);
1498
1499 // If this entry has a host kind, copy the input file to the output file.
1500 auto OffloadInfo = OffloadTargetInfo(E.getKey(), BundlerConfig);
1501 if (OffloadInfo.hasHostKind())
1502 OutputFile.write(Input.getBufferStart(), Input.getBufferSize());
1503 }
1504 return Error::success();
1505 }
1506
1507 // If we found elements, we emit an error if none of those were for the host
1508 // in case host bundle name was provided in command line.
1509 if (!(FoundHostBundle || BundlerConfig.HostInputIndex == ~0u ||
1511 return createStringError(inconvertibleErrorCode(),
1512 "Can't find bundle for the host target");
1513
1514 // If we still have any elements in the worklist, create empty files for them.
1515 for (auto &E : Worklist) {
1516 std::error_code EC;
1517 raw_fd_ostream OutputFile(E.second, EC, sys::fs::OF_None);
1518 if (EC)
1519 return createFileError(E.second, EC);
1520 }
1521
1522 return Error::success();
1523}
1524
1525static Archive::Kind getDefaultArchiveKindForHost() {
1526 return Triple(sys::getDefaultTargetTriple()).isOSDarwin() ? Archive::K_DARWIN
1527 : Archive::K_GNU;
1528}
1529
1530/// @brief Computes a list of targets among all given targets which are
1531/// compatible with this code object
1532/// @param [in] CodeObjectInfo Code Object
1533/// @param [out] CompatibleTargets List of all compatible targets among all
1534/// given targets
1535/// @return false, if no compatible target is found.
1536static bool
1538 SmallVectorImpl<StringRef> &CompatibleTargets,
1539 const OffloadBundlerConfig &BundlerConfig) {
1540 if (!CompatibleTargets.empty()) {
1541 DEBUG_WITH_TYPE("CodeObjectCompatibility",
1542 dbgs() << "CompatibleTargets list should be empty\n");
1543 return false;
1544 }
1545 for (auto &Target : BundlerConfig.TargetNames) {
1546 auto TargetInfo = OffloadTargetInfo(Target, BundlerConfig);
1547 if (isCodeObjectCompatible(CodeObjectInfo, TargetInfo))
1548 CompatibleTargets.push_back(Target);
1549 }
1550 return !CompatibleTargets.empty();
1551}
1552
1553// Check that each code object file in the input archive conforms to following
1554// rule: for a specific processor, a feature either shows up in all target IDs,
1555// or does not show up in any target IDs. Otherwise the target ID combination is
1556// invalid.
1557static Error
1558CheckHeterogeneousArchive(StringRef ArchiveName,
1559 const OffloadBundlerConfig &BundlerConfig) {
1560 std::vector<std::unique_ptr<MemoryBuffer>> ArchiveBuffers;
1561 ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
1562 MemoryBuffer::getFileOrSTDIN(ArchiveName, true, false);
1563 if (std::error_code EC = BufOrErr.getError())
1564 return createFileError(ArchiveName, EC);
1565
1566 ArchiveBuffers.push_back(std::move(*BufOrErr));
1568 Archive::create(ArchiveBuffers.back()->getMemBufferRef());
1569 if (!LibOrErr)
1570 return LibOrErr.takeError();
1571
1572 auto Archive = std::move(*LibOrErr);
1573
1574 Error ArchiveErr = Error::success();
1575 auto ChildEnd = Archive->child_end();
1576
1577 /// Iterate over all bundled code object files in the input archive.
1578 for (auto ArchiveIter = Archive->child_begin(ArchiveErr);
1579 ArchiveIter != ChildEnd; ++ArchiveIter) {
1580 if (ArchiveErr)
1581 return ArchiveErr;
1582 auto ArchiveChildNameOrErr = (*ArchiveIter).getName();
1583 if (!ArchiveChildNameOrErr)
1584 return ArchiveChildNameOrErr.takeError();
1585
1586 auto CodeObjectBufferRefOrErr = (*ArchiveIter).getMemoryBufferRef();
1587 if (!CodeObjectBufferRefOrErr)
1588 return CodeObjectBufferRefOrErr.takeError();
1589
1590 auto CodeObjectBuffer =
1591 MemoryBuffer::getMemBuffer(*CodeObjectBufferRefOrErr, false);
1592
1593 Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
1594 CreateFileHandler(*CodeObjectBuffer, BundlerConfig);
1595 if (!FileHandlerOrErr)
1596 return FileHandlerOrErr.takeError();
1597
1598 std::unique_ptr<FileHandler> &FileHandler = *FileHandlerOrErr;
1599 assert(FileHandler);
1600
1601 std::set<StringRef> BundleIds;
1602 auto CodeObjectFileError =
1603 FileHandler->getBundleIDs(*CodeObjectBuffer, BundleIds);
1604 if (CodeObjectFileError)
1605 return CodeObjectFileError;
1606
1607 auto &&ConflictingArchs = clang::getConflictTargetIDCombination(BundleIds);
1608 if (ConflictingArchs) {
1609 std::string ErrMsg =
1610 Twine("conflicting TargetIDs [" + ConflictingArchs.value().first +
1611 ", " + ConflictingArchs.value().second + "] found in " +
1612 ArchiveChildNameOrErr.get() + " of " + ArchiveName)
1613 .str();
1614 return createStringError(inconvertibleErrorCode(), ErrMsg);
1615 }
1616 }
1617
1618 return ArchiveErr;
1619}
1620
1621/// UnbundleArchive takes an archive file (".a") as input containing bundled
1622/// code object files, and a list of offload targets (not host), and extracts
1623/// the code objects into a new archive file for each offload target. Each
1624/// resulting archive file contains all code object files corresponding to that
1625/// particular offload target. The created archive file does not
1626/// contain an index of the symbols and code object files are named as
1627/// <<Parent Bundle Name>-<CodeObject's TargetID>>, with ':' replaced with '_'.
1629 std::vector<std::unique_ptr<MemoryBuffer>> ArchiveBuffers;
1630
1631 /// Map of target names with list of object files that will form the device
1632 /// specific archive for that target
1633 StringMap<std::vector<NewArchiveMember>> OutputArchivesMap;
1634
1635 // Map of target names and output archive filenames
1636 StringMap<StringRef> TargetOutputFileNameMap;
1637
1638 auto Output = BundlerConfig.OutputFileNames.begin();
1639 for (auto &Target : BundlerConfig.TargetNames) {
1640 TargetOutputFileNameMap[Target] = *Output;
1641 ++Output;
1642 }
1643
1644 StringRef IFName = BundlerConfig.InputFileNames.front();
1645
1647 // For a specific processor, a feature either shows up in all target IDs, or
1648 // does not show up in any target IDs. Otherwise the target ID combination
1649 // is invalid.
1650 auto ArchiveError = CheckHeterogeneousArchive(IFName, BundlerConfig);
1651 if (ArchiveError) {
1652 return ArchiveError;
1653 }
1654 }
1655
1656 ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
1657 MemoryBuffer::getFileOrSTDIN(IFName, true, false);
1658 if (std::error_code EC = BufOrErr.getError())
1659 return createFileError(BundlerConfig.InputFileNames.front(), EC);
1660
1661 ArchiveBuffers.push_back(std::move(*BufOrErr));
1663 Archive::create(ArchiveBuffers.back()->getMemBufferRef());
1664 if (!LibOrErr)
1665 return LibOrErr.takeError();
1666
1667 auto Archive = std::move(*LibOrErr);
1668
1669 Error ArchiveErr = Error::success();
1670 auto ChildEnd = Archive->child_end();
1671
1672 /// Iterate over all bundled code object files in the input archive.
1673 for (auto ArchiveIter = Archive->child_begin(ArchiveErr);
1674 ArchiveIter != ChildEnd; ++ArchiveIter) {
1675 if (ArchiveErr)
1676 return ArchiveErr;
1677 auto ArchiveChildNameOrErr = (*ArchiveIter).getName();
1678 if (!ArchiveChildNameOrErr)
1679 return ArchiveChildNameOrErr.takeError();
1680
1681 StringRef BundledObjectFile = sys::path::filename(*ArchiveChildNameOrErr);
1682
1683 auto CodeObjectBufferRefOrErr = (*ArchiveIter).getMemoryBufferRef();
1684 if (!CodeObjectBufferRefOrErr)
1685 return CodeObjectBufferRefOrErr.takeError();
1686
1687 auto TempCodeObjectBuffer =
1688 MemoryBuffer::getMemBuffer(*CodeObjectBufferRefOrErr, false);
1689
1690 // Decompress the buffer if necessary.
1691 Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr =
1692 CompressedOffloadBundle::decompress(*TempCodeObjectBuffer,
1694 if (!DecompressedBufferOrErr)
1695 return createStringError(
1696 inconvertibleErrorCode(),
1697 "Failed to decompress code object: " +
1698 llvm::toString(DecompressedBufferOrErr.takeError()));
1699
1700 MemoryBuffer &CodeObjectBuffer = **DecompressedBufferOrErr;
1701
1702 Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
1703 CreateFileHandler(CodeObjectBuffer, BundlerConfig);
1704 if (!FileHandlerOrErr)
1705 return FileHandlerOrErr.takeError();
1706
1707 std::unique_ptr<FileHandler> &FileHandler = *FileHandlerOrErr;
1708 assert(FileHandler &&
1709 "FileHandle creation failed for file in the archive!");
1710
1711 if (Error ReadErr = FileHandler->ReadHeader(CodeObjectBuffer))
1712 return ReadErr;
1713
1714 Expected<std::optional<StringRef>> CurBundleIDOrErr =
1715 FileHandler->ReadBundleStart(CodeObjectBuffer);
1716 if (!CurBundleIDOrErr)
1717 return CurBundleIDOrErr.takeError();
1718
1719 std::optional<StringRef> OptionalCurBundleID = *CurBundleIDOrErr;
1720 // No device code in this child, skip.
1721 if (!OptionalCurBundleID)
1722 continue;
1723 StringRef CodeObject = *OptionalCurBundleID;
1724
1725 // Process all bundle entries (CodeObjects) found in this child of input
1726 // archive.
1727 while (!CodeObject.empty()) {
1728 SmallVector<StringRef> CompatibleTargets;
1729 auto CodeObjectInfo = OffloadTargetInfo(CodeObject, BundlerConfig);
1730 if (getCompatibleOffloadTargets(CodeObjectInfo, CompatibleTargets,
1731 BundlerConfig)) {
1732 std::string BundleData;
1733 raw_string_ostream DataStream(BundleData);
1734 if (Error Err = FileHandler->ReadBundle(DataStream, CodeObjectBuffer))
1735 return Err;
1736
1737 for (auto &CompatibleTarget : CompatibleTargets) {
1738 SmallString<128> BundledObjectFileName;
1739 BundledObjectFileName.assign(BundledObjectFile);
1740 auto OutputBundleName =
1741 Twine(llvm::sys::path::stem(BundledObjectFileName) + "-" +
1742 CodeObject +
1743 getDeviceLibraryFileName(BundledObjectFileName,
1744 CodeObjectInfo.TargetID))
1745 .str();
1746 // Replace ':' in optional target feature list with '_' to ensure
1747 // cross-platform validity.
1748 std::replace(OutputBundleName.begin(), OutputBundleName.end(), ':',
1749 '_');
1750
1751 std::unique_ptr<MemoryBuffer> MemBuf = MemoryBuffer::getMemBufferCopy(
1752 DataStream.str(), OutputBundleName);
1753 ArchiveBuffers.push_back(std::move(MemBuf));
1754 llvm::MemoryBufferRef MemBufRef =
1755 MemoryBufferRef(*(ArchiveBuffers.back()));
1756
1757 // For inserting <CompatibleTarget, list<CodeObject>> entry in
1758 // OutputArchivesMap.
1759 if (!OutputArchivesMap.contains(CompatibleTarget)) {
1760
1761 std::vector<NewArchiveMember> ArchiveMembers;
1762 ArchiveMembers.push_back(NewArchiveMember(MemBufRef));
1763 OutputArchivesMap.insert_or_assign(CompatibleTarget,
1764 std::move(ArchiveMembers));
1765 } else {
1766 OutputArchivesMap[CompatibleTarget].push_back(
1767 NewArchiveMember(MemBufRef));
1768 }
1769 }
1770 }
1771
1772 if (Error Err = FileHandler->ReadBundleEnd(CodeObjectBuffer))
1773 return Err;
1774
1775 Expected<std::optional<StringRef>> NextTripleOrErr =
1776 FileHandler->ReadBundleStart(CodeObjectBuffer);
1777 if (!NextTripleOrErr)
1778 return NextTripleOrErr.takeError();
1779
1780 CodeObject = ((*NextTripleOrErr).has_value()) ? **NextTripleOrErr : "";
1781 } // End of processing of all bundle entries of this child of input archive.
1782 } // End of while over children of input archive.
1783
1784 assert(!ArchiveErr && "Error occurred while reading archive!");
1785
1786 /// Write out an archive for each target
1787 for (auto &Target : BundlerConfig.TargetNames) {
1788 StringRef FileName = TargetOutputFileNameMap[Target];
1789 StringMapIterator<std::vector<llvm::NewArchiveMember>> CurArchiveMembers =
1790 OutputArchivesMap.find(Target);
1791 if (CurArchiveMembers != OutputArchivesMap.end()) {
1792 if (Error WriteErr = writeArchive(FileName, CurArchiveMembers->getValue(),
1793 SymtabWritingMode::NormalSymtab,
1795 false, nullptr))
1796 return WriteErr;
1797 } else if (!BundlerConfig.AllowMissingBundles) {
1798 std::string ErrMsg =
1799 Twine("no compatible code object found for the target '" + Target +
1800 "' in heterogeneous archive library: " + IFName)
1801 .str();
1802 return createStringError(inconvertibleErrorCode(), ErrMsg);
1803 } else { // Create an empty archive file if no compatible code object is
1804 // found and "allow-missing-bundles" is enabled. It ensures that
1805 // the linker using output of this step doesn't complain about
1806 // the missing input file.
1807 std::vector<llvm::NewArchiveMember> EmptyArchive;
1808 EmptyArchive.clear();
1809 if (Error WriteErr = writeArchive(
1810 FileName, EmptyArchive, SymtabWritingMode::NormalSymtab,
1811 getDefaultArchiveKindForHost(), true, false, nullptr))
1812 return WriteErr;
1813 }
1814 }
1815
1816 return Error::success();
1817}
StringRef P
llvm::MachO::Target Target
Definition: MachO.h:48
static std::string getDeviceLibraryFileName(StringRef BundleFileName, StringRef Device)
static StringRef getDeviceFileExtension(StringRef Device, StringRef BundleFileName)
static Expected< std::unique_ptr< FileHandler > > CreateFileHandler(MemoryBuffer &FirstInput, const OffloadBundlerConfig &BundlerConfig)
Return an appropriate handler given the input files and options.
#define OFFLOAD_BUNDLER_MAGIC_STR
Magic string that marks the existence of offloading data.
bool isCodeObjectCompatible(const OffloadTargetInfo &CodeObjectInfo, const OffloadTargetInfo &TargetInfo)
Checks if a code object CodeObjectInfo is compatible with a given target TargetInfo.
static llvm::TimerGroup ClangOffloadBundlerTimerGroup("Clang Offload Bundler Timer Group", "Timer group for clang offload bundler")
static Error CheckHeterogeneousArchive(StringRef ArchiveName, const OffloadBundlerConfig &BundlerConfig)
static std::unique_ptr< FileHandler > CreateObjectFileHandler(MemoryBuffer &FirstInput, const OffloadBundlerConfig &BundlerConfig)
Return an appropriate object file handler.
static Archive::Kind getDefaultArchiveKindForHost()
static std::string formatWithCommas(unsigned long long Value)
static bool getCompatibleOffloadTargets(OffloadTargetInfo &CodeObjectInfo, SmallVectorImpl< StringRef > &CompatibleTargets, const OffloadBundlerConfig &BundlerConfig)
Computes a list of targets among all given targets which are compatible with this code object.
This file defines an offload bundling API that bundles different files that relate with the same sour...
Defines version macros and version-related utility functions for Clang.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
static llvm::Expected< std::unique_ptr< llvm::MemoryBuffer > > compress(llvm::compression::Params P, const llvm::MemoryBuffer &Input, bool Verbose=false)
static llvm::Expected< std::unique_ptr< llvm::MemoryBuffer > > decompress(const llvm::MemoryBuffer &Input, bool Verbose=false)
llvm::compression::Format CompressionFormat
std::vector< std::string > OutputFileNames
std::vector< std::string > TargetNames
std::vector< std::string > InputFileNames
llvm::Error BundleFiles()
Bundle the files. Return true if an error was found.
llvm::Error UnbundleArchive()
UnbundleArchive takes an archive file (".a") as input containing bundled code object files,...
static llvm::Error ListBundleIDsInFile(llvm::StringRef InputFileName, const OffloadBundlerConfig &BundlerConfig)
const OffloadBundlerConfig & BundlerConfig
Exposes information about the current target.
Definition: TargetInfo.h:213
The JSON file list parser is used to communicate input to InstallAPI.
@ Create
'copyin' clause, allowed on Compute and Combined constructs, plus 'data', 'enter data',...
std::optional< llvm::StringRef > parseTargetID(const llvm::Triple &T, llvm::StringRef OffloadArch, llvm::StringMap< bool > *FeatureMap)
Parse a target ID to get processor and feature map.
Definition: TargetID.cpp:105
std::optional< std::pair< llvm::StringRef, llvm::StringRef > > getConflictTargetIDCombination(const std::set< llvm::StringRef > &TargetIDs)
Get the conflicted pair of target IDs for a compilation or a bundled code object, assuming TargetIDs ...
Definition: TargetID.cpp:145
CudaArch StringToCudaArch(llvm::StringRef S)
Definition: Cuda.cpp:168
@ Result
The result type of a method or function.
const FunctionProtoType * T
unsigned long uint64_t
Diagnostic wrappers for TextAPI types for error reporting.
Definition: Dominators.h:30
Definition: Format.h:5394
Obtain the offload kind, real machine triple, and an optional TargetID out of the target information ...
bool operator==(const OffloadTargetInfo &Target) const
bool isOffloadKindCompatible(const llvm::StringRef TargetOffloadKind) const
OffloadTargetInfo(const llvm::StringRef Target, const OffloadBundlerConfig &BC)
llvm::StringRef OffloadKind
std::string str() const
llvm::StringRef TargetID
const OffloadBundlerConfig & BundlerConfig