clang 19.0.0git
OffloadBundler.cpp
Go to the documentation of this file.
1//===- OffloadBundler.cpp - File Bundling and Unbundling ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements an offload bundling API that bundles different files
11/// that relate with the same source code but different targets into a single
12/// one. Also the implements the opposite functionality, i.e. unbundle files
13/// previous created by this API.
14///
15//===----------------------------------------------------------------------===//
16
18#include "clang/Basic/Cuda.h"
20#include "clang/Basic/Version.h"
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringExtras.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringRef.h"
27#include "llvm/BinaryFormat/Magic.h"
28#include "llvm/Object/Archive.h"
29#include "llvm/Object/ArchiveWriter.h"
30#include "llvm/Object/Binary.h"
31#include "llvm/Object/ObjectFile.h"
32#include "llvm/Support/Casting.h"
33#include "llvm/Support/Compression.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/EndianStream.h"
36#include "llvm/Support/Errc.h"
37#include "llvm/Support/Error.h"
38#include "llvm/Support/ErrorOr.h"
39#include "llvm/Support/FileSystem.h"
40#include "llvm/Support/MD5.h"
41#include "llvm/Support/MemoryBuffer.h"
42#include "llvm/Support/Path.h"
43#include "llvm/Support/Program.h"
44#include "llvm/Support/Signals.h"
45#include "llvm/Support/StringSaver.h"
46#include "llvm/Support/Timer.h"
47#include "llvm/Support/WithColor.h"
48#include "llvm/Support/raw_ostream.h"
49#include "llvm/TargetParser/Host.h"
50#include "llvm/TargetParser/Triple.h"
51#include <algorithm>
52#include <cassert>
53#include <cstddef>
54#include <cstdint>
55#include <forward_list>
56#include <llvm/Support/Process.h>
57#include <memory>
58#include <set>
59#include <string>
60#include <system_error>
61#include <utility>
62
63using namespace llvm;
64using namespace llvm::object;
65using namespace clang;
66
67static llvm::TimerGroup
68 ClangOffloadBundlerTimerGroup("Clang Offload Bundler Timer Group",
69 "Timer group for clang offload bundler");
70
71/// Magic string that marks the existence of offloading data.
72#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
73
75 const OffloadBundlerConfig &BC)
76 : BundlerConfig(BC) {
77
78 // TODO: Add error checking from ClangOffloadBundler.cpp
79 auto TargetFeatures = Target.split(':');
80 auto TripleOrGPU = TargetFeatures.first.rsplit('-');
81
82 if (clang::StringToCudaArch(TripleOrGPU.second) != clang::CudaArch::UNKNOWN) {
83 auto KindTriple = TripleOrGPU.first.split('-');
84 this->OffloadKind = KindTriple.first;
85
86 // Enforce optional env field to standardize bundles
87 llvm::Triple t = llvm::Triple(KindTriple.second);
88 this->Triple = llvm::Triple(t.getArchName(), t.getVendorName(),
89 t.getOSName(), t.getEnvironmentName());
90
91 this->TargetID = Target.substr(Target.find(TripleOrGPU.second));
92 } else {
93 auto KindTriple = TargetFeatures.first.split('-');
94 this->OffloadKind = KindTriple.first;
95
96 // Enforce optional env field to standardize bundles
97 llvm::Triple t = llvm::Triple(KindTriple.second);
98 this->Triple = llvm::Triple(t.getArchName(), t.getVendorName(),
99 t.getOSName(), t.getEnvironmentName());
100
101 this->TargetID = "";
102 }
103}
104
106 return this->OffloadKind == "host";
107}
108
110 return OffloadKind == "host" || OffloadKind == "openmp" ||
111 OffloadKind == "hip" || OffloadKind == "hipv4";
112}
113
115 const StringRef TargetOffloadKind) const {
116 if (OffloadKind == TargetOffloadKind)
117 return true;
119 bool HIPCompatibleWithOpenMP = OffloadKind.starts_with_insensitive("hip") &&
120 TargetOffloadKind == "openmp";
121 bool OpenMPCompatibleWithHIP =
122 OffloadKind == "openmp" &&
123 TargetOffloadKind.starts_with_insensitive("hip");
124 return HIPCompatibleWithOpenMP || OpenMPCompatibleWithHIP;
125 }
126 return false;
127}
128
130 return !Triple.str().empty() && Triple.getArch() != Triple::UnknownArch;
131}
132
134 return OffloadKind == Target.OffloadKind &&
135 Triple.isCompatibleWith(Target.Triple) && TargetID == Target.TargetID;
136}
137
138std::string OffloadTargetInfo::str() const {
139 return Twine(OffloadKind + "-" + Triple.str() + "-" + TargetID).str();
140}
141
142static StringRef getDeviceFileExtension(StringRef Device,
143 StringRef BundleFileName) {
144 if (Device.contains("gfx"))
145 return ".bc";
146 if (Device.contains("sm_"))
147 return ".cubin";
148 return sys::path::extension(BundleFileName);
149}
150
151static std::string getDeviceLibraryFileName(StringRef BundleFileName,
152 StringRef Device) {
153 StringRef LibName = sys::path::stem(BundleFileName);
154 StringRef Extension = getDeviceFileExtension(Device, BundleFileName);
155
156 std::string Result;
157 Result += LibName;
158 Result += Extension;
159 return Result;
160}
161
162namespace {
163/// Generic file handler interface.
164class FileHandler {
165public:
166 struct BundleInfo {
167 StringRef BundleID;
168 };
169
170 FileHandler() {}
171
172 virtual ~FileHandler() {}
173
174 /// Update the file handler with information from the header of the bundled
175 /// file.
176 virtual Error ReadHeader(MemoryBuffer &Input) = 0;
177
178 /// Read the marker of the next bundled to be read in the file. The bundle
179 /// name is returned if there is one in the file, or `std::nullopt` if there
180 /// are no more bundles to be read.
182 ReadBundleStart(MemoryBuffer &Input) = 0;
183
184 /// Read the marker that closes the current bundle.
185 virtual Error ReadBundleEnd(MemoryBuffer &Input) = 0;
186
187 /// Read the current bundle and write the result into the stream \a OS.
188 virtual Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) = 0;
189
190 /// Write the header of the bundled file to \a OS based on the information
191 /// gathered from \a Inputs.
192 virtual Error WriteHeader(raw_ostream &OS,
193 ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) = 0;
194
195 /// Write the marker that initiates a bundle for the triple \a TargetTriple to
196 /// \a OS.
197 virtual Error WriteBundleStart(raw_ostream &OS, StringRef TargetTriple) = 0;
198
199 /// Write the marker that closes a bundle for the triple \a TargetTriple to \a
200 /// OS.
201 virtual Error WriteBundleEnd(raw_ostream &OS, StringRef TargetTriple) = 0;
202
203 /// Write the bundle from \a Input into \a OS.
204 virtual Error WriteBundle(raw_ostream &OS, MemoryBuffer &Input) = 0;
205
206 /// Finalize output file.
207 virtual Error finalizeOutputFile() { return Error::success(); }
208
209 /// List bundle IDs in \a Input.
210 virtual Error listBundleIDs(MemoryBuffer &Input) {
211 if (Error Err = ReadHeader(Input))
212 return Err;
213 return forEachBundle(Input, [&](const BundleInfo &Info) -> Error {
214 llvm::outs() << Info.BundleID << '\n';
215 Error Err = listBundleIDsCallback(Input, Info);
216 if (Err)
217 return Err;
218 return Error::success();
219 });
220 }
221
222 /// Get bundle IDs in \a Input in \a BundleIds.
223 virtual Error getBundleIDs(MemoryBuffer &Input,
224 std::set<StringRef> &BundleIds) {
225 if (Error Err = ReadHeader(Input))
226 return Err;
227 return forEachBundle(Input, [&](const BundleInfo &Info) -> Error {
228 BundleIds.insert(Info.BundleID);
229 Error Err = listBundleIDsCallback(Input, Info);
230 if (Err)
231 return Err;
232 return Error::success();
233 });
234 }
235
236 /// For each bundle in \a Input, do \a Func.
237 Error forEachBundle(MemoryBuffer &Input,
238 std::function<Error(const BundleInfo &)> Func) {
239 while (true) {
240 Expected<std::optional<StringRef>> CurTripleOrErr =
241 ReadBundleStart(Input);
242 if (!CurTripleOrErr)
243 return CurTripleOrErr.takeError();
244
245 // No more bundles.
246 if (!*CurTripleOrErr)
247 break;
248
249 StringRef CurTriple = **CurTripleOrErr;
250 assert(!CurTriple.empty());
251
252 BundleInfo Info{CurTriple};
253 if (Error Err = Func(Info))
254 return Err;
255 }
256 return Error::success();
257 }
258
259protected:
260 virtual Error listBundleIDsCallback(MemoryBuffer &Input,
261 const BundleInfo &Info) {
262 return Error::success();
263 }
264};
265
266/// Handler for binary files. The bundled file will have the following format
267/// (all integers are stored in little-endian format):
268///
269/// "OFFLOAD_BUNDLER_MAGIC_STR" (ASCII encoding of the string)
270///
271/// NumberOfOffloadBundles (8-byte integer)
272///
273/// OffsetOfBundle1 (8-byte integer)
274/// SizeOfBundle1 (8-byte integer)
275/// NumberOfBytesInTripleOfBundle1 (8-byte integer)
276/// TripleOfBundle1 (byte length defined before)
277///
278/// ...
279///
280/// OffsetOfBundleN (8-byte integer)
281/// SizeOfBundleN (8-byte integer)
282/// NumberOfBytesInTripleOfBundleN (8-byte integer)
283/// TripleOfBundleN (byte length defined before)
284///
285/// Bundle1
286/// ...
287/// BundleN
288
289/// Read 8-byte integers from a buffer in little-endian format.
290static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) {
291 return llvm::support::endian::read64le(Buffer.data() + pos);
292}
293
294/// Write 8-byte integers to a buffer in little-endian format.
295static void Write8byteIntegerToBuffer(raw_ostream &OS, uint64_t Val) {
296 llvm::support::endian::write(OS, Val, llvm::endianness::little);
297}
298
299class BinaryFileHandler final : public FileHandler {
300 /// Information about the bundles extracted from the header.
301 struct BinaryBundleInfo final : public BundleInfo {
302 /// Size of the bundle.
303 uint64_t Size = 0u;
304 /// Offset at which the bundle starts in the bundled file.
305 uint64_t Offset = 0u;
306
307 BinaryBundleInfo() {}
308 BinaryBundleInfo(uint64_t Size, uint64_t Offset)
309 : Size(Size), Offset(Offset) {}
310 };
311
312 /// Map between a triple and the corresponding bundle information.
313 StringMap<BinaryBundleInfo> BundlesInfo;
314
315 /// Iterator for the bundle information that is being read.
316 StringMap<BinaryBundleInfo>::iterator CurBundleInfo;
317 StringMap<BinaryBundleInfo>::iterator NextBundleInfo;
318
319 /// Current bundle target to be written.
320 std::string CurWriteBundleTarget;
321
322 /// Configuration options and arrays for this bundler job
323 const OffloadBundlerConfig &BundlerConfig;
324
325public:
326 // TODO: Add error checking from ClangOffloadBundler.cpp
327 BinaryFileHandler(const OffloadBundlerConfig &BC) : BundlerConfig(BC) {}
328
329 ~BinaryFileHandler() final {}
330
331 Error ReadHeader(MemoryBuffer &Input) final {
332 StringRef FC = Input.getBuffer();
333
334 // Initialize the current bundle with the end of the container.
335 CurBundleInfo = BundlesInfo.end();
336
337 // Check if buffer is smaller than magic string.
338 size_t ReadChars = sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1;
339 if (ReadChars > FC.size())
340 return Error::success();
341
342 // Check if no magic was found.
343 if (llvm::identify_magic(FC) != llvm::file_magic::offload_bundle)
344 return Error::success();
345
346 // Read number of bundles.
347 if (ReadChars + 8 > FC.size())
348 return Error::success();
349
350 uint64_t NumberOfBundles = Read8byteIntegerFromBuffer(FC, ReadChars);
351 ReadChars += 8;
352
353 // Read bundle offsets, sizes and triples.
354 for (uint64_t i = 0; i < NumberOfBundles; ++i) {
355
356 // Read offset.
357 if (ReadChars + 8 > FC.size())
358 return Error::success();
359
360 uint64_t Offset = Read8byteIntegerFromBuffer(FC, ReadChars);
361 ReadChars += 8;
362
363 // Read size.
364 if (ReadChars + 8 > FC.size())
365 return Error::success();
366
367 uint64_t Size = Read8byteIntegerFromBuffer(FC, ReadChars);
368 ReadChars += 8;
369
370 // Read triple size.
371 if (ReadChars + 8 > FC.size())
372 return Error::success();
373
374 uint64_t TripleSize = Read8byteIntegerFromBuffer(FC, ReadChars);
375 ReadChars += 8;
376
377 // Read triple.
378 if (ReadChars + TripleSize > FC.size())
379 return Error::success();
380
381 StringRef Triple(&FC.data()[ReadChars], TripleSize);
382 ReadChars += TripleSize;
383
384 // Check if the offset and size make sense.
385 if (!Offset || Offset + Size > FC.size())
386 return Error::success();
387
388 assert(!BundlesInfo.contains(Triple) && "Triple is duplicated??");
389 BundlesInfo[Triple] = BinaryBundleInfo(Size, Offset);
390 }
391 // Set the iterator to where we will start to read.
392 CurBundleInfo = BundlesInfo.end();
393 NextBundleInfo = BundlesInfo.begin();
394 return Error::success();
395 }
396
398 ReadBundleStart(MemoryBuffer &Input) final {
399 if (NextBundleInfo == BundlesInfo.end())
400 return std::nullopt;
401 CurBundleInfo = NextBundleInfo++;
402 return CurBundleInfo->first();
403 }
404
405 Error ReadBundleEnd(MemoryBuffer &Input) final {
406 assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
407 return Error::success();
408 }
409
410 Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
411 assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
412 StringRef FC = Input.getBuffer();
413 OS.write(FC.data() + CurBundleInfo->second.Offset,
414 CurBundleInfo->second.Size);
415 return Error::success();
416 }
417
418 Error WriteHeader(raw_ostream &OS,
419 ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
420
421 // Compute size of the header.
422 uint64_t HeaderSize = 0;
423
424 HeaderSize += sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1;
425 HeaderSize += 8; // Number of Bundles
426
427 for (auto &T : BundlerConfig.TargetNames) {
428 HeaderSize += 3 * 8; // Bundle offset, Size of bundle and size of triple.
429 HeaderSize += T.size(); // The triple.
430 }
431
432 // Write to the buffer the header.
434
435 Write8byteIntegerToBuffer(OS, BundlerConfig.TargetNames.size());
436
437 unsigned Idx = 0;
438 for (auto &T : BundlerConfig.TargetNames) {
439 MemoryBuffer &MB = *Inputs[Idx++];
440 HeaderSize = alignTo(HeaderSize, BundlerConfig.BundleAlignment);
441 // Bundle offset.
442 Write8byteIntegerToBuffer(OS, HeaderSize);
443 // Size of the bundle (adds to the next bundle's offset)
444 Write8byteIntegerToBuffer(OS, MB.getBufferSize());
445 BundlesInfo[T] = BinaryBundleInfo(MB.getBufferSize(), HeaderSize);
446 HeaderSize += MB.getBufferSize();
447 // Size of the triple
448 Write8byteIntegerToBuffer(OS, T.size());
449 // Triple
450 OS << T;
451 }
452 return Error::success();
453 }
454
455 Error WriteBundleStart(raw_ostream &OS, StringRef TargetTriple) final {
456 CurWriteBundleTarget = TargetTriple.str();
457 return Error::success();
458 }
459
460 Error WriteBundleEnd(raw_ostream &OS, StringRef TargetTriple) final {
461 return Error::success();
462 }
463
464 Error WriteBundle(raw_ostream &OS, MemoryBuffer &Input) final {
465 auto BI = BundlesInfo[CurWriteBundleTarget];
466
467 // Pad with 0 to reach specified offset.
468 size_t CurrentPos = OS.tell();
469 size_t PaddingSize = BI.Offset > CurrentPos ? BI.Offset - CurrentPos : 0;
470 for (size_t I = 0; I < PaddingSize; ++I)
471 OS.write('\0');
472 assert(OS.tell() == BI.Offset);
473
474 OS.write(Input.getBufferStart(), Input.getBufferSize());
475
476 return Error::success();
477 }
478};
479
480// This class implements a list of temporary files that are removed upon
481// object destruction.
482class TempFileHandlerRAII {
483public:
484 ~TempFileHandlerRAII() {
485 for (const auto &File : Files)
486 sys::fs::remove(File);
487 }
488
489 // Creates temporary file with given contents.
490 Expected<StringRef> Create(std::optional<ArrayRef<char>> Contents) {
492 if (std::error_code EC =
493 sys::fs::createTemporaryFile("clang-offload-bundler", "tmp", File))
494 return createFileError(File, EC);
495 Files.push_front(File);
496
497 if (Contents) {
498 std::error_code EC;
499 raw_fd_ostream OS(File, EC);
500 if (EC)
501 return createFileError(File, EC);
502 OS.write(Contents->data(), Contents->size());
503 }
504 return Files.front().str();
505 }
506
507private:
508 std::forward_list<SmallString<128u>> Files;
509};
510
511/// Handler for object files. The bundles are organized by sections with a
512/// designated name.
513///
514/// To unbundle, we just copy the contents of the designated section.
515class ObjectFileHandler final : public FileHandler {
516
517 /// The object file we are currently dealing with.
518 std::unique_ptr<ObjectFile> Obj;
519
520 /// Return the input file contents.
521 StringRef getInputFileContents() const { return Obj->getData(); }
522
523 /// Return bundle name (<kind>-<triple>) if the provided section is an offload
524 /// section.
526 IsOffloadSection(SectionRef CurSection) {
527 Expected<StringRef> NameOrErr = CurSection.getName();
528 if (!NameOrErr)
529 return NameOrErr.takeError();
530
531 // If it does not start with the reserved suffix, just skip this section.
532 if (llvm::identify_magic(*NameOrErr) != llvm::file_magic::offload_bundle)
533 return std::nullopt;
534
535 // Return the triple that is right after the reserved prefix.
536 return NameOrErr->substr(sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1);
537 }
538
539 /// Total number of inputs.
540 unsigned NumberOfInputs = 0;
541
542 /// Total number of processed inputs, i.e, inputs that were already
543 /// read from the buffers.
544 unsigned NumberOfProcessedInputs = 0;
545
546 /// Iterator of the current and next section.
547 section_iterator CurrentSection;
548 section_iterator NextSection;
549
550 /// Configuration options and arrays for this bundler job
551 const OffloadBundlerConfig &BundlerConfig;
552
553public:
554 // TODO: Add error checking from ClangOffloadBundler.cpp
555 ObjectFileHandler(std::unique_ptr<ObjectFile> ObjIn,
556 const OffloadBundlerConfig &BC)
557 : Obj(std::move(ObjIn)), CurrentSection(Obj->section_begin()),
558 NextSection(Obj->section_begin()), BundlerConfig(BC) {}
559
560 ~ObjectFileHandler() final {}
561
562 Error ReadHeader(MemoryBuffer &Input) final { return Error::success(); }
563
565 ReadBundleStart(MemoryBuffer &Input) final {
566 while (NextSection != Obj->section_end()) {
567 CurrentSection = NextSection;
568 ++NextSection;
569
570 // Check if the current section name starts with the reserved prefix. If
571 // so, return the triple.
573 IsOffloadSection(*CurrentSection);
574 if (!TripleOrErr)
575 return TripleOrErr.takeError();
576 if (*TripleOrErr)
577 return **TripleOrErr;
578 }
579 return std::nullopt;
580 }
581
582 Error ReadBundleEnd(MemoryBuffer &Input) final { return Error::success(); }
583
584 Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
585 Expected<StringRef> ContentOrErr = CurrentSection->getContents();
586 if (!ContentOrErr)
587 return ContentOrErr.takeError();
588 StringRef Content = *ContentOrErr;
589
590 // Copy fat object contents to the output when extracting host bundle.
591 std::string ModifiedContent;
592 if (Content.size() == 1u && Content.front() == 0) {
593 auto HostBundleOrErr = getHostBundle(
594 StringRef(Input.getBufferStart(), Input.getBufferSize()));
595 if (!HostBundleOrErr)
596 return HostBundleOrErr.takeError();
597
598 ModifiedContent = std::move(*HostBundleOrErr);
599 Content = ModifiedContent;
600 }
601
602 OS.write(Content.data(), Content.size());
603 return Error::success();
604 }
605
606 Error WriteHeader(raw_ostream &OS,
607 ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
608 assert(BundlerConfig.HostInputIndex != ~0u &&
609 "Host input index not defined.");
610
611 // Record number of inputs.
612 NumberOfInputs = Inputs.size();
613 return Error::success();
614 }
615
616 Error WriteBundleStart(raw_ostream &OS, StringRef TargetTriple) final {
617 ++NumberOfProcessedInputs;
618 return Error::success();
619 }
620
621 Error WriteBundleEnd(raw_ostream &OS, StringRef TargetTriple) final {
622 return Error::success();
623 }
624
625 Error finalizeOutputFile() final {
626 assert(NumberOfProcessedInputs <= NumberOfInputs &&
627 "Processing more inputs that actually exist!");
628 assert(BundlerConfig.HostInputIndex != ~0u &&
629 "Host input index not defined.");
630
631 // If this is not the last output, we don't have to do anything.
632 if (NumberOfProcessedInputs != NumberOfInputs)
633 return Error::success();
634
635 // We will use llvm-objcopy to add target objects sections to the output
636 // fat object. These sections should have 'exclude' flag set which tells
637 // link editor to remove them from linker inputs when linking executable or
638 // shared library.
639
640 assert(BundlerConfig.ObjcopyPath != "" &&
641 "llvm-objcopy path not specified");
642
643 // Temporary files that need to be removed.
644 TempFileHandlerRAII TempFiles;
645
646 // Compose llvm-objcopy command line for add target objects' sections with
647 // appropriate flags.
648 BumpPtrAllocator Alloc;
649 StringSaver SS{Alloc};
650 SmallVector<StringRef, 8u> ObjcopyArgs{"llvm-objcopy"};
651
652 for (unsigned I = 0; I < NumberOfInputs; ++I) {
653 StringRef InputFile = BundlerConfig.InputFileNames[I];
654 if (I == BundlerConfig.HostInputIndex) {
655 // Special handling for the host bundle. We do not need to add a
656 // standard bundle for the host object since we are going to use fat
657 // object as a host object. Therefore use dummy contents (one zero byte)
658 // when creating section for the host bundle.
659 Expected<StringRef> TempFileOrErr = TempFiles.Create(ArrayRef<char>(0));
660 if (!TempFileOrErr)
661 return TempFileOrErr.takeError();
662 InputFile = *TempFileOrErr;
663 }
664
665 ObjcopyArgs.push_back(
666 SS.save(Twine("--add-section=") + OFFLOAD_BUNDLER_MAGIC_STR +
667 BundlerConfig.TargetNames[I] + "=" + InputFile));
668 ObjcopyArgs.push_back(
669 SS.save(Twine("--set-section-flags=") + OFFLOAD_BUNDLER_MAGIC_STR +
670 BundlerConfig.TargetNames[I] + "=readonly,exclude"));
671 }
672 ObjcopyArgs.push_back("--");
673 ObjcopyArgs.push_back(
674 BundlerConfig.InputFileNames[BundlerConfig.HostInputIndex]);
675 ObjcopyArgs.push_back(BundlerConfig.OutputFileNames.front());
676
677 if (Error Err = executeObjcopy(BundlerConfig.ObjcopyPath, ObjcopyArgs))
678 return Err;
679
680 return Error::success();
681 }
682
683 Error WriteBundle(raw_ostream &OS, MemoryBuffer &Input) final {
684 return Error::success();
685 }
686
687private:
688 Error executeObjcopy(StringRef Objcopy, ArrayRef<StringRef> Args) {
689 // If the user asked for the commands to be printed out, we do that
690 // instead of executing it.
691 if (BundlerConfig.PrintExternalCommands) {
692 errs() << "\"" << Objcopy << "\"";
693 for (StringRef Arg : drop_begin(Args, 1))
694 errs() << " \"" << Arg << "\"";
695 errs() << "\n";
696 } else {
697 if (sys::ExecuteAndWait(Objcopy, Args))
698 return createStringError(inconvertibleErrorCode(),
699 "'llvm-objcopy' tool failed");
700 }
701 return Error::success();
702 }
703
704 Expected<std::string> getHostBundle(StringRef Input) {
705 TempFileHandlerRAII TempFiles;
706
707 auto ModifiedObjPathOrErr = TempFiles.Create(std::nullopt);
708 if (!ModifiedObjPathOrErr)
709 return ModifiedObjPathOrErr.takeError();
710 StringRef ModifiedObjPath = *ModifiedObjPathOrErr;
711
712 BumpPtrAllocator Alloc;
713 StringSaver SS{Alloc};
714 SmallVector<StringRef, 16> ObjcopyArgs{"llvm-objcopy"};
715
716 ObjcopyArgs.push_back("--regex");
717 ObjcopyArgs.push_back("--remove-section=__CLANG_OFFLOAD_BUNDLE__.*");
718 ObjcopyArgs.push_back("--");
719
720 StringRef ObjcopyInputFileName;
721 // When unbundling an archive, the content of each object file in the
722 // archive is passed to this function by parameter Input, which is different
723 // from the content of the original input archive file, therefore it needs
724 // to be saved to a temporary file before passed to llvm-objcopy. Otherwise,
725 // Input is the same as the content of the original input file, therefore
726 // temporary file is not needed.
727 if (StringRef(BundlerConfig.FilesType).starts_with("a")) {
728 auto InputFileOrErr =
729 TempFiles.Create(ArrayRef<char>(Input.data(), Input.size()));
730 if (!InputFileOrErr)
731 return InputFileOrErr.takeError();
732 ObjcopyInputFileName = *InputFileOrErr;
733 } else
734 ObjcopyInputFileName = BundlerConfig.InputFileNames.front();
735
736 ObjcopyArgs.push_back(ObjcopyInputFileName);
737 ObjcopyArgs.push_back(ModifiedObjPath);
738
739 if (Error Err = executeObjcopy(BundlerConfig.ObjcopyPath, ObjcopyArgs))
740 return std::move(Err);
741
742 auto BufOrErr = MemoryBuffer::getFile(ModifiedObjPath);
743 if (!BufOrErr)
744 return createStringError(BufOrErr.getError(),
745 "Failed to read back the modified object file");
746
747 return BufOrErr->get()->getBuffer().str();
748 }
749};
750
751/// Handler for text files. The bundled file will have the following format.
752///
753/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
754/// Bundle 1
755/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
756/// ...
757/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
758/// Bundle N
759/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
760class TextFileHandler final : public FileHandler {
761 /// String that begins a line comment.
762 StringRef Comment;
763
764 /// String that initiates a bundle.
765 std::string BundleStartString;
766
767 /// String that closes a bundle.
768 std::string BundleEndString;
769
770 /// Number of chars read from input.
771 size_t ReadChars = 0u;
772
773protected:
774 Error ReadHeader(MemoryBuffer &Input) final { return Error::success(); }
775
777 ReadBundleStart(MemoryBuffer &Input) final {
778 StringRef FC = Input.getBuffer();
779
780 // Find start of the bundle.
781 ReadChars = FC.find(BundleStartString, ReadChars);
782 if (ReadChars == FC.npos)
783 return std::nullopt;
784
785 // Get position of the triple.
786 size_t TripleStart = ReadChars = ReadChars + BundleStartString.size();
787
788 // Get position that closes the triple.
789 size_t TripleEnd = ReadChars = FC.find("\n", ReadChars);
790 if (TripleEnd == FC.npos)
791 return std::nullopt;
792
793 // Next time we read after the new line.
794 ++ReadChars;
795
796 return StringRef(&FC.data()[TripleStart], TripleEnd - TripleStart);
797 }
798
799 Error ReadBundleEnd(MemoryBuffer &Input) final {
800 StringRef FC = Input.getBuffer();
801
802 // Read up to the next new line.
803 assert(FC[ReadChars] == '\n' && "The bundle should end with a new line.");
804
805 size_t TripleEnd = ReadChars = FC.find("\n", ReadChars + 1);
806 if (TripleEnd != FC.npos)
807 // Next time we read after the new line.
808 ++ReadChars;
809
810 return Error::success();
811 }
812
813 Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
814 StringRef FC = Input.getBuffer();
815 size_t BundleStart = ReadChars;
816
817 // Find end of the bundle.
818 size_t BundleEnd = ReadChars = FC.find(BundleEndString, ReadChars);
819
820 StringRef Bundle(&FC.data()[BundleStart], BundleEnd - BundleStart);
821 OS << Bundle;
822
823 return Error::success();
824 }
825
826 Error WriteHeader(raw_ostream &OS,
827 ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
828 return Error::success();
829 }
830
831 Error WriteBundleStart(raw_ostream &OS, StringRef TargetTriple) final {
832 OS << BundleStartString << TargetTriple << "\n";
833 return Error::success();
834 }
835
836 Error WriteBundleEnd(raw_ostream &OS, StringRef TargetTriple) final {
837 OS << BundleEndString << TargetTriple << "\n";
838 return Error::success();
839 }
840
841 Error WriteBundle(raw_ostream &OS, MemoryBuffer &Input) final {
842 OS << Input.getBuffer();
843 return Error::success();
844 }
845
846public:
847 TextFileHandler(StringRef Comment) : Comment(Comment), ReadChars(0) {
848 BundleStartString =
849 "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__START__ ";
850 BundleEndString =
851 "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__END__ ";
852 }
853
854 Error listBundleIDsCallback(MemoryBuffer &Input,
855 const BundleInfo &Info) final {
856 // TODO: To list bundle IDs in a bundled text file we need to go through
857 // all bundles. The format of bundled text file may need to include a
858 // header if the performance of listing bundle IDs of bundled text file is
859 // important.
860 ReadChars = Input.getBuffer().find(BundleEndString, ReadChars);
861 if (Error Err = ReadBundleEnd(Input))
862 return Err;
863 return Error::success();
864 }
865};
866} // namespace
867
868/// Return an appropriate object file handler. We use the specific object
869/// handler if we know how to deal with that format, otherwise we use a default
870/// binary file handler.
871static std::unique_ptr<FileHandler>
872CreateObjectFileHandler(MemoryBuffer &FirstInput,
873 const OffloadBundlerConfig &BundlerConfig) {
874 // Check if the input file format is one that we know how to deal with.
875 Expected<std::unique_ptr<Binary>> BinaryOrErr = createBinary(FirstInput);
876
877 // We only support regular object files. If failed to open the input as a
878 // known binary or this is not an object file use the default binary handler.
879 if (errorToBool(BinaryOrErr.takeError()) || !isa<ObjectFile>(*BinaryOrErr))
880 return std::make_unique<BinaryFileHandler>(BundlerConfig);
881
882 // Otherwise create an object file handler. The handler will be owned by the
883 // client of this function.
884 return std::make_unique<ObjectFileHandler>(
885 std::unique_ptr<ObjectFile>(cast<ObjectFile>(BinaryOrErr->release())),
886 BundlerConfig);
887}
888
889/// Return an appropriate handler given the input files and options.
891CreateFileHandler(MemoryBuffer &FirstInput,
892 const OffloadBundlerConfig &BundlerConfig) {
893 std::string FilesType = BundlerConfig.FilesType;
894
895 if (FilesType == "i")
896 return std::make_unique<TextFileHandler>(/*Comment=*/"//");
897 if (FilesType == "ii")
898 return std::make_unique<TextFileHandler>(/*Comment=*/"//");
899 if (FilesType == "cui")
900 return std::make_unique<TextFileHandler>(/*Comment=*/"//");
901 if (FilesType == "hipi")
902 return std::make_unique<TextFileHandler>(/*Comment=*/"//");
903 // TODO: `.d` should be eventually removed once `-M` and its variants are
904 // handled properly in offload compilation.
905 if (FilesType == "d")
906 return std::make_unique<TextFileHandler>(/*Comment=*/"#");
907 if (FilesType == "ll")
908 return std::make_unique<TextFileHandler>(/*Comment=*/";");
909 if (FilesType == "bc")
910 return std::make_unique<BinaryFileHandler>(BundlerConfig);
911 if (FilesType == "s")
912 return std::make_unique<TextFileHandler>(/*Comment=*/"#");
913 if (FilesType == "o")
914 return CreateObjectFileHandler(FirstInput, BundlerConfig);
915 if (FilesType == "a")
916 return CreateObjectFileHandler(FirstInput, BundlerConfig);
917 if (FilesType == "gch")
918 return std::make_unique<BinaryFileHandler>(BundlerConfig);
919 if (FilesType == "ast")
920 return std::make_unique<BinaryFileHandler>(BundlerConfig);
921
922 return createStringError(errc::invalid_argument,
923 "'" + FilesType + "': invalid file type specified");
924}
925
927 if (llvm::compression::zstd::isAvailable()) {
928 CompressionFormat = llvm::compression::Format::Zstd;
929 // Compression level 3 is usually sufficient for zstd since long distance
930 // matching is enabled.
932 } else if (llvm::compression::zlib::isAvailable()) {
933 CompressionFormat = llvm::compression::Format::Zlib;
934 // Use default level for zlib since higher level does not have significant
935 // improvement.
936 CompressionLevel = llvm::compression::zlib::DefaultCompression;
937 }
938 auto IgnoreEnvVarOpt =
939 llvm::sys::Process::GetEnv("OFFLOAD_BUNDLER_IGNORE_ENV_VAR");
940 if (IgnoreEnvVarOpt.has_value() && IgnoreEnvVarOpt.value() == "1")
941 return;
942
943 auto VerboseEnvVarOpt = llvm::sys::Process::GetEnv("OFFLOAD_BUNDLER_VERBOSE");
944 if (VerboseEnvVarOpt.has_value())
945 Verbose = VerboseEnvVarOpt.value() == "1";
946
947 auto CompressEnvVarOpt =
948 llvm::sys::Process::GetEnv("OFFLOAD_BUNDLER_COMPRESS");
949 if (CompressEnvVarOpt.has_value())
950 Compress = CompressEnvVarOpt.value() == "1";
951
952 auto CompressionLevelEnvVarOpt =
953 llvm::sys::Process::GetEnv("OFFLOAD_BUNDLER_COMPRESSION_LEVEL");
954 if (CompressionLevelEnvVarOpt.has_value()) {
955 llvm::StringRef CompressionLevelStr = CompressionLevelEnvVarOpt.value();
956 int Level;
957 if (!CompressionLevelStr.getAsInteger(10, Level))
958 CompressionLevel = Level;
959 else
960 llvm::errs()
961 << "Warning: Invalid value for OFFLOAD_BUNDLER_COMPRESSION_LEVEL: "
962 << CompressionLevelStr.str() << ". Ignoring it.\n";
963 }
964}
965
966// Utility function to format numbers with commas
967static std::string formatWithCommas(unsigned long long Value) {
968 std::string Num = std::to_string(Value);
969 int InsertPosition = Num.length() - 3;
970 while (InsertPosition > 0) {
971 Num.insert(InsertPosition, ",");
972 InsertPosition -= 3;
973 }
974 return Num;
975}
976
978CompressedOffloadBundle::compress(llvm::compression::Params P,
979 const llvm::MemoryBuffer &Input,
980 bool Verbose) {
981 if (!llvm::compression::zstd::isAvailable() &&
982 !llvm::compression::zlib::isAvailable())
983 return createStringError(llvm::inconvertibleErrorCode(),
984 "Compression not supported");
985
986 llvm::Timer HashTimer("Hash Calculation Timer", "Hash calculation time",
988 if (Verbose)
989 HashTimer.startTimer();
990 llvm::MD5 Hash;
991 llvm::MD5::MD5Result Result;
992 Hash.update(Input.getBuffer());
993 Hash.final(Result);
994 uint64_t TruncatedHash = Result.low();
995 if (Verbose)
996 HashTimer.stopTimer();
997
998 SmallVector<uint8_t, 0> CompressedBuffer;
999 auto BufferUint8 = llvm::ArrayRef<uint8_t>(
1000 reinterpret_cast<const uint8_t *>(Input.getBuffer().data()),
1001 Input.getBuffer().size());
1002
1003 llvm::Timer CompressTimer("Compression Timer", "Compression time",
1005 if (Verbose)
1006 CompressTimer.startTimer();
1007 llvm::compression::compress(P, BufferUint8, CompressedBuffer);
1008 if (Verbose)
1009 CompressTimer.stopTimer();
1010
1011 uint16_t CompressionMethod = static_cast<uint16_t>(P.format);
1012 uint32_t UncompressedSize = Input.getBuffer().size();
1013
1014 SmallVector<char, 0> FinalBuffer;
1015 llvm::raw_svector_ostream OS(FinalBuffer);
1016 OS << MagicNumber;
1017 OS.write(reinterpret_cast<const char *>(&Version), sizeof(Version));
1018 OS.write(reinterpret_cast<const char *>(&CompressionMethod),
1019 sizeof(CompressionMethod));
1020 OS.write(reinterpret_cast<const char *>(&UncompressedSize),
1021 sizeof(UncompressedSize));
1022 OS.write(reinterpret_cast<const char *>(&TruncatedHash),
1023 sizeof(TruncatedHash));
1024 OS.write(reinterpret_cast<const char *>(CompressedBuffer.data()),
1025 CompressedBuffer.size());
1026
1027 if (Verbose) {
1028 auto MethodUsed =
1029 P.format == llvm::compression::Format::Zstd ? "zstd" : "zlib";
1030 double CompressionRate =
1031 static_cast<double>(UncompressedSize) / CompressedBuffer.size();
1032 double CompressionTimeSeconds = CompressTimer.getTotalTime().getWallTime();
1033 double CompressionSpeedMBs =
1034 (UncompressedSize / (1024.0 * 1024.0)) / CompressionTimeSeconds;
1035
1036 llvm::errs() << "Compressed bundle format version: " << Version << "\n"
1037 << "Compression method used: " << MethodUsed << "\n"
1038 << "Compression level: " << P.level << "\n"
1039 << "Binary size before compression: "
1040 << formatWithCommas(UncompressedSize) << " bytes\n"
1041 << "Binary size after compression: "
1042 << formatWithCommas(CompressedBuffer.size()) << " bytes\n"
1043 << "Compression rate: "
1044 << llvm::format("%.2lf", CompressionRate) << "\n"
1045 << "Compression ratio: "
1046 << llvm::format("%.2lf%%", 100.0 / CompressionRate) << "\n"
1047 << "Compression speed: "
1048 << llvm::format("%.2lf MB/s", CompressionSpeedMBs) << "\n"
1049 << "Truncated MD5 hash: "
1050 << llvm::format_hex(TruncatedHash, 16) << "\n";
1051 }
1052 return llvm::MemoryBuffer::getMemBufferCopy(
1053 llvm::StringRef(FinalBuffer.data(), FinalBuffer.size()));
1054}
1055
1057CompressedOffloadBundle::decompress(const llvm::MemoryBuffer &Input,
1058 bool Verbose) {
1059
1060 StringRef Blob = Input.getBuffer();
1061
1062 if (Blob.size() < HeaderSize) {
1063 return llvm::MemoryBuffer::getMemBufferCopy(Blob);
1064 }
1065 if (llvm::identify_magic(Blob) !=
1066 llvm::file_magic::offload_bundle_compressed) {
1067 if (Verbose)
1068 llvm::errs() << "Uncompressed bundle.\n";
1069 return llvm::MemoryBuffer::getMemBufferCopy(Blob);
1070 }
1071
1072 uint16_t ThisVersion;
1073 uint16_t CompressionMethod;
1074 uint32_t UncompressedSize;
1075 uint64_t StoredHash;
1076 memcpy(&ThisVersion, Input.getBuffer().data() + MagicNumber.size(),
1077 sizeof(uint16_t));
1078 memcpy(&CompressionMethod, Blob.data() + MagicSize + VersionFieldSize,
1079 sizeof(uint16_t));
1080 memcpy(&UncompressedSize,
1081 Blob.data() + MagicSize + VersionFieldSize + MethodFieldSize,
1082 sizeof(uint32_t));
1083 memcpy(&StoredHash,
1084 Blob.data() + MagicSize + VersionFieldSize + MethodFieldSize +
1085 SizeFieldSize,
1086 sizeof(uint64_t));
1087
1088 llvm::compression::Format CompressionFormat;
1089 if (CompressionMethod ==
1090 static_cast<uint16_t>(llvm::compression::Format::Zlib))
1091 CompressionFormat = llvm::compression::Format::Zlib;
1092 else if (CompressionMethod ==
1093 static_cast<uint16_t>(llvm::compression::Format::Zstd))
1094 CompressionFormat = llvm::compression::Format::Zstd;
1095 else
1096 return createStringError(inconvertibleErrorCode(),
1097 "Unknown compressing method");
1098
1099 llvm::Timer DecompressTimer("Decompression Timer", "Decompression time",
1101 if (Verbose)
1102 DecompressTimer.startTimer();
1103
1104 SmallVector<uint8_t, 0> DecompressedData;
1105 StringRef CompressedData = Blob.substr(HeaderSize);
1106 if (llvm::Error DecompressionError = llvm::compression::decompress(
1107 CompressionFormat, llvm::arrayRefFromStringRef(CompressedData),
1108 DecompressedData, UncompressedSize))
1109 return createStringError(inconvertibleErrorCode(),
1110 "Could not decompress embedded file contents: " +
1111 llvm::toString(std::move(DecompressionError)));
1112
1113 if (Verbose) {
1114 DecompressTimer.stopTimer();
1115
1116 double DecompressionTimeSeconds =
1117 DecompressTimer.getTotalTime().getWallTime();
1118
1119 // Recalculate MD5 hash for integrity check
1120 llvm::Timer HashRecalcTimer("Hash Recalculation Timer",
1121 "Hash recalculation time",
1123 HashRecalcTimer.startTimer();
1124 llvm::MD5 Hash;
1125 llvm::MD5::MD5Result Result;
1126 Hash.update(llvm::ArrayRef<uint8_t>(DecompressedData.data(),
1127 DecompressedData.size()));
1128 Hash.final(Result);
1129 uint64_t RecalculatedHash = Result.low();
1130 HashRecalcTimer.stopTimer();
1131 bool HashMatch = (StoredHash == RecalculatedHash);
1132
1133 double CompressionRate =
1134 static_cast<double>(UncompressedSize) / CompressedData.size();
1135 double DecompressionSpeedMBs =
1136 (UncompressedSize / (1024.0 * 1024.0)) / DecompressionTimeSeconds;
1137
1138 llvm::errs() << "Compressed bundle format version: " << ThisVersion << "\n"
1139 << "Decompression method: "
1140 << (CompressionFormat == llvm::compression::Format::Zlib
1141 ? "zlib"
1142 : "zstd")
1143 << "\n"
1144 << "Size before decompression: "
1145 << formatWithCommas(CompressedData.size()) << " bytes\n"
1146 << "Size after decompression: "
1147 << formatWithCommas(UncompressedSize) << " bytes\n"
1148 << "Compression rate: "
1149 << llvm::format("%.2lf", CompressionRate) << "\n"
1150 << "Compression ratio: "
1151 << llvm::format("%.2lf%%", 100.0 / CompressionRate) << "\n"
1152 << "Decompression speed: "
1153 << llvm::format("%.2lf MB/s", DecompressionSpeedMBs) << "\n"
1154 << "Stored hash: " << llvm::format_hex(StoredHash, 16) << "\n"
1155 << "Recalculated hash: "
1156 << llvm::format_hex(RecalculatedHash, 16) << "\n"
1157 << "Hashes match: " << (HashMatch ? "Yes" : "No") << "\n";
1158 }
1159
1160 return llvm::MemoryBuffer::getMemBufferCopy(
1161 llvm::toStringRef(DecompressedData));
1162}
1163
1164// List bundle IDs. Return true if an error was found.
1166 StringRef InputFileName, const OffloadBundlerConfig &BundlerConfig) {
1167 // Open Input file.
1168 ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
1169 MemoryBuffer::getFileOrSTDIN(InputFileName);
1170 if (std::error_code EC = CodeOrErr.getError())
1171 return createFileError(InputFileName, EC);
1172
1173 // Decompress the input if necessary.
1174 Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr =
1176 if (!DecompressedBufferOrErr)
1177 return createStringError(
1178 inconvertibleErrorCode(),
1179 "Failed to decompress input: " +
1180 llvm::toString(DecompressedBufferOrErr.takeError()));
1181
1182 MemoryBuffer &DecompressedInput = **DecompressedBufferOrErr;
1183
1184 // Select the right files handler.
1185 Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
1186 CreateFileHandler(DecompressedInput, BundlerConfig);
1187 if (!FileHandlerOrErr)
1188 return FileHandlerOrErr.takeError();
1189
1190 std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr;
1191 assert(FH);
1192 return FH->listBundleIDs(DecompressedInput);
1193}
1194
1195/// @brief Checks if a code object \p CodeObjectInfo is compatible with a given
1196/// target \p TargetInfo.
1197/// @link https://clang.llvm.org/docs/ClangOffloadBundler.html#bundle-entry-id
1200
1201 // Compatible in case of exact match.
1202 if (CodeObjectInfo == TargetInfo) {
1203 DEBUG_WITH_TYPE("CodeObjectCompatibility",
1204 dbgs() << "Compatible: Exact match: \t[CodeObject: "
1205 << CodeObjectInfo.str()
1206 << "]\t:\t[Target: " << TargetInfo.str() << "]\n");
1207 return true;
1208 }
1209
1210 // Incompatible if Kinds or Triples mismatch.
1211 if (!CodeObjectInfo.isOffloadKindCompatible(TargetInfo.OffloadKind) ||
1212 !CodeObjectInfo.Triple.isCompatibleWith(TargetInfo.Triple)) {
1213 DEBUG_WITH_TYPE(
1214 "CodeObjectCompatibility",
1215 dbgs() << "Incompatible: Kind/Triple mismatch \t[CodeObject: "
1216 << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str()
1217 << "]\n");
1218 return false;
1219 }
1220
1221 // Incompatible if Processors mismatch.
1222 llvm::StringMap<bool> CodeObjectFeatureMap, TargetFeatureMap;
1223 std::optional<StringRef> CodeObjectProc = clang::parseTargetID(
1224 CodeObjectInfo.Triple, CodeObjectInfo.TargetID, &CodeObjectFeatureMap);
1225 std::optional<StringRef> TargetProc = clang::parseTargetID(
1226 TargetInfo.Triple, TargetInfo.TargetID, &TargetFeatureMap);
1227
1228 // Both TargetProc and CodeObjectProc can't be empty here.
1229 if (!TargetProc || !CodeObjectProc ||
1230 CodeObjectProc.value() != TargetProc.value()) {
1231 DEBUG_WITH_TYPE("CodeObjectCompatibility",
1232 dbgs() << "Incompatible: Processor mismatch \t[CodeObject: "
1233 << CodeObjectInfo.str()
1234 << "]\t:\t[Target: " << TargetInfo.str() << "]\n");
1235 return false;
1236 }
1237
1238 // Incompatible if CodeObject has more features than Target, irrespective of
1239 // type or sign of features.
1240 if (CodeObjectFeatureMap.getNumItems() > TargetFeatureMap.getNumItems()) {
1241 DEBUG_WITH_TYPE("CodeObjectCompatibility",
1242 dbgs() << "Incompatible: CodeObject has more features "
1243 "than target \t[CodeObject: "
1244 << CodeObjectInfo.str()
1245 << "]\t:\t[Target: " << TargetInfo.str() << "]\n");
1246 return false;
1247 }
1248
1249 // Compatible if each target feature specified by target is compatible with
1250 // target feature of code object. The target feature is compatible if the
1251 // code object does not specify it (meaning Any), or if it specifies it
1252 // with the same value (meaning On or Off).
1253 for (const auto &CodeObjectFeature : CodeObjectFeatureMap) {
1254 auto TargetFeature = TargetFeatureMap.find(CodeObjectFeature.getKey());
1255 if (TargetFeature == TargetFeatureMap.end()) {
1256 DEBUG_WITH_TYPE(
1257 "CodeObjectCompatibility",
1258 dbgs()
1259 << "Incompatible: Value of CodeObject's non-ANY feature is "
1260 "not matching with Target feature's ANY value \t[CodeObject: "
1261 << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str()
1262 << "]\n");
1263 return false;
1264 } else if (TargetFeature->getValue() != CodeObjectFeature.getValue()) {
1265 DEBUG_WITH_TYPE(
1266 "CodeObjectCompatibility",
1267 dbgs() << "Incompatible: Value of CodeObject's non-ANY feature is "
1268 "not matching with Target feature's non-ANY value "
1269 "\t[CodeObject: "
1270 << CodeObjectInfo.str()
1271 << "]\t:\t[Target: " << TargetInfo.str() << "]\n");
1272 return false;
1273 }
1274 }
1275
1276 // CodeObject is compatible if all features of Target are:
1277 // - either, present in the Code Object's features map with the same sign,
1278 // - or, the feature is missing from CodeObjects's features map i.e. it is
1279 // set to ANY
1280 DEBUG_WITH_TYPE(
1281 "CodeObjectCompatibility",
1282 dbgs() << "Compatible: Target IDs are compatible \t[CodeObject: "
1283 << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str()
1284 << "]\n");
1285 return true;
1286}
1287
1288/// Bundle the files. Return true if an error was found.
1290 std::error_code EC;
1291
1292 // Create a buffer to hold the content before compressing.
1293 SmallVector<char, 0> Buffer;
1294 llvm::raw_svector_ostream BufferStream(Buffer);
1295
1296 // Open input files.
1298 InputBuffers.reserve(BundlerConfig.InputFileNames.size());
1299 for (auto &I : BundlerConfig.InputFileNames) {
1300 ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
1301 MemoryBuffer::getFileOrSTDIN(I);
1302 if (std::error_code EC = CodeOrErr.getError())
1303 return createFileError(I, EC);
1304 InputBuffers.emplace_back(std::move(*CodeOrErr));
1305 }
1306
1307 // Get the file handler. We use the host buffer as reference.
1309 "Host input index undefined??");
1311 *InputBuffers[BundlerConfig.AllowNoHost ? 0
1314 if (!FileHandlerOrErr)
1315 return FileHandlerOrErr.takeError();
1316
1317 std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr;
1318 assert(FH);
1319
1320 // Write header.
1321 if (Error Err = FH->WriteHeader(BufferStream, InputBuffers))
1322 return Err;
1323
1324 // Write all bundles along with the start/end markers. If an error was found
1325 // writing the end of the bundle component, abort the bundle writing.
1326 auto Input = InputBuffers.begin();
1327 for (auto &Triple : BundlerConfig.TargetNames) {
1328 if (Error Err = FH->WriteBundleStart(BufferStream, Triple))
1329 return Err;
1330 if (Error Err = FH->WriteBundle(BufferStream, **Input))
1331 return Err;
1332 if (Error Err = FH->WriteBundleEnd(BufferStream, Triple))
1333 return Err;
1334 ++Input;
1335 }
1336
1337 raw_fd_ostream OutputFile(BundlerConfig.OutputFileNames.front(), EC,
1338 sys::fs::OF_None);
1339 if (EC)
1340 return createFileError(BundlerConfig.OutputFileNames.front(), EC);
1341
1342 SmallVector<char, 0> CompressedBuffer;
1343 if (BundlerConfig.Compress) {
1344 std::unique_ptr<llvm::MemoryBuffer> BufferMemory =
1345 llvm::MemoryBuffer::getMemBufferCopy(
1346 llvm::StringRef(Buffer.data(), Buffer.size()));
1347 auto CompressionResult = CompressedOffloadBundle::compress(
1349 /*zstdEnableLdm=*/true},
1350 *BufferMemory, BundlerConfig.Verbose);
1351 if (auto Error = CompressionResult.takeError())
1352 return Error;
1353
1354 auto CompressedMemBuffer = std::move(CompressionResult.get());
1355 CompressedBuffer.assign(CompressedMemBuffer->getBufferStart(),
1356 CompressedMemBuffer->getBufferEnd());
1357 } else
1358 CompressedBuffer = Buffer;
1359
1360 OutputFile.write(CompressedBuffer.data(), CompressedBuffer.size());
1361
1362 return FH->finalizeOutputFile();
1363}
1364
1365// Unbundle the files. Return true if an error was found.
1367 // Open Input file.
1368 ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
1369 MemoryBuffer::getFileOrSTDIN(BundlerConfig.InputFileNames.front());
1370 if (std::error_code EC = CodeOrErr.getError())
1371 return createFileError(BundlerConfig.InputFileNames.front(), EC);
1372
1373 // Decompress the input if necessary.
1374 Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr =
1376 if (!DecompressedBufferOrErr)
1377 return createStringError(
1378 inconvertibleErrorCode(),
1379 "Failed to decompress input: " +
1380 llvm::toString(DecompressedBufferOrErr.takeError()));
1381
1382 MemoryBuffer &Input = **DecompressedBufferOrErr;
1383
1384 // Select the right files handler.
1385 Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
1387 if (!FileHandlerOrErr)
1388 return FileHandlerOrErr.takeError();
1389
1390 std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr;
1391 assert(FH);
1392
1393 // Read the header of the bundled file.
1394 if (Error Err = FH->ReadHeader(Input))
1395 return Err;
1396
1397 // Create a work list that consist of the map triple/output file.
1398 StringMap<StringRef> Worklist;
1399 auto Output = BundlerConfig.OutputFileNames.begin();
1400 for (auto &Triple : BundlerConfig.TargetNames) {
1401 Worklist[Triple] = *Output;
1402 ++Output;
1403 }
1404
1405 // Read all the bundles that are in the work list. If we find no bundles we
1406 // assume the file is meant for the host target.
1407 bool FoundHostBundle = false;
1408 while (!Worklist.empty()) {
1409 Expected<std::optional<StringRef>> CurTripleOrErr =
1410 FH->ReadBundleStart(Input);
1411 if (!CurTripleOrErr)
1412 return CurTripleOrErr.takeError();
1413
1414 // We don't have more bundles.
1415 if (!*CurTripleOrErr)
1416 break;
1417
1418 StringRef CurTriple = **CurTripleOrErr;
1419 assert(!CurTriple.empty());
1420
1421 auto Output = Worklist.begin();
1422 for (auto E = Worklist.end(); Output != E; Output++) {
1424 OffloadTargetInfo(CurTriple, BundlerConfig),
1425 OffloadTargetInfo((*Output).first(), BundlerConfig))) {
1426 break;
1427 }
1428 }
1429
1430 if (Output == Worklist.end())
1431 continue;
1432 // Check if the output file can be opened and copy the bundle to it.
1433 std::error_code EC;
1434 raw_fd_ostream OutputFile((*Output).second, EC, sys::fs::OF_None);
1435 if (EC)
1436 return createFileError((*Output).second, EC);
1437 if (Error Err = FH->ReadBundle(OutputFile, Input))
1438 return Err;
1439 if (Error Err = FH->ReadBundleEnd(Input))
1440 return Err;
1441 Worklist.erase(Output);
1442
1443 // Record if we found the host bundle.
1444 auto OffloadInfo = OffloadTargetInfo(CurTriple, BundlerConfig);
1445 if (OffloadInfo.hasHostKind())
1446 FoundHostBundle = true;
1447 }
1448
1449 if (!BundlerConfig.AllowMissingBundles && !Worklist.empty()) {
1450 std::string ErrMsg = "Can't find bundles for";
1451 std::set<StringRef> Sorted;
1452 for (auto &E : Worklist)
1453 Sorted.insert(E.first());
1454 unsigned I = 0;
1455 unsigned Last = Sorted.size() - 1;
1456 for (auto &E : Sorted) {
1457 if (I != 0 && Last > 1)
1458 ErrMsg += ",";
1459 ErrMsg += " ";
1460 if (I == Last && I != 0)
1461 ErrMsg += "and ";
1462 ErrMsg += E.str();
1463 ++I;
1464 }
1465 return createStringError(inconvertibleErrorCode(), ErrMsg);
1466 }
1467
1468 // If no bundles were found, assume the input file is the host bundle and
1469 // create empty files for the remaining targets.
1470 if (Worklist.size() == BundlerConfig.TargetNames.size()) {
1471 for (auto &E : Worklist) {
1472 std::error_code EC;
1473 raw_fd_ostream OutputFile(E.second, EC, sys::fs::OF_None);
1474 if (EC)
1475 return createFileError(E.second, EC);
1476
1477 // If this entry has a host kind, copy the input file to the output file.
1478 auto OffloadInfo = OffloadTargetInfo(E.getKey(), BundlerConfig);
1479 if (OffloadInfo.hasHostKind())
1480 OutputFile.write(Input.getBufferStart(), Input.getBufferSize());
1481 }
1482 return Error::success();
1483 }
1484
1485 // If we found elements, we emit an error if none of those were for the host
1486 // in case host bundle name was provided in command line.
1487 if (!(FoundHostBundle || BundlerConfig.HostInputIndex == ~0u ||
1489 return createStringError(inconvertibleErrorCode(),
1490 "Can't find bundle for the host target");
1491
1492 // If we still have any elements in the worklist, create empty files for them.
1493 for (auto &E : Worklist) {
1494 std::error_code EC;
1495 raw_fd_ostream OutputFile(E.second, EC, sys::fs::OF_None);
1496 if (EC)
1497 return createFileError(E.second, EC);
1498 }
1499
1500 return Error::success();
1501}
1502
1503static Archive::Kind getDefaultArchiveKindForHost() {
1504 return Triple(sys::getDefaultTargetTriple()).isOSDarwin() ? Archive::K_DARWIN
1505 : Archive::K_GNU;
1506}
1507
1508/// @brief Computes a list of targets among all given targets which are
1509/// compatible with this code object
1510/// @param [in] CodeObjectInfo Code Object
1511/// @param [out] CompatibleTargets List of all compatible targets among all
1512/// given targets
1513/// @return false, if no compatible target is found.
1514static bool
1516 SmallVectorImpl<StringRef> &CompatibleTargets,
1517 const OffloadBundlerConfig &BundlerConfig) {
1518 if (!CompatibleTargets.empty()) {
1519 DEBUG_WITH_TYPE("CodeObjectCompatibility",
1520 dbgs() << "CompatibleTargets list should be empty\n");
1521 return false;
1522 }
1523 for (auto &Target : BundlerConfig.TargetNames) {
1524 auto TargetInfo = OffloadTargetInfo(Target, BundlerConfig);
1525 if (isCodeObjectCompatible(CodeObjectInfo, TargetInfo))
1526 CompatibleTargets.push_back(Target);
1527 }
1528 return !CompatibleTargets.empty();
1529}
1530
1531// Check that each code object file in the input archive conforms to following
1532// rule: for a specific processor, a feature either shows up in all target IDs,
1533// or does not show up in any target IDs. Otherwise the target ID combination is
1534// invalid.
1535static Error
1536CheckHeterogeneousArchive(StringRef ArchiveName,
1537 const OffloadBundlerConfig &BundlerConfig) {
1538 std::vector<std::unique_ptr<MemoryBuffer>> ArchiveBuffers;
1539 ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
1540 MemoryBuffer::getFileOrSTDIN(ArchiveName, true, false);
1541 if (std::error_code EC = BufOrErr.getError())
1542 return createFileError(ArchiveName, EC);
1543
1544 ArchiveBuffers.push_back(std::move(*BufOrErr));
1546 Archive::create(ArchiveBuffers.back()->getMemBufferRef());
1547 if (!LibOrErr)
1548 return LibOrErr.takeError();
1549
1550 auto Archive = std::move(*LibOrErr);
1551
1552 Error ArchiveErr = Error::success();
1553 auto ChildEnd = Archive->child_end();
1554
1555 /// Iterate over all bundled code object files in the input archive.
1556 for (auto ArchiveIter = Archive->child_begin(ArchiveErr);
1557 ArchiveIter != ChildEnd; ++ArchiveIter) {
1558 if (ArchiveErr)
1559 return ArchiveErr;
1560 auto ArchiveChildNameOrErr = (*ArchiveIter).getName();
1561 if (!ArchiveChildNameOrErr)
1562 return ArchiveChildNameOrErr.takeError();
1563
1564 auto CodeObjectBufferRefOrErr = (*ArchiveIter).getMemoryBufferRef();
1565 if (!CodeObjectBufferRefOrErr)
1566 return CodeObjectBufferRefOrErr.takeError();
1567
1568 auto CodeObjectBuffer =
1569 MemoryBuffer::getMemBuffer(*CodeObjectBufferRefOrErr, false);
1570
1571 Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
1572 CreateFileHandler(*CodeObjectBuffer, BundlerConfig);
1573 if (!FileHandlerOrErr)
1574 return FileHandlerOrErr.takeError();
1575
1576 std::unique_ptr<FileHandler> &FileHandler = *FileHandlerOrErr;
1577 assert(FileHandler);
1578
1579 std::set<StringRef> BundleIds;
1580 auto CodeObjectFileError =
1581 FileHandler->getBundleIDs(*CodeObjectBuffer, BundleIds);
1582 if (CodeObjectFileError)
1583 return CodeObjectFileError;
1584
1585 auto &&ConflictingArchs = clang::getConflictTargetIDCombination(BundleIds);
1586 if (ConflictingArchs) {
1587 std::string ErrMsg =
1588 Twine("conflicting TargetIDs [" + ConflictingArchs.value().first +
1589 ", " + ConflictingArchs.value().second + "] found in " +
1590 ArchiveChildNameOrErr.get() + " of " + ArchiveName)
1591 .str();
1592 return createStringError(inconvertibleErrorCode(), ErrMsg);
1593 }
1594 }
1595
1596 return ArchiveErr;
1597}
1598
1599/// UnbundleArchive takes an archive file (".a") as input containing bundled
1600/// code object files, and a list of offload targets (not host), and extracts
1601/// the code objects into a new archive file for each offload target. Each
1602/// resulting archive file contains all code object files corresponding to that
1603/// particular offload target. The created archive file does not
1604/// contain an index of the symbols and code object files are named as
1605/// <<Parent Bundle Name>-<CodeObject's TargetID>>, with ':' replaced with '_'.
1607 std::vector<std::unique_ptr<MemoryBuffer>> ArchiveBuffers;
1608
1609 /// Map of target names with list of object files that will form the device
1610 /// specific archive for that target
1611 StringMap<std::vector<NewArchiveMember>> OutputArchivesMap;
1612
1613 // Map of target names and output archive filenames
1614 StringMap<StringRef> TargetOutputFileNameMap;
1615
1616 auto Output = BundlerConfig.OutputFileNames.begin();
1617 for (auto &Target : BundlerConfig.TargetNames) {
1618 TargetOutputFileNameMap[Target] = *Output;
1619 ++Output;
1620 }
1621
1622 StringRef IFName = BundlerConfig.InputFileNames.front();
1623
1625 // For a specific processor, a feature either shows up in all target IDs, or
1626 // does not show up in any target IDs. Otherwise the target ID combination
1627 // is invalid.
1628 auto ArchiveError = CheckHeterogeneousArchive(IFName, BundlerConfig);
1629 if (ArchiveError) {
1630 return ArchiveError;
1631 }
1632 }
1633
1634 ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
1635 MemoryBuffer::getFileOrSTDIN(IFName, true, false);
1636 if (std::error_code EC = BufOrErr.getError())
1637 return createFileError(BundlerConfig.InputFileNames.front(), EC);
1638
1639 ArchiveBuffers.push_back(std::move(*BufOrErr));
1641 Archive::create(ArchiveBuffers.back()->getMemBufferRef());
1642 if (!LibOrErr)
1643 return LibOrErr.takeError();
1644
1645 auto Archive = std::move(*LibOrErr);
1646
1647 Error ArchiveErr = Error::success();
1648 auto ChildEnd = Archive->child_end();
1649
1650 /// Iterate over all bundled code object files in the input archive.
1651 for (auto ArchiveIter = Archive->child_begin(ArchiveErr);
1652 ArchiveIter != ChildEnd; ++ArchiveIter) {
1653 if (ArchiveErr)
1654 return ArchiveErr;
1655 auto ArchiveChildNameOrErr = (*ArchiveIter).getName();
1656 if (!ArchiveChildNameOrErr)
1657 return ArchiveChildNameOrErr.takeError();
1658
1659 StringRef BundledObjectFile = sys::path::filename(*ArchiveChildNameOrErr);
1660
1661 auto CodeObjectBufferRefOrErr = (*ArchiveIter).getMemoryBufferRef();
1662 if (!CodeObjectBufferRefOrErr)
1663 return CodeObjectBufferRefOrErr.takeError();
1664
1665 auto TempCodeObjectBuffer =
1666 MemoryBuffer::getMemBuffer(*CodeObjectBufferRefOrErr, false);
1667
1668 // Decompress the buffer if necessary.
1669 Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr =
1670 CompressedOffloadBundle::decompress(*TempCodeObjectBuffer,
1672 if (!DecompressedBufferOrErr)
1673 return createStringError(
1674 inconvertibleErrorCode(),
1675 "Failed to decompress code object: " +
1676 llvm::toString(DecompressedBufferOrErr.takeError()));
1677
1678 MemoryBuffer &CodeObjectBuffer = **DecompressedBufferOrErr;
1679
1680 Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
1681 CreateFileHandler(CodeObjectBuffer, BundlerConfig);
1682 if (!FileHandlerOrErr)
1683 return FileHandlerOrErr.takeError();
1684
1685 std::unique_ptr<FileHandler> &FileHandler = *FileHandlerOrErr;
1686 assert(FileHandler &&
1687 "FileHandle creation failed for file in the archive!");
1688
1689 if (Error ReadErr = FileHandler->ReadHeader(CodeObjectBuffer))
1690 return ReadErr;
1691
1692 Expected<std::optional<StringRef>> CurBundleIDOrErr =
1693 FileHandler->ReadBundleStart(CodeObjectBuffer);
1694 if (!CurBundleIDOrErr)
1695 return CurBundleIDOrErr.takeError();
1696
1697 std::optional<StringRef> OptionalCurBundleID = *CurBundleIDOrErr;
1698 // No device code in this child, skip.
1699 if (!OptionalCurBundleID)
1700 continue;
1701 StringRef CodeObject = *OptionalCurBundleID;
1702
1703 // Process all bundle entries (CodeObjects) found in this child of input
1704 // archive.
1705 while (!CodeObject.empty()) {
1706 SmallVector<StringRef> CompatibleTargets;
1707 auto CodeObjectInfo = OffloadTargetInfo(CodeObject, BundlerConfig);
1708 if (getCompatibleOffloadTargets(CodeObjectInfo, CompatibleTargets,
1709 BundlerConfig)) {
1710 std::string BundleData;
1711 raw_string_ostream DataStream(BundleData);
1712 if (Error Err = FileHandler->ReadBundle(DataStream, CodeObjectBuffer))
1713 return Err;
1714
1715 for (auto &CompatibleTarget : CompatibleTargets) {
1716 SmallString<128> BundledObjectFileName;
1717 BundledObjectFileName.assign(BundledObjectFile);
1718 auto OutputBundleName =
1719 Twine(llvm::sys::path::stem(BundledObjectFileName) + "-" +
1720 CodeObject +
1721 getDeviceLibraryFileName(BundledObjectFileName,
1722 CodeObjectInfo.TargetID))
1723 .str();
1724 // Replace ':' in optional target feature list with '_' to ensure
1725 // cross-platform validity.
1726 std::replace(OutputBundleName.begin(), OutputBundleName.end(), ':',
1727 '_');
1728
1729 std::unique_ptr<MemoryBuffer> MemBuf = MemoryBuffer::getMemBufferCopy(
1730 DataStream.str(), OutputBundleName);
1731 ArchiveBuffers.push_back(std::move(MemBuf));
1732 llvm::MemoryBufferRef MemBufRef =
1733 MemoryBufferRef(*(ArchiveBuffers.back()));
1734
1735 // For inserting <CompatibleTarget, list<CodeObject>> entry in
1736 // OutputArchivesMap.
1737 if (!OutputArchivesMap.contains(CompatibleTarget)) {
1738
1739 std::vector<NewArchiveMember> ArchiveMembers;
1740 ArchiveMembers.push_back(NewArchiveMember(MemBufRef));
1741 OutputArchivesMap.insert_or_assign(CompatibleTarget,
1742 std::move(ArchiveMembers));
1743 } else {
1744 OutputArchivesMap[CompatibleTarget].push_back(
1745 NewArchiveMember(MemBufRef));
1746 }
1747 }
1748 }
1749
1750 if (Error Err = FileHandler->ReadBundleEnd(CodeObjectBuffer))
1751 return Err;
1752
1753 Expected<std::optional<StringRef>> NextTripleOrErr =
1754 FileHandler->ReadBundleStart(CodeObjectBuffer);
1755 if (!NextTripleOrErr)
1756 return NextTripleOrErr.takeError();
1757
1758 CodeObject = ((*NextTripleOrErr).has_value()) ? **NextTripleOrErr : "";
1759 } // End of processing of all bundle entries of this child of input archive.
1760 } // End of while over children of input archive.
1761
1762 assert(!ArchiveErr && "Error occurred while reading archive!");
1763
1764 /// Write out an archive for each target
1765 for (auto &Target : BundlerConfig.TargetNames) {
1766 StringRef FileName = TargetOutputFileNameMap[Target];
1767 StringMapIterator<std::vector<llvm::NewArchiveMember>> CurArchiveMembers =
1768 OutputArchivesMap.find(Target);
1769 if (CurArchiveMembers != OutputArchivesMap.end()) {
1770 if (Error WriteErr = writeArchive(FileName, CurArchiveMembers->getValue(),
1771 SymtabWritingMode::NormalSymtab,
1773 false, nullptr))
1774 return WriteErr;
1775 } else if (!BundlerConfig.AllowMissingBundles) {
1776 std::string ErrMsg =
1777 Twine("no compatible code object found for the target '" + Target +
1778 "' in heterogeneous archive library: " + IFName)
1779 .str();
1780 return createStringError(inconvertibleErrorCode(), ErrMsg);
1781 } else { // Create an empty archive file if no compatible code object is
1782 // found and "allow-missing-bundles" is enabled. It ensures that
1783 // the linker using output of this step doesn't complain about
1784 // the missing input file.
1785 std::vector<llvm::NewArchiveMember> EmptyArchive;
1786 EmptyArchive.clear();
1787 if (Error WriteErr = writeArchive(
1788 FileName, EmptyArchive, SymtabWritingMode::NormalSymtab,
1789 getDefaultArchiveKindForHost(), true, false, nullptr))
1790 return WriteErr;
1791 }
1792 }
1793
1794 return Error::success();
1795}
StringRef P
llvm::MachO::Target Target
Definition: MachO.h:44
static std::string getDeviceLibraryFileName(StringRef BundleFileName, StringRef Device)
static StringRef getDeviceFileExtension(StringRef Device, StringRef BundleFileName)
static Expected< std::unique_ptr< FileHandler > > CreateFileHandler(MemoryBuffer &FirstInput, const OffloadBundlerConfig &BundlerConfig)
Return an appropriate handler given the input files and options.
#define OFFLOAD_BUNDLER_MAGIC_STR
Magic string that marks the existence of offloading data.
bool isCodeObjectCompatible(const OffloadTargetInfo &CodeObjectInfo, const OffloadTargetInfo &TargetInfo)
Checks if a code object CodeObjectInfo is compatible with a given target TargetInfo.
static llvm::TimerGroup ClangOffloadBundlerTimerGroup("Clang Offload Bundler Timer Group", "Timer group for clang offload bundler")
static Error CheckHeterogeneousArchive(StringRef ArchiveName, const OffloadBundlerConfig &BundlerConfig)
static std::unique_ptr< FileHandler > CreateObjectFileHandler(MemoryBuffer &FirstInput, const OffloadBundlerConfig &BundlerConfig)
Return an appropriate object file handler.
static Archive::Kind getDefaultArchiveKindForHost()
static std::string formatWithCommas(unsigned long long Value)
static bool getCompatibleOffloadTargets(OffloadTargetInfo &CodeObjectInfo, SmallVectorImpl< StringRef > &CompatibleTargets, const OffloadBundlerConfig &BundlerConfig)
Computes a list of targets among all given targets which are compatible with this code object.
This file defines an offload bundling API that bundles different files that relate with the same sour...
Defines version macros and version-related utility functions for Clang.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
static llvm::Expected< std::unique_ptr< llvm::MemoryBuffer > > compress(llvm::compression::Params P, const llvm::MemoryBuffer &Input, bool Verbose=false)
static llvm::Expected< std::unique_ptr< llvm::MemoryBuffer > > decompress(const llvm::MemoryBuffer &Input, bool Verbose=false)
llvm::compression::Format CompressionFormat
std::vector< std::string > OutputFileNames
std::vector< std::string > TargetNames
std::vector< std::string > InputFileNames
llvm::Error BundleFiles()
Bundle the files. Return true if an error was found.
llvm::Error UnbundleArchive()
UnbundleArchive takes an archive file (".a") as input containing bundled code object files,...
static llvm::Error ListBundleIDsInFile(llvm::StringRef InputFileName, const OffloadBundlerConfig &BundlerConfig)
const OffloadBundlerConfig & BundlerConfig
Exposes information about the current target.
Definition: TargetInfo.h:213
The JSON file list parser is used to communicate input to InstallAPI.
@ Create
'copyin' clause, allowed on Compute and Combined constructs, plus 'data', 'enter data',...
@ Device
'device' clause, allowed on the 'update' construct.
std::optional< llvm::StringRef > parseTargetID(const llvm::Triple &T, llvm::StringRef OffloadArch, llvm::StringMap< bool > *FeatureMap)
Parse a target ID to get processor and feature map.
Definition: TargetID.cpp:105
std::optional< std::pair< llvm::StringRef, llvm::StringRef > > getConflictTargetIDCombination(const std::set< llvm::StringRef > &TargetIDs)
Get the conflicted pair of target IDs for a compilation or a bundled code object, assuming TargetIDs ...
Definition: TargetID.cpp:145
CudaArch StringToCudaArch(llvm::StringRef S)
Definition: Cuda.cpp:168
@ Result
The result type of a method or function.
unsigned long uint64_t
YAML serialization mapping.
Definition: Dominators.h:30
Definition: Format.h:5378
Obtain the offload kind, real machine triple, and an optional TargetID out of the target information ...
bool operator==(const OffloadTargetInfo &Target) const
bool isOffloadKindCompatible(const llvm::StringRef TargetOffloadKind) const
OffloadTargetInfo(const llvm::StringRef Target, const OffloadBundlerConfig &BC)
llvm::StringRef OffloadKind
std::string str() const
llvm::StringRef TargetID
const OffloadBundlerConfig & BundlerConfig