clang  16.0.0git
OffloadBundler.cpp
Go to the documentation of this file.
1 //===- OffloadBundler.cpp - File Bundling and Unbundling ------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements an offload bundling API that bundles different files
11 /// that relate with the same source code but different targets into a single
12 /// one. Also the implements the opposite functionality, i.e. unbundle files
13 /// previous created by this API.
14 ///
15 //===----------------------------------------------------------------------===//
16 
18 #include "clang/Basic/Cuda.h"
19 #include "clang/Basic/TargetID.h"
20 #include "clang/Basic/Version.h"
21 #include "llvm/ADT/ArrayRef.h"
22 #include "llvm/ADT/SmallString.h"
23 #include "llvm/ADT/SmallVector.h"
24 #include "llvm/ADT/StringMap.h"
25 #include "llvm/ADT/StringRef.h"
26 #include "llvm/ADT/Triple.h"
27 #include "llvm/Object/Archive.h"
28 #include "llvm/Object/ArchiveWriter.h"
29 #include "llvm/Object/Binary.h"
30 #include "llvm/Object/ObjectFile.h"
31 #include "llvm/Support/Casting.h"
32 #include "llvm/Support/CommandLine.h"
33 #include "llvm/Support/Debug.h"
34 #include "llvm/Support/Errc.h"
35 #include "llvm/Support/Error.h"
36 #include "llvm/Support/ErrorOr.h"
37 #include "llvm/Support/FileSystem.h"
38 #include "llvm/Support/Host.h"
39 #include "llvm/Support/MemoryBuffer.h"
40 #include "llvm/Support/Path.h"
41 #include "llvm/Support/Program.h"
42 #include "llvm/Support/Signals.h"
43 #include "llvm/Support/StringSaver.h"
44 #include "llvm/Support/WithColor.h"
45 #include "llvm/Support/raw_ostream.h"
46 #include <algorithm>
47 #include <cassert>
48 #include <cstddef>
49 #include <cstdint>
50 #include <forward_list>
51 #include <memory>
52 #include <set>
53 #include <string>
54 #include <system_error>
55 #include <utility>
56 
57 using namespace llvm;
58 using namespace llvm::object;
59 using namespace clang;
60 
61 /// Magic string that marks the existence of offloading data.
62 #define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
63 
64 OffloadTargetInfo::OffloadTargetInfo(const StringRef Target,
65  const OffloadBundlerConfig &BC)
66  : BundlerConfig(BC) {
67 
68  // TODO: Add error checking from ClangOffloadBundler.cpp
69  auto TargetFeatures = Target.split(':');
70  auto TripleOrGPU = TargetFeatures.first.rsplit('-');
71 
72  if (clang::StringToCudaArch(TripleOrGPU.second) != clang::CudaArch::UNKNOWN) {
73  auto KindTriple = TripleOrGPU.first.split('-');
74  this->OffloadKind = KindTriple.first;
75  this->Triple = llvm::Triple(KindTriple.second);
76  this->TargetID = Target.substr(Target.find(TripleOrGPU.second));
77  } else {
78  auto KindTriple = TargetFeatures.first.split('-');
79  this->OffloadKind = KindTriple.first;
80  this->Triple = llvm::Triple(KindTriple.second);
81  this->TargetID = "";
82  }
83 }
84 
86  return this->OffloadKind == "host";
87 }
88 
90  return OffloadKind == "host" || OffloadKind == "openmp" ||
91  OffloadKind == "hip" || OffloadKind == "hipv4";
92 }
93 
95  const StringRef TargetOffloadKind) const {
96  if (OffloadKind == TargetOffloadKind)
97  return true;
99  bool HIPCompatibleWithOpenMP = OffloadKind.startswith_insensitive("hip") &&
100  TargetOffloadKind == "openmp";
101  bool OpenMPCompatibleWithHIP =
102  OffloadKind == "openmp" &&
103  TargetOffloadKind.startswith_insensitive("hip");
104  return HIPCompatibleWithOpenMP || OpenMPCompatibleWithHIP;
105  }
106  return false;
107 }
108 
110  return !Triple.str().empty() && Triple.getArch() != Triple::UnknownArch;
111 }
112 
114  return OffloadKind == Target.OffloadKind &&
115  Triple.isCompatibleWith(Target.Triple) && TargetID == Target.TargetID;
116 }
117 
119  return Twine(OffloadKind + "-" + Triple.str() + "-" + TargetID).str();
120 }
121 
122 static StringRef getDeviceFileExtension(StringRef Device,
123  StringRef BundleFileName) {
124  if (Device.contains("gfx"))
125  return ".bc";
126  if (Device.contains("sm_"))
127  return ".cubin";
128  return sys::path::extension(BundleFileName);
129 }
130 
131 static std::string getDeviceLibraryFileName(StringRef BundleFileName,
132  StringRef Device) {
133  StringRef LibName = sys::path::stem(BundleFileName);
134  StringRef Extension = getDeviceFileExtension(Device, BundleFileName);
135 
136  std::string Result;
137  Result += LibName;
138  Result += Extension;
139  return Result;
140 }
141 
142 /// @brief Checks if a code object \p CodeObjectInfo is compatible with a given
143 /// target \p TargetInfo.
144 /// @link https://clang.llvm.org/docs/ClangOffloadBundler.html#bundle-entry-id
145 bool isCodeObjectCompatible(const OffloadTargetInfo &CodeObjectInfo,
146  const OffloadTargetInfo &TargetInfo) {
147 
148  // Compatible in case of exact match.
149  if (CodeObjectInfo == TargetInfo) {
150  DEBUG_WITH_TYPE("CodeObjectCompatibility",
151  dbgs() << "Compatible: Exact match: \t[CodeObject: "
152  << CodeObjectInfo.str()
153  << "]\t:\t[Target: " << TargetInfo.str() << "]\n");
154  return true;
155  }
156 
157  // Incompatible if Kinds or Triples mismatch.
158  if (!CodeObjectInfo.isOffloadKindCompatible(TargetInfo.OffloadKind) ||
159  !CodeObjectInfo.Triple.isCompatibleWith(TargetInfo.Triple)) {
160  DEBUG_WITH_TYPE(
161  "CodeObjectCompatibility",
162  dbgs() << "Incompatible: Kind/Triple mismatch \t[CodeObject: "
163  << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str()
164  << "]\n");
165  return false;
166  }
167 
168  // Incompatible if target IDs are incompatible.
169  if (!clang::isCompatibleTargetID(CodeObjectInfo.TargetID,
170  TargetInfo.TargetID)) {
171  DEBUG_WITH_TYPE(
172  "CodeObjectCompatibility",
173  dbgs() << "Incompatible: target IDs are incompatible \t[CodeObject: "
174  << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str()
175  << "]\n");
176  return false;
177  }
178 
179  DEBUG_WITH_TYPE(
180  "CodeObjectCompatibility",
181  dbgs() << "Compatible: Code Objects are compatible \t[CodeObject: "
182  << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str()
183  << "]\n");
184  return true;
185 }
186 
187 /// Generic file handler interface.
188 class FileHandler {
189 public:
190  struct BundleInfo {
191  StringRef BundleID;
192  };
193 
195 
196  virtual ~FileHandler() {}
197 
198  /// Update the file handler with information from the header of the bundled
199  /// file.
200  virtual Error ReadHeader(MemoryBuffer &Input) = 0;
201 
202  /// Read the marker of the next bundled to be read in the file. The bundle
203  /// name is returned if there is one in the file, or `None` if there are no
204  /// more bundles to be read.
206  ReadBundleStart(MemoryBuffer &Input) = 0;
207 
208  /// Read the marker that closes the current bundle.
209  virtual Error ReadBundleEnd(MemoryBuffer &Input) = 0;
210 
211  /// Read the current bundle and write the result into the stream \a OS.
212  virtual Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) = 0;
213 
214  /// Write the header of the bundled file to \a OS based on the information
215  /// gathered from \a Inputs.
216  virtual Error WriteHeader(raw_fd_ostream &OS,
217  ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) = 0;
218 
219  /// Write the marker that initiates a bundle for the triple \a TargetTriple to
220  /// \a OS.
221  virtual Error WriteBundleStart(raw_fd_ostream &OS,
222  StringRef TargetTriple) = 0;
223 
224  /// Write the marker that closes a bundle for the triple \a TargetTriple to \a
225  /// OS.
226  virtual Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) = 0;
227 
228  /// Write the bundle from \a Input into \a OS.
229  virtual Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0;
230 
231  /// List bundle IDs in \a Input.
232  virtual Error listBundleIDs(MemoryBuffer &Input) {
233  if (Error Err = ReadHeader(Input))
234  return Err;
235  return forEachBundle(Input, [&](const BundleInfo &Info) -> Error {
236  llvm::outs() << Info.BundleID << '\n';
237  Error Err = listBundleIDsCallback(Input, Info);
238  if (Err)
239  return Err;
240  return Error::success();
241  });
242  }
243 
244  /// For each bundle in \a Input, do \a Func.
245  Error forEachBundle(MemoryBuffer &Input,
246  std::function<Error(const BundleInfo &)> Func) {
247  while (true) {
248  Expected<Optional<StringRef>> CurTripleOrErr = ReadBundleStart(Input);
249  if (!CurTripleOrErr)
250  return CurTripleOrErr.takeError();
251 
252  // No more bundles.
253  if (!*CurTripleOrErr)
254  break;
255 
256  StringRef CurTriple = **CurTripleOrErr;
257  assert(!CurTriple.empty());
258 
259  BundleInfo Info{CurTriple};
260  if (Error Err = Func(Info))
261  return Err;
262  }
263  return Error::success();
264  }
265 
266 protected:
267  virtual Error listBundleIDsCallback(MemoryBuffer &Input,
268  const BundleInfo &Info) {
269  return Error::success();
270  }
271 };
272 
273 /// Handler for binary files. The bundled file will have the following format
274 /// (all integers are stored in little-endian format):
275 ///
276 /// "OFFLOAD_BUNDLER_MAGIC_STR" (ASCII encoding of the string)
277 ///
278 /// NumberOfOffloadBundles (8-byte integer)
279 ///
280 /// OffsetOfBundle1 (8-byte integer)
281 /// SizeOfBundle1 (8-byte integer)
282 /// NumberOfBytesInTripleOfBundle1 (8-byte integer)
283 /// TripleOfBundle1 (byte length defined before)
284 ///
285 /// ...
286 ///
287 /// OffsetOfBundleN (8-byte integer)
288 /// SizeOfBundleN (8-byte integer)
289 /// NumberOfBytesInTripleOfBundleN (8-byte integer)
290 /// TripleOfBundleN (byte length defined before)
291 ///
292 /// Bundle1
293 /// ...
294 /// BundleN
295 
296 /// Read 8-byte integers from a buffer in little-endian format.
297 static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) {
298  uint64_t Res = 0;
299  const char *Data = Buffer.data();
300 
301  for (unsigned i = 0; i < 8; ++i) {
302  Res <<= 8;
303  uint64_t Char = (uint64_t)Data[pos + 7 - i];
304  Res |= 0xffu & Char;
305  }
306  return Res;
307 }
308 
309 /// Write 8-byte integers to a buffer in little-endian format.
310 static void Write8byteIntegerToBuffer(raw_fd_ostream &OS, uint64_t Val) {
311  for (unsigned i = 0; i < 8; ++i) {
312  char Char = (char)(Val & 0xffu);
313  OS.write(&Char, 1);
314  Val >>= 8;
315  }
316 }
317 
318 class BinaryFileHandler final : public FileHandler {
319  /// Information about the bundles extracted from the header.
320  struct BinaryBundleInfo final : public BundleInfo {
321  /// Size of the bundle.
322  uint64_t Size = 0u;
323  /// Offset at which the bundle starts in the bundled file.
324  uint64_t Offset = 0u;
325 
326  BinaryBundleInfo() {}
327  BinaryBundleInfo(uint64_t Size, uint64_t Offset)
328  : Size(Size), Offset(Offset) {}
329  };
330 
331  /// Map between a triple and the corresponding bundle information.
332  StringMap<BinaryBundleInfo> BundlesInfo;
333 
334  /// Iterator for the bundle information that is being read.
335  StringMap<BinaryBundleInfo>::iterator CurBundleInfo;
336  StringMap<BinaryBundleInfo>::iterator NextBundleInfo;
337 
338  /// Current bundle target to be written.
339  std::string CurWriteBundleTarget;
340 
341  /// Configuration options and arrays for this bundler job
342  const OffloadBundlerConfig &BundlerConfig;
343 
344 public:
345  // TODO: Add error checking from ClangOffloadBundler.cpp
346  BinaryFileHandler(const OffloadBundlerConfig &BC) : BundlerConfig(BC) {}
347 
349 
350  Error ReadHeader(MemoryBuffer &Input) final {
351  StringRef FC = Input.getBuffer();
352 
353  // Initialize the current bundle with the end of the container.
354  CurBundleInfo = BundlesInfo.end();
355 
356  // Check if buffer is smaller than magic string.
357  size_t ReadChars = sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1;
358  if (ReadChars > FC.size())
359  return Error::success();
360 
361  // Check if no magic was found.
362  StringRef Magic(FC.data(), sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1);
363  if (!Magic.equals(OFFLOAD_BUNDLER_MAGIC_STR))
364  return Error::success();
365 
366  // Read number of bundles.
367  if (ReadChars + 8 > FC.size())
368  return Error::success();
369 
370  uint64_t NumberOfBundles = Read8byteIntegerFromBuffer(FC, ReadChars);
371  ReadChars += 8;
372 
373  // Read bundle offsets, sizes and triples.
374  for (uint64_t i = 0; i < NumberOfBundles; ++i) {
375 
376  // Read offset.
377  if (ReadChars + 8 > FC.size())
378  return Error::success();
379 
380  uint64_t Offset = Read8byteIntegerFromBuffer(FC, ReadChars);
381  ReadChars += 8;
382 
383  // Read size.
384  if (ReadChars + 8 > FC.size())
385  return Error::success();
386 
387  uint64_t Size = Read8byteIntegerFromBuffer(FC, ReadChars);
388  ReadChars += 8;
389 
390  // Read triple size.
391  if (ReadChars + 8 > FC.size())
392  return Error::success();
393 
394  uint64_t TripleSize = Read8byteIntegerFromBuffer(FC, ReadChars);
395  ReadChars += 8;
396 
397  // Read triple.
398  if (ReadChars + TripleSize > FC.size())
399  return Error::success();
400 
401  StringRef Triple(&FC.data()[ReadChars], TripleSize);
402  ReadChars += TripleSize;
403 
404  // Check if the offset and size make sense.
405  if (!Offset || Offset + Size > FC.size())
406  return Error::success();
407 
408  assert(BundlesInfo.find(Triple) == BundlesInfo.end() &&
409  "Triple is duplicated??");
410  BundlesInfo[Triple] = BinaryBundleInfo(Size, Offset);
411  }
412  // Set the iterator to where we will start to read.
413  CurBundleInfo = BundlesInfo.end();
414  NextBundleInfo = BundlesInfo.begin();
415  return Error::success();
416  }
417 
418  Expected<Optional<StringRef>> ReadBundleStart(MemoryBuffer &Input) final {
419  if (NextBundleInfo == BundlesInfo.end())
420  return None;
421  CurBundleInfo = NextBundleInfo++;
422  return CurBundleInfo->first();
423  }
424 
425  Error ReadBundleEnd(MemoryBuffer &Input) final {
426  assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
427  return Error::success();
428  }
429 
430  Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
431  assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
432  StringRef FC = Input.getBuffer();
433  OS.write(FC.data() + CurBundleInfo->second.Offset,
434  CurBundleInfo->second.Size);
435  return Error::success();
436  }
437 
438  Error WriteHeader(raw_fd_ostream &OS,
439  ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
440 
441  // Compute size of the header.
442  uint64_t HeaderSize = 0;
443 
444  HeaderSize += sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1;
445  HeaderSize += 8; // Number of Bundles
446 
447  for (auto &T : BundlerConfig.TargetNames) {
448  HeaderSize += 3 * 8; // Bundle offset, Size of bundle and size of triple.
449  HeaderSize += T.size(); // The triple.
450  }
451 
452  // Write to the buffer the header.
454 
455  Write8byteIntegerToBuffer(OS, BundlerConfig.TargetNames.size());
456 
457  unsigned Idx = 0;
458  for (auto &T : BundlerConfig.TargetNames) {
459  MemoryBuffer &MB = *Inputs[Idx++];
460  HeaderSize = alignTo(HeaderSize, BundlerConfig.BundleAlignment);
461  // Bundle offset.
462  Write8byteIntegerToBuffer(OS, HeaderSize);
463  // Size of the bundle (adds to the next bundle's offset)
464  Write8byteIntegerToBuffer(OS, MB.getBufferSize());
465  BundlesInfo[T] = BinaryBundleInfo(MB.getBufferSize(), HeaderSize);
466  HeaderSize += MB.getBufferSize();
467  // Size of the triple
468  Write8byteIntegerToBuffer(OS, T.size());
469  // Triple
470  OS << T;
471  }
472  return Error::success();
473  }
474 
475  Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final {
476  CurWriteBundleTarget = TargetTriple.str();
477  return Error::success();
478  }
479 
480  Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final {
481  return Error::success();
482  }
483 
484  Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
485  auto BI = BundlesInfo[CurWriteBundleTarget];
486  OS.seek(BI.Offset);
487  OS.write(Input.getBufferStart(), Input.getBufferSize());
488  return Error::success();
489  }
490 };
491 
492 namespace {
493 
494 // This class implements a list of temporary files that are removed upon
495 // object destruction.
496 class TempFileHandlerRAII {
497 public:
498  ~TempFileHandlerRAII() {
499  for (const auto &File : Files)
500  sys::fs::remove(File);
501  }
502 
503  // Creates temporary file with given contents.
504  Expected<StringRef> Create(Optional<ArrayRef<char>> Contents) {
506  if (std::error_code EC =
507  sys::fs::createTemporaryFile("clang-offload-bundler", "tmp", File))
508  return createFileError(File, EC);
509  Files.push_front(File);
510 
511  if (Contents) {
512  std::error_code EC;
513  raw_fd_ostream OS(File, EC);
514  if (EC)
515  return createFileError(File, EC);
516  OS.write(Contents->data(), Contents->size());
517  }
518  return Files.front().str();
519  }
520 
521 private:
522  std::forward_list<SmallString<128u>> Files;
523 };
524 
525 } // end anonymous namespace
526 
527 /// Handler for object files. The bundles are organized by sections with a
528 /// designated name.
529 ///
530 /// To unbundle, we just copy the contents of the designated section.
531 class ObjectFileHandler final : public FileHandler {
532 
533  /// The object file we are currently dealing with.
534  std::unique_ptr<ObjectFile> Obj;
535 
536  /// Return the input file contents.
537  StringRef getInputFileContents() const { return Obj->getData(); }
538 
539  /// Return bundle name (<kind>-<triple>) if the provided section is an offload
540  /// section.
541  static Expected<Optional<StringRef>> IsOffloadSection(SectionRef CurSection) {
542  Expected<StringRef> NameOrErr = CurSection.getName();
543  if (!NameOrErr)
544  return NameOrErr.takeError();
545 
546  // If it does not start with the reserved suffix, just skip this section.
547  if (!NameOrErr->startswith(OFFLOAD_BUNDLER_MAGIC_STR))
548  return None;
549 
550  // Return the triple that is right after the reserved prefix.
551  return NameOrErr->substr(sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1);
552  }
553 
554  /// Total number of inputs.
555  unsigned NumberOfInputs = 0;
556 
557  /// Total number of processed inputs, i.e, inputs that were already
558  /// read from the buffers.
559  unsigned NumberOfProcessedInputs = 0;
560 
561  /// Iterator of the current and next section.
562  section_iterator CurrentSection;
563  section_iterator NextSection;
564 
565  /// Configuration options and arrays for this bundler job
566  const OffloadBundlerConfig &BundlerConfig;
567 
568 public:
569  // TODO: Add error checking from ClangOffloadBundler.cpp
570  ObjectFileHandler(std::unique_ptr<ObjectFile> ObjIn,
571  const OffloadBundlerConfig &BC)
572  : Obj(std::move(ObjIn)), CurrentSection(Obj->section_begin()),
573  NextSection(Obj->section_begin()), BundlerConfig(BC) {}
574 
576 
577  Error ReadHeader(MemoryBuffer &Input) final { return Error::success(); }
578 
579  Expected<Optional<StringRef>> ReadBundleStart(MemoryBuffer &Input) final {
580  while (NextSection != Obj->section_end()) {
581  CurrentSection = NextSection;
582  ++NextSection;
583 
584  // Check if the current section name starts with the reserved prefix. If
585  // so, return the triple.
586  Expected<Optional<StringRef>> TripleOrErr =
587  IsOffloadSection(*CurrentSection);
588  if (!TripleOrErr)
589  return TripleOrErr.takeError();
590  if (*TripleOrErr)
591  return **TripleOrErr;
592  }
593  return None;
594  }
595 
596  Error ReadBundleEnd(MemoryBuffer &Input) final { return Error::success(); }
597 
598  Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
599  Expected<StringRef> ContentOrErr = CurrentSection->getContents();
600  if (!ContentOrErr)
601  return ContentOrErr.takeError();
602  StringRef Content = *ContentOrErr;
603 
604  // Copy fat object contents to the output when extracting host bundle.
605  if (Content.size() == 1u && Content.front() == 0)
606  Content = StringRef(Input.getBufferStart(), Input.getBufferSize());
607 
608  OS.write(Content.data(), Content.size());
609  return Error::success();
610  }
611 
612  Error WriteHeader(raw_fd_ostream &OS,
613  ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
614  assert(BundlerConfig.HostInputIndex != ~0u &&
615  "Host input index not defined.");
616 
617  // Record number of inputs.
618  NumberOfInputs = Inputs.size();
619  return Error::success();
620  }
621 
622  Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final {
623  ++NumberOfProcessedInputs;
624  return Error::success();
625  }
626 
627  Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final {
628  assert(NumberOfProcessedInputs <= NumberOfInputs &&
629  "Processing more inputs that actually exist!");
630  assert(BundlerConfig.HostInputIndex != ~0u &&
631  "Host input index not defined.");
632 
633  // If this is not the last output, we don't have to do anything.
634  if (NumberOfProcessedInputs != NumberOfInputs)
635  return Error::success();
636 
637  // We will use llvm-objcopy to add target objects sections to the output
638  // fat object. These sections should have 'exclude' flag set which tells
639  // link editor to remove them from linker inputs when linking executable or
640  // shared library.
641 
642  assert(BundlerConfig.ObjcopyPath != "" &&
643  "llvm-objcopy path not specified");
644 
645  // We write to the output file directly. So, we close it and use the name
646  // to pass down to llvm-objcopy.
647  OS.close();
648 
649  // Temporary files that need to be removed.
650  TempFileHandlerRAII TempFiles;
651 
652  // Compose llvm-objcopy command line for add target objects' sections with
653  // appropriate flags.
654  BumpPtrAllocator Alloc;
655  StringSaver SS{Alloc};
656  SmallVector<StringRef, 8u> ObjcopyArgs{"llvm-objcopy"};
657 
658  for (unsigned I = 0; I < NumberOfInputs; ++I) {
659  StringRef InputFile = BundlerConfig.InputFileNames[I];
660  if (I == BundlerConfig.HostInputIndex) {
661  // Special handling for the host bundle. We do not need to add a
662  // standard bundle for the host object since we are going to use fat
663  // object as a host object. Therefore use dummy contents (one zero byte)
664  // when creating section for the host bundle.
665  Expected<StringRef> TempFileOrErr = TempFiles.Create(ArrayRef<char>(0));
666  if (!TempFileOrErr)
667  return TempFileOrErr.takeError();
668  InputFile = *TempFileOrErr;
669  }
670 
671  ObjcopyArgs.push_back(
672  SS.save(Twine("--add-section=") + OFFLOAD_BUNDLER_MAGIC_STR +
673  BundlerConfig.TargetNames[I] + "=" + InputFile));
674  ObjcopyArgs.push_back(
675  SS.save(Twine("--set-section-flags=") + OFFLOAD_BUNDLER_MAGIC_STR +
676  BundlerConfig.TargetNames[I] + "=readonly,exclude"));
677  }
678  ObjcopyArgs.push_back("--");
679  ObjcopyArgs.push_back(
680  BundlerConfig.InputFileNames[BundlerConfig.HostInputIndex]);
681  ObjcopyArgs.push_back(BundlerConfig.OutputFileNames.front());
682 
683  if (Error Err = executeObjcopy(BundlerConfig.ObjcopyPath, ObjcopyArgs))
684  return Err;
685 
686  return Error::success();
687  }
688 
689  Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
690  return Error::success();
691  }
692 
693 private:
694  Error executeObjcopy(StringRef Objcopy, ArrayRef<StringRef> Args) {
695  // If the user asked for the commands to be printed out, we do that
696  // instead of executing it.
697  if (BundlerConfig.PrintExternalCommands) {
698  errs() << "\"" << Objcopy << "\"";
699  for (StringRef Arg : drop_begin(Args, 1))
700  errs() << " \"" << Arg << "\"";
701  errs() << "\n";
702  } else {
703  if (sys::ExecuteAndWait(Objcopy, Args))
704  return createStringError(inconvertibleErrorCode(),
705  "'llvm-objcopy' tool failed");
706  }
707  return Error::success();
708  }
709 };
710 
711 /// Handler for text files. The bundled file will have the following format.
712 ///
713 /// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
714 /// Bundle 1
715 /// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
716 /// ...
717 /// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
718 /// Bundle N
719 /// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
720 class TextFileHandler final : public FileHandler {
721  /// String that begins a line comment.
722  StringRef Comment;
723 
724  /// String that initiates a bundle.
725  std::string BundleStartString;
726 
727  /// String that closes a bundle.
728  std::string BundleEndString;
729 
730  /// Number of chars read from input.
731  size_t ReadChars = 0u;
732 
733 protected:
734  Error ReadHeader(MemoryBuffer &Input) final { return Error::success(); }
735 
736  Expected<Optional<StringRef>> ReadBundleStart(MemoryBuffer &Input) final {
737  StringRef FC = Input.getBuffer();
738 
739  // Find start of the bundle.
740  ReadChars = FC.find(BundleStartString, ReadChars);
741  if (ReadChars == FC.npos)
742  return None;
743 
744  // Get position of the triple.
745  size_t TripleStart = ReadChars = ReadChars + BundleStartString.size();
746 
747  // Get position that closes the triple.
748  size_t TripleEnd = ReadChars = FC.find("\n", ReadChars);
749  if (TripleEnd == FC.npos)
750  return None;
751 
752  // Next time we read after the new line.
753  ++ReadChars;
754 
755  return StringRef(&FC.data()[TripleStart], TripleEnd - TripleStart);
756  }
757 
758  Error ReadBundleEnd(MemoryBuffer &Input) final {
759  StringRef FC = Input.getBuffer();
760 
761  // Read up to the next new line.
762  assert(FC[ReadChars] == '\n' && "The bundle should end with a new line.");
763 
764  size_t TripleEnd = ReadChars = FC.find("\n", ReadChars + 1);
765  if (TripleEnd != FC.npos)
766  // Next time we read after the new line.
767  ++ReadChars;
768 
769  return Error::success();
770  }
771 
772  Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
773  StringRef FC = Input.getBuffer();
774  size_t BundleStart = ReadChars;
775 
776  // Find end of the bundle.
777  size_t BundleEnd = ReadChars = FC.find(BundleEndString, ReadChars);
778 
779  StringRef Bundle(&FC.data()[BundleStart], BundleEnd - BundleStart);
780  OS << Bundle;
781 
782  return Error::success();
783  }
784 
785  Error WriteHeader(raw_fd_ostream &OS,
786  ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
787  return Error::success();
788  }
789 
790  Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final {
791  OS << BundleStartString << TargetTriple << "\n";
792  return Error::success();
793  }
794 
795  Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final {
796  OS << BundleEndString << TargetTriple << "\n";
797  return Error::success();
798  }
799 
800  Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
801  OS << Input.getBuffer();
802  return Error::success();
803  }
804 
805 public:
806  TextFileHandler(StringRef Comment) : Comment(Comment), ReadChars(0) {
807  BundleStartString =
808  "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__START__ ";
809  BundleEndString =
810  "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__END__ ";
811  }
812 
813  Error listBundleIDsCallback(MemoryBuffer &Input,
814  const BundleInfo &Info) final {
815  // TODO: To list bundle IDs in a bundled text file we need to go through
816  // all bundles. The format of bundled text file may need to include a
817  // header if the performance of listing bundle IDs of bundled text file is
818  // important.
819  ReadChars = Input.getBuffer().find(BundleEndString, ReadChars);
820  if (Error Err = ReadBundleEnd(Input))
821  return Err;
822  return Error::success();
823  }
824 };
825 
826 /// Return an appropriate object file handler. We use the specific object
827 /// handler if we know how to deal with that format, otherwise we use a default
828 /// binary file handler.
829 static std::unique_ptr<FileHandler>
830 CreateObjectFileHandler(MemoryBuffer &FirstInput,
831  const OffloadBundlerConfig &BundlerConfig) {
832  // Check if the input file format is one that we know how to deal with.
833  Expected<std::unique_ptr<Binary>> BinaryOrErr = createBinary(FirstInput);
834 
835  // We only support regular object files. If failed to open the input as a
836  // known binary or this is not an object file use the default binary handler.
837  if (errorToBool(BinaryOrErr.takeError()) || !isa<ObjectFile>(*BinaryOrErr))
838  return std::make_unique<BinaryFileHandler>(BundlerConfig);
839 
840  // Otherwise create an object file handler. The handler will be owned by the
841  // client of this function.
842  return std::make_unique<ObjectFileHandler>(
843  std::unique_ptr<ObjectFile>(cast<ObjectFile>(BinaryOrErr->release())),
844  BundlerConfig);
845 }
846 
847 /// Return an appropriate handler given the input files and options.
849 CreateFileHandler(MemoryBuffer &FirstInput,
850  const OffloadBundlerConfig &BundlerConfig) {
851  std::string FilesType = BundlerConfig.FilesType;
852 
853  if (FilesType == "i")
854  return std::make_unique<TextFileHandler>(/*Comment=*/"//");
855  if (FilesType == "ii")
856  return std::make_unique<TextFileHandler>(/*Comment=*/"//");
857  if (FilesType == "cui")
858  return std::make_unique<TextFileHandler>(/*Comment=*/"//");
859  // TODO: `.d` should be eventually removed once `-M` and its variants are
860  // handled properly in offload compilation.
861  if (FilesType == "d")
862  return std::make_unique<TextFileHandler>(/*Comment=*/"#");
863  if (FilesType == "ll")
864  return std::make_unique<TextFileHandler>(/*Comment=*/";");
865  if (FilesType == "bc")
866  return std::make_unique<BinaryFileHandler>(BundlerConfig);
867  if (FilesType == "s")
868  return std::make_unique<TextFileHandler>(/*Comment=*/"#");
869  if (FilesType == "o")
870  return CreateObjectFileHandler(FirstInput, BundlerConfig);
871  if (FilesType == "a")
872  return CreateObjectFileHandler(FirstInput, BundlerConfig);
873  if (FilesType == "gch")
874  return std::make_unique<BinaryFileHandler>(BundlerConfig);
875  if (FilesType == "ast")
876  return std::make_unique<BinaryFileHandler>(BundlerConfig);
877 
878  return createStringError(errc::invalid_argument,
879  "'" + FilesType + "': invalid file type specified");
880 }
881 
882 // List bundle IDs. Return true if an error was found.
884  StringRef InputFileName, const OffloadBundlerConfig &BundlerConfig) {
885  // Open Input file.
886  ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
887  MemoryBuffer::getFileOrSTDIN(InputFileName);
888  if (std::error_code EC = CodeOrErr.getError())
889  return createFileError(InputFileName, EC);
890 
891  MemoryBuffer &Input = **CodeOrErr;
892 
893  // Select the right files handler.
894  Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
896  if (!FileHandlerOrErr)
897  return FileHandlerOrErr.takeError();
898 
899  std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr;
900  assert(FH);
901  return FH->listBundleIDs(Input);
902 }
903 
904 /// Bundle the files. Return true if an error was found.
906  std::error_code EC;
907 
908  // Create output file.
909  raw_fd_ostream OutputFile(BundlerConfig.OutputFileNames.front(), EC,
910  sys::fs::OF_None);
911  if (EC)
912  return createFileError(BundlerConfig.OutputFileNames.front(), EC);
913 
914  // Open input files.
916  InputBuffers.reserve(BundlerConfig.InputFileNames.size());
917  for (auto &I : BundlerConfig.InputFileNames) {
918  ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
919  MemoryBuffer::getFileOrSTDIN(I);
920  if (std::error_code EC = CodeOrErr.getError())
921  return createFileError(I, EC);
922  InputBuffers.emplace_back(std::move(*CodeOrErr));
923  }
924 
925  // Get the file handler. We use the host buffer as reference.
927  "Host input index undefined??");
929  *InputBuffers[BundlerConfig.AllowNoHost ? 0
931  BundlerConfig);
932  if (!FileHandlerOrErr)
933  return FileHandlerOrErr.takeError();
934 
935  std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr;
936  assert(FH);
937 
938  // Write header.
939  if (Error Err = FH->WriteHeader(OutputFile, InputBuffers))
940  return Err;
941 
942  // Write all bundles along with the start/end markers. If an error was found
943  // writing the end of the bundle component, abort the bundle writing.
944  auto Input = InputBuffers.begin();
945  for (auto &Triple : BundlerConfig.TargetNames) {
946  if (Error Err = FH->WriteBundleStart(OutputFile, Triple))
947  return Err;
948  if (Error Err = FH->WriteBundle(OutputFile, **Input))
949  return Err;
950  if (Error Err = FH->WriteBundleEnd(OutputFile, Triple))
951  return Err;
952  ++Input;
953  }
954  return Error::success();
955 }
956 
957 // Unbundle the files. Return true if an error was found.
959  // Open Input file.
960  ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
961  MemoryBuffer::getFileOrSTDIN(BundlerConfig.InputFileNames.front());
962  if (std::error_code EC = CodeOrErr.getError())
963  return createFileError(BundlerConfig.InputFileNames.front(), EC);
964 
965  MemoryBuffer &Input = **CodeOrErr;
966 
967  // Select the right files handler.
968  Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
970  if (!FileHandlerOrErr)
971  return FileHandlerOrErr.takeError();
972 
973  std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr;
974  assert(FH);
975 
976  // Read the header of the bundled file.
977  if (Error Err = FH->ReadHeader(Input))
978  return Err;
979 
980  // Create a work list that consist of the map triple/output file.
981  StringMap<StringRef> Worklist;
982  auto Output = BundlerConfig.OutputFileNames.begin();
983  for (auto &Triple : BundlerConfig.TargetNames) {
984  Worklist[Triple] = *Output;
985  ++Output;
986  }
987 
988  // Read all the bundles that are in the work list. If we find no bundles we
989  // assume the file is meant for the host target.
990  bool FoundHostBundle = false;
991  while (!Worklist.empty()) {
992  Expected<Optional<StringRef>> CurTripleOrErr = FH->ReadBundleStart(Input);
993  if (!CurTripleOrErr)
994  return CurTripleOrErr.takeError();
995 
996  // We don't have more bundles.
997  if (!*CurTripleOrErr)
998  break;
999 
1000  StringRef CurTriple = **CurTripleOrErr;
1001  assert(!CurTriple.empty());
1002 
1003  auto Output = Worklist.begin();
1004  for (auto E = Worklist.end(); Output != E; Output++) {
1006  OffloadTargetInfo(CurTriple, BundlerConfig),
1007  OffloadTargetInfo((*Output).first(), BundlerConfig))) {
1008  break;
1009  }
1010  }
1011 
1012  if (Output == Worklist.end())
1013  continue;
1014  // Check if the output file can be opened and copy the bundle to it.
1015  std::error_code EC;
1016  raw_fd_ostream OutputFile((*Output).second, EC, sys::fs::OF_None);
1017  if (EC)
1018  return createFileError((*Output).second, EC);
1019  if (Error Err = FH->ReadBundle(OutputFile, Input))
1020  return Err;
1021  if (Error Err = FH->ReadBundleEnd(Input))
1022  return Err;
1023  Worklist.erase(Output);
1024 
1025  // Record if we found the host bundle.
1026  auto OffloadInfo = OffloadTargetInfo(CurTriple, BundlerConfig);
1027  if (OffloadInfo.hasHostKind())
1028  FoundHostBundle = true;
1029  }
1030 
1031  if (!BundlerConfig.AllowMissingBundles && !Worklist.empty()) {
1032  std::string ErrMsg = "Can't find bundles for";
1033  std::set<StringRef> Sorted;
1034  for (auto &E : Worklist)
1035  Sorted.insert(E.first());
1036  unsigned I = 0;
1037  unsigned Last = Sorted.size() - 1;
1038  for (auto &E : Sorted) {
1039  if (I != 0 && Last > 1)
1040  ErrMsg += ",";
1041  ErrMsg += " ";
1042  if (I == Last && I != 0)
1043  ErrMsg += "and ";
1044  ErrMsg += E.str();
1045  ++I;
1046  }
1047  return createStringError(inconvertibleErrorCode(), ErrMsg);
1048  }
1049 
1050  // If no bundles were found, assume the input file is the host bundle and
1051  // create empty files for the remaining targets.
1052  if (Worklist.size() == BundlerConfig.TargetNames.size()) {
1053  for (auto &E : Worklist) {
1054  std::error_code EC;
1055  raw_fd_ostream OutputFile(E.second, EC, sys::fs::OF_None);
1056  if (EC)
1057  return createFileError(E.second, EC);
1058 
1059  // If this entry has a host kind, copy the input file to the output file.
1060  auto OffloadInfo = OffloadTargetInfo(E.getKey(), BundlerConfig);
1061  if (OffloadInfo.hasHostKind())
1062  OutputFile.write(Input.getBufferStart(), Input.getBufferSize());
1063  }
1064  return Error::success();
1065  }
1066 
1067  // If we found elements, we emit an error if none of those were for the host
1068  // in case host bundle name was provided in command line.
1069  if (!FoundHostBundle && BundlerConfig.HostInputIndex != ~0u)
1070  return createStringError(inconvertibleErrorCode(),
1071  "Can't find bundle for the host target");
1072 
1073  // If we still have any elements in the worklist, create empty files for them.
1074  for (auto &E : Worklist) {
1075  std::error_code EC;
1076  raw_fd_ostream OutputFile(E.second, EC, sys::fs::OF_None);
1077  if (EC)
1078  return createFileError(E.second, EC);
1079  }
1080 
1081  return Error::success();
1082 }
1083 
1085  return Triple(sys::getDefaultTargetTriple()).isOSDarwin() ? Archive::K_DARWIN
1086  : Archive::K_GNU;
1087 }
1088 
1089 /// @brief Computes a list of targets among all given targets which are
1090 /// compatible with this code object
1091 /// @param [in] CodeObjectInfo Code Object
1092 /// @param [out] CompatibleTargets List of all compatible targets among all
1093 /// given targets
1094 /// @return false, if no compatible target is found.
1095 static bool
1097  SmallVectorImpl<StringRef> &CompatibleTargets,
1098  const OffloadBundlerConfig &BundlerConfig) {
1099  if (!CompatibleTargets.empty()) {
1100  DEBUG_WITH_TYPE("CodeObjectCompatibility",
1101  dbgs() << "CompatibleTargets list should be empty\n");
1102  return false;
1103  }
1104  for (auto &Target : BundlerConfig.TargetNames) {
1105  auto TargetInfo = OffloadTargetInfo(Target, BundlerConfig);
1106  if (isCodeObjectCompatible(CodeObjectInfo, TargetInfo))
1107  CompatibleTargets.push_back(Target);
1108  }
1109  return !CompatibleTargets.empty();
1110 }
1111 
1112 /// UnbundleArchive takes an archive file (".a") as input containing bundled
1113 /// code object files, and a list of offload targets (not host), and extracts
1114 /// the code objects into a new archive file for each offload target. Each
1115 /// resulting archive file contains all code object files corresponding to that
1116 /// particular offload target. The created archive file does not
1117 /// contain an index of the symbols and code object files are named as
1118 /// <<Parent Bundle Name>-<CodeObject's GPUArch>>, with ':' replaced with '_'.
1120  std::vector<std::unique_ptr<MemoryBuffer>> ArchiveBuffers;
1121 
1122  /// Map of target names with list of object files that will form the device
1123  /// specific archive for that target
1124  StringMap<std::vector<NewArchiveMember>> OutputArchivesMap;
1125 
1126  // Map of target names and output archive filenames
1127  StringMap<StringRef> TargetOutputFileNameMap;
1128 
1129  auto Output = BundlerConfig.OutputFileNames.begin();
1130  for (auto &Target : BundlerConfig.TargetNames) {
1131  TargetOutputFileNameMap[Target] = *Output;
1132  ++Output;
1133  }
1134 
1135  StringRef IFName = BundlerConfig.InputFileNames.front();
1136 
1137  ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
1138  MemoryBuffer::getFileOrSTDIN(IFName, true, false);
1139  if (std::error_code EC = BufOrErr.getError())
1140  return createFileError(BundlerConfig.InputFileNames.front(), EC);
1141 
1142  ArchiveBuffers.push_back(std::move(*BufOrErr));
1144  Archive::create(ArchiveBuffers.back()->getMemBufferRef());
1145  if (!LibOrErr)
1146  return LibOrErr.takeError();
1147 
1148  auto Archive = std::move(*LibOrErr);
1149 
1150  Error ArchiveErr = Error::success();
1151  auto ChildEnd = Archive->child_end();
1152 
1153  /// Iterate over all bundled code object files in the input archive.
1154  for (auto ArchiveIter = Archive->child_begin(ArchiveErr);
1155  ArchiveIter != ChildEnd; ++ArchiveIter) {
1156  if (ArchiveErr)
1157  return ArchiveErr;
1158  auto ArchiveChildNameOrErr = (*ArchiveIter).getName();
1159  if (!ArchiveChildNameOrErr)
1160  return ArchiveChildNameOrErr.takeError();
1161 
1162  StringRef BundledObjectFile = sys::path::filename(*ArchiveChildNameOrErr);
1163 
1164  auto CodeObjectBufferRefOrErr = (*ArchiveIter).getMemoryBufferRef();
1165  if (!CodeObjectBufferRefOrErr)
1166  return CodeObjectBufferRefOrErr.takeError();
1167 
1168  auto CodeObjectBuffer =
1169  MemoryBuffer::getMemBuffer(*CodeObjectBufferRefOrErr, false);
1170 
1171  Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
1172  CreateFileHandler(*CodeObjectBuffer, BundlerConfig);
1173  if (!FileHandlerOrErr)
1174  return FileHandlerOrErr.takeError();
1175 
1176  std::unique_ptr<FileHandler> &FileHandler = *FileHandlerOrErr;
1177  assert(FileHandler &&
1178  "FileHandle creation failed for file in the archive!");
1179 
1180  if (Error ReadErr = FileHandler.get()->ReadHeader(*CodeObjectBuffer))
1181  return ReadErr;
1182 
1183  Expected<Optional<StringRef>> CurBundleIDOrErr =
1184  FileHandler->ReadBundleStart(*CodeObjectBuffer);
1185  if (!CurBundleIDOrErr)
1186  return CurBundleIDOrErr.takeError();
1187 
1188  Optional<StringRef> OptionalCurBundleID = *CurBundleIDOrErr;
1189  // No device code in this child, skip.
1190  if (!OptionalCurBundleID)
1191  continue;
1192  StringRef CodeObject = *OptionalCurBundleID;
1193 
1194  // Process all bundle entries (CodeObjects) found in this child of input
1195  // archive.
1196  while (!CodeObject.empty()) {
1197  SmallVector<StringRef> CompatibleTargets;
1198  auto CodeObjectInfo = OffloadTargetInfo(CodeObject, BundlerConfig);
1199  if (CodeObjectInfo.hasHostKind()) {
1200  // Do nothing, we don't extract host code yet.
1201  } else if (getCompatibleOffloadTargets(CodeObjectInfo, CompatibleTargets,
1202  BundlerConfig)) {
1203  std::string BundleData;
1204  raw_string_ostream DataStream(BundleData);
1205  if (Error Err =
1206  FileHandler.get()->ReadBundle(DataStream, *CodeObjectBuffer))
1207  return Err;
1208 
1209  for (auto &CompatibleTarget : CompatibleTargets) {
1210  SmallString<128> BundledObjectFileName;
1211  BundledObjectFileName.assign(BundledObjectFile);
1212  auto OutputBundleName =
1213  Twine(llvm::sys::path::stem(BundledObjectFileName) + "-" +
1214  CodeObject +
1215  getDeviceLibraryFileName(BundledObjectFileName,
1216  CodeObjectInfo.TargetID))
1217  .str();
1218  // Replace ':' in optional target feature list with '_' to ensure
1219  // cross-platform validity.
1220  std::replace(OutputBundleName.begin(), OutputBundleName.end(), ':',
1221  '_');
1222 
1223  std::unique_ptr<MemoryBuffer> MemBuf = MemoryBuffer::getMemBufferCopy(
1224  DataStream.str(), OutputBundleName);
1225  ArchiveBuffers.push_back(std::move(MemBuf));
1226  llvm::MemoryBufferRef MemBufRef =
1227  MemoryBufferRef(*(ArchiveBuffers.back()));
1228 
1229  // For inserting <CompatibleTarget, list<CodeObject>> entry in
1230  // OutputArchivesMap.
1231  if (OutputArchivesMap.find(CompatibleTarget) ==
1232  OutputArchivesMap.end()) {
1233 
1234  std::vector<NewArchiveMember> ArchiveMembers;
1235  ArchiveMembers.push_back(NewArchiveMember(MemBufRef));
1236  OutputArchivesMap.insert_or_assign(CompatibleTarget,
1237  std::move(ArchiveMembers));
1238  } else {
1239  OutputArchivesMap[CompatibleTarget].push_back(
1240  NewArchiveMember(MemBufRef));
1241  }
1242  }
1243  }
1244 
1245  if (Error Err = FileHandler.get()->ReadBundleEnd(*CodeObjectBuffer))
1246  return Err;
1247 
1248  Expected<Optional<StringRef>> NextTripleOrErr =
1249  FileHandler->ReadBundleStart(*CodeObjectBuffer);
1250  if (!NextTripleOrErr)
1251  return NextTripleOrErr.takeError();
1252 
1253  CodeObject = ((*NextTripleOrErr).has_value()) ? **NextTripleOrErr : "";
1254  } // End of processing of all bundle entries of this child of input archive.
1255  } // End of while over children of input archive.
1256 
1257  assert(!ArchiveErr && "Error occurred while reading archive!");
1258 
1259  /// Write out an archive for each target
1260  for (auto &Target : BundlerConfig.TargetNames) {
1261  StringRef FileName = TargetOutputFileNameMap[Target];
1262  StringMapIterator<std::vector<llvm::NewArchiveMember>> CurArchiveMembers =
1263  OutputArchivesMap.find(Target);
1264  if (CurArchiveMembers != OutputArchivesMap.end()) {
1265  if (Error WriteErr = writeArchive(FileName, CurArchiveMembers->getValue(),
1267  true, false, nullptr))
1268  return WriteErr;
1269  } else if (!BundlerConfig.AllowMissingBundles) {
1270  std::string ErrMsg =
1271  Twine("no compatible code object found for the target '" + Target +
1272  "' in heterogeneous archive library: " + IFName)
1273  .str();
1274  return createStringError(inconvertibleErrorCode(), ErrMsg);
1275  } else { // Create an empty archive file if no compatible code object is
1276  // found and "allow-missing-bundles" is enabled. It ensures that
1277  // the linker using output of this step doesn't complain about
1278  // the missing input file.
1279  std::vector<llvm::NewArchiveMember> EmptyArchive;
1280  EmptyArchive.clear();
1281  if (Error WriteErr = writeArchive(FileName, EmptyArchive, true,
1283  false, nullptr))
1284  return WriteErr;
1285  }
1286  }
1287 
1288  return Error::success();
1289 }
ObjectFileHandler::ReadBundleStart
Expected< Optional< StringRef > > ReadBundleStart(MemoryBuffer &Input) final
Read the marker of the next bundled to be read in the file.
Definition: OffloadBundler.cpp:579
Write8byteIntegerToBuffer
static void Write8byteIntegerToBuffer(raw_fd_ostream &OS, uint64_t Val)
Write 8-byte integers to a buffer in little-endian format.
Definition: OffloadBundler.cpp:310
CreateObjectFileHandler
static std::unique_ptr< FileHandler > CreateObjectFileHandler(MemoryBuffer &FirstInput, const OffloadBundlerConfig &BundlerConfig)
Return an appropriate object file handler.
Definition: OffloadBundler.cpp:830
clang::OffloadBundlerConfig::AllowNoHost
bool AllowNoHost
Definition: OffloadBundler.h:29
llvm
YAML serialization mapping.
Definition: Dominators.h:30
clang::OffloadBundlerConfig::HostInputIndex
unsigned HostInputIndex
Definition: OffloadBundler.h:36
clang::OffloadBundler::BundlerConfig
const OffloadBundlerConfig & BundlerConfig
Definition: OffloadBundler.h:49
clang::OffloadBundler::UnbundleArchive
llvm::Error UnbundleArchive()
UnbundleArchive takes an archive file (".a") as input containing bundled code object files,...
Definition: OffloadBundler.cpp:1119
clang::OffloadBundlerConfig::InputFileNames
std::vector< std::string > InputFileNames
Definition: OffloadBundler.h:43
clang::DeclaratorContext::File
@ File
Error
llvm::Error Error
Definition: ByteCodeEmitter.cpp:20
BinaryFileHandler::WriteBundle
Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final
Write the bundle from Input into OS.
Definition: OffloadBundler.cpp:484
string
string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} 0 ${PATH_LIB_START} PATH_HEAD) string(SUBSTRING $
Definition: CMakeLists.txt:22
clang::OffloadTargetInfo::OffloadKind
llvm::StringRef OffloadKind
Definition: OffloadBundler.h:71
clang::OffloadTargetInfo::isOffloadKindValid
bool isOffloadKindValid() const
Definition: OffloadBundler.cpp:89
llvm::SmallVector
Definition: LLVM.h:38
clang::OffloadBundlerConfig::OutputFileNames
std::vector< std::string > OutputFileNames
Definition: OffloadBundler.h:44
clang::OffloadBundler::UnbundleFiles
llvm::Error UnbundleFiles()
Definition: OffloadBundler.cpp:958
TextFileHandler::WriteBundleEnd
Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final
Write the marker that closes a bundle for the triple TargetTriple to OS.
Definition: OffloadBundler.cpp:795
clang::MultiVersionKind::Target
@ Target
TextFileHandler
Handler for text files.
Definition: OffloadBundler.cpp:720
BinaryFileHandler::ReadBundleEnd
Error ReadBundleEnd(MemoryBuffer &Input) final
Read the marker that closes the current bundle.
Definition: OffloadBundler.cpp:425
BinaryFileHandler::ReadBundle
Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final
Read the current bundle and write the result into the stream OS.
Definition: OffloadBundler.cpp:430
clang::OffloadTargetInfo::TargetID
llvm::StringRef TargetID
Definition: OffloadBundler.h:73
clang::TargetInfo
Exposes information about the current target.
Definition: TargetInfo.h:205
clang::OffloadBundler::BundleFiles
llvm::Error BundleFiles()
Bundle the files. Return true if an error was found.
Definition: OffloadBundler.cpp:905
clang::StringToCudaArch
CudaArch StringToCudaArch(llvm::StringRef S)
Definition: Cuda.cpp:158
BinaryFileHandler::WriteBundleEnd
Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final
Write the marker that closes a bundle for the triple TargetTriple to OS.
Definition: OffloadBundler.cpp:480
llvm::Optional
Definition: LLVM.h:40
clang::OffloadBundlerConfig::HipOpenmpCompatible
bool HipOpenmpCompatible
Definition: OffloadBundler.h:33
BinaryFileHandler::WriteBundleStart
Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final
Write the marker that initiates a bundle for the triple TargetTriple to OS.
Definition: OffloadBundler.cpp:475
llvm::Expected
Definition: LLVM.h:41
clang::OffloadTargetInfo::BundlerConfig
const OffloadBundlerConfig & BundlerConfig
Definition: OffloadBundler.h:75
Read8byteIntegerFromBuffer
static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos)
Handler for binary files.
Definition: OffloadBundler.cpp:297
Offset
unsigned Offset
Definition: Format.cpp:2717
ObjectFileHandler
Handler for object files.
Definition: OffloadBundler.cpp:531
ObjectFileHandler::WriteBundle
Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final
Write the bundle from Input into OS.
Definition: OffloadBundler.cpp:689
clang::OffloadBundlerConfig::AllowMissingBundles
bool AllowMissingBundles
Definition: OffloadBundler.h:30
TextFileHandler::ReadBundleStart
Expected< Optional< StringRef > > ReadBundleStart(MemoryBuffer &Input) final
Read the marker of the next bundled to be read in the file.
Definition: OffloadBundler.cpp:736
hlsl::uint64_t
unsigned long uint64_t
Definition: hlsl_basic_types.h:25
BinaryFileHandler::WriteHeader
Error WriteHeader(raw_fd_ostream &OS, ArrayRef< std::unique_ptr< MemoryBuffer >> Inputs) final
Write the header of the bundled file to OS based on the information gathered from Inputs.
Definition: OffloadBundler.cpp:438
Version.h
clang::OffloadBundlerConfig
Definition: OffloadBundler.h:27
OFFLOAD_BUNDLER_MAGIC_STR
#define OFFLOAD_BUNDLER_MAGIC_STR
Magic string that marks the existence of offloading data.
Definition: OffloadBundler.cpp:62
ObjectFileHandler::ObjectFileHandler
ObjectFileHandler(std::unique_ptr< ObjectFile > ObjIn, const OffloadBundlerConfig &BC)
Definition: OffloadBundler.cpp:570
ObjectFileHandler::WriteBundleStart
Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final
Write the marker that initiates a bundle for the triple TargetTriple to OS.
Definition: OffloadBundler.cpp:622
FileHandler::ReadBundleEnd
virtual Error ReadBundleEnd(MemoryBuffer &Input)=0
Read the marker that closes the current bundle.
clang::OffloadTargetInfo::operator==
bool operator==(const OffloadTargetInfo &Target) const
Definition: OffloadBundler.cpp:113
FileHandler::BundleInfo
Definition: OffloadBundler.cpp:190
getDeviceLibraryFileName
static std::string getDeviceLibraryFileName(StringRef BundleFileName, StringRef Device)
Definition: OffloadBundler.cpp:131
llvm::SmallString
Definition: LLVM.h:37
TargetID.h
BinaryFileHandler::~BinaryFileHandler
~BinaryFileHandler() final
Definition: OffloadBundler.cpp:348
FileHandler::~FileHandler
virtual ~FileHandler()
Definition: OffloadBundler.cpp:196
FileHandler::listBundleIDs
virtual Error listBundleIDs(MemoryBuffer &Input)
List bundle IDs in Input.
Definition: OffloadBundler.cpp:232
getDeviceFileExtension
static StringRef getDeviceFileExtension(StringRef Device, StringRef BundleFileName)
Definition: OffloadBundler.cpp:122
OffloadBundler.h
FileHandler::ReadBundleStart
virtual Expected< Optional< StringRef > > ReadBundleStart(MemoryBuffer &Input)=0
Read the marker of the next bundled to be read in the file.
ObjectFileHandler::~ObjectFileHandler
~ObjectFileHandler() final
Definition: OffloadBundler.cpp:575
BinaryFileHandler::ReadHeader
Error ReadHeader(MemoryBuffer &Input) final
Update the file handler with information from the header of the bundled file.
Definition: OffloadBundler.cpp:350
BinaryFileHandler::BinaryFileHandler
BinaryFileHandler(const OffloadBundlerConfig &BC)
Definition: OffloadBundler.cpp:346
FileHandler
Generic file handler interface.
Definition: OffloadBundler.cpp:188
clang::transformer::remove
ASTEdit remove(RangeSelector S)
Removes the source selected by S.
Definition: RewriteRule.cpp:163
clang::OffloadTargetInfo::str
std::string str() const
Definition: OffloadBundler.cpp:118
BinaryFileHandler::ReadBundleStart
Expected< Optional< StringRef > > ReadBundleStart(MemoryBuffer &Input) final
Read the marker of the next bundled to be read in the file.
Definition: OffloadBundler.cpp:418
FileHandler::BundleInfo::BundleID
StringRef BundleID
Definition: OffloadBundler.cpp:191
clang::serialized_diags::create
std::unique_ptr< DiagnosticConsumer > create(StringRef OutputFile, DiagnosticOptions *Diags, bool MergeChildRecords=false)
Returns a DiagnosticConsumer that serializes diagnostics to a bitcode file.
Definition: SerializedDiagnosticPrinter.cpp:301
TextFileHandler::TextFileHandler
TextFileHandler(StringRef Comment)
Definition: OffloadBundler.cpp:806
clang::OffloadTargetInfo
Obtain the offload kind, real machine triple, and an optional GPUArch out of the target information s...
Definition: OffloadBundler.h:70
TextFileHandler::WriteBundle
Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final
Write the bundle from Input into OS.
Definition: OffloadBundler.cpp:800
clang::OffloadBundlerConfig::TargetNames
std::vector< std::string > TargetNames
Definition: OffloadBundler.h:42
llvm::ArrayRef
Definition: LLVM.h:34
clang::OffloadBundlerConfig::ObjcopyPath
std::string ObjcopyPath
Definition: OffloadBundler.h:39
getDefaultArchiveKindForHost
static Archive::Kind getDefaultArchiveKindForHost()
Definition: OffloadBundler.cpp:1084
getCompatibleOffloadTargets
static bool getCompatibleOffloadTargets(OffloadTargetInfo &CodeObjectInfo, SmallVectorImpl< StringRef > &CompatibleTargets, const OffloadBundlerConfig &BundlerConfig)
Computes a list of targets among all given targets which are compatible with this code object.
Definition: OffloadBundler.cpp:1096
FileHandler::ReadBundle
virtual Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input)=0
Read the current bundle and write the result into the stream OS.
clang::OffloadBundlerConfig::FilesType
std::string FilesType
Definition: OffloadBundler.h:38
clang::CudaArch::UNKNOWN
@ UNKNOWN
FileHandler::FileHandler
FileHandler()
Definition: OffloadBundler.cpp:194
clang::isCompatibleTargetID
bool isCompatibleTargetID(llvm::StringRef Provided, llvm::StringRef Requested)
Check whether the provided target ID is compatible with the requested target ID.
Definition: TargetID.cpp:167
clang::OffloadTargetInfo::isTripleValid
bool isTripleValid() const
Definition: OffloadBundler.cpp:109
ObjectFileHandler::WriteHeader
Error WriteHeader(raw_fd_ostream &OS, ArrayRef< std::unique_ptr< MemoryBuffer >> Inputs) final
Write the header of the bundled file to OS based on the information gathered from Inputs.
Definition: OffloadBundler.cpp:612
ObjectFileHandler::ReadBundle
Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final
Read the current bundle and write the result into the stream OS.
Definition: OffloadBundler.cpp:598
ObjectFileHandler::ReadHeader
Error ReadHeader(MemoryBuffer &Input) final
Update the file handler with information from the header of the bundled file.
Definition: OffloadBundler.cpp:577
ObjectFileHandler::ReadBundleEnd
Error ReadBundleEnd(MemoryBuffer &Input) final
Read the marker that closes the current bundle.
Definition: OffloadBundler.cpp:596
TextFileHandler::ReadHeader
Error ReadHeader(MemoryBuffer &Input) final
Update the file handler with information from the header of the bundled file.
Definition: OffloadBundler.cpp:734
clang::ObjCPropertyAttribute::Kind
Kind
Definition: DeclObjCCommon.h:22
std
Definition: Format.h:4477
FileHandler::listBundleIDsCallback
virtual Error listBundleIDsCallback(MemoryBuffer &Input, const BundleInfo &Info)
Definition: OffloadBundler.cpp:267
Cuda.h
FileHandler::forEachBundle
Error forEachBundle(MemoryBuffer &Input, std::function< Error(const BundleInfo &)> Func)
For each bundle in Input, do Func.
Definition: OffloadBundler.cpp:245
clang::OffloadTargetInfo::Triple
llvm::Triple Triple
Definition: OffloadBundler.h:72
clang
Definition: CalledOnceCheck.h:17
clang::OffloadTargetInfo::hasHostKind
bool hasHostKind() const
Definition: OffloadBundler.cpp:85
clang::ComparisonCategoryType::Last
@ Last
TextFileHandler::listBundleIDsCallback
Error listBundleIDsCallback(MemoryBuffer &Input, const BundleInfo &Info) final
Definition: OffloadBundler.cpp:813
isCodeObjectCompatible
bool isCodeObjectCompatible(const OffloadTargetInfo &CodeObjectInfo, const OffloadTargetInfo &TargetInfo)
Checks if a code object CodeObjectInfo is compatible with a given target TargetInfo.
Definition: OffloadBundler.cpp:145
TextFileHandler::ReadBundleEnd
Error ReadBundleEnd(MemoryBuffer &Input) final
Read the marker that closes the current bundle.
Definition: OffloadBundler.cpp:758
clang::OffloadBundlerConfig::BundleAlignment
unsigned BundleAlignment
Definition: OffloadBundler.h:35
FileHandler::ReadHeader
virtual Error ReadHeader(MemoryBuffer &Input)=0
Update the file handler with information from the header of the bundled file.
TextFileHandler::WriteHeader
Error WriteHeader(raw_fd_ostream &OS, ArrayRef< std::unique_ptr< MemoryBuffer >> Inputs) final
Write the header of the bundled file to OS based on the information gathered from Inputs.
Definition: OffloadBundler.cpp:785
clang::OffloadBundler::ListBundleIDsInFile
static llvm::Error ListBundleIDsInFile(llvm::StringRef InputFileName, const OffloadBundlerConfig &BundlerConfig)
Definition: OffloadBundler.cpp:883
llvm::SmallVectorImpl
Definition: Randstruct.h:18
BinaryFileHandler
Definition: OffloadBundler.cpp:318
clang::OffloadTargetInfo::isOffloadKindCompatible
bool isOffloadKindCompatible(const llvm::StringRef TargetOffloadKind) const
Definition: OffloadBundler.cpp:94
ObjectFileHandler::WriteBundleEnd
Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final
Write the marker that closes a bundle for the triple TargetTriple to OS.
Definition: OffloadBundler.cpp:627
TextFileHandler::ReadBundle
Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final
Read the current bundle and write the result into the stream OS.
Definition: OffloadBundler.cpp:772
clang::ento::ObjKind::OS
@ OS
Indicates that the tracking object is a descendant of a referenced-counted OSObject,...
TextFileHandler::WriteBundleStart
Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final
Write the marker that initiates a bundle for the triple TargetTriple to OS.
Definition: OffloadBundler.cpp:790
clang::OffloadBundlerConfig::PrintExternalCommands
bool PrintExternalCommands
Definition: OffloadBundler.h:32
CreateFileHandler
static Expected< std::unique_ptr< FileHandler > > CreateFileHandler(MemoryBuffer &FirstInput, const OffloadBundlerConfig &BundlerConfig)
Return an appropriate handler given the input files and options.
Definition: OffloadBundler.cpp:849