Cuda.cpp

Go to the documentation of this file.

//===--- Cuda.cpp - Cuda Tool and ToolChain Implementations -----*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#include "Cuda.h"
#include "clang/Basic/Cuda.h"
#include "clang/Config/config.h"
#include "clang/Driver/CommonArgs.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/Distro.h"
#include "clang/Driver/Driver.h"
#include "clang/Driver/InputInfo.h"
#include "clang/Options/Options.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Config/llvm-config.h" // for LLVM_HOST_TRIPLE
#include "llvm/Option/ArgList.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Program.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/TargetParser/Host.h"
#include "llvm/TargetParser/TargetParser.h"
#include <system_error>
 
using namespace clang::driver;
using namespace clang::driver::toolchains;
using namespace clang::driver::tools;
using namespace clang;
using namespace llvm::opt;
 
namespace {
 
CudaVersion getCudaVersion(uint32_t raw_version) {
  if (raw_version < 7050)
    return CudaVersion::CUDA_70;
  if (raw_version < 8000)
    return CudaVersion::CUDA_75;
  if (raw_version < 9000)
    return CudaVersion::CUDA_80;
  if (raw_version < 9010)
    return CudaVersion::CUDA_90;
  if (raw_version < 9020)
    return CudaVersion::CUDA_91;
  if (raw_version < 10000)
    return CudaVersion::CUDA_92;
  if (raw_version < 10010)
    return CudaVersion::CUDA_100;
  if (raw_version < 10020)
    return CudaVersion::CUDA_101;
  if (raw_version < 11000)
    return CudaVersion::CUDA_102;
  if (raw_version < 11010)
    return CudaVersion::CUDA_110;
  if (raw_version < 11020)
    return CudaVersion::CUDA_111;
  if (raw_version < 11030)
    return CudaVersion::CUDA_112;
  if (raw_version < 11040)
    return CudaVersion::CUDA_113;
  if (raw_version < 11050)
    return CudaVersion::CUDA_114;
  if (raw_version < 11060)
    return CudaVersion::CUDA_115;
  if (raw_version < 11070)
    return CudaVersion::CUDA_116;
  if (raw_version < 11080)
    return CudaVersion::CUDA_117;
  if (raw_version < 11090)
    return CudaVersion::CUDA_118;
  if (raw_version < 12010)
    return CudaVersion::CUDA_120;
  if (raw_version < 12020)
    return CudaVersion::CUDA_121;
  if (raw_version < 12030)
    return CudaVersion::CUDA_122;
  if (raw_version < 12040)
    return CudaVersion::CUDA_123;
  if (raw_version < 12050)
    return CudaVersion::CUDA_124;
  if (raw_version < 12060)
    return CudaVersion::CUDA_125;
  if (raw_version < 12070)
    return CudaVersion::CUDA_126;
  if (raw_version < 12090)
    return CudaVersion::CUDA_128;
  if (raw_version < 13000)
    return CudaVersion::CUDA_129;
  if (raw_version < 13010)
    return CudaVersion::CUDA_130;
  if (raw_version < 13020)
    return CudaVersion::CUDA_131;
  if (raw_version < 13030)
    return CudaVersion::CUDA_132;
  if (raw_version < 13040)
    return CudaVersion::CUDA_133;
  if (raw_version < 13050)
    return CudaVersion::CUDA_134;
  return CudaVersion::NEW;
}
 
CudaVersion parseCudaHFile(llvm::StringRef Input) {
  // Helper lambda which skips the words if the line starts with them or returns
  // std::nullopt otherwise.
  auto StartsWithWords =
      [](llvm::StringRef Line,
         const SmallVector<StringRef, 3> words) -> std::optional<StringRef> {
    for (StringRef word : words) {
      if (!Line.consume_front(word))
        return {};
      Line = Line.ltrim();
    }
    return Line;
  };
 
  Input = Input.ltrim();
  while (!Input.empty()) {
    if (auto Line =
            StartsWithWords(Input.ltrim(), {"#", "define", "CUDA_VERSION"})) {
      uint32_t RawVersion;
      Line->consumeInteger(10, RawVersion);
      return getCudaVersion(RawVersion);
    }
    // Find next non-empty line.
    Input = Input.drop_front(Input.find_first_of("\n\r")).ltrim();
  }
  return CudaVersion::UNKNOWN;
}
} // namespace
 
void CudaInstallationDetector::WarnIfUnsupportedVersion() const {
  if (Version > CudaVersion::PARTIALLY_SUPPORTED) {
    std::string VersionString = CudaVersionToString(Version);
    if (!VersionString.empty())
      VersionString.insert(0, " ");
    D.Diag(diag::warn_drv_new_cuda_version)
        << VersionString
        << (CudaVersion::PARTIALLY_SUPPORTED != CudaVersion::FULLY_SUPPORTED)
        << CudaVersionToString(CudaVersion::PARTIALLY_SUPPORTED);
  } else if (Version > CudaVersion::FULLY_SUPPORTED)
    D.Diag(diag::warn_drv_partially_supported_cuda_version)
        << CudaVersionToString(Version);
}
 
CudaInstallationDetector::CudaInstallationDetector(
    const Driver &D, const llvm::Triple &HostTriple,
    const llvm::opt::ArgList &Args)
    : D(D) {
  struct Candidate {
    std::string Path;
    bool StrictChecking;
 
    Candidate(std::string Path, bool StrictChecking = false)
        : Path(Path), StrictChecking(StrictChecking) {}
  };
  SmallVector<Candidate, 4> Candidates;
 
  // In decreasing order so we prefer newer versions to older versions.
  std::initializer_list<const char *> Versions = {"8.0", "7.5", "7.0"};
  auto &FS = D.getVFS();
 
  if (Args.hasArg(options::OPT_cuda_path_EQ)) {
    Candidates.emplace_back(
        Args.getLastArgValue(options::OPT_cuda_path_EQ).str());
  } else if (HostTriple.isOSWindows()) {
    for (const char *Ver : Versions)
      Candidates.emplace_back(
          D.SysRoot + "/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v" +
          Ver);
  } else {
    if (!Args.hasArg(options::OPT_cuda_path_ignore_env)) {
      // Try to find ptxas binary. If the executable is located in a directory
      // called 'bin/', its parent directory might be a good guess for a valid
      // CUDA installation.
      // However, some distributions might installs 'ptxas' to /usr/bin. In that
      // case the candidate would be '/usr' which passes the following checks
      // because '/usr/include' exists as well. To avoid this case, we always
      // check for the directory potentially containing files for libdevice,
      // even if the user passes -nocudalib.
      if (llvm::ErrorOr<std::string> ptxas =
              llvm::sys::findProgramByName("ptxas")) {
        SmallString<256> ptxasAbsolutePath;
        llvm::sys::fs::real_path(*ptxas, ptxasAbsolutePath);
 
        StringRef ptxasDir = llvm::sys::path::parent_path(ptxasAbsolutePath);
        if (llvm::sys::path::filename(ptxasDir) == "bin")
          Candidates.emplace_back(
              std::string(llvm::sys::path::parent_path(ptxasDir)),
              /*StrictChecking=*/true);
      }
    }
 
    Candidates.emplace_back(D.SysRoot + "/usr/local/cuda");
    for (const char *Ver : Versions)
      Candidates.emplace_back(D.SysRoot + "/usr/local/cuda-" + Ver);
 
    Distro Dist(FS, llvm::Triple(llvm::sys::getProcessTriple()));
    if (Dist.IsDebian() || Dist.IsUbuntu())
      // Special case for Debian to have nvidia-cuda-toolkit work
      // out of the box. More info on http://bugs.debian.org/882505
      Candidates.emplace_back(D.SysRoot + "/usr/lib/cuda");
  }
 
  bool NoCudaLib =
      !Args.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib, true);
 
  for (const auto &Candidate : Candidates) {
    InstallPath = Candidate.Path;
    if (InstallPath.empty() || !FS.exists(InstallPath))
      continue;
 
    BinPath = InstallPath + "/bin";
    IncludePath = InstallPath + "/include";
    LibDevicePath = InstallPath + "/nvvm/libdevice";
 
    if (!(FS.exists(IncludePath) && FS.exists(BinPath)))
      continue;
    bool CheckLibDevice = (!NoCudaLib || Candidate.StrictChecking);
    if (CheckLibDevice && !FS.exists(LibDevicePath))
      continue;
 
    Version = CudaVersion::UNKNOWN;
    if (auto CudaHFile = FS.getBufferForFile(InstallPath + "/include/cuda.h"))
      Version = parseCudaHFile((*CudaHFile)->getBuffer());
    // As the last resort, make an educated guess between CUDA-7.0, which had
    // old-style libdevice bitcode, and an unknown recent CUDA version.
    if (Version == CudaVersion::UNKNOWN) {
      Version = FS.exists(LibDevicePath + "/libdevice.10.bc")
                    ? CudaVersion::NEW
                    : CudaVersion::CUDA_70;
    }
 
    if (Version >= CudaVersion::CUDA_90) {
      // CUDA-9+ uses single libdevice file for all GPU variants.
      std::string FilePath = LibDevicePath + "/libdevice.10.bc";
      if (FS.exists(FilePath)) {
        // CUDA-9+ uses a single libdevice file for every NVIDIA GPU variant
        // (sm_30 and newer).
#define NVPTX_GPU(NAME, KIND, VIRTUAL, SM_ID, MIN_VER, MAX_VER, SUFFIX)        \
  if ((SM_ID) >= 300)                                                          \
    LibDeviceMap[NAME] = FilePath;
#include "llvm/TargetParser/NVPTXTargetParser.def"
      }
    } else {
      std::error_code EC;
      for (llvm::vfs::directory_iterator LI = FS.dir_begin(LibDevicePath, EC),
                                         LE;
           !EC && LI != LE; LI = LI.increment(EC)) {
        StringRef FilePath = LI->path();
        StringRef FileName = llvm::sys::path::filename(FilePath);
        // Process all bitcode filenames that look like
        // libdevice.compute_XX.YY.bc
        const StringRef LibDeviceName = "libdevice.";
        if (!(FileName.starts_with(LibDeviceName) && FileName.ends_with(".bc")))
          continue;
        StringRef GpuArch = FileName.slice(
            LibDeviceName.size(), FileName.find('.', LibDeviceName.size()));
        LibDeviceMap[GpuArch] = FilePath.str();
        // Insert map entries for specific devices with this compute
        // capability. NVCC's choice of the libdevice library version is
        // rather peculiar and depends on the CUDA version.
        if (GpuArch == "compute_20") {
          LibDeviceMap["sm_20"] = std::string(FilePath);
          LibDeviceMap["sm_21"] = std::string(FilePath);
          LibDeviceMap["sm_32"] = std::string(FilePath);
        } else if (GpuArch == "compute_30") {
          LibDeviceMap["sm_30"] = std::string(FilePath);
          if (Version < CudaVersion::CUDA_80) {
            LibDeviceMap["sm_50"] = std::string(FilePath);
            LibDeviceMap["sm_52"] = std::string(FilePath);
            LibDeviceMap["sm_53"] = std::string(FilePath);
          }
          LibDeviceMap["sm_60"] = std::string(FilePath);
          LibDeviceMap["sm_61"] = std::string(FilePath);
          LibDeviceMap["sm_62"] = std::string(FilePath);
        } else if (GpuArch == "compute_35") {
          LibDeviceMap["sm_35"] = std::string(FilePath);
          LibDeviceMap["sm_37"] = std::string(FilePath);
        } else if (GpuArch == "compute_50") {
          if (Version >= CudaVersion::CUDA_80) {
            LibDeviceMap["sm_50"] = std::string(FilePath);
            LibDeviceMap["sm_52"] = std::string(FilePath);
            LibDeviceMap["sm_53"] = std::string(FilePath);
          }
        }
      }
    }
 
    // Check that we have found at least one libdevice that we can link in if
    // -nocudalib hasn't been specified.
    if (LibDeviceMap.empty() && !NoCudaLib)
      continue;
 
    IsValid = true;
    break;
  }
}
 
void CudaInstallationDetector::AddCudaIncludeArgs(
    const ArgList &DriverArgs, ArgStringList &CC1Args) const {
  if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
                         options::OPT_fno_offload_via_llvm, false))
    return;
 
  if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {
    // Add cuda_wrappers/* to our system include path.  This lets us wrap
    // standard library headers.
    SmallString<128> P(D.ResourceDir);
    llvm::sys::path::append(P, "include");
    llvm::sys::path::append(P, "cuda_wrappers");
    CC1Args.push_back("-internal-isystem");
    CC1Args.push_back(DriverArgs.MakeArgString(P));
  }
 
  if (!DriverArgs.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,
                          true))
    return;
 
  if (!isValid()) {
    D.Diag(diag::err_drv_no_cuda_installation);
    return;
  }
 
  CC1Args.push_back("-include");
  CC1Args.push_back("__clang_cuda_runtime_wrapper.h");
 
  // From CUDA 13.0 on, libcu++, CUB and Thrust are in include/cccl, and some
  // of the toolkit's own headers, such as cooperative_groups.h, include them.
  SmallString<128> CCCLPath(getIncludePath());
  llvm::sys::path::append(CCCLPath, "cccl");
  if (D.getVFS().exists(CCCLPath)) {
    CC1Args.push_back("-internal-isystem");
    CC1Args.push_back(DriverArgs.MakeArgString(CCCLPath));
  }
}
 
void CudaInstallationDetector::CheckCudaVersionSupportsArch(
    OffloadArch Arch) const {
  // Only NVIDIA architectures depend on the CUDA toolkit version.
  if (!Arch.isNVPTX() || Version == CudaVersion::UNKNOWN ||
      ArchsWithBadVersion[Arch.nvptxKind()])
    return;
 
  auto MinVersion = MinVersionForOffloadArch(Arch);
  auto MaxVersion = MaxVersionForOffloadArch(Arch);
  if (Version < MinVersion || Version > MaxVersion) {
    ArchsWithBadVersion[Arch.nvptxKind()] = true;
    D.Diag(diag::err_drv_cuda_version_unsupported)
        << OffloadArchToString(Arch) << CudaVersionToString(MinVersion)
        << CudaVersionToString(MaxVersion) << InstallPath
        << CudaVersionToString(Version);
  }
}
 
void CudaInstallationDetector::print(raw_ostream &OS) const {
  if (isValid())
    OS << "Found CUDA installation: " << InstallPath << ", version "
       << CudaVersionToString(Version) << "\n";
}
 
namespace {
/// Debug info level for the NVPTX devices. We may need to emit different debug
/// info level for the host and for the device itselfi. This type controls
/// emission of the debug info for the devices. It either prohibits disable info
/// emission completely, or emits debug directives only, or emits same debug
/// info as for the host.
enum DeviceDebugInfoLevel {
  DisableDebugInfo,        /// Do not emit debug info for the devices.
  DebugDirectivesOnly,     /// Emit only debug directives.
  EmitSameDebugInfoAsHost, /// Use the same debug info level just like for the
                           /// host.
};
} // anonymous namespace
 
/// Define debug info level for the NVPTX devices. If the debug info for both
/// the host and device are disabled (-g0/-ggdb0 or no debug options at all). If
/// only debug directives are requested for the both host and device
/// (-gline-directvies-only), or the debug info only for the device is disabled
/// (optimization is on and --cuda-noopt-device-debug was not specified), the
/// debug directves only must be emitted for the device. Otherwise, use the same
/// debug info level just like for the host (with the limitations of only
/// supported DWARF2 standard).
static DeviceDebugInfoLevel mustEmitDebugInfo(const ArgList &Args) {
  const Arg *A = Args.getLastArg(options::OPT_O_Group);
  bool IsDebugEnabled = !A || A->getOption().matches(options::OPT_O0) ||
                        Args.hasFlag(options::OPT_cuda_noopt_device_debug,
                                     options::OPT_no_cuda_noopt_device_debug,
                                     /*Default=*/false);
  if (const Arg *A = Args.getLastArg(options::OPT_g_Group)) {
    const Option &Opt = A->getOption();
    if (Opt.matches(options::OPT_gN_Group)) {
      if (Opt.matches(options::OPT_g0) || Opt.matches(options::OPT_ggdb0))
        return DisableDebugInfo;
      if (Opt.matches(options::OPT_gline_directives_only))
        return DebugDirectivesOnly;
    }
    return IsDebugEnabled ? EmitSameDebugInfoAsHost : DebugDirectivesOnly;
  }
  return willEmitRemarks(Args) ? DebugDirectivesOnly : DisableDebugInfo;
}
 
void NVPTX::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
                                    const InputInfo &Output,
                                    const InputInfoList &Inputs,
                                    const ArgList &Args,
                                    const char *LinkingOutput) const {
  const auto &TC =
      static_cast<const toolchains::NVPTXToolChain &>(getToolChain());
 
  bool UsesLLVMOffloading = Args.hasFlag(
      options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
  assert((TC.getTriple().isNVPTX() || UsesLLVMOffloading) && "Wrong platform");
 
  BoundArch GPUArch;
  // If this is a CUDA action we need to extract the device architecture
  // from the Job's associated architecture, otherwise use the -march=arch
  // option. This option may come from -Xopenmp-target flag or the default
  // value.
  if (JA.isDeviceOffloading(Action::OFK_Cuda)) {
    GPUArch = JA.getOffloadingArch();
  } else {
    GPUArch = BoundArch(Args.getLastArgValue(options::OPT_march_EQ));
    if (GPUArch.empty()) {
      C.getDriver().Diag(diag::err_drv_offload_missing_gpu_arch)
          << getToolChain().getArchName() << getShortName();
      return;
    }
  }
 
  // Obtain architecture from the action.
  assert(!GPUArch.Arch.isUnknown() &&
         "Device action expected to have an architecture.");
 
  // Check that our installation's ptxas supports gpu_arch.
  if (!UsesLLVMOffloading && !Args.hasArg(options::OPT_no_cuda_version_check)) {
    TC.CudaInstallation.CheckCudaVersionSupportsArch(GPUArch.Arch);
  }
 
  ArgStringList CmdArgs;
  CmdArgs.push_back(TC.getTriple().isArch64Bit() ? "-m64" : "-m32");
  DeviceDebugInfoLevel DIKind = mustEmitDebugInfo(Args);
  if (DIKind == EmitSameDebugInfoAsHost) {
    // ptxas does not accept -g option if optimization is enabled, so
    // we ignore the compiler's -O* options if we want debug info.
    CmdArgs.push_back("-g");
    CmdArgs.push_back("--dont-merge-basicblocks");
    CmdArgs.push_back("--return-at-end");
  } else if (Arg *A = Args.getLastArg(options::OPT_O_Group)) {
    // Map the -O we received to -O{0,1,2,3}.
    //
    // TODO: Perhaps we should map host -O2 to ptxas -O3. -O3 is ptxas's
    // default, so it may correspond more closely to the spirit of clang -O2.
 
    // -O3 seems like the least-bad option when -Osomething is specified to
    // clang but it isn't handled below.
    StringRef OOpt = "3";
    if (A->getOption().matches(options::OPT_O4) ||
        A->getOption().matches(options::OPT_Ofast))
      OOpt = "3";
    else if (A->getOption().matches(options::OPT_O0))
      OOpt = "0";
    else if (A->getOption().matches(options::OPT_O)) {
      // -Os, -Oz, and -O(anything else) map to -O2, for lack of better options.
      OOpt = llvm::StringSwitch<const char *>(A->getValue())
                 .Case("1", "1")
                 .Case("2", "2")
                 .Case("3", "3")
                 .Case("s", "2")
                 .Case("z", "2")
                 .Default("2");
    }
    CmdArgs.push_back(Args.MakeArgString(llvm::Twine("-O") + OOpt));
  } else {
    // If no -O was passed, pass -O0 to ptxas -- no opt flag should correspond
    // to no optimizations, but ptxas's default is -O3.
    CmdArgs.push_back("-O0");
  }
  if (DIKind == DebugDirectivesOnly)
    CmdArgs.push_back("-lineinfo");
 
  // Pass -v to ptxas if it was passed to the driver.
  if (Args.hasArg(options::OPT_v))
    CmdArgs.push_back("-v");
 
  CmdArgs.push_back("--gpu-name");
  CmdArgs.push_back(Args.MakeArgString(GPUArch.ArchName));
  CmdArgs.push_back("--output-file");
  std::string OutputFileName = TC.getInputFilename(Output);
 
  if (Output.isFilename() && OutputFileName != Output.getFilename())
    C.addTempFile(Args.MakeArgString(OutputFileName));
 
  CmdArgs.push_back(Args.MakeArgString(OutputFileName));
  for (const auto &II : Inputs)
    CmdArgs.push_back(Args.MakeArgString(II.getFilename()));
 
  for (const auto &A : Args.getAllArgValues(options::OPT_Xcuda_ptxas))
    CmdArgs.push_back(Args.MakeArgString(A));
 
  bool Relocatable;
  if (JA.isOffloading(Action::OFK_OpenMP))
    // In OpenMP we need to generate relocatable code.
    Relocatable = Args.hasFlag(options::OPT_fopenmp_relocatable_target,
                               options::OPT_fnoopenmp_relocatable_target,
                               /*Default=*/true);
  else if (JA.isOffloading(Action::OFK_Cuda))
    // In CUDA we generate relocatable code by default.
    Relocatable = UsesLLVMOffloading ||
                  Args.hasFlag(options::OPT_fgpu_rdc, options::OPT_fno_gpu_rdc,
                               /*Default=*/false);
  else
    // Otherwise, we are compiling directly and should create linkable output.
    Relocatable = true;
 
  if (Relocatable)
    CmdArgs.push_back("-c");
 
  const char *Exec;
  if (Arg *A = Args.getLastArg(options::OPT_ptxas_path_EQ))
    Exec = A->getValue();
  else
    Exec = Args.MakeArgString(TC.GetProgramPath("ptxas"));
  C.addCommand(std::make_unique<Command>(
      JA, *this,
      ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8,
                          "--options-file="},
      Exec, CmdArgs, Inputs, Output));
}
 
static bool shouldIncludePTX(const ArgList &Args, StringRef InputArch) {
  bool includePTX = false;
  for (Arg *A : Args.filtered(options::OPT_cuda_include_ptx_EQ,
                              options::OPT_no_cuda_include_ptx_EQ)) {
    A->claim();
    const StringRef ArchStr = A->getValue();
    if (A->getOption().matches(options::OPT_cuda_include_ptx_EQ) &&
        (ArchStr == "all" || ArchStr == InputArch))
      includePTX = true;
    else if (A->getOption().matches(options::OPT_no_cuda_include_ptx_EQ) &&
             (ArchStr == "all" || ArchStr == InputArch))
      includePTX = false;
  }
  return includePTX;
}
 
// All inputs to this linker must be from CudaDeviceActions, as we need to look
// at the Inputs' Actions in order to figure out which GPU architecture they
// correspond to.
void NVPTX::FatBinary::ConstructJob(Compilation &C, const JobAction &JA,
                                    const InputInfo &Output,
                                    const InputInfoList &Inputs,
                                    const ArgList &Args,
                                    const char *LinkingOutput) const {
  const auto &TC =
      static_cast<const toolchains::CudaToolChain &>(getToolChain());
  [[maybe_unused]] bool UsesLLVMOffloading = Args.hasFlag(
      options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
  assert((UsesLLVMOffloading || TC.getTriple().isNVPTX()) && "Wrong platform");
 
  ArgStringList CmdArgs;
  if (TC.CudaInstallation.version() <= CudaVersion::CUDA_100)
    CmdArgs.push_back("--cuda");
  CmdArgs.push_back(TC.getTriple().isArch64Bit() ? "-64" : "-32");
  CmdArgs.push_back(Args.MakeArgString("--create"));
  CmdArgs.push_back(Args.MakeArgString(Output.getFilename()));
  if (mustEmitDebugInfo(Args) == EmitSameDebugInfoAsHost)
    CmdArgs.push_back("-g");
 
  for (const auto &II : Inputs) {
    auto *A = II.getAction();
    assert(A->getInputs().size() == 1 &&
           "Device offload action is expected to have a single input");
    BoundArch GpuArch = A->getOffloadingArch();
    assert(!GpuArch.empty() &&
           "Device action expected to have associated a GPU architecture!");
 
    if (II.getType() == types::TY_PP_Asm &&
        !shouldIncludePTX(Args, GpuArch.ArchName))
      continue;
    StringRef Kind = (II.getType() == types::TY_PP_Asm) ? "ptx" : "elf";
    CmdArgs.push_back(Args.MakeArgString(
        "--image3=kind=" + Kind + ",sm=" + GpuArch.ArchName.drop_front(3) +
        ",file=" + getToolChain().getInputFilename(II)));
  }
 
  for (const auto &A : Args.getAllArgValues(options::OPT_Xcuda_fatbinary))
    CmdArgs.push_back(Args.MakeArgString(A));
 
  const char *Exec = Args.MakeArgString(TC.GetProgramPath("fatbinary"));
  C.addCommand(std::make_unique<Command>(
      JA, *this,
      ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8,
                          "--options-file="},
      Exec, CmdArgs, Inputs, Output));
}
 
void NVPTX::Linker::ConstructJob(Compilation &C, const JobAction &JA,
                                 const InputInfo &Output,
                                 const InputInfoList &Inputs,
                                 const ArgList &Args,
                                 const char *LinkingOutput) const {
  const auto &TC =
      static_cast<const toolchains::NVPTXToolChain &>(getToolChain());
  ArgStringList CmdArgs;
 
  [[maybe_unused]] bool UsesLLVMOffloading = Args.hasFlag(
      options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
  assert((UsesLLVMOffloading || TC.getTriple().isNVPTX()) && "Wrong platform");
 
  assert((Output.isFilename() || Output.isNothing()) && "Invalid output.");
  if (Output.isFilename()) {
    CmdArgs.push_back("-o");
    CmdArgs.push_back(Output.getFilename());
  }
 
  if (mustEmitDebugInfo(Args) == EmitSameDebugInfoAsHost)
    CmdArgs.push_back("-g");
 
  if (Args.hasArg(options::OPT_v))
    CmdArgs.push_back("-v");
 
  StringRef GPUArch = Args.getLastArgValue(options::OPT_march_EQ);
  if (GPUArch.empty() && !getToolChain().isUsingLTO(Args)) {
    C.getDriver().Diag(diag::err_drv_offload_missing_gpu_arch)
        << getToolChain().getArchName() << getShortName();
    return;
  }
 
  if (!GPUArch.empty()) {
    CmdArgs.push_back("-arch");
    CmdArgs.push_back(Args.MakeArgString(GPUArch));
  }
 
  if (Args.hasArg(options::OPT_ptxas_path_EQ))
    CmdArgs.push_back(Args.MakeArgString(
        "--ptxas-path=" + Args.getLastArgValue(options::OPT_ptxas_path_EQ)));
 
  // The wrapper runs 'ptxas' itself when doing LTO, so it needs these.
  for (const Arg *A : Args.filtered(options::OPT_Xcuda_ptxas)) {
    A->claim();
    CmdArgs.append({"-Xptxas", A->getValue()});
  }
 
  if (Args.hasArg(options::OPT_cuda_path_EQ) || TC.CudaInstallation.isValid()) {
    StringRef CudaPath = Args.getLastArgValue(
        options::OPT_cuda_path_EQ,
        llvm::sys::path::parent_path(TC.CudaInstallation.getBinPath()));
    CmdArgs.push_back(Args.MakeArgString("--cuda-path=" + CudaPath));
  }
 
  // Add paths specified in LIBRARY_PATH environment variable as -L options.
  addDirectoryList(Args, CmdArgs, "-L", "LIBRARY_PATH");
 
  // Add standard library search paths passed on the command line.
  Args.AddAllArgs(CmdArgs, options::OPT_L);
  getToolChain().AddFilePathLibArgs(Args, CmdArgs);
  AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs, JA);
 
  if (auto LTO = getToolChain().getLTOMode(Args); LTO != LTOK_None)
    addLTOOptions(getToolChain(), Args, CmdArgs, Output, Inputs,
                  LTO == LTOK_Thin);
 
  // Forward the PTX features if the nvlink-wrapper needs it.
  std::vector<StringRef> Features;
  getNVPTXTargetFeatures(C.getDriver(), getToolChain().getTriple(), Args,
                         Features);
  CmdArgs.push_back(
      Args.MakeArgString("--plugin-opt=-mattr=" + llvm::join(Features, ",")));
 
  // Add paths for the default clang library path.
  SmallString<256> DefaultLibPath =
      llvm::sys::path::parent_path(TC.getDriver().Dir);
  llvm::sys::path::append(DefaultLibPath, CLANG_INSTALL_LIBDIR_BASENAME);
  CmdArgs.push_back(Args.MakeArgString(Twine("-L") + DefaultLibPath));
 
  getToolChain().addProfileRTLibs(Args, CmdArgs);
  addSanitizerRuntimes(getToolChain(), Args, CmdArgs, C);
 
  if (Args.hasArg(options::OPT_stdlib))
    CmdArgs.append({"-lc", "-lm"});
  if (Args.hasArg(options::OPT_startfiles)) {
    std::optional<std::string> IncludePath = getToolChain().getStdlibPath();
    if (!IncludePath)
      IncludePath = "/lib";
    SmallString<128> P(*IncludePath);
    llvm::sys::path::append(P, "crt1.o");
    CmdArgs.push_back(Args.MakeArgString(P));
  }
 
  C.addCommand(std::make_unique<Command>(
      JA, *this, ResponseFileSupport::AtFileUTF8(),
      Args.MakeArgString(getToolChain().GetProgramPath("clang-nvlink-wrapper")),
      CmdArgs, Inputs, Output));
}
 
void NVPTX::getNVPTXTargetFeatures(const Driver &D, const llvm::Triple &Triple,
                                   const llvm::opt::ArgList &Args,
                                   std::vector<StringRef> &Features) {
  if (Args.hasArg(options::OPT_cuda_feature_EQ)) {
    StringRef PtxFeature = Args.getLastArgValue(options::OPT_cuda_feature_EQ);
    Features.push_back(Args.MakeArgString(PtxFeature));
    return;
  }
  CudaInstallationDetector CudaInstallation(D, Triple, Args);
 
  // New CUDA versions often introduce new instructions that are only supported
  // by new PTX version, so we need to raise PTX level to enable them in NVPTX
  // back-end.
  const char *PtxFeature = nullptr;
  switch (CudaInstallation.version()) {
#define CASE_CUDA_VERSION(CUDA_VER, PTX_VER)                                   \
  case CudaVersion::CUDA_##CUDA_VER:                                           \
    PtxFeature = "+ptx" #PTX_VER;                                              \
    break;
    CASE_CUDA_VERSION(134, 94);
    CASE_CUDA_VERSION(133, 93);
    CASE_CUDA_VERSION(132, 92);
    CASE_CUDA_VERSION(131, 91);
    CASE_CUDA_VERSION(130, 90);
    CASE_CUDA_VERSION(129, 88);
    CASE_CUDA_VERSION(128, 87);
    CASE_CUDA_VERSION(126, 85);
    CASE_CUDA_VERSION(125, 85);
    CASE_CUDA_VERSION(124, 84);
    CASE_CUDA_VERSION(123, 83);
    CASE_CUDA_VERSION(122, 82);
    CASE_CUDA_VERSION(121, 81);
    CASE_CUDA_VERSION(120, 80);
    CASE_CUDA_VERSION(118, 78);
    CASE_CUDA_VERSION(117, 77);
    CASE_CUDA_VERSION(116, 76);
    CASE_CUDA_VERSION(115, 75);
    CASE_CUDA_VERSION(114, 74);
    CASE_CUDA_VERSION(113, 73);
    CASE_CUDA_VERSION(112, 72);
    CASE_CUDA_VERSION(111, 71);
    CASE_CUDA_VERSION(110, 70);
    CASE_CUDA_VERSION(102, 65);
    CASE_CUDA_VERSION(101, 64);
    CASE_CUDA_VERSION(100, 63);
    CASE_CUDA_VERSION(92, 61);
    CASE_CUDA_VERSION(91, 61);
    CASE_CUDA_VERSION(90, 60);
    CASE_CUDA_VERSION(80, 50);
    CASE_CUDA_VERSION(75, 43);
    CASE_CUDA_VERSION(70, 42);
#undef CASE_CUDA_VERSION
  // TODO: Use specific CUDA version once it's public.
  case clang::CudaVersion::NEW:
    PtxFeature = "+ptx86";
    break;
  default:
    // No PTX feature specified; let the backend choose based on the target SM.
    break;
  }
  if (PtxFeature)
    Features.push_back(PtxFeature);
}
 
/// NVPTX toolchain. Our assembler is ptxas, and our linker is nvlink. This
/// operates as a stand-alone version of the NVPTX tools without the host
/// toolchain.
NVPTXToolChain::NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,
                               const llvm::Triple &HostTriple,
                               const ArgList &Args)
    : ToolChain(D, Triple, Args), CudaInstallation(D, HostTriple, Args) {
  if (CudaInstallation.isValid())
    getProgramPaths().push_back(std::string(CudaInstallation.getBinPath()));
  // Lookup binaries into the driver directory, this is used to
  // discover the 'nvptx-arch' executable.
  getProgramPaths().push_back(getDriver().Dir);
}
 
/// We only need the host triple to locate the CUDA binary utilities, use the
/// system's default triple if not provided.
NVPTXToolChain::NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,
                               const ArgList &Args)
    : NVPTXToolChain(D, Triple, llvm::Triple(LLVM_HOST_TRIPLE), Args) {
  loadMultilibsFromYAML(Args, D);
}
 
llvm::opt::DerivedArgList *
NVPTXToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args,
                              BoundArch BA,
                              Action::OffloadKind OffloadKind) const {
  DerivedArgList *DAL = ToolChain::TranslateArgs(Args, BA, OffloadKind);
  if (!DAL)
    DAL = new DerivedArgList(Args.getBaseArgs());
 
  const OptTable &Opts = getDriver().getOpts();
 
  for (Arg *A : Args)
    if (!llvm::is_contained(*DAL, A))
      DAL->append(A);
 
  if (!DAL->hasArg(options::OPT_march_EQ) && OffloadKind != Action::OFK_None) {
    DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_march_EQ),
                      OffloadArchToString(OffloadArch::CudaDefault()));
  } else if (DAL->getLastArgValue(options::OPT_march_EQ) == "generic" &&
             OffloadKind == Action::OFK_None) {
    DAL->eraseArg(options::OPT_march_EQ);
  } else if (DAL->getLastArgValue(options::OPT_march_EQ) == "native") {
    auto GPUsOrErr = getSystemGPUArchs(Args);
    if (!GPUsOrErr) {
      getDriver().Diag(diag::err_drv_undetermined_gpu_arch)
          << getArchName() << llvm::toString(GPUsOrErr.takeError()) << "-march";
    } else {
      auto &GPUs = *GPUsOrErr;
      if (llvm::SmallSet<std::string, 1>(GPUs.begin(), GPUs.end()).size() > 1)
        getDriver().Diag(diag::warn_drv_multi_gpu_arch)
            << getArchName() << llvm::join(GPUs, ", ") << "-march";
      DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_march_EQ),
                        Args.MakeArgString(GPUs.front()));
    }
  }
 
  return DAL;
}
 
void NVPTXToolChain::addClangTargetOptions(
    const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
    BoundArch BA, Action::OffloadKind DeviceOffloadingKind) const {}
 
void NVPTXToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
                                               ArgStringList &CC1Args) const {
  if (DriverArgs.hasArg(options::OPT_nostdinc) ||
      DriverArgs.hasArg(options::OPT_nostdlibinc))
    return;
 
  // Add multilib variant include paths in priority order.
  for (const Multilib &M : getOrderedMultilibs()) {
    if (M.isDefault())
      continue;
    if (std::optional<std::string> StdlibIncDir = getStdlibIncludePath()) {
      SmallString<128> Dir(*StdlibIncDir);
      llvm::sys::path::append(Dir, M.includeSuffix());
      if (getDriver().getVFS().exists(Dir))
        addSystemInclude(DriverArgs, CC1Args, Dir);
    }
  }
 
  if (std::optional<std::string> Path = getStdlibIncludePath())
    addSystemInclude(DriverArgs, CC1Args, *Path);
}
 
bool NVPTXToolChain::supportsDebugInfoOption(const llvm::opt::Arg *A) const {
  const Option &O = A->getOption();
  return (O.matches(options::OPT_gN_Group) &&
          !O.matches(options::OPT_gmodules)) ||
         O.matches(options::OPT_g_Flag) ||
         O.matches(options::OPT_ggdbN_Group) || O.matches(options::OPT_ggdb) ||
         O.matches(options::OPT_gdwarf) || O.matches(options::OPT_gdwarf_2) ||
         O.matches(options::OPT_gdwarf_3) || O.matches(options::OPT_gdwarf_4) ||
         O.matches(options::OPT_gdwarf_5) ||
         O.matches(options::OPT_gcolumn_info);
}
 
void NVPTXToolChain::adjustDebugInfoKind(
    llvm::codegenoptions::DebugInfoKind &DebugInfoKind,
    const ArgList &Args) const {
  switch (mustEmitDebugInfo(Args)) {
  case DisableDebugInfo:
    DebugInfoKind = llvm::codegenoptions::NoDebugInfo;
    break;
  case DebugDirectivesOnly:
    DebugInfoKind = llvm::codegenoptions::DebugDirectivesOnly;
    break;
  case EmitSameDebugInfoAsHost:
    // Use same debug info level as the host.
    break;
  }
}
 
Expected<SmallVector<std::string>>
NVPTXToolChain::getSystemGPUArchs(const ArgList &Args) const {
  // Detect NVIDIA GPUs availible on the system.
  std::string Program;
  if (Arg *A = Args.getLastArg(options::OPT_offload_arch_tool_EQ))
    Program = A->getValue();
  else
    Program = GetProgramPath("nvptx-arch");
 
  auto StdoutOrErr = getDriver().executeProgram({Program});
  if (!StdoutOrErr)
    return StdoutOrErr.takeError();
 
  SmallVector<std::string, 1> GPUArchs;
  for (StringRef Arch : llvm::split((*StdoutOrErr)->getBuffer(), "\n"))
    if (!Arch.empty())
      GPUArchs.push_back(Arch.str());
 
  if (GPUArchs.empty())
    return llvm::createStringError(std::error_code(),
                                   "No NVIDIA GPU detected in the system");
 
  return std::move(GPUArchs);
}
 
/// CUDA toolchain.  Our assembler is ptxas, and our "linker" is fatbinary,
/// which isn't properly a linker but nonetheless performs the step of stitching
/// together object files from the assembler into a single blob.
 
CudaToolChain::CudaToolChain(const Driver &D, const llvm::Triple &Triple,
                             const ToolChain &HostTC, const ArgList &Args)
    : NVPTXToolChain(D, Triple, HostTC.getTriple(), Args), HostTC(HostTC) {}
 
void CudaToolChain::addClangTargetOptions(
    const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
    BoundArch BA, Action::OffloadKind DeviceOffloadingKind) const {
  HostTC.addClangTargetOptions(DriverArgs, CC1Args, BA, DeviceOffloadingKind);
 
  bool UsesLLVMOffloading = DriverArgs.hasFlag(
      options::OPT_foffload_via_llvm, options::OPT_fno_offload_via_llvm, false);
 
  StringRef GpuArch = DriverArgs.getLastArgValue(options::OPT_march_EQ);
  assert((DeviceOffloadingKind == Action::OFK_OpenMP ||
          DeviceOffloadingKind == Action::OFK_Cuda || UsesLLVMOffloading) &&
         "Only OpenMP or CUDA offloading kinds are supported for NVIDIA GPUs.");
 
  CC1Args.append({"-fcuda-is-device", "-fno-threadsafe-statics"});
 
  if (DriverArgs.hasFlag(options::OPT_fcuda_short_ptr,
                         options::OPT_fno_cuda_short_ptr, false))
    CC1Args.append({"-target-abi", "shortptr"});
 
  if (!DriverArgs.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib,
                          true))
    return;
 
  if (DeviceOffloadingKind == Action::OFK_OpenMP &&
      DriverArgs.hasArg(options::OPT_S))
    return;
 
  if (UsesLLVMOffloading)
    return;
 
  std::string LibDeviceFile = CudaInstallation.getLibDeviceFile(GpuArch);
  if (LibDeviceFile.empty()) {
    getDriver().Diag(diag::err_drv_no_cuda_libdevice) << GpuArch;
    return;
  }
 
  CC1Args.push_back("-mlink-builtin-bitcode");
  CC1Args.push_back(DriverArgs.MakeArgString(LibDeviceFile));
 
  clang::CudaVersion CudaInstallationVersion = CudaInstallation.version();
 
  if (CudaInstallationVersion >= CudaVersion::UNKNOWN)
    CC1Args.push_back(
        DriverArgs.MakeArgString(Twine("-target-sdk-version=") +
                                 CudaVersionToString(CudaInstallationVersion)));
 
  if (DeviceOffloadingKind == Action::OFK_OpenMP) {
    if (CudaInstallationVersion < CudaVersion::CUDA_92) {
      getDriver().Diag(
          diag::err_drv_omp_offload_target_cuda_version_not_support)
          << CudaVersionToString(CudaInstallationVersion);
      return;
    }
 
    // Link the bitcode library late if we're using device LTO.
    if (isUsingLTO(DriverArgs, DeviceOffloadingKind))
      return;
 
    addOpenMPDeviceRTL(getDriver(), DriverArgs, CC1Args, GpuArch.str(),
                       getTriple(), HostTC);
  }
}
 
llvm::DenormalMode CudaToolChain::getDefaultDenormalModeForType(
    const llvm::opt::ArgList &DriverArgs, const JobAction &JA,
    const llvm::fltSemantics *FPType) const {
  if (JA.getOffloadingDeviceKind() == Action::OFK_Cuda) {
    if (FPType && FPType == &llvm::APFloat::IEEEsingle() &&
        DriverArgs.hasFlag(options::OPT_fgpu_flush_denormals_to_zero,
                           options::OPT_fno_gpu_flush_denormals_to_zero, false))
      return llvm::DenormalMode::getPreserveSign();
  }
 
  assert(JA.getOffloadingDeviceKind() != Action::OFK_Host);
  return llvm::DenormalMode::getIEEE();
}
 
void CudaToolChain::AddCudaIncludeArgs(const ArgList &DriverArgs,
                                       ArgStringList &CC1Args) const {
  if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
                         options::OPT_fno_offload_via_llvm, false))
    return;
 
  // Check our CUDA version if we're going to include the CUDA headers.
  if (DriverArgs.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,
                         true) &&
      !DriverArgs.hasArg(options::OPT_no_cuda_version_check)) {
    StringRef Arch = DriverArgs.getLastArgValue(options::OPT_march_EQ);
    assert(!Arch.empty() && "Must have an explicit GPU arch.");
    CudaInstallation.CheckCudaVersionSupportsArch(StringToOffloadArch(Arch));
  }
  CudaInstallation.AddCudaIncludeArgs(DriverArgs, CC1Args);
}
 
std::string CudaToolChain::getInputFilename(const InputInfo &Input) const {
  // Only object files are changed, for example assembly files keep their .s
  // extensions. If the user requested device-only compilation don't change it.
  if (Input.getType() != types::TY_Object || getDriver().offloadDeviceOnly())
    return ToolChain::getInputFilename(Input);
 
  return ToolChain::getInputFilename(Input);
}
 
llvm::opt::DerivedArgList *
CudaToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args,
                             BoundArch BA,
                             Action::OffloadKind DeviceOffloadKind) const {
  DerivedArgList *DAL = HostTC.TranslateArgs(Args, BA, DeviceOffloadKind);
  if (!DAL)
    DAL = new DerivedArgList(Args.getBaseArgs());
 
  const OptTable &Opts = getDriver().getOpts();
 
  for (Arg *A : Args) {
    // Make sure flags are not duplicated.
    if (!llvm::is_contained(*DAL, A)) {
      DAL->append(A);
    }
  }
 
  if (BA) {
    DAL->eraseArg(options::OPT_march_EQ);
    DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_march_EQ),
                      BA.ArchName);
  }
  return DAL;
}
 
Tool *NVPTXToolChain::buildAssembler() const {
  return new tools::NVPTX::Assembler(*this);
}
 
Tool *NVPTXToolChain::buildLinker() const {
  return new tools::NVPTX::Linker(*this);
}
 
Tool *CudaToolChain::buildAssembler() const {
  return new tools::NVPTX::Assembler(*this);
}
 
Tool *CudaToolChain::buildLinker() const {
  return new tools::NVPTX::FatBinary(*this);
}
 
void CudaToolChain::addClangWarningOptions(ArgStringList &CC1Args) const {
  HostTC.addClangWarningOptions(CC1Args);
}
 
ToolChain::CXXStdlibType
CudaToolChain::GetCXXStdlibType(const ArgList &Args) const {
  return HostTC.GetCXXStdlibType(Args);
}
 
void CudaToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
                                              ArgStringList &CC1Args) const {
  if (DriverArgs.hasFlag(options::OPT_foffload_via_llvm,
                         options::OPT_fno_offload_via_llvm, false))
    return;
 
  HostTC.AddClangSystemIncludeArgs(DriverArgs, CC1Args);
 
  if (DriverArgs.hasFlag(options::OPT_offload_inc, options::OPT_no_offload_inc,
                         true) &&
      CudaInstallation.isValid())
    CC1Args.append(
        {"-internal-isystem",
         DriverArgs.MakeArgString(CudaInstallation.getIncludePath())});
}
 
void CudaToolChain::AddClangCXXStdlibIncludeArgs(const ArgList &Args,
                                                 ArgStringList &CC1Args) const {
  HostTC.AddClangCXXStdlibIncludeArgs(Args, CC1Args);
}
 
void CudaToolChain::AddIAMCUIncludeArgs(const ArgList &Args,
                                        ArgStringList &CC1Args) const {
  HostTC.AddIAMCUIncludeArgs(Args, CC1Args);
}
 
SanitizerMask CudaToolChain::getSupportedSanitizers(
    BoundArch BA, Action::OffloadKind DeviceOffloadKind) const {
  // The CudaToolChain only supports sanitizers in the sense that it allows
  // sanitizer arguments on the command line if they are supported by the host
  // toolchain. The CudaToolChain will actually ignore any command line
  // arguments for any of these "supported" sanitizers. That means that no
  // sanitization of device code is actually supported at this time.
  //
  // This behavior is necessary because the host and device toolchains
  // invocations often share the command line, so the device toolchain must
  // tolerate flags meant only for the host toolchain.
 
  // FIXME: Be accurate and use DeviceOffloadKind.
  return HostTC.getSupportedSanitizers(BA, DeviceOffloadKind);
}
 
VersionTuple CudaToolChain::computeMSVCVersion(const Driver *D,
                                               const ArgList &Args) const {
  return HostTC.computeMSVCVersion(D, Args);
}