CGHLSLRuntime.cpp

Go to the documentation of this file.

//===----- CGHLSLRuntime.cpp - Interface to HLSL Runtimes -----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This provides an abstract class for HLSL code generation.  Concrete
// subclasses of this implement code generation for specific HLSL
// runtime libraries.
//
//===----------------------------------------------------------------------===//
 
#include "CGHLSLRuntime.h"
#include "CGDebugInfo.h"
#include "CGRecordLayout.h"
#include "CodeGenFunction.h"
#include "CodeGenModule.h"
#include "HLSLBufferLayoutBuilder.h"
#include "TargetInfo.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/Attr.h"
#include "clang/AST/Decl.h"
#include "clang/AST/Expr.h"
#include "clang/AST/HLSLResource.h"
#include "clang/AST/RecursiveASTVisitor.h"
#include "clang/AST/Type.h"
#include "clang/Basic/DiagnosticDriver.h"
#include "clang/Basic/DiagnosticFrontend.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/TargetOptions.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Enum.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Frontend/HLSL/HLSLResource.h"
#include "llvm/Frontend/HLSL/RootSignatureMetadata.h"
#include "llvm/Frontend/HLSL/SemanticSignaturePacking.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Alignment.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/Path.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <array>
#include <cstdint>
#include <optional>
 
using namespace clang;
using namespace CodeGen;
using namespace clang::hlsl;
using namespace llvm;
 
using llvm::hlsl::CBufferRowSizeInBytes;
 
namespace {
 
void addDxilValVersion(StringRef ValVersionStr, llvm::Module &M) {
  // The validation of ValVersionStr is done at HLSLToolChain::TranslateArgs.
  // Assume ValVersionStr is legal here.
  VersionTuple Version;
  if (Version.tryParse(ValVersionStr) || Version.getBuild() ||
      Version.getSubminor() || !Version.getMinor()) {
    return;
  }
 
  uint64_t Major = Version.getMajor();
  uint64_t Minor = *Version.getMinor();
 
  auto &Ctx = M.getContext();
  IRBuilder<> B(M);
  MDNode *Val = MDNode::get(Ctx, {ConstantAsMetadata::get(B.getInt32(Major)),
                                  ConstantAsMetadata::get(B.getInt32(Minor))});
  StringRef DXILValKey = "dx.valver";
  auto *DXILValMD = M.getOrInsertNamedMetadata(DXILValKey);
  DXILValMD->addOperand(Val);
}
 
void addRootSignatureMD(llvm::dxbc::RootSignatureVersion RootSigVer,
                        ArrayRef<llvm::hlsl::rootsig::RootElement> Elements,
                        llvm::Function *Fn, llvm::Module &M) {
  auto &Ctx = M.getContext();
 
  llvm::hlsl::rootsig::MetadataBuilder RSBuilder(M, Elements);
  MDNode *RootSignature = RSBuilder.BuildRootSignature();
 
  ConstantAsMetadata *Version = ConstantAsMetadata::get(ConstantInt::get(
      llvm::Type::getInt32Ty(Ctx), llvm::to_underlying(RootSigVer)));
  ValueAsMetadata *EntryFunc = Fn ? ValueAsMetadata::get(Fn) : nullptr;
  MDNode *MDVals = MDNode::get(Ctx, {EntryFunc, RootSignature, Version});
 
  StringRef RootSignatureValKey = "dx.rootsignatures";
  auto *RootSignatureValMD = M.getOrInsertNamedMetadata(RootSignatureValKey);
  RootSignatureValMD->addOperand(MDVals);
}
 
Expected<unsigned> packSemanticSignature(
    MutableArrayRef<llvm::hlsl::SemanticSignatureElement> Elements,
    llvm::Triple::EnvironmentType Stage, llvm::hlsl::IOType IOTy,
    CodeGenOptions::HLSLSemanticSignaturePackingMode Mode,
    bool UseNative16BitTypes) {
  using namespace llvm::hlsl;
  // Vertex inputs and pixel outputs have fixed packing rules, independent of
  // the packing mode used between programmable stages.
  if (Stage == llvm::Triple::Vertex && IOTy == IOType::In)
    return packSignatureStacked(Elements, Stage, IOTy);
  if (Stage == llvm::Triple::Pixel && IOTy == IOType::Out)
    return packSignatureIndexed(Elements, Stage, IOTy);
  if (Mode == CodeGenOptions::HLSLSemanticSignaturePackingMode::Optimized)
    return packSignatureOptimized(Elements, Stage, IOTy, UseNative16BitTypes);
  return packSignaturePrefixStable(Elements, Stage, IOTy, UseNative16BitTypes);
}
 
MDNode *buildSemanticSignatureMD(
    ArrayRef<llvm::hlsl::SemanticSignatureElement> Elements, LLVMContext &Ctx) {
  if (Elements.empty())
    return nullptr;
 
  SmallVector<Metadata *> ElementMD;
  for (const llvm::hlsl::SemanticSignatureElement &Element : Elements)
    ElementMD.push_back(Element.toMetadata(Ctx));
  return MDNode::get(Ctx, ElementMD);
}
 
void addSemanticSignatureMD(
    ArrayRef<llvm::hlsl::SemanticSignatureElement> InputElements,
    ArrayRef<llvm::hlsl::SemanticSignatureElement> OutputElements,
    llvm::Function *Fn, llvm::Module &M) {
  if (InputElements.empty() && OutputElements.empty())
    return;
 
  LLVMContext &Ctx = M.getContext();
  MDNode *InputSignature = buildSemanticSignatureMD(InputElements, Ctx);
  MDNode *OutputSignature = buildSemanticSignatureMD(OutputElements, Ctx);
  MDNode *MDVals = MDNode::get(
      Ctx, {ValueAsMetadata::get(Fn), InputSignature, OutputSignature});
 
  M.getOrInsertNamedMetadata("dx.semantic.signatures")->addOperand(MDVals);
}
 
static void copyGlobalResource(CodeGenFunction &CGF, const VarDecl *ResourceVD,
                               AggValueSlot &DestSlot) {
  GlobalVariable *ResGV =
      cast<GlobalVariable>(CGF.CGM.GetAddrOfGlobalVar(ResourceVD));
  assert(ResGV && "expected valid global variable");
  CGF.Builder.CreateStore(ResGV, DestSlot.getAddress());
}
 
// Given a MemberExpr of a resource or resource array type, find the parent
// VarDecl of the struct or class instance that contains this resource and
// build the full resource name based on the member access path.
//
// For example, for a member access like "myStructArray[0].memberA",
// this function will find the VarDecl of "myStructArray" and use the
// EmbeddedResourceNameBuilder to build the resource name
// "myStructArray.0.memberA".
//
// This also works for a record type expression that has some embedded
// resources. It finds the parent VarDecl of that record and builds a partial
// name which is the prefix of the resource globals associated with the
// declaration.
static const VarDecl *findStructResourceParentDeclAndBuildName(
    const Expr *E, EmbeddedResourceNameBuilder &NameBuilder) {
 
  SmallVector<const Expr *> WorkList;
  const VarDecl *VD = nullptr;
 
  for (;;) {
    if (const auto *DRE = dyn_cast<DeclRefExpr>(E)) {
      assert(isa<VarDecl>(DRE->getDecl()) &&
             "member expr base is not a var decl");
      VD = cast<VarDecl>(DRE->getDecl());
      NameBuilder.pushName(VD->getName());
      break;
    }
 
    WorkList.push_back(E);
    if (const auto *MExp = dyn_cast<MemberExpr>(E))
      E = MExp->getBase();
    else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(E))
      E = ICE->getSubExpr();
    else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(E))
      E = ASE->getBase();
    else if (isa<CXXThisExpr>(E))
      // Resource member access on "this" pointer not yet implemented
      // (llvm/llvm-project#190299)
      return nullptr;
    else
      llvm_unreachable("unexpected expr type in resource member access");
 
    assert(E && "expected valid expression");
  }
 
  while (!WorkList.empty()) {
    E = WorkList.pop_back_val();
    if (const auto *ME = dyn_cast<MemberExpr>(E)) {
      NameBuilder.pushName(
          ME->getMemberNameInfo().getName().getAsIdentifierInfo()->getName());
    } else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(E)) {
      if (ICE->getCastKind() == CK_UncheckedDerivedToBase) {
        CXXRecordDecl *DerivedRD =
            ICE->getSubExpr()->getType()->getAsCXXRecordDecl();
        CXXRecordDecl *BaseRD = ICE->getType()->getAsCXXRecordDecl();
        NameBuilder.pushBaseNameHierarchy(DerivedRD, BaseRD);
      }
    } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(E)) {
      const Expr *IdxExpr = ASE->getIdx();
      std::optional<llvm::APSInt> Value =
          IdxExpr->getIntegerConstantExpr(VD->getASTContext());
      assert(Value &&
             "expected constant index in struct with resource array access");
      NameBuilder.pushArrayIndex(Value->getZExtValue());
    } else {
      llvm_unreachable("unexpected expr type in resource member access");
    }
  }
  return VD;
}
 
// Given a MemberExpr of a resource or resource array type, find the
// corresponding global resource declaration associated with the owning struct
// or class instance via HLSLAssociatedResourceDeclAttr.
static const VarDecl *
findAssociatedResourceDeclForStruct(ASTContext &AST, const MemberExpr *ME) {
 
  EmbeddedResourceNameBuilder NameBuilder;
  const VarDecl *ParentVD =
      findStructResourceParentDeclAndBuildName(ME, NameBuilder);
  if (!ParentVD)
    return nullptr;
 
  if (!ParentVD->hasGlobalStorage())
    return nullptr;
 
  IdentifierInfo *II = NameBuilder.getNameAsIdentifier(AST);
  for (const Attr *A : ParentVD->getAttrs()) {
    if (const auto *ADA = dyn_cast<HLSLAssociatedResourceDeclAttr>(A)) {
      VarDecl *AssocResVD = ADA->getResDecl();
      if (AssocResVD->getIdentifier() == II)
        return AssocResVD;
    }
  }
  return nullptr;
}
 
void addSourceInfo(CodeGenModule &CGM, llvm::Module &M) {
  auto &SM = CGM.getContext().getSourceManager();
  auto &Macros = CGM.getPreprocessorOpts().Macros;
  auto &CodeGenOpts = CGM.getCodeGenOpts();
  auto &Ctx = M.getContext();
 
  // Names and content of shader source code files.
  llvm::NamedMDNode *DXContents =
      M.getOrInsertNamedMetadata("dx.source.contents");
  auto addFile = [&](const std::pair<StringRef, StringRef> &NameContent) {
    llvm::MDTuple *FileInfo =
        llvm::MDNode::get(Ctx, {llvm::MDString::get(Ctx, NameContent.first),
                                llvm::MDString::get(Ctx, NameContent.second)});
    DXContents->addOperand(FileInfo);
  };
 
  bool Invalid = false;
  const SrcMgr::SLocEntry *MainLocEntry =
      &SM.getSLocEntry(SM.getMainFileID(), &Invalid);
  assert(!Invalid && "Main file SLocEntry must not be invalid!");
  const SrcMgr::ContentCache &MainCCEntry =
      MainLocEntry->getFile().getContentCache();
 
  SmallVector<std::pair<std::string, StringRef>> Files;
  std::optional<SmallString<256>> MainFileName;
  Files.reserve(SM.local_sloc_entry_size());
  for (unsigned I : llvm::seq(SM.local_sloc_entry_size())) {
    const SrcMgr::SLocEntry &LocEntry = SM.getLocalSLocEntry(I);
    if (!LocEntry.isFile())
      continue;
 
    const SrcMgr::FileInfo &FInfo = LocEntry.getFile();
    if (isSystem(FInfo.getFileCharacteristic()))
      continue;
 
    const SrcMgr::ContentCache &CCEntry = FInfo.getContentCache();
    OptionalFileEntryRef FEntry = CCEntry.OrigEntry;
    if (!FEntry)
      continue;
 
    llvm::SmallString<256> Path = FEntry->getName();
    llvm::sys::path::native(Path);
    std::optional<llvm::MemoryBufferRef> Buffer = CCEntry.getBufferOrNone(
        SM.getDiagnostics(), SM.getFileManager(), SourceLocation());
    if (!Buffer) {
      SM.getDiagnostics().Report(diag::warn_hlsl_failed_to_embed_source)
          << Path;
      continue;
    }
 
    if (&MainCCEntry != &CCEntry) {
      Files.emplace_back(Path, Buffer->getBuffer());
    } else {
      // Main file should be at first position.
      addFile(std::make_pair(Path, Buffer->getBuffer()));
      MainFileName.emplace(Path);
    }
  }
  assert(MainFileName && "Main file not found.");
 
  // Files other that main one should be sorted by name.
  llvm::sort(Files);
#ifndef NDEBUG
  for (unsigned I = 1; I < Files.size(); ++I)
    assert((Files[I - 1].first != Files[I].first) &&
           "duplicate files in dx.source.contents");
#endif
  llvm::for_each(Files, addFile);
 
  SmallVector<llvm::Metadata *> Defines;
  Defines.reserve(Macros.size());
  for (const auto &Macro : Macros) {
    // Ignore undefs.
    if (!Macro.second)
      Defines.emplace_back(llvm::MDString::get(Ctx, Macro.first));
  }
  M.getOrInsertNamedMetadata("dx.source.defines")
      ->addOperand(llvm::MDNode::get(Ctx, Defines));
 
  if (!CodeGenOpts.MainFileName.empty())
    llvm::sys::path::native(CodeGenOpts.MainFileName, *MainFileName);
  M.getOrInsertNamedMetadata("dx.source.mainFileName")
      ->addOperand(
          llvm::MDNode::get(Ctx, llvm::MDString::get(Ctx, *MainFileName)));
 
  SmallVector<llvm::Metadata *> Args;
  Args.reserve(CodeGenOpts.HLSLParsedCommandLine.size());
  if (!CodeGenOpts.HLSLParsedCommandLine.empty())
    for (const auto &Arg : llvm::drop_begin(CodeGenOpts.HLSLParsedCommandLine))
      Args.push_back(llvm::MDString::get(Ctx, Arg));
  M.getOrInsertNamedMetadata("dx.source.args")
      ->addOperand(llvm::MDNode::get(Ctx, Args));
}
 
// Find array variable declaration from DeclRef expression
static const ValueDecl *getArrayDecl(ASTContext &AST, const Expr *E) {
  E = E->IgnoreImpCasts();
  if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(E))
    return DRE->getDecl();
  if (auto *OVE = dyn_cast<OpaqueValueExpr>(E))
    E = OVE->getSourceExpr()->IgnoreImpCasts();
  if (isa<MemberExpr>(E))
    return findAssociatedResourceDeclForStruct(AST, cast<MemberExpr>(E));
  return nullptr;
}
 
// Find array variable declaration from nested array subscript AST nodes
static const ValueDecl *getArrayDecl(ASTContext &AST,
                                     const ArraySubscriptExpr *ASE) {
  const Expr *E = nullptr;
  while (ASE != nullptr) {
    E = ASE->getBase()->IgnoreImpCasts();
    if (!E)
      return nullptr;
    ASE = dyn_cast<ArraySubscriptExpr>(E);
  }
  return getArrayDecl(AST, E);
}
 
// Get the total size of the array, or 0 if the array is unbounded.
static int getTotalArraySize(ASTContext &AST, const clang::Type *Ty) {
  Ty = Ty->getUnqualifiedDesugaredType();
  assert(Ty->isArrayType() && "expected array type");
  if (Ty->isIncompleteArrayType())
    return 0;
  return AST.getConstantArrayElementCount(cast<ConstantArrayType>(Ty));
}
 
static Value *buildNameForResource(llvm::StringRef BaseName,
                                   CodeGenModule &CGM) {
  llvm::SmallString<64> GlobalName = {BaseName, ".str"};
  return CGM.GetAddrOfConstantCString(BaseName.str(), GlobalName.c_str())
      .getPointer();
}
 
static CXXMethodDecl *lookupMethod(CXXRecordDecl *Record, StringRef Name,
                                   StorageClass SC = SC_None) {
  for (auto *Method : Record->methods()) {
    if (Method->getStorageClass() == SC && Method->getName() == Name)
      return Method;
  }
  return nullptr;
}
 
static CXXMethodDecl *lookupResourceInitMethodAndSetupArgs(
    CodeGenModule &CGM, CXXRecordDecl *ResourceDecl, llvm::Value *Range,
    llvm::Value *Index, StringRef Name, ResourceBindingAttrs &Binding,
    CallArgList &Args) {
  assert(Binding.hasBinding() && "at least one binding attribute expected");
 
  ASTContext &AST = CGM.getContext();
  CXXMethodDecl *CreateMethod = nullptr;
  Value *NameStr = buildNameForResource(Name, CGM);
  Value *Space = llvm::ConstantInt::get(CGM.IntTy, Binding.getSpace());
 
  bool HasCounter = hasCounterHandle(ResourceDecl);
  assert((!HasCounter || Binding.hasCounterImplicitOrderID()) &&
         "resources with counter handle must have a binding with counter "
         "implicit order ID");
  if (Binding.isExplicit()) {
    // explicit binding
    auto *RegSlot = llvm::ConstantInt::get(CGM.IntTy, Binding.getSlot());
    Args.add(RValue::get(RegSlot), AST.UnsignedIntTy);
    const char *Name = Binding.hasCounterImplicitOrderID()
                           ? "__createFromBindingWithImplicitCounter"
                           : "__createFromBinding";
    CreateMethod = lookupMethod(ResourceDecl, Name, SC_Static);
  } else {
    // implicit binding
    auto *OrderID =
        llvm::ConstantInt::get(CGM.IntTy, Binding.getImplicitOrderID());
    Args.add(RValue::get(OrderID), AST.UnsignedIntTy);
    const char *Name = Binding.hasCounterImplicitOrderID()
                           ? "__createFromImplicitBindingWithImplicitCounter"
                           : "__createFromImplicitBinding";
    CreateMethod = lookupMethod(ResourceDecl, Name, SC_Static);
  }
  Args.add(RValue::get(Space), AST.UnsignedIntTy);
  Args.add(RValue::get(Range), AST.IntTy);
  Args.add(RValue::get(Index), AST.UnsignedIntTy);
  Args.add(RValue::get(NameStr), AST.getPointerType(AST.CharTy.withConst()));
  if (HasCounter) {
    uint32_t CounterBinding = Binding.getCounterImplicitOrderID();
    auto *CounterOrderID = llvm::ConstantInt::get(CGM.IntTy, CounterBinding);
    Args.add(RValue::get(CounterOrderID), AST.UnsignedIntTy);
  }
 
  return CreateMethod;
}
 
static void callResourceInitMethod(CodeGenFunction &CGF,
                                   CXXMethodDecl *CreateMethod,
                                   CallArgList &Args, Address ReturnAddress) {
  llvm::Constant *CalleeFn = CGF.CGM.GetAddrOfFunction(CreateMethod);
  const FunctionProtoType *Proto =
      CreateMethod->getType()->getAs<FunctionProtoType>();
  // HLSL code generation is restricted to DXIL and SPIR-V targets, so no
  // caller declaration is needed for x86 SysV ABI selection.
  const CGFunctionInfo &FnInfo = CGF.CGM.getTypes().arrangeFreeFunctionCall(
      Args, Proto, false, /*ABIInfoFD=*/nullptr);
  ReturnValueSlot ReturnValue(ReturnAddress, false);
  CGCallee Callee(CGCalleeInfo(Proto), CalleeFn);
  CGF.EmitCall(FnInfo, Callee, ReturnValue, Args, nullptr);
}
 
// Initializes local resource array variable with global resource array
// elements. For multi-dimensional arrays it calls itself recursively to
// initialize its sub-arrays. The Index used in the resource constructor calls
// will begin at StartIndex and will be incremented for each array element. The
// last used resource Index is returned to the caller. If the function returns
// std::nullopt, it indicates an error.
static std::optional<llvm::Value *> initializeResourceArrayFromGlobal(
    CodeGenFunction &CGF, CXXRecordDecl *ResourceDecl,
    const ConstantArrayType *ArrayTy, AggValueSlot &ValueSlot,
    llvm::Value *Range, llvm::Value *StartIndex, StringRef ResourceName,
    ResourceBindingAttrs &Binding, ArrayRef<llvm::Value *> PrevGEPIndices) {
 
  ASTContext &AST = CGF.getContext();
  llvm::IntegerType *IntTy = CGF.CGM.IntTy;
  llvm::Value *Index = StartIndex;
  llvm::Value *One = llvm::ConstantInt::get(IntTy, 1);
  const uint64_t ArraySize = ArrayTy->getSExtSize();
  QualType ElemType = ArrayTy->getElementType();
  Address TmpArrayAddr = ValueSlot.getAddress();
 
  // Add additional index to the getelementptr call indices.
  // This index will be updated for each array element in the loops below.
  SmallVector<llvm::Value *> GEPIndices(PrevGEPIndices);
  GEPIndices.push_back(llvm::ConstantInt::get(IntTy, 0));
 
  // For array of arrays, recursively initialize the sub-arrays.
  if (ElemType->isArrayType()) {
    const ConstantArrayType *SubArrayTy = cast<ConstantArrayType>(ElemType);
    for (uint64_t I = 0; I < ArraySize; I++) {
      if (I > 0) {
        Index = CGF.Builder.CreateAdd(Index, One);
        GEPIndices.back() = llvm::ConstantInt::get(IntTy, I);
      }
      std::optional<llvm::Value *> MaybeIndex =
          initializeResourceArrayFromGlobal(CGF, ResourceDecl, SubArrayTy,
                                            ValueSlot, Range, Index,
                                            ResourceName, Binding, GEPIndices);
      if (!MaybeIndex)
        return std::nullopt;
      Index = *MaybeIndex;
    }
    return Index;
  }
 
  // For array of resources, initialize each resource in the array.
  llvm::Type *Ty = CGF.ConvertTypeForMem(ElemType);
  CharUnits ElemSize = AST.getTypeSizeInChars(ElemType);
  CharUnits Align =
      TmpArrayAddr.getAlignment().alignmentOfArrayElement(ElemSize);
 
  for (uint64_t I = 0; I < ArraySize; I++) {
    if (I > 0) {
      Index = CGF.Builder.CreateAdd(Index, One);
      GEPIndices.back() = llvm::ConstantInt::get(IntTy, I);
    }
    Address ReturnAddress =
        CGF.Builder.CreateGEP(TmpArrayAddr, GEPIndices, Ty, Align);
 
    CallArgList Args;
    CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
        CGF.CGM, ResourceDecl, Range, Index, ResourceName, Binding, Args);
 
    if (!CreateMethod)
      // This can happen if someone creates an array of structs that looks like
      // an HLSL resource record array but it does not have the required static
      // create method. No binding will be generated for it.
      return std::nullopt;
 
    callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress);
  }
  return Index;
}
 
/// Utility for emitting copies following the HLSL buffer layout rules (ie,
/// copying out of a cbuffer).
class HLSLBufferCopyEmitter {
  CodeGenFunction &CGF;
  Address DstPtr;
  Address SrcPtr;
  llvm::Type *LayoutTy = nullptr;
 
  SmallVector<llvm::Value *> CurStoreIndices;
  SmallVector<llvm::Value *> CurLoadIndices;
 
  using EmitResourceFnTy = llvm::function_ref<void(AggValueSlot &)>;
 
  // Creates & returns either a structured.gep or a ptradd/gep depending on
  // langopts.
  llvm::Value *emitAccessChain(llvm::Type *BaseTy, llvm::Value *Base,
                               ArrayRef<llvm::Value *> Indices) {
    bool EmitLogical = CGF.getLangOpts().EmitLogicalPointer;
    if (EmitLogical)
      return CGF.Builder.CreateAccessChain(EmitLogical, BaseTy, Base, Indices);
 
    llvm::SmallVector<llvm::Value *> GEPIndices;
    GEPIndices.reserve(Indices.size() + 1);
    GEPIndices.push_back(llvm::ConstantInt::get(CGF.IntTy, 0));
    GEPIndices.append(Indices.begin(), Indices.end());
    return CGF.Builder.CreateAccessChain(EmitLogical, BaseTy, Base, GEPIndices);
  }
 
  bool isBufferLayoutArray(llvm::StructType *ST) {
    // A buffer layout array is a struct with two elements: the padded array,
    // and the last element. That is, is should look something like this:
    //
    //   { [%n x { %type, %padding }], %type }
    //
    if (!ST || ST->getNumElements() != 2)
      return false;
 
    auto *PaddedEltsTy = dyn_cast<llvm::ArrayType>(ST->getElementType(0));
    if (!PaddedEltsTy)
      return false;
 
    auto *PaddedTy = dyn_cast<llvm::StructType>(PaddedEltsTy->getElementType());
    if (!PaddedTy || PaddedTy->getNumElements() != 2)
      return false;
 
    if (!CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(
            PaddedTy->getElementType(1)))
      return false;
 
    llvm::Type *ElementTy = ST->getElementType(1);
    if (PaddedTy->getElementType(0) != ElementTy)
      return false;
    return true;
  }
 
  // Returns true if the type is either a struct representing a resource record,
  // or an array of structs that are resource records. This assumes a struct is
  // a resource record if the first element is a target type (resource handle).
  // This is the case for all target types used by HLSL except the padding type
  // ("{dx|spirv.Padding"), but padding will never be the first element of a
  // struct.
  bool isResourceOrResourceArray(llvm::Type *Ty) {
    while (auto *AT = dyn_cast<llvm::ArrayType>(Ty))
      Ty = AT->getElementType();
 
    auto *ST = dyn_cast<llvm::StructType>(Ty);
    if (!ST || ST->getNumElements() < 1)
      return false;
 
    auto *TargetTy = dyn_cast<llvm::TargetExtType>(ST->getElementType(0));
    return TargetTy != nullptr;
  }
 
  void emitResourceOrResourceArray(Value *Dst, llvm::Type *DstTy,
                                   EmitResourceFnTy EmitResFn) {
    CharUnits DstAlign =
        CharUnits::fromQuantity(CGF.CGM.getDataLayout().getABITypeAlign(DstTy));
    Address DstAddr(Dst, DstTy, DstAlign);
    AggValueSlot Slot = AggValueSlot::forAddr(
        DstAddr, Qualifiers(), AggValueSlot::IsDestructed_t(true),
        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsAliased_t(false),
        AggValueSlot::DoesNotOverlap);
 
    EmitResFn(Slot);
  }
 
  void emitBufferLayoutCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
                            llvm::ArrayType *DstTy,
                            EmitResourceFnTy EmitResFn) {
    // Those assumptions are checked by isBufferLayoutArray.
    auto *SrcPaddedArrayTy = cast<llvm::ArrayType>(SrcTy->getElementType(0));
    assert(SrcPaddedArrayTy->getNumElements() + 1 == DstTy->getNumElements());
    assert(cast<llvm::StructType>(SrcPaddedArrayTy->getElementType())
               ->getElementType(0) == SrcTy->getElementType(1));
 
    auto *SrcDataTy = SrcTy->getElementType(1);
    auto Zero = llvm::ConstantInt::get(CGF.IntTy, 0);
 
    for (unsigned I = 0; I < SrcPaddedArrayTy->getNumElements(); ++I) {
      auto Index = llvm::ConstantInt::get(CGF.IntTy, I);
      auto *SrcElt = emitAccessChain(SrcTy, Src, {Zero, Index, Zero});
      auto *DstElt = emitAccessChain(DstTy, Dst, {Index});
      emitElementCopy(SrcElt, SrcDataTy, DstElt, DstTy->getElementType(),
                      EmitResFn);
    }
 
    auto *SrcElt =
        emitAccessChain(SrcTy, Src, {llvm::ConstantInt::get(CGF.IntTy, 1)});
    auto *DstElt = emitAccessChain(
        DstTy, Dst,
        {llvm::ConstantInt::get(CGF.IntTy, DstTy->getNumElements() - 1)});
    emitElementCopy(SrcElt, SrcDataTy, DstElt, DstTy->getElementType(),
                    EmitResFn);
  }
 
  void emitCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
                llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
    assert(!isResourceOrResourceArray(DstTy) &&
           "direct access to resources or resource arrays should be handled "
           "separately");
 
    if (isBufferLayoutArray(SrcTy))
      return emitBufferLayoutCopy(Src, SrcTy, Dst, cast<llvm::ArrayType>(DstTy),
                                  EmitResFn);
 
    unsigned SrcIndex = 0;
    unsigned DstIndex = 0;
 
    // DstTy layout is in default address space and can include resource types.
    // SrcTy is in cbuffer layout where resources are filtered out, so the
    // number of elements in SrcTy can be less than the number of elements in
    // DstTy.
    auto *DstST = cast<llvm::StructType>(DstTy);
    while (DstIndex < DstST->getNumElements()) {
      llvm::Type *DstEltTy = DstST->getElementType(DstIndex);
      if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(DstEltTy)) {
        DstIndex += 1;
        continue;
      }
      if (isResourceOrResourceArray(DstEltTy)) {
        auto *DstElt = emitAccessChain(
            DstTy, Dst, {llvm::ConstantInt::get(CGF.IntTy, DstIndex)});
        emitResourceOrResourceArray(DstElt, DstEltTy, EmitResFn);
        DstIndex += 1;
        continue;
      }
 
      assert(SrcIndex < SrcTy->getNumElements());
      llvm::Type *SrcEltTy = SrcTy->getElementType(SrcIndex);
      if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(SrcEltTy)) {
        SrcIndex += 1;
        continue;
      }
 
      auto *SrcElt = emitAccessChain(
          SrcTy, Src, {llvm::ConstantInt::get(CGF.IntTy, SrcIndex)});
      auto *DstElt = emitAccessChain(
          DstTy, Dst, {llvm::ConstantInt::get(CGF.IntTy, DstIndex)});
      emitElementCopy(SrcElt, SrcEltTy, DstElt, DstEltTy, EmitResFn);
      DstIndex += 1;
      SrcIndex += 1;
    }
  }
 
  void emitCopy(Value *Src, llvm::ArrayType *SrcTy, Value *Dst,
                llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
    for (unsigned I = 0, E = SrcTy->getNumElements(); I < E; ++I) {
      auto *SrcElt =
          emitAccessChain(SrcTy, Src, {llvm::ConstantInt::get(CGF.IntTy, I)});
      auto *DstElt =
          emitAccessChain(DstTy, Dst, {llvm::ConstantInt::get(CGF.IntTy, I)});
      emitElementCopy(SrcElt, SrcTy->getElementType(), DstElt,
                      cast<llvm::ArrayType>(DstTy)->getElementType(),
                      EmitResFn);
    }
  }
 
  void emitElementCopy(Value *Src, llvm::Type *SrcTy, Value *Dst,
                       llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
    if (auto *AT = dyn_cast<llvm::ArrayType>(SrcTy))
      return emitCopy(Src, AT, Dst, DstTy, EmitResFn);
    if (auto *ST = dyn_cast<llvm::StructType>(SrcTy))
      return emitCopy(Src, ST, Dst, DstTy, EmitResFn);
 
    // When we have a scalar or vector element we can emit the copy.
    CharUnits SrcAlign =
        CharUnits::fromQuantity(CGF.CGM.getDataLayout().getABITypeAlign(SrcTy));
    CharUnits DstAlign =
        CharUnits::fromQuantity(CGF.CGM.getDataLayout().getABITypeAlign(DstTy));
    Address SrcAddr(Src, SrcTy, SrcAlign);
    Address DstAddr(Dst, DstTy, DstAlign);
    llvm::Value *Load = CGF.Builder.CreateLoad(SrcAddr, "cbuf.load");
    CGF.Builder.CreateStore(Load, DstAddr);
  }
 
public:
  HLSLBufferCopyEmitter(CodeGenFunction &CGF, Address DstPtr, Address SrcPtr)
      : CGF(CGF), DstPtr(DstPtr), SrcPtr(SrcPtr) {}
 
  bool emitCopy(QualType CType, EmitResourceFnTy EmitResFn = nullptr) {
    LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(CType);
 
    // TODO: We should be able to fall back to a regular memcpy if the layout
    // type doesn't have any padding, but that runs into issues in the backend
    // currently.
    //
    // See https://github.com/llvm/wg-hlsl/issues/351
    emitElementCopy(SrcPtr.getBasePointer(), LayoutTy, DstPtr.getBasePointer(),
                    DstPtr.getElementType(), EmitResFn);
    return true;
  }
};
 
// Represents a list resources associated with a global struct whose name
// starts with the specified prefix.
// The order of HLSLAssociatedResourceDeclAttr attributes is identical to the
// order of the depth-first traversal of the corresponding fields in the struct.
// The resources are always returned in that order, which is the same order
// we need when a struct is copied element-by-element.
class AssociatedResourcesList {
  // Iterator pointers for the associated resource attributes that match the
  // prefix. Begin = begin of the range of attributes that match the prefix End
  // = end of the range of attributes that match the prefix Next = the current
  // attribute in the iteration to be returned by getNextResource
  specific_attr_iterator<HLSLAssociatedResourceDeclAttr> Begin, End, Next;
 
public:
  AssociatedResourcesList(const VarDecl *StructVD,
                          StringRef ResourceNamePrefix) {
    auto I = StructVD->specific_attr_begin<HLSLAssociatedResourceDeclAttr>();
    auto E = StructVD->specific_attr_end<HLSLAssociatedResourceDeclAttr>();
 
    // Skip over associated resources that don't match the prefix.
    while (I != E &&
           !I->getResDecl()->getName().starts_with(ResourceNamePrefix))
      ++I;
    assert(I != E && "expected associated resource not found");
    Begin = End = I;
 
    // Scan over associated resources that do match the prefix to find the end
    // of the range.
    while (I != E && ((HLSLAssociatedResourceDeclAttr *)*I)
                         ->getResDecl()
                         ->getName()
                         .starts_with(ResourceNamePrefix))
      End = ++I;
 
    Next = Begin;
  }
 
  const VarDecl *getNextResource() {
    if (Next == End)
      return nullptr;
 
    const VarDecl *Res = Next->getResDecl();
    ++Next;
    return Res;
  }
};
 
} // namespace
 
llvm::Type *
CGHLSLRuntime::convertHLSLSpecificType(const Type *T,
                                       const CGHLSLOffsetInfo &OffsetInfo) {
  assert(T->isHLSLSpecificType() && "Not an HLSL specific type!");
 
  // Check if the target has a specific translation for this type first.
  if (llvm::Type *TargetTy =
          CGM.getTargetCodeGenInfo().getHLSLType(CGM, T, OffsetInfo))
    return TargetTy;
 
  llvm_unreachable("Generic handling of HLSL types is not supported.");
}
 
llvm::Triple::ArchType CGHLSLRuntime::getArch() {
  return CGM.getTarget().getTriple().getArch();
}
 
// Emits constant global variables for buffer constants declarations
// and creates metadata linking the constant globals with the buffer global.
void CGHLSLRuntime::emitBufferGlobalsAndMetadata(
    const HLSLBufferDecl *BufDecl, llvm::GlobalVariable *BufGV,
    const CGHLSLOffsetInfo &OffsetInfo) {
  LLVMContext &Ctx = CGM.getLLVMContext();
 
  // get the layout struct from constant buffer target type
  llvm::Type *BufType = BufGV->getValueType();
  llvm::StructType *LayoutStruct = cast<llvm::StructType>(
      cast<llvm::TargetExtType>(BufType)->getTypeParameter(0));
 
  SmallVector<std::pair<VarDecl *, uint32_t>> DeclsWithOffset;
  size_t OffsetIdx = 0;
  for (Decl *D : BufDecl->buffer_decls()) {
    if (isa<CXXRecordDecl, EmptyDecl>(D))
      // Nothing to do for this declaration.
      continue;
    if (isa<FunctionDecl>(D)) {
      // A function within an cbuffer is effectively a top-level function.
      CGM.EmitTopLevelDecl(D);
      continue;
    }
    VarDecl *VD = dyn_cast<VarDecl>(D);
    if (!VD)
      continue;
 
    QualType VDTy = VD->getType();
    if (VDTy.getAddressSpace() != LangAS::hlsl_constant) {
      if (VD->getStorageClass() == SC_Static ||
          VDTy.getAddressSpace() == LangAS::hlsl_groupshared ||
          VDTy->isHLSLResourceRecord() || VDTy->isHLSLResourceRecordArray()) {
        // Emit static and groupshared variables and resource classes inside
        // cbuffer as regular globals
        CGM.EmitGlobal(VD);
      }
      continue;
    }
 
    DeclsWithOffset.emplace_back(VD, OffsetInfo[OffsetIdx++]);
  }
 
  if (!OffsetInfo.empty())
    llvm::stable_sort(DeclsWithOffset, [](const auto &LHS, const auto &RHS) {
      return CGHLSLOffsetInfo::compareOffsets(LHS.second, RHS.second);
    });
 
  // Associate the buffer global variable with its constants
  SmallVector<llvm::Metadata *> BufGlobals;
  BufGlobals.reserve(DeclsWithOffset.size() + 1);
  BufGlobals.push_back(ValueAsMetadata::get(BufGV));
 
  auto ElemIt = LayoutStruct->element_begin();
  for (auto &[VD, _] : DeclsWithOffset) {
    if (CGM.getTargetCodeGenInfo().isHLSLPadding(*ElemIt))
      ++ElemIt;
 
    assert(ElemIt != LayoutStruct->element_end() &&
           "number of elements in layout struct does not match");
    llvm::Type *LayoutType = *ElemIt++;
 
    GlobalVariable *ElemGV =
        cast<GlobalVariable>(CGM.GetAddrOfGlobalVar(VD, LayoutType));
    BufGlobals.push_back(ValueAsMetadata::get(ElemGV));
  }
  assert(ElemIt == LayoutStruct->element_end() &&
         "number of elements in layout struct does not match");
 
  // add buffer metadata to the module
  CGM.getModule()
      .getOrInsertNamedMetadata("hlsl.cbs")
      ->addOperand(MDNode::get(Ctx, BufGlobals));
}
 
// Creates resource handle type for the HLSL buffer declaration
static const clang::HLSLAttributedResourceType *
createBufferHandleType(const HLSLBufferDecl *BufDecl) {
  ASTContext &AST = BufDecl->getASTContext();
  QualType QT = AST.getHLSLAttributedResourceType(
      AST.HLSLResourceTy, AST.getCanonicalTagType(BufDecl->getLayoutStruct()),
      HLSLAttributedResourceType::Attributes(ResourceClass::CBuffer));
  return cast<HLSLAttributedResourceType>(QT.getTypePtr());
}
 
CGHLSLOffsetInfo CGHLSLOffsetInfo::fromDecl(const HLSLBufferDecl &BufDecl) {
  CGHLSLOffsetInfo Result;
 
  // If we don't have packoffset info, just return an empty result.
  if (!BufDecl.hasValidPackoffset())
    return Result;
 
  for (Decl *D : BufDecl.buffer_decls()) {
    if (isa<CXXRecordDecl, EmptyDecl>(D) || isa<FunctionDecl>(D)) {
      continue;
    }
    VarDecl *VD = dyn_cast<VarDecl>(D);
    if (!VD || VD->getType().getAddressSpace() != LangAS::hlsl_constant)
      continue;
 
    if (!VD->hasAttrs()) {
      Result.Offsets.push_back(Unspecified);
      continue;
    }
 
    uint32_t Offset = Unspecified;
    for (auto *Attr : VD->getAttrs()) {
      if (auto *POA = dyn_cast<HLSLPackOffsetAttr>(Attr)) {
        Offset = POA->getOffsetInBytes();
        break;
      }
      auto *RBA = dyn_cast<HLSLResourceBindingAttr>(Attr);
      if (RBA &&
          RBA->getRegisterType() == HLSLResourceBindingAttr::RegisterType::C) {
        Offset = RBA->getSlotNumber() * CBufferRowSizeInBytes;
        break;
      }
    }
    Result.Offsets.push_back(Offset);
  }
  return Result;
}
 
// Codegen for HLSLBufferDecl
void CGHLSLRuntime::addBuffer(const HLSLBufferDecl *BufDecl) {
 
  assert(BufDecl->isCBuffer() && "tbuffer codegen is not supported yet");
 
  // create resource handle type for the buffer
  const clang::HLSLAttributedResourceType *ResHandleTy =
      createBufferHandleType(BufDecl);
 
  // empty constant buffer is ignored
  if (ResHandleTy->getContainedType()->getAsCXXRecordDecl()->isEmpty())
    return;
 
  // create global variable for the constant buffer
  CGHLSLOffsetInfo OffsetInfo = CGHLSLOffsetInfo::fromDecl(*BufDecl);
  llvm::Type *LayoutTy = convertHLSLSpecificType(ResHandleTy, OffsetInfo);
  llvm::GlobalVariable *BufGV = new GlobalVariable(
      LayoutTy, /*isConstant*/ false,
      GlobalValue::LinkageTypes::InternalLinkage, PoisonValue::get(LayoutTy),
      llvm::formatv("{0}{1}", BufDecl->getName(),
                    BufDecl->isCBuffer() ? ".cb" : ".tb"),
      GlobalValue::NotThreadLocal);
 
  llvm::Module &M = CGM.getModule();
  M.insertGlobalVariable(BufGV);
 
  // Add the global variable to the compiler used list so it does not
  // get optimized away by GlobalOptPass before it reaches
  // {DXIL|SPIRV}CBufferAccess pass.
  llvm::appendToCompilerUsed(M, {BufGV});
 
  // Add globals for constant buffer elements and create metadata nodes
  emitBufferGlobalsAndMetadata(BufDecl, BufGV, OffsetInfo);
 
  // Initialize cbuffer from binding (implicit or explicit)
  initializeBufferFromBinding(BufDecl, BufGV);
}
 
void CGHLSLRuntime::addRootSignature(
    const HLSLRootSignatureDecl *SignatureDecl) {
  llvm::Module &M = CGM.getModule();
  Triple T(M.getTargetTriple());
 
  // Generated later with the function decl if not targeting root signature
  if (T.getEnvironment() != Triple::EnvironmentType::RootSignature)
    return;
 
  addRootSignatureMD(SignatureDecl->getVersion(),
                     SignatureDecl->getRootElements(), nullptr, M);
}
 
llvm::StructType *
CGHLSLRuntime::getHLSLBufferLayoutType(const RecordType *StructType) {
  const auto Entry = LayoutTypes.find(StructType);
  if (Entry != LayoutTypes.end())
    return Entry->getSecond();
  return nullptr;
}
 
void CGHLSLRuntime::addHLSLBufferLayoutType(const RecordType *StructType,
                                            llvm::StructType *LayoutTy) {
  assert(getHLSLBufferLayoutType(StructType) == nullptr &&
         "layout type for this struct already exist");
  LayoutTypes[StructType] = LayoutTy;
}
 
void CGHLSLRuntime::finishCodeGen() {
  auto &TargetOpts = CGM.getTarget().getTargetOpts();
  auto &CodeGenOpts = CGM.getCodeGenOpts();
  auto &LangOpts = CGM.getLangOpts();
  llvm::Module &M = CGM.getModule();
  Triple T(M.getTargetTriple());
  if (T.getArch() == Triple::ArchType::dxil)
    addDxilValVersion(TargetOpts.DxilValidatorVersion, M);
  if (!CodeGenOpts.DisableDXSourceMetadata &&
      CodeGenOpts.getDebugInfo() >=
          llvm::codegenoptions::DebugInfoKind::DebugInfoConstructor)
    addSourceInfo(CGM, M);
  if (CodeGenOpts.ResMayAlias)
    M.setModuleFlag(llvm::Module::ModFlagBehavior::Error, "dx.resmayalias", 1);
  if (CodeGenOpts.AllResourcesBound)
    M.setModuleFlag(llvm::Module::ModFlagBehavior::Error,
                    "dx.allresourcesbound", 1);
  if (CodeGenOpts.OptimizationLevel == 0)
    M.addModuleFlag(llvm::Module::ModFlagBehavior::Override,
                    "dx.disable_optimizations", 1);
 
  // NativeHalfType corresponds to the -fnative-half-type clang option which is
  // aliased by clang-dxc's -enable-16bit-types option. This option is used to
  // set the UseNativeLowPrecision DXIL module flag in the DirectX backend
  if (LangOpts.NativeHalfType)
    M.setModuleFlag(llvm::Module::ModFlagBehavior::Error, "dx.nativelowprec",
                    1);
 
  if (LangOpts.HLSLSpvPreserveInterface && T.isSPIRV()) {
    // Runs before optimization. Keeps Input/Output globals from GlobalDCE.
    const ASTContext &Ctx = CGM.getContext();
    unsigned InputAS = Ctx.getTargetAddressSpace(LangAS::hlsl_input);
    unsigned OutputAS = Ctx.getTargetAddressSpace(LangAS::hlsl_output);
    SmallVector<GlobalValue *, 8> InterfaceVars;
    for (GlobalVariable &GV : M.globals()) {
      unsigned AS = GV.getAddressSpace();
      if (AS == InputAS || AS == OutputAS)
        InterfaceVars.push_back(&GV);
    }
    if (!InterfaceVars.empty())
      appendToCompilerUsed(M, InterfaceVars);
  }
 
  generateGlobalCtorDtorCalls();
}
 
void clang::CodeGen::CGHLSLRuntime::setHLSLEntryAttributes(
    const FunctionDecl *FD, llvm::Function *Fn) {
  const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
  assert(ShaderAttr && "All entry functions must have a HLSLShaderAttr");
  const StringRef ShaderAttrKindStr = "hlsl.shader";
  Fn->addFnAttr(ShaderAttrKindStr,
                llvm::Triple::getEnvironmentTypeName(ShaderAttr->getType()));
  if (HLSLNumThreadsAttr *NumThreadsAttr = FD->getAttr<HLSLNumThreadsAttr>()) {
    const StringRef NumThreadsKindStr = "hlsl.numthreads";
    std::string NumThreadsStr =
        formatv("{0},{1},{2}", NumThreadsAttr->getX(), NumThreadsAttr->getY(),
                NumThreadsAttr->getZ());
    Fn->addFnAttr(NumThreadsKindStr, NumThreadsStr);
  }
  if (HLSLWaveSizeAttr *WaveSizeAttr = FD->getAttr<HLSLWaveSizeAttr>()) {
    const StringRef WaveSizeKindStr = "hlsl.wavesize";
    std::string WaveSizeStr =
        formatv("{0},{1},{2}", WaveSizeAttr->getMin(), WaveSizeAttr->getMax(),
                WaveSizeAttr->getPreferred());
    Fn->addFnAttr(WaveSizeKindStr, WaveSizeStr);
  }
  // HLSL entry functions are materialized for module functions with
  // HLSLShaderAttr attribute. SetLLVMFunctionAttributesForDefinition called
  // later in the compiler-flow for such module functions is not aware of and
  // hence not able to set attributes of the newly materialized entry functions.
  // So, set attributes of entry function here, as appropriate.
  Fn->addFnAttr(llvm::Attribute::NoInline);
 
  if (CGM.getLangOpts().HLSLSpvEnableMaximalReconvergence) {
    Fn->addFnAttr("enable-maximal-reconvergence", "true");
  }
}
 
static Value *buildVectorInput(IRBuilder<> &B, Function *F, llvm::Type *Ty) {
  // Compute ID intrinsics return i32 components, but the semantic may use
  // 16-bit integers. Narrow each component before assembling the input.
  if (const auto *VT = dyn_cast<FixedVectorType>(Ty)) {
    Value *Result = PoisonValue::get(Ty);
    for (unsigned I = 0; I < VT->getNumElements(); ++I) {
      Value *Elt = B.CreateCall(F, {B.getInt32(I)});
      Elt = B.CreateTrunc(Elt, VT->getElementType());
      Result = B.CreateInsertElement(Result, Elt, I);
    }
    return Result;
  }
  return B.CreateTrunc(B.CreateCall(F, {B.getInt32(0)}), Ty);
}
 
static void addSPIRVBuiltinDecoration(llvm::GlobalVariable *GV,
                                      unsigned BuiltIn) {
  LLVMContext &Ctx = GV->getContext();
  IRBuilder<> B(*GV->getParent());
  MDNode *Operands = MDNode::get(
      Ctx,
      {ConstantAsMetadata::get(B.getInt32(/* Spirv::Decoration::BuiltIn */ 11)),
       ConstantAsMetadata::get(B.getInt32(BuiltIn))});
  MDNode *Decoration = MDNode::get(Ctx, {Operands});
  GV->addMetadata("spirv.Decorations", *Decoration);
}
 
static void addLocationDecoration(llvm::GlobalVariable *GV, unsigned Location) {
  LLVMContext &Ctx = GV->getContext();
  IRBuilder<> B(*GV->getParent());
  MDNode *Operands =
      MDNode::get(Ctx, {ConstantAsMetadata::get(B.getInt32(/* Location */ 30)),
                        ConstantAsMetadata::get(B.getInt32(Location))});
  MDNode *Decoration = MDNode::get(Ctx, {Operands});
  GV->addMetadata("spirv.Decorations", *Decoration);
}
 
// A fragment shader input interface variable whose base type is an integer or
// a 64-bit float (double) cannot be interpolated by the rasterizer. The Vulkan
// specification requires these variables to be decorated with Flat (see
// VUID-StandaloneSpirv-Flat-04744). Arrays and vectors are unwrapped to inspect
// their base scalar type.
static bool inputRequiresFlatDecoration(llvm::Type *Ty) {
  while (true) {
    if (auto *AT = dyn_cast<llvm::ArrayType>(Ty)) {
      Ty = AT->getElementType();
      continue;
    }
    if (auto *VT = dyn_cast<llvm::FixedVectorType>(Ty)) {
      Ty = VT->getElementType();
      continue;
    }
    break;
  }
  return Ty->isIntegerTy() || Ty->isDoubleTy();
}
 
static llvm::Value *createSPIRVBuiltinLoad(IRBuilder<> &B, llvm::Module &M,
                                           llvm::Type *Ty, const Twine &Name,
                                           unsigned BuiltInID) {
  auto *GV = new llvm::GlobalVariable(
      M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
      /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
      llvm::GlobalVariable::GeneralDynamicTLSModel,
      /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
  addSPIRVBuiltinDecoration(GV, BuiltInID);
  GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
  return B.CreateLoad(Ty, GV);
}
 
static llvm::Value *createSPIRVLocationLoad(IRBuilder<> &B, llvm::Module &M,
                                            llvm::Type *Ty, unsigned Location,
                                            StringRef Name, bool NeedsFlat) {
  auto *GV = new llvm::GlobalVariable(
      M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
      /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr,
      llvm::GlobalVariable::GeneralDynamicTLSModel,
      /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
  GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
 
  // Emit all decorations as a single `spirv.Decorations` node. Attaching
  // multiple `spirv.Decorations` metadata nodes to the same global is not
  // supported by the SPIR-V backend and results in all but one being dropped.
  LLVMContext &Ctx = GV->getContext();
  SmallVector<Metadata *, 2> Decorations;
  Decorations.push_back(
      MDNode::get(Ctx, {ConstantAsMetadata::get(
                            B.getInt32(/* SPIRV::Decoration::Location */ 30)),
                        ConstantAsMetadata::get(B.getInt32(Location))}));
  if (NeedsFlat)
    Decorations.push_back(
        MDNode::get(Ctx, {ConstantAsMetadata::get(
                             B.getInt32(/* SPIRV::Decoration::Flat */ 14))}));
  GV->addMetadata("spirv.Decorations", *MDNode::get(Ctx, Decorations));
 
  return B.CreateLoad(Ty, GV);
}
 
llvm::Value *CGHLSLRuntime::emitSPIRVUserSemanticLoad(
    llvm::IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
    const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
    std::optional<unsigned> Index) {
  Twine BaseName = Twine(Semantic->getAttrName()->getName());
  Twine VariableName = BaseName.concat(Twine(Index.value_or(0)));
 
  unsigned Location = SPIRVLastAssignedInputSemanticLocation;
  if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
    Location = L->getLocation();
 
  // DXC completely ignores the semantic/index pair. Location are assigned from
  // the first semantic to the last.
  llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Type);
  unsigned ElementCount = AT ? AT->getNumElements() : 1;
  SPIRVLastAssignedInputSemanticLocation += ElementCount;
 
  const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
  bool NeedsFlat =
      ShaderAttr &&
      ShaderAttr->getType() == llvm::Triple::EnvironmentType::Pixel &&
      inputRequiresFlatDecoration(Type);
 
  return createSPIRVLocationLoad(B, CGM.getModule(), Type, Location,
                                 VariableName.str(), NeedsFlat);
}
 
static void createSPIRVLocationStore(IRBuilder<> &B, llvm::Module &M,
                                     llvm::Value *Source, unsigned Location,
                                     StringRef Name) {
  auto *GV = new llvm::GlobalVariable(
      M, Source->getType(), /* isConstant= */ false,
      llvm::GlobalValue::ExternalLinkage,
      /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr,
      llvm::GlobalVariable::GeneralDynamicTLSModel,
      /* AddressSpace */ 8, /* isExternallyInitialized= */ false);
  GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
  addLocationDecoration(GV, Location);
  B.CreateStore(Source, GV);
}
 
void CGHLSLRuntime::emitSPIRVUserSemanticStore(
    llvm::IRBuilder<> &B, llvm::Value *Source,
    const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
    std::optional<unsigned> Index) {
  Twine BaseName = Twine(Semantic->getAttrName()->getName());
  Twine VariableName = BaseName.concat(Twine(Index.value_or(0)));
 
  unsigned Location = SPIRVLastAssignedOutputSemanticLocation;
  if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
    Location = L->getLocation();
 
  // DXC completely ignores the semantic/index pair. Location are assigned from
  // the first semantic to the last.
  llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Source->getType());
  unsigned ElementCount = AT ? AT->getNumElements() : 1;
  SPIRVLastAssignedOutputSemanticLocation += ElementCount;
  createSPIRVLocationStore(B, CGM.getModule(), Source, Location,
                           VariableName.str());
}
 
namespace {
// Describes how a semantic leaf lowers to signature rows
struct SemanticShape {
  SmallVector<unsigned> Dimensions; // Empty dims denotes a scalar
  unsigned Cols;
  QualType RowType;
 
  unsigned getNumRows() const {
    unsigned Rows = 1;
    for (unsigned Dimension : Dimensions)
      Rows *= Dimension;
    return Rows;
  }
 
  SmallVector<unsigned> getArrayIndicesForRow(unsigned Row) const {
    assert(Row < getNumRows() && "row exceeds semantic shape");
 
    SmallVector<unsigned> Indices(Dimensions.size());
    for (auto [Index, Dimension] :
         llvm::zip_equal(llvm::reverse(Indices), llvm::reverse(Dimensions))) {
      Index = Row % Dimension;
      Row /= Dimension;
    }
    return Indices;
  }
};
} // namespace
 
// Returns the QualType of a semantic leaf declarator. For a function the
// declared return type is used, otherwise the declared type.
static QualType getSemanticLeafType(const clang::DeclaratorDecl *Decl) {
  if (const auto *FD = dyn_cast<clang::FunctionDecl>(Decl))
    return FD->getDeclaredReturnType();
  return Decl->getType();
}
 
// Walks through the surrounding constant array types of \p Ty, collecting their
// dimensions until reaching a scalar, vector, or matrix leaf.
static SemanticShape getSemanticShape(ASTContext &Ctx, QualType Ty) {
  SemanticShape Shape{{}, 1, Ty};
  while (const ConstantArrayType *CAT =
             Ctx.getAsConstantArrayType(Shape.RowType)) {
    Shape.Dimensions.push_back(CAT->getSize().getZExtValue());
    Shape.RowType = CAT->getElementType();
  }
 
  if (const auto *VT = Shape.RowType->getAs<clang::VectorType>()) {
    Shape.Cols = VT->getNumElements();
  } else if (const auto *MT =
                 Shape.RowType->getAs<clang::ConstantMatrixType>()) {
    // FIXME: a matrix leaf lowers to one row per matrix row but if column_major
    // is specified we transpose the num rows and num cols, this depends on
    // #211977 to resolve
    Shape.Cols = MT->getNumColumns();
  }
 
  return Shape;
}
 
static llvm::dxil::ElementType getSignatureComponentType(CodeGenModule &CGM,
                                                         QualType Ty) {
  if (const auto *VT = Ty->getAs<clang::VectorType>())
    Ty = VT->getElementType();
  else if (const auto *MT = Ty->getAs<clang::ConstantMatrixType>())
    Ty = MT->getElementType();
 
  llvm::Type *IRTy = CGM.getTypes().ConvertTypeForMem(Ty);
  bool IsSigned = Ty->isSignedIntegerOrEnumerationType();
  return llvm::hlsl::getDXILElementType(IRTy, IsSigned);
}
 
static llvm::hlsl::SemanticSignatureElement createSemanticSignatureElement(
    CodeGenModule &CGM, uint32_t SigId, HLSLAppliedSemanticAttr *Semantic,
    std::optional<unsigned> Index, const SemanticShape &Shape) {
  StringRef Name = Semantic->getAttrName()->getName();
 
  // One semantic index per row, starting from the declared index.
  SmallVector<uint32_t> SemanticIndices;
  uint32_t FirstSemanticIndex = Index.value_or(0);
  for (uint32_t I = 0, E = Shape.getNumRows(); I < E; ++I)
    SemanticIndices.push_back(FirstSemanticIndex + I);
 
  // The remaining members keep their default value and will be filled at a
  // later stage, either during packing or analysis of usage. Interpolation
  // is set by the load/store traversal after visiting each leaf.
  return llvm::hlsl::SemanticSignatureElement(
      SigId, Name, getSignatureComponentType(CGM, Shape.RowType),
      llvm::hlsl::getSemanticKind(Name), SemanticIndices,
      static_cast<uint8_t>(Shape.Cols));
}
 
llvm::Value *CGHLSLRuntime::emitDXILUserSemanticLoad(
    llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
    HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index,
    SemanticSignatures &Signature) {
  StringRef Name = Semantic->getAttrName()->getName();
  SemanticShape Shape =
      getSemanticShape(CGM.getContext(), getSemanticLeafType(Decl));
 
  uint32_t SigId = Signature.size();
  Signature.push_back(
      createSemanticSignatureElement(CGM, SigId, Semantic, Index, Shape));
 
  llvm::Type *RowTy = CGM.getTypes().ConvertTypeForMem(Shape.RowType);
 
  llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
      B.getModule(), llvm::Intrinsic::dx_load_input, {RowTy});
 
  SmallVector<OperandBundleDef, 1> OB;
  if (auto *Token = getConvergenceToken(*B.GetInsertBlock())) {
    llvm::Value *bundleArgs[] = {Token};
    OB.emplace_back("convergencectrl", bundleArgs);
  }
 
  llvm::Type *LeafTy = CGM.getTypes().ConvertType(Shape.RowType);
  llvm::Value *Result = llvm::PoisonValue::get(Type);
 
  const unsigned NumRows = Shape.getNumRows();
 
  for (unsigned Row = 0; Row < NumRows; ++Row) {
    SmallVector<unsigned> Indices = Shape.getArrayIndicesForRow(Row);
    std::array<Value *, 4> Args{
        /*SigElementId=*/B.getInt32(SigId),
        /*RowIndex=*/B.getInt32(Row),
        /*ColIndex=*/B.getInt8(0),
        /*GsVertexOrPrimIndex=*/llvm::PoisonValue::get(B.getInt32Ty())};
    llvm::Value *Value =
        B.CreateCall(IntrFn, Args, OB, Twine(Name).concat(Twine(Row)));
    // Booleans use their memory representation in DXIL signatures, but
    // function parameters use their value representation.
    if (Value->getType() != LeafTy) {
      assert(Shape.RowType->hasBooleanRepresentation() &&
             "unexpected semantic load type mismatch");
      Value = B.CreateICmpNE(
          Value, llvm::Constant::getNullValue(Value->getType()), "loadedv");
    }
 
    Result =
        Indices.empty() ? Value : B.CreateInsertValue(Result, Value, Indices);
  }
  return Result;
}
 
void CGHLSLRuntime::emitDXILUserSemanticStore(llvm::IRBuilder<> &B,
                                              llvm::Value *Source,
                                              const clang::DeclaratorDecl *Decl,
                                              HLSLAppliedSemanticAttr *Semantic,
                                              std::optional<unsigned> Index,
                                              SemanticSignatures &Signature) {
  SemanticShape Shape =
      getSemanticShape(CGM.getContext(), getSemanticLeafType(Decl));
 
  uint32_t SigId = Signature.size();
  Signature.push_back(
      createSemanticSignatureElement(CGM, SigId, Semantic, Index, Shape));
 
  llvm::Type *RowTy = CGM.getTypes().ConvertTypeForMem(Shape.RowType);
 
  llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
      B.getModule(), llvm::Intrinsic::dx_store_output, {RowTy});
 
  SmallVector<OperandBundleDef, 1> OB;
  if (auto *Token = getConvergenceToken(*B.GetInsertBlock())) {
    llvm::Value *bundleArgs[] = {Token};
    OB.emplace_back("convergencectrl", bundleArgs);
  }
 
  const unsigned NumRows = Shape.getNumRows();
  for (unsigned Row = 0; Row < NumRows; ++Row) {
    SmallVector<unsigned> Indices = Shape.getArrayIndicesForRow(Row);
    llvm::Value *Val =
        Indices.empty() ? Source : B.CreateExtractValue(Source, Indices);
 
    // Booleans use their memory representation in DXIL signatures, but direct
    // function results use their value representation.
    if (Val->getType() != RowTy) {
      assert(Shape.RowType->hasBooleanRepresentation() &&
             "unexpected semantic store type mismatch");
      Val = B.CreateZExt(Val, RowTy, "storedv");
    }
 
    std::array<Value *, 4> Args{/*SigElementId=*/B.getInt32(SigId),
                                /*RowIndex=*/B.getInt32(Row),
                                /*ColIndex=*/B.getInt8(0), /*Value=*/Val};
    B.CreateCall(IntrFn, Args, OB);
  }
}
 
llvm::Value *CGHLSLRuntime::emitUserSemanticLoad(
    IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
    const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
    std::optional<unsigned> Index, SemanticSignatures &Signature) {
  if (CGM.getTarget().getTriple().isSPIRV())
    return emitSPIRVUserSemanticLoad(B, FD, Type, Decl, Semantic, Index);
 
  if (CGM.getTarget().getTriple().isDXIL())
    return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index, Signature);
 
  llvm_unreachable("Unsupported target for user-semantic load.");
}
 
void CGHLSLRuntime::emitUserSemanticStore(IRBuilder<> &B, llvm::Value *Source,
                                          const clang::DeclaratorDecl *Decl,
                                          HLSLAppliedSemanticAttr *Semantic,
                                          std::optional<unsigned> Index,
                                          SemanticSignatures &Signature) {
  if (CGM.getTarget().getTriple().isSPIRV())
    return emitSPIRVUserSemanticStore(B, Source, Decl, Semantic, Index);
 
  if (CGM.getTarget().getTriple().isDXIL())
    return emitDXILUserSemanticStore(B, Source, Decl, Semantic, Index,
                                     Signature);
 
  llvm_unreachable("Unsupported target for user-semantic load.");
}
 
llvm::Value *CGHLSLRuntime::emitSystemSemanticLoad(
    IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
    HLSLAppliedSemanticAttr *Semantic,
    llvm::dxbc::PSV::SemanticKind SemanticKind,
    llvm::Triple::EnvironmentType Stage, std::optional<unsigned> Index,
    SemanticSignatures &Signature) {
  switch (SemanticKind) {
  case llvm::dxbc::PSV::SemanticKind::GroupIndex: {
    assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
                               llvm::Triple::Amplification},
                              Stage) &&
           "SV_GroupIndex is in an unavailable stage and should have been "
           "diagnosed by Sema");
    assert(Stage != llvm::Triple::Mesh &&
           Stage != llvm::Triple::Amplification &&
           "FIXME: SV_GroupIndex is not yet implemented for this shader "
           "stage");
    llvm::Function *GroupIndex =
        CGM.getIntrinsic(getFlattenedThreadIdInGroupIntrinsic());
    return B.CreateCall(FunctionCallee(GroupIndex));
  }
  case llvm::dxbc::PSV::SemanticKind::DispatchThreadID: {
    assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
                               llvm::Triple::Amplification},
                              Stage) &&
           "SV_DispatchThreadID is in an unavailable stage and should have "
           "been diagnosed by Sema");
    assert(Stage != llvm::Triple::Mesh &&
           Stage != llvm::Triple::Amplification &&
           "FIXME: SV_DispatchThreadID is not yet implemented for this "
           "shader stage");
    llvm::Intrinsic::ID IntrinID = getThreadIdIntrinsic();
    llvm::Function *ThreadIDIntrinsic =
        llvm::Intrinsic::isOverloaded(IntrinID)
            ? CGM.getIntrinsic(IntrinID, {CGM.Int32Ty})
            : CGM.getIntrinsic(IntrinID);
    return buildVectorInput(B, ThreadIDIntrinsic, Type);
  }
  case llvm::dxbc::PSV::SemanticKind::GroupThreadID: {
    assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
                               llvm::Triple::Amplification},
                              Stage) &&
           "SV_GroupThreadID is in an unavailable stage and should have been "
           "diagnosed by Sema");
    assert(Stage != llvm::Triple::Mesh &&
           Stage != llvm::Triple::Amplification &&
           "FIXME: SV_GroupThreadID is not yet implemented for this shader "
           "stage");
    llvm::Intrinsic::ID IntrinID = getGroupThreadIdIntrinsic();
    llvm::Function *GroupThreadIDIntrinsic =
        llvm::Intrinsic::isOverloaded(IntrinID)
            ? CGM.getIntrinsic(IntrinID, {CGM.Int32Ty})
            : CGM.getIntrinsic(IntrinID);
    return buildVectorInput(B, GroupThreadIDIntrinsic, Type);
  }
  case llvm::dxbc::PSV::SemanticKind::GroupID: {
    assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
                               llvm::Triple::Amplification},
                              Stage) &&
           "SV_GroupID is in an unavailable stage and should have been "
           "diagnosed by Sema");
    assert(Stage != llvm::Triple::Mesh &&
           Stage != llvm::Triple::Amplification &&
           "FIXME: SV_GroupID is not yet implemented for this shader stage");
    llvm::Intrinsic::ID IntrinID = getGroupIdIntrinsic();
    llvm::Function *GroupIDIntrinsic =
        llvm::Intrinsic::isOverloaded(IntrinID)
            ? CGM.getIntrinsic(IntrinID, {CGM.Int32Ty})
            : CGM.getIntrinsic(IntrinID);
    return buildVectorInput(B, GroupIDIntrinsic, Type);
  }
  case llvm::dxbc::PSV::SemanticKind::Position:
    assert(llvm::is_contained({llvm::Triple::Hull, llvm::Triple::Domain,
                               llvm::Triple::Geometry, llvm::Triple::Pixel},
                              Stage) &&
           "SV_Position is in an unavailable stage and should have been "
           "diagnosed by Sema");
    assert(Stage != llvm::Triple::Hull && Stage != llvm::Triple::Domain &&
           Stage != llvm::Triple::Geometry &&
           "FIXME: loading SV_Position is not yet implemented for this "
           "shader stage");
    if (CGM.getTarget().getTriple().isSPIRV())
      return createSPIRVBuiltinLoad(B, CGM.getModule(), Type,
                                    Semantic->getAttrName()->getName(),
                                    /* BuiltIn::FragCoord */ 15);
    if (CGM.getTarget().getTriple().isDXIL())
      return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
                                      Signature);
    break;
  case llvm::dxbc::PSV::SemanticKind::VertexID:
    assert(Stage == llvm::Triple::Vertex &&
           "SV_VertexID is in an unavailable stage and should have been "
           "diagnosed by Sema");
    if (CGM.getTarget().getTriple().isSPIRV())
      return createSPIRVBuiltinLoad(B, CGM.getModule(), Type,
                                    Semantic->getAttrName()->getName(),
                                    /* BuiltIn::VertexIndex */ 42);
    if (CGM.getTarget().getTriple().isDXIL())
      return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
                                      Signature);
    break;
  case llvm::dxbc::PSV::SemanticKind::InstanceID:
    assert(Stage == llvm::Triple::Vertex &&
           "SV_InstanceID is in an unavailable stage and should have been "
           "diagnosed by Sema");
    if (CGM.getTarget().getTriple().isSPIRV())
      return createSPIRVBuiltinLoad(B, CGM.getModule(), Type,
                                    Semantic->getAttrName()->getName(),
                                    /* BuiltIn::InstanceIndex */ 43);
    if (CGM.getTarget().getTriple().isDXIL())
      return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
                                      Signature);
    break;
  default:
    break;
  }
 
  llvm_unreachable(
      "Load hasn't been implemented yet for this system semantic. FIXME");
}
 
static void createSPIRVBuiltinStore(IRBuilder<> &B, llvm::Module &M,
                                    llvm::Value *Source, const Twine &Name,
                                    unsigned BuiltInID) {
  auto *GV = new llvm::GlobalVariable(
      M, Source->getType(), /* isConstant= */ false,
      llvm::GlobalValue::ExternalLinkage,
      /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
      llvm::GlobalVariable::GeneralDynamicTLSModel,
      /* AddressSpace */ 8, /* isExternallyInitialized= */ false);
  addSPIRVBuiltinDecoration(GV, BuiltInID);
  GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
  B.CreateStore(Source, GV);
}
 
void CGHLSLRuntime::emitSystemSemanticStore(
    IRBuilder<> &B, llvm::Value *Source, const clang::DeclaratorDecl *Decl,
    HLSLAppliedSemanticAttr *Semantic,
    llvm::dxbc::PSV::SemanticKind SemanticKind,
    llvm::Triple::EnvironmentType Stage, std::optional<unsigned> Index,
    SemanticSignatures &Signature) {
  switch (SemanticKind) {
  case llvm::dxbc::PSV::SemanticKind::Position:
    assert(llvm::is_contained({llvm::Triple::Vertex, llvm::Triple::Hull,
                               llvm::Triple::Domain, llvm::Triple::Geometry,
                               llvm::Triple::Mesh},
                              Stage) &&
           "SV_Position is in an unavailable stage and should have been "
           "diagnosed by Sema");
    assert(Stage != llvm::Triple::Hull && Stage != llvm::Triple::Domain &&
           Stage != llvm::Triple::Geometry && Stage != llvm::Triple::Mesh &&
           "FIXME: storing SV_Position is not yet implemented for this "
           "shader stage");
    if (CGM.getTarget().getTriple().isDXIL()) {
      emitDXILUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
      return;
    }
    if (CGM.getTarget().getTriple().isSPIRV()) {
      createSPIRVBuiltinStore(B, CGM.getModule(), Source,
                              Semantic->getAttrName()->getName(),
                              /* BuiltIn::Position */ 0);
      return;
    }
    break;
  case llvm::dxbc::PSV::SemanticKind::Target:
    assert(Stage == llvm::Triple::Pixel &&
           "SV_Target is in an unavailable stage and should have been "
           "diagnosed by Sema");
    emitUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
    return;
  default:
    break;
  }
 
  llvm_unreachable(
      "Store hasn't been implemented yet for this system semantic. FIXME");
}
 
llvm::Value *CGHLSLRuntime::handleScalarSemanticLoad(
    IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
    const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
    SemanticSignatures &Signature) {
 
  std::optional<unsigned> Index = Semantic->getSemanticIndex();
  llvm::dxbc::PSV::SemanticKind SemanticKind =
      llvm::hlsl::getSemanticKind(Semantic->getAttrName()->getName());
  const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
  assert(ShaderAttr && "Entry point has no shader attribute");
  llvm::hlsl::SemanticInterpretation Interpretation =
      llvm::hlsl::getInterpretationKind(SemanticKind, ShaderAttr->getType(),
                                        llvm::hlsl::IOType::In);
  assert(Interpretation != llvm::hlsl::SemanticInterpretation::Invalid &&
         "invalid semantic should have been diagnosed by Sema");
  if (Interpretation == llvm::hlsl::SemanticInterpretation::Arbitrary)
    return emitUserSemanticLoad(B, FD, Type, Decl, Semantic, Index, Signature);
  return emitSystemSemanticLoad(B, Type, Decl, Semantic, SemanticKind,
                                ShaderAttr->getType(), Index, Signature);
}
 
void CGHLSLRuntime::handleScalarSemanticStore(IRBuilder<> &B,
                                              const FunctionDecl *FD,
                                              llvm::Value *Source,
                                              const clang::DeclaratorDecl *Decl,
                                              HLSLAppliedSemanticAttr *Semantic,
                                              SemanticSignatures &Signature) {
  std::optional<unsigned> Index = Semantic->getSemanticIndex();
  llvm::dxbc::PSV::SemanticKind SemanticKind =
      llvm::hlsl::getSemanticKind(Semantic->getAttrName()->getName());
  const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
  assert(ShaderAttr && "Entry point has no shader attribute");
 
  llvm::hlsl::SemanticInterpretation Interpretation =
      llvm::hlsl::getInterpretationKind(SemanticKind, ShaderAttr->getType(),
                                        llvm::hlsl::IOType::Out);
  assert(Interpretation != llvm::hlsl::SemanticInterpretation::Invalid &&
         "invalid semantic should have been diagnosed by Sema");
 
  if (Interpretation == llvm::hlsl::SemanticInterpretation::Arbitrary)
    return emitUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
  emitSystemSemanticStore(B, Source, Decl, Semantic, SemanticKind,
                          ShaderAttr->getType(), Index, Signature);
}
 
std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
CGHLSLRuntime::handleStructSemanticLoad(
    IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
    const clang::DeclaratorDecl *Decl,
    specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
    specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
    SemanticSignatures &Signature,
    llvm::hlsl::InterpolationModifier Modifiers) {
  const llvm::StructType *ST = cast<StructType>(Type);
  const clang::RecordDecl *RD = Decl->getType()->getAsRecordDecl();
 
  assert(RD->getNumFields() == ST->getNumElements());
 
  llvm::Value *Aggregate = llvm::PoisonValue::get(Type);
  auto FieldDecl = RD->field_begin();
  for (unsigned I = 0; I < ST->getNumElements(); ++I) {
    auto [ChildValue, NextAttr] =
        handleSemanticLoad(B, FD, ST->getElementType(I), *FieldDecl, AttrBegin,
                           AttrEnd, Signature, Modifiers);
    AttrBegin = NextAttr;
    assert(ChildValue);
    Aggregate = B.CreateInsertValue(Aggregate, ChildValue, I);
    ++FieldDecl;
  }
 
  return std::make_pair(Aggregate, AttrBegin);
}
 
specific_attr_iterator<HLSLAppliedSemanticAttr>
CGHLSLRuntime::handleStructSemanticStore(
    IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
    const clang::DeclaratorDecl *Decl,
    specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
    specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
    SemanticSignatures &Signature,
    llvm::hlsl::InterpolationModifier Modifiers) {
 
  const llvm::StructType *ST = cast<StructType>(Source->getType());
 
  const clang::RecordDecl *RD = nullptr;
  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Decl))
    RD = FD->getDeclaredReturnType()->getAsRecordDecl();
  else
    RD = Decl->getType()->getAsRecordDecl();
  assert(RD);
 
  assert(RD->getNumFields() == ST->getNumElements());
 
  auto FieldDecl = RD->field_begin();
  for (unsigned I = 0; I < ST->getNumElements(); ++I, ++FieldDecl) {
    llvm::Value *Extract = B.CreateExtractValue(Source, I);
    AttrBegin = handleSemanticStore(B, FD, Extract, *FieldDecl, AttrBegin,
                                    AttrEnd, Signature, Modifiers);
  }
 
  return AttrBegin;
}
 
std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
CGHLSLRuntime::handleSemanticLoad(
    IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
    const clang::DeclaratorDecl *Decl,
    specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
    specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
    SemanticSignatures &Signature,
    llvm::hlsl::InterpolationModifier Modifiers) {
  assert(AttrBegin != AttrEnd);
  // Pass the enclosing declaration's mask down the traversal, replacing (not
  // merging) it when an inner field has its own interpolation modifiers.
  if (const auto *A = Decl->getAttr<HLSLInterpolationModifierAttr>())
    Modifiers =
        static_cast<llvm::hlsl::InterpolationModifier>(A->getModifiers());
  if (Type->isStructTy())
    return handleStructSemanticLoad(B, FD, Type, Decl, AttrBegin, AttrEnd,
                                    Signature, Modifiers);
 
  HLSLAppliedSemanticAttr *Attr = *AttrBegin;
  ++AttrBegin;
  size_t PreviousSize = Signature.size();
  llvm::Value *Value =
      handleScalarSemanticLoad(B, FD, Type, Decl, Attr, Signature);
  // Intrinsic-only system values and SPIR-V loads do not add DXIL signature
  // elements. Non-pixel inputs retain Undefined.
  if (Signature.size() != PreviousSize) {
    auto &Element = Signature.back();
    Element.InterpMode = llvm::hlsl::normalizeInterpolationMode(
        llvm::hlsl::getInterpolationMode(Modifiers), Element.CompType,
        Element.SemanticKind, FD->getAttr<HLSLShaderAttr>()->getType(),
        llvm::hlsl::IOType::In);
  }
  return std::make_pair(Value, AttrBegin);
}
 
specific_attr_iterator<HLSLAppliedSemanticAttr>
CGHLSLRuntime::handleSemanticStore(
    IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
    const clang::DeclaratorDecl *Decl,
    specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
    specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
    SemanticSignatures &Signature,
    llvm::hlsl::InterpolationModifier Modifiers) {
  assert(AttrBegin != AttrEnd);
  // An inner field overrides the enclosing return declaration's modifiers.
  if (const auto *A = Decl->getAttr<HLSLInterpolationModifierAttr>())
    Modifiers =
        static_cast<llvm::hlsl::InterpolationModifier>(A->getModifiers());
  if (Source->getType()->isStructTy())
    return handleStructSemanticStore(B, FD, Source, Decl, AttrBegin, AttrEnd,
                                     Signature, Modifiers);
 
  HLSLAppliedSemanticAttr *Attr = *AttrBegin;
  ++AttrBegin;
  size_t PreviousSize = Signature.size();
  handleScalarSemanticStore(B, FD, Source, Decl, Attr, Signature);
  if (Signature.size() != PreviousSize) {
    auto &Element = Signature.back();
    Element.InterpMode = llvm::hlsl::normalizeInterpolationMode(
        llvm::hlsl::getInterpolationMode(Modifiers), Element.CompType,
        Element.SemanticKind, FD->getAttr<HLSLShaderAttr>()->getType(),
        llvm::hlsl::IOType::Out);
  }
  return AttrBegin;
}
 
void CGHLSLRuntime::emitEntryFunction(const FunctionDecl *FD,
                                      llvm::Function *Fn) {
  SmallVector<llvm::hlsl::SemanticSignatureElement> InputSignature;
  SmallVector<llvm::hlsl::SemanticSignatureElement> OutputSignature;
 
  llvm::Module &M = CGM.getModule();
  llvm::LLVMContext &Ctx = M.getContext();
  auto *EntryTy = llvm::FunctionType::get(llvm::Type::getVoidTy(Ctx), false);
  Function *EntryFn =
      Function::Create(EntryTy, Function::ExternalLinkage, FD->getName(), &M);
 
  // Copy function attributes over, we have no argument or return attributes
  // that can be valid on the real entry.
  AttributeList NewAttrs = AttributeList::get(Ctx, AttributeList::FunctionIndex,
                                              Fn->getAttributes().getFnAttrs());
  EntryFn->setAttributes(NewAttrs);
  setHLSLEntryAttributes(FD, EntryFn);
 
  // Set the called function as internal linkage.
  Fn->setLinkage(GlobalValue::InternalLinkage);
 
  BasicBlock *BB = BasicBlock::Create(Ctx, "entry", EntryFn);
  IRBuilder<> B(BB);
  llvm::SmallVector<Value *> Args;
 
  SmallVector<OperandBundleDef, 1> OB;
  if (CGM.shouldEmitConvergenceTokens()) {
    assert(EntryFn->isConvergent());
    llvm::Value *I =
        B.CreateIntrinsic(llvm::Intrinsic::experimental_convergence_entry, {});
    llvm::Value *bundleArgs[] = {I};
    OB.emplace_back("convergencectrl", bundleArgs);
  }
 
  SmallVector<std::pair<llvm::Value *, llvm::Type *>> OutputSemantic;
 
  unsigned SRetOffset = 0;
  for (const auto &Param : Fn->args()) {
    if (Param.hasStructRetAttr()) {
      SRetOffset = 1;
      llvm::Type *VarType = Param.getParamStructRetType();
      llvm::Value *Var =
          CGM.getLangOpts().EmitLogicalPointer
              ? cast<Instruction>(B.CreateStructuredAlloca(VarType))
              : cast<Instruction>(B.CreateAlloca(VarType));
      OutputSemantic.push_back(std::make_pair(Var, VarType));
      Args.push_back(Var);
      continue;
    }
 
    const ParmVarDecl *PD = FD->getParamDecl(Param.getArgNo() - SRetOffset);
    llvm::Value *SemanticValue = nullptr;
    // FIXME: support inout/out parameters for semantics.
    if ([[maybe_unused]] HLSLParamModifierAttr *MA =
            PD->getAttr<HLSLParamModifierAttr>()) {
      llvm_unreachable("Not handled yet");
    } else {
      llvm::Type *ParamType = nullptr;
      if (Param.hasByValAttr())
        ParamType = Param.getParamByValType();
      else if (PD->getType()->isRecordType())
        ParamType = CGM.getTypes().ConvertType(PD->getType());
      else
        ParamType = Param.getType();
 
      auto AttrBegin = PD->specific_attr_begin<HLSLAppliedSemanticAttr>();
      auto AttrEnd = PD->specific_attr_end<HLSLAppliedSemanticAttr>();
      auto Result = handleSemanticLoad(B, FD, ParamType, PD, AttrBegin, AttrEnd,
                                       InputSignature);
      SemanticValue = Result.first;
      if (!SemanticValue)
        return;
      if (Param.hasByValAttr() || PD->getType()->isRecordType()) {
        llvm::Value *Var =
            CGM.getLangOpts().EmitLogicalPointer
                ? cast<Instruction>(B.CreateStructuredAlloca(ParamType))
                : cast<Instruction>(B.CreateAlloca(ParamType));
        B.CreateStore(SemanticValue, Var);
        SemanticValue = Var;
      }
    }
 
    assert(SemanticValue);
    Args.push_back(SemanticValue);
  }
 
  CallInst *CI = B.CreateCall(FunctionCallee(Fn), Args, OB);
  CI->setCallingConv(Fn->getCallingConv());
 
  if (Fn->getReturnType() != CGM.VoidTy)
    // Element type is unused, so set to dummy value (NULL).
    OutputSemantic.push_back(std::make_pair(CI, nullptr));
 
  for (auto &SourcePair : OutputSemantic) {
    llvm::Value *Source = SourcePair.first;
    llvm::Type *ElementType = SourcePair.second;
    AllocaInst *AI = dyn_cast<AllocaInst>(Source);
    llvm::Value *SourceValue = AI ? B.CreateLoad(ElementType, Source) : Source;
 
    auto AttrBegin = FD->specific_attr_begin<HLSLAppliedSemanticAttr>();
    auto AttrEnd = FD->specific_attr_end<HLSLAppliedSemanticAttr>();
    handleSemanticStore(B, FD, SourceValue, FD, AttrBegin, AttrEnd,
                        OutputSignature);
  }
 
  B.CreateRetVoid();
 
  // Add and identify root signature to function, if applicable
  for (const Attr *Attr : FD->getAttrs()) {
    if (const auto *RSAttr = dyn_cast<RootSignatureAttr>(Attr)) {
      auto *RSDecl = RSAttr->getSignatureDecl();
      addRootSignatureMD(RSDecl->getVersion(), RSDecl->getRootElements(),
                         EntryFn, M);
    }
  }
 
  if (CGM.getTarget().getTriple().isDXIL()) {
    // Use the entry's shader stage, not the target environment, which may be
    // 'library' when compiling multiple entry points.
    auto Stage = FD->getAttr<HLSLShaderAttr>()->getType();
    using PackingMode = CodeGenOptions::HLSLSemanticSignaturePackingMode;
    auto Mode = CGM.getCodeGenOpts().getHLSLSemanticSignaturePacking();
    for (auto IOTy : {llvm::hlsl::IOType::In, llvm::hlsl::IOType::Out}) {
      bool IsOutput = IOTy == llvm::hlsl::IOType::Out;
      auto &Signature = IsOutput ? OutputSignature : InputSignature;
      auto Packed = packSemanticSignature(Signature, Stage, IOTy, Mode,
                                          CGM.getLangOpts().NativeHalfType);
      if (!Packed) {
        CGM.getDiags().Report(FD->getLocation(),
                              diag::err_hlsl_signature_packing)
            << IsOutput << llvm::toString(Packed.takeError());
        return;
      }
      // Library entry points retain stage-specific packing without warnings.
      if (Mode != PackingMode::Default &&
          CGM.getTarget().getTriple().getEnvironment() !=
              llvm::Triple::Library &&
          ((Stage == llvm::Triple::Vertex && !IsOutput) ||
           (Stage == llvm::Triple::Pixel && IsOutput)))
        CGM.getDiags().Report(FD->getLocation(),
                              diag::warn_hlsl_signature_packing_ignored)
            << (Mode == PackingMode::Optimized ? "optimized" : "prefix-stable")
            << IsOutput;
    }
  }
 
  addSemanticSignatureMD(InputSignature, OutputSignature, EntryFn, M);
}
 
static void gatherFunctions(SmallVectorImpl<Function *> &Fns, llvm::Module &M,
                            bool CtorOrDtor) {
  const auto *GV =
      M.getNamedGlobal(CtorOrDtor ? "llvm.global_ctors" : "llvm.global_dtors");
  if (!GV)
    return;
  const auto *CA = dyn_cast<ConstantArray>(GV->getInitializer());
  if (!CA)
    return;
  // The global_ctor array elements are a struct [Priority, Fn *, COMDat].
  // HLSL neither supports priorities or COMDat values, so we will check those
  // in an assert but not handle them.
 
  for (const auto &Ctor : CA->operands()) {
    if (isa<ConstantAggregateZero>(Ctor))
      continue;
    ConstantStruct *CS = cast<ConstantStruct>(Ctor);
 
    assert(cast<ConstantInt>(CS->getOperand(0))->getValue() == 65535 &&
           "HLSL doesn't support setting priority for global ctors.");
    assert(isa<ConstantPointerNull>(CS->getOperand(2)) &&
           "HLSL doesn't support COMDat for global ctors.");
    Fns.push_back(cast<Function>(CS->getOperand(1)));
  }
}
 
void CGHLSLRuntime::generateGlobalCtorDtorCalls() {
  llvm::Module &M = CGM.getModule();
  SmallVector<Function *> CtorFns;
  SmallVector<Function *> DtorFns;
  gatherFunctions(CtorFns, M, true);
  gatherFunctions(DtorFns, M, false);
 
  // Insert a call to the global constructor at the beginning of the entry block
  // to externally exported functions. This is a bit of a hack, but HLSL allows
  // global constructors, but doesn't support driver initialization of globals.
  for (auto &F : M.functions()) {
    if (!F.hasFnAttribute("hlsl.shader"))
      continue;
    auto *Token = getConvergenceToken(F.getEntryBlock());
    Instruction *IP = &*F.getEntryBlock().begin();
    SmallVector<OperandBundleDef, 1> OB;
    if (Token) {
      llvm::Value *bundleArgs[] = {Token};
      OB.emplace_back("convergencectrl", bundleArgs);
      IP = Token->getNextNode();
    }
    IRBuilder<> B(IP);
    for (auto *Fn : CtorFns) {
      auto CI = B.CreateCall(FunctionCallee(Fn), {}, OB);
      CI->setCallingConv(Fn->getCallingConv());
    }
 
    // Insert global dtors before the terminator of the last instruction
    B.SetInsertPoint(F.back().getTerminator());
    for (auto *Fn : DtorFns) {
      auto CI = B.CreateCall(FunctionCallee(Fn), {}, OB);
      CI->setCallingConv(Fn->getCallingConv());
    }
  }
 
  // No need to keep global ctors/dtors for non-lib profile after call to
  // ctors/dtors added for entry.
  Triple T(M.getTargetTriple());
  if (T.getEnvironment() != Triple::EnvironmentType::Library) {
    if (auto *GV = M.getNamedGlobal("llvm.global_ctors"))
      GV->eraseFromParent();
    if (auto *GV = M.getNamedGlobal("llvm.global_dtors"))
      GV->eraseFromParent();
  }
}
 
static void initializeBuffer(CodeGenModule &CGM, llvm::GlobalVariable *GV,
                             Intrinsic::ID IntrID,
                             ArrayRef<llvm::Value *> Args) {
 
  LLVMContext &Ctx = CGM.getLLVMContext();
  llvm::Function *InitResFunc =
      llvm::Function::Create(llvm::FunctionType::get(CGM.VoidTy, false),
                             llvm::GlobalValue::InternalLinkage,
                             "_init_buffer_" + GV->getName(), CGM.getModule());
  InitResFunc->addFnAttr(llvm::Attribute::AlwaysInline);
 
  llvm::BasicBlock *EntryBB =
      llvm::BasicBlock::Create(Ctx, "entry", InitResFunc);
  CGBuilderTy Builder(CGM, Ctx);
  const DataLayout &DL = CGM.getModule().getDataLayout();
  Builder.SetInsertPoint(EntryBB);
 
  // Make sure the global variable is buffer resource handle
  llvm::Type *HandleTy = GV->getValueType();
  assert(HandleTy->isTargetExtTy() && "unexpected type of the buffer global");
 
  llvm::Value *CreateHandle = Builder.CreateIntrinsic(
      /*ReturnType=*/HandleTy, IntrID, Args, nullptr,
      Twine(GV->getName()).concat("_h"));
 
  Builder.CreateAlignedStore(CreateHandle, GV, GV->getPointerAlignment(DL));
  Builder.CreateRetVoid();
 
  CGM.AddCXXGlobalInit(InitResFunc);
}
 
void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
                                                llvm::GlobalVariable *GV) {
  ResourceBindingAttrs Binding(BufDecl);
  assert(Binding.hasBinding() &&
         "cbuffer/tbuffer should always have resource binding attribute");
 
  auto *Index = llvm::ConstantInt::get(CGM.IntTy, 0);
  auto *RangeSize = llvm::ConstantInt::get(CGM.IntTy, 1);
  auto *Space = llvm::ConstantInt::get(CGM.IntTy, Binding.getSpace());
  Value *Name = buildNameForResource(BufDecl->getName(), CGM);
 
  // buffer with explicit binding
  if (Binding.isExplicit()) {
    llvm::Intrinsic::ID IntrinsicID =
        CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
    auto *RegSlot = llvm::ConstantInt::get(CGM.IntTy, Binding.getSlot());
    SmallVector<Value *> Args{Space, RegSlot, RangeSize, Index, Name};
    initializeBuffer(CGM, GV, IntrinsicID, Args);
  } else {
    // buffer with implicit binding
    llvm::Intrinsic::ID IntrinsicID =
        CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
    auto *OrderID =
        llvm::ConstantInt::get(CGM.IntTy, Binding.getImplicitOrderID());
    SmallVector<Value *> Args{OrderID, Space, RangeSize, Index, Name};
    initializeBuffer(CGM, GV, IntrinsicID, Args);
  }
}
 
void CGHLSLRuntime::handleGlobalVarDefinition(const VarDecl *VD,
                                              llvm::GlobalVariable *GV) {
  if (auto Attr = VD->getAttr<HLSLVkExtBuiltinInputAttr>())
    addSPIRVBuiltinDecoration(GV, Attr->getBuiltIn());
  if (auto Attr = VD->getAttr<HLSLVkExtBuiltinOutputAttr>())
    addSPIRVBuiltinDecoration(GV, Attr->getBuiltIn());
}
 
llvm::Instruction *CGHLSLRuntime::getConvergenceToken(BasicBlock &BB) {
  if (!CGM.shouldEmitConvergenceTokens())
    return nullptr;
 
  auto E = BB.end();
  for (auto I = BB.begin(); I != E; ++I) {
    auto *II = dyn_cast<llvm::IntrinsicInst>(&*I);
    if (II && llvm::isConvergenceControlIntrinsic(II->getIntrinsicID())) {
      return II;
    }
  }
  llvm_unreachable("Convergence token should have been emitted.");
  return nullptr;
}
 
class OpaqueValueVisitor : public RecursiveASTVisitor<OpaqueValueVisitor> {
public:
  llvm::SmallVector<OpaqueValueExpr *, 8> OVEs;
  llvm::SmallPtrSet<OpaqueValueExpr *, 8> Visited;
  OpaqueValueVisitor() {}
 
  bool VisitHLSLOutArgExpr(HLSLOutArgExpr *) {
    // These need to be bound in CodeGenFunction::EmitHLSLOutArgLValues
    // or CodeGenFunction::EmitHLSLOutArgExpr. If they are part of this
    // traversal, the temporary containing the copy out will not have
    // been created yet.
    return false;
  }
 
  bool VisitOpaqueValueExpr(OpaqueValueExpr *E) {
    // Traverse the source expression first.
    if (E->getSourceExpr())
      TraverseStmt(E->getSourceExpr());
 
    // Then add this OVE if we haven't seen it before.
    if (Visited.insert(E).second)
      OVEs.push_back(E);
 
    return true;
  }
};
 
void CGHLSLRuntime::emitInitListOpaqueValues(CodeGenFunction &CGF,
                                             InitListExpr *E) {
 
  typedef CodeGenFunction::OpaqueValueMappingData OpaqueValueMappingData;
  OpaqueValueVisitor Visitor;
  Visitor.TraverseStmt(E);
  for (auto *OVE : Visitor.OVEs) {
    if (CGF.isOpaqueValueEmitted(OVE))
      continue;
    if (OpaqueValueMappingData::shouldBindAsLValue(OVE)) {
      LValue LV = CGF.EmitLValue(OVE->getSourceExpr());
      OpaqueValueMappingData::bind(CGF, OVE, LV);
    } else {
      RValue RV = CGF.EmitAnyExpr(OVE->getSourceExpr());
      OpaqueValueMappingData::bind(CGF, OVE, RV);
    }
  }
}
 
std::optional<LValue> CGHLSLRuntime::emitResourceArraySubscriptExpr(
    const ArraySubscriptExpr *ArraySubsExpr, CodeGenFunction &CGF) {
  assert((ArraySubsExpr->getType()->isHLSLResourceRecord() ||
          ArraySubsExpr->getType()->isHLSLResourceRecordArray()) &&
         "expected resource array subscript expression");
 
  // Let clang codegen handle local and static resource array subscripts,
  // or when the subscript references on opaque expression (as part of
  // ArrayInitLoopExpr AST node).
  const VarDecl *ArrayDecl = dyn_cast_or_null<VarDecl>(
      getArrayDecl(CGF.CGM.getContext(), ArraySubsExpr));
  if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
      ArrayDecl->getStorageClass() == SC_Static)
    return std::nullopt;
 
  // get the resource array type
  ASTContext &AST = ArrayDecl->getASTContext();
  const Type *ResArrayTy = ArrayDecl->getType().getTypePtr();
  assert(ResArrayTy->isHLSLResourceRecordArray() &&
         "expected array of resource classes");
 
  // Iterate through all nested array subscript expressions to calculate
  // the index in the flattened resource array (if this is a multi-
  // dimensional array). The index is calculated as a sum of all indices
  // multiplied by the total size of the array at that level.
  Value *Index = nullptr;
  const ArraySubscriptExpr *ASE = ArraySubsExpr;
  while (ASE != nullptr) {
    Value *SubIndex = CGF.EmitScalarExpr(ASE->getIdx());
    if (const auto *ArrayTy =
            dyn_cast<ConstantArrayType>(ASE->getType().getTypePtr())) {
      Value *Multiplier = llvm::ConstantInt::get(
          CGM.IntTy, AST.getConstantArrayElementCount(ArrayTy));
      SubIndex = CGF.Builder.CreateMul(SubIndex, Multiplier);
    }
    Index = Index ? CGF.Builder.CreateAdd(Index, SubIndex) : SubIndex;
    ASE = dyn_cast<ArraySubscriptExpr>(ASE->getBase()->IgnoreParenImpCasts());
  }
 
  // Find binding info for the resource array. For implicit binding
  // an HLSLResourceBindingAttr should have been added by SemaHLSL.
  ResourceBindingAttrs Binding(ArrayDecl);
  assert(Binding.hasBinding() &&
         "resource array must have a binding attribute");
 
  // Find the individual resource type.
  QualType ResultTy = ArraySubsExpr->getType();
  QualType ResourceTy =
      ResultTy->isArrayType() ? AST.getBaseElementType(ResultTy) : ResultTy;
 
  // Create a temporary variable for the result, which is either going
  // to be a single resource instance or a local array of resources (we need to
  // return an LValue).
  RawAddress TmpVar = CGF.CreateMemTempWithoutCast(ResultTy);
  if (CGF.EmitLifetimeStart(TmpVar.getPointer()))
    CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
        NormalEHLifetimeMarker, TmpVar);
 
  AggValueSlot ValueSlot = AggValueSlot::forAddr(
      TmpVar, Qualifiers(), AggValueSlot::IsDestructed_t(true),
      AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsAliased_t(false),
      AggValueSlot::DoesNotOverlap);
 
  // Calculate total array size (= range size).
  llvm::Value *Range = llvm::ConstantInt::getSigned(
      CGM.IntTy, getTotalArraySize(AST, ResArrayTy));
 
  // If the result of the subscript operation is a single resource, call the
  // constructor.
  if (ResultTy == ResourceTy) {
    CallArgList Args;
    CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
        CGF.CGM, ResourceTy->getAsCXXRecordDecl(), Range, Index,
        ArrayDecl->getName(), Binding, Args);
 
    if (!CreateMethod) {
      // This can happen if someone creates an array of structs that looks like
      // an HLSL resource record array but it does not have the required static
      // create method. No binding will be generated for it.
      assert(!ResourceTy->getAsCXXRecordDecl()->isImplicit() &&
             "create method lookup should always succeed for built-in resource "
             "records");
      return std::nullopt;
    }
 
    callResourceInitMethod(CGF, CreateMethod, Args, ValueSlot.getAddress());
 
  } else {
    // The result of the subscript operation is a local resource array which
    // needs to be initialized.
    const ConstantArrayType *ArrayTy =
        cast<ConstantArrayType>(ResultTy.getTypePtr());
    std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
        CGF, ResourceTy->getAsCXXRecordDecl(), ArrayTy, ValueSlot, Range, Index,
        ArrayDecl->getName(), Binding, {llvm::ConstantInt::get(CGM.IntTy, 0)});
    if (!EndIndex)
      return std::nullopt;
  }
  return CGF.MakeAddrLValue(TmpVar, ResultTy, AlignmentSource::Decl);
}
 
// Initialize all resources of a global resource array into provided slot.
bool CGHLSLRuntime::initializeGlobalResourceArray(CodeGenFunction &CGF,
                                                  const VarDecl *ArrayDecl,
                                                  AggValueSlot &DestSlot) {
  assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
         ArrayDecl->hasGlobalStorage() &&
         ArrayDecl->getStorageClass() != SC_Static &&
         "expected global non-static resource array");
 
  // Find binding info for the resource array. For implicit binding
  // the HLSLResourceBindingAttr should have been added by SemaHLSL.
  ResourceBindingAttrs Binding(ArrayDecl);
  assert(Binding.hasBinding() &&
         "resource array must have a binding attribute");
 
  // Find the individual resource type.
  ASTContext &AST = ArrayDecl->getASTContext();
  QualType ResTy = AST.getBaseElementType(ArrayDecl->getType());
  const auto *ResArrayTy =
      cast<ConstantArrayType>(ArrayDecl->getType().getTypePtr());
 
  // Create Value for index and total array size (= range size).
  int Size = getTotalArraySize(AST, ResArrayTy);
  llvm::Value *Zero = llvm::ConstantInt::get(CGM.IntTy, 0);
  llvm::Value *Range = llvm::ConstantInt::get(CGM.IntTy, Size);
 
  // Initialize individual resources in the array into DestSlot.
  std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
      CGF, ResTy->getAsCXXRecordDecl(), ResArrayTy, DestSlot, Range, Zero,
      ArrayDecl->getName(), Binding, {Zero});
  return EndIndex.has_value();
}
 
// If the expression is a global resource array, initialize all of its resources
// into Dest. Returns false if no initialization has been performed and the
// array copy should be handled by the default codegen.
bool CGHLSLRuntime::emitGlobalResourceArray(CodeGenFunction &CGF, const Expr *E,
                                            AggValueSlot &DestSlot) {
  assert(E->getType()->isHLSLResourceRecordArray() &&
         "expected resource array");
 
  // Find the array declaration for the expression. Fallback to the default
  // handling if it's not a global resource array.
  const VarDecl *ArrayDecl =
      dyn_cast_or_null<VarDecl>(getArrayDecl(CGF.CGM.getContext(), E));
  if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
      ArrayDecl->getStorageClass() == SC_Static)
    return false;
 
  return initializeGlobalResourceArray(CGF, ArrayDecl, DestSlot);
}
 
// If the expression is a global resource array, create a temporary and
// initialize all of its resources, and return it as an LValue. Returns nullopt
// if no initialization has been performed and the handling should follow the
// default path.
std::optional<LValue>
CGHLSLRuntime::emitGlobalResourceArrayAsLValue(CodeGenFunction &CGF,
                                               const VarDecl *ArrayDecl) {
  assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
         "expected resource array declaration");
 
  if (!ArrayDecl->hasGlobalStorage() ||
      ArrayDecl->getStorageClass() == SC_Static)
    return std::nullopt;
 
  AggValueSlot TmpArraySlot =
      CGF.CreateAggTemp(ArrayDecl->getType(), "tmpResArray");
  if (initializeGlobalResourceArray(CGF, ArrayDecl, TmpArraySlot))
    return CGF.MakeAddrLValue(TmpArraySlot.getAddress(), ArrayDecl->getType(),
                              AlignmentSource::Decl);
  return std::nullopt;
}
 
RawAddress CGHLSLRuntime::createBufferMatrixTempAddress(const LValue &LV,
                                                        CodeGenFunction &CGF) {
 
  assert(LV.getType()->isConstantMatrixType() && "expected matrix type");
  assert(LV.getType().getAddressSpace() == LangAS::hlsl_constant &&
         "expected cbuffer matrix");
 
  QualType MatQualTy = LV.getType();
  llvm::Type *LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(MatQualTy);
  Address SrcAddr = LV.getAddress();
 
  if (LayoutTy == CGF.ConvertTypeForMem(MatQualTy))
    return SrcAddr;
 
  RawAddress DestAlloca =
      CGF.CreateMemTempWithoutCast(MatQualTy, "matrix.buf.copy");
  HLSLBufferCopyEmitter(CGF, DestAlloca, SrcAddr).emitCopy(MatQualTy);
  return DestAlloca;
}
 
std::optional<LValue> CGHLSLRuntime::emitBufferArraySubscriptExpr(
    const ArraySubscriptExpr *E, CodeGenFunction &CGF,
    llvm::function_ref<llvm::Value *(bool Promote)> EmitIdxAfterBase) {
  // Find the element type to index by first padding the element type per HLSL
  // buffer rules, and then padding out to a 16-byte register boundary if
  // necessary.
  llvm::Type *LayoutTy =
      HLSLBufferLayoutBuilder(CGF.CGM).layOutType(E->getType());
  uint64_t LayoutSizeInBits =
      CGM.getDataLayout().getTypeSizeInBits(LayoutTy).getFixedValue();
  CharUnits ElementSize = CharUnits::fromQuantity(LayoutSizeInBits / 8);
  CharUnits RowAlignedSize = ElementSize.alignTo(CharUnits::fromQuantity(16));
  if (RowAlignedSize > ElementSize) {
    llvm::Type *Padding = CGM.getTargetCodeGenInfo().getHLSLPadding(
        CGM, RowAlignedSize - ElementSize);
    assert(Padding && "No padding type for target?");
    LayoutTy = llvm::StructType::get(CGF.getLLVMContext(), {LayoutTy, Padding},
                                     /*isPacked=*/true);
  }
 
  // If the layout type doesn't introduce any padding, we don't need to do
  // anything special.
  llvm::Type *OrigTy = CGF.CGM.getTypes().ConvertTypeForMem(E->getType());
  if (LayoutTy == OrigTy)
    return std::nullopt;
 
  LValueBaseInfo EltBaseInfo;
  TBAAAccessInfo EltTBAAInfo;
 
  // Index into the object as-if we have an array of the padded element type,
  // and then dereference the element itself to avoid reading padding that may
  // be past the end of the in-memory object.
  SmallVector<llvm::Value *, 2> Indices;
  llvm::Value *Idx = EmitIdxAfterBase(/*Promote*/ true);
  Indices.push_back(Idx);
  Indices.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 0));
 
  if (CGF.getLangOpts().EmitLogicalPointer) {
    // The fact that we emit an array-to-pointer decay might be an oversight,
    // but for now, we simply ignore it (see #179951).
    const CastExpr *CE = cast<CastExpr>(E->getBase());
    assert(CE->getCastKind() == CastKind::CK_ArrayToPointerDecay);
 
    LValue LV = CGF.EmitLValue(CE->getSubExpr());
    Address Addr = LV.getAddress();
    LayoutTy = llvm::ArrayType::get(
        LayoutTy,
        cast<llvm::ArrayType>(Addr.getElementType())->getNumElements());
    auto *GEP = cast<StructuredGEPInst>(CGF.Builder.CreateStructuredGEP(
        LayoutTy, Addr.emitRawPointer(CGF), Indices, "cbufferidx"));
    Addr =
        Address(GEP, GEP->getResultElementType(), RowAlignedSize, KnownNonNull);
    return CGF.MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo);
  }
 
  Address Addr =
      CGF.EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
  llvm::Value *GEP = CGF.Builder.CreateGEP(LayoutTy, Addr.emitRawPointer(CGF),
                                           Indices, "cbufferidx");
  Addr = Address(GEP, Addr.getElementType(), RowAlignedSize, KnownNonNull);
  return CGF.MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo);
}
 
std::optional<LValue>
CGHLSLRuntime::emitResourceMemberExpr(CodeGenFunction &CGF,
                                      const MemberExpr *ME) {
  assert((ME->getType()->isHLSLResourceRecord() ||
          ME->getType()->isHLSLResourceRecordArray()) &&
         "expected resource member expression");
 
  const VarDecl *ResourceVD =
      findAssociatedResourceDeclForStruct(CGF.CGM.getContext(), ME);
  if (!ResourceVD)
    return std::nullopt;
 
  // Handle member of resource array type.
  if (ResourceVD->getType()->isHLSLResourceRecordArray())
    return emitGlobalResourceArrayAsLValue(CGF, ResourceVD);
 
  GlobalVariable *ResGV =
      cast<GlobalVariable>(CGM.GetAddrOfGlobalVar(ResourceVD));
  const DataLayout &DL = CGM.getDataLayout();
  llvm::Type *Ty = ResGV->getValueType();
  CharUnits Align = CharUnits::fromQuantity(DL.getABITypeAlign(Ty));
  Address Addr = Address(ResGV, Ty, Align);
  LValue LV = LValue::MakeAddr(Addr, ME->getType(), CGM.getContext(),
                               LValueBaseInfo(AlignmentSource::Type),
                               CGM.getTBAAAccessInfo(ME->getType()));
  return LV;
}
 
bool CGHLSLRuntime::emitBufferCopy(CodeGenFunction &CGF, const Expr *E,
                                   const LValue &SrcLV,
                                   AggValueSlot &DestSlot) {
  assert(E->getType().getAddressSpace() == LangAS::hlsl_constant &&
         "expected expression in HLSL constant address space");
  assert(!E->getType()->isHLSLResourceRecord() &&
         !E->getType()->isHLSLResourceRecordArray() &&
         "direct accesses to resource types should be handled separately");
 
  if (DestSlot.isIgnored())
    return false;
 
  QualType Ty = E->getType();
  Address DstPtr = DestSlot.getAddress();
  Address SrcPtr = SrcLV.getAddress();
 
  // If there are no intangible types, we don't need to lookup associated
  // resources.
  if (!Ty->isHLSLIntangibleType())
    return HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(Ty);
 
  // Handle structs with intangible types by setting the resource fields
  // of the destination struct with the resources associated with the global
  // struct.
  EmbeddedResourceNameBuilder NameBuilder;
  const VarDecl *VD = findStructResourceParentDeclAndBuildName(E, NameBuilder);
  AssociatedResourcesList AssociatedResources(VD, NameBuilder.getName());
 
  // Callback to fill in the associated resource.
  auto EmitResFn = [&](AggValueSlot &ResSlot) {
    const VarDecl *ResDecl = AssociatedResources.getNextResource();
    assert(ResDecl && "associated resource declaration not found");
 
    // Check that the resource type of dest and src matches.
    [[maybe_unused]] llvm::Type *DestType =
        ResSlot.getAddress().getElementType();
    [[maybe_unused]] llvm::Type *SrcConvertedType =
        CGM.getTypes().ConvertTypeForMem(ResDecl->getType());
    assert(DestType == SrcConvertedType && "resource slot type mismatch");
 
    if (ResDecl->getType()->isHLSLResourceRecord())
      copyGlobalResource(CGF, ResDecl, ResSlot);
    else
      initializeGlobalResourceArray(CGF, ResDecl, ResSlot);
  };
 
  auto Result =
      HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(Ty, EmitResFn);
  assert(AssociatedResources.getNextResource() == nullptr &&
         "expected all associated resources to be processed");
  return Result;
}
 
LValue CGHLSLRuntime::emitBufferMemberExpr(CodeGenFunction &CGF,
                                           const MemberExpr *E) {
  LValue Base =
      CGF.EmitCheckedLValue(E->getBase(), CodeGenFunction::TCK_MemberAccess);
  auto *Field = dyn_cast<FieldDecl>(E->getMemberDecl());
  assert(Field && "Unexpected access into HLSL buffer");
 
  const RecordDecl *Rec = Field->getParent();
 
  // Work out the buffer layout type to index into.
  QualType RecType = CGM.getContext().getCanonicalTagType(Rec);
  assert(RecType->isStructureOrClassType() && "Invalid type in HLSL buffer");
  // Since this is a member of an object in the buffer and not the buffer's
  // struct/class itself, we shouldn't have any offsets on the members we need
  // to contend with.
  CGHLSLOffsetInfo EmptyOffsets;
  llvm::StructType *LayoutTy = HLSLBufferLayoutBuilder(CGM).layOutStruct(
      RecType->getAsCanonical<RecordType>(), EmptyOffsets);
 
  // Get the field index for the layout struct, accounting for padding.
  unsigned FieldIdx =
      CGM.getTypes().getCGRecordLayout(Rec).getLLVMFieldNo(Field);
  assert(FieldIdx < LayoutTy->getNumElements() &&
         "Layout struct is smaller than member struct");
  unsigned Skipped = 0;
  for (unsigned I = 0; I <= FieldIdx;) {
    llvm::Type *ElementTy = LayoutTy->getElementType(I + Skipped);
    if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(ElementTy))
      ++Skipped;
    else
      ++I;
  }
  FieldIdx += Skipped;
  assert(FieldIdx < LayoutTy->getNumElements() && "Access out of bounds");
 
  // Now index into the struct, making sure that the type we return is the
  // buffer layout type rather than the original type in the AST.
  QualType FieldType = Field->getType();
  llvm::Type *FieldLLVMTy = CGM.getTypes().ConvertTypeForMem(FieldType);
  CharUnits Align = CharUnits::fromQuantity(
      CGF.CGM.getDataLayout().getABITypeAlign(FieldLLVMTy));
 
  Value *Ptr = CGF.getLangOpts().EmitLogicalPointer
                   ? CGF.Builder.CreateStructuredGEP(
                         LayoutTy, Base.getPointer(CGF),
                         llvm::ConstantInt::get(CGM.IntTy, FieldIdx))
                   : CGF.Builder.CreateStructGEP(LayoutTy, Base.getPointer(CGF),
                                                 FieldIdx, Field->getName());
  Address Addr(Ptr, FieldLLVMTy, Align, KnownNonNull);
 
  LValue LV = LValue::MakeAddr(Addr, FieldType, CGM.getContext(),
                               LValueBaseInfo(AlignmentSource::Type),
                               CGM.getTBAAAccessInfo(FieldType));
  LV.getQuals().addCVRQualifiers(Base.getVRQualifiers());
 
  return LV;
}