CGHLSLRuntime.h

Go to the documentation of this file.

//===----- CGHLSLRuntime.h - Interface to HLSL Runtimes -----*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This provides an abstract class for HLSL code generation.  Concrete
// subclasses of this implement code generation for specific HLSL
// runtime libraries.
//
//===----------------------------------------------------------------------===//
 
#ifndef LLVM_CLANG_LIB_CODEGEN_CGHLSLRUNTIME_H
#define LLVM_CLANG_LIB_CODEGEN_CGHLSLRUNTIME_H
 
#include "Address.h"
#include "clang/AST/Attr.h"
#include "clang/AST/Decl.h"
#include "clang/Basic/Builtins.h"
#include "clang/Basic/HLSLRuntime.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Frontend/HLSL/HLSLResource.h"
#include "llvm/Frontend/HLSL/SemanticSignatures.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsDirectX.h"
#include "llvm/IR/IntrinsicsSPIRV.h"
 
#include <optional>
#include <vector>
 
// Function generator macros for picking the right intrinsic for the target.
#define GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(FunctionName, DxilPostfix,       \
                                              SpirvPostfix)                    \
  llvm::Intrinsic::ID get##FunctionName##Intrinsic() {                         \
    llvm::Triple::ArchType Arch = getArch();                                   \
    switch (Arch) {                                                            \
    case llvm::Triple::dxil:                                                   \
      return llvm::Intrinsic::dx_##DxilPostfix;                                \
    case llvm::Triple::spirv:                                                  \
      return llvm::Intrinsic::spv_##SpirvPostfix;                              \
    default:                                                                   \
      llvm_unreachable(#FunctionName " not supported by target architecture"); \
    }                                                                          \
  }
 
#define GENERATE_HLSL_INTRINSIC_FUNCTION(FunctionName, IntrinsicPostfix)       \
  GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(FunctionName, IntrinsicPostfix,        \
                                        IntrinsicPostfix)
 
using ResourceClass = llvm::dxil::ResourceClass;
 
namespace llvm {
class GlobalVariable;
class Function;
class StructType;
class Metadata;
} // namespace llvm
 
namespace clang {
class NamedDecl;
class VarDecl;
class ParmVarDecl;
class InitListExpr;
class HLSLBufferDecl;
class HLSLRootSignatureDecl;
class HLSLVkBindingAttr;
class Type;
class RecordType;
class DeclContext;
class HLSLPackOffsetAttr;
class ArraySubscriptExpr;
 
class FunctionDecl;
 
namespace CodeGen {
 
class CodeGenModule;
class CodeGenFunction;
class LValue;
class AggValueSlot;
 
class CGHLSLOffsetInfo {
  SmallVector<uint32_t> Offsets;
 
public:
  static const uint32_t Unspecified = ~0U;
 
  /// Iterates over all declarations in the HLSL buffer and based on the
  /// packoffset or register(c#) annotations it fills outs the Offsets vector
  /// with the user-specified layout offsets. The buffer offsets can be
  /// specified 2 ways: 1. declarations in cbuffer {} block can have a
  /// packoffset annotation (translates to HLSLPackOffsetAttr) 2. default
  /// constant buffer declarations at global scope can have register(c#)
  /// annotations (translates to HLSLResourceBindingAttr with RegisterType::C)
  /// It is not guaranteed that all declarations in a buffer have an annotation.
  /// For those where it is not specified a `~0U` value is added to the Offsets
  /// vector. In the final layout these declarations will be placed at the end
  /// of the HLSL buffer after all of the elements with specified offset.
  static CGHLSLOffsetInfo fromDecl(const HLSLBufferDecl &BufDecl);
 
  /// Comparison function for offsets received from `operator[]` suitable for
  /// use in a `stable_sort`. This will order implicit bindings after explicit
  /// offsets.
  static bool compareOffsets(uint32_t LHS, uint32_t RHS) { return LHS < RHS; }
 
  /// Get the given offset, or `~0U` if there is no offset for the member.
  uint32_t operator[](size_t I) const {
    if (Offsets.empty())
      return Unspecified;
    return Offsets[I];
  }
 
  bool empty() const { return Offsets.empty(); }
};
 
class CGHLSLRuntime {
public:
  //===----------------------------------------------------------------------===//
  // Start of reserved area for HLSL intrinsic getters.
  //===----------------------------------------------------------------------===//
 
  GENERATE_HLSL_INTRINSIC_FUNCTION(All, all)
  GENERATE_HLSL_INTRINSIC_FUNCTION(Any, any)
  GENERATE_HLSL_INTRINSIC_FUNCTION(Frac, frac)
  GENERATE_HLSL_INTRINSIC_FUNCTION(FlattenedThreadIdInGroup,
                                   flattened_thread_id_in_group)
  GENERATE_HLSL_INTRINSIC_FUNCTION(IsFinite, isfinite)
  GENERATE_HLSL_INTRINSIC_FUNCTION(IsInf, isinf)
  GENERATE_HLSL_INTRINSIC_FUNCTION(IsNaN, isnan)
  GENERATE_HLSL_INTRINSIC_FUNCTION(Rsqrt, rsqrt)
  GENERATE_HLSL_INTRINSIC_FUNCTION(Saturate, saturate)
  GENERATE_HLSL_INTRINSIC_FUNCTION(Sign, sign)
  GENERATE_HLSL_INTRINSIC_FUNCTION(ThreadId, thread_id)
  GENERATE_HLSL_INTRINSIC_FUNCTION(GroupThreadId, thread_id_in_group)
  GENERATE_HLSL_INTRINSIC_FUNCTION(GroupId, group_id)
  GENERATE_HLSL_INTRINSIC_FUNCTION(FDot, fdot)
  GENERATE_HLSL_INTRINSIC_FUNCTION(SDot, sdot)
  GENERATE_HLSL_INTRINSIC_FUNCTION(UDot, udot)
  GENERATE_HLSL_INTRINSIC_FUNCTION(Dot4AddI8Packed, dot4add_i8packed)
  GENERATE_HLSL_INTRINSIC_FUNCTION(Dot4AddU8Packed, dot4add_u8packed)
  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAllEqual, wave_all_equal)
  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAllTrue, wave_all)
  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAnyTrue, wave_any)
  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitOr, wave_reduce_or)
  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitXor, wave_reduce_xor)
  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitAnd, wave_reduce_and)
  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMax, wave_reduce_max)
  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMax, wave_reduce_umax)
  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMin, wave_reduce_min)
  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMin, wave_reduce_umin)
  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveCountBits, wave_active_countbits)
  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveIsFirstLane, wave_is_first_lane)
  GENERATE_HLSL_INTRINSIC_FUNCTION_BASE(WaveGetLaneCount, wave_get_lane_count,
                                        subgroup_size)
  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveReadLaneAt, wave_readlane)
  GENERATE_HLSL_INTRINSIC_FUNCTION(WaveReadLaneFirst, wave_readlane_first)
  GENERATE_HLSL_INTRINSIC_FUNCTION(QuadReadAcrossX, quad_read_across_x)
  GENERATE_HLSL_INTRINSIC_FUNCTION(QuadReadAcrossY, quad_read_across_y)
  GENERATE_HLSL_INTRINSIC_FUNCTION(QuadReadAcrossDiagonal,
                                   quad_read_across_diagonal)
  GENERATE_HLSL_INTRINSIC_FUNCTION(FirstBitUHigh, firstbituhigh)
  GENERATE_HLSL_INTRINSIC_FUNCTION(FirstBitSHigh, firstbitshigh)
  GENERATE_HLSL_INTRINSIC_FUNCTION(FirstBitLow, firstbitlow)
  GENERATE_HLSL_INTRINSIC_FUNCTION(NClamp, nclamp)
  GENERATE_HLSL_INTRINSIC_FUNCTION(SClamp, sclamp)
  GENERATE_HLSL_INTRINSIC_FUNCTION(UClamp, uclamp)
 
  GENERATE_HLSL_INTRINSIC_FUNCTION(CreateResourceGetBasePointer,
                                   resource_getbasepointer)
  GENERATE_HLSL_INTRINSIC_FUNCTION(CreateResourceGetPointer,
                                   resource_getpointer)
  GENERATE_HLSL_INTRINSIC_FUNCTION(Sample, resource_sample)
  GENERATE_HLSL_INTRINSIC_FUNCTION(SampleClamp, resource_sample_clamp)
  GENERATE_HLSL_INTRINSIC_FUNCTION(SampleBias, resource_samplebias)
  GENERATE_HLSL_INTRINSIC_FUNCTION(SampleBiasClamp, resource_samplebias_clamp)
  GENERATE_HLSL_INTRINSIC_FUNCTION(SampleGrad, resource_samplegrad)
  GENERATE_HLSL_INTRINSIC_FUNCTION(SampleGradClamp, resource_samplegrad_clamp)
  GENERATE_HLSL_INTRINSIC_FUNCTION(SampleLevel, resource_samplelevel)
  GENERATE_HLSL_INTRINSIC_FUNCTION(SampleCmp, resource_samplecmp)
  GENERATE_HLSL_INTRINSIC_FUNCTION(SampleCmpClamp, resource_samplecmp_clamp)
  GENERATE_HLSL_INTRINSIC_FUNCTION(SampleCmpLevelZero,
                                   resource_samplecmplevelzero)
  GENERATE_HLSL_INTRINSIC_FUNCTION(Gather, resource_gather)
  GENERATE_HLSL_INTRINSIC_FUNCTION(GatherCmp, resource_gather_cmp)
  GENERATE_HLSL_INTRINSIC_FUNCTION(CreateHandleFromBinding,
                                   resource_handlefrombinding)
  GENERATE_HLSL_INTRINSIC_FUNCTION(CreateHandleFromImplicitBinding,
                                   resource_handlefromimplicitbinding)
  GENERATE_HLSL_INTRINSIC_FUNCTION(CreateHandleFromHeap,
                                   resource_handlefromheap)
  GENERATE_HLSL_INTRINSIC_FUNCTION(NonUniformResourceIndex,
                                   resource_nonuniformindex)
  GENERATE_HLSL_INTRINSIC_FUNCTION(BufferUpdateCounter, resource_updatecounter)
  GENERATE_HLSL_INTRINSIC_FUNCTION(AllMemoryBarrier, all_memory_barrier)
  GENERATE_HLSL_INTRINSIC_FUNCTION(AllMemoryBarrierWithGroupSync,
                                   all_memory_barrier_with_group_sync)
  GENERATE_HLSL_INTRINSIC_FUNCTION(DeviceMemoryBarrier, device_memory_barrier)
  GENERATE_HLSL_INTRINSIC_FUNCTION(DeviceMemoryBarrierWithGroupSync,
                                   device_memory_barrier_with_group_sync)
  GENERATE_HLSL_INTRINSIC_FUNCTION(GroupMemoryBarrier, group_memory_barrier)
  GENERATE_HLSL_INTRINSIC_FUNCTION(GroupMemoryBarrierWithGroupSync,
                                   group_memory_barrier_with_group_sync)
  llvm::Intrinsic::ID getBarrierByMemoryTypeIntrinsic() {
    assert(getArch() == llvm::Triple::dxil &&
           "Barrier by memory type is only supported for DXIL");
    return llvm::Intrinsic::dx_barrier_by_memory_type;
  }
  llvm::Intrinsic::ID getBarrierByMemoryHandleIntrinsic() {
    assert(getArch() == llvm::Triple::dxil &&
           "Barrier by memory handle is only supported for DXIL");
    return llvm::Intrinsic::dx_barrier_by_memory_handle;
  }
  GENERATE_HLSL_INTRINSIC_FUNCTION(GetDimensionsX, resource_getdimensions_x)
  GENERATE_HLSL_INTRINSIC_FUNCTION(GetDimensionsXY, resource_getdimensions_xy)
  GENERATE_HLSL_INTRINSIC_FUNCTION(GetDimensionsLevelsXY,
                                   resource_getdimensions_levels_xy)
  GENERATE_HLSL_INTRINSIC_FUNCTION(LoadLevel, resource_load_level)
  GENERATE_HLSL_INTRINSIC_FUNCTION(LoadMS, resource_load_ms)
  GENERATE_HLSL_INTRINSIC_FUNCTION(CalculateLod, resource_calculate_lod)
  GENERATE_HLSL_INTRINSIC_FUNCTION(CalculateLodUnclamped,
                                   resource_calculate_lod_unclamped)
  GENERATE_HLSL_INTRINSIC_FUNCTION(DdxCoarse, ddx_coarse)
  GENERATE_HLSL_INTRINSIC_FUNCTION(DdyCoarse, ddy_coarse)
  GENERATE_HLSL_INTRINSIC_FUNCTION(DdxFine, ddx_fine)
  GENERATE_HLSL_INTRINSIC_FUNCTION(DdyFine, ddy_fine)
 
  //===----------------------------------------------------------------------===//
  // End of reserved area for HLSL intrinsic getters.
  //===----------------------------------------------------------------------===//
 
protected:
  using SemanticSignatures =
      llvm::SmallVectorImpl<llvm::hlsl::SemanticSignatureElement>;
 
  CodeGenModule &CGM;
 
  llvm::Value *emitSystemSemanticLoad(
      llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
      HLSLAppliedSemanticAttr *Semantic,
      llvm::dxbc::PSV::SemanticKind SemanticKind,
      llvm::Triple::EnvironmentType Stage, std::optional<unsigned> Index,
      SemanticSignatures &Signature, llvm::hlsl::InterpolationMode InterpMode);
 
  void emitSystemSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
                               const clang::DeclaratorDecl *Decl,
                               HLSLAppliedSemanticAttr *Semantic,
                               llvm::dxbc::PSV::SemanticKind SemanticKind,
                               llvm::Triple::EnvironmentType Stage,
                               std::optional<unsigned> Index,
                               SemanticSignatures &Signature,
                               llvm::hlsl::InterpolationMode InterpMode);
 
  llvm::Value *handleScalarSemanticLoad(
      llvm::IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
      const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
      SemanticSignatures &Signature, llvm::hlsl::InterpolationMode InterpMode);
 
  void handleScalarSemanticStore(llvm::IRBuilder<> &B, const FunctionDecl *FD,
                                 llvm::Value *Source,
                                 const clang::DeclaratorDecl *Decl,
                                 HLSLAppliedSemanticAttr *Semantic,
                                 SemanticSignatures &Signature,
                                 llvm::hlsl::InterpolationMode InterpMode);
 
  std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
  handleStructSemanticLoad(
      llvm::IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
      const clang::DeclaratorDecl *Decl,
      specific_attr_iterator<HLSLAppliedSemanticAttr> begin,
      specific_attr_iterator<HLSLAppliedSemanticAttr> end,
      SemanticSignatures &Signature,
      llvm::hlsl::InterpolationModifier Modifiers);
 
  specific_attr_iterator<HLSLAppliedSemanticAttr> handleStructSemanticStore(
      llvm::IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
      const clang::DeclaratorDecl *Decl,
      specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
      specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
      SemanticSignatures &Signature,
      llvm::hlsl::InterpolationModifier Modifiers);
 
  std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
  handleSemanticLoad(llvm::IRBuilder<> &B, const FunctionDecl *FD,
                     llvm::Type *Type, const clang::DeclaratorDecl *Decl,
                     specific_attr_iterator<HLSLAppliedSemanticAttr> begin,
                     specific_attr_iterator<HLSLAppliedSemanticAttr> end,
                     SemanticSignatures &Signature,
                     llvm::hlsl::InterpolationModifier Modifiers =
                         llvm::hlsl::InterpolationModifier::None);
 
  specific_attr_iterator<HLSLAppliedSemanticAttr>
  handleSemanticStore(llvm::IRBuilder<> &B, const FunctionDecl *FD,
                      llvm::Value *Source, const clang::DeclaratorDecl *Decl,
                      specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
                      specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
                      SemanticSignatures &Signature,
                      llvm::hlsl::InterpolationModifier Modifiers =
                          llvm::hlsl::InterpolationModifier::None);
 
public:
  CGHLSLRuntime(CodeGenModule &CGM) : CGM(CGM) {}
  virtual ~CGHLSLRuntime() {}
 
  llvm::Type *convertHLSLSpecificType(const Type *T,
                                      const CGHLSLOffsetInfo &OffsetInfo);
  llvm::Type *convertHLSLSpecificType(const Type *T) {
    return convertHLSLSpecificType(T, CGHLSLOffsetInfo());
  }
 
  void generateGlobalCtorDtorCalls();
 
  void addBuffer(const HLSLBufferDecl *D);
  void addRootSignature(const HLSLRootSignatureDecl *D);
  void finishCodeGen();
 
  void setHLSLEntryAttributes(const FunctionDecl *FD, llvm::Function *Fn);
 
  void emitEntryFunction(const FunctionDecl *FD, llvm::Function *Fn);
  void handleGlobalVarDefinition(const VarDecl *VD, llvm::GlobalVariable *Var);
 
  llvm::Instruction *getConvergenceToken(llvm::BasicBlock &BB);
 
  llvm::StructType *getHLSLBufferLayoutType(const RecordType *LayoutStructTy);
  void addHLSLBufferLayoutType(const RecordType *LayoutStructTy,
                               llvm::StructType *LayoutTy);
  void emitInitListOpaqueValues(CodeGenFunction &CGF, InitListExpr *E);
 
  std::optional<LValue>
  emitResourceArraySubscriptExpr(const ArraySubscriptExpr *E,
                                 CodeGenFunction &CGF);
 
  bool emitGlobalResourceArray(CodeGenFunction &CGF, const Expr *E,
                               AggValueSlot &DestSlot);
  std::optional<LValue>
  emitGlobalResourceArrayAsLValue(CodeGenFunction &CGF,
                                  const VarDecl *ArrayDecl);
 
  std::optional<LValue> emitBufferArraySubscriptExpr(
      const ArraySubscriptExpr *E, CodeGenFunction &CGF,
      llvm::function_ref<llvm::Value *(bool Promote)> EmitIdxAfterBase);
 
  RawAddress createBufferMatrixTempAddress(const LValue &LV,
                                           CodeGenFunction &CGF);
 
  bool emitBufferCopy(CodeGenFunction &CGF, const Expr *E, const LValue &SrcLV,
                      AggValueSlot &DestSlot);
 
  LValue emitBufferMemberExpr(CodeGenFunction &CGF, const MemberExpr *E);
  std::optional<LValue> emitResourceMemberExpr(CodeGenFunction &CGF,
                                               const MemberExpr *E);
 
private:
  void emitBufferGlobalsAndMetadata(const HLSLBufferDecl *BufDecl,
                                    llvm::GlobalVariable *BufGV,
                                    const CGHLSLOffsetInfo &OffsetInfo);
  void initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
                                   llvm::GlobalVariable *GV);
 
  llvm::Value *emitSPIRVUserSemanticLoad(
      llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
      HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index,
      llvm::hlsl::InterpolationMode InterpMode);
  llvm::Value *emitDXILUserSemanticLoad(llvm::IRBuilder<> &B, llvm::Type *Type,
                                        const clang::DeclaratorDecl *Decl,
                                        HLSLAppliedSemanticAttr *Semantic,
                                        std::optional<unsigned> Index,
                                        SemanticSignatures &Signature);
  llvm::Value *emitUserSemanticLoad(llvm::IRBuilder<> &B, llvm::Type *Type,
                                    const clang::DeclaratorDecl *Decl,
                                    HLSLAppliedSemanticAttr *Semantic,
                                    std::optional<unsigned> Index,
                                    SemanticSignatures &Signature,
                                    llvm::hlsl::InterpolationMode InterpMode);
 
  void emitSPIRVUserSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
                                  const clang::DeclaratorDecl *Decl,
                                  HLSLAppliedSemanticAttr *Semantic,
                                  std::optional<unsigned> Index,
                                  llvm::hlsl::InterpolationMode InterpMode);
  void emitDXILUserSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
                                 const clang::DeclaratorDecl *Decl,
                                 HLSLAppliedSemanticAttr *Semantic,
                                 std::optional<unsigned> Index,
                                 SemanticSignatures &Signature);
  void emitUserSemanticStore(llvm::IRBuilder<> &B, llvm::Value *Source,
                             const clang::DeclaratorDecl *Decl,
                             HLSLAppliedSemanticAttr *Semantic,
                             std::optional<unsigned> Index,
                             SemanticSignatures &Signature,
                             llvm::hlsl::InterpolationMode InterpMode);
 
  bool initializeGlobalResourceArray(CodeGenFunction &CGF,
                                     const VarDecl *ArrayDecl,
                                     AggValueSlot &DestSlot);
 
  llvm::Triple::ArchType getArch();
 
  llvm::DenseMap<const clang::RecordType *, llvm::StructType *> LayoutTypes;
  unsigned SPIRVLastAssignedInputSemanticLocation = 0;
  unsigned SPIRVLastAssignedOutputSemanticLocation = 0;
};
 
} // namespace CodeGen
} // namespace clang
 
#endif