97 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(E->
getArg(1));
98 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(E->
getArg(2));
101 LValue Op1TmpLValue =
103 LValue Op2TmpLValue =
109 Value *LowBits =
nullptr;
110 Value *HighBits =
nullptr;
113 llvm::Type *RetElementTy = CGF->
Int32Ty;
115 RetElementTy = llvm::VectorType::get(
116 CGF->
Int32Ty, ElementCount::getFixed(Op0VecTy->getNumElements()));
117 auto *RetTy = llvm::StructType::get(RetElementTy, RetElementTy);
119 CallInst *CI = CGF->
Builder.CreateIntrinsic(
120 RetTy, Intrinsic::dx_splitdouble, {Op0},
nullptr,
"hlsl.splitdouble");
122 LowBits = CGF->
Builder.CreateExtractValue(CI, 0);
123 HighBits = CGF->
Builder.CreateExtractValue(CI, 1);
127 if (!Op0->
getType()->isVectorTy()) {
128 FixedVectorType *DestTy = FixedVectorType::get(CGF->
Int32Ty, 2);
129 Value *Bitcast = CGF->
Builder.CreateBitCast(Op0, DestTy);
131 LowBits = CGF->
Builder.CreateExtractElement(Bitcast, (uint64_t)0);
132 HighBits = CGF->
Builder.CreateExtractElement(Bitcast, 1);
135 if (
const auto *VecTy =
137 NumElements = VecTy->getNumElements();
139 FixedVectorType *Uint32VecTy =
140 FixedVectorType::get(CGF->
Int32Ty, NumElements * 2);
141 Value *Uint32Vec = CGF->
Builder.CreateBitCast(Op0, Uint32VecTy);
142 if (NumElements == 1) {
143 LowBits = CGF->
Builder.CreateExtractElement(Uint32Vec, (uint64_t)0);
144 HighBits = CGF->
Builder.CreateExtractElement(Uint32Vec, 1);
147 for (
int I = 0, E = NumElements; I != E; ++I) {
148 EvenMask.push_back(I * 2);
149 OddMask.push_back(I * 2 + 1);
151 LowBits = CGF->
Builder.CreateShuffleVector(Uint32Vec, EvenMask);
152 HighBits = CGF->
Builder.CreateShuffleVector(Uint32Vec, OddMask);
263 case Builtin::BI__builtin_hlsl_adduint64: {
269 "AddUint64 operand types must match");
271 "AddUint64 operands must have an integer representation");
272 assert((NumElements == 2 || NumElements == 4) &&
273 "AddUint64 operands must have 2 or 4 elements");
281 if (NumElements == 2) {
282 LowA =
Builder.CreateExtractElement(OpA, (uint64_t)0,
"LowA");
283 HighA =
Builder.CreateExtractElement(OpA, (uint64_t)1,
"HighA");
284 LowB =
Builder.CreateExtractElement(OpB, (uint64_t)0,
"LowB");
285 HighB =
Builder.CreateExtractElement(OpB, (uint64_t)1,
"HighB");
287 LowA =
Builder.CreateShuffleVector(OpA, {0, 2},
"LowA");
288 HighA =
Builder.CreateShuffleVector(OpA, {1, 3},
"HighA");
289 LowB =
Builder.CreateShuffleVector(OpB, {0, 2},
"LowB");
290 HighB =
Builder.CreateShuffleVector(OpB, {1, 3},
"HighB");
297 *
this, Intrinsic::uadd_with_overflow, LowA, LowB, Carry);
298 llvm::Value *ZExtCarry =
299 Builder.CreateZExt(Carry, HighA->getType(),
"CarryZExt");
302 llvm::Value *HighSum =
Builder.CreateAdd(HighA, HighB,
"HighSum");
303 llvm::Value *HighSumPlusCarry =
304 Builder.CreateAdd(HighSum, ZExtCarry,
"HighSumPlusCarry");
306 if (NumElements == 4) {
307 return Builder.CreateShuffleVector(LowSum, HighSumPlusCarry, {0, 2, 1, 3},
313 "hlsl.AddUint64.upto0");
318 case Builtin::BI__builtin_hlsl_resource_getpointer: {
323 return Builder.CreateIntrinsic(
324 RetTy,
CGM.getHLSLRuntime().getCreateResourceGetPointerIntrinsic(),
327 case Builtin::BI__builtin_hlsl_resource_uninitializedhandle: {
328 llvm::Type *HandleTy =
CGM.getTypes().ConvertType(E->
getType());
329 return llvm::PoisonValue::get(HandleTy);
331 case Builtin::BI__builtin_hlsl_resource_handlefrombinding: {
332 llvm::Type *HandleTy =
CGM.getTypes().ConvertType(E->
getType());
338 llvm::Intrinsic::ID IntrinsicID =
339 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
341 return Builder.CreateIntrinsic(HandleTy, IntrinsicID, Args);
343 case Builtin::BI__builtin_hlsl_resource_handlefromimplicitbinding: {
344 llvm::Type *HandleTy =
CGM.getTypes().ConvertType(E->
getType());
350 llvm::Intrinsic::ID IntrinsicID =
351 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
353 return Builder.CreateIntrinsic(HandleTy, IntrinsicID, Args);
355 case Builtin::BI__builtin_hlsl_resource_nonuniformindex: {
358 return Builder.CreateIntrinsic(
359 RetTy,
CGM.getHLSLRuntime().getNonUniformResourceIndexIntrinsic(),
362 case Builtin::BI__builtin_hlsl_all: {
364 return Builder.CreateIntrinsic(
369 case Builtin::BI__builtin_hlsl_and: {
372 return Builder.CreateAnd(Op0, Op1,
"hlsl.and");
374 case Builtin::BI__builtin_hlsl_or: {
377 return Builder.CreateOr(Op0, Op1,
"hlsl.or");
379 case Builtin::BI__builtin_hlsl_any: {
381 return Builder.CreateIntrinsic(
386 case Builtin::BI__builtin_hlsl_asdouble:
388 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
395 Ty = VecTy->getElementType();
399 Intr =
CGM.getHLSLRuntime().getNClampIntrinsic();
401 Intr =
CGM.getHLSLRuntime().getUClampIntrinsic();
404 Intr =
CGM.getHLSLRuntime().getSClampIntrinsic();
406 return Builder.CreateIntrinsic(
410 case Builtin::BI__builtin_hlsl_crossf16:
411 case Builtin::BI__builtin_hlsl_crossf32: {
416 "cross operands must have a float representation");
421 "input vectors must have 3 elements each");
422 return Builder.CreateIntrinsic(
423 Op0->
getType(),
CGM.getHLSLRuntime().getCrossIntrinsic(),
426 case Builtin::BI__builtin_hlsl_dot: {
429 llvm::Type *T0 = Op0->
getType();
430 llvm::Type *T1 = Op1->
getType();
433 if (!T0->isVectorTy() && !T1->isVectorTy()) {
434 if (T0->isFloatingPointTy())
435 return Builder.CreateFMul(Op0, Op1,
"hlsl.dot");
437 if (T0->isIntegerTy())
438 return Builder.CreateMul(Op0, Op1,
"hlsl.dot");
441 "Scalar dot product is only supported on ints and floats.");
446 "Dot product operands must have the same type.");
449 assert(VecTy0 &&
"Dot product argument must be a vector.");
451 return Builder.CreateIntrinsic(
456 case Builtin::BI__builtin_hlsl_dot4add_i8packed: {
461 Intrinsic::ID ID =
CGM.getHLSLRuntime().getDot4AddI8PackedIntrinsic();
464 return Builder.CreateIntrinsic(
466 nullptr,
"hlsl.dot4add.i8packed");
468 case Builtin::BI__builtin_hlsl_dot4add_u8packed: {
473 Intrinsic::ID ID =
CGM.getHLSLRuntime().getDot4AddU8PackedIntrinsic();
476 return Builder.CreateIntrinsic(
478 nullptr,
"hlsl.dot4add.u8packed");
480 case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {
483 return Builder.CreateIntrinsic(
488 case Builtin::BI__builtin_hlsl_elementwise_firstbitlow: {
491 return Builder.CreateIntrinsic(
494 nullptr,
"hlsl.firstbitlow");
496 case Builtin::BI__builtin_hlsl_lerp: {
501 llvm_unreachable(
"lerp operand must have a float representation");
502 return Builder.CreateIntrinsic(
503 X->getType(),
CGM.getHLSLRuntime().getLerpIntrinsic(),
506 case Builtin::BI__builtin_hlsl_normalize: {
510 "normalize operand must have a float representation");
512 return Builder.CreateIntrinsic(
515 nullptr,
"hlsl.normalize");
517 case Builtin::BI__builtin_hlsl_elementwise_degrees: {
521 "degree operand must have a float representation");
523 return Builder.CreateIntrinsic(
524 X->getType(),
CGM.getHLSLRuntime().getDegreesIntrinsic(),
527 case Builtin::BI__builtin_hlsl_elementwise_frac: {
530 llvm_unreachable(
"frac operand must have a float representation");
531 return Builder.CreateIntrinsic(
532 Op0->
getType(),
CGM.getHLSLRuntime().getFracIntrinsic(),
535 case Builtin::BI__builtin_hlsl_elementwise_isinf: {
537 llvm::Type *Xty = Op0->
getType();
538 llvm::Type *retType = llvm::Type::getInt1Ty(this->
getLLVMContext());
539 if (Xty->isVectorTy()) {
541 retType = llvm::VectorType::get(
542 retType, ElementCount::getFixed(XVecTy->getNumElements()));
545 llvm_unreachable(
"isinf operand must have a float representation");
546 return Builder.CreateIntrinsic(
547 retType,
CGM.getHLSLRuntime().getIsInfIntrinsic(),
550 case Builtin::BI__builtin_hlsl_mad: {
555 return Builder.CreateIntrinsic(
556 M->
getType(), Intrinsic::fmuladd,
560 if (
CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
561 return Builder.CreateIntrinsic(
562 M->
getType(), Intrinsic::dx_imad,
566 return Builder.CreateNSWAdd(Mul, B);
569 if (
CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
570 return Builder.CreateIntrinsic(
571 M->
getType(), Intrinsic::dx_umad,
575 return Builder.CreateNUWAdd(Mul, B);
577 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
580 llvm_unreachable(
"rcp operand must have a float representation");
581 llvm::Type *Ty = Op0->
getType();
582 llvm::Type *EltTy = Ty->getScalarType();
583 Constant *One = Ty->isVectorTy()
584 ? ConstantVector::getSplat(
585 ElementCount::getFixed(
587 ConstantFP::get(EltTy, 1.0))
588 : ConstantFP::get(EltTy, 1.0);
589 return Builder.CreateFDiv(One, Op0,
"hlsl.rcp");
591 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
594 llvm_unreachable(
"rsqrt operand must have a float representation");
595 return Builder.CreateIntrinsic(
596 Op0->
getType(),
CGM.getHLSLRuntime().getRsqrtIntrinsic(),
599 case Builtin::BI__builtin_hlsl_elementwise_saturate: {
602 "saturate operand must have a float representation");
603 return Builder.CreateIntrinsic(
606 nullptr,
"hlsl.saturate");
608 case Builtin::BI__builtin_hlsl_select: {
621 if (!OpTrue->
getType()->isVectorTy())
623 Builder.CreateVectorSplat(VTy->getNumElements(), OpTrue,
"splat");
624 if (!OpFalse->
getType()->isVectorTy())
626 Builder.CreateVectorSplat(VTy->getNumElements(), OpFalse,
"splat");
630 Builder.CreateSelect(OpCond, OpTrue, OpFalse,
"hlsl.select");
637 case Builtin::BI__builtin_hlsl_step: {
642 "step operands must have a float representation");
643 return Builder.CreateIntrinsic(
644 Op0->
getType(),
CGM.getHLSLRuntime().getStepIntrinsic(),
647 case Builtin::BI__builtin_hlsl_wave_active_all_true: {
649 assert(Op->
getType()->isIntegerTy(1) &&
650 "Intrinsic WaveActiveAllTrue operand must be a bool");
652 Intrinsic::ID ID =
CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();
654 Intrinsic::getOrInsertDeclaration(&
CGM.getModule(), ID), {Op});
656 case Builtin::BI__builtin_hlsl_wave_active_any_true: {
658 assert(Op->
getType()->isIntegerTy(1) &&
659 "Intrinsic WaveActiveAnyTrue operand must be a bool");
661 Intrinsic::ID ID =
CGM.getHLSLRuntime().getWaveActiveAnyTrueIntrinsic();
663 Intrinsic::getOrInsertDeclaration(&
CGM.getModule(), ID), {Op});
665 case Builtin::BI__builtin_hlsl_wave_active_count_bits: {
667 Intrinsic::ID ID =
CGM.getHLSLRuntime().getWaveActiveCountBitsIntrinsic();
669 Intrinsic::getOrInsertDeclaration(&
CGM.getModule(), ID),
672 case Builtin::BI__builtin_hlsl_wave_active_sum: {
676 getTarget().getTriple().getArch(),
CGM.getHLSLRuntime(),
680 &
CGM.getModule(), IID, {OpExpr->getType()}),
681 ArrayRef{OpExpr},
"hlsl.wave.active.sum");
683 case Builtin::BI__builtin_hlsl_wave_active_max: {
687 getTarget().getTriple().getArch(),
CGM.getHLSLRuntime(),
691 &
CGM.getModule(), IID, {OpExpr->getType()}),
692 ArrayRef{OpExpr},
"hlsl.wave.active.max");
694 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
698 switch (
CGM.getTarget().getTriple().getArch()) {
699 case llvm::Triple::dxil:
701 &
CGM.getModule(), Intrinsic::dx_wave_getlaneindex));
702 case llvm::Triple::spirv:
704 llvm::FunctionType::get(
IntTy, {},
false),
705 "__hlsl_wave_get_lane_index", {},
false,
true));
708 "Intrinsic WaveGetLaneIndex not supported by target architecture");
711 case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
712 Intrinsic::ID ID =
CGM.getHLSLRuntime().getWaveIsFirstLaneIntrinsic();
714 Intrinsic::getOrInsertDeclaration(&
CGM.getModule(), ID));
716 case Builtin::BI__builtin_hlsl_wave_get_lane_count: {
717 Intrinsic::ID ID =
CGM.getHLSLRuntime().getWaveGetLaneCountIntrinsic();
719 Intrinsic::getOrInsertDeclaration(&
CGM.getModule(), ID));
721 case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
727 Intrinsic::getOrInsertDeclaration(
728 &
CGM.getModule(),
CGM.getHLSLRuntime().getWaveReadLaneAtIntrinsic(),
729 {OpExpr->getType()}),
730 ArrayRef{OpExpr, OpIndex},
"hlsl.wave.readlane");
732 case Builtin::BI__builtin_hlsl_elementwise_sign: {
733 auto *Arg0 = E->
getArg(0);
735 llvm::Type *Xty = Op0->
getType();
736 llvm::Type *retType = llvm::Type::getInt32Ty(this->
getLLVMContext());
737 if (Xty->isVectorTy()) {
738 auto *XVecTy = Arg0->getType()->castAs<
VectorType>();
739 retType = llvm::VectorType::get(
740 retType, ElementCount::getFixed(XVecTy->getNumElements()));
742 assert((Arg0->getType()->hasFloatingRepresentation() ||
743 Arg0->getType()->hasIntegerRepresentation()) &&
744 "sign operand must have a float or int representation");
746 if (Arg0->getType()->hasUnsignedIntegerRepresentation()) {
747 Value *Cmp =
Builder.CreateICmpEQ(Op0, ConstantInt::get(Xty, 0));
748 return Builder.CreateSelect(Cmp, ConstantInt::get(retType, 0),
749 ConstantInt::get(retType, 1),
"hlsl.sign");
752 return Builder.CreateIntrinsic(
753 retType,
CGM.getHLSLRuntime().getSignIntrinsic(),
756 case Builtin::BI__builtin_hlsl_elementwise_radians: {
759 "radians operand must have a float representation");
760 return Builder.CreateIntrinsic(
763 nullptr,
"hlsl.radians");
765 case Builtin::BI__builtin_hlsl_buffer_update_counter: {
769 return Builder.CreateIntrinsic(
771 CGM.getHLSLRuntime().getBufferUpdateCounterIntrinsic(),
774 case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
779 "asuint operands types mismatch");
782 case Builtin::BI__builtin_hlsl_elementwise_clip:
784 "clip operands types mismatch");
786 case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
788 CGM.getHLSLRuntime().getGroupMemoryBarrierWithGroupSyncIntrinsic();
790 Intrinsic::getOrInsertDeclaration(&
CGM.getModule(), ID));
792 case Builtin::BI__builtin_get_spirv_spec_constant_bool:
793 case Builtin::BI__builtin_get_spirv_spec_constant_short:
794 case Builtin::BI__builtin_get_spirv_spec_constant_ushort:
795 case Builtin::BI__builtin_get_spirv_spec_constant_int:
796 case Builtin::BI__builtin_get_spirv_spec_constant_uint:
797 case Builtin::BI__builtin_get_spirv_spec_constant_longlong:
798 case Builtin::BI__builtin_get_spirv_spec_constant_ulonglong:
799 case Builtin::BI__builtin_get_spirv_spec_constant_half:
800 case Builtin::BI__builtin_get_spirv_spec_constant_float:
801 case Builtin::BI__builtin_get_spirv_spec_constant_double: {
805 llvm::Value *Args[] = {SpecId, DefaultVal};
806 return Builder.CreateCall(SpecConstantFn, Args);