18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/IR/Constants.h"
20 #include "llvm/IR/Instructions.h"
21 #include "llvm/IR/MDBuilder.h"
22 #include "llvm/IR/Metadata.h"
24 using namespace clang;
25 using namespace CodeGen;
39 return cast<ComplexType>(cast<AtomicType>(
type)->getValueType());
44 class ComplexExprEmitter
45 :
public StmtVisitor<ComplexExprEmitter, ComplexPairTy> {
52 : CGF(cgf), Builder(CGF.Builder), IgnoreReal(ir), IgnoreImag(ii) {
60 bool TestAndClearIgnoreReal() {
65 bool TestAndClearIgnoreImag() {
102 llvm_unreachable(
"Stmt can't have complex result type!");
106 if (llvm::Constant *Result =
ConstantEmitter(CGF).tryEmitConstantExpr(E))
108 Result->getAggregateElement(1
U));
113 return Visit(
GE->getResultExpr());
132 assert(Constant &&
"not a constant");
137 llvm::Constant *pair = Constant.
getValue();
139 pair->getAggregateElement(1
U));
145 return emitConstant(Constant, E);
146 return EmitLoadOfLValue(E);
149 return EmitLoadOfLValue(E);
159 return emitConstant(Constant, ME);
161 return EmitLoadOfLValue(ME);
183 if (
const auto *ECE = dyn_cast<ExplicitCastExpr>(E))
192 bool isInc,
bool isPre) {
197 return VisitPrePostIncDec(E,
false,
false);
200 return VisitPrePostIncDec(E,
true,
false);
203 return VisitPrePostIncDec(E,
false,
true);
206 return VisitPrePostIncDec(E,
true,
true);
210 TestAndClearIgnoreReal();
211 TestAndClearIgnoreImag();
233 Scope.ForceCleanup({&Vals.first, &Vals.second});
239 llvm::Constant *
Null = llvm::Constant::getNullValue(CGF.
ConvertType(Elem));
245 llvm::Constant *
Null =
246 llvm::Constant::getNullValue(CGF.
ConvertType(Elem));
263 (
const BinOpInfo &));
271 const BinOpInfo &Op);
274 return EmitBinAdd(EmitBinOps(E));
277 return EmitBinSub(EmitBinOps(E));
280 return EmitBinMul(EmitBinOps(E));
283 return EmitBinDiv(EmitBinOps(E));
292 return EmitCompoundAssign(E, &ComplexExprEmitter::EmitBinAdd);
295 return EmitCompoundAssign(E, &ComplexExprEmitter::EmitBinSub);
298 return EmitCompoundAssign(E, &ComplexExprEmitter::EmitBinMul);
301 return EmitCompoundAssign(E, &ComplexExprEmitter::EmitBinDiv);
322 return EmitLoadOfLValue(E);
351 assert(lvalue.
isSimple() &&
"non-simple complex l-value?");
358 llvm::Value *Real =
nullptr, *Imag =
nullptr;
360 if (!IgnoreReal || isVolatile) {
362 Real = Builder.CreateLoad(RealP, isVolatile, SrcPtr.
getName() +
".real");
365 if (!IgnoreImag || isVolatile) {
367 Imag = Builder.CreateLoad(ImagP, isVolatile, SrcPtr.
getName() +
".imag");
399 llvm::Value *
U = llvm::UndefValue::get(EltTy);
406 return ComplexPairTy(llvm::Constant::getNullValue(Imag->getType()), Imag);
412 return EmitLoadOfLValue(E);
420 assert(RetAlloca.
isValid() &&
"Expected complex return value");
444 ComplexPairTy ComplexExprEmitter::EmitScalarToComplexCast(llvm::Value *Val,
453 return ComplexPairTy(Val, llvm::Constant::getNullValue(Val->getType()));
459 case CK_Dependent: llvm_unreachable(
"dependent cast kind in IR gen!");
463 case CK_AtomicToNonAtomic:
464 case CK_NonAtomicToAtomic:
466 case CK_LValueToRValue:
467 case CK_UserDefinedConversion:
470 case CK_LValueBitCast: {
473 V = Builder.CreateElementBitCast(
V, CGF.
ConvertType(DestTy));
477 case CK_LValueToRValueBitCast: {
483 return EmitLoadOfLValue(DestLV, Op->
getExprLoc());
487 case CK_BaseToDerived:
488 case CK_DerivedToBase:
489 case CK_UncheckedDerivedToBase:
492 case CK_ArrayToPointerDecay:
493 case CK_FunctionToPointerDecay:
494 case CK_NullToPointer:
495 case CK_NullToMemberPointer:
496 case CK_BaseToDerivedMemberPointer:
497 case CK_DerivedToBaseMemberPointer:
498 case CK_MemberPointerToBoolean:
499 case CK_ReinterpretMemberPointer:
500 case CK_ConstructorConversion:
501 case CK_IntegralToPointer:
502 case CK_PointerToIntegral:
503 case CK_PointerToBoolean:
506 case CK_IntegralCast:
507 case CK_BooleanToSignedIntegral:
508 case CK_IntegralToBoolean:
509 case CK_IntegralToFloating:
510 case CK_FloatingToIntegral:
511 case CK_FloatingToBoolean:
512 case CK_FloatingCast:
513 case CK_CPointerToObjCPointerCast:
514 case CK_BlockPointerToObjCPointerCast:
515 case CK_AnyPointerToBlockPointerCast:
516 case CK_ObjCObjectLValueCast:
517 case CK_FloatingComplexToReal:
518 case CK_FloatingComplexToBoolean:
519 case CK_IntegralComplexToReal:
520 case CK_IntegralComplexToBoolean:
521 case CK_ARCProduceObject:
522 case CK_ARCConsumeObject:
523 case CK_ARCReclaimReturnedObject:
524 case CK_ARCExtendBlockObject:
525 case CK_CopyAndAutoreleaseBlockObject:
526 case CK_BuiltinFnToFnPtr:
527 case CK_ZeroToOCLOpaqueType:
528 case CK_AddressSpaceConversion:
529 case CK_IntToOCLSampler:
530 case CK_FloatingToFixedPoint:
531 case CK_FixedPointToFloating:
532 case CK_FixedPointCast:
533 case CK_FixedPointToBoolean:
534 case CK_FixedPointToIntegral:
535 case CK_IntegralToFixedPoint:
537 llvm_unreachable(
"invalid cast kind for complex value");
539 case CK_FloatingRealToComplex:
540 case CK_IntegralRealToComplex: {
546 case CK_FloatingComplexCast:
547 case CK_FloatingComplexToIntegralComplex:
548 case CK_IntegralComplexCast:
549 case CK_IntegralComplexToFloatingComplex: {
551 return EmitComplexToComplexCast(Visit(Op), Op->
getType(), DestTy,
556 llvm_unreachable(
"unknown cast resulting in complex value");
560 TestAndClearIgnoreReal();
561 TestAndClearIgnoreImag();
564 llvm::Value *ResR, *ResI;
565 if (Op.first->getType()->isFloatingPointTy()) {
566 ResR = Builder.CreateFNeg(Op.first,
"neg.r");
567 ResI = Builder.CreateFNeg(Op.second,
"neg.i");
569 ResR = Builder.CreateNeg(Op.first,
"neg.r");
570 ResI = Builder.CreateNeg(Op.second,
"neg.i");
576 TestAndClearIgnoreReal();
577 TestAndClearIgnoreImag();
581 if (Op.second->getType()->isFloatingPointTy())
582 ResI = Builder.CreateFNeg(Op.second,
"conj.i");
584 ResI = Builder.CreateNeg(Op.second,
"conj.i");
589 ComplexPairTy ComplexExprEmitter::EmitBinAdd(
const BinOpInfo &Op) {
590 llvm::Value *ResR, *ResI;
592 if (Op.LHS.first->getType()->isFloatingPointTy()) {
593 ResR = Builder.CreateFAdd(Op.LHS.first, Op.RHS.first,
"add.r");
594 if (Op.LHS.second && Op.RHS.second)
595 ResI = Builder.CreateFAdd(Op.LHS.second, Op.RHS.second,
"add.i");
597 ResI = Op.LHS.second ? Op.LHS.second : Op.RHS.second;
598 assert(ResI &&
"Only one operand may be real!");
600 ResR = Builder.CreateAdd(Op.LHS.first, Op.RHS.first,
"add.r");
601 assert(Op.LHS.second && Op.RHS.second &&
602 "Both operands of integer complex operators must be complex!");
603 ResI = Builder.CreateAdd(Op.LHS.second, Op.RHS.second,
"add.i");
608 ComplexPairTy ComplexExprEmitter::EmitBinSub(
const BinOpInfo &Op) {
609 llvm::Value *ResR, *ResI;
610 if (Op.LHS.first->getType()->isFloatingPointTy()) {
611 ResR = Builder.CreateFSub(Op.LHS.first, Op.RHS.first,
"sub.r");
612 if (Op.LHS.second && Op.RHS.second)
613 ResI = Builder.CreateFSub(Op.LHS.second, Op.RHS.second,
"sub.i");
615 ResI = Op.LHS.second ? Op.LHS.second
616 : Builder.CreateFNeg(Op.RHS.second,
"sub.i");
617 assert(ResI &&
"Only one operand may be real!");
619 ResR = Builder.CreateSub(Op.LHS.first, Op.RHS.first,
"sub.r");
620 assert(Op.LHS.second && Op.RHS.second &&
621 "Both operands of integer complex operators must be complex!");
622 ResI = Builder.CreateSub(Op.LHS.second, Op.RHS.second,
"sub.i");
628 ComplexPairTy ComplexExprEmitter::EmitComplexBinOpLibCall(StringRef LibCallName,
629 const BinOpInfo &Op) {
653 Args, cast<FunctionType>(FQTy.
getTypePtr()),
false);
657 FTy, LibCallName, llvm::AttributeList(),
true);
660 llvm::CallBase *
Call;
669 switch (Ty->getTypeID()) {
671 llvm_unreachable(
"Unsupported floating point type!");
672 case llvm::Type::HalfTyID:
674 case llvm::Type::FloatTyID:
676 case llvm::Type::DoubleTyID:
678 case llvm::Type::PPC_FP128TyID:
680 case llvm::Type::X86_FP80TyID:
682 case llvm::Type::FP128TyID:
689 ComplexPairTy ComplexExprEmitter::EmitBinMul(
const BinOpInfo &Op) {
694 if (Op.LHS.first->getType()->isFloatingPointTy()) {
703 if (Op.LHS.second && Op.RHS.second) {
714 Value *AC = Builder.CreateFMul(Op.LHS.first, Op.RHS.first,
"mul_ac");
715 Value *BD = Builder.CreateFMul(Op.LHS.second, Op.RHS.second,
"mul_bd");
716 Value *AD = Builder.CreateFMul(Op.LHS.first, Op.RHS.second,
"mul_ad");
717 Value *BC = Builder.CreateFMul(Op.LHS.second, Op.RHS.first,
"mul_bc");
721 ResR = Builder.CreateFSub(AC, BD,
"mul_r");
722 ResI = Builder.CreateFAdd(AD, BC,
"mul_i");
726 Value *IsRNaN = Builder.CreateFCmpUNO(ResR, ResR,
"isnan_cmp");
729 llvm::Instruction *Branch = Builder.CreateCondBr(IsRNaN, INaNBB, ContBB);
730 llvm::BasicBlock *OrigBB = Branch->getParent();
734 llvm::MDNode *BrWeight = MDHelper.createBranchWeights(1, (1
U << 20) - 1);
735 Branch->setMetadata(llvm::LLVMContext::MD_prof, BrWeight);
739 Value *IsINaN = Builder.CreateFCmpUNO(ResI, ResI,
"isnan_cmp");
741 Branch = Builder.CreateCondBr(IsINaN, LibCallBB, ContBB);
742 Branch->setMetadata(llvm::LLVMContext::MD_prof, BrWeight);
746 Value *LibCallR, *LibCallI;
747 std::tie(LibCallR, LibCallI) = EmitComplexBinOpLibCall(
749 Builder.CreateBr(ContBB);
754 llvm::PHINode *RealPHI = Builder.CreatePHI(ResR->getType(), 3,
"real_mul_phi");
755 RealPHI->addIncoming(ResR, OrigBB);
756 RealPHI->addIncoming(ResR, INaNBB);
757 RealPHI->addIncoming(LibCallR, LibCallBB);
758 llvm::PHINode *ImagPHI = Builder.CreatePHI(ResI->getType(), 3,
"imag_mul_phi");
759 ImagPHI->addIncoming(ResI, OrigBB);
760 ImagPHI->addIncoming(ResI, INaNBB);
761 ImagPHI->addIncoming(LibCallI, LibCallBB);
764 assert((Op.LHS.second || Op.RHS.second) &&
765 "At least one operand must be complex!");
770 ResR = Builder.CreateFMul(Op.LHS.first, Op.RHS.first,
"mul.rl");
773 ? Builder.CreateFMul(Op.LHS.second, Op.RHS.first,
"mul.il")
774 : Builder.CreateFMul(Op.LHS.first, Op.RHS.second,
"mul.ir");
776 assert(Op.LHS.second && Op.RHS.second &&
777 "Both operands of integer complex operators must be complex!");
778 Value *ResRl = Builder.CreateMul(Op.LHS.first, Op.RHS.first,
"mul.rl");
779 Value *ResRr = Builder.CreateMul(Op.LHS.second, Op.RHS.second,
"mul.rr");
780 ResR = Builder.CreateSub(ResRl, ResRr,
"mul.r");
782 Value *ResIl = Builder.CreateMul(Op.LHS.second, Op.RHS.first,
"mul.il");
783 Value *ResIr = Builder.CreateMul(Op.LHS.first, Op.RHS.second,
"mul.ir");
784 ResI = Builder.CreateAdd(ResIl, ResIr,
"mul.i");
791 ComplexPairTy ComplexExprEmitter::EmitBinDiv(
const BinOpInfo &Op) {
792 llvm::Value *LHSr = Op.LHS.first, *LHSi = Op.LHS.second;
793 llvm::Value *RHSr = Op.RHS.first, *RHSi = Op.RHS.second;
795 llvm::Value *DSTr, *DSTi;
796 if (LHSr->getType()->isFloatingPointTy()) {
805 BinOpInfo LibCallOp = Op;
808 LibCallOp.LHS.second = llvm::Constant::getNullValue(LHSr->getType());
810 switch (LHSr->getType()->getTypeID()) {
812 llvm_unreachable(
"Unsupported floating point type!");
813 case llvm::Type::HalfTyID:
814 return EmitComplexBinOpLibCall(
"__divhc3", LibCallOp);
815 case llvm::Type::FloatTyID:
816 return EmitComplexBinOpLibCall(
"__divsc3", LibCallOp);
817 case llvm::Type::DoubleTyID:
818 return EmitComplexBinOpLibCall(
"__divdc3", LibCallOp);
819 case llvm::Type::PPC_FP128TyID:
820 return EmitComplexBinOpLibCall(
"__divtc3", LibCallOp);
821 case llvm::Type::X86_FP80TyID:
822 return EmitComplexBinOpLibCall(
"__divxc3", LibCallOp);
823 case llvm::Type::FP128TyID:
824 return EmitComplexBinOpLibCall(
"__divtc3", LibCallOp);
828 LHSi = llvm::Constant::getNullValue(RHSi->getType());
831 llvm::Value *AC = Builder.CreateFMul(LHSr, RHSr);
832 llvm::Value *BD = Builder.CreateFMul(LHSi, RHSi);
833 llvm::Value *ACpBD = Builder.CreateFAdd(AC, BD);
835 llvm::Value *CC = Builder.CreateFMul(RHSr, RHSr);
836 llvm::Value *DD = Builder.CreateFMul(RHSi, RHSi);
837 llvm::Value *CCpDD = Builder.CreateFAdd(CC, DD);
839 llvm::Value *BC = Builder.CreateFMul(LHSi, RHSr);
840 llvm::Value *AD = Builder.CreateFMul(LHSr, RHSi);
841 llvm::Value *BCmAD = Builder.CreateFSub(BC, AD);
843 DSTr = Builder.CreateFDiv(ACpBD, CCpDD);
844 DSTi = Builder.CreateFDiv(BCmAD, CCpDD);
846 assert(LHSi &&
"Can have at most one non-complex operand!");
848 DSTr = Builder.CreateFDiv(LHSr, RHSr);
849 DSTi = Builder.CreateFDiv(LHSi, RHSr);
852 assert(Op.LHS.second && Op.RHS.second &&
853 "Both operands of integer complex operators must be complex!");
855 llvm::Value *Tmp1 = Builder.CreateMul(LHSr, RHSr);
856 llvm::Value *Tmp2 = Builder.CreateMul(LHSi, RHSi);
857 llvm::Value *Tmp3 = Builder.CreateAdd(Tmp1, Tmp2);
859 llvm::Value *Tmp4 = Builder.CreateMul(RHSr, RHSr);
860 llvm::Value *Tmp5 = Builder.CreateMul(RHSi, RHSi);
861 llvm::Value *Tmp6 = Builder.CreateAdd(Tmp4, Tmp5);
863 llvm::Value *Tmp7 = Builder.CreateMul(LHSi, RHSr);
864 llvm::Value *Tmp8 = Builder.CreateMul(LHSr, RHSi);
865 llvm::Value *Tmp9 = Builder.CreateSub(Tmp7, Tmp8);
868 DSTr = Builder.CreateUDiv(Tmp3, Tmp6);
869 DSTi = Builder.CreateUDiv(Tmp9, Tmp6);
871 DSTr = Builder.CreateSDiv(Tmp3, Tmp6);
872 DSTi = Builder.CreateSDiv(Tmp9, Tmp6);
879 ComplexExprEmitter::BinOpInfo
881 TestAndClearIgnoreReal();
882 TestAndClearIgnoreImag();
887 Ops.LHS = Visit(E->
getLHS());
891 Ops.RHS = Visit(E->
getRHS());
898 LValue ComplexExprEmitter::
902 TestAndClearIgnoreReal();
903 TestAndClearIgnoreImag();
906 LHSTy = AT->getValueType();
915 QualType ComplexElementTy = cast<ComplexType>(OpInfo.Ty)->getElementType();
926 OpInfo.RHS = Visit(E->
getRHS());
935 OpInfo.LHS = EmitComplexToComplexCast(LHSVal, LHSTy, OpInfo.Ty, Loc);
945 OpInfo.LHS = EmitScalarToComplexCast(LHSVal, LHSTy, OpInfo.Ty, Loc);
955 EmitComplexToComplexCast(Result, OpInfo.Ty, LHSTy, Loc);
956 EmitStoreOfComplex(ResVal, LHS,
false);
959 llvm::Value *ResVal =
971 ComplexPairTy (ComplexExprEmitter::*Func)(
const BinOpInfo&)){
973 LValue LV = EmitCompoundAssignLValue(E, Func, Val);
990 "Invalid assignment");
991 TestAndClearIgnoreReal();
992 TestAndClearIgnoreImag();
1001 EmitStoreOfComplex(Val, LHS,
false);
1008 LValue LV = EmitBinAssignLValue(E, Val);
1018 return EmitLoadOfLValue(LV, E->
getExprLoc());
1023 return Visit(E->
getRHS());
1028 TestAndClearIgnoreReal();
1029 TestAndClearIgnoreImag();
1046 LHSBlock = Builder.GetInsertBlock();
1053 RHSBlock = Builder.GetInsertBlock();
1058 llvm::PHINode *RealPN = Builder.CreatePHI(LHS.first->getType(), 2,
"cond.r");
1059 RealPN->addIncoming(LHS.first, LHSBlock);
1060 RealPN->addIncoming(RHS.first, RHSBlock);
1063 llvm::PHINode *ImagPN = Builder.CreatePHI(LHS.first->getType(), 2,
"cond.i");
1064 ImagPN->addIncoming(LHS.second, LHSBlock);
1065 ImagPN->addIncoming(RHS.second, RHSBlock);
1075 bool Ignore = TestAndClearIgnoreReal();
1077 assert (Ignore ==
false &&
"init list ignored");
1078 Ignore = TestAndClearIgnoreImag();
1080 assert (Ignore ==
false &&
"init list ignored");
1091 assert(E->
getNumInits() == 0 &&
"Unexpected number of inits");
1094 llvm::Value* zeroConstant = llvm::Constant::getNullValue(LTy);
1106 llvm::Value *
U = llvm::UndefValue::get(EltTy);
1123 "Invalid complex expression to emit");
1125 return ComplexExprEmitter(*
this, IgnoreReal, IgnoreImag)
1126 .Visit(
const_cast<Expr *
>(E));
1132 "Invalid complex expression to emit");
1133 ComplexExprEmitter
Emitter(*
this);
1135 Emitter.EmitStoreOfComplex(Val, dest, isInit);
1141 ComplexExprEmitter(*this).EmitStoreOfComplex(
V, dest, isInit);
1147 return ComplexExprEmitter(*this).EmitLoadOfLValue(src, loc);
1153 LValue LVal = ComplexExprEmitter(*this).EmitBinAssignLValue(E, Val);
1161 const ComplexExprEmitter::BinOpInfo &);
1165 case BO_MulAssign:
return &ComplexExprEmitter::EmitBinMul;
1166 case BO_DivAssign:
return &ComplexExprEmitter::EmitBinDiv;
1167 case BO_SubAssign:
return &ComplexExprEmitter::EmitBinSub;
1168 case BO_AddAssign:
return &ComplexExprEmitter::EmitBinAdd;
1170 llvm_unreachable(
"unexpected complex compound assignment");
1178 return ComplexExprEmitter(*this).EmitCompoundAssignLValue(E, Op, Val);
1183 llvm::Value *&Result) {
1186 LValue Ret = ComplexExprEmitter(*this).EmitCompoundAssignLValue(E, Op, Val);