#include "CIRGenBuilder.h"
#include "CIRGenFunction.h"
#include "clang/Basic/AArch64CodeGenUtils.h"
#include "clang/Basic/TargetBuiltins.h"
#include "clang/CIR/Dialect/IR/CIRTypes.h"
#include "clang/CIR/MissingFeatures.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/Value.h"
#include "clang/AST/GlobalDecl.h"
#include "clang/Basic/Builtins.h"
#include "clang/Basic/arm_sve_builtin_cg.inc"
#include "clang/Basic/arm_fp16.inc"
#include "clang/Basic/arm_neon.inc"

Macros
#define	SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier)
#define	SVEMAP2(NameBase, TypeModifier)
#define	GET_SVE_LLVM_INTRINSIC_MAP
#define	GET_NEON_OVERLOAD_CHECK

Functions
static mlir::Value	genVscaleTimesFactor (mlir::Location loc, CIRGenBuilderTy builder, mlir::Type cirTy, int32_t scalingFactor)
static const ARMVectorIntrinsicInfo *	findARMVectorIntrinsicInMap (ArrayRef< ARMVectorIntrinsicInfo > intrinsicMap, unsigned builtinID, bool &mapProvenSorted)
static llvm::StringRef	getLLVMIntrNameNoPrefix (llvm::Intrinsic::ID intrID)
static bool	hasExtraNeonArgument (unsigned builtinID)
	Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the vector type.
static cir::VectorType	getFloatNeonType (CIRGenFunction &cgf, NeonTypeFlags intTypeFlags)
static int64_t	getIntValueFromConstOp (mlir::Value val)
static mlir::Value	emitNeonShiftVector (CIRGenBuilderTy &builder, mlir::Value shiftVal, cir::VectorType vecTy, mlir::Location loc, bool neg)
	Build a constant shift amount vector of vecTy to shift a vector Here shiftVal is a constant integer that will be broadcast into a a const vector of vecTy which is the return value of this function If neg is true, the shift amount is negated before splatting (used when encoding a right shift as a left shift by a negative amount for intrinsics like aarch64.neon.
template<typename Operation>
static mlir::Value	emitNeonCallToOp (CIRGenModule &cgm, CIRGenBuilderTy &builder, llvm::SmallVector< mlir::Type > argTypes, llvm::SmallVectorImpl< mlir::Value > &args, std::optional< llvm::StringRef > intrinsicName, mlir::Type funcResTy, mlir::Location loc, bool isConstrainedFPIntrinsic=false, unsigned shift=0, bool rightshift=false)
static mlir::Value	emitNeonCall (CIRGenModule &cgm, CIRGenBuilderTy &builder, llvm::SmallVector< mlir::Type > argTypes, llvm::SmallVectorImpl< mlir::Value > &args, llvm::StringRef intrinsicName, mlir::Type funcResTy, mlir::Location loc, bool isConstrainedFPIntrinsic=false, unsigned shift=0, bool rightshift=false)
static mlir::Value	emitCommonNeonSISDBuiltinExpr (CIRGenFunction &cgf, const ARMVectorIntrinsicInfo &info, llvm::SmallVectorImpl< mlir::Value > &ops, const CallExpr *expr)
static mlir::Value	emitAArch64CompareBuiltinExpr (CIRGenFunction &cgf, CIRGenBuilderTy &builder, mlir::Location loc, mlir::Value src, mlir::Type retTy, const cir::CmpOpKind kind)
static cir::VectorType	getNeonType (CIRGenFunction *cgf, NeonTypeFlags typeFlags, mlir::Location loc, bool hasLegalHalfType=true, bool v1Ty=false, bool allowBFloatArgsAndRet=true)
static mlir::Value	emitNeonSplat (CIRGenBuilderTy &builder, mlir::Location loc, mlir::Value v, mlir::Value lane, unsigned int resEltCnt)
static cir::VectorType	getSignChangedVectorType (CIRGenBuilderTy &builder, cir::VectorType vecTy)
	Flip the signedness of vecTy's element type, keeping the width and number of lanes the same.
static mlir::Value	emitCommonNeonShift (CIRGenBuilderTy &builder, mlir::Location loc, cir::VectorType resTy, mlir::Value shifTgt, mlir::Value shiftAmt, bool shiftLeft)
static mlir::Value	emitNeonRShiftImm (CIRGenFunction &cgf, mlir::Value shiftVec, mlir::Value shiftVal, cir::VectorType vecTy, bool usgn, mlir::Location loc)
static cir::VectorType	getIntVecFromVecTy (CIRGenBuilderTy &builder, cir::VectorType vecTy)
static mlir::Value	emitCommonNeonBuiltinExpr (CIRGenFunction &cgf, unsigned builtinID, unsigned llvmIntrinsic, unsigned altLLVMIntrinsic, const char nameHint, unsigned modifier, const CallExpr expr, llvm::SmallVectorImpl< mlir::Value > &ops)
static mlir::Value	emitCallMaybeConstrainedBuiltin (CIRGenBuilderTy &builder, mlir::Location loc, StringRef intrName, mlir::Type retTy, llvm::SmallVector< mlir::Value > &ops)
static unsigned	getSVEMinEltCount (clang::SVETypeFlags::EltType sveType)
static cir::VectorType	getSVEVectorForElementType (CIRGenModule &cgm, mlir::Type eltTy)

Variables
static const ARMVectorIntrinsicInfo	aarch64SVEIntrinsicMap []
static bool	aarch64SIMDIntrinsicsProvenSorted = false
static bool	aarch64SISDIntrinsicsProvenSorted = false
static bool	aarch64SVEIntrinsicsProvenSorted = false
constexpr unsigned	sveBitsPerBlock = 128
static const std::pair< unsigned, unsigned >	neonEquivalentIntrinsicMap []

Macro Definition Documentation

◆ GET_NEON_OVERLOAD_CHECK

#define GET_NEON_OVERLOAD_CHECK

◆ GET_SVE_LLVM_INTRINSIC_MAP

#define GET_SVE_LLVM_INTRINSIC_MAP

◆ SVEMAP1

#define SVEMAP1	(	NameBase,
		LLVMIntrinsic,
		TypeModifier )

Value:

{#NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \

TypeModifier}

Definition at line 47 of file CIRGenBuiltinAArch64.cpp.

◆ SVEMAP2

#define SVEMAP2	(		NameBase,
			TypeModifier )

Value:

{#NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier}

Definition at line 51 of file CIRGenBuiltinAArch64.cpp.

Function Documentation

◆ emitAArch64CompareBuiltinExpr()

mlir::Value emitAArch64CompareBuiltinExpr	(	CIRGenFunction &	cgf,
		CIRGenBuilderTy &	builder,
		mlir::Location	loc,
		mlir::Value	src,
		mlir::Type	retTy,
		const cir::CmpOpKind	kind )

static

Definition at line 283 of file CIRGenBuiltinAArch64.cpp.

References clang::cast(), cir::CIRBaseBuilderTy::createBitcast(), cir::CIRBaseBuilderTy::createCast(), cir::CIRBaseBuilderTy::createCompare(), cir::CIRBaseBuilderTy::createVecCompare(), cir::CIRBaseBuilderTy::getNullValue(), clang::CIRGen::CIRGenBuilderTy::getSIntNTy(), clang::isa(), and kind.

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr(), and emitCommonNeonBuiltinExpr().

◆ emitCallMaybeConstrainedBuiltin()

mlir::Value emitCallMaybeConstrainedBuiltin	(	CIRGenBuilderTy &	builder,
		mlir::Location	loc,
		StringRef	intrName,
		mlir::Type	retTy,
		llvm::SmallVector< mlir::Value > &	ops )

static

Definition at line 794 of file CIRGenBuiltinAArch64.cpp.

References cir::MissingFeatures::emitConstrainedFPCall(), and clang::CIRGen::CIRGenBuilderTy::emitIntrinsicCallOp().

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr().

◆ emitCommonNeonBuiltinExpr()

mlir::Value emitCommonNeonBuiltinExpr	(	CIRGenFunction &	cgf,
		unsigned	builtinID,
		unsigned	llvmIntrinsic,
		unsigned	altLLVMIntrinsic,
		const char *	nameHint,
		unsigned	modifier,
		const CallExpr *	expr,
		llvm::SmallVectorImpl< mlir::Value > &	ops )

static

Definition at line 438 of file CIRGenBuiltinAArch64.cpp.

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr().

◆ emitCommonNeonShift()

mlir::Value emitCommonNeonShift	(	CIRGenBuilderTy &	builder,
		mlir::Location	loc,
		cir::VectorType	resTy,
		mlir::Value	shifTgt,
		mlir::Value	shiftAmt,
		bool	shiftLeft )

static

Definition at line 387 of file CIRGenBuiltinAArch64.cpp.

References cir::CIRBaseBuilderTy::createBitcast(), and emitNeonShiftVector().

Referenced by emitCommonNeonBuiltinExpr(), and emitNeonRShiftImm().

◆ emitCommonNeonSISDBuiltinExpr()

mlir::Value emitCommonNeonSISDBuiltinExpr	(	CIRGenFunction &	cgf,
		const ARMVectorIntrinsicInfo &	info,
		llvm::SmallVectorImpl< mlir::Value > &	ops,
		const CallExpr *	expr )

static

Definition at line 226 of file CIRGenBuiltinAArch64.cpp.

References clang::aarch64::ARMVectorIntrinsicInfo::BuiltinID, clang::ASTContext::BuiltinInfo, clang::CIRGen::CIRGenFunction::cgm, clang::CIRGen::CIRGenFunction::convertType(), emitNeonCall(), clang::CIRGen::CIRGenModule::errorNYI(), clang::ast_matchers::expr, clang::CIRGen::CIRGenFunction::getBuilder(), clang::CIRGen::CIRGenFunction::getContext(), getLLVMIntrNameNoPrefix(), clang::CIRGen::CIRGenFunction::getLoc(), clang::Builtin::Context::getName(), and clang::aarch64::ARMVectorIntrinsicInfo::LLVMIntrinsic.

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr().

◆ emitNeonCall()

mlir::Value emitNeonCall	(	CIRGenModule &	cgm,
		CIRGenBuilderTy &	builder,
		llvm::SmallVector< mlir::Type >	argTypes,
		llvm::SmallVectorImpl< mlir::Value > &	args,
		llvm::StringRef	intrinsicName,
		mlir::Type	funcResTy,
		mlir::Location	loc,
		bool	isConstrainedFPIntrinsic = false,
		unsigned	shift = 0,
		bool	rightshift = false )

static

Definition at line 214 of file CIRGenBuiltinAArch64.cpp.

References emitNeonCallToOp().

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr(), emitCommonNeonBuiltinExpr(), and emitCommonNeonSISDBuiltinExpr().

◆ emitNeonCallToOp()

template<typename Operation>

mlir::Value emitNeonCallToOp	(	CIRGenModule &	cgm,
		CIRGenBuilderTy &	builder,
		llvm::SmallVector< mlir::Type >	argTypes,
		llvm::SmallVectorImpl< mlir::Value > &	args,
		std::optional< llvm::StringRef >	intrinsicName,
		mlir::Type	funcResTy,
		mlir::Location	loc,
		bool	isConstrainedFPIntrinsic = false,
		unsigned	shift = 0,
		bool	rightshift = false )

static

Definition at line 174 of file CIRGenBuiltinAArch64.cpp.

References cir::CIRBaseBuilderTy::createBitcast(), cir::MissingFeatures::emitConstrainedFPCall(), emitNeonShiftVector(), and clang::CIRGen::CIRGenModule::errorNYI().

Referenced by emitNeonCall().

◆ emitNeonRShiftImm()

mlir::Value emitNeonRShiftImm	(	CIRGenFunction &	cgf,
		mlir::Value	shiftVec,
		mlir::Value	shiftVal,
		cir::VectorType	vecTy,
		bool	usgn,
		mlir::Location	loc )

static

Definition at line 399 of file CIRGenBuiltinAArch64.cpp.

References clang::CIRGen::CIRGenFunction::cgm, cir::CIRBaseBuilderTy::createBitcast(), emitCommonNeonShift(), clang::CIRGen::CIRGenFunction::getBuilder(), clang::CIRGen::CIRGenBuilderTy::getConstInt(), clang::CIRGen::CIRGenModule::getDataLayout(), getIntValueFromConstOp(), cir::CIRDataLayout::getTypeSizeInBits(), and clang::CIRGen::CIRGenBuilderTy::getZero().

Referenced by emitCommonNeonBuiltinExpr().

◆ emitNeonShiftVector()

mlir::Value emitNeonShiftVector	(	CIRGenBuilderTy &	builder,
		mlir::Value	shiftVal,
		cir::VectorType	vecTy,
		mlir::Location	loc,
		bool	neg )

static

Build a constant shift amount vector of vecTy to shift a vector Here shiftVal is a constant integer that will be broadcast into a a const vector of vecTy which is the return value of this function If neg is true, the shift amount is negated before splatting (used when encoding a right shift as a left shift by a negative amount for intrinsics like aarch64.neon.

{s,u}rshl).

Definition at line 156 of file CIRGenBuiltinAArch64.cpp.

References cir::CIRBaseBuilderTy::createIntCast(), cir::CIRBaseBuilderTy::getConstantInt(), and getIntValueFromConstOp().

Referenced by emitCommonNeonShift(), and emitNeonCallToOp().

◆ emitNeonSplat()

mlir::Value emitNeonSplat	(	CIRGenBuilderTy &	builder,
		mlir::Location	loc,
		mlir::Value	v,
		mlir::Value	lane,
		unsigned int	resEltCnt )

static

Definition at line 365 of file CIRGenBuiltinAArch64.cpp.

References clang::CIRGen::CIRGenBuilderTy::createVecShuffle(), getIntValueFromConstOp(), and clang::isa().

Referenced by emitCommonNeonBuiltinExpr().

◆ findARMVectorIntrinsicInMap()

const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap	(	ArrayRef< ARMVectorIntrinsicInfo >	intrinsicMap,
		unsigned	builtinID,
		bool &	mapProvenSorted )

static

Definition at line 66 of file CIRGenBuiltinAArch64.cpp.

References clang::aarch64::ARMVectorIntrinsicInfo::BuiltinID.

Referenced by clang::CodeGen::CodeGenFunction::EmitAArch64BuiltinExpr(), clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr(), clang::CodeGen::CodeGenFunction::EmitAArch64SMEBuiltinExpr(), clang::CodeGen::CodeGenFunction::EmitAArch64SVEBuiltinExpr(), clang::CIRGen::CIRGenFunction::emitAArch64SVEBuiltinExpr(), and clang::CodeGen::CodeGenFunction::EmitARMBuiltinExpr().

◆ genVscaleTimesFactor()

mlir::Value genVscaleTimesFactor	(	mlir::Location	loc,
		CIRGenBuilderTy	builder,
		mlir::Type	cirTy,
		int32_t	scalingFactor )

static

Definition at line 38 of file CIRGenBuiltinAArch64.cpp.

References cir::CIRBaseBuilderTy::createNUWAMul(), clang::CIRGen::CIRGenBuilderTy::emitIntrinsicCallOp(), and clang::CIRGen::CIRGenBuilderTy::getUInt64().

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64SVEBuiltinExpr().

◆ getFloatNeonType()

cir::VectorType getFloatNeonType	(	CIRGenFunction &	cgf,
		NeonTypeFlags	intTypeFlags )

static

Definition at line 131 of file CIRGenBuiltinAArch64.cpp.

References clang::CIRGen::CIRGenTypeCache::doubleTy, clang::CIRGen::CIRGenTypeCache::floatTy, clang::CIRGen::CIRGenTypeCache::fP16Ty, clang::NeonTypeFlags::getEltType(), clang::NeonTypeFlags::Int16, clang::NeonTypeFlags::Int32, clang::NeonTypeFlags::Int64, and clang::NeonTypeFlags::isQuad().

Referenced by emitCommonNeonBuiltinExpr().

◆ getIntValueFromConstOp()

int64_t getIntValueFromConstOp ( mlir::Value val )

static

Definition at line 146 of file CIRGenBuiltinAArch64.cpp.

Referenced by emitNeonRShiftImm(), emitNeonShiftVector(), and emitNeonSplat().

◆ getIntVecFromVecTy()

cir::VectorType getIntVecFromVecTy	(	CIRGenBuilderTy &	builder,
		cir::VectorType	vecTy )

static

Definition at line 425 of file CIRGenBuiltinAArch64.cpp.

References clang::CIRGen::CIRGenBuilderTy::getSInt32Ty(), and clang::CIRGen::CIRGenBuilderTy::getSInt64Ty().

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr().

◆ getLLVMIntrNameNoPrefix()

llvm::StringRef getLLVMIntrNameNoPrefix ( llvm::Intrinsic::ID intrID )

static

Definition at line 88 of file CIRGenBuiltinAArch64.cpp.

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64SVEBuiltinExpr(), emitCommonNeonBuiltinExpr(), emitCommonNeonSISDBuiltinExpr(), and clang::CIRGen::CIRGenFunction::emitSVEPredicateCast().

◆ getNeonType()

cir::VectorType getNeonType	(	CIRGenFunction *	cgf,
		NeonTypeFlags	typeFlags,
		mlir::Location	loc,
		bool	hasLegalHalfType = true,
		bool	v1Ty = false,
		bool	allowBFloatArgsAndRet = true )

static

Definition at line 311 of file CIRGenBuiltinAArch64.cpp.

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr(), and emitCommonNeonBuiltinExpr().

◆ getSignChangedVectorType()

cir::VectorType getSignChangedVectorType	(	CIRGenBuilderTy &	builder,
		cir::VectorType	vecTy )

static

Flip the signedness of vecTy's element type, keeping the width and number of lanes the same.

Used when a NEON intrinsic takes a shift amount vector that must be signed (e.g. aarch64.neon.urshl takes a signed amount even though the data vector is unsigned).

Definition at line 379 of file CIRGenBuiltinAArch64.cpp.

References clang::CIRGen::CIRGenBuilderTy::getSIntNTy(), and clang::CIRGen::CIRGenBuilderTy::getUIntNTy().

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr().

◆ getSVEMinEltCount()

unsigned getSVEMinEltCount ( clang::SVETypeFlags::EltType sveType )

static

Definition at line 870 of file CIRGenBuiltinAArch64.cpp.

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64SVEBuiltinExpr().

◆ getSVEVectorForElementType()

cir::VectorType getSVEVectorForElementType	(	CIRGenModule &	cgm,
		mlir::Type	eltTy )

static

Definition at line 908 of file CIRGenBuiltinAArch64.cpp.

References clang::CIRGen::CIRGenModule::getDataLayout(), cir::CIRDataLayout::getTypeSizeInBits(), and sveBitsPerBlock.

Referenced by clang::CodeGen::CodeGenFunction::EmitAArch64SVEBuiltinExpr(), clang::CIRGen::CIRGenFunction::emitAArch64SVEBuiltinExpr(), clang::CodeGen::CodeGenFunction::EmitSMELd1St1(), and clang::CodeGen::CodeGenFunction::EmitSVEPrefetchLoad().

◆ hasExtraNeonArgument()

bool hasExtraNeonArgument ( unsigned builtinID )

static

Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the vector type.

The additional argument is meant for Sema checking (see CheckNeonBuiltinFunctionCall) and this function should be kept consistent with the logic in Sema. TODO: Make this return false for SISD builtins. TODO(cir): Share this with ARM.cpp

Definition at line 103 of file CIRGenBuiltinAArch64.cpp.

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr().

Variable Documentation

◆ aarch64SIMDIntrinsicsProvenSorted

bool aarch64SIMDIntrinsicsProvenSorted = false

static

Definition at line 59 of file CIRGenBuiltinAArch64.cpp.

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr().

◆ aarch64SISDIntrinsicsProvenSorted

bool aarch64SISDIntrinsicsProvenSorted = false

static

Definition at line 60 of file CIRGenBuiltinAArch64.cpp.

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr().

◆ aarch64SVEIntrinsicMap

const ARMVectorIntrinsicInfo aarch64SVEIntrinsicMap[]

static

Initial value:

= {
#define GET_SVE_LLVM_INTRINSIC_MAP
 
 
}

Definition at line 53 of file CIRGenBuiltinAArch64.cpp.

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64SVEBuiltinExpr().

◆ aarch64SVEIntrinsicsProvenSorted

bool aarch64SVEIntrinsicsProvenSorted = false

static

Definition at line 61 of file CIRGenBuiltinAArch64.cpp.

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64SVEBuiltinExpr().

◆ neonEquivalentIntrinsicMap

const std::pair<unsigned, unsigned> neonEquivalentIntrinsicMap[]

static

Definition at line 1205 of file CIRGenBuiltinAArch64.cpp.

Referenced by clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr().

◆ sveBitsPerBlock

unsigned sveBitsPerBlock = 128

constexpr

Definition at line 906 of file CIRGenBuiltinAArch64.cpp.

Referenced by getSVEVectorForElementType().

Macros

Functions

Variables

Macro Definition Documentation

◆ GET_NEON_OVERLOAD_CHECK

◆ GET_SVE_LLVM_INTRINSIC_MAP

◆ SVEMAP1

◆ SVEMAP2

Function Documentation

◆ emitAArch64CompareBuiltinExpr()

◆ emitCallMaybeConstrainedBuiltin()

◆ emitCommonNeonBuiltinExpr()

◆ emitCommonNeonShift()

◆ emitCommonNeonSISDBuiltinExpr()

◆ emitNeonCall()

◆ emitNeonCallToOp()

◆ emitNeonRShiftImm()

◆ emitNeonShiftVector()

◆ emitNeonSplat()

◆ findARMVectorIntrinsicInMap()

◆ genVscaleTimesFactor()

◆ getFloatNeonType()

◆ getIntValueFromConstOp()

◆ getIntVecFromVecTy()

◆ getLLVMIntrNameNoPrefix()

◆ getNeonType()

◆ getSignChangedVectorType()

◆ getSVEMinEltCount()

◆ getSVEVectorForElementType()

◆ hasExtraNeonArgument()

Variable Documentation

◆ aarch64SIMDIntrinsicsProvenSorted

◆ aarch64SISDIntrinsicsProvenSorted

◆ aarch64SVEIntrinsicMap

◆ aarch64SVEIntrinsicsProvenSorted

◆ neonEquivalentIntrinsicMap

◆ sveBitsPerBlock