clang 22.0.0git
avx512bf16intrin.h
Go to the documentation of this file.
1/*===------------ avx512bf16intrin.h - AVX512_BF16 intrinsics --------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9#ifndef __IMMINTRIN_H
10#error "Never use <avx512bf16intrin.h> directly; include <immintrin.h> instead."
11#endif
12
13#ifdef __SSE2__
14
15#ifndef __AVX512BF16INTRIN_H
16#define __AVX512BF16INTRIN_H
17
18typedef __bf16 __v32bf __attribute__((__vector_size__(64), __aligned__(64)));
19typedef __bf16 __m512bh __attribute__((__vector_size__(64), __aligned__(64)));
20typedef __bf16 __bfloat16 __attribute__((deprecated("use __bf16 instead")));
21
22#define __DEFAULT_FN_ATTRS512 \
23 __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16"), \
24 __min_vector_width__(512)))
25#define __DEFAULT_FN_ATTRS \
26 __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16")))
27
28/// Convert One BF16 Data to One Single Float Data.
29///
30/// \headerfile <x86intrin.h>
31///
32/// This intrinsic does not correspond to a specific instruction.
33///
34/// \param __A
35/// A bfloat data.
36/// \returns A float data whose sign field and exponent field keep unchanged,
37/// and fraction field is extended to 23 bits.
38static __inline__ float __DEFAULT_FN_ATTRS _mm_cvtsbh_ss(__bf16 __A) {
39 return __builtin_ia32_cvtsbf162ss_32(__A);
40}
41
42/// Convert Two Packed Single Data to One Packed BF16 Data.
43///
44/// \headerfile <x86intrin.h>
45///
46/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
47///
48/// \param __A
49/// A 512-bit vector of [16 x float].
50/// \param __B
51/// A 512-bit vector of [16 x float].
52/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from
53/// conversion of __B, and higher 256 bits come from conversion of __A.
54static __inline__ __m512bh __DEFAULT_FN_ATTRS512
55_mm512_cvtne2ps_pbh(__m512 __A, __m512 __B) {
56 return (__m512bh)__builtin_ia32_cvtne2ps2bf16_512((__v16sf) __A,
57 (__v16sf) __B);
58}
59
60/// Convert Two Packed Single Data to One Packed BF16 Data.
61///
62/// \headerfile <x86intrin.h>
63///
64/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
65///
66/// \param __A
67/// A 512-bit vector of [16 x float].
68/// \param __B
69/// A 512-bit vector of [16 x float].
70/// \param __W
71/// A 512-bit vector of [32 x bfloat].
72/// \param __U
73/// A 32-bit mask value specifying what is chosen for each element.
74/// A 1 means conversion of __A or __B. A 0 means element from __W.
75/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from
76/// conversion of __B, and higher 256 bits come from conversion of __A.
77static __inline__ __m512bh __DEFAULT_FN_ATTRS512
78_mm512_mask_cvtne2ps_pbh(__m512bh __W, __mmask32 __U, __m512 __A, __m512 __B) {
79 return (__m512bh)__builtin_ia32_selectpbf_512((__mmask32)__U,
80 (__v32bf)_mm512_cvtne2ps_pbh(__A, __B),
81 (__v32bf)__W);
82}
83
84/// Convert Two Packed Single Data to One Packed BF16 Data.
85///
86/// \headerfile <x86intrin.h>
87///
88/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
89///
90/// \param __A
91/// A 512-bit vector of [16 x float].
92/// \param __B
93/// A 512-bit vector of [16 x float].
94/// \param __U
95/// A 32-bit mask value specifying what is chosen for each element.
96/// A 1 means conversion of __A or __B. A 0 means element is zero.
97/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from
98/// conversion of __B, and higher 256 bits come from conversion of __A.
99static __inline__ __m512bh __DEFAULT_FN_ATTRS512
100_mm512_maskz_cvtne2ps_pbh(__mmask32 __U, __m512 __A, __m512 __B) {
101 return (__m512bh)__builtin_ia32_selectpbf_512((__mmask32)__U,
102 (__v32bf)_mm512_cvtne2ps_pbh(__A, __B),
103 (__v32bf)_mm512_setzero_si512());
104}
105
106/// Convert Packed Single Data to Packed BF16 Data.
107///
108/// \headerfile <x86intrin.h>
109///
110/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
111///
112/// \param __A
113/// A 512-bit vector of [16 x float].
114/// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.
115static __inline__ __m256bh __DEFAULT_FN_ATTRS512
116_mm512_cvtneps_pbh(__m512 __A) {
117 return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,
118 (__v16bf)_mm256_undefined_si256(),
119 (__mmask16)-1);
120}
121
122/// Convert Packed Single Data to Packed BF16 Data.
123///
124/// \headerfile <x86intrin.h>
125///
126/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
127///
128/// \param __A
129/// A 512-bit vector of [16 x float].
130/// \param __W
131/// A 256-bit vector of [16 x bfloat].
132/// \param __U
133/// A 16-bit mask value specifying what is chosen for each element.
134/// A 1 means conversion of __A. A 0 means element from __W.
135/// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.
136static __inline__ __m256bh __DEFAULT_FN_ATTRS512
137_mm512_mask_cvtneps_pbh(__m256bh __W, __mmask16 __U, __m512 __A) {
138 return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,
139 (__v16bf)__W,
140 (__mmask16)__U);
141}
142
143/// Convert Packed Single Data to Packed BF16 Data.
144///
145/// \headerfile <x86intrin.h>
146///
147/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
148///
149/// \param __A
150/// A 512-bit vector of [16 x float].
151/// \param __U
152/// A 16-bit mask value specifying what is chosen for each element.
153/// A 1 means conversion of __A. A 0 means element is zero.
154/// \returns A 256-bit vector of [16 x bfloat] come from conversion of __A.
155static __inline__ __m256bh __DEFAULT_FN_ATTRS512
156_mm512_maskz_cvtneps_pbh(__mmask16 __U, __m512 __A) {
157 return (__m256bh)__builtin_ia32_cvtneps2bf16_512_mask((__v16sf)__A,
158 (__v16bf)_mm256_setzero_si256(),
159 (__mmask16)__U);
160}
161
162/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
163///
164/// \headerfile <x86intrin.h>
165///
166/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
167///
168/// \param __A
169/// A 512-bit vector of [32 x bfloat].
170/// \param __B
171/// A 512-bit vector of [32 x bfloat].
172/// \param __D
173/// A 512-bit vector of [16 x float].
174/// \returns A 512-bit vector of [16 x float] comes from Dot Product of
175/// __A, __B and __D
176static __inline__ __m512 __DEFAULT_FN_ATTRS512
177_mm512_dpbf16_ps(__m512 __D, __m512bh __A, __m512bh __B) {
178 return (__m512)__builtin_ia32_dpbf16ps_512((__v16sf) __D,
179 (__v32bf) __A,
180 (__v32bf) __B);
181}
182
183/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
184///
185/// \headerfile <x86intrin.h>
186///
187/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
188///
189/// \param __A
190/// A 512-bit vector of [32 x bfloat].
191/// \param __B
192/// A 512-bit vector of [32 x bfloat].
193/// \param __D
194/// A 512-bit vector of [16 x float].
195/// \param __U
196/// A 16-bit mask value specifying what is chosen for each element.
197/// A 1 means __A and __B's dot product accumulated with __D. A 0 means __D.
198/// \returns A 512-bit vector of [16 x float] comes from Dot Product of
199/// __A, __B and __D
200static __inline__ __m512 __DEFAULT_FN_ATTRS512
201_mm512_mask_dpbf16_ps(__m512 __D, __mmask16 __U, __m512bh __A, __m512bh __B) {
202 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
203 (__v16sf)_mm512_dpbf16_ps(__D, __A, __B),
204 (__v16sf)__D);
205}
206
207/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
208///
209/// \headerfile <x86intrin.h>
210///
211/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
212///
213/// \param __A
214/// A 512-bit vector of [32 x bfloat].
215/// \param __B
216/// A 512-bit vector of [32 x bfloat].
217/// \param __D
218/// A 512-bit vector of [16 x float].
219/// \param __U
220/// A 16-bit mask value specifying what is chosen for each element.
221/// A 1 means __A and __B's dot product accumulated with __D. A 0 means 0.
222/// \returns A 512-bit vector of [16 x float] comes from Dot Product of
223/// __A, __B and __D
224static __inline__ __m512 __DEFAULT_FN_ATTRS512
225_mm512_maskz_dpbf16_ps(__mmask16 __U, __m512 __D, __m512bh __A, __m512bh __B) {
226 return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
227 (__v16sf)_mm512_dpbf16_ps(__D, __A, __B),
228 (__v16sf)_mm512_setzero_si512());
229}
230
231/// Convert Packed BF16 Data to Packed float Data.
232///
233/// \headerfile <x86intrin.h>
234///
235/// \param __A
236/// A 256-bit vector of [16 x bfloat].
237/// \returns A 512-bit vector of [16 x float] come from conversion of __A
238static __inline__ __m512 __DEFAULT_FN_ATTRS512 _mm512_cvtpbh_ps(__m256bh __A) {
240 (__m512i)_mm512_cvtepi16_epi32((__m256i)__A), 16));
241}
242
243/// Convert Packed BF16 Data to Packed float Data using zeroing mask.
244///
245/// \headerfile <x86intrin.h>
246///
247/// \param __U
248/// A 16-bit mask. Elements are zeroed out when the corresponding mask
249/// bit is not set.
250/// \param __A
251/// A 256-bit vector of [16 x bfloat].
252/// \returns A 512-bit vector of [16 x float] come from conversion of __A
253static __inline__ __m512 __DEFAULT_FN_ATTRS512
254_mm512_maskz_cvtpbh_ps(__mmask16 __U, __m256bh __A) {
256 (__m512i)_mm512_maskz_cvtepi16_epi32((__mmask16)__U, (__m256i)__A), 16));
257}
258
259/// Convert Packed BF16 Data to Packed float Data using merging mask.
260///
261/// \headerfile <x86intrin.h>
262///
263/// \param __S
264/// A 512-bit vector of [16 x float]. Elements are copied from __S when
265/// the corresponding mask bit is not set.
266/// \param __U
267/// A 16-bit mask.
268/// \param __A
269/// A 256-bit vector of [16 x bfloat].
270/// \returns A 512-bit vector of [16 x float] come from conversion of __A
271static __inline__ __m512 __DEFAULT_FN_ATTRS512
272_mm512_mask_cvtpbh_ps(__m512 __S, __mmask16 __U, __m256bh __A) {
274 (__m512i)__S, (__mmask16)__U,
275 (__m512i)_mm512_cvtepi16_epi32((__m256i)__A), 16));
276}
277
278#undef __DEFAULT_FN_ATTRS
279#undef __DEFAULT_FN_ATTRS512
280
281#endif
282#endif
_Float16 __2f16 __attribute__((ext_vector_type(2)))
Zeroes the upper 128 bits (bits 255:128) of all YMM registers.
#define __DEFAULT_FN_ATTRS
#define __DEFAULT_FN_ATTRS512
unsigned int __mmask32
static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_slli_epi32(__m512i __A, unsigned int __B)
static __inline __m512 __DEFAULT_FN_ATTRS512 _mm512_castsi512_ps(__m512i __A)
static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_maskz_cvtepi16_epi32(__mmask16 __U, __m256i __A)
static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_mask_slli_epi32(__m512i __W, __mmask16 __U, __m512i __A, unsigned int __B)
unsigned short __mmask16
static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_setzero_si512(void)
static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_cvtepi16_epi32(__m256i __A)
static __inline__ __m256i __DEFAULT_FN_ATTRS _mm256_undefined_si256(void)
Create a 256-bit integer vector with undefined values.
Definition avxintrin.h:3657
static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_setzero_si256(void)
Constructs a 256-bit integer vector initialized to zero.
Definition avxintrin.h:4340
static __inline__ void short __D
Definition immintrin.h:342