clang  9.0.0svn
immintrin.h
Go to the documentation of this file.
1 /*===---- immintrin.h - Intel intrinsics -----------------------------------===
2  *
3  * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4  * See https://llvm.org/LICENSE.txt for license information.
5  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6  *
7  *===-----------------------------------------------------------------------===
8  */
9 
10 #ifndef __IMMINTRIN_H
11 #define __IMMINTRIN_H
12 
13 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__MMX__)
14 #include <mmintrin.h>
15 #endif
16 
17 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE__)
18 #include <xmmintrin.h>
19 #endif
20 
21 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE2__)
22 #include <emmintrin.h>
23 #endif
24 
25 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE3__)
26 #include <pmmintrin.h>
27 #endif
28 
29 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSSE3__)
30 #include <tmmintrin.h>
31 #endif
32 
33 #if !defined(_MSC_VER) || __has_feature(modules) || \
34  (defined(__SSE4_2__) || defined(__SSE4_1__))
35 #include <smmintrin.h>
36 #endif
37 
38 #if !defined(_MSC_VER) || __has_feature(modules) || \
39  (defined(__AES__) || defined(__PCLMUL__))
40 #include <wmmintrin.h>
41 #endif
42 
43 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLFLUSHOPT__)
44 #include <clflushoptintrin.h>
45 #endif
46 
47 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLWB__)
48 #include <clwbintrin.h>
49 #endif
50 
51 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX__)
52 #include <avxintrin.h>
53 #endif
54 
55 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX2__)
56 #include <avx2intrin.h>
57 #endif
58 
59 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__F16C__)
60 #include <f16cintrin.h>
61 #endif
62 
63 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__VPCLMULQDQ__)
64 #include <vpclmulqdqintrin.h>
65 #endif
66 
67 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__BMI__)
68 #include <bmiintrin.h>
69 #endif
70 
71 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__BMI2__)
72 #include <bmi2intrin.h>
73 #endif
74 
75 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__LZCNT__)
76 #include <lzcntintrin.h>
77 #endif
78 
79 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__POPCNT__)
80 #include <popcntintrin.h>
81 #endif
82 
83 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FMA__)
84 #include <fmaintrin.h>
85 #endif
86 
87 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512F__)
88 #include <avx512fintrin.h>
89 #endif
90 
91 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VL__)
92 #include <avx512vlintrin.h>
93 #endif
94 
95 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512BW__)
96 #include <avx512bwintrin.h>
97 #endif
98 
99 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512BITALG__)
100 #include <avx512bitalgintrin.h>
101 #endif
102 
103 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512CD__)
104 #include <avx512cdintrin.h>
105 #endif
106 
107 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VPOPCNTDQ__)
108 #include <avx512vpopcntdqintrin.h>
109 #endif
110 
111 #if !defined(_MSC_VER) || __has_feature(modules) || \
112  (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__))
113 #include <avx512vpopcntdqvlintrin.h>
114 #endif
115 
116 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VNNI__)
117 #include <avx512vnniintrin.h>
118 #endif
119 
120 #if !defined(_MSC_VER) || __has_feature(modules) || \
121  (defined(__AVX512VL__) && defined(__AVX512VNNI__))
122 #include <avx512vlvnniintrin.h>
123 #endif
124 
125 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512DQ__)
126 #include <avx512dqintrin.h>
127 #endif
128 
129 #if !defined(_MSC_VER) || __has_feature(modules) || \
130  (defined(__AVX512VL__) && defined(__AVX512BITALG__))
131 #include <avx512vlbitalgintrin.h>
132 #endif
133 
134 #if !defined(_MSC_VER) || __has_feature(modules) || \
135  (defined(__AVX512VL__) && defined(__AVX512BW__))
136 #include <avx512vlbwintrin.h>
137 #endif
138 
139 #if !defined(_MSC_VER) || __has_feature(modules) || \
140  (defined(__AVX512VL__) && defined(__AVX512CD__))
141 #include <avx512vlcdintrin.h>
142 #endif
143 
144 #if !defined(_MSC_VER) || __has_feature(modules) || \
145  (defined(__AVX512VL__) && defined(__AVX512DQ__))
146 #include <avx512vldqintrin.h>
147 #endif
148 
149 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512ER__)
150 #include <avx512erintrin.h>
151 #endif
152 
153 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512IFMA__)
154 #include <avx512ifmaintrin.h>
155 #endif
156 
157 #if !defined(_MSC_VER) || __has_feature(modules) || \
158  (defined(__AVX512IFMA__) && defined(__AVX512VL__))
159 #include <avx512ifmavlintrin.h>
160 #endif
161 
162 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VBMI__)
163 #include <avx512vbmiintrin.h>
164 #endif
165 
166 #if !defined(_MSC_VER) || __has_feature(modules) || \
167  (defined(__AVX512VBMI__) && defined(__AVX512VL__))
168 #include <avx512vbmivlintrin.h>
169 #endif
170 
171 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VBMI2__)
172 #include <avx512vbmi2intrin.h>
173 #endif
174 
175 #if !defined(_MSC_VER) || __has_feature(modules) || \
176  (defined(__AVX512VBMI2__) && defined(__AVX512VL__))
177 #include <avx512vlvbmi2intrin.h>
178 #endif
179 
180 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512PF__)
181 #include <avx512pfintrin.h>
182 #endif
183 
184 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__PKU__)
185 #include <pkuintrin.h>
186 #endif
187 
188 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__VAES__)
189 #include <vaesintrin.h>
190 #endif
191 
192 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__GFNI__)
193 #include <gfniintrin.h>
194 #endif
195 
196 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RDPID__)
197 /// Returns the value of the IA32_TSC_AUX MSR (0xc0000103).
198 ///
199 /// \headerfile <immintrin.h>
200 ///
201 /// This intrinsic corresponds to the <c> RDPID </c> instruction.
202 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid")))
203 _rdpid_u32(void) {
204  return __builtin_ia32_rdpid();
205 }
206 #endif // __RDPID__
207 
208 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RDRND__)
209 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
210 _rdrand16_step(unsigned short *__p)
211 {
212  return __builtin_ia32_rdrand16_step(__p);
213 }
214 
215 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
216 _rdrand32_step(unsigned int *__p)
217 {
218  return __builtin_ia32_rdrand32_step(__p);
219 }
220 
221 #ifdef __x86_64__
222 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
223 _rdrand64_step(unsigned long long *__p)
224 {
225  return __builtin_ia32_rdrand64_step(__p);
226 }
227 #endif
228 #endif /* __RDRND__ */
229 
230 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FSGSBASE__)
231 #ifdef __x86_64__
232 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
233 _readfsbase_u32(void)
234 {
235  return __builtin_ia32_rdfsbase32();
236 }
237 
238 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
239 _readfsbase_u64(void)
240 {
241  return __builtin_ia32_rdfsbase64();
242 }
243 
244 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
245 _readgsbase_u32(void)
246 {
247  return __builtin_ia32_rdgsbase32();
248 }
249 
250 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
251 _readgsbase_u64(void)
252 {
253  return __builtin_ia32_rdgsbase64();
254 }
255 
256 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
257 _writefsbase_u32(unsigned int __V)
258 {
259  __builtin_ia32_wrfsbase32(__V);
260 }
261 
262 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
263 _writefsbase_u64(unsigned long long __V)
264 {
265  __builtin_ia32_wrfsbase64(__V);
266 }
267 
268 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
269 _writegsbase_u32(unsigned int __V)
270 {
271  __builtin_ia32_wrgsbase32(__V);
272 }
273 
274 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
275 _writegsbase_u64(unsigned long long __V)
276 {
277  __builtin_ia32_wrgsbase64(__V);
278 }
279 
280 #endif
281 #endif /* __FSGSBASE__ */
282 
283 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__MOVBE__)
284 
285 /* The structs used below are to force the load/store to be unaligned. This
286  * is accomplished with the __packed__ attribute. The __may_alias__ prevents
287  * tbaa metadata from being generated based on the struct and the type of the
288  * field inside of it.
289  */
290 
291 static __inline__ short __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
292 _loadbe_i16(void const * __P) {
293  struct __loadu_i16 {
294  short __v;
295  } __attribute__((__packed__, __may_alias__));
296  return __builtin_bswap16(((struct __loadu_i16*)__P)->__v);
297 }
298 
299 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
300 _storebe_i16(void * __P, short __D) {
301  struct __storeu_i16 {
302  short __v;
303  } __attribute__((__packed__, __may_alias__));
304  ((struct __storeu_i16*)__P)->__v = __builtin_bswap16(__D);
305 }
306 
307 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
308 _loadbe_i32(void const * __P) {
309  struct __loadu_i32 {
310  int __v;
311  } __attribute__((__packed__, __may_alias__));
312  return __builtin_bswap32(((struct __loadu_i32*)__P)->__v);
313 }
314 
315 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
316 _storebe_i32(void * __P, int __D) {
317  struct __storeu_i32 {
318  int __v;
319  } __attribute__((__packed__, __may_alias__));
320  ((struct __storeu_i32*)__P)->__v = __builtin_bswap32(__D);
321 }
322 
323 #ifdef __x86_64__
324 static __inline__ long long __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
325 _loadbe_i64(void const * __P) {
326  struct __loadu_i64 {
327  long long __v;
328  } __attribute__((__packed__, __may_alias__));
329  return __builtin_bswap64(((struct __loadu_i64*)__P)->__v);
330 }
331 
332 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
333 _storebe_i64(void * __P, long long __D) {
334  struct __storeu_i64 {
335  long long __v;
336  } __attribute__((__packed__, __may_alias__));
337  ((struct __storeu_i64*)__P)->__v = __builtin_bswap64(__D);
338 }
339 #endif
340 #endif /* __MOVBE */
341 
342 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RTM__)
343 #include <rtmintrin.h>
344 #include <xtestintrin.h>
345 #endif
346 
347 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SHA__)
348 #include <shaintrin.h>
349 #endif
350 
351 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FXSR__)
352 #include <fxsrintrin.h>
353 #endif
354 
355 /* No feature check desired due to internal MSC_VER checks */
356 #include <xsaveintrin.h>
357 
358 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVEOPT__)
359 #include <xsaveoptintrin.h>
360 #endif
361 
362 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVEC__)
363 #include <xsavecintrin.h>
364 #endif
365 
366 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVES__)
367 #include <xsavesintrin.h>
368 #endif
369 
370 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SHSTK__)
371 #include <cetintrin.h>
372 #endif
373 
374 /* Some intrinsics inside adxintrin.h are available only on processors with ADX,
375  * whereas others are also available at all times. */
376 #include <adxintrin.h>
377 
378 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RDSEED__)
379 #include <rdseedintrin.h>
380 #endif
381 
382 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__WBNOINVD__)
383 #include <wbnoinvdintrin.h>
384 #endif
385 
386 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLDEMOTE__)
387 #include <cldemoteintrin.h>
388 #endif
389 
390 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__WAITPKG__)
391 #include <waitpkgintrin.h>
392 #endif
393 
394 #if !defined(_MSC_VER) || __has_feature(modules) || \
395  defined(__MOVDIRI__) || defined(__MOVDIR64B__)
396 #include <movdirintrin.h>
397 #endif
398 
399 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__PCONFIG__)
400 #include <pconfigintrin.h>
401 #endif
402 
403 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SGX__)
404 #include <sgxintrin.h>
405 #endif
406 
407 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__PTWRITE__)
408 #include <ptwriteintrin.h>
409 #endif
410 
411 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__INVPCID__)
412 #include <invpcidintrin.h>
413 #endif
414 
415 #ifdef _MSC_VER
416 /* Define the default attributes for these intrinsics */
417 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
418 #ifdef __cplusplus
419 extern "C" {
420 #endif
421 /*----------------------------------------------------------------------------*\
422 |* Interlocked Exchange HLE
423 \*----------------------------------------------------------------------------*/
424 #if defined(__i386__) || defined(__x86_64__)
425 static __inline__ long __DEFAULT_FN_ATTRS
426 _InterlockedExchange_HLEAcquire(long volatile *_Target, long _Value) {
427  __asm__ __volatile__(".byte 0xf2 ; lock ; xchg %0, %1"
428  : "+r" (_Value), "+m" (*_Target) :: "memory");
429  return _Value;
430 }
431 static __inline__ long __DEFAULT_FN_ATTRS
432 _InterlockedExchange_HLERelease(long volatile *_Target, long _Value) {
433  __asm__ __volatile__(".byte 0xf3 ; lock ; xchg %0, %1"
434  : "+r" (_Value), "+m" (*_Target) :: "memory");
435  return _Value;
436 }
437 #endif
438 #if defined(__x86_64__)
439 static __inline__ __int64 __DEFAULT_FN_ATTRS
440 _InterlockedExchange64_HLEAcquire(__int64 volatile *_Target, __int64 _Value) {
441  __asm__ __volatile__(".byte 0xf2 ; lock ; xchg %0, %1"
442  : "+r" (_Value), "+m" (*_Target) :: "memory");
443  return _Value;
444 }
445 static __inline__ __int64 __DEFAULT_FN_ATTRS
446 _InterlockedExchange64_HLERelease(__int64 volatile *_Target, __int64 _Value) {
447  __asm__ __volatile__(".byte 0xf3 ; lock ; xchg %0, %1"
448  : "+r" (_Value), "+m" (*_Target) :: "memory");
449  return _Value;
450 }
451 #endif
452 /*----------------------------------------------------------------------------*\
453 |* Interlocked Compare Exchange HLE
454 \*----------------------------------------------------------------------------*/
455 #if defined(__i386__) || defined(__x86_64__)
456 static __inline__ long __DEFAULT_FN_ATTRS
457 _InterlockedCompareExchange_HLEAcquire(long volatile *_Destination,
458  long _Exchange, long _Comparand) {
459  __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg %2, %1"
460  : "+a" (_Comparand), "+m" (*_Destination)
461  : "r" (_Exchange) : "memory");
462  return _Comparand;
463 }
464 static __inline__ long __DEFAULT_FN_ATTRS
465 _InterlockedCompareExchange_HLERelease(long volatile *_Destination,
466  long _Exchange, long _Comparand) {
467  __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg %2, %1"
468  : "+a" (_Comparand), "+m" (*_Destination)
469  : "r" (_Exchange) : "memory");
470  return _Comparand;
471 }
472 #endif
473 #if defined(__x86_64__)
474 static __inline__ __int64 __DEFAULT_FN_ATTRS
475 _InterlockedCompareExchange64_HLEAcquire(__int64 volatile *_Destination,
476  __int64 _Exchange, __int64 _Comparand) {
477  __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg %2, %1"
478  : "+a" (_Comparand), "+m" (*_Destination)
479  : "r" (_Exchange) : "memory");
480  return _Comparand;
481 }
482 static __inline__ __int64 __DEFAULT_FN_ATTRS
483 _InterlockedCompareExchange64_HLERelease(__int64 volatile *_Destination,
484  __int64 _Exchange, __int64 _Comparand) {
485  __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg %2, %1"
486  : "+a" (_Comparand), "+m" (*_Destination)
487  : "r" (_Exchange) : "memory");
488  return _Comparand;
489 }
490 #endif
491 #ifdef __cplusplus
492 }
493 #endif
494 
495 #undef __DEFAULT_FN_ATTRS
496 
497 #endif /* _MSC_VER */
498 
499 #endif /* __IMMINTRIN_H */
struct __storeu_i16 *__P __v
Definition: immintrin.h:304
static __inline__ void short __D
Definition: immintrin.h:300
static __inline unsigned char unsigned int unsigned int unsigned int * __p
Definition: adxintrin.h:24
#define __DEFAULT_FN_ATTRS
static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid"))) _rdpid_u32(void)
Returns the value of the IA32_TSC_AUX MSR (0xc0000103).
Definition: immintrin.h:202