clang  16.0.0git
immintrin.h
Go to the documentation of this file.
1 /*===---- immintrin.h - Intel intrinsics -----------------------------------===
2  *
3  * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4  * See https://llvm.org/LICENSE.txt for license information.
5  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6  *
7  *===-----------------------------------------------------------------------===
8  */
9 
10 #ifndef __IMMINTRIN_H
11 #define __IMMINTRIN_H
12 
13 #if !defined(__i386__) && !defined(__x86_64__)
14 #error "This header is only meant to be used on x86 and x64 architecture"
15 #endif
16 
17 #include <x86gprintrin.h>
18 
19 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
20  defined(__MMX__)
21 #include <mmintrin.h>
22 #endif
23 
24 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
25  defined(__SSE__)
26 #include <xmmintrin.h>
27 #endif
28 
29 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
30  defined(__SSE2__)
31 #include <emmintrin.h>
32 #endif
33 
34 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
35  defined(__SSE3__)
36 #include <pmmintrin.h>
37 #endif
38 
39 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
40  defined(__SSSE3__)
41 #include <tmmintrin.h>
42 #endif
43 
44 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
45  (defined(__SSE4_2__) || defined(__SSE4_1__))
46 #include <smmintrin.h>
47 #endif
48 
49 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
50  (defined(__AES__) || defined(__PCLMUL__))
51 #include <wmmintrin.h>
52 #endif
53 
54 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
55  defined(__CLFLUSHOPT__)
56 #include <clflushoptintrin.h>
57 #endif
58 
59 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
60  defined(__CLWB__)
61 #include <clwbintrin.h>
62 #endif
63 
64 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
65  defined(__AVX__)
66 #include <avxintrin.h>
67 #endif
68 
69 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
70  defined(__AVX2__)
71 #include <avx2intrin.h>
72 #endif
73 
74 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
75  defined(__F16C__)
76 #include <f16cintrin.h>
77 #endif
78 
79 /* No feature check desired due to internal checks */
80 #include <bmiintrin.h>
81 
82 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
83  defined(__BMI2__)
84 #include <bmi2intrin.h>
85 #endif
86 
87 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
88  defined(__LZCNT__)
89 #include <lzcntintrin.h>
90 #endif
91 
92 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
93  defined(__POPCNT__)
94 #include <popcntintrin.h>
95 #endif
96 
97 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
98  defined(__FMA__)
99 #include <fmaintrin.h>
100 #endif
101 
102 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
103  defined(__AVX512F__)
104 #include <avx512fintrin.h>
105 #endif
106 
107 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
108  defined(__AVX512VL__)
109 #include <avx512vlintrin.h>
110 #endif
111 
112 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
113  defined(__AVX512BW__)
114 #include <avx512bwintrin.h>
115 #endif
116 
117 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
118  defined(__AVX512BITALG__)
119 #include <avx512bitalgintrin.h>
120 #endif
121 
122 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
123  defined(__AVX512CD__)
124 #include <avx512cdintrin.h>
125 #endif
126 
127 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
128  defined(__AVX512VPOPCNTDQ__)
129 #include <avx512vpopcntdqintrin.h>
130 #endif
131 
132 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
133  (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__))
134 #include <avx512vpopcntdqvlintrin.h>
135 #endif
136 
137 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
138  defined(__AVX512VNNI__)
139 #include <avx512vnniintrin.h>
140 #endif
141 
142 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
143  (defined(__AVX512VL__) && defined(__AVX512VNNI__))
144 #include <avx512vlvnniintrin.h>
145 #endif
146 
147 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
148  defined(__AVXVNNI__)
149 #include <avxvnniintrin.h>
150 #endif
151 
152 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
153  defined(__AVX512DQ__)
154 #include <avx512dqintrin.h>
155 #endif
156 
157 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
158  (defined(__AVX512VL__) && defined(__AVX512BITALG__))
159 #include <avx512vlbitalgintrin.h>
160 #endif
161 
162 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
163  (defined(__AVX512VL__) && defined(__AVX512BW__))
164 #include <avx512vlbwintrin.h>
165 #endif
166 
167 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
168  (defined(__AVX512VL__) && defined(__AVX512CD__))
169 #include <avx512vlcdintrin.h>
170 #endif
171 
172 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
173  (defined(__AVX512VL__) && defined(__AVX512DQ__))
174 #include <avx512vldqintrin.h>
175 #endif
176 
177 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
178  defined(__AVX512ER__)
179 #include <avx512erintrin.h>
180 #endif
181 
182 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
183  defined(__AVX512IFMA__)
184 #include <avx512ifmaintrin.h>
185 #endif
186 
187 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
188  (defined(__AVX512IFMA__) && defined(__AVX512VL__))
189 #include <avx512ifmavlintrin.h>
190 #endif
191 
192 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
193  defined(__AVXIFMA__)
194 #include <avxifmaintrin.h>
195 #endif
196 
197 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
198  defined(__AVX512VBMI__)
199 #include <avx512vbmiintrin.h>
200 #endif
201 
202 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
203  (defined(__AVX512VBMI__) && defined(__AVX512VL__))
204 #include <avx512vbmivlintrin.h>
205 #endif
206 
207 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
208  defined(__AVX512VBMI2__)
209 #include <avx512vbmi2intrin.h>
210 #endif
211 
212 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
213  (defined(__AVX512VBMI2__) && defined(__AVX512VL__))
214 #include <avx512vlvbmi2intrin.h>
215 #endif
216 
217 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
218  defined(__AVX512PF__)
219 #include <avx512pfintrin.h>
220 #endif
221 
222 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
223  defined(__AVX512FP16__)
224 #include <avx512fp16intrin.h>
225 #endif
226 
227 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
228  (defined(__AVX512VL__) && defined(__AVX512FP16__))
229 #include <avx512vlfp16intrin.h>
230 #endif
231 
232 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
233  defined(__AVX512BF16__)
234 #include <avx512bf16intrin.h>
235 #endif
236 
237 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
238  (defined(__AVX512VL__) && defined(__AVX512BF16__))
239 #include <avx512vlbf16intrin.h>
240 #endif
241 
242 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
243  defined(__PKU__)
244 #include <pkuintrin.h>
245 #endif
246 
247 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
248  defined(__VPCLMULQDQ__)
249 #include <vpclmulqdqintrin.h>
250 #endif
251 
252 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
253  defined(__VAES__)
254 #include <vaesintrin.h>
255 #endif
256 
257 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
258  defined(__GFNI__)
259 #include <gfniintrin.h>
260 #endif
261 
262 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
263  defined(__AVXVNNIINT8__)
264 #include <avxvnniint8intrin.h>
265 #endif
266 
267 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
268  defined(__AVXNECONVERT__)
269 #include <avxneconvertintrin.h>
270 #endif
271 
272 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
273  defined(__RDPID__)
274 /// Returns the value of the IA32_TSC_AUX MSR (0xc0000103).
275 ///
276 /// \headerfile <immintrin.h>
277 ///
278 /// This intrinsic corresponds to the <c> RDPID </c> instruction.
279 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid")))
280 _rdpid_u32(void) {
281  return __builtin_ia32_rdpid();
282 }
283 #endif // __RDPID__
284 
285 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
286  defined(__RDRND__)
287 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
288 _rdrand16_step(unsigned short *__p)
289 {
290  return (int)__builtin_ia32_rdrand16_step(__p);
291 }
292 
293 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
294 _rdrand32_step(unsigned int *__p)
295 {
296  return (int)__builtin_ia32_rdrand32_step(__p);
297 }
298 
299 #ifdef __x86_64__
300 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
301 _rdrand64_step(unsigned long long *__p)
302 {
303  return (int)__builtin_ia32_rdrand64_step(__p);
304 }
305 #else
306 // We need to emulate the functionality of 64-bit rdrand with 2 32-bit
307 // rdrand instructions.
308 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
309 _rdrand64_step(unsigned long long *__p)
310 {
311  unsigned int __lo, __hi;
312  unsigned int __res_lo = __builtin_ia32_rdrand32_step(&__lo);
313  unsigned int __res_hi = __builtin_ia32_rdrand32_step(&__hi);
314  if (__res_lo && __res_hi) {
315  *__p = ((unsigned long long)__hi << 32) | (unsigned long long)__lo;
316  return 1;
317  } else {
318  *__p = 0;
319  return 0;
320  }
321 }
322 #endif
323 #endif /* __RDRND__ */
324 
325 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
326  defined(__FSGSBASE__)
327 #ifdef __x86_64__
328 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
329 _readfsbase_u32(void)
330 {
331  return __builtin_ia32_rdfsbase32();
332 }
333 
334 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
335 _readfsbase_u64(void)
336 {
337  return __builtin_ia32_rdfsbase64();
338 }
339 
340 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
341 _readgsbase_u32(void)
342 {
343  return __builtin_ia32_rdgsbase32();
344 }
345 
346 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
347 _readgsbase_u64(void)
348 {
349  return __builtin_ia32_rdgsbase64();
350 }
351 
352 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
353 _writefsbase_u32(unsigned int __V)
354 {
355  __builtin_ia32_wrfsbase32(__V);
356 }
357 
358 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
359 _writefsbase_u64(unsigned long long __V)
360 {
361  __builtin_ia32_wrfsbase64(__V);
362 }
363 
364 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
365 _writegsbase_u32(unsigned int __V)
366 {
367  __builtin_ia32_wrgsbase32(__V);
368 }
369 
370 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
371 _writegsbase_u64(unsigned long long __V)
372 {
373  __builtin_ia32_wrgsbase64(__V);
374 }
375 
376 #endif
377 #endif /* __FSGSBASE__ */
378 
379 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
380  defined(__MOVBE__)
381 
382 /* The structs used below are to force the load/store to be unaligned. This
383  * is accomplished with the __packed__ attribute. The __may_alias__ prevents
384  * tbaa metadata from being generated based on the struct and the type of the
385  * field inside of it.
386  */
387 
388 static __inline__ short __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
389 _loadbe_i16(void const * __P) {
390  struct __loadu_i16 {
391  unsigned short __v;
392  } __attribute__((__packed__, __may_alias__));
393  return (short)__builtin_bswap16(((const struct __loadu_i16*)__P)->__v);
394 }
395 
396 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
397 _storebe_i16(void * __P, short __D) {
398  struct __storeu_i16 {
399  unsigned short __v;
400  } __attribute__((__packed__, __may_alias__));
401  ((struct __storeu_i16*)__P)->__v = __builtin_bswap16((unsigned short)__D);
402 }
403 
404 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
405 _loadbe_i32(void const * __P) {
406  struct __loadu_i32 {
407  unsigned int __v;
408  } __attribute__((__packed__, __may_alias__));
409  return (int)__builtin_bswap32(((const struct __loadu_i32*)__P)->__v);
410 }
411 
412 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
413 _storebe_i32(void * __P, int __D) {
414  struct __storeu_i32 {
415  unsigned int __v;
416  } __attribute__((__packed__, __may_alias__));
417  ((struct __storeu_i32*)__P)->__v = __builtin_bswap32((unsigned int)__D);
418 }
419 
420 #ifdef __x86_64__
421 static __inline__ long long __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
422 _loadbe_i64(void const * __P) {
423  struct __loadu_i64 {
424  unsigned long long __v;
425  } __attribute__((__packed__, __may_alias__));
426  return (long long)__builtin_bswap64(((const struct __loadu_i64*)__P)->__v);
427 }
428 
429 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
430 _storebe_i64(void * __P, long long __D) {
431  struct __storeu_i64 {
432  unsigned long long __v;
433  } __attribute__((__packed__, __may_alias__));
434  ((struct __storeu_i64*)__P)->__v = __builtin_bswap64((unsigned long long)__D);
435 }
436 #endif
437 #endif /* __MOVBE */
438 
439 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
440  defined(__RTM__)
441 #include <rtmintrin.h>
442 #include <xtestintrin.h>
443 #endif
444 
445 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
446  defined(__SHA__)
447 #include <shaintrin.h>
448 #endif
449 
450 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
451  defined(__FXSR__)
452 #include <fxsrintrin.h>
453 #endif
454 
455 /* No feature check desired due to internal MSC_VER checks */
456 #include <xsaveintrin.h>
457 
458 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
459  defined(__XSAVEOPT__)
460 #include <xsaveoptintrin.h>
461 #endif
462 
463 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
464  defined(__XSAVEC__)
465 #include <xsavecintrin.h>
466 #endif
467 
468 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
469  defined(__XSAVES__)
470 #include <xsavesintrin.h>
471 #endif
472 
473 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
474  defined(__SHSTK__)
475 #include <cetintrin.h>
476 #endif
477 
478 /* Some intrinsics inside adxintrin.h are available only on processors with ADX,
479  * whereas others are also available at all times. */
480 #include <adxintrin.h>
481 
482 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
483  defined(__RDSEED__)
484 #include <rdseedintrin.h>
485 #endif
486 
487 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
488  defined(__WBNOINVD__)
489 #include <wbnoinvdintrin.h>
490 #endif
491 
492 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
493  defined(__CLDEMOTE__)
494 #include <cldemoteintrin.h>
495 #endif
496 
497 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
498  defined(__WAITPKG__)
499 #include <waitpkgintrin.h>
500 #endif
501 
502 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
503  defined(__MOVDIRI__) || defined(__MOVDIR64B__)
504 #include <movdirintrin.h>
505 #endif
506 
507 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
508  defined(__PCONFIG__)
509 #include <pconfigintrin.h>
510 #endif
511 
512 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
513  defined(__SGX__)
514 #include <sgxintrin.h>
515 #endif
516 
517 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
518  defined(__PTWRITE__)
519 #include <ptwriteintrin.h>
520 #endif
521 
522 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
523  defined(__INVPCID__)
524 #include <invpcidintrin.h>
525 #endif
526 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
527  defined(__AMXFP16__)
528 #include <amxfp16intrin.h>
529 #endif
530 
531 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
532  defined(__CMPCCXADD__)
533 #include <cmpccxaddintrin.h>
534 #endif
535 
536 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
537  defined(__KL__) || defined(__WIDEKL__)
538 #include <keylockerintrin.h>
539 #endif
540 
541 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
542  defined(__AMXTILE__) || defined(__AMXINT8__) || defined(__AMXBF16__)
543 #include <amxintrin.h>
544 #endif
545 
546 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
547  defined(__AVX512VP2INTERSECT__)
549 #endif
550 
551 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
552  (defined(__AVX512VL__) && defined(__AVX512VP2INTERSECT__))
554 #endif
555 
556 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
557  defined(__ENQCMD__)
558 #include <enqcmdintrin.h>
559 #endif
560 
561 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
562  defined(__SERIALIZE__)
563 #include <serializeintrin.h>
564 #endif
565 
566 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
567  defined(__TSXLDTRK__)
568 #include <tsxldtrkintrin.h>
569 #endif
570 
571 #if defined(_MSC_VER) && __has_extension(gnu_asm)
572 /* Define the default attributes for these intrinsics */
573 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
574 #ifdef __cplusplus
575 extern "C" {
576 #endif
577 /*----------------------------------------------------------------------------*\
578 |* Interlocked Exchange HLE
579 \*----------------------------------------------------------------------------*/
580 #if defined(__i386__) || defined(__x86_64__)
581 static __inline__ long __DEFAULT_FN_ATTRS
582 _InterlockedExchange_HLEAcquire(long volatile *_Target, long _Value) {
583  __asm__ __volatile__(".byte 0xf2 ; lock ; xchg {%0, %1|%1, %0}"
584  : "+r" (_Value), "+m" (*_Target) :: "memory");
585  return _Value;
586 }
587 static __inline__ long __DEFAULT_FN_ATTRS
588 _InterlockedExchange_HLERelease(long volatile *_Target, long _Value) {
589  __asm__ __volatile__(".byte 0xf3 ; lock ; xchg {%0, %1|%1, %0}"
590  : "+r" (_Value), "+m" (*_Target) :: "memory");
591  return _Value;
592 }
593 #endif
594 #if defined(__x86_64__)
595 static __inline__ __int64 __DEFAULT_FN_ATTRS
596 _InterlockedExchange64_HLEAcquire(__int64 volatile *_Target, __int64 _Value) {
597  __asm__ __volatile__(".byte 0xf2 ; lock ; xchg {%0, %1|%1, %0}"
598  : "+r" (_Value), "+m" (*_Target) :: "memory");
599  return _Value;
600 }
601 static __inline__ __int64 __DEFAULT_FN_ATTRS
602 _InterlockedExchange64_HLERelease(__int64 volatile *_Target, __int64 _Value) {
603  __asm__ __volatile__(".byte 0xf3 ; lock ; xchg {%0, %1|%1, %0}"
604  : "+r" (_Value), "+m" (*_Target) :: "memory");
605  return _Value;
606 }
607 #endif
608 /*----------------------------------------------------------------------------*\
609 |* Interlocked Compare Exchange HLE
610 \*----------------------------------------------------------------------------*/
611 #if defined(__i386__) || defined(__x86_64__)
612 static __inline__ long __DEFAULT_FN_ATTRS
613 _InterlockedCompareExchange_HLEAcquire(long volatile *_Destination,
614  long _Exchange, long _Comparand) {
615  __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg {%2, %1|%1, %2}"
616  : "+a" (_Comparand), "+m" (*_Destination)
617  : "r" (_Exchange) : "memory");
618  return _Comparand;
619 }
620 static __inline__ long __DEFAULT_FN_ATTRS
621 _InterlockedCompareExchange_HLERelease(long volatile *_Destination,
622  long _Exchange, long _Comparand) {
623  __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg {%2, %1|%1, %2}"
624  : "+a" (_Comparand), "+m" (*_Destination)
625  : "r" (_Exchange) : "memory");
626  return _Comparand;
627 }
628 #endif
629 #if defined(__x86_64__)
630 static __inline__ __int64 __DEFAULT_FN_ATTRS
631 _InterlockedCompareExchange64_HLEAcquire(__int64 volatile *_Destination,
632  __int64 _Exchange, __int64 _Comparand) {
633  __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg {%2, %1|%1, %2}"
634  : "+a" (_Comparand), "+m" (*_Destination)
635  : "r" (_Exchange) : "memory");
636  return _Comparand;
637 }
638 static __inline__ __int64 __DEFAULT_FN_ATTRS
639 _InterlockedCompareExchange64_HLERelease(__int64 volatile *_Destination,
640  __int64 _Exchange, __int64 _Comparand) {
641  __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg {%2, %1|%1, %2}"
642  : "+a" (_Comparand), "+m" (*_Destination)
643  : "r" (_Exchange) : "memory");
644  return _Comparand;
645 }
646 #endif
647 #ifdef __cplusplus
648 }
649 #endif
650 
651 #undef __DEFAULT_FN_ATTRS
652 
653 #endif /* defined(_MSC_VER) && __has_extension(gnu_asm) */
654 
655 #endif /* __IMMINTRIN_H */
avx512vlvbmi2intrin.h
avx512vbmi2intrin.h
smmintrin.h
avx512vbmivlintrin.h
avx512bwintrin.h
cldemoteintrin.h
avx512vlintrin.h
lzcntintrin.h
avx512cdintrin.h
avx512ifmaintrin.h
__v
struct __storeu_i16 *__P __v
Definition: immintrin.h:401
avx512vpopcntdqvlintrin.h
xsaveintrin.h
avx512bitalgintrin.h
wbnoinvdintrin.h
__DEFAULT_FN_ATTRS
#define __DEFAULT_FN_ATTRS
Definition: __wmmintrin_aes.h:18
rtmintrin.h
avx512erintrin.h
waitpkgintrin.h
__P
__inline unsigned int unsigned int unsigned int * __P
Definition: bmi2intrin.h:25
xmmintrin.h
avxifmaintrin.h
vaesintrin.h
xsavesintrin.h
tsxldtrkintrin.h
avx512vldqintrin.h
avx512vpopcntdqintrin.h
avx512vbmiintrin.h
gfniintrin.h
avx512bf16intrin.h
__attribute__
static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid"))) _rdpid_u32(void)
Returns the value of the IA32_TSC_AUX MSR (0xc0000103).
Definition: immintrin.h:279
amxfp16intrin.h
adxintrin.h
tmmintrin.h
cetintrin.h
movdirintrin.h
bmiintrin.h
__D
static __inline__ void short __D
Definition: immintrin.h:397
xtestintrin.h
avx512vp2intersectintrin.h
bmi2intrin.h
avx512dqintrin.h
avxvnniintrin.h
emmintrin.h
avxintrin.h
avx512vlfp16intrin.h
serializeintrin.h
clflushoptintrin.h
clwbintrin.h
xsavecintrin.h
shaintrin.h
cmpccxaddintrin.h
sgxintrin.h
ptwriteintrin.h
avx2intrin.h
f16cintrin.h
avx512vlcdintrin.h
keylockerintrin.h
avx512vlvp2intersectintrin.h
pkuintrin.h
x86gprintrin.h
invpcidintrin.h
pmmintrin.h
avxneconvertintrin.h
fmaintrin.h
__p
static __inline unsigned char unsigned int unsigned int unsigned int * __p
Definition: adxintrin.h:24
xsaveoptintrin.h
avx512ifmavlintrin.h
avx512vlbwintrin.h
avx512vlbf16intrin.h
avx512pfintrin.h
avx512fp16intrin.h
avx512fintrin.h
popcntintrin.h
avx512vlvnniintrin.h
wmmintrin.h
mmintrin.h
amxintrin.h
enqcmdintrin.h
avx512vnniintrin.h
pconfigintrin.h
avx512vlbitalgintrin.h
fxsrintrin.h
vpclmulqdqintrin.h
avxvnniint8intrin.h
rdseedintrin.h