clang 23.0.0git
InterpBuiltin.cpp
Go to the documentation of this file.
1//===--- InterpBuiltin.cpp - Interpreter for the constexpr VM ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
9#include "Boolean.h"
10#include "Char.h"
11#include "EvalEmitter.h"
13#include "InterpHelpers.h"
14#include "PrimType.h"
15#include "Program.h"
17#include "clang/AST/OSLog.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/Support/AllocToken.h"
24#include "llvm/Support/ErrorHandling.h"
25#include "llvm/Support/SipHash.h"
26
27namespace clang {
28namespace interp {
29
30[[maybe_unused]] static bool isNoopBuiltin(unsigned ID) {
31 switch (ID) {
32 case Builtin::BIas_const:
33 case Builtin::BIforward:
34 case Builtin::BIforward_like:
35 case Builtin::BImove:
36 case Builtin::BImove_if_noexcept:
37 case Builtin::BIaddressof:
38 case Builtin::BI__addressof:
39 case Builtin::BI__builtin_addressof:
40 case Builtin::BI__builtin_launder:
41 return true;
42 default:
43 return false;
44 }
45 return false;
46}
47
48static void discard(InterpStack &Stk, PrimType T) {
49 TYPE_SWITCH(T, { Stk.discard<T>(); });
50}
51
52static uint64_t popToUInt64(const InterpState &S, const Expr *E) {
54 return static_cast<uint64_t>(S.Stk.pop<T>()));
55}
56
58 INT_TYPE_SWITCH(T, return Stk.pop<T>().toAPSInt());
59}
60
61static APSInt popToAPSInt(InterpState &S, const Expr *E) {
62 return popToAPSInt(S.Stk, *S.getContext().classify(E->getType()));
63}
65 return popToAPSInt(S.Stk, *S.getContext().classify(T));
66}
67
68/// Check for common reasons a pointer can't be read from, which
69/// are usually not diagnosed in a builtin function.
70static bool isReadable(const Pointer &P) {
71 if (P.isDummy())
72 return false;
73 if (!P.isBlockPointer())
74 return false;
75 if (!P.isLive())
76 return false;
77 if (P.isOnePastEnd())
78 return false;
79 return true;
80}
81
82/// Pushes \p Val on the stack as the type given by \p QT.
83static void pushInteger(InterpState &S, const APSInt &Val, QualType QT) {
87 assert(T);
88 unsigned BitWidth = S.getASTContext().getIntWidth(QT);
89
90 if (T == PT_IntAPS) {
91 auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
92 Result.copy(Val);
94 return;
95 }
96
97 if (T == PT_IntAP) {
98 auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
99 Result.copy(Val);
101 return;
102 }
103
105 int64_t V = Val.getSExtValue();
106 INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V, BitWidth)); });
107 } else {
109 uint64_t V = Val.getZExtValue();
110 INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V, BitWidth)); });
111 }
112}
113
114template <typename T>
115static void pushInteger(InterpState &S, T Val, QualType QT) {
116 if constexpr (std::is_same_v<T, APInt>)
117 pushInteger(S, APSInt(Val, !std::is_signed_v<T>), QT);
118 else if constexpr (std::is_same_v<T, APSInt>)
119 pushInteger(S, Val, QT);
120 else
121 pushInteger(S,
122 APSInt(APInt(sizeof(T) * 8, static_cast<uint64_t>(Val),
123 std::is_signed_v<T>),
124 !std::is_signed_v<T>),
125 QT);
126}
127
128static void assignInteger(InterpState &S, const Pointer &Dest, PrimType ValueT,
129 const APSInt &Value) {
130
131 if (ValueT == PT_IntAPS) {
132 Dest.deref<IntegralAP<true>>() =
133 S.allocAP<IntegralAP<true>>(Value.getBitWidth());
134 Dest.deref<IntegralAP<true>>().copy(Value);
135 } else if (ValueT == PT_IntAP) {
136 Dest.deref<IntegralAP<false>>() =
137 S.allocAP<IntegralAP<false>>(Value.getBitWidth());
138 Dest.deref<IntegralAP<false>>().copy(Value);
139 } else {
141 ValueT, { Dest.deref<T>() = T::from(static_cast<T>(Value)); });
142 }
143}
144
145static QualType getElemType(const Pointer &P) {
146 const Descriptor *Desc = P.getFieldDesc();
147 QualType T = Desc->getType();
148 if (Desc->isPrimitive())
149 return T;
150 if (T->isPointerType())
151 return T->castAs<PointerType>()->getPointeeType();
152 if (Desc->isArray())
153 return Desc->getElemQualType();
154 if (const auto *AT = T->getAsArrayTypeUnsafe())
155 return AT->getElementType();
156 return T;
157}
158
160 unsigned ID) {
161 if (!S.diagnosing())
162 return;
163
164 auto Loc = S.Current->getSource(OpPC);
165 if (S.getLangOpts().CPlusPlus11)
166 S.CCEDiag(Loc, diag::note_constexpr_invalid_function)
167 << /*isConstexpr=*/0 << /*isConstructor=*/0
169 else
170 S.CCEDiag(Loc, diag::note_invalid_subexpr_in_const_expr);
171}
172
173static llvm::APSInt convertBoolVectorToInt(const Pointer &Val) {
174 assert(Val.getFieldDesc()->isPrimitiveArray() &&
176 "Not a boolean vector");
177 unsigned NumElems = Val.getNumElems();
178
179 // Each element is one bit, so create an integer with NumElts bits.
180 llvm::APSInt Result(NumElems, 0);
181 for (unsigned I = 0; I != NumElems; ++I) {
182 if (Val.elem<bool>(I))
183 Result.setBit(I);
184 }
185
186 return Result;
187}
188
189// Strict double -> float conversion used for X86 PD2PS/cvtsd2ss intrinsics.
190// Reject NaN/Inf/Subnormal inputs and any lossy/inexact conversions.
192 InterpState &S, const Expr *DiagExpr) {
193 if (Src.isInfinity()) {
194 if (S.diagnosing())
195 S.CCEDiag(DiagExpr, diag::note_constexpr_float_arithmetic) << 0;
196 return false;
197 }
198 if (Src.isNaN()) {
199 if (S.diagnosing())
200 S.CCEDiag(DiagExpr, diag::note_constexpr_float_arithmetic) << 1;
201 return false;
202 }
203 APFloat Val = Src;
204 bool LosesInfo = false;
205 APFloat::opStatus Status = Val.convert(
206 APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven, &LosesInfo);
207 if (LosesInfo || Val.isDenormal()) {
208 if (S.diagnosing())
209 S.CCEDiag(DiagExpr, diag::note_constexpr_float_arithmetic_strict);
210 return false;
211 }
212 if (Status != APFloat::opOK) {
213 if (S.diagnosing())
214 S.CCEDiag(DiagExpr, diag::note_invalid_subexpr_in_const_expr);
215 return false;
216 }
217 Dst.copy(Val);
218 return true;
219}
220
222 const InterpFrame *Frame,
223 const CallExpr *Call) {
224 unsigned Depth = S.Current->getDepth();
225 auto isStdCall = [](const FunctionDecl *F) -> bool {
226 return F && F->isInStdNamespace() && F->getIdentifier() &&
227 F->getIdentifier()->isStr("is_constant_evaluated");
228 };
229 const InterpFrame *Caller = Frame->Caller;
230 // The current frame is the one for __builtin_is_constant_evaluated.
231 // The one above that, potentially the one for std::is_constant_evaluated().
233 S.getEvalStatus().Diag &&
234 (Depth == 0 || (Depth == 1 && isStdCall(Frame->getCallee())))) {
235 if (Caller && isStdCall(Frame->getCallee())) {
236 const Expr *E = Caller->getExpr(Caller->getRetPC());
237 S.report(E->getExprLoc(),
238 diag::warn_is_constant_evaluated_always_true_constexpr)
239 << "std::is_constant_evaluated" << E->getSourceRange();
240 } else {
241 S.report(Call->getExprLoc(),
242 diag::warn_is_constant_evaluated_always_true_constexpr)
243 << "__builtin_is_constant_evaluated" << Call->getSourceRange();
244 }
245 }
246
248 return true;
249}
250
251// __builtin_assume
252// __assume (MS extension)
254 const InterpFrame *Frame,
255 const CallExpr *Call) {
256 // Nothing to be done here since the argument is NOT evaluated.
257 assert(Call->getNumArgs() == 1);
258 return true;
259}
260
262 const InterpFrame *Frame,
263 const CallExpr *Call, unsigned ID) {
264 uint64_t Limit = ~static_cast<uint64_t>(0);
265 if (ID == Builtin::BIstrncmp || ID == Builtin::BI__builtin_strncmp ||
266 ID == Builtin::BIwcsncmp || ID == Builtin::BI__builtin_wcsncmp)
267 Limit = popToUInt64(S, Call->getArg(2));
268
269 const Pointer &B = S.Stk.pop<Pointer>();
270 const Pointer &A = S.Stk.pop<Pointer>();
271 if (ID == Builtin::BIstrcmp || ID == Builtin::BIstrncmp ||
272 ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp)
273 diagnoseNonConstexprBuiltin(S, OpPC, ID);
274
275 if (Limit == 0) {
276 pushInteger(S, 0, Call->getType());
277 return true;
278 }
279
280 if (!CheckLive(S, OpPC, A, AK_Read) || !CheckLive(S, OpPC, B, AK_Read))
281 return false;
282
283 if (A.isDummy() || B.isDummy())
284 return false;
285 if (!A.isBlockPointer() || !B.isBlockPointer())
286 return false;
287 if (!A.getFieldDesc()->isPrimitiveArray() ||
289 return false;
290
291 bool IsWide = ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp ||
292 ID == Builtin::BI__builtin_wcscmp ||
293 ID == Builtin::BI__builtin_wcsncmp;
294 assert(A.getFieldDesc()->isPrimitiveArray());
295 assert(B.getFieldDesc()->isPrimitiveArray());
296
297 // Different element types shouldn't happen, but with casts they can.
299 return false;
300
301 PrimType ElemT = *S.getContext().classify(getElemType(A));
302
303 auto returnResult = [&](int V) -> bool {
304 pushInteger(S, V, Call->getType());
305 return true;
306 };
307
308 unsigned IndexA = A.getIndex();
309 unsigned IndexB = B.getIndex();
310 uint64_t Steps = 0;
311 for (;; ++IndexA, ++IndexB, ++Steps) {
312
313 if (Steps >= Limit)
314 break;
315 const Pointer &PA = A.atIndex(IndexA);
316 const Pointer &PB = B.atIndex(IndexB);
317 if (!CheckRange(S, OpPC, PA, AK_Read) ||
318 !CheckRange(S, OpPC, PB, AK_Read)) {
319 return false;
320 }
321
322 if (IsWide) {
323 INT_TYPE_SWITCH(ElemT, {
324 T CA = PA.deref<T>();
325 T CB = PB.deref<T>();
326 if (CA > CB)
327 return returnResult(1);
328 if (CA < CB)
329 return returnResult(-1);
330 if (CA.isZero() || CB.isZero())
331 return returnResult(0);
332 });
333 continue;
334 }
335
336 uint8_t CA = PA.deref<uint8_t>();
337 uint8_t CB = PB.deref<uint8_t>();
338
339 if (CA > CB)
340 return returnResult(1);
341 if (CA < CB)
342 return returnResult(-1);
343 if (CA == 0 || CB == 0)
344 return returnResult(0);
345 }
346
347 return returnResult(0);
348}
349
351 const InterpFrame *Frame,
352 const CallExpr *Call, unsigned ID) {
353 const Pointer &StrPtr = S.Stk.pop<Pointer>().expand();
354
355 if (ID == Builtin::BIstrlen || ID == Builtin::BIwcslen)
356 diagnoseNonConstexprBuiltin(S, OpPC, ID);
357
358 if (!CheckArray(S, OpPC, StrPtr))
359 return false;
360
361 if (!CheckLive(S, OpPC, StrPtr, AK_Read))
362 return false;
363
364 if (!StrPtr.isBlockPointer())
365 return false;
366
367 if (!CheckDummy(S, OpPC, StrPtr.block(), AK_Read))
368 return false;
369
370 if (!StrPtr.getFieldDesc()->isPrimitiveArray())
371 return false;
372
373 assert(StrPtr.getFieldDesc()->isPrimitiveArray());
374 PrimType ElemT = StrPtr.getFieldDesc()->getPrimType();
375 unsigned ElemSize = StrPtr.getFieldDesc()->getElemDataSize();
376 if (ElemSize != 1 && ElemSize != 2 && ElemSize != 4)
377 return Invalid(S, OpPC);
378
379 if (ID == Builtin::BI__builtin_wcslen || ID == Builtin::BIwcslen) {
380 const ASTContext &AC = S.getASTContext();
381 unsigned WCharSize = AC.getTypeSizeInChars(AC.getWCharType()).getQuantity();
382 if (StrPtr.getFieldDesc()->getElemDataSize() != WCharSize)
383 return false;
384 }
385
386 size_t Len = 0;
387 for (size_t I = StrPtr.getIndex();; ++I, ++Len) {
388 const Pointer &ElemPtr = StrPtr.atIndex(I);
389
390 if (!CheckRange(S, OpPC, ElemPtr, AK_Read))
391 return false;
392
393 uint32_t Val;
395 ElemT, { Val = static_cast<uint32_t>(ElemPtr.deref<T>()); });
396 if (Val == 0)
397 break;
398 }
399
400 pushInteger(S, Len, Call->getType());
401
402 return true;
403}
404
406 const InterpFrame *Frame, const CallExpr *Call,
407 bool Signaling) {
408 const Pointer &Arg = S.Stk.pop<Pointer>();
409
410 if (!CheckLoad(S, OpPC, Arg))
411 return false;
412
413 if (!Arg.getFieldDesc()->isPrimitiveArray())
414 return Invalid(S, OpPC);
415
416 // Convert the given string to an integer using StringRef's API.
417 llvm::APInt Fill;
418 std::string Str;
419 unsigned ArgLength = Arg.getNumElems();
420 bool FoundZero = false;
421 for (unsigned I = 0; I != ArgLength; ++I) {
422 if (!Arg.isElementInitialized(I))
423 return false;
424
425 if (Arg.elem<int8_t>(I) == 0) {
426 FoundZero = true;
427 break;
428 }
429 Str += Arg.elem<char>(I);
430 }
431
432 // If we didn't find a NUL byte, diagnose as a one-past-the-end read.
433 if (!FoundZero)
434 return CheckRange(S, OpPC, Arg.atIndex(ArgLength), AK_Read);
435
436 // Treat empty strings as if they were zero.
437 if (Str.empty())
438 Fill = llvm::APInt(32, 0);
439 else if (StringRef(Str).getAsInteger(0, Fill))
440 return false;
441
442 const llvm::fltSemantics &TargetSemantics =
444 Call->getDirectCallee()->getReturnType());
445
446 Floating Result = S.allocFloat(TargetSemantics);
448 if (Signaling)
449 Result.copy(
450 llvm::APFloat::getSNaN(TargetSemantics, /*Negative=*/false, &Fill));
451 else
452 Result.copy(
453 llvm::APFloat::getQNaN(TargetSemantics, /*Negative=*/false, &Fill));
454 } else {
455 // Prior to IEEE 754-2008, architectures were allowed to choose whether
456 // the first bit of their significand was set for qNaN or sNaN. MIPS chose
457 // a different encoding to what became a standard in 2008, and for pre-
458 // 2008 revisions, MIPS interpreted sNaN-2008 as qNan and qNaN-2008 as
459 // sNaN. This is now known as "legacy NaN" encoding.
460 if (Signaling)
461 Result.copy(
462 llvm::APFloat::getQNaN(TargetSemantics, /*Negative=*/false, &Fill));
463 else
464 Result.copy(
465 llvm::APFloat::getSNaN(TargetSemantics, /*Negative=*/false, &Fill));
466 }
467
469 return true;
470}
471
473 const InterpFrame *Frame,
474 const CallExpr *Call) {
475 const llvm::fltSemantics &TargetSemantics =
477 Call->getDirectCallee()->getReturnType());
478
479 Floating Result = S.allocFloat(TargetSemantics);
480 Result.copy(APFloat::getInf(TargetSemantics));
482 return true;
483}
484
486 const InterpFrame *Frame) {
487 const Floating &Arg2 = S.Stk.pop<Floating>();
488 const Floating &Arg1 = S.Stk.pop<Floating>();
489 Floating Result = S.allocFloat(Arg1.getSemantics());
490
491 APFloat Copy = Arg1.getAPFloat();
492 Copy.copySign(Arg2.getAPFloat());
493 Result.copy(Copy);
495
496 return true;
497}
498
500 const InterpFrame *Frame, bool IsNumBuiltin) {
501 const Floating &RHS = S.Stk.pop<Floating>();
502 const Floating &LHS = S.Stk.pop<Floating>();
503 Floating Result = S.allocFloat(LHS.getSemantics());
504
505 if (IsNumBuiltin)
506 Result.copy(llvm::minimumnum(LHS.getAPFloat(), RHS.getAPFloat()));
507 else
508 Result.copy(minnum(LHS.getAPFloat(), RHS.getAPFloat()));
510 return true;
511}
512
514 const InterpFrame *Frame, bool IsNumBuiltin) {
515 const Floating &RHS = S.Stk.pop<Floating>();
516 const Floating &LHS = S.Stk.pop<Floating>();
517 Floating Result = S.allocFloat(LHS.getSemantics());
518
519 if (IsNumBuiltin)
520 Result.copy(llvm::maximumnum(LHS.getAPFloat(), RHS.getAPFloat()));
521 else
522 Result.copy(maxnum(LHS.getAPFloat(), RHS.getAPFloat()));
524 return true;
525}
526
527/// Defined as __builtin_isnan(...), to accommodate the fact that it can
528/// take a float, double, long double, etc.
529/// But for us, that's all a Floating anyway.
531 const InterpFrame *Frame,
532 const CallExpr *Call) {
533 const Floating &Arg = S.Stk.pop<Floating>();
534
535 pushInteger(S, Arg.isNan(), Call->getType());
536 return true;
537}
538
540 const InterpFrame *Frame,
541 const CallExpr *Call) {
542 const Floating &Arg = S.Stk.pop<Floating>();
543
544 pushInteger(S, Arg.isSignaling(), Call->getType());
545 return true;
546}
547
549 const InterpFrame *Frame, bool CheckSign,
550 const CallExpr *Call) {
551 const Floating &Arg = S.Stk.pop<Floating>();
552 APFloat F = Arg.getAPFloat();
553 bool IsInf = F.isInfinity();
554
555 if (CheckSign)
556 pushInteger(S, IsInf ? (F.isNegative() ? -1 : 1) : 0, Call->getType());
557 else
558 pushInteger(S, IsInf, Call->getType());
559 return true;
560}
561
563 const InterpFrame *Frame,
564 const CallExpr *Call) {
565 const Floating &Arg = S.Stk.pop<Floating>();
566
567 pushInteger(S, Arg.isFinite(), Call->getType());
568 return true;
569}
570
572 const InterpFrame *Frame,
573 const CallExpr *Call) {
574 const Floating &Arg = S.Stk.pop<Floating>();
575
576 pushInteger(S, Arg.isNormal(), Call->getType());
577 return true;
578}
579
581 const InterpFrame *Frame,
582 const CallExpr *Call) {
583 const Floating &Arg = S.Stk.pop<Floating>();
584
585 pushInteger(S, Arg.isDenormal(), Call->getType());
586 return true;
587}
588
590 const InterpFrame *Frame,
591 const CallExpr *Call) {
592 const Floating &Arg = S.Stk.pop<Floating>();
593
594 pushInteger(S, Arg.isZero(), Call->getType());
595 return true;
596}
597
599 const InterpFrame *Frame,
600 const CallExpr *Call) {
601 const Floating &Arg = S.Stk.pop<Floating>();
602
603 pushInteger(S, Arg.isNegative(), Call->getType());
604 return true;
605}
606
608 const CallExpr *Call, unsigned ID) {
609 const Floating &RHS = S.Stk.pop<Floating>();
610 const Floating &LHS = S.Stk.pop<Floating>();
611
613 S,
614 [&] {
615 switch (ID) {
616 case Builtin::BI__builtin_isgreater:
617 return LHS > RHS;
618 case Builtin::BI__builtin_isgreaterequal:
619 return LHS >= RHS;
620 case Builtin::BI__builtin_isless:
621 return LHS < RHS;
622 case Builtin::BI__builtin_islessequal:
623 return LHS <= RHS;
624 case Builtin::BI__builtin_islessgreater: {
625 ComparisonCategoryResult Cmp = LHS.compare(RHS);
628 }
629 case Builtin::BI__builtin_isunordered:
631 default:
632 llvm_unreachable("Unexpected builtin ID: Should be a floating point "
633 "comparison function");
634 }
635 }(),
636 Call->getType());
637 return true;
638}
639
640/// First parameter to __builtin_isfpclass is the floating value, the
641/// second one is an integral value.
643 const InterpFrame *Frame,
644 const CallExpr *Call) {
645 APSInt FPClassArg = popToAPSInt(S, Call->getArg(1));
646 const Floating &F = S.Stk.pop<Floating>();
647
648 int32_t Result = static_cast<int32_t>(
649 (F.classify() & std::move(FPClassArg)).getZExtValue());
650 pushInteger(S, Result, Call->getType());
651
652 return true;
653}
654
655/// Five int values followed by one floating value.
656/// __builtin_fpclassify(int, int, int, int, int, float)
658 const InterpFrame *Frame,
659 const CallExpr *Call) {
660 const Floating &Val = S.Stk.pop<Floating>();
661
662 PrimType IntT = *S.getContext().classify(Call->getArg(0));
663 APSInt Values[5];
664 for (unsigned I = 0; I != 5; ++I)
665 Values[4 - I] = popToAPSInt(S.Stk, IntT);
666
667 unsigned Index;
668 switch (Val.getCategory()) {
669 case APFloat::fcNaN:
670 Index = 0;
671 break;
672 case APFloat::fcInfinity:
673 Index = 1;
674 break;
675 case APFloat::fcNormal:
676 Index = Val.isDenormal() ? 3 : 2;
677 break;
678 case APFloat::fcZero:
679 Index = 4;
680 break;
681 }
682
683 // The last argument is first on the stack.
684 assert(Index <= 4);
685
686 pushInteger(S, Values[Index], Call->getType());
687 return true;
688}
689
690static inline Floating abs(InterpState &S, const Floating &In) {
691 if (!In.isNegative())
692 return In;
693
694 Floating Output = S.allocFloat(In.getSemantics());
695 APFloat New = In.getAPFloat();
696 New.changeSign();
697 Output.copy(New);
698 return Output;
699}
700
701// The C standard says "fabs raises no floating-point exceptions,
702// even if x is a signaling NaN. The returned value is independent of
703// the current rounding direction mode." Therefore constant folding can
704// proceed without regard to the floating point settings.
705// Reference, WG14 N2478 F.10.4.3
707 const InterpFrame *Frame) {
708 const Floating &Val = S.Stk.pop<Floating>();
709 S.Stk.push<Floating>(abs(S, Val));
710 return true;
711}
712
714 const InterpFrame *Frame,
715 const CallExpr *Call) {
716 APSInt Val = popToAPSInt(S, Call->getArg(0));
717 if (Val ==
718 APSInt(APInt::getSignedMinValue(Val.getBitWidth()), /*IsUnsigned=*/false))
719 return false;
720 if (Val.isNegative())
721 Val.negate();
722 pushInteger(S, Val, Call->getType());
723 return true;
724}
725
727 const InterpFrame *Frame,
728 const CallExpr *Call) {
729 APSInt Val;
730 if (Call->getArg(0)->getType()->isExtVectorBoolType()) {
731 const Pointer &Arg = S.Stk.pop<Pointer>();
732 Val = convertBoolVectorToInt(Arg);
733 } else {
734 Val = popToAPSInt(S, Call->getArg(0));
735 }
736 pushInteger(S, Val.popcount(), Call->getType());
737 return true;
738}
739
741 const InterpFrame *Frame,
742 const CallExpr *Call,
743 unsigned DataBytes) {
744 uint64_t DataVal = popToUInt64(S, Call->getArg(1));
745 uint64_t CRCVal = popToUInt64(S, Call->getArg(0));
746
747 // CRC32C polynomial (iSCSI polynomial, bit-reversed)
748 static const uint32_t CRC32C_POLY = 0x82F63B78;
749
750 // Process each byte
751 uint32_t Result = static_cast<uint32_t>(CRCVal);
752 for (unsigned I = 0; I != DataBytes; ++I) {
753 uint8_t Byte = static_cast<uint8_t>((DataVal >> (I * 8)) & 0xFF);
754 Result ^= Byte;
755 for (int J = 0; J != 8; ++J) {
756 Result = (Result >> 1) ^ ((Result & 1) ? CRC32C_POLY : 0);
757 }
758 }
759
760 pushInteger(S, Result, Call->getType());
761 return true;
762}
763
765 const InterpFrame *Frame,
766 const CallExpr *Call) {
767 // This is an unevaluated call, so there are no arguments on the stack.
768 assert(Call->getNumArgs() == 1);
769 const Expr *Arg = Call->getArg(0);
770
771 GCCTypeClass ResultClass =
773 int32_t ReturnVal = static_cast<int32_t>(ResultClass);
774 pushInteger(S, ReturnVal, Call->getType());
775 return true;
776}
777
778// __builtin_expect(long, long)
779// __builtin_expect_with_probability(long, long, double)
781 const InterpFrame *Frame,
782 const CallExpr *Call) {
783 // The return value is simply the value of the first parameter.
784 // We ignore the probability.
785 unsigned NumArgs = Call->getNumArgs();
786 assert(NumArgs == 2 || NumArgs == 3);
787
788 PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
789 if (NumArgs == 3)
790 S.Stk.discard<Floating>();
791 discard(S.Stk, ArgT);
792
793 APSInt Val = popToAPSInt(S.Stk, ArgT);
794 pushInteger(S, Val, Call->getType());
795 return true;
796}
797
799 const InterpFrame *Frame,
800 const CallExpr *Call) {
801#ifndef NDEBUG
802 assert(Call->getArg(0)->isLValue());
803 PrimType PtrT = S.getContext().classify(Call->getArg(0)).value_or(PT_Ptr);
804 assert(PtrT == PT_Ptr &&
805 "Unsupported pointer type passed to __builtin_addressof()");
806#endif
807 return true;
808}
809
811 const InterpFrame *Frame,
812 const CallExpr *Call) {
813 return Call->getDirectCallee()->isConstexpr();
814}
815
817 const InterpFrame *Frame,
818 const CallExpr *Call) {
819 APSInt Arg = popToAPSInt(S, Call->getArg(0));
820
822 Arg.getZExtValue());
823 pushInteger(S, Result, Call->getType());
824 return true;
825}
826
827// Two integral values followed by a pointer (lhs, rhs, resultOut)
829 const CallExpr *Call,
830 unsigned BuiltinOp) {
831 const Pointer &ResultPtr = S.Stk.pop<Pointer>();
832 if (ResultPtr.isDummy() || !ResultPtr.isBlockPointer())
833 return false;
834
835 PrimType RHST = *S.getContext().classify(Call->getArg(1)->getType());
836 PrimType LHST = *S.getContext().classify(Call->getArg(0)->getType());
837 APSInt RHS = popToAPSInt(S.Stk, RHST);
838 APSInt LHS = popToAPSInt(S.Stk, LHST);
839 QualType ResultType = Call->getArg(2)->getType()->getPointeeType();
840 PrimType ResultT = *S.getContext().classify(ResultType);
841 bool Overflow;
842
844 if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
845 BuiltinOp == Builtin::BI__builtin_sub_overflow ||
846 BuiltinOp == Builtin::BI__builtin_mul_overflow) {
847 bool IsSigned = LHS.isSigned() || RHS.isSigned() ||
849 bool AllSigned = LHS.isSigned() && RHS.isSigned() &&
851 uint64_t LHSSize = LHS.getBitWidth();
852 uint64_t RHSSize = RHS.getBitWidth();
853 uint64_t ResultSize = S.getASTContext().getTypeSize(ResultType);
854 uint64_t MaxBits = std::max(std::max(LHSSize, RHSSize), ResultSize);
855
856 // Add an additional bit if the signedness isn't uniformly agreed to. We
857 // could do this ONLY if there is a signed and an unsigned that both have
858 // MaxBits, but the code to check that is pretty nasty. The issue will be
859 // caught in the shrink-to-result later anyway.
860 if (IsSigned && !AllSigned)
861 ++MaxBits;
862
863 LHS = APSInt(LHS.extOrTrunc(MaxBits), !IsSigned);
864 RHS = APSInt(RHS.extOrTrunc(MaxBits), !IsSigned);
865 Result = APSInt(MaxBits, !IsSigned);
866 }
867
868 // Find largest int.
869 switch (BuiltinOp) {
870 default:
871 llvm_unreachable("Invalid value for BuiltinOp");
872 case Builtin::BI__builtin_add_overflow:
873 case Builtin::BI__builtin_sadd_overflow:
874 case Builtin::BI__builtin_saddl_overflow:
875 case Builtin::BI__builtin_saddll_overflow:
876 case Builtin::BI__builtin_uadd_overflow:
877 case Builtin::BI__builtin_uaddl_overflow:
878 case Builtin::BI__builtin_uaddll_overflow:
879 Result = LHS.isSigned() ? LHS.sadd_ov(RHS, Overflow)
880 : LHS.uadd_ov(RHS, Overflow);
881 break;
882 case Builtin::BI__builtin_sub_overflow:
883 case Builtin::BI__builtin_ssub_overflow:
884 case Builtin::BI__builtin_ssubl_overflow:
885 case Builtin::BI__builtin_ssubll_overflow:
886 case Builtin::BI__builtin_usub_overflow:
887 case Builtin::BI__builtin_usubl_overflow:
888 case Builtin::BI__builtin_usubll_overflow:
889 Result = LHS.isSigned() ? LHS.ssub_ov(RHS, Overflow)
890 : LHS.usub_ov(RHS, Overflow);
891 break;
892 case Builtin::BI__builtin_mul_overflow:
893 case Builtin::BI__builtin_smul_overflow:
894 case Builtin::BI__builtin_smull_overflow:
895 case Builtin::BI__builtin_smulll_overflow:
896 case Builtin::BI__builtin_umul_overflow:
897 case Builtin::BI__builtin_umull_overflow:
898 case Builtin::BI__builtin_umulll_overflow:
899 Result = LHS.isSigned() ? LHS.smul_ov(RHS, Overflow)
900 : LHS.umul_ov(RHS, Overflow);
901 break;
902 }
903
904 // In the case where multiple sizes are allowed, truncate and see if
905 // the values are the same.
906 if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
907 BuiltinOp == Builtin::BI__builtin_sub_overflow ||
908 BuiltinOp == Builtin::BI__builtin_mul_overflow) {
909 // APSInt doesn't have a TruncOrSelf, so we use extOrTrunc instead,
910 // since it will give us the behavior of a TruncOrSelf in the case where
911 // its parameter <= its size. We previously set Result to be at least the
912 // type-size of the result, so getTypeSize(ResultType) <= Resu
913 APSInt Temp = Result.extOrTrunc(S.getASTContext().getTypeSize(ResultType));
914 Temp.setIsSigned(ResultType->isSignedIntegerOrEnumerationType());
915
916 if (!APSInt::isSameValue(Temp, Result))
917 Overflow = true;
918 Result = std::move(Temp);
919 }
920
921 // Write Result to ResultPtr and put Overflow on the stack.
922 assignInteger(S, ResultPtr, ResultT, Result);
923 if (ResultPtr.canBeInitialized())
924 ResultPtr.initialize();
925
926 assert(Call->getDirectCallee()->getReturnType()->isBooleanType());
927 S.Stk.push<Boolean>(Overflow);
928 return true;
929}
930
931/// Three integral values followed by a pointer (lhs, rhs, carry, carryOut).
933 const InterpFrame *Frame,
934 const CallExpr *Call, unsigned BuiltinOp) {
935 const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
936 PrimType LHST = *S.getContext().classify(Call->getArg(0)->getType());
937 PrimType RHST = *S.getContext().classify(Call->getArg(1)->getType());
938 APSInt CarryIn = popToAPSInt(S.Stk, LHST);
939 APSInt RHS = popToAPSInt(S.Stk, RHST);
940 APSInt LHS = popToAPSInt(S.Stk, LHST);
941
942 if (CarryOutPtr.isDummy() || !CarryOutPtr.isBlockPointer())
943 return false;
944
945 APSInt CarryOut;
946
948 // Copy the number of bits and sign.
949 Result = LHS;
950 CarryOut = LHS;
951
952 bool FirstOverflowed = false;
953 bool SecondOverflowed = false;
954 switch (BuiltinOp) {
955 default:
956 llvm_unreachable("Invalid value for BuiltinOp");
957 case Builtin::BI__builtin_addcb:
958 case Builtin::BI__builtin_addcs:
959 case Builtin::BI__builtin_addc:
960 case Builtin::BI__builtin_addcl:
961 case Builtin::BI__builtin_addcll:
962 Result =
963 LHS.uadd_ov(RHS, FirstOverflowed).uadd_ov(CarryIn, SecondOverflowed);
964 break;
965 case Builtin::BI__builtin_subcb:
966 case Builtin::BI__builtin_subcs:
967 case Builtin::BI__builtin_subc:
968 case Builtin::BI__builtin_subcl:
969 case Builtin::BI__builtin_subcll:
970 Result =
971 LHS.usub_ov(RHS, FirstOverflowed).usub_ov(CarryIn, SecondOverflowed);
972 break;
973 }
974 // It is possible for both overflows to happen but CGBuiltin uses an OR so
975 // this is consistent.
976 CarryOut = (uint64_t)(FirstOverflowed | SecondOverflowed);
977
978 QualType CarryOutType = Call->getArg(3)->getType()->getPointeeType();
979 PrimType CarryOutT = *S.getContext().classify(CarryOutType);
980 assignInteger(S, CarryOutPtr, CarryOutT, CarryOut);
981 CarryOutPtr.initialize();
982
983 assert(Call->getType() == Call->getArg(0)->getType());
984 pushInteger(S, Result, Call->getType());
985 return true;
986}
987
989 const InterpFrame *Frame, const CallExpr *Call,
990 unsigned BuiltinOp) {
991
992 std::optional<APSInt> Fallback;
993 if (BuiltinOp == Builtin::BI__builtin_clzg && Call->getNumArgs() == 2)
994 Fallback = popToAPSInt(S, Call->getArg(1));
995
996 APSInt Val;
997 if (Call->getArg(0)->getType()->isExtVectorBoolType()) {
998 const Pointer &Arg = S.Stk.pop<Pointer>();
999 Val = convertBoolVectorToInt(Arg);
1000 } else {
1001 Val = popToAPSInt(S, Call->getArg(0));
1002 }
1003
1004 // When the argument is 0, the result of GCC builtins is undefined, whereas
1005 // for Microsoft intrinsics, the result is the bit-width of the argument.
1006 bool ZeroIsUndefined = BuiltinOp != Builtin::BI__lzcnt16 &&
1007 BuiltinOp != Builtin::BI__lzcnt &&
1008 BuiltinOp != Builtin::BI__lzcnt64;
1009
1010 if (Val == 0) {
1011 if (Fallback) {
1012 pushInteger(S, *Fallback, Call->getType());
1013 return true;
1014 }
1015
1016 if (ZeroIsUndefined)
1017 return false;
1018 }
1019
1020 pushInteger(S, Val.countl_zero(), Call->getType());
1021 return true;
1022}
1023
1025 const InterpFrame *Frame, const CallExpr *Call,
1026 unsigned BuiltinID) {
1027 std::optional<APSInt> Fallback;
1028 if (BuiltinID == Builtin::BI__builtin_ctzg && Call->getNumArgs() == 2)
1029 Fallback = popToAPSInt(S, Call->getArg(1));
1030
1031 APSInt Val;
1032 if (Call->getArg(0)->getType()->isExtVectorBoolType()) {
1033 const Pointer &Arg = S.Stk.pop<Pointer>();
1034 Val = convertBoolVectorToInt(Arg);
1035 } else {
1036 Val = popToAPSInt(S, Call->getArg(0));
1037 }
1038
1039 if (Val == 0) {
1040 if (Fallback) {
1041 pushInteger(S, *Fallback, Call->getType());
1042 return true;
1043 }
1044 return false;
1045 }
1046
1047 pushInteger(S, Val.countr_zero(), Call->getType());
1048 return true;
1049}
1050
1052 const InterpFrame *Frame,
1053 const CallExpr *Call) {
1054 const APSInt &Val = popToAPSInt(S, Call->getArg(0));
1055 if (Val.getBitWidth() == 8 || Val.getBitWidth() == 1)
1056 pushInteger(S, Val, Call->getType());
1057 else
1058 pushInteger(S, Val.byteSwap(), Call->getType());
1059 return true;
1060}
1061
1062/// bool __atomic_always_lock_free(size_t, void const volatile*)
1063/// bool __atomic_is_lock_free(size_t, void const volatile*)
1065 const InterpFrame *Frame,
1066 const CallExpr *Call,
1067 unsigned BuiltinOp) {
1068 auto returnBool = [&S](bool Value) -> bool {
1069 S.Stk.push<Boolean>(Value);
1070 return true;
1071 };
1072
1073 const Pointer &Ptr = S.Stk.pop<Pointer>();
1074 uint64_t SizeVal = popToUInt64(S, Call->getArg(0));
1075
1076 // For __atomic_is_lock_free(sizeof(_Atomic(T))), if the size is a power
1077 // of two less than or equal to the maximum inline atomic width, we know it
1078 // is lock-free. If the size isn't a power of two, or greater than the
1079 // maximum alignment where we promote atomics, we know it is not lock-free
1080 // (at least not in the sense of atomic_is_lock_free). Otherwise,
1081 // the answer can only be determined at runtime; for example, 16-byte
1082 // atomics have lock-free implementations on some, but not all,
1083 // x86-64 processors.
1084
1085 // Check power-of-two.
1086 CharUnits Size = CharUnits::fromQuantity(SizeVal);
1087 if (Size.isPowerOfTwo()) {
1088 // Check against inlining width.
1089 unsigned InlineWidthBits =
1091 if (Size <= S.getASTContext().toCharUnitsFromBits(InlineWidthBits)) {
1092
1093 // OK, we will inline appropriately-aligned operations of this size,
1094 // and _Atomic(T) is appropriately-aligned.
1095 if (Size == CharUnits::One())
1096 return returnBool(true);
1097
1098 // Same for null pointers.
1099 assert(BuiltinOp != Builtin::BI__c11_atomic_is_lock_free);
1100 if (Ptr.isZero())
1101 return returnBool(true);
1102
1103 if (Ptr.isIntegralPointer()) {
1104 uint64_t IntVal = Ptr.getIntegerRepresentation();
1105 if (APSInt(APInt(64, IntVal, false), true).isAligned(Size.getAsAlign()))
1106 return returnBool(true);
1107 }
1108
1109 const Expr *PtrArg = Call->getArg(1);
1110 // Otherwise, check if the type's alignment against Size.
1111 if (const auto *ICE = dyn_cast<ImplicitCastExpr>(PtrArg)) {
1112 // Drop the potential implicit-cast to 'const volatile void*', getting
1113 // the underlying type.
1114 if (ICE->getCastKind() == CK_BitCast)
1115 PtrArg = ICE->getSubExpr();
1116 }
1117
1118 if (const auto *PtrTy = PtrArg->getType()->getAs<PointerType>()) {
1119 QualType PointeeType = PtrTy->getPointeeType();
1120 if (!PointeeType->isIncompleteType() &&
1121 S.getASTContext().getTypeAlignInChars(PointeeType) >= Size) {
1122 // OK, we will inline operations on this object.
1123 return returnBool(true);
1124 }
1125 }
1126 }
1127 }
1128
1129 if (BuiltinOp == Builtin::BI__atomic_always_lock_free)
1130 return returnBool(false);
1131
1132 return Invalid(S, OpPC);
1133}
1134
1135/// bool __c11_atomic_is_lock_free(size_t)
1137 CodePtr OpPC,
1138 const InterpFrame *Frame,
1139 const CallExpr *Call) {
1140 uint64_t SizeVal = popToUInt64(S, Call->getArg(0));
1141
1142 CharUnits Size = CharUnits::fromQuantity(SizeVal);
1143 if (Size.isPowerOfTwo()) {
1144 // Check against inlining width.
1145 unsigned InlineWidthBits =
1147 if (Size <= S.getASTContext().toCharUnitsFromBits(InlineWidthBits)) {
1148 S.Stk.push<Boolean>(true);
1149 return true;
1150 }
1151 }
1152
1153 return false; // returnBool(false);
1154}
1155
1156/// __builtin_complex(Float A, float B);
1158 const InterpFrame *Frame,
1159 const CallExpr *Call) {
1160 const Floating &Arg2 = S.Stk.pop<Floating>();
1161 const Floating &Arg1 = S.Stk.pop<Floating>();
1162 Pointer &Result = S.Stk.peek<Pointer>();
1163
1164 Result.elem<Floating>(0) = Arg1;
1165 Result.elem<Floating>(1) = Arg2;
1166 Result.initializeAllElements();
1167
1168 return true;
1169}
1170
1171/// __builtin_is_aligned()
1172/// __builtin_align_up()
1173/// __builtin_align_down()
1174/// The first parameter is either an integer or a pointer.
1175/// The second parameter is the requested alignment as an integer.
1177 const InterpFrame *Frame,
1178 const CallExpr *Call,
1179 unsigned BuiltinOp) {
1180 const APSInt &Alignment = popToAPSInt(S, Call->getArg(1));
1181
1182 if (Alignment < 0 || !Alignment.isPowerOf2()) {
1183 S.FFDiag(Call, diag::note_constexpr_invalid_alignment) << Alignment;
1184 return false;
1185 }
1186 unsigned SrcWidth = S.getASTContext().getIntWidth(Call->getArg(0)->getType());
1187 APSInt MaxValue(APInt::getOneBitSet(SrcWidth, SrcWidth - 1));
1188 if (APSInt::compareValues(Alignment, MaxValue) > 0) {
1189 S.FFDiag(Call, diag::note_constexpr_alignment_too_big)
1190 << MaxValue << Call->getArg(0)->getType() << Alignment;
1191 return false;
1192 }
1193
1194 // The first parameter is either an integer or a pointer.
1195 PrimType FirstArgT = *S.Ctx.classify(Call->getArg(0));
1196
1197 if (isIntegerType(FirstArgT)) {
1198 const APSInt &Src = popToAPSInt(S.Stk, FirstArgT);
1199 APInt AlignMinusOne = Alignment.extOrTrunc(Src.getBitWidth()) - 1;
1200 if (BuiltinOp == Builtin::BI__builtin_align_up) {
1201 APSInt AlignedVal =
1202 APSInt((Src + AlignMinusOne) & ~AlignMinusOne, Src.isUnsigned());
1203 pushInteger(S, AlignedVal, Call->getType());
1204 } else if (BuiltinOp == Builtin::BI__builtin_align_down) {
1205 APSInt AlignedVal = APSInt(Src & ~AlignMinusOne, Src.isUnsigned());
1206 pushInteger(S, AlignedVal, Call->getType());
1207 } else {
1208 assert(*S.Ctx.classify(Call->getType()) == PT_Bool);
1209 S.Stk.push<Boolean>((Src & AlignMinusOne) == 0);
1210 }
1211 return true;
1212 }
1213 assert(FirstArgT == PT_Ptr);
1214 const Pointer &Ptr = S.Stk.pop<Pointer>();
1215 if (!Ptr.isBlockPointer())
1216 return false;
1217
1218 const ValueDecl *PtrDecl = Ptr.getDeclDesc()->asValueDecl();
1219 // We need a pointer for a declaration here.
1220 if (!PtrDecl) {
1221 if (BuiltinOp == Builtin::BI__builtin_is_aligned)
1222 S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_compute)
1223 << Alignment;
1224 else
1225 S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_adjust)
1226 << Alignment;
1227 return false;
1228 }
1229
1230 // For one-past-end pointers, we can't call getIndex() since it asserts.
1231 // Use getNumElems() instead which gives the correct index for past-end.
1232 unsigned PtrOffset =
1233 Ptr.isElementPastEnd() ? Ptr.getNumElems() : Ptr.getIndex();
1234 CharUnits BaseAlignment = S.getASTContext().getDeclAlign(PtrDecl);
1235 CharUnits PtrAlign =
1236 BaseAlignment.alignmentAtOffset(CharUnits::fromQuantity(PtrOffset));
1237
1238 if (BuiltinOp == Builtin::BI__builtin_is_aligned) {
1239 if (PtrAlign.getQuantity() >= Alignment) {
1240 S.Stk.push<Boolean>(true);
1241 return true;
1242 }
1243 // If the alignment is not known to be sufficient, some cases could still
1244 // be aligned at run time. However, if the requested alignment is less or
1245 // equal to the base alignment and the offset is not aligned, we know that
1246 // the run-time value can never be aligned.
1247 if (BaseAlignment.getQuantity() >= Alignment &&
1248 PtrAlign.getQuantity() < Alignment) {
1249 S.Stk.push<Boolean>(false);
1250 return true;
1251 }
1252
1253 S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_compute)
1254 << Alignment;
1255 return false;
1256 }
1257
1258 assert(BuiltinOp == Builtin::BI__builtin_align_down ||
1259 BuiltinOp == Builtin::BI__builtin_align_up);
1260
1261 // For align_up/align_down, we can return the same value if the alignment
1262 // is known to be greater or equal to the requested value.
1263 if (PtrAlign.getQuantity() >= Alignment) {
1264 S.Stk.push<Pointer>(Ptr);
1265 return true;
1266 }
1267
1268 // The alignment could be greater than the minimum at run-time, so we cannot
1269 // infer much about the resulting pointer value. One case is possible:
1270 // For `_Alignas(32) char buf[N]; __builtin_align_down(&buf[idx], 32)` we
1271 // can infer the correct index if the requested alignment is smaller than
1272 // the base alignment so we can perform the computation on the offset.
1273 if (BaseAlignment.getQuantity() >= Alignment) {
1274 assert(Alignment.getBitWidth() <= 64 &&
1275 "Cannot handle > 64-bit address-space");
1276 uint64_t Alignment64 = Alignment.getZExtValue();
1277 CharUnits NewOffset =
1278 CharUnits::fromQuantity(BuiltinOp == Builtin::BI__builtin_align_down
1279 ? llvm::alignDown(PtrOffset, Alignment64)
1280 : llvm::alignTo(PtrOffset, Alignment64));
1281
1282 S.Stk.push<Pointer>(Ptr.atIndex(NewOffset.getQuantity()));
1283 return true;
1284 }
1285
1286 // Otherwise, we cannot constant-evaluate the result.
1287 S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_adjust) << Alignment;
1288 return false;
1289}
1290
1291/// __builtin_assume_aligned(Ptr, Alignment[, ExtraOffset])
1293 const InterpFrame *Frame,
1294 const CallExpr *Call) {
1295 assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
1296
1297 std::optional<APSInt> ExtraOffset;
1298 if (Call->getNumArgs() == 3)
1299 ExtraOffset = popToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(2)));
1300
1301 APSInt Alignment = popToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(1)));
1302 const Pointer &Ptr = S.Stk.pop<Pointer>();
1303
1304 CharUnits Align = CharUnits::fromQuantity(Alignment.getZExtValue());
1305
1306 // If there is a base object, then it must have the correct alignment.
1307 if (Ptr.isBlockPointer()) {
1308 CharUnits BaseAlignment;
1309 if (const auto *VD = Ptr.getDeclDesc()->asValueDecl())
1310 BaseAlignment = S.getASTContext().getDeclAlign(VD);
1311 else if (const auto *E = Ptr.getDeclDesc()->asExpr())
1312 BaseAlignment = GetAlignOfExpr(S.getASTContext(), E, UETT_AlignOf);
1313
1314 if (BaseAlignment < Align) {
1315 S.CCEDiag(Call->getArg(0),
1316 diag::note_constexpr_baa_insufficient_alignment)
1317 << 0 << BaseAlignment.getQuantity() << Align.getQuantity();
1318 return false;
1319 }
1320 }
1321
1322 APValue AV = Ptr.toAPValue(S.getASTContext());
1323 CharUnits AVOffset = AV.getLValueOffset();
1324 if (ExtraOffset)
1325 AVOffset -= CharUnits::fromQuantity(ExtraOffset->getZExtValue());
1326 if (AVOffset.alignTo(Align) != AVOffset) {
1327 if (Ptr.isBlockPointer())
1328 S.CCEDiag(Call->getArg(0),
1329 diag::note_constexpr_baa_insufficient_alignment)
1330 << 1 << AVOffset.getQuantity() << Align.getQuantity();
1331 else
1332 S.CCEDiag(Call->getArg(0),
1333 diag::note_constexpr_baa_value_insufficient_alignment)
1334 << AVOffset.getQuantity() << Align.getQuantity();
1335 return false;
1336 }
1337
1338 S.Stk.push<Pointer>(Ptr);
1339 return true;
1340}
1341
1342/// (CarryIn, LHS, RHS, Result)
1344 CodePtr OpPC,
1345 const InterpFrame *Frame,
1346 const CallExpr *Call,
1347 unsigned BuiltinOp) {
1348 if (Call->getNumArgs() != 4 || !Call->getArg(0)->getType()->isIntegerType() ||
1349 !Call->getArg(1)->getType()->isIntegerType() ||
1350 !Call->getArg(2)->getType()->isIntegerType())
1351 return false;
1352
1353 const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
1354
1355 APSInt RHS = popToAPSInt(S, Call->getArg(2));
1356 APSInt LHS = popToAPSInt(S, Call->getArg(1));
1357 APSInt CarryIn = popToAPSInt(S, Call->getArg(0));
1358
1359 bool IsAdd = BuiltinOp == clang::X86::BI__builtin_ia32_addcarryx_u32 ||
1360 BuiltinOp == clang::X86::BI__builtin_ia32_addcarryx_u64;
1361
1362 unsigned BitWidth = LHS.getBitWidth();
1363 unsigned CarryInBit = CarryIn.ugt(0) ? 1 : 0;
1364 APInt ExResult =
1365 IsAdd ? (LHS.zext(BitWidth + 1) + (RHS.zext(BitWidth + 1) + CarryInBit))
1366 : (LHS.zext(BitWidth + 1) - (RHS.zext(BitWidth + 1) + CarryInBit));
1367
1368 APInt Result = ExResult.extractBits(BitWidth, 0);
1369 APSInt CarryOut =
1370 APSInt(ExResult.extractBits(1, BitWidth), /*IsUnsigned=*/true);
1371
1372 QualType CarryOutType = Call->getArg(3)->getType()->getPointeeType();
1373 PrimType CarryOutT = *S.getContext().classify(CarryOutType);
1374 assignInteger(S, CarryOutPtr, CarryOutT, APSInt(std::move(Result), true));
1375
1376 pushInteger(S, CarryOut, Call->getType());
1377
1378 return true;
1379}
1380
1382 CodePtr OpPC,
1383 const InterpFrame *Frame,
1384 const CallExpr *Call) {
1387 pushInteger(S, Layout.size().getQuantity(), Call->getType());
1388 return true;
1389}
1390
1391static bool
1393 const InterpFrame *Frame,
1394 const CallExpr *Call) {
1395 const auto &Ptr = S.Stk.pop<Pointer>();
1396 assert(Ptr.getFieldDesc()->isPrimitiveArray());
1397
1398 // This should be created for a StringLiteral, so always holds at least
1399 // one array element.
1400 assert(Ptr.getFieldDesc()->getNumElems() >= 1);
1401 uint64_t Result = getPointerAuthStableSipHash(
1402 cast<StringLiteral>(Ptr.getFieldDesc()->asExpr())->getString());
1403 pushInteger(S, Result, Call->getType());
1404 return true;
1405}
1406
1408 const InterpFrame *Frame,
1409 const CallExpr *Call) {
1410 const ASTContext &ASTCtx = S.getASTContext();
1411 uint64_t BitWidth = ASTCtx.getTypeSize(ASTCtx.getSizeType());
1412 auto Mode =
1413 ASTCtx.getLangOpts().AllocTokenMode.value_or(llvm::DefaultAllocTokenMode);
1414 auto MaxTokensOpt = ASTCtx.getLangOpts().AllocTokenMax;
1415 uint64_t MaxTokens =
1416 MaxTokensOpt.value_or(0) ? *MaxTokensOpt : (~0ULL >> (64 - BitWidth));
1417
1418 // We do not read any of the arguments; discard them.
1419 for (int I = Call->getNumArgs() - 1; I >= 0; --I)
1420 discard(S.Stk, S.getContext().classify(Call->getArg(I)).value_or(PT_Ptr));
1421
1422 // Note: Type inference from a surrounding cast is not supported in
1423 // constexpr evaluation.
1424 QualType AllocType = infer_alloc::inferPossibleType(Call, ASTCtx, nullptr);
1425 if (AllocType.isNull()) {
1426 S.CCEDiag(Call,
1427 diag::note_constexpr_infer_alloc_token_type_inference_failed);
1428 return false;
1429 }
1430
1431 auto ATMD = infer_alloc::getAllocTokenMetadata(AllocType, ASTCtx);
1432 if (!ATMD) {
1433 S.CCEDiag(Call, diag::note_constexpr_infer_alloc_token_no_metadata);
1434 return false;
1435 }
1436
1437 auto MaybeToken = llvm::getAllocToken(Mode, *ATMD, MaxTokens);
1438 if (!MaybeToken) {
1439 S.CCEDiag(Call, diag::note_constexpr_infer_alloc_token_stateful_mode);
1440 return false;
1441 }
1442
1443 pushInteger(S, llvm::APInt(BitWidth, *MaybeToken), ASTCtx.getSizeType());
1444 return true;
1445}
1446
1448 const InterpFrame *Frame,
1449 const CallExpr *Call) {
1450 // A call to __operator_new is only valid within std::allocate<>::allocate.
1451 // Walk up the call stack to find the appropriate caller and get the
1452 // element type from it.
1453 auto [NewCall, ElemType] = S.getStdAllocatorCaller("allocate");
1454
1455 if (ElemType.isNull()) {
1456 S.FFDiag(Call, S.getLangOpts().CPlusPlus20
1457 ? diag::note_constexpr_new_untyped
1458 : diag::note_constexpr_new);
1459 return false;
1460 }
1461 assert(NewCall);
1462
1463 if (ElemType->isIncompleteType() || ElemType->isFunctionType()) {
1464 S.FFDiag(Call, diag::note_constexpr_new_not_complete_object_type)
1465 << (ElemType->isIncompleteType() ? 0 : 1) << ElemType;
1466 return false;
1467 }
1468
1469 // We only care about the first parameter (the size), so discard all the
1470 // others.
1471 {
1472 unsigned NumArgs = Call->getNumArgs();
1473 assert(NumArgs >= 1);
1474
1475 // The std::nothrow_t arg never gets put on the stack.
1476 if (Call->getArg(NumArgs - 1)->getType()->isNothrowT())
1477 --NumArgs;
1478 auto Args = ArrayRef(Call->getArgs(), Call->getNumArgs());
1479 // First arg is needed.
1480 Args = Args.drop_front();
1481
1482 // Discard the rest.
1483 for (const Expr *Arg : Args)
1484 discard(S.Stk, *S.getContext().classify(Arg));
1485 }
1486
1487 APSInt Bytes = popToAPSInt(S, Call->getArg(0));
1488 CharUnits ElemSize = S.getASTContext().getTypeSizeInChars(ElemType);
1489 assert(!ElemSize.isZero());
1490 // Divide the number of bytes by sizeof(ElemType), so we get the number of
1491 // elements we should allocate.
1492 APInt NumElems, Remainder;
1493 APInt ElemSizeAP(Bytes.getBitWidth(), ElemSize.getQuantity());
1494 APInt::udivrem(Bytes, ElemSizeAP, NumElems, Remainder);
1495 if (Remainder != 0) {
1496 // This likely indicates a bug in the implementation of 'std::allocator'.
1497 S.FFDiag(Call, diag::note_constexpr_operator_new_bad_size)
1498 << Bytes << APSInt(ElemSizeAP, true) << ElemType;
1499 return false;
1500 }
1501
1502 // NB: The same check we're using in CheckArraySize()
1503 if (NumElems.getActiveBits() >
1505 NumElems.ugt(Descriptor::MaxArrayElemBytes / ElemSize.getQuantity())) {
1506 // FIXME: NoThrow check?
1507 const SourceInfo &Loc = S.Current->getSource(OpPC);
1508 S.FFDiag(Loc, diag::note_constexpr_new_too_large)
1509 << NumElems.getZExtValue();
1510 return false;
1511 }
1512
1513 if (!CheckArraySize(S, OpPC, NumElems.getZExtValue()))
1514 return false;
1515
1516 bool IsArray = NumElems.ugt(1);
1517 OptPrimType ElemT = S.getContext().classify(ElemType);
1518 DynamicAllocator &Allocator = S.getAllocator();
1519 if (ElemT) {
1520 Block *B =
1521 Allocator.allocate(NewCall, *ElemT, NumElems.getZExtValue(),
1523 assert(B);
1524 S.Stk.push<Pointer>(Pointer(B).atIndex(0));
1525 return true;
1526 }
1527
1528 assert(!ElemT);
1529
1530 // Composite arrays
1531 if (IsArray) {
1532 const Descriptor *Desc =
1533 S.P.createDescriptor(NewCall, ElemType.getTypePtr(), std::nullopt);
1534 Block *B =
1535 Allocator.allocate(Desc, NumElems.getZExtValue(), S.Ctx.getEvalID(),
1537 assert(B);
1538 S.Stk.push<Pointer>(Pointer(B).atIndex(0).narrow());
1539 return true;
1540 }
1541
1542 // Records. Still allocate them as single-element arrays.
1544 ElemType, NumElems, nullptr, ArraySizeModifier::Normal, 0);
1545
1546 const Descriptor *Desc = S.P.createDescriptor(NewCall, AllocType.getTypePtr(),
1548 Block *B = Allocator.allocate(Desc, S.getContext().getEvalID(),
1550 assert(B);
1551 S.Stk.push<Pointer>(Pointer(B).atIndex(0).narrow());
1552 return true;
1553}
1554
1556 const InterpFrame *Frame,
1557 const CallExpr *Call) {
1558 const Expr *Source = nullptr;
1559 const Block *BlockToDelete = nullptr;
1560
1562 S.Stk.discard<Pointer>();
1563 return false;
1564 }
1565
1566 // This is permitted only within a call to std::allocator<T>::deallocate.
1567 if (!S.getStdAllocatorCaller("deallocate")) {
1568 S.FFDiag(Call);
1569 S.Stk.discard<Pointer>();
1570 return true;
1571 }
1572
1573 {
1574 const Pointer &Ptr = S.Stk.pop<Pointer>();
1575
1576 if (Ptr.isZero()) {
1577 S.CCEDiag(Call, diag::note_constexpr_deallocate_null);
1578 return true;
1579 }
1580
1581 Source = Ptr.getDeclDesc()->asExpr();
1582 BlockToDelete = Ptr.block();
1583
1584 if (!BlockToDelete->isDynamic()) {
1585 S.FFDiag(Call, diag::note_constexpr_delete_not_heap_alloc)
1587 if (const auto *D = Ptr.getFieldDesc()->asDecl())
1588 S.Note(D->getLocation(), diag::note_declared_at);
1589 }
1590 }
1591 assert(BlockToDelete);
1592
1593 DynamicAllocator &Allocator = S.getAllocator();
1594 const Descriptor *BlockDesc = BlockToDelete->getDescriptor();
1595 std::optional<DynamicAllocator::Form> AllocForm =
1596 Allocator.getAllocationForm(Source);
1597
1598 if (!Allocator.deallocate(Source, BlockToDelete, S)) {
1599 // Nothing has been deallocated, this must be a double-delete.
1600 const SourceInfo &Loc = S.Current->getSource(OpPC);
1601 S.FFDiag(Loc, diag::note_constexpr_double_delete);
1602 return false;
1603 }
1604 assert(AllocForm);
1605
1606 return CheckNewDeleteForms(
1607 S, OpPC, *AllocForm, DynamicAllocator::Form::Operator, BlockDesc, Source);
1608}
1609
1611 const InterpFrame *Frame,
1612 const CallExpr *Call) {
1613 const Floating &Arg0 = S.Stk.pop<Floating>();
1614 S.Stk.push<Floating>(Arg0);
1615 return true;
1616}
1617
1619 const CallExpr *Call, unsigned ID) {
1620 const Pointer &Arg = S.Stk.pop<Pointer>();
1621 assert(Arg.getFieldDesc()->isPrimitiveArray());
1622
1623 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1624 assert(Call->getType() == ElemType);
1625 PrimType ElemT = *S.getContext().classify(ElemType);
1626 unsigned NumElems = Arg.getNumElems();
1627
1629 T Result = Arg.elem<T>(0);
1630 unsigned BitWidth = Result.bitWidth();
1631 for (unsigned I = 1; I != NumElems; ++I) {
1632 T Elem = Arg.elem<T>(I);
1633 T PrevResult = Result;
1634
1635 if (ID == Builtin::BI__builtin_reduce_add) {
1636 if (T::add(Result, Elem, BitWidth, &Result)) {
1637 unsigned OverflowBits = BitWidth + 1;
1638 (void)handleOverflow(S, OpPC,
1639 (PrevResult.toAPSInt(OverflowBits) +
1640 Elem.toAPSInt(OverflowBits)));
1641 return false;
1642 }
1643 } else if (ID == Builtin::BI__builtin_reduce_mul) {
1644 if (T::mul(Result, Elem, BitWidth, &Result)) {
1645 unsigned OverflowBits = BitWidth * 2;
1646 (void)handleOverflow(S, OpPC,
1647 (PrevResult.toAPSInt(OverflowBits) *
1648 Elem.toAPSInt(OverflowBits)));
1649 return false;
1650 }
1651
1652 } else if (ID == Builtin::BI__builtin_reduce_and) {
1653 (void)T::bitAnd(Result, Elem, BitWidth, &Result);
1654 } else if (ID == Builtin::BI__builtin_reduce_or) {
1655 (void)T::bitOr(Result, Elem, BitWidth, &Result);
1656 } else if (ID == Builtin::BI__builtin_reduce_xor) {
1657 (void)T::bitXor(Result, Elem, BitWidth, &Result);
1658 } else if (ID == Builtin::BI__builtin_reduce_min) {
1659 if (Elem < Result)
1660 Result = Elem;
1661 } else if (ID == Builtin::BI__builtin_reduce_max) {
1662 if (Elem > Result)
1663 Result = Elem;
1664 } else {
1665 llvm_unreachable("Unhandled vector reduce builtin");
1666 }
1667 }
1668 pushInteger(S, Result.toAPSInt(), Call->getType());
1669 });
1670
1671 return true;
1672}
1673
1675 const InterpFrame *Frame,
1676 const CallExpr *Call,
1677 unsigned BuiltinID) {
1678 assert(Call->getNumArgs() == 1);
1679 QualType Ty = Call->getArg(0)->getType();
1680 if (Ty->isIntegerType()) {
1681 APSInt Val = popToAPSInt(S, Call->getArg(0));
1682 pushInteger(S, Val.abs(), Call->getType());
1683 return true;
1684 }
1685
1686 if (Ty->isFloatingType()) {
1687 Floating Val = S.Stk.pop<Floating>();
1688 Floating Result = abs(S, Val);
1689 S.Stk.push<Floating>(Result);
1690 return true;
1691 }
1692
1693 // Otherwise, the argument must be a vector.
1694 assert(Call->getArg(0)->getType()->isVectorType());
1695 const Pointer &Arg = S.Stk.pop<Pointer>();
1696 assert(Arg.getFieldDesc()->isPrimitiveArray());
1697 const Pointer &Dst = S.Stk.peek<Pointer>();
1698 assert(Dst.getFieldDesc()->isPrimitiveArray());
1699 assert(Arg.getFieldDesc()->getNumElems() ==
1700 Dst.getFieldDesc()->getNumElems());
1701
1702 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1703 PrimType ElemT = *S.getContext().classify(ElemType);
1704 unsigned NumElems = Arg.getNumElems();
1705 // we can either have a vector of integer or a vector of floating point
1706 for (unsigned I = 0; I != NumElems; ++I) {
1707 if (ElemType->isIntegerType()) {
1709 Dst.elem<T>(I) = T::from(static_cast<T>(
1710 APSInt(Arg.elem<T>(I).toAPSInt().abs(),
1712 });
1713 } else {
1714 Floating Val = Arg.elem<Floating>(I);
1715 Dst.elem<Floating>(I) = abs(S, Val);
1716 }
1717 }
1719
1720 return true;
1721}
1722
1723/// Can be called with an integer or vector as the first and only parameter.
1725 CodePtr OpPC,
1726 const InterpFrame *Frame,
1727 const CallExpr *Call,
1728 unsigned BuiltinID) {
1729 bool HasZeroArg = Call->getNumArgs() == 2;
1730 bool IsCTTZ = BuiltinID == Builtin::BI__builtin_elementwise_ctzg;
1731 assert(Call->getNumArgs() == 1 || HasZeroArg);
1732 if (Call->getArg(0)->getType()->isIntegerType()) {
1733 PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
1734 APSInt Val = popToAPSInt(S.Stk, ArgT);
1735 std::optional<APSInt> ZeroVal;
1736 if (HasZeroArg) {
1737 ZeroVal = Val;
1738 Val = popToAPSInt(S.Stk, ArgT);
1739 }
1740
1741 if (Val.isZero()) {
1742 if (ZeroVal) {
1743 pushInteger(S, *ZeroVal, Call->getType());
1744 return true;
1745 }
1746 // If we haven't been provided the second argument, the result is
1747 // undefined
1748 S.FFDiag(S.Current->getSource(OpPC),
1749 diag::note_constexpr_countzeroes_zero)
1750 << /*IsTrailing=*/IsCTTZ;
1751 return false;
1752 }
1753
1754 if (BuiltinID == Builtin::BI__builtin_elementwise_clzg) {
1755 pushInteger(S, Val.countLeadingZeros(), Call->getType());
1756 } else {
1757 pushInteger(S, Val.countTrailingZeros(), Call->getType());
1758 }
1759 return true;
1760 }
1761 // Otherwise, the argument must be a vector.
1762 const ASTContext &ASTCtx = S.getASTContext();
1763 Pointer ZeroArg;
1764 if (HasZeroArg) {
1765 assert(Call->getArg(1)->getType()->isVectorType() &&
1766 ASTCtx.hasSameUnqualifiedType(Call->getArg(0)->getType(),
1767 Call->getArg(1)->getType()));
1768 (void)ASTCtx;
1769 ZeroArg = S.Stk.pop<Pointer>();
1770 assert(ZeroArg.getFieldDesc()->isPrimitiveArray());
1771 }
1772 assert(Call->getArg(0)->getType()->isVectorType());
1773 const Pointer &Arg = S.Stk.pop<Pointer>();
1774 assert(Arg.getFieldDesc()->isPrimitiveArray());
1775 const Pointer &Dst = S.Stk.peek<Pointer>();
1776 assert(Dst.getFieldDesc()->isPrimitiveArray());
1777 assert(Arg.getFieldDesc()->getNumElems() ==
1778 Dst.getFieldDesc()->getNumElems());
1779
1780 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1781 PrimType ElemT = *S.getContext().classify(ElemType);
1782 unsigned NumElems = Arg.getNumElems();
1783
1784 // FIXME: Reading from uninitialized vector elements?
1785 for (unsigned I = 0; I != NumElems; ++I) {
1787 APInt EltVal = Arg.atIndex(I).deref<T>().toAPSInt();
1788 if (EltVal.isZero()) {
1789 if (HasZeroArg) {
1790 Dst.atIndex(I).deref<T>() = ZeroArg.atIndex(I).deref<T>();
1791 } else {
1792 // If we haven't been provided the second argument, the result is
1793 // undefined
1794 S.FFDiag(S.Current->getSource(OpPC),
1795 diag::note_constexpr_countzeroes_zero)
1796 << /*IsTrailing=*/IsCTTZ;
1797 return false;
1798 }
1799 } else if (IsCTTZ) {
1800 Dst.atIndex(I).deref<T>() = T::from(EltVal.countTrailingZeros());
1801 } else {
1802 Dst.atIndex(I).deref<T>() = T::from(EltVal.countLeadingZeros());
1803 }
1804 Dst.atIndex(I).initialize();
1805 });
1806 }
1807
1808 return true;
1809}
1810
1812 const InterpFrame *Frame,
1813 const CallExpr *Call, unsigned ID) {
1814 assert(Call->getNumArgs() == 3);
1815 const ASTContext &ASTCtx = S.getASTContext();
1816 uint64_t Size = popToUInt64(S, Call->getArg(2));
1817 Pointer SrcPtr = S.Stk.pop<Pointer>().expand();
1818 Pointer DestPtr = S.Stk.pop<Pointer>().expand();
1819
1820 if (ID == Builtin::BImemcpy || ID == Builtin::BImemmove)
1821 diagnoseNonConstexprBuiltin(S, OpPC, ID);
1822
1823 bool Move =
1824 (ID == Builtin::BI__builtin_memmove || ID == Builtin::BImemmove ||
1825 ID == Builtin::BI__builtin_wmemmove || ID == Builtin::BIwmemmove);
1826 bool WChar = ID == Builtin::BIwmemcpy || ID == Builtin::BIwmemmove ||
1827 ID == Builtin::BI__builtin_wmemcpy ||
1828 ID == Builtin::BI__builtin_wmemmove;
1829
1830 // If the size is zero, we treat this as always being a valid no-op.
1831 if (Size == 0) {
1832 S.Stk.push<Pointer>(DestPtr);
1833 return true;
1834 }
1835
1836 if (SrcPtr.isZero() || DestPtr.isZero()) {
1837 Pointer DiagPtr = (SrcPtr.isZero() ? SrcPtr : DestPtr);
1838 S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_null)
1839 << /*IsMove=*/Move << /*IsWchar=*/WChar << !SrcPtr.isZero()
1840 << DiagPtr.toDiagnosticString(ASTCtx);
1841 return false;
1842 }
1843
1844 // Diagnose integral src/dest pointers specially.
1845 if (SrcPtr.isIntegralPointer() || DestPtr.isIntegralPointer()) {
1846 std::string DiagVal = "(void *)";
1847 DiagVal += SrcPtr.isIntegralPointer()
1848 ? std::to_string(SrcPtr.getIntegerRepresentation())
1849 : std::to_string(DestPtr.getIntegerRepresentation());
1850 S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_null)
1851 << Move << WChar << DestPtr.isIntegralPointer() << DiagVal;
1852 return false;
1853 }
1854
1855 if (!isReadable(DestPtr) || !isReadable(SrcPtr))
1856 return false;
1857
1858 if (DestPtr.getType()->isIncompleteType()) {
1859 S.FFDiag(S.Current->getSource(OpPC),
1860 diag::note_constexpr_memcpy_incomplete_type)
1861 << Move << DestPtr.getType();
1862 return false;
1863 }
1864 if (SrcPtr.getType()->isIncompleteType()) {
1865 S.FFDiag(S.Current->getSource(OpPC),
1866 diag::note_constexpr_memcpy_incomplete_type)
1867 << Move << SrcPtr.getType();
1868 return false;
1869 }
1870
1871 QualType DestElemType = getElemType(DestPtr);
1872 if (DestElemType->isIncompleteType()) {
1873 S.FFDiag(S.Current->getSource(OpPC),
1874 diag::note_constexpr_memcpy_incomplete_type)
1875 << Move << DestElemType;
1876 return false;
1877 }
1878
1879 size_t RemainingDestElems;
1880 if (DestPtr.getFieldDesc()->isArray()) {
1881 RemainingDestElems = DestPtr.isUnknownSizeArray()
1882 ? 0
1883 : (DestPtr.getNumElems() - DestPtr.getIndex());
1884 } else {
1885 RemainingDestElems = 1;
1886 }
1887 unsigned DestElemSize = ASTCtx.getTypeSizeInChars(DestElemType).getQuantity();
1888
1889 if (WChar) {
1890 uint64_t WCharSize =
1891 ASTCtx.getTypeSizeInChars(ASTCtx.getWCharType()).getQuantity();
1892 Size *= WCharSize;
1893 }
1894
1895 if (Size % DestElemSize != 0) {
1896 S.FFDiag(S.Current->getSource(OpPC),
1897 diag::note_constexpr_memcpy_unsupported)
1898 << Move << WChar << 0 << DestElemType << Size << DestElemSize;
1899 return false;
1900 }
1901
1902 QualType SrcElemType = getElemType(SrcPtr);
1903 size_t RemainingSrcElems;
1904 if (SrcPtr.getFieldDesc()->isArray()) {
1905 RemainingSrcElems = SrcPtr.isUnknownSizeArray()
1906 ? 0
1907 : (SrcPtr.getNumElems() - SrcPtr.getIndex());
1908 } else {
1909 RemainingSrcElems = 1;
1910 }
1911 unsigned SrcElemSize = ASTCtx.getTypeSizeInChars(SrcElemType).getQuantity();
1912
1913 if (!ASTCtx.hasSameUnqualifiedType(DestElemType, SrcElemType)) {
1914 S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_type_pun)
1915 << Move << SrcElemType << DestElemType;
1916 return false;
1917 }
1918
1919 if (!DestElemType.isTriviallyCopyableType(ASTCtx)) {
1920 S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_nontrivial)
1921 << Move << DestElemType;
1922 return false;
1923 }
1924
1925 // Check if we have enough elements to read from and write to.
1926 size_t RemainingDestBytes = RemainingDestElems * DestElemSize;
1927 size_t RemainingSrcBytes = RemainingSrcElems * SrcElemSize;
1928 if (Size > RemainingDestBytes || Size > RemainingSrcBytes) {
1929 APInt N = APInt(64, Size / DestElemSize);
1930 S.FFDiag(S.Current->getSource(OpPC),
1931 diag::note_constexpr_memcpy_unsupported)
1932 << Move << WChar << (Size > RemainingSrcBytes ? 1 : 2) << DestElemType
1933 << toString(N, 10, /*Signed=*/false);
1934 return false;
1935 }
1936
1937 // Check for overlapping memory regions.
1938 if (!Move && Pointer::pointToSameBlock(SrcPtr, DestPtr)) {
1939 // Remove base casts.
1940 Pointer SrcP = SrcPtr.stripBaseCasts();
1941 Pointer DestP = DestPtr.stripBaseCasts();
1942
1943 unsigned SrcIndex = SrcP.expand().getIndex() * SrcElemSize;
1944 unsigned DstIndex = DestP.expand().getIndex() * DestElemSize;
1945
1946 if ((SrcIndex <= DstIndex && (SrcIndex + Size) > DstIndex) ||
1947 (DstIndex <= SrcIndex && (DstIndex + Size) > SrcIndex)) {
1948 S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_overlap)
1949 << /*IsWChar=*/false;
1950 return false;
1951 }
1952 }
1953
1954 assert(Size % DestElemSize == 0);
1955 if (!DoMemcpy(S, OpPC, SrcPtr, DestPtr, Bytes(Size).toBits()))
1956 return false;
1957
1958 S.Stk.push<Pointer>(DestPtr);
1959 return true;
1960}
1961
1962/// Determine if T is a character type for which we guarantee that
1963/// sizeof(T) == 1.
1965 return T->isCharType() || T->isChar8Type();
1966}
1967
1969 const InterpFrame *Frame,
1970 const CallExpr *Call, unsigned ID) {
1971 assert(Call->getNumArgs() == 3);
1972 uint64_t Size = popToUInt64(S, Call->getArg(2));
1973 const Pointer &PtrB = S.Stk.pop<Pointer>();
1974 const Pointer &PtrA = S.Stk.pop<Pointer>();
1975
1976 if (ID == Builtin::BImemcmp || ID == Builtin::BIbcmp ||
1977 ID == Builtin::BIwmemcmp)
1978 diagnoseNonConstexprBuiltin(S, OpPC, ID);
1979
1980 if (Size == 0) {
1981 pushInteger(S, 0, Call->getType());
1982 return true;
1983 }
1984
1985 if (!PtrA.isBlockPointer() || !PtrB.isBlockPointer())
1986 return false;
1987
1988 bool IsWide =
1989 (ID == Builtin::BIwmemcmp || ID == Builtin::BI__builtin_wmemcmp);
1990
1991 const ASTContext &ASTCtx = S.getASTContext();
1992 QualType ElemTypeA = getElemType(PtrA);
1993 QualType ElemTypeB = getElemType(PtrB);
1994 // FIXME: This is an arbitrary limitation the current constant interpreter
1995 // had. We could remove this.
1996 if (!IsWide && (!isOneByteCharacterType(ElemTypeA) ||
1997 !isOneByteCharacterType(ElemTypeB))) {
1998 S.FFDiag(S.Current->getSource(OpPC),
1999 diag::note_constexpr_memcmp_unsupported)
2000 << ASTCtx.BuiltinInfo.getQuotedName(ID) << PtrA.getType()
2001 << PtrB.getType();
2002 return false;
2003 }
2004
2005 if (!CheckLoad(S, OpPC, PtrA, AK_Read) || !CheckLoad(S, OpPC, PtrB, AK_Read))
2006 return false;
2007
2008 // Now, read both pointers to a buffer and compare those.
2009 BitcastBuffer BufferA(
2010 Bits(ASTCtx.getTypeSize(ElemTypeA) * PtrA.getNumElems()));
2011 readPointerToBuffer(S.getContext(), PtrA, BufferA, false);
2012
2013 // FIXME: The swapping here is UNDOING something we do when reading the
2014 // data into the buffer.
2015 if (ASTCtx.getTargetInfo().isBigEndian())
2016 swapBytes(BufferA.Data.get(), BufferA.byteSize().getQuantity());
2017
2018 BitcastBuffer BufferB(
2019 Bits(ASTCtx.getTypeSize(ElemTypeB) * PtrB.getNumElems()));
2020 readPointerToBuffer(S.getContext(), PtrB, BufferB, false);
2021 // FIXME: The swapping here is UNDOING something we do when reading the
2022 // data into the buffer.
2023 if (ASTCtx.getTargetInfo().isBigEndian())
2024 swapBytes(BufferB.Data.get(), BufferB.byteSize().getQuantity());
2025
2026 size_t MinBufferSize = std::min(BufferA.byteSize().getQuantity(),
2027 BufferB.byteSize().getQuantity());
2028
2029 unsigned ElemSize = 1;
2030 if (IsWide)
2031 ElemSize = ASTCtx.getTypeSizeInChars(ASTCtx.getWCharType()).getQuantity();
2032 // The Size given for the wide variants is in wide-char units. Convert it
2033 // to bytes.
2034 size_t ByteSize = Size * ElemSize;
2035 size_t CmpSize = std::min(MinBufferSize, ByteSize);
2036
2037 for (size_t I = 0; I != CmpSize; I += ElemSize) {
2038 if (IsWide) {
2040 *S.getContext().classify(ASTCtx.getWCharType()), {
2041 T A = T::bitcastFromMemory(BufferA.atByte(I), T::bitWidth());
2042 T B = T::bitcastFromMemory(BufferB.atByte(I), T::bitWidth());
2043 if (A < B) {
2044 pushInteger(S, -1, Call->getType());
2045 return true;
2046 }
2047 if (A > B) {
2048 pushInteger(S, 1, Call->getType());
2049 return true;
2050 }
2051 });
2052 } else {
2053 auto A = BufferA.deref<std::byte>(Bytes(I));
2054 auto B = BufferB.deref<std::byte>(Bytes(I));
2055
2056 if (A < B) {
2057 pushInteger(S, -1, Call->getType());
2058 return true;
2059 }
2060 if (A > B) {
2061 pushInteger(S, 1, Call->getType());
2062 return true;
2063 }
2064 }
2065 }
2066
2067 // We compared CmpSize bytes above. If the limiting factor was the Size
2068 // passed, we're done and the result is equality (0).
2069 if (ByteSize <= CmpSize) {
2070 pushInteger(S, 0, Call->getType());
2071 return true;
2072 }
2073
2074 // However, if we read all the available bytes but were instructed to read
2075 // even more, diagnose this as a "read of dereferenced one-past-the-end
2076 // pointer". This is what would happen if we called CheckLoad() on every array
2077 // element.
2078 S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_access_past_end)
2079 << AK_Read << S.Current->getRange(OpPC);
2080 return false;
2081}
2082
2083// __builtin_memchr(ptr, int, int)
2084// __builtin_strchr(ptr, int)
2086 const CallExpr *Call, unsigned ID) {
2087 if (ID == Builtin::BImemchr || ID == Builtin::BIwcschr ||
2088 ID == Builtin::BIstrchr || ID == Builtin::BIwmemchr)
2089 diagnoseNonConstexprBuiltin(S, OpPC, ID);
2090
2091 std::optional<APSInt> MaxLength;
2092 if (Call->getNumArgs() == 3)
2093 MaxLength = popToAPSInt(S, Call->getArg(2));
2094
2095 APSInt Desired = popToAPSInt(S, Call->getArg(1));
2096 const Pointer &Ptr = S.Stk.pop<Pointer>();
2097
2098 if (MaxLength && MaxLength->isZero()) {
2099 S.Stk.push<Pointer>();
2100 return true;
2101 }
2102
2103 if (Ptr.isDummy()) {
2104 if (Ptr.getType()->isIncompleteType())
2105 S.FFDiag(S.Current->getSource(OpPC),
2106 diag::note_constexpr_ltor_incomplete_type)
2107 << Ptr.getType();
2108 return false;
2109 }
2110
2111 // Null is only okay if the given size is 0.
2112 if (Ptr.isZero()) {
2113 S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_access_null)
2114 << AK_Read;
2115 return false;
2116 }
2117
2118 if (!Ptr.isBlockPointer())
2119 return false;
2120
2121 QualType ElemTy = Ptr.getFieldDesc()->isArray()
2122 ? Ptr.getFieldDesc()->getElemQualType()
2123 : Ptr.getFieldDesc()->getType();
2124 bool IsRawByte = ID == Builtin::BImemchr || ID == Builtin::BI__builtin_memchr;
2125
2126 // Give up on byte-oriented matching against multibyte elements.
2127 if (IsRawByte && !isOneByteCharacterType(ElemTy)) {
2128 S.FFDiag(S.Current->getSource(OpPC),
2129 diag::note_constexpr_memchr_unsupported)
2130 << S.getASTContext().BuiltinInfo.getQuotedName(ID) << ElemTy;
2131 return false;
2132 }
2133
2134 if (!isReadable(Ptr))
2135 return false;
2136
2137 if (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr) {
2138 int64_t DesiredTrunc;
2139 if (S.getASTContext().CharTy->isSignedIntegerType())
2140 DesiredTrunc =
2141 Desired.trunc(S.getASTContext().getCharWidth()).getSExtValue();
2142 else
2143 DesiredTrunc =
2144 Desired.trunc(S.getASTContext().getCharWidth()).getZExtValue();
2145 // strchr compares directly to the passed integer, and therefore
2146 // always fails if given an int that is not a char.
2147 if (Desired != DesiredTrunc) {
2148 S.Stk.push<Pointer>();
2149 return true;
2150 }
2151 }
2152
2153 uint64_t DesiredVal;
2154 if (ID == Builtin::BIwmemchr || ID == Builtin::BI__builtin_wmemchr ||
2155 ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr) {
2156 // wcschr and wmemchr are given a wchar_t to look for. Just use it.
2157 DesiredVal = Desired.getZExtValue();
2158 } else {
2159 DesiredVal = Desired.trunc(S.getASTContext().getCharWidth()).getZExtValue();
2160 }
2161
2162 bool StopAtZero =
2163 (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr ||
2164 ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr);
2165
2166 PrimType ElemT =
2167 IsRawByte ? PT_Sint8 : *S.getContext().classify(getElemType(Ptr));
2168
2169 size_t Index = Ptr.getIndex();
2170 size_t Step = 0;
2171 for (;;) {
2172 const Pointer &ElemPtr =
2173 (Index + Step) > 0 ? Ptr.atIndex(Index + Step) : Ptr;
2174
2175 if (!CheckLoad(S, OpPC, ElemPtr))
2176 return false;
2177
2178 uint64_t V;
2180 ElemT, { V = static_cast<uint64_t>(ElemPtr.deref<T>().toUnsigned()); });
2181
2182 if (V == DesiredVal) {
2183 S.Stk.push<Pointer>(ElemPtr);
2184 return true;
2185 }
2186
2187 if (StopAtZero && V == 0)
2188 break;
2189
2190 ++Step;
2191 if (MaxLength && Step == MaxLength->getZExtValue())
2192 break;
2193 }
2194
2195 S.Stk.push<Pointer>();
2196 return true;
2197}
2198
2199static std::optional<unsigned> computeFullDescSize(const ASTContext &ASTCtx,
2200 const Descriptor *Desc) {
2201 if (Desc->isPrimitive())
2202 return ASTCtx.getTypeSizeInChars(Desc->getType()).getQuantity();
2203 if (Desc->isArray())
2204 return ASTCtx.getTypeSizeInChars(Desc->getElemQualType()).getQuantity() *
2205 Desc->getNumElems();
2206 if (Desc->isRecord()) {
2207 // Can't use Descriptor::getType() as that may return a pointer type. Look
2208 // at the decl directly.
2209 return ASTCtx
2211 ASTCtx.getCanonicalTagType(Desc->ElemRecord->getDecl()))
2212 .getQuantity();
2213 }
2214
2215 return std::nullopt;
2216}
2217
2218/// Compute the byte offset of \p Ptr in the full declaration.
2219static unsigned computePointerOffset(const ASTContext &ASTCtx,
2220 const Pointer &Ptr) {
2221 unsigned Result = 0;
2222
2223 Pointer P = Ptr;
2224 while (P.isField() || P.isArrayElement()) {
2225 P = P.expand();
2226 const Descriptor *D = P.getFieldDesc();
2227
2228 if (P.isArrayElement()) {
2229 unsigned ElemSize =
2231 if (P.isOnePastEnd())
2232 Result += ElemSize * P.getNumElems();
2233 else
2234 Result += ElemSize * P.getIndex();
2235 P = P.expand().getArray();
2236 } else if (P.isBaseClass()) {
2237 const auto *RD = cast<CXXRecordDecl>(D->asDecl());
2238 bool IsVirtual = Ptr.isVirtualBaseClass();
2239 P = P.getBase();
2240 const Record *BaseRecord = P.getRecord();
2241
2242 const ASTRecordLayout &Layout =
2243 ASTCtx.getASTRecordLayout(cast<CXXRecordDecl>(BaseRecord->getDecl()));
2244 if (IsVirtual)
2245 Result += Layout.getVBaseClassOffset(RD).getQuantity();
2246 else
2247 Result += Layout.getBaseClassOffset(RD).getQuantity();
2248 } else if (P.isField()) {
2249 const FieldDecl *FD = P.getField();
2250 const ASTRecordLayout &Layout =
2251 ASTCtx.getASTRecordLayout(FD->getParent());
2252 unsigned FieldIndex = FD->getFieldIndex();
2253 uint64_t FieldOffset =
2254 ASTCtx.toCharUnitsFromBits(Layout.getFieldOffset(FieldIndex))
2255 .getQuantity();
2256 Result += FieldOffset;
2257 P = P.getBase();
2258 } else
2259 llvm_unreachable("Unhandled descriptor type");
2260 }
2261
2262 return Result;
2263}
2264
2265/// Does Ptr point to the last subobject?
2266static bool pointsToLastObject(const Pointer &Ptr) {
2267 Pointer P = Ptr;
2268 while (!P.isRoot()) {
2269
2270 if (P.isArrayElement()) {
2271 P = P.expand().getArray();
2272 continue;
2273 }
2274 if (P.isBaseClass()) {
2275 if (P.getRecord()->getNumFields() > 0)
2276 return false;
2277 P = P.getBase();
2278 continue;
2279 }
2280
2281 Pointer Base = P.getBase();
2282 if (const Record *R = Base.getRecord()) {
2283 assert(P.getField());
2284 if (P.getField()->getFieldIndex() != R->getNumFields() - 1)
2285 return false;
2286 }
2287 P = Base;
2288 }
2289
2290 return true;
2291}
2292
2293/// Does Ptr point to the last object AND to a flexible array member?
2294static bool isUserWritingOffTheEnd(const ASTContext &Ctx, const Pointer &Ptr,
2295 bool InvalidBase) {
2296 auto isFlexibleArrayMember = [&](const Descriptor *FieldDesc) {
2298 FAMKind StrictFlexArraysLevel =
2299 Ctx.getLangOpts().getStrictFlexArraysLevel();
2300
2301 if (StrictFlexArraysLevel == FAMKind::Default)
2302 return true;
2303
2304 unsigned NumElems = FieldDesc->getNumElems();
2305 if (NumElems == 0 && StrictFlexArraysLevel != FAMKind::IncompleteOnly)
2306 return true;
2307
2308 if (NumElems == 1 && StrictFlexArraysLevel == FAMKind::OneZeroOrIncomplete)
2309 return true;
2310 return false;
2311 };
2312
2313 const Descriptor *FieldDesc = Ptr.getFieldDesc();
2314 if (!FieldDesc->isArray())
2315 return false;
2316
2317 return InvalidBase && pointsToLastObject(Ptr) &&
2318 isFlexibleArrayMember(FieldDesc);
2319}
2320
2322 unsigned Kind, Pointer &Ptr) {
2323 if (Ptr.isZero() || !Ptr.isBlockPointer())
2324 return std::nullopt;
2325
2326 if (Ptr.isDummy() && Ptr.getType()->isPointerType())
2327 return std::nullopt;
2328
2329 bool InvalidBase = false;
2330
2331 if (Ptr.isDummy()) {
2332 if (const VarDecl *VD = Ptr.getDeclDesc()->asVarDecl();
2333 VD && VD->getType()->isPointerType())
2334 InvalidBase = true;
2335 }
2336
2337 // According to the GCC documentation, we want the size of the subobject
2338 // denoted by the pointer. But that's not quite right -- what we actually
2339 // want is the size of the immediately-enclosing array, if there is one.
2340 if (Ptr.isArrayElement())
2341 Ptr = Ptr.expand();
2342
2343 bool DetermineForCompleteObject = Ptr.getFieldDesc() == Ptr.getDeclDesc();
2344 const Descriptor *DeclDesc = Ptr.getDeclDesc();
2345 assert(DeclDesc);
2346
2347 bool UseFieldDesc = (Kind & 1u);
2348 bool ReportMinimum = (Kind & 2u);
2349 if (!UseFieldDesc || DetermineForCompleteObject) {
2350 // Can't read beyond the pointer decl desc.
2351 if (!ReportMinimum && DeclDesc->getType()->isPointerType())
2352 return std::nullopt;
2353
2354 if (InvalidBase)
2355 return std::nullopt;
2356 } else {
2357 if (isUserWritingOffTheEnd(ASTCtx, Ptr, InvalidBase)) {
2358 // If we cannot determine the size of the initial allocation, then we
2359 // can't given an accurate upper-bound. However, we are still able to give
2360 // conservative lower-bounds for Type=3.
2361 if (Kind == 1)
2362 return std::nullopt;
2363 }
2364 }
2365
2366 // The "closest surrounding subobject" is NOT a base class,
2367 // so strip the base class casts.
2368 if (UseFieldDesc && Ptr.isBaseClass())
2369 Ptr = Ptr.stripBaseCasts();
2370
2371 const Descriptor *Desc = UseFieldDesc ? Ptr.getFieldDesc() : DeclDesc;
2372 assert(Desc);
2373
2374 std::optional<unsigned> FullSize = computeFullDescSize(ASTCtx, Desc);
2375 if (!FullSize)
2376 return std::nullopt;
2377
2378 unsigned ByteOffset;
2379 if (UseFieldDesc) {
2380 if (Ptr.isBaseClass()) {
2381 assert(computePointerOffset(ASTCtx, Ptr.getBase()) <=
2382 computePointerOffset(ASTCtx, Ptr));
2383 ByteOffset = computePointerOffset(ASTCtx, Ptr.getBase()) -
2384 computePointerOffset(ASTCtx, Ptr);
2385 } else {
2386 if (Ptr.inArray())
2387 ByteOffset =
2388 computePointerOffset(ASTCtx, Ptr) -
2389 computePointerOffset(ASTCtx, Ptr.expand().atIndex(0).narrow());
2390 else
2391 ByteOffset = 0;
2392 }
2393 } else
2394 ByteOffset = computePointerOffset(ASTCtx, Ptr);
2395
2396 assert(ByteOffset <= *FullSize);
2397 return *FullSize - ByteOffset;
2398}
2399
2401 const InterpFrame *Frame,
2402 const CallExpr *Call) {
2403 const ASTContext &ASTCtx = S.getASTContext();
2404 // From the GCC docs:
2405 // Kind is an integer constant from 0 to 3. If the least significant bit is
2406 // clear, objects are whole variables. If it is set, a closest surrounding
2407 // subobject is considered the object a pointer points to. The second bit
2408 // determines if maximum or minimum of remaining bytes is computed.
2409 unsigned Kind = popToUInt64(S, Call->getArg(1));
2410 assert(Kind <= 3 && "unexpected kind");
2411 Pointer Ptr = S.Stk.pop<Pointer>();
2412
2413 if (Call->getArg(0)->HasSideEffects(ASTCtx)) {
2414 // "If there are any side effects in them, it returns (size_t) -1
2415 // for type 0 or 1 and (size_t) 0 for type 2 or 3."
2416 pushInteger(S, Kind <= 1 ? -1 : 0, Call->getType());
2417 return true;
2418 }
2419
2420 if (auto Result = evaluateBuiltinObjectSize(ASTCtx, Kind, Ptr)) {
2421 pushInteger(S, *Result, Call->getType());
2422 return true;
2423 }
2424 return false;
2425}
2426
2428 const CallExpr *Call) {
2429
2430 if (!S.inConstantContext())
2431 return false;
2432
2433 const Pointer &Ptr = S.Stk.pop<Pointer>();
2434
2435 auto Error = [&](int Diag) {
2436 bool CalledFromStd = false;
2437 const auto *Callee = S.Current->getCallee();
2438 if (Callee && Callee->isInStdNamespace()) {
2439 const IdentifierInfo *Identifier = Callee->getIdentifier();
2440 CalledFromStd = Identifier && Identifier->isStr("is_within_lifetime");
2441 }
2442 S.CCEDiag(CalledFromStd
2444 : S.Current->getSource(OpPC),
2445 diag::err_invalid_is_within_lifetime)
2446 << (CalledFromStd ? "std::is_within_lifetime"
2447 : "__builtin_is_within_lifetime")
2448 << Diag;
2449 return false;
2450 };
2451
2452 if (Ptr.isZero())
2453 return Error(0);
2454 if (Ptr.isOnePastEnd())
2455 return Error(1);
2456
2457 bool Result = Ptr.getLifetime() != Lifetime::Ended;
2458 if (!Ptr.isActive()) {
2459 Result = false;
2460 } else {
2461 if (!CheckLive(S, OpPC, Ptr, AK_Read))
2462 return false;
2463 if (!CheckMutable(S, OpPC, Ptr))
2464 return false;
2465 if (!CheckDummy(S, OpPC, Ptr.block(), AK_Read))
2466 return false;
2467 }
2468
2469 // Check if we're currently running an initializer.
2470 if (llvm::is_contained(S.InitializingBlocks, Ptr.block()))
2471 return Error(2);
2472 if (S.EvaluatingDecl && Ptr.getDeclDesc()->asVarDecl() == S.EvaluatingDecl)
2473 return Error(2);
2474
2475 pushInteger(S, Result, Call->getType());
2476 return true;
2477}
2478
2480 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2481 llvm::function_ref<APInt(const APSInt &)> Fn) {
2482 assert(Call->getNumArgs() == 1);
2483
2484 // Single integer case.
2485 if (!Call->getArg(0)->getType()->isVectorType()) {
2486 assert(Call->getType()->isIntegerType());
2487 APSInt Src = popToAPSInt(S, Call->getArg(0));
2488 APInt Result = Fn(Src);
2489 pushInteger(S, APSInt(std::move(Result), !Src.isSigned()), Call->getType());
2490 return true;
2491 }
2492
2493 // Vector case.
2494 const Pointer &Arg = S.Stk.pop<Pointer>();
2495 assert(Arg.getFieldDesc()->isPrimitiveArray());
2496 const Pointer &Dst = S.Stk.peek<Pointer>();
2497 assert(Dst.getFieldDesc()->isPrimitiveArray());
2498 assert(Arg.getFieldDesc()->getNumElems() ==
2499 Dst.getFieldDesc()->getNumElems());
2500
2501 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
2502 PrimType ElemT = *S.getContext().classify(ElemType);
2503 unsigned NumElems = Arg.getNumElems();
2504 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2505
2506 for (unsigned I = 0; I != NumElems; ++I) {
2508 APSInt Src = Arg.elem<T>(I).toAPSInt();
2509 APInt Result = Fn(Src);
2510 Dst.elem<T>(I) = static_cast<T>(APSInt(std::move(Result), DestUnsigned));
2511 });
2512 }
2514
2515 return true;
2516}
2517
2519 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2520 llvm::function_ref<std::optional<APFloat>(
2521 const APFloat &, const APFloat &, std::optional<APSInt> RoundingMode)>
2522 Fn,
2523 bool IsScalar = false) {
2524 assert((Call->getNumArgs() == 2) || (Call->getNumArgs() == 3));
2525 const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
2526 assert(VT->getElementType()->isFloatingType());
2527 unsigned NumElems = VT->getNumElements();
2528
2529 // Vector case.
2530 assert(Call->getArg(0)->getType()->isVectorType() &&
2531 Call->getArg(1)->getType()->isVectorType());
2532 assert(VT->getElementType() ==
2533 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2534 assert(VT->getNumElements() ==
2535 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2536
2537 std::optional<APSInt> RoundingMode = std::nullopt;
2538 if (Call->getNumArgs() == 3)
2539 RoundingMode = popToAPSInt(S, Call->getArg(2));
2540
2541 const Pointer &BPtr = S.Stk.pop<Pointer>();
2542 const Pointer &APtr = S.Stk.pop<Pointer>();
2543 const Pointer &Dst = S.Stk.peek<Pointer>();
2544 for (unsigned ElemIdx = 0; ElemIdx != NumElems; ++ElemIdx) {
2545 using T = PrimConv<PT_Float>::T;
2546 if (IsScalar && ElemIdx > 0) {
2547 Dst.elem<T>(ElemIdx) = APtr.elem<T>(ElemIdx);
2548 continue;
2549 }
2550 APFloat ElemA = APtr.elem<T>(ElemIdx).getAPFloat();
2551 APFloat ElemB = BPtr.elem<T>(ElemIdx).getAPFloat();
2552 std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
2553 if (!Result)
2554 return false;
2555 Dst.elem<T>(ElemIdx) = static_cast<T>(*Result);
2556 }
2557
2559
2560 return true;
2561}
2562
2564 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2565 llvm::function_ref<std::optional<APFloat>(const APFloat &, const APFloat &,
2566 std::optional<APSInt>)>
2567 Fn) {
2568 assert(Call->getNumArgs() == 5);
2569 const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
2570 unsigned NumElems = VT->getNumElements();
2571
2572 APSInt RoundingMode = popToAPSInt(S, Call->getArg(4));
2573 uint64_t MaskVal = popToUInt64(S, Call->getArg(3));
2574 const Pointer &SrcPtr = S.Stk.pop<Pointer>();
2575 const Pointer &BPtr = S.Stk.pop<Pointer>();
2576 const Pointer &APtr = S.Stk.pop<Pointer>();
2577 const Pointer &Dst = S.Stk.peek<Pointer>();
2578
2579 using T = PrimConv<PT_Float>::T;
2580
2581 if (MaskVal & 1) {
2582 APFloat ElemA = APtr.elem<T>(0).getAPFloat();
2583 APFloat ElemB = BPtr.elem<T>(0).getAPFloat();
2584 std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
2585 if (!Result)
2586 return false;
2587 Dst.elem<T>(0) = static_cast<T>(*Result);
2588 } else {
2589 Dst.elem<T>(0) = SrcPtr.elem<T>(0);
2590 }
2591
2592 for (unsigned I = 1; I < NumElems; ++I)
2593 Dst.elem<T>(I) = APtr.elem<T>(I);
2594
2595 Dst.initializeAllElements();
2596
2597 return true;
2598}
2599
2601 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2602 llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
2603 assert(Call->getNumArgs() == 2);
2604
2605 // Single integer case.
2606 if (!Call->getArg(0)->getType()->isVectorType()) {
2607 assert(!Call->getArg(1)->getType()->isVectorType());
2608 APSInt RHS = popToAPSInt(S, Call->getArg(1));
2609 APSInt LHS = popToAPSInt(S, Call->getArg(0));
2610 APInt Result = Fn(LHS, RHS);
2611 pushInteger(S, APSInt(std::move(Result), !LHS.isSigned()), Call->getType());
2612 return true;
2613 }
2614
2615 const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
2616 assert(VT->getElementType()->isIntegralOrEnumerationType());
2617 PrimType ElemT = *S.getContext().classify(VT->getElementType());
2618 unsigned NumElems = VT->getNumElements();
2619 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2620
2621 // Vector + Scalar case.
2622 if (!Call->getArg(1)->getType()->isVectorType()) {
2623 assert(Call->getArg(1)->getType()->isIntegralOrEnumerationType());
2624
2625 APSInt RHS = popToAPSInt(S, Call->getArg(1));
2626 const Pointer &LHS = S.Stk.pop<Pointer>();
2627 const Pointer &Dst = S.Stk.peek<Pointer>();
2628
2629 for (unsigned I = 0; I != NumElems; ++I) {
2631 Dst.elem<T>(I) = static_cast<T>(
2632 APSInt(Fn(LHS.elem<T>(I).toAPSInt(), RHS), DestUnsigned));
2633 });
2634 }
2636 return true;
2637 }
2638
2639 // Vector case.
2640 assert(Call->getArg(0)->getType()->isVectorType() &&
2641 Call->getArg(1)->getType()->isVectorType());
2642 assert(VT->getElementType() ==
2643 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2644 assert(VT->getNumElements() ==
2645 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2646 assert(VT->getElementType()->isIntegralOrEnumerationType());
2647
2648 const Pointer &RHS = S.Stk.pop<Pointer>();
2649 const Pointer &LHS = S.Stk.pop<Pointer>();
2650 const Pointer &Dst = S.Stk.peek<Pointer>();
2651 for (unsigned I = 0; I != NumElems; ++I) {
2653 APSInt Elem1 = LHS.elem<T>(I).toAPSInt();
2654 APSInt Elem2 = RHS.elem<T>(I).toAPSInt();
2655 Dst.elem<T>(I) = static_cast<T>(APSInt(Fn(Elem1, Elem2), DestUnsigned));
2656 });
2657 }
2659
2660 return true;
2661}
2662
2663static bool
2665 llvm::function_ref<APInt(const APSInt &)> PackFn) {
2666 const auto *VT0 = E->getArg(0)->getType()->castAs<VectorType>();
2667 [[maybe_unused]] const auto *VT1 =
2668 E->getArg(1)->getType()->castAs<VectorType>();
2669 assert(VT0 && VT1 && "pack builtin VT0 and VT1 must be VectorType");
2670 assert(VT0->getElementType() == VT1->getElementType() &&
2671 VT0->getNumElements() == VT1->getNumElements() &&
2672 "pack builtin VT0 and VT1 ElementType must be same");
2673
2674 const Pointer &RHS = S.Stk.pop<Pointer>();
2675 const Pointer &LHS = S.Stk.pop<Pointer>();
2676 const Pointer &Dst = S.Stk.peek<Pointer>();
2677
2678 const ASTContext &ASTCtx = S.getASTContext();
2679 unsigned SrcBits = ASTCtx.getIntWidth(VT0->getElementType());
2680 unsigned LHSVecLen = VT0->getNumElements();
2681 unsigned SrcPerLane = 128 / SrcBits;
2682 unsigned Lanes = LHSVecLen * SrcBits / 128;
2683
2684 PrimType SrcT = *S.getContext().classify(VT0->getElementType());
2685 PrimType DstT = *S.getContext().classify(getElemType(Dst));
2686 bool IsUnsigend = getElemType(Dst)->isUnsignedIntegerType();
2687
2688 for (unsigned Lane = 0; Lane != Lanes; ++Lane) {
2689 unsigned BaseSrc = Lane * SrcPerLane;
2690 unsigned BaseDst = Lane * (2 * SrcPerLane);
2691
2692 for (unsigned I = 0; I != SrcPerLane; ++I) {
2694 APSInt A = LHS.elem<T>(BaseSrc + I).toAPSInt();
2695 APSInt B = RHS.elem<T>(BaseSrc + I).toAPSInt();
2696
2697 assignInteger(S, Dst.atIndex(BaseDst + I), DstT,
2698 APSInt(PackFn(A), IsUnsigend));
2699 assignInteger(S, Dst.atIndex(BaseDst + SrcPerLane + I), DstT,
2700 APSInt(PackFn(B), IsUnsigend));
2701 });
2702 }
2703 }
2704
2705 Dst.initializeAllElements();
2706 return true;
2707}
2708
2710 const CallExpr *Call,
2711 unsigned BuiltinID) {
2712 assert(Call->getNumArgs() == 2);
2713
2714 QualType Arg0Type = Call->getArg(0)->getType();
2715
2716 // TODO: Support floating-point types.
2717 if (!(Arg0Type->isIntegerType() ||
2718 (Arg0Type->isVectorType() &&
2719 Arg0Type->castAs<VectorType>()->getElementType()->isIntegerType())))
2720 return false;
2721
2722 if (!Arg0Type->isVectorType()) {
2723 assert(!Call->getArg(1)->getType()->isVectorType());
2724 APSInt RHS = popToAPSInt(S, Call->getArg(1));
2725 APSInt LHS = popToAPSInt(S, Arg0Type);
2726 APInt Result;
2727 if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
2728 Result = std::max(LHS, RHS);
2729 } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
2730 Result = std::min(LHS, RHS);
2731 } else {
2732 llvm_unreachable("Wrong builtin ID");
2733 }
2734
2735 pushInteger(S, APSInt(Result, !LHS.isSigned()), Call->getType());
2736 return true;
2737 }
2738
2739 // Vector case.
2740 assert(Call->getArg(0)->getType()->isVectorType() &&
2741 Call->getArg(1)->getType()->isVectorType());
2742 const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
2743 assert(VT->getElementType() ==
2744 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2745 assert(VT->getNumElements() ==
2746 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2747 assert(VT->getElementType()->isIntegralOrEnumerationType());
2748
2749 const Pointer &RHS = S.Stk.pop<Pointer>();
2750 const Pointer &LHS = S.Stk.pop<Pointer>();
2751 const Pointer &Dst = S.Stk.peek<Pointer>();
2752 PrimType ElemT = *S.getContext().classify(VT->getElementType());
2753 unsigned NumElems = VT->getNumElements();
2754 for (unsigned I = 0; I != NumElems; ++I) {
2755 APSInt Elem1;
2756 APSInt Elem2;
2758 Elem1 = LHS.elem<T>(I).toAPSInt();
2759 Elem2 = RHS.elem<T>(I).toAPSInt();
2760 });
2761
2762 APSInt Result;
2763 if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
2764 Result = APSInt(std::max(Elem1, Elem2),
2765 Call->getType()->isUnsignedIntegerOrEnumerationType());
2766 } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
2767 Result = APSInt(std::min(Elem1, Elem2),
2768 Call->getType()->isUnsignedIntegerOrEnumerationType());
2769 } else {
2770 llvm_unreachable("Wrong builtin ID");
2771 }
2772
2774 { Dst.elem<T>(I) = static_cast<T>(Result); });
2775 }
2776 Dst.initializeAllElements();
2777
2778 return true;
2779}
2780
2782 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2783 llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &,
2784 const APSInt &)>
2785 Fn) {
2786 assert(Call->getArg(0)->getType()->isVectorType() &&
2787 Call->getArg(1)->getType()->isVectorType());
2788 const Pointer &RHS = S.Stk.pop<Pointer>();
2789 const Pointer &LHS = S.Stk.pop<Pointer>();
2790 const Pointer &Dst = S.Stk.peek<Pointer>();
2791
2792 const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
2793 PrimType ElemT = *S.getContext().classify(VT->getElementType());
2794 unsigned NumElems = VT->getNumElements();
2795 const auto *DestVT = Call->getType()->castAs<VectorType>();
2796 PrimType DestElemT = *S.getContext().classify(DestVT->getElementType());
2797 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2798
2799 unsigned DstElem = 0;
2800 for (unsigned I = 0; I != NumElems; I += 2) {
2801 APSInt Result;
2803 APSInt LoLHS = LHS.elem<T>(I).toAPSInt();
2804 APSInt HiLHS = LHS.elem<T>(I + 1).toAPSInt();
2805 APSInt LoRHS = RHS.elem<T>(I).toAPSInt();
2806 APSInt HiRHS = RHS.elem<T>(I + 1).toAPSInt();
2807 Result = APSInt(Fn(LoLHS, HiLHS, LoRHS, HiRHS), DestUnsigned);
2808 });
2809
2810 INT_TYPE_SWITCH_NO_BOOL(DestElemT,
2811 { Dst.elem<T>(DstElem) = static_cast<T>(Result); });
2812 ++DstElem;
2813 }
2814
2815 Dst.initializeAllElements();
2816 return true;
2817}
2818
2820 const CallExpr *Call) {
2821 assert(Call->getNumArgs() == 3);
2822 unsigned Imm = popToUInt64(S, Call->getArg(2));
2823
2824 const Pointer &Src2 = S.Stk.pop<Pointer>();
2825 const Pointer &Src1 = S.Stk.pop<Pointer>();
2826 const Pointer &Dst = S.Stk.peek<Pointer>();
2827
2828 const auto *SrcVT = Call->getArg(0)->getType()->castAs<VectorType>();
2829 PrimType SrcElemT = *S.getContext().classify(SrcVT->getElementType());
2830 unsigned SourceLen = SrcVT->getNumElements();
2831
2832 const auto *DestVT = Call->getType()->castAs<VectorType>();
2833 PrimType DestElemT = *S.getContext().classify(DestVT->getElementType());
2834 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2835
2836 constexpr unsigned LaneSize = 16; // 128-bit lane = 16 bytes
2837
2838 // Phase 1: Shuffle Src2 using all four 2-bit fields of imm8.
2839 // Within each 128-bit lane, for group j (0..3), select a 4-byte block
2840 // from Src2 based on bits [2*j+1:2*j] of imm8.
2841 SmallVector<uint8_t, 64> Shuffled(SourceLen);
2842 for (unsigned I = 0; I < SourceLen; I += LaneSize) {
2843 for (unsigned J = 0; J < 4; ++J) {
2844 unsigned Part = (Imm >> (2 * J)) & 3;
2845 for (unsigned K = 0; K < 4; ++K) {
2846 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
2847 Shuffled[I + 4 * J + K] =
2848 static_cast<uint8_t>(Src2.elem<T>(I + 4 * Part + K));
2849 });
2850 }
2851 }
2852 }
2853
2854 // Phase 2: Sliding SAD computation.
2855 // For every group of 4 output u16 values, compute absolute differences
2856 // using overlapping windows into Src1 and the shuffled array.
2857 unsigned Size = SourceLen / 2; // number of output u16 elements
2858 for (unsigned I = 0; I < Size; I += 4) {
2859 unsigned Sad[4] = {0, 0, 0, 0};
2860 for (unsigned J = 0; J < 4; ++J) {
2861 uint8_t A1, A2;
2862 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
2863 A1 = static_cast<uint8_t>(Src1.elem<T>(2 * I + J));
2864 A2 = static_cast<uint8_t>(Src1.elem<T>(2 * I + J + 4));
2865 });
2866 uint8_t B0 = Shuffled[2 * I + J];
2867 uint8_t B1 = Shuffled[2 * I + J + 1];
2868 uint8_t B2 = Shuffled[2 * I + J + 2];
2869 uint8_t B3 = Shuffled[2 * I + J + 3];
2870 Sad[0] += (A1 > B0) ? (A1 - B0) : (B0 - A1);
2871 Sad[1] += (A1 > B1) ? (A1 - B1) : (B1 - A1);
2872 Sad[2] += (A2 > B2) ? (A2 - B2) : (B2 - A2);
2873 Sad[3] += (A2 > B3) ? (A2 - B3) : (B3 - A2);
2874 }
2875 for (unsigned R = 0; R < 4; ++R) {
2876 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
2877 Dst.elem<T>(I + R) =
2878 static_cast<T>(APSInt(APInt(16, Sad[R]), DestUnsigned));
2879 });
2880 }
2881 }
2882
2883 Dst.initializeAllElements();
2884 return true;
2885}
2886
2888 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2889 llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
2890 const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
2891 PrimType ElemT = *S.getContext().classify(VT->getElementType());
2892 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2893
2894 const Pointer &RHS = S.Stk.pop<Pointer>();
2895 const Pointer &LHS = S.Stk.pop<Pointer>();
2896 const Pointer &Dst = S.Stk.peek<Pointer>();
2897 unsigned NumElts = VT->getNumElements();
2898 unsigned EltBits = S.getASTContext().getIntWidth(VT->getElementType());
2899 unsigned EltsPerLane = 128 / EltBits;
2900 unsigned Lanes = NumElts * EltBits / 128;
2901 unsigned DestIndex = 0;
2902
2903 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
2904 unsigned LaneStart = Lane * EltsPerLane;
2905 for (unsigned I = 0; I < EltsPerLane; I += 2) {
2907 APSInt Elem1 = LHS.elem<T>(LaneStart + I).toAPSInt();
2908 APSInt Elem2 = LHS.elem<T>(LaneStart + I + 1).toAPSInt();
2909 APSInt ResL = APSInt(Fn(Elem1, Elem2), DestUnsigned);
2910 Dst.elem<T>(DestIndex++) = static_cast<T>(ResL);
2911 });
2912 }
2913
2914 for (unsigned I = 0; I < EltsPerLane; I += 2) {
2916 APSInt Elem1 = RHS.elem<T>(LaneStart + I).toAPSInt();
2917 APSInt Elem2 = RHS.elem<T>(LaneStart + I + 1).toAPSInt();
2918 APSInt ResR = APSInt(Fn(Elem1, Elem2), DestUnsigned);
2919 Dst.elem<T>(DestIndex++) = static_cast<T>(ResR);
2920 });
2921 }
2922 }
2923 Dst.initializeAllElements();
2924 return true;
2925}
2926
2928 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2929 llvm::function_ref<APFloat(const APFloat &, const APFloat &,
2930 llvm::RoundingMode)>
2931 Fn) {
2932 const Pointer &RHS = S.Stk.pop<Pointer>();
2933 const Pointer &LHS = S.Stk.pop<Pointer>();
2934 const Pointer &Dst = S.Stk.peek<Pointer>();
2935 FPOptions FPO = Call->getFPFeaturesInEffect(S.Ctx.getLangOpts());
2936 llvm::RoundingMode RM = getRoundingMode(FPO);
2937 const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
2938
2939 unsigned NumElts = VT->getNumElements();
2940 unsigned EltBits = S.getASTContext().getTypeSize(VT->getElementType());
2941 unsigned NumLanes = NumElts * EltBits / 128;
2942 unsigned NumElemsPerLane = NumElts / NumLanes;
2943 unsigned HalfElemsPerLane = NumElemsPerLane / 2;
2944
2945 for (unsigned L = 0; L != NumElts; L += NumElemsPerLane) {
2946 using T = PrimConv<PT_Float>::T;
2947 for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
2948 APFloat Elem1 = LHS.elem<T>(L + (2 * E) + 0).getAPFloat();
2949 APFloat Elem2 = LHS.elem<T>(L + (2 * E) + 1).getAPFloat();
2950 Dst.elem<T>(L + E) = static_cast<T>(Fn(Elem1, Elem2, RM));
2951 }
2952 for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
2953 APFloat Elem1 = RHS.elem<T>(L + (2 * E) + 0).getAPFloat();
2954 APFloat Elem2 = RHS.elem<T>(L + (2 * E) + 1).getAPFloat();
2955 Dst.elem<T>(L + E + HalfElemsPerLane) =
2956 static_cast<T>(Fn(Elem1, Elem2, RM));
2957 }
2958 }
2959 Dst.initializeAllElements();
2960 return true;
2961}
2962
2964 const CallExpr *Call) {
2965 // Addsub: alternates between subtraction and addition
2966 // Result[i] = (i % 2 == 0) ? (a[i] - b[i]) : (a[i] + b[i])
2967 const Pointer &RHS = S.Stk.pop<Pointer>();
2968 const Pointer &LHS = S.Stk.pop<Pointer>();
2969 const Pointer &Dst = S.Stk.peek<Pointer>();
2970 FPOptions FPO = Call->getFPFeaturesInEffect(S.Ctx.getLangOpts());
2971 llvm::RoundingMode RM = getRoundingMode(FPO);
2972 const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
2973 unsigned NumElems = VT->getNumElements();
2974
2975 using T = PrimConv<PT_Float>::T;
2976 for (unsigned I = 0; I != NumElems; ++I) {
2977 APFloat LElem = LHS.elem<T>(I).getAPFloat();
2978 APFloat RElem = RHS.elem<T>(I).getAPFloat();
2979 if (I % 2 == 0) {
2980 // Even indices: subtract
2981 LElem.subtract(RElem, RM);
2982 } else {
2983 // Odd indices: add
2984 LElem.add(RElem, RM);
2985 }
2986 Dst.elem<T>(I) = static_cast<T>(LElem);
2987 }
2988 Dst.initializeAllElements();
2989 return true;
2990}
2991
2993 const CallExpr *Call) {
2994 // PCLMULQDQ: carry-less multiplication of selected 64-bit halves
2995 // imm8 bit 0: selects lower (0) or upper (1) 64 bits of first operand
2996 // imm8 bit 4: selects lower (0) or upper (1) 64 bits of second operand
2997 assert(Call->getArg(0)->getType()->isVectorType() &&
2998 Call->getArg(1)->getType()->isVectorType());
2999
3000 // Extract imm8 argument
3001 APSInt Imm8 = popToAPSInt(S, Call->getArg(2));
3002 bool SelectUpperA = (Imm8 & 0x01) != 0;
3003 bool SelectUpperB = (Imm8 & 0x10) != 0;
3004
3005 const Pointer &RHS = S.Stk.pop<Pointer>();
3006 const Pointer &LHS = S.Stk.pop<Pointer>();
3007 const Pointer &Dst = S.Stk.peek<Pointer>();
3008
3009 const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
3010 PrimType ElemT = *S.getContext().classify(VT->getElementType());
3011 unsigned NumElems = VT->getNumElements();
3012 const auto *DestVT = Call->getType()->castAs<VectorType>();
3013 PrimType DestElemT = *S.getContext().classify(DestVT->getElementType());
3014 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3015
3016 // Process each 128-bit lane (2 elements at a time)
3017 for (unsigned Lane = 0; Lane < NumElems; Lane += 2) {
3018 APSInt A0, A1, B0, B1;
3020 A0 = LHS.elem<T>(Lane + 0).toAPSInt();
3021 A1 = LHS.elem<T>(Lane + 1).toAPSInt();
3022 B0 = RHS.elem<T>(Lane + 0).toAPSInt();
3023 B1 = RHS.elem<T>(Lane + 1).toAPSInt();
3024 });
3025
3026 // Select the appropriate 64-bit values based on imm8
3027 APInt A = SelectUpperA ? A1 : A0;
3028 APInt B = SelectUpperB ? B1 : B0;
3029
3030 // Extend both operands to 128 bits for carry-less multiplication
3031 APInt A128 = A.zext(128);
3032 APInt B128 = B.zext(128);
3033
3034 // Use APIntOps::clmul for carry-less multiplication
3035 APInt Result = llvm::APIntOps::clmul(A128, B128);
3036
3037 // Split the 128-bit result into two 64-bit halves
3038 APSInt ResultLow(Result.extractBits(64, 0), DestUnsigned);
3039 APSInt ResultHigh(Result.extractBits(64, 64), DestUnsigned);
3040
3041 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3042 Dst.elem<T>(Lane + 0) = static_cast<T>(ResultLow);
3043 Dst.elem<T>(Lane + 1) = static_cast<T>(ResultHigh);
3044 });
3045 }
3046
3047 Dst.initializeAllElements();
3048 return true;
3049}
3050
3052 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3053 llvm::function_ref<APFloat(const APFloat &, const APFloat &,
3054 const APFloat &, llvm::RoundingMode)>
3055 Fn) {
3056 assert(Call->getNumArgs() == 3);
3057
3058 FPOptions FPO = Call->getFPFeaturesInEffect(S.Ctx.getLangOpts());
3059 llvm::RoundingMode RM = getRoundingMode(FPO);
3060 QualType Arg1Type = Call->getArg(0)->getType();
3061 QualType Arg2Type = Call->getArg(1)->getType();
3062 QualType Arg3Type = Call->getArg(2)->getType();
3063
3064 // Non-vector floating point types.
3065 if (!Arg1Type->isVectorType()) {
3066 assert(!Arg2Type->isVectorType());
3067 assert(!Arg3Type->isVectorType());
3068 (void)Arg2Type;
3069 (void)Arg3Type;
3070
3071 const Floating &Z = S.Stk.pop<Floating>();
3072 const Floating &Y = S.Stk.pop<Floating>();
3073 const Floating &X = S.Stk.pop<Floating>();
3074 APFloat F = Fn(X.getAPFloat(), Y.getAPFloat(), Z.getAPFloat(), RM);
3075 Floating Result = S.allocFloat(X.getSemantics());
3076 Result.copy(F);
3077 S.Stk.push<Floating>(Result);
3078 return true;
3079 }
3080
3081 // Vector type.
3082 assert(Arg1Type->isVectorType() && Arg2Type->isVectorType() &&
3083 Arg3Type->isVectorType());
3084
3085 const VectorType *VecTy = Arg1Type->castAs<VectorType>();
3086 QualType ElemQT = VecTy->getElementType();
3087 unsigned NumElems = VecTy->getNumElements();
3088
3089 assert(ElemQT == Arg2Type->castAs<VectorType>()->getElementType() &&
3090 ElemQT == Arg3Type->castAs<VectorType>()->getElementType());
3091 assert(NumElems == Arg2Type->castAs<VectorType>()->getNumElements() &&
3092 NumElems == Arg3Type->castAs<VectorType>()->getNumElements());
3093 assert(ElemQT->isRealFloatingType());
3094 (void)ElemQT;
3095
3096 const Pointer &VZ = S.Stk.pop<Pointer>();
3097 const Pointer &VY = S.Stk.pop<Pointer>();
3098 const Pointer &VX = S.Stk.pop<Pointer>();
3099 const Pointer &Dst = S.Stk.peek<Pointer>();
3100 for (unsigned I = 0; I != NumElems; ++I) {
3101 using T = PrimConv<PT_Float>::T;
3102 APFloat X = VX.elem<T>(I).getAPFloat();
3103 APFloat Y = VY.elem<T>(I).getAPFloat();
3104 APFloat Z = VZ.elem<T>(I).getAPFloat();
3105 APFloat F = Fn(X, Y, Z, RM);
3106 Dst.elem<Floating>(I) = Floating(F);
3107 }
3109 return true;
3110}
3111
3112/// AVX512 predicated move: "Result = Mask[] ? LHS[] : RHS[]".
3114 const CallExpr *Call) {
3115 const Pointer &RHS = S.Stk.pop<Pointer>();
3116 const Pointer &LHS = S.Stk.pop<Pointer>();
3117 APSInt Mask = popToAPSInt(S, Call->getArg(0));
3118 const Pointer &Dst = S.Stk.peek<Pointer>();
3119
3120 assert(LHS.getNumElems() == RHS.getNumElems());
3121 assert(LHS.getNumElems() == Dst.getNumElems());
3122 unsigned NumElems = LHS.getNumElems();
3123 PrimType ElemT = LHS.getFieldDesc()->getPrimType();
3124 PrimType DstElemT = Dst.getFieldDesc()->getPrimType();
3125
3126 for (unsigned I = 0; I != NumElems; ++I) {
3127 if (ElemT == PT_Float) {
3128 assert(DstElemT == PT_Float);
3129 Dst.elem<Floating>(I) =
3130 Mask[I] ? LHS.elem<Floating>(I) : RHS.elem<Floating>(I);
3131 } else {
3132 APSInt Elem;
3133 INT_TYPE_SWITCH(ElemT, {
3134 Elem = Mask[I] ? LHS.elem<T>(I).toAPSInt() : RHS.elem<T>(I).toAPSInt();
3135 });
3136 INT_TYPE_SWITCH_NO_BOOL(DstElemT,
3137 { Dst.elem<T>(I) = static_cast<T>(Elem); });
3138 }
3139 }
3141
3142 return true;
3143}
3144
3145/// Scalar variant of AVX512 predicated select:
3146/// Result[i] = (Mask bit 0) ? LHS[i] : RHS[i], but only element 0 may change.
3147/// All other elements are taken from RHS.
3149 const CallExpr *Call) {
3150 unsigned N =
3151 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements();
3152
3153 const Pointer &W = S.Stk.pop<Pointer>();
3154 const Pointer &A = S.Stk.pop<Pointer>();
3155 APSInt U = popToAPSInt(S, Call->getArg(0));
3156 const Pointer &Dst = S.Stk.peek<Pointer>();
3157
3158 bool TakeA0 = U.getZExtValue() & 1ULL;
3159
3160 for (unsigned I = TakeA0; I != N; ++I)
3161 Dst.elem<Floating>(I) = W.elem<Floating>(I);
3162 if (TakeA0)
3163 Dst.elem<Floating>(0) = A.elem<Floating>(0);
3164
3166 return true;
3167}
3168
3170 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3171 llvm::function_ref<bool(const APInt &A, const APInt &B)> Fn) {
3172 const Pointer &RHS = S.Stk.pop<Pointer>();
3173 const Pointer &LHS = S.Stk.pop<Pointer>();
3174
3175 assert(LHS.getNumElems() == RHS.getNumElems());
3176
3177 unsigned SourceLen = LHS.getNumElems();
3178 QualType ElemQT = getElemType(LHS);
3179 OptPrimType ElemPT = S.getContext().classify(ElemQT);
3180 unsigned LaneWidth = S.getASTContext().getTypeSize(ElemQT);
3181
3182 APInt AWide(LaneWidth * SourceLen, 0);
3183 APInt BWide(LaneWidth * SourceLen, 0);
3184
3185 for (unsigned I = 0; I != SourceLen; ++I) {
3186 APInt ALane;
3187 APInt BLane;
3188
3189 if (ElemQT->isIntegerType()) { // Get value.
3190 INT_TYPE_SWITCH_NO_BOOL(*ElemPT, {
3191 ALane = LHS.elem<T>(I).toAPSInt();
3192 BLane = RHS.elem<T>(I).toAPSInt();
3193 });
3194 } else if (ElemQT->isFloatingType()) { // Get only sign bit.
3195 using T = PrimConv<PT_Float>::T;
3196 ALane = LHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
3197 BLane = RHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
3198 } else { // Must be integer or floating type.
3199 return false;
3200 }
3201 AWide.insertBits(ALane, I * LaneWidth);
3202 BWide.insertBits(BLane, I * LaneWidth);
3203 }
3204 pushInteger(S, Fn(AWide, BWide), Call->getType());
3205 return true;
3206}
3207
3209 const CallExpr *Call) {
3210 assert(Call->getNumArgs() == 1);
3211
3212 const Pointer &Source = S.Stk.pop<Pointer>();
3213
3214 unsigned SourceLen = Source.getNumElems();
3215 QualType ElemQT = getElemType(Source);
3216 OptPrimType ElemT = S.getContext().classify(ElemQT);
3217 unsigned ResultLen =
3218 S.getASTContext().getTypeSize(Call->getType()); // Always 32-bit integer.
3219 APInt Result(ResultLen, 0);
3220
3221 for (unsigned I = 0; I != SourceLen; ++I) {
3222 APInt Elem;
3223 if (ElemQT->isIntegerType()) {
3224 INT_TYPE_SWITCH_NO_BOOL(*ElemT, { Elem = Source.elem<T>(I).toAPSInt(); });
3225 } else if (ElemQT->isRealFloatingType()) {
3226 using T = PrimConv<PT_Float>::T;
3227 Elem = Source.elem<T>(I).getAPFloat().bitcastToAPInt();
3228 } else {
3229 return false;
3230 }
3231 Result.setBitVal(I, Elem.isNegative());
3232 }
3233 pushInteger(S, Result, Call->getType());
3234 return true;
3235}
3236
3238 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3239 llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &)>
3240 Fn) {
3241 assert(Call->getNumArgs() == 3);
3242
3243 QualType Arg0Type = Call->getArg(0)->getType();
3244 QualType Arg2Type = Call->getArg(2)->getType();
3245 // Non-vector integer types.
3246 if (!Arg0Type->isVectorType()) {
3247 const APSInt &Op2 = popToAPSInt(S, Arg2Type);
3248 const APSInt &Op1 = popToAPSInt(S, Call->getArg(1));
3249 const APSInt &Op0 = popToAPSInt(S, Arg0Type);
3250 APSInt Result = APSInt(Fn(Op0, Op1, Op2), Op0.isUnsigned());
3251 pushInteger(S, Result, Call->getType());
3252 return true;
3253 }
3254
3255 const auto *VecT = Arg0Type->castAs<VectorType>();
3256 PrimType ElemT = *S.getContext().classify(VecT->getElementType());
3257 unsigned NumElems = VecT->getNumElements();
3258 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3259
3260 // Vector + Vector + Scalar case.
3261 if (!Arg2Type->isVectorType()) {
3262 APSInt Op2 = popToAPSInt(S, Arg2Type);
3263
3264 const Pointer &Op1 = S.Stk.pop<Pointer>();
3265 const Pointer &Op0 = S.Stk.pop<Pointer>();
3266 const Pointer &Dst = S.Stk.peek<Pointer>();
3267 for (unsigned I = 0; I != NumElems; ++I) {
3269 Dst.elem<T>(I) = static_cast<T>(APSInt(
3270 Fn(Op0.elem<T>(I).toAPSInt(), Op1.elem<T>(I).toAPSInt(), Op2),
3271 DestUnsigned));
3272 });
3273 }
3275
3276 return true;
3277 }
3278
3279 // Vector type.
3280 const Pointer &Op2 = S.Stk.pop<Pointer>();
3281 const Pointer &Op1 = S.Stk.pop<Pointer>();
3282 const Pointer &Op0 = S.Stk.pop<Pointer>();
3283 const Pointer &Dst = S.Stk.peek<Pointer>();
3284 for (unsigned I = 0; I != NumElems; ++I) {
3285 APSInt Val0, Val1, Val2;
3287 Val0 = Op0.elem<T>(I).toAPSInt();
3288 Val1 = Op1.elem<T>(I).toAPSInt();
3289 Val2 = Op2.elem<T>(I).toAPSInt();
3290 });
3291 APSInt Result = APSInt(Fn(Val0, Val1, Val2), Val0.isUnsigned());
3293 { Dst.elem<T>(I) = static_cast<T>(Result); });
3294 }
3296
3297 return true;
3298}
3299
3301 const CallExpr *Call,
3302 unsigned ID) {
3303 assert(Call->getNumArgs() == 2);
3304
3305 APSInt ImmAPS = popToAPSInt(S, Call->getArg(1));
3306 uint64_t Index = ImmAPS.getZExtValue();
3307
3308 const Pointer &Src = S.Stk.pop<Pointer>();
3309 if (!Src.getFieldDesc()->isPrimitiveArray())
3310 return false;
3311
3312 const Pointer &Dst = S.Stk.peek<Pointer>();
3313 if (!Dst.getFieldDesc()->isPrimitiveArray())
3314 return false;
3315
3316 unsigned SrcElems = Src.getNumElems();
3317 unsigned DstElems = Dst.getNumElems();
3318
3319 unsigned NumLanes = SrcElems / DstElems;
3320 unsigned Lane = static_cast<unsigned>(Index % NumLanes);
3321 unsigned ExtractPos = Lane * DstElems;
3322
3323 PrimType ElemT = Src.getFieldDesc()->getPrimType();
3324
3325 TYPE_SWITCH(ElemT, {
3326 for (unsigned I = 0; I != DstElems; ++I) {
3327 Dst.elem<T>(I) = Src.elem<T>(ExtractPos + I);
3328 }
3329 });
3330
3332 return true;
3333}
3334
3336 CodePtr OpPC,
3337 const CallExpr *Call,
3338 unsigned ID) {
3339 assert(Call->getNumArgs() == 4);
3340
3341 APSInt MaskAPS = popToAPSInt(S, Call->getArg(3));
3342 const Pointer &Merge = S.Stk.pop<Pointer>();
3343 APSInt ImmAPS = popToAPSInt(S, Call->getArg(1));
3344 const Pointer &Src = S.Stk.pop<Pointer>();
3345
3346 if (!Src.getFieldDesc()->isPrimitiveArray() ||
3347 !Merge.getFieldDesc()->isPrimitiveArray())
3348 return false;
3349
3350 const Pointer &Dst = S.Stk.peek<Pointer>();
3351 if (!Dst.getFieldDesc()->isPrimitiveArray())
3352 return false;
3353
3354 unsigned SrcElems = Src.getNumElems();
3355 unsigned DstElems = Dst.getNumElems();
3356
3357 unsigned NumLanes = SrcElems / DstElems;
3358 unsigned Lane = static_cast<unsigned>(ImmAPS.getZExtValue() % NumLanes);
3359 unsigned Base = Lane * DstElems;
3360
3361 PrimType ElemT = Src.getFieldDesc()->getPrimType();
3362
3363 TYPE_SWITCH(ElemT, {
3364 for (unsigned I = 0; I != DstElems; ++I) {
3365 if (MaskAPS[I])
3366 Dst.elem<T>(I) = Src.elem<T>(Base + I);
3367 else
3368 Dst.elem<T>(I) = Merge.elem<T>(I);
3369 }
3370 });
3371
3373 return true;
3374}
3375
3377 const CallExpr *Call,
3378 unsigned ID) {
3379 assert(Call->getNumArgs() == 3);
3380
3381 APSInt ImmAPS = popToAPSInt(S, Call->getArg(2));
3382 uint64_t Index = ImmAPS.getZExtValue();
3383
3384 const Pointer &SubVec = S.Stk.pop<Pointer>();
3385 if (!SubVec.getFieldDesc()->isPrimitiveArray())
3386 return false;
3387
3388 const Pointer &BaseVec = S.Stk.pop<Pointer>();
3389 if (!BaseVec.getFieldDesc()->isPrimitiveArray())
3390 return false;
3391
3392 const Pointer &Dst = S.Stk.peek<Pointer>();
3393
3394 unsigned BaseElements = BaseVec.getNumElems();
3395 unsigned SubElements = SubVec.getNumElems();
3396
3397 assert(SubElements != 0 && BaseElements != 0 &&
3398 (BaseElements % SubElements) == 0);
3399
3400 unsigned NumLanes = BaseElements / SubElements;
3401 unsigned Lane = static_cast<unsigned>(Index % NumLanes);
3402 unsigned InsertPos = Lane * SubElements;
3403
3404 PrimType ElemT = BaseVec.getFieldDesc()->getPrimType();
3405
3406 TYPE_SWITCH(ElemT, {
3407 for (unsigned I = 0; I != BaseElements; ++I)
3408 Dst.elem<T>(I) = BaseVec.elem<T>(I);
3409 for (unsigned I = 0; I != SubElements; ++I)
3410 Dst.elem<T>(InsertPos + I) = SubVec.elem<T>(I);
3411 });
3412
3414 return true;
3415}
3416
3418 const CallExpr *Call) {
3419 assert(Call->getNumArgs() == 1);
3420
3421 const Pointer &Source = S.Stk.pop<Pointer>();
3422 const Pointer &Dest = S.Stk.peek<Pointer>();
3423
3424 unsigned SourceLen = Source.getNumElems();
3425 QualType ElemQT = getElemType(Source);
3426 OptPrimType ElemT = S.getContext().classify(ElemQT);
3427 unsigned ElemBitWidth = S.getASTContext().getTypeSize(ElemQT);
3428
3429 bool DestUnsigned = Call->getCallReturnType(S.getASTContext())
3430 ->castAs<VectorType>()
3431 ->getElementType()
3433
3434 INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
3435 APSInt MinIndex(ElemBitWidth, DestUnsigned);
3436 APSInt MinVal = Source.elem<T>(0).toAPSInt();
3437
3438 for (unsigned I = 1; I != SourceLen; ++I) {
3439 APSInt Val = Source.elem<T>(I).toAPSInt();
3440 if (MinVal.ugt(Val)) {
3441 MinVal = Val;
3442 MinIndex = I;
3443 }
3444 }
3445
3446 Dest.elem<T>(0) = static_cast<T>(MinVal);
3447 Dest.elem<T>(1) = static_cast<T>(MinIndex);
3448 for (unsigned I = 2; I != SourceLen; ++I) {
3449 Dest.elem<T>(I) = static_cast<T>(APSInt(ElemBitWidth, DestUnsigned));
3450 }
3451 });
3452 Dest.initializeAllElements();
3453 return true;
3454}
3455
3457 const CallExpr *Call, bool MaskZ) {
3458 assert(Call->getNumArgs() == 5);
3459
3460 APInt U = popToAPSInt(S, Call->getArg(4)); // Lane mask
3461 APInt Imm = popToAPSInt(S, Call->getArg(3)); // Ternary truth table
3462 const Pointer &C = S.Stk.pop<Pointer>();
3463 const Pointer &B = S.Stk.pop<Pointer>();
3464 const Pointer &A = S.Stk.pop<Pointer>();
3465 const Pointer &Dst = S.Stk.peek<Pointer>();
3466
3467 unsigned DstLen = A.getNumElems();
3468 QualType ElemQT = getElemType(A);
3469 OptPrimType ElemT = S.getContext().classify(ElemQT);
3470 unsigned LaneWidth = S.getASTContext().getTypeSize(ElemQT);
3471 bool DstUnsigned = ElemQT->isUnsignedIntegerOrEnumerationType();
3472
3473 INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
3474 for (unsigned I = 0; I != DstLen; ++I) {
3475 APInt ALane = A.elem<T>(I).toAPSInt();
3476 APInt BLane = B.elem<T>(I).toAPSInt();
3477 APInt CLane = C.elem<T>(I).toAPSInt();
3478 APInt RLane(LaneWidth, 0);
3479 if (U[I]) { // If lane not masked, compute ternary logic.
3480 for (unsigned Bit = 0; Bit != LaneWidth; ++Bit) {
3481 unsigned ABit = ALane[Bit];
3482 unsigned BBit = BLane[Bit];
3483 unsigned CBit = CLane[Bit];
3484 unsigned Idx = (ABit << 2) | (BBit << 1) | (CBit);
3485 RLane.setBitVal(Bit, Imm[Idx]);
3486 }
3487 Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
3488 } else if (MaskZ) { // If zero masked, zero the lane.
3489 Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
3490 } else { // Just masked, put in A lane.
3491 Dst.elem<T>(I) = static_cast<T>(APSInt(ALane, DstUnsigned));
3492 }
3493 }
3494 });
3495 Dst.initializeAllElements();
3496 return true;
3497}
3498
3500 const CallExpr *Call, unsigned ID) {
3501 assert(Call->getNumArgs() == 2);
3502
3503 APSInt ImmAPS = popToAPSInt(S, Call->getArg(1));
3504 const Pointer &Vec = S.Stk.pop<Pointer>();
3505 if (!Vec.getFieldDesc()->isPrimitiveArray())
3506 return false;
3507
3508 unsigned NumElems = Vec.getNumElems();
3509 unsigned Index =
3510 static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
3511
3512 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3513 // FIXME(#161685): Replace float+int split with a numeric-only type switch
3514 if (ElemT == PT_Float) {
3515 S.Stk.push<Floating>(Vec.elem<Floating>(Index));
3516 return true;
3517 }
3519 APSInt V = Vec.elem<T>(Index).toAPSInt();
3520 pushInteger(S, V, Call->getType());
3521 });
3522
3523 return true;
3524}
3525
3527 const CallExpr *Call, unsigned ID) {
3528 assert(Call->getNumArgs() == 3);
3529
3530 APSInt ImmAPS = popToAPSInt(S, Call->getArg(2));
3531 APSInt ValAPS = popToAPSInt(S, Call->getArg(1));
3532
3533 const Pointer &Base = S.Stk.pop<Pointer>();
3534 if (!Base.getFieldDesc()->isPrimitiveArray())
3535 return false;
3536
3537 const Pointer &Dst = S.Stk.peek<Pointer>();
3538
3539 unsigned NumElems = Base.getNumElems();
3540 unsigned Index =
3541 static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
3542
3543 PrimType ElemT = Base.getFieldDesc()->getPrimType();
3545 for (unsigned I = 0; I != NumElems; ++I)
3546 Dst.elem<T>(I) = Base.elem<T>(I);
3547 Dst.elem<T>(Index) = static_cast<T>(ValAPS);
3548 });
3549
3551 return true;
3552}
3553
3554static bool evalICmpImm(uint8_t Imm, const APSInt &A, const APSInt &B,
3555 bool IsUnsigned) {
3556 switch (Imm & 0x7) {
3557 case 0x00: // _MM_CMPINT_EQ
3558 return (A == B);
3559 case 0x01: // _MM_CMPINT_LT
3560 return IsUnsigned ? A.ult(B) : A.slt(B);
3561 case 0x02: // _MM_CMPINT_LE
3562 return IsUnsigned ? A.ule(B) : A.sle(B);
3563 case 0x03: // _MM_CMPINT_FALSE
3564 return false;
3565 case 0x04: // _MM_CMPINT_NE
3566 return (A != B);
3567 case 0x05: // _MM_CMPINT_NLT
3568 return IsUnsigned ? A.ugt(B) : A.sgt(B);
3569 case 0x06: // _MM_CMPINT_NLE
3570 return IsUnsigned ? A.uge(B) : A.sge(B);
3571 case 0x07: // _MM_CMPINT_TRUE
3572 return true;
3573 default:
3574 llvm_unreachable("Invalid Op");
3575 }
3576}
3577
3579 const CallExpr *Call, unsigned ID,
3580 bool IsUnsigned) {
3581 assert(Call->getNumArgs() == 4);
3582
3583 APSInt Mask = popToAPSInt(S, Call->getArg(3));
3584 APSInt Opcode = popToAPSInt(S, Call->getArg(2));
3585 unsigned CmpOp = static_cast<unsigned>(Opcode.getZExtValue());
3586 const Pointer &RHS = S.Stk.pop<Pointer>();
3587 const Pointer &LHS = S.Stk.pop<Pointer>();
3588
3589 assert(LHS.getNumElems() == RHS.getNumElems());
3590
3591 APInt RetMask = APInt::getZero(LHS.getNumElems());
3592 unsigned VectorLen = LHS.getNumElems();
3593 PrimType ElemT = LHS.getFieldDesc()->getPrimType();
3594
3595 for (unsigned ElemNum = 0; ElemNum < VectorLen; ++ElemNum) {
3596 APSInt A, B;
3598 A = LHS.elem<T>(ElemNum).toAPSInt();
3599 B = RHS.elem<T>(ElemNum).toAPSInt();
3600 });
3601 RetMask.setBitVal(ElemNum,
3602 Mask[ElemNum] && evalICmpImm(CmpOp, A, B, IsUnsigned));
3603 }
3604 pushInteger(S, RetMask, Call->getType());
3605 return true;
3606}
3607
3609 const CallExpr *Call) {
3610 assert(Call->getNumArgs() == 1);
3611
3612 QualType Arg0Type = Call->getArg(0)->getType();
3613 const auto *VecT = Arg0Type->castAs<VectorType>();
3614 PrimType ElemT = *S.getContext().classify(VecT->getElementType());
3615 unsigned NumElems = VecT->getNumElements();
3616 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3617 const Pointer &Src = S.Stk.pop<Pointer>();
3618 const Pointer &Dst = S.Stk.peek<Pointer>();
3619
3620 for (unsigned I = 0; I != NumElems; ++I) {
3622 APSInt ElemI = Src.elem<T>(I).toAPSInt();
3623 APInt ConflictMask(ElemI.getBitWidth(), 0);
3624 for (unsigned J = 0; J != I; ++J) {
3625 APSInt ElemJ = Src.elem<T>(J).toAPSInt();
3626 ConflictMask.setBitVal(J, ElemI == ElemJ);
3627 }
3628 Dst.elem<T>(I) = static_cast<T>(APSInt(ConflictMask, DestUnsigned));
3629 });
3630 }
3632 return true;
3633}
3634
3636 const CallExpr *Call,
3637 unsigned ID) {
3638 assert(Call->getNumArgs() == 1);
3639
3640 const Pointer &Vec = S.Stk.pop<Pointer>();
3641 unsigned RetWidth = S.getASTContext().getIntWidth(Call->getType());
3642 APInt RetMask(RetWidth, 0);
3643
3644 unsigned VectorLen = Vec.getNumElems();
3645 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3646
3647 for (unsigned ElemNum = 0; ElemNum != VectorLen; ++ElemNum) {
3648 APSInt A;
3649 INT_TYPE_SWITCH_NO_BOOL(ElemT, { A = Vec.elem<T>(ElemNum).toAPSInt(); });
3650 unsigned MSB = A[A.getBitWidth() - 1];
3651 RetMask.setBitVal(ElemNum, MSB);
3652 }
3653 pushInteger(S, RetMask, Call->getType());
3654 return true;
3655}
3656
3658 const CallExpr *Call,
3659 unsigned ID) {
3660 assert(Call->getNumArgs() == 1);
3661
3662 APSInt Mask = popToAPSInt(S, Call->getArg(0));
3663
3664 const Pointer &Vec = S.Stk.peek<Pointer>();
3665 unsigned NumElems = Vec.getNumElems();
3666 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3667
3668 for (unsigned I = 0; I != NumElems; ++I) {
3669 bool BitSet = Mask[I];
3670
3672 ElemT, { Vec.elem<T>(I) = BitSet ? T::from(-1) : T::from(0); });
3673 }
3674
3676
3677 return true;
3678}
3679
3681 const CallExpr *Call,
3682 bool HasRoundingMask) {
3683 APSInt Rounding, MaskInt;
3684 Pointer Src, B, A;
3685
3686 if (HasRoundingMask) {
3687 assert(Call->getNumArgs() == 5);
3688 Rounding = popToAPSInt(S, Call->getArg(4));
3689 MaskInt = popToAPSInt(S, Call->getArg(3));
3690 Src = S.Stk.pop<Pointer>();
3691 B = S.Stk.pop<Pointer>();
3692 A = S.Stk.pop<Pointer>();
3693 if (!CheckLoad(S, OpPC, A) || !CheckLoad(S, OpPC, B) ||
3694 !CheckLoad(S, OpPC, Src))
3695 return false;
3696 } else {
3697 assert(Call->getNumArgs() == 2);
3698 B = S.Stk.pop<Pointer>();
3699 A = S.Stk.pop<Pointer>();
3700 if (!CheckLoad(S, OpPC, A) || !CheckLoad(S, OpPC, B))
3701 return false;
3702 }
3703
3704 const auto *DstVTy = Call->getType()->castAs<VectorType>();
3705 unsigned NumElems = DstVTy->getNumElements();
3706 const Pointer &Dst = S.Stk.peek<Pointer>();
3707
3708 // Copy all elements except lane 0 (overwritten below) from A to Dst.
3709 for (unsigned I = 1; I != NumElems; ++I)
3710 Dst.elem<Floating>(I) = A.elem<Floating>(I);
3711
3712 // Convert element 0 from double to float, or use Src if masked off.
3713 if (!HasRoundingMask || (MaskInt.getZExtValue() & 0x1)) {
3714 assert(S.getASTContext().FloatTy == DstVTy->getElementType() &&
3715 "cvtsd2ss requires float element type in destination vector");
3716
3717 Floating Conv = S.allocFloat(
3718 S.getASTContext().getFloatTypeSemantics(DstVTy->getElementType()));
3719 APFloat SrcVal = B.elem<Floating>(0).getAPFloat();
3720 if (!convertDoubleToFloatStrict(SrcVal, Conv, S, Call))
3721 return false;
3722 Dst.elem<Floating>(0) = Conv;
3723 } else {
3724 Dst.elem<Floating>(0) = Src.elem<Floating>(0);
3725 }
3726
3728 return true;
3729}
3730
3732 const CallExpr *Call, bool IsMasked,
3733 bool HasRounding) {
3734
3735 APSInt MaskVal;
3736 Pointer PassThrough;
3737 Pointer Src;
3738 APSInt Rounding;
3739
3740 if (IsMasked) {
3741 // Pop in reverse order.
3742 if (HasRounding) {
3743 Rounding = popToAPSInt(S, Call->getArg(3));
3744 MaskVal = popToAPSInt(S, Call->getArg(2));
3745 PassThrough = S.Stk.pop<Pointer>();
3746 Src = S.Stk.pop<Pointer>();
3747 } else {
3748 MaskVal = popToAPSInt(S, Call->getArg(2));
3749 PassThrough = S.Stk.pop<Pointer>();
3750 Src = S.Stk.pop<Pointer>();
3751 }
3752
3753 if (!CheckLoad(S, OpPC, PassThrough))
3754 return false;
3755 } else {
3756 // Pop source only.
3757 Src = S.Stk.pop<Pointer>();
3758 }
3759
3760 if (!CheckLoad(S, OpPC, Src))
3761 return false;
3762
3763 const auto *RetVTy = Call->getType()->castAs<VectorType>();
3764 unsigned RetElems = RetVTy->getNumElements();
3765 unsigned SrcElems = Src.getNumElems();
3766 const Pointer &Dst = S.Stk.peek<Pointer>();
3767
3768 // Initialize destination with passthrough or zeros.
3769 for (unsigned I = 0; I != RetElems; ++I)
3770 if (IsMasked)
3771 Dst.elem<Floating>(I) = PassThrough.elem<Floating>(I);
3772 else
3773 Dst.elem<Floating>(I) = Floating(APFloat(0.0f));
3774
3775 assert(S.getASTContext().FloatTy == RetVTy->getElementType() &&
3776 "cvtpd2ps requires float element type in return vector");
3777
3778 // Convert double to float for enabled elements (only process source elements
3779 // that exist).
3780 for (unsigned I = 0; I != SrcElems; ++I) {
3781 if (IsMasked && !MaskVal[I])
3782 continue;
3783
3784 APFloat SrcVal = Src.elem<Floating>(I).getAPFloat();
3785
3786 Floating Conv = S.allocFloat(
3787 S.getASTContext().getFloatTypeSemantics(RetVTy->getElementType()));
3788 if (!convertDoubleToFloatStrict(SrcVal, Conv, S, Call))
3789 return false;
3790 Dst.elem<Floating>(I) = Conv;
3791 }
3792
3794 return true;
3795}
3796
3798 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3799 llvm::function_ref<std::pair<unsigned, int>(unsigned, const APInt &)>
3800 GetSourceIndex) {
3801
3802 assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
3803
3804 APInt ShuffleMask;
3805 Pointer A, MaskVector, B;
3806 bool IsVectorMask = false;
3807 bool IsSingleOperand = (Call->getNumArgs() == 2);
3808
3809 if (IsSingleOperand) {
3810 QualType MaskType = Call->getArg(1)->getType();
3811 if (MaskType->isVectorType()) {
3812 IsVectorMask = true;
3813 MaskVector = S.Stk.pop<Pointer>();
3814 A = S.Stk.pop<Pointer>();
3815 B = A;
3816 } else if (MaskType->isIntegerType()) {
3817 ShuffleMask = popToAPSInt(S, Call->getArg(1));
3818 A = S.Stk.pop<Pointer>();
3819 B = A;
3820 } else {
3821 return false;
3822 }
3823 } else {
3824 QualType Arg2Type = Call->getArg(2)->getType();
3825 if (Arg2Type->isVectorType()) {
3826 IsVectorMask = true;
3827 B = S.Stk.pop<Pointer>();
3828 MaskVector = S.Stk.pop<Pointer>();
3829 A = S.Stk.pop<Pointer>();
3830 } else if (Arg2Type->isIntegerType()) {
3831 ShuffleMask = popToAPSInt(S, Call->getArg(2));
3832 B = S.Stk.pop<Pointer>();
3833 A = S.Stk.pop<Pointer>();
3834 } else {
3835 return false;
3836 }
3837 }
3838
3839 QualType Arg0Type = Call->getArg(0)->getType();
3840 const auto *VecT = Arg0Type->castAs<VectorType>();
3841 PrimType ElemT = *S.getContext().classify(VecT->getElementType());
3842 unsigned NumElems = VecT->getNumElements();
3843
3844 const Pointer &Dst = S.Stk.peek<Pointer>();
3845
3846 PrimType MaskElemT = PT_Uint32;
3847 if (IsVectorMask) {
3848 QualType Arg1Type = Call->getArg(1)->getType();
3849 const auto *MaskVecT = Arg1Type->castAs<VectorType>();
3850 QualType MaskElemType = MaskVecT->getElementType();
3851 MaskElemT = *S.getContext().classify(MaskElemType);
3852 }
3853
3854 for (unsigned DstIdx = 0; DstIdx != NumElems; ++DstIdx) {
3855 if (IsVectorMask) {
3856 INT_TYPE_SWITCH(MaskElemT,
3857 { ShuffleMask = MaskVector.elem<T>(DstIdx).toAPSInt(); });
3858 }
3859
3860 auto [SrcVecIdx, SrcIdx] = GetSourceIndex(DstIdx, ShuffleMask);
3861
3862 if (SrcIdx < 0) {
3863 // Zero out this element
3864 if (ElemT == PT_Float) {
3865 Dst.elem<Floating>(DstIdx) = Floating(
3866 S.getASTContext().getFloatTypeSemantics(VecT->getElementType()));
3867 } else {
3868 INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(DstIdx) = T::from(0); });
3869 }
3870 } else {
3871 const Pointer &Src = (SrcVecIdx == 0) ? A : B;
3872 TYPE_SWITCH(ElemT, { Dst.elem<T>(DstIdx) = Src.elem<T>(SrcIdx); });
3873 }
3874 }
3876
3877 return true;
3878}
3879
3881 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3882 llvm::function_ref<std::pair<unsigned, int>(unsigned, unsigned)>
3883 GetSourceIndex) {
3885 S, OpPC, Call,
3886 [&GetSourceIndex](unsigned DstIdx,
3887 const APInt &Mask) -> std::pair<unsigned, int> {
3888 return GetSourceIndex(DstIdx, Mask.getZExtValue());
3889 });
3890}
3891
3893 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3894 llvm::function_ref<APInt(const APInt &, uint64_t)> ShiftOp,
3895 llvm::function_ref<APInt(const APInt &, unsigned)> OverflowOp) {
3896
3897 assert(Call->getNumArgs() == 2);
3898
3899 const Pointer &Count = S.Stk.pop<Pointer>();
3900 const Pointer &Source = S.Stk.pop<Pointer>();
3901
3902 QualType SourceType = Call->getArg(0)->getType();
3903 QualType CountType = Call->getArg(1)->getType();
3904 assert(SourceType->isVectorType() && CountType->isVectorType());
3905
3906 const auto *SourceVecT = SourceType->castAs<VectorType>();
3907 const auto *CountVecT = CountType->castAs<VectorType>();
3908 PrimType SourceElemT = *S.getContext().classify(SourceVecT->getElementType());
3909 PrimType CountElemT = *S.getContext().classify(CountVecT->getElementType());
3910
3911 const Pointer &Dst = S.Stk.peek<Pointer>();
3912
3913 unsigned DestEltWidth =
3914 S.getASTContext().getTypeSize(SourceVecT->getElementType());
3915 bool IsDestUnsigned = SourceVecT->getElementType()->isUnsignedIntegerType();
3916 unsigned DestLen = SourceVecT->getNumElements();
3917 unsigned CountEltWidth =
3918 S.getASTContext().getTypeSize(CountVecT->getElementType());
3919 unsigned NumBitsInQWord = 64;
3920 unsigned NumCountElts = NumBitsInQWord / CountEltWidth;
3921
3922 uint64_t CountLQWord = 0;
3923 for (unsigned EltIdx = 0; EltIdx != NumCountElts; ++EltIdx) {
3924 uint64_t Elt = 0;
3925 INT_TYPE_SWITCH(CountElemT,
3926 { Elt = static_cast<uint64_t>(Count.elem<T>(EltIdx)); });
3927 CountLQWord |= (Elt << (EltIdx * CountEltWidth));
3928 }
3929
3930 for (unsigned EltIdx = 0; EltIdx != DestLen; ++EltIdx) {
3931 APSInt Elt;
3932 INT_TYPE_SWITCH(SourceElemT, { Elt = Source.elem<T>(EltIdx).toAPSInt(); });
3933
3934 APInt Result;
3935 if (CountLQWord < DestEltWidth) {
3936 Result = ShiftOp(Elt, CountLQWord);
3937 } else {
3938 Result = OverflowOp(Elt, DestEltWidth);
3939 }
3940 if (IsDestUnsigned) {
3941 INT_TYPE_SWITCH(SourceElemT, {
3942 Dst.elem<T>(EltIdx) = T::from(Result.getZExtValue());
3943 });
3944 } else {
3945 INT_TYPE_SWITCH(SourceElemT, {
3946 Dst.elem<T>(EltIdx) = T::from(Result.getSExtValue());
3947 });
3948 }
3949 }
3950
3952 return true;
3953}
3954
3956 const CallExpr *Call) {
3957
3958 assert(Call->getNumArgs() == 3);
3959
3960 QualType SourceType = Call->getArg(0)->getType();
3961 QualType ShuffleMaskType = Call->getArg(1)->getType();
3962 QualType ZeroMaskType = Call->getArg(2)->getType();
3963 if (!SourceType->isVectorType() || !ShuffleMaskType->isVectorType() ||
3964 !ZeroMaskType->isIntegerType()) {
3965 return false;
3966 }
3967
3968 Pointer Source, ShuffleMask;
3969 APSInt ZeroMask = popToAPSInt(S, Call->getArg(2));
3970 ShuffleMask = S.Stk.pop<Pointer>();
3971 Source = S.Stk.pop<Pointer>();
3972
3973 const auto *SourceVecT = SourceType->castAs<VectorType>();
3974 const auto *ShuffleMaskVecT = ShuffleMaskType->castAs<VectorType>();
3975 assert(SourceVecT->getNumElements() == ShuffleMaskVecT->getNumElements());
3976 assert(ZeroMask.getBitWidth() == SourceVecT->getNumElements());
3977
3978 PrimType SourceElemT = *S.getContext().classify(SourceVecT->getElementType());
3979 PrimType ShuffleMaskElemT =
3980 *S.getContext().classify(ShuffleMaskVecT->getElementType());
3981
3982 unsigned NumBytesInQWord = 8;
3983 unsigned NumBitsInByte = 8;
3984 unsigned NumBytes = SourceVecT->getNumElements();
3985 unsigned NumQWords = NumBytes / NumBytesInQWord;
3986 unsigned RetWidth = ZeroMask.getBitWidth();
3987 APSInt RetMask(llvm::APInt(RetWidth, 0), /*isUnsigned=*/true);
3988
3989 for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
3990 APInt SourceQWord(64, 0);
3991 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
3992 uint64_t Byte = 0;
3993 INT_TYPE_SWITCH(SourceElemT, {
3994 Byte = static_cast<uint64_t>(
3995 Source.elem<T>(QWordId * NumBytesInQWord + ByteIdx));
3996 });
3997 SourceQWord.insertBits(APInt(8, Byte & 0xFF), ByteIdx * NumBitsInByte);
3998 }
3999
4000 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4001 unsigned SelIdx = QWordId * NumBytesInQWord + ByteIdx;
4002 unsigned M = 0;
4003 INT_TYPE_SWITCH(ShuffleMaskElemT, {
4004 M = static_cast<unsigned>(ShuffleMask.elem<T>(SelIdx)) & 0x3F;
4005 });
4006
4007 if (ZeroMask[SelIdx]) {
4008 RetMask.setBitVal(SelIdx, SourceQWord[M]);
4009 }
4010 }
4011 }
4012
4013 pushInteger(S, RetMask, Call->getType());
4014 return true;
4015}
4016
4018 const CallExpr *Call) {
4019 // Arguments are: vector of floats, rounding immediate
4020 assert(Call->getNumArgs() == 2);
4021
4022 APSInt Imm = popToAPSInt(S, Call->getArg(1));
4023 const Pointer &Src = S.Stk.pop<Pointer>();
4024 const Pointer &Dst = S.Stk.peek<Pointer>();
4025
4026 assert(Src.getFieldDesc()->isPrimitiveArray());
4027 assert(Dst.getFieldDesc()->isPrimitiveArray());
4028
4029 const auto *SrcVTy = Call->getArg(0)->getType()->castAs<VectorType>();
4030 unsigned SrcNumElems = SrcVTy->getNumElements();
4031 const auto *DstVTy = Call->getType()->castAs<VectorType>();
4032 unsigned DstNumElems = DstVTy->getNumElements();
4033
4034 const llvm::fltSemantics &HalfSem =
4036
4037 // imm[2] == 1 means use MXCSR rounding mode.
4038 // In that case, we can only evaluate if the conversion is exact.
4039 int ImmVal = Imm.getZExtValue();
4040 bool UseMXCSR = (ImmVal & 4) != 0;
4041 bool IsFPConstrained =
4042 Call->getFPFeaturesInEffect(S.getASTContext().getLangOpts())
4043 .isFPConstrained();
4044
4045 llvm::RoundingMode RM;
4046 if (!UseMXCSR) {
4047 switch (ImmVal & 3) {
4048 case 0:
4049 RM = llvm::RoundingMode::NearestTiesToEven;
4050 break;
4051 case 1:
4052 RM = llvm::RoundingMode::TowardNegative;
4053 break;
4054 case 2:
4055 RM = llvm::RoundingMode::TowardPositive;
4056 break;
4057 case 3:
4058 RM = llvm::RoundingMode::TowardZero;
4059 break;
4060 default:
4061 llvm_unreachable("Invalid immediate rounding mode");
4062 }
4063 } else {
4064 // For MXCSR, we must check for exactness. We can use any rounding mode
4065 // for the trial conversion since the result is the same if it's exact.
4066 RM = llvm::RoundingMode::NearestTiesToEven;
4067 }
4068
4069 QualType DstElemQT = Dst.getFieldDesc()->getElemQualType();
4070 PrimType DstElemT = *S.getContext().classify(DstElemQT);
4071
4072 for (unsigned I = 0; I != SrcNumElems; ++I) {
4073 Floating SrcVal = Src.elem<Floating>(I);
4074 APFloat DstVal = SrcVal.getAPFloat();
4075
4076 bool LostInfo;
4077 APFloat::opStatus St = DstVal.convert(HalfSem, RM, &LostInfo);
4078
4079 if (UseMXCSR && IsFPConstrained && St != APFloat::opOK) {
4080 S.FFDiag(S.Current->getSource(OpPC),
4081 diag::note_constexpr_dynamic_rounding);
4082 return false;
4083 }
4084
4085 INT_TYPE_SWITCH_NO_BOOL(DstElemT, {
4086 // Convert the destination value's bit pattern to an unsigned integer,
4087 // then reconstruct the element using the target type's 'from' method.
4088 uint64_t RawBits = DstVal.bitcastToAPInt().getZExtValue();
4089 Dst.elem<T>(I) = T::from(RawBits);
4090 });
4091 }
4092
4093 // Zero out remaining elements if the destination has more elements
4094 // (e.g., vcvtps2ph converting 4 floats to 8 shorts).
4095 if (DstNumElems > SrcNumElems) {
4096 for (unsigned I = SrcNumElems; I != DstNumElems; ++I) {
4097 INT_TYPE_SWITCH_NO_BOOL(DstElemT, { Dst.elem<T>(I) = T::from(0); });
4098 }
4099 }
4100
4101 Dst.initializeAllElements();
4102 return true;
4103}
4104
4106 const CallExpr *Call) {
4107 assert(Call->getNumArgs() == 2);
4108
4109 QualType ATy = Call->getArg(0)->getType();
4110 QualType BTy = Call->getArg(1)->getType();
4111 if (!ATy->isVectorType() || !BTy->isVectorType()) {
4112 return false;
4113 }
4114
4115 const Pointer &BPtr = S.Stk.pop<Pointer>();
4116 const Pointer &APtr = S.Stk.pop<Pointer>();
4117 const auto *AVecT = ATy->castAs<VectorType>();
4118 assert(AVecT->getNumElements() ==
4119 BTy->castAs<VectorType>()->getNumElements());
4120
4121 PrimType ElemT = *S.getContext().classify(AVecT->getElementType());
4122
4123 unsigned NumBytesInQWord = 8;
4124 unsigned NumBitsInByte = 8;
4125 unsigned NumBytes = AVecT->getNumElements();
4126 unsigned NumQWords = NumBytes / NumBytesInQWord;
4127 const Pointer &Dst = S.Stk.peek<Pointer>();
4128
4129 for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
4130 APInt BQWord(64, 0);
4131 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4132 unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
4133 INT_TYPE_SWITCH(ElemT, {
4134 uint64_t Byte = static_cast<uint64_t>(BPtr.elem<T>(Idx));
4135 BQWord.insertBits(APInt(8, Byte & 0xFF), ByteIdx * NumBitsInByte);
4136 });
4137 }
4138
4139 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4140 unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
4141 uint64_t Ctrl = 0;
4143 ElemT, { Ctrl = static_cast<uint64_t>(APtr.elem<T>(Idx)) & 0x3F; });
4144
4145 APInt Byte(8, 0);
4146 for (unsigned BitIdx = 0; BitIdx != NumBitsInByte; ++BitIdx) {
4147 Byte.setBitVal(BitIdx, BQWord[(Ctrl + BitIdx) & 0x3F]);
4148 }
4149 INT_TYPE_SWITCH(ElemT,
4150 { Dst.elem<T>(Idx) = T::from(Byte.getZExtValue()); });
4151 }
4152 }
4153
4155
4156 return true;
4157}
4158
4160 const CallExpr *Call,
4161 bool Inverse) {
4162 assert(Call->getNumArgs() == 3);
4163 QualType XType = Call->getArg(0)->getType();
4164 QualType AType = Call->getArg(1)->getType();
4165 QualType ImmType = Call->getArg(2)->getType();
4166 if (!XType->isVectorType() || !AType->isVectorType() ||
4167 !ImmType->isIntegerType()) {
4168 return false;
4169 }
4170
4171 Pointer X, A;
4172 APSInt Imm = popToAPSInt(S, Call->getArg(2));
4173 A = S.Stk.pop<Pointer>();
4174 X = S.Stk.pop<Pointer>();
4175
4176 const Pointer &Dst = S.Stk.peek<Pointer>();
4177 const auto *AVecT = AType->castAs<VectorType>();
4178 assert(XType->castAs<VectorType>()->getNumElements() ==
4179 AVecT->getNumElements());
4180 unsigned NumBytesInQWord = 8;
4181 unsigned NumBytes = AVecT->getNumElements();
4182 unsigned NumBitsInQWord = 64;
4183 unsigned NumQWords = NumBytes / NumBytesInQWord;
4184 unsigned NumBitsInByte = 8;
4185 PrimType AElemT = *S.getContext().classify(AVecT->getElementType());
4186
4187 // computing A*X + Imm
4188 for (unsigned QWordIdx = 0; QWordIdx != NumQWords; ++QWordIdx) {
4189 // Extract the QWords from X, A
4190 APInt XQWord(NumBitsInQWord, 0);
4191 APInt AQWord(NumBitsInQWord, 0);
4192 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4193 unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
4194 uint8_t XByte;
4195 uint8_t AByte;
4196 INT_TYPE_SWITCH(AElemT, {
4197 XByte = static_cast<uint8_t>(X.elem<T>(Idx));
4198 AByte = static_cast<uint8_t>(A.elem<T>(Idx));
4199 });
4200
4201 XQWord.insertBits(APInt(NumBitsInByte, XByte), ByteIdx * NumBitsInByte);
4202 AQWord.insertBits(APInt(NumBitsInByte, AByte), ByteIdx * NumBitsInByte);
4203 }
4204
4205 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4206 unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
4207 uint8_t XByte =
4208 XQWord.lshr(ByteIdx * NumBitsInByte).getLoBits(8).getZExtValue();
4209 INT_TYPE_SWITCH(AElemT, {
4210 Dst.elem<T>(Idx) = T::from(GFNIAffine(XByte, AQWord, Imm, Inverse));
4211 });
4212 }
4213 }
4214 Dst.initializeAllElements();
4215 return true;
4216}
4217
4219 const CallExpr *Call) {
4220 assert(Call->getNumArgs() == 2);
4221
4222 QualType AType = Call->getArg(0)->getType();
4223 QualType BType = Call->getArg(1)->getType();
4224 if (!AType->isVectorType() || !BType->isVectorType()) {
4225 return false;
4226 }
4227
4228 Pointer A, B;
4229 B = S.Stk.pop<Pointer>();
4230 A = S.Stk.pop<Pointer>();
4231
4232 const Pointer &Dst = S.Stk.peek<Pointer>();
4233 const auto *AVecT = AType->castAs<VectorType>();
4234 assert(AVecT->getNumElements() ==
4235 BType->castAs<VectorType>()->getNumElements());
4236
4237 PrimType AElemT = *S.getContext().classify(AVecT->getElementType());
4238 unsigned NumBytes = A.getNumElems();
4239
4240 for (unsigned ByteIdx = 0; ByteIdx != NumBytes; ++ByteIdx) {
4241 uint8_t AByte, BByte;
4242 INT_TYPE_SWITCH(AElemT, {
4243 AByte = static_cast<uint8_t>(A.elem<T>(ByteIdx));
4244 BByte = static_cast<uint8_t>(B.elem<T>(ByteIdx));
4245 Dst.elem<T>(ByteIdx) = T::from(GFNIMul(AByte, BByte));
4246 });
4247 }
4248
4249 Dst.initializeAllElements();
4250 return true;
4251}
4252
4254 uint32_t BuiltinID) {
4255 if (!S.getASTContext().BuiltinInfo.isConstantEvaluated(BuiltinID))
4256 return Invalid(S, OpPC);
4257
4258 const InterpFrame *Frame = S.Current;
4259 switch (BuiltinID) {
4260 case Builtin::BI__builtin_is_constant_evaluated:
4262
4263 case Builtin::BI__builtin_assume:
4264 case Builtin::BI__assume:
4265 return interp__builtin_assume(S, OpPC, Frame, Call);
4266
4267 case Builtin::BI__builtin_strcmp:
4268 case Builtin::BIstrcmp:
4269 case Builtin::BI__builtin_strncmp:
4270 case Builtin::BIstrncmp:
4271 case Builtin::BI__builtin_wcsncmp:
4272 case Builtin::BIwcsncmp:
4273 case Builtin::BI__builtin_wcscmp:
4274 case Builtin::BIwcscmp:
4275 return interp__builtin_strcmp(S, OpPC, Frame, Call, BuiltinID);
4276
4277 case Builtin::BI__builtin_strlen:
4278 case Builtin::BIstrlen:
4279 case Builtin::BI__builtin_wcslen:
4280 case Builtin::BIwcslen:
4281 return interp__builtin_strlen(S, OpPC, Frame, Call, BuiltinID);
4282
4283 case Builtin::BI__builtin_nan:
4284 case Builtin::BI__builtin_nanf:
4285 case Builtin::BI__builtin_nanl:
4286 case Builtin::BI__builtin_nanf16:
4287 case Builtin::BI__builtin_nanf128:
4288 return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/false);
4289
4290 case Builtin::BI__builtin_nans:
4291 case Builtin::BI__builtin_nansf:
4292 case Builtin::BI__builtin_nansl:
4293 case Builtin::BI__builtin_nansf16:
4294 case Builtin::BI__builtin_nansf128:
4295 return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/true);
4296
4297 case Builtin::BI__builtin_huge_val:
4298 case Builtin::BI__builtin_huge_valf:
4299 case Builtin::BI__builtin_huge_vall:
4300 case Builtin::BI__builtin_huge_valf16:
4301 case Builtin::BI__builtin_huge_valf128:
4302 case Builtin::BI__builtin_inf:
4303 case Builtin::BI__builtin_inff:
4304 case Builtin::BI__builtin_infl:
4305 case Builtin::BI__builtin_inff16:
4306 case Builtin::BI__builtin_inff128:
4307 return interp__builtin_inf(S, OpPC, Frame, Call);
4308
4309 case Builtin::BI__builtin_copysign:
4310 case Builtin::BI__builtin_copysignf:
4311 case Builtin::BI__builtin_copysignl:
4312 case Builtin::BI__builtin_copysignf128:
4313 return interp__builtin_copysign(S, OpPC, Frame);
4314
4315 case Builtin::BI__builtin_fmin:
4316 case Builtin::BI__builtin_fminf:
4317 case Builtin::BI__builtin_fminl:
4318 case Builtin::BI__builtin_fminf16:
4319 case Builtin::BI__builtin_fminf128:
4320 return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/false);
4321
4322 case Builtin::BI__builtin_fminimum_num:
4323 case Builtin::BI__builtin_fminimum_numf:
4324 case Builtin::BI__builtin_fminimum_numl:
4325 case Builtin::BI__builtin_fminimum_numf16:
4326 case Builtin::BI__builtin_fminimum_numf128:
4327 return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/true);
4328
4329 case Builtin::BI__builtin_fmax:
4330 case Builtin::BI__builtin_fmaxf:
4331 case Builtin::BI__builtin_fmaxl:
4332 case Builtin::BI__builtin_fmaxf16:
4333 case Builtin::BI__builtin_fmaxf128:
4334 return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/false);
4335
4336 case Builtin::BI__builtin_fmaximum_num:
4337 case Builtin::BI__builtin_fmaximum_numf:
4338 case Builtin::BI__builtin_fmaximum_numl:
4339 case Builtin::BI__builtin_fmaximum_numf16:
4340 case Builtin::BI__builtin_fmaximum_numf128:
4341 return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/true);
4342
4343 case Builtin::BI__builtin_isnan:
4344 return interp__builtin_isnan(S, OpPC, Frame, Call);
4345
4346 case Builtin::BI__builtin_issignaling:
4347 return interp__builtin_issignaling(S, OpPC, Frame, Call);
4348
4349 case Builtin::BI__builtin_isinf:
4350 return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/false, Call);
4351
4352 case Builtin::BI__builtin_isinf_sign:
4353 return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/true, Call);
4354
4355 case Builtin::BI__builtin_isfinite:
4356 return interp__builtin_isfinite(S, OpPC, Frame, Call);
4357
4358 case Builtin::BI__builtin_isnormal:
4359 return interp__builtin_isnormal(S, OpPC, Frame, Call);
4360
4361 case Builtin::BI__builtin_issubnormal:
4362 return interp__builtin_issubnormal(S, OpPC, Frame, Call);
4363
4364 case Builtin::BI__builtin_iszero:
4365 return interp__builtin_iszero(S, OpPC, Frame, Call);
4366
4367 case Builtin::BI__builtin_signbit:
4368 case Builtin::BI__builtin_signbitf:
4369 case Builtin::BI__builtin_signbitl:
4370 return interp__builtin_signbit(S, OpPC, Frame, Call);
4371
4372 case Builtin::BI__builtin_isgreater:
4373 case Builtin::BI__builtin_isgreaterequal:
4374 case Builtin::BI__builtin_isless:
4375 case Builtin::BI__builtin_islessequal:
4376 case Builtin::BI__builtin_islessgreater:
4377 case Builtin::BI__builtin_isunordered:
4378 return interp_floating_comparison(S, OpPC, Call, BuiltinID);
4379
4380 case Builtin::BI__builtin_isfpclass:
4381 return interp__builtin_isfpclass(S, OpPC, Frame, Call);
4382
4383 case Builtin::BI__builtin_fpclassify:
4384 return interp__builtin_fpclassify(S, OpPC, Frame, Call);
4385
4386 case Builtin::BI__builtin_fabs:
4387 case Builtin::BI__builtin_fabsf:
4388 case Builtin::BI__builtin_fabsl:
4389 case Builtin::BI__builtin_fabsf128:
4390 return interp__builtin_fabs(S, OpPC, Frame);
4391
4392 case Builtin::BI__builtin_abs:
4393 case Builtin::BI__builtin_labs:
4394 case Builtin::BI__builtin_llabs:
4395 return interp__builtin_abs(S, OpPC, Frame, Call);
4396
4397 case Builtin::BI__builtin_popcount:
4398 case Builtin::BI__builtin_popcountl:
4399 case Builtin::BI__builtin_popcountll:
4400 case Builtin::BI__builtin_popcountg:
4401 case Builtin::BI__popcnt16: // Microsoft variants of popcount
4402 case Builtin::BI__popcnt:
4403 case Builtin::BI__popcnt64:
4404 return interp__builtin_popcount(S, OpPC, Frame, Call);
4405
4406 case Builtin::BI__builtin_parity:
4407 case Builtin::BI__builtin_parityl:
4408 case Builtin::BI__builtin_parityll:
4410 S, OpPC, Call, [](const APSInt &Val) {
4411 return APInt(Val.getBitWidth(), Val.popcount() % 2);
4412 });
4413 case Builtin::BI__builtin_clrsb:
4414 case Builtin::BI__builtin_clrsbl:
4415 case Builtin::BI__builtin_clrsbll:
4417 S, OpPC, Call, [](const APSInt &Val) {
4418 return APInt(Val.getBitWidth(),
4419 Val.getBitWidth() - Val.getSignificantBits());
4420 });
4421 case Builtin::BI__builtin_bitreverseg:
4422 case Builtin::BI__builtin_bitreverse8:
4423 case Builtin::BI__builtin_bitreverse16:
4424 case Builtin::BI__builtin_bitreverse32:
4425 case Builtin::BI__builtin_bitreverse64:
4427 S, OpPC, Call, [](const APSInt &Val) { return Val.reverseBits(); });
4428
4429 case Builtin::BI__builtin_classify_type:
4430 return interp__builtin_classify_type(S, OpPC, Frame, Call);
4431
4432 case Builtin::BI__builtin_expect:
4433 case Builtin::BI__builtin_expect_with_probability:
4434 return interp__builtin_expect(S, OpPC, Frame, Call);
4435
4436 case Builtin::BI__builtin_rotateleft8:
4437 case Builtin::BI__builtin_rotateleft16:
4438 case Builtin::BI__builtin_rotateleft32:
4439 case Builtin::BI__builtin_rotateleft64:
4440 case Builtin::BI__builtin_stdc_rotate_left:
4441 case Builtin::BI_rotl8: // Microsoft variants of rotate left
4442 case Builtin::BI_rotl16:
4443 case Builtin::BI_rotl:
4444 case Builtin::BI_lrotl:
4445 case Builtin::BI_rotl64:
4446 case Builtin::BI__builtin_rotateright8:
4447 case Builtin::BI__builtin_rotateright16:
4448 case Builtin::BI__builtin_rotateright32:
4449 case Builtin::BI__builtin_rotateright64:
4450 case Builtin::BI__builtin_stdc_rotate_right:
4451 case Builtin::BI_rotr8: // Microsoft variants of rotate right
4452 case Builtin::BI_rotr16:
4453 case Builtin::BI_rotr:
4454 case Builtin::BI_lrotr:
4455 case Builtin::BI_rotr64: {
4456 // Determine if this is a rotate right operation
4457 bool IsRotateRight;
4458 switch (BuiltinID) {
4459 case Builtin::BI__builtin_rotateright8:
4460 case Builtin::BI__builtin_rotateright16:
4461 case Builtin::BI__builtin_rotateright32:
4462 case Builtin::BI__builtin_rotateright64:
4463 case Builtin::BI__builtin_stdc_rotate_right:
4464 case Builtin::BI_rotr8:
4465 case Builtin::BI_rotr16:
4466 case Builtin::BI_rotr:
4467 case Builtin::BI_lrotr:
4468 case Builtin::BI_rotr64:
4469 IsRotateRight = true;
4470 break;
4471 default:
4472 IsRotateRight = false;
4473 break;
4474 }
4475
4477 S, OpPC, Call, [IsRotateRight](const APSInt &Value, APSInt Amount) {
4478 Amount = NormalizeRotateAmount(Value, Amount);
4479 return IsRotateRight ? Value.rotr(Amount.getZExtValue())
4480 : Value.rotl(Amount.getZExtValue());
4481 });
4482 }
4483
4484 case Builtin::BIstdc_leading_zeros_uc:
4485 case Builtin::BIstdc_leading_zeros_us:
4486 case Builtin::BIstdc_leading_zeros_ui:
4487 case Builtin::BIstdc_leading_zeros_ul:
4488 case Builtin::BIstdc_leading_zeros_ull:
4489 case Builtin::BIstdc_leading_zeros:
4490 case Builtin::BI__builtin_stdc_leading_zeros: {
4491 unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
4493 S, OpPC, Call, [ResWidth](const APSInt &Val) {
4494 return APInt(ResWidth, Val.countl_zero());
4495 });
4496 }
4497
4498 case Builtin::BIstdc_leading_ones_uc:
4499 case Builtin::BIstdc_leading_ones_us:
4500 case Builtin::BIstdc_leading_ones_ui:
4501 case Builtin::BIstdc_leading_ones_ul:
4502 case Builtin::BIstdc_leading_ones_ull:
4503 case Builtin::BIstdc_leading_ones:
4504 case Builtin::BI__builtin_stdc_leading_ones: {
4505 unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
4507 S, OpPC, Call, [ResWidth](const APSInt &Val) {
4508 return APInt(ResWidth, Val.countl_one());
4509 });
4510 }
4511
4512 case Builtin::BIstdc_trailing_zeros_uc:
4513 case Builtin::BIstdc_trailing_zeros_us:
4514 case Builtin::BIstdc_trailing_zeros_ui:
4515 case Builtin::BIstdc_trailing_zeros_ul:
4516 case Builtin::BIstdc_trailing_zeros_ull:
4517 case Builtin::BIstdc_trailing_zeros:
4518 case Builtin::BI__builtin_stdc_trailing_zeros: {
4519 unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
4521 S, OpPC, Call, [ResWidth](const APSInt &Val) {
4522 return APInt(ResWidth, Val.countr_zero());
4523 });
4524 }
4525
4526 case Builtin::BIstdc_trailing_ones_uc:
4527 case Builtin::BIstdc_trailing_ones_us:
4528 case Builtin::BIstdc_trailing_ones_ui:
4529 case Builtin::BIstdc_trailing_ones_ul:
4530 case Builtin::BIstdc_trailing_ones_ull:
4531 case Builtin::BIstdc_trailing_ones:
4532 case Builtin::BI__builtin_stdc_trailing_ones: {
4533 unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
4535 S, OpPC, Call, [ResWidth](const APSInt &Val) {
4536 return APInt(ResWidth, Val.countr_one());
4537 });
4538 }
4539
4540 case Builtin::BIstdc_first_leading_zero_uc:
4541 case Builtin::BIstdc_first_leading_zero_us:
4542 case Builtin::BIstdc_first_leading_zero_ui:
4543 case Builtin::BIstdc_first_leading_zero_ul:
4544 case Builtin::BIstdc_first_leading_zero_ull:
4545 case Builtin::BIstdc_first_leading_zero:
4546 case Builtin::BI__builtin_stdc_first_leading_zero: {
4547 unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
4549 S, OpPC, Call, [ResWidth](const APSInt &Val) {
4550 return APInt(ResWidth, Val.isAllOnes() ? 0 : Val.countl_one() + 1);
4551 });
4552 }
4553
4554 case Builtin::BIstdc_first_leading_one_uc:
4555 case Builtin::BIstdc_first_leading_one_us:
4556 case Builtin::BIstdc_first_leading_one_ui:
4557 case Builtin::BIstdc_first_leading_one_ul:
4558 case Builtin::BIstdc_first_leading_one_ull:
4559 case Builtin::BIstdc_first_leading_one:
4560 case Builtin::BI__builtin_stdc_first_leading_one: {
4561 unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
4563 S, OpPC, Call, [ResWidth](const APSInt &Val) {
4564 return APInt(ResWidth, Val.isZero() ? 0 : Val.countl_zero() + 1);
4565 });
4566 }
4567
4568 case Builtin::BIstdc_first_trailing_zero_uc:
4569 case Builtin::BIstdc_first_trailing_zero_us:
4570 case Builtin::BIstdc_first_trailing_zero_ui:
4571 case Builtin::BIstdc_first_trailing_zero_ul:
4572 case Builtin::BIstdc_first_trailing_zero_ull:
4573 case Builtin::BIstdc_first_trailing_zero:
4574 case Builtin::BI__builtin_stdc_first_trailing_zero: {
4575 unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
4577 S, OpPC, Call, [ResWidth](const APSInt &Val) {
4578 return APInt(ResWidth, Val.isAllOnes() ? 0 : Val.countr_one() + 1);
4579 });
4580 }
4581
4582 case Builtin::BIstdc_first_trailing_one_uc:
4583 case Builtin::BIstdc_first_trailing_one_us:
4584 case Builtin::BIstdc_first_trailing_one_ui:
4585 case Builtin::BIstdc_first_trailing_one_ul:
4586 case Builtin::BIstdc_first_trailing_one_ull:
4587 case Builtin::BIstdc_first_trailing_one:
4588 case Builtin::BI__builtin_stdc_first_trailing_one: {
4589 unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
4591 S, OpPC, Call, [ResWidth](const APSInt &Val) {
4592 return APInt(ResWidth, Val.isZero() ? 0 : Val.countr_zero() + 1);
4593 });
4594 }
4595
4596 case Builtin::BIstdc_count_zeros_uc:
4597 case Builtin::BIstdc_count_zeros_us:
4598 case Builtin::BIstdc_count_zeros_ui:
4599 case Builtin::BIstdc_count_zeros_ul:
4600 case Builtin::BIstdc_count_zeros_ull:
4601 case Builtin::BIstdc_count_zeros:
4602 case Builtin::BI__builtin_stdc_count_zeros: {
4603 unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
4605 S, OpPC, Call, [ResWidth](const APSInt &Val) {
4606 unsigned BitWidth = Val.getBitWidth();
4607 return APInt(ResWidth, BitWidth - Val.popcount());
4608 });
4609 }
4610
4611 case Builtin::BIstdc_count_ones_uc:
4612 case Builtin::BIstdc_count_ones_us:
4613 case Builtin::BIstdc_count_ones_ui:
4614 case Builtin::BIstdc_count_ones_ul:
4615 case Builtin::BIstdc_count_ones_ull:
4616 case Builtin::BIstdc_count_ones:
4617 case Builtin::BI__builtin_stdc_count_ones: {
4618 unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
4620 S, OpPC, Call, [ResWidth](const APSInt &Val) {
4621 return APInt(ResWidth, Val.popcount());
4622 });
4623 }
4624
4625 case Builtin::BIstdc_has_single_bit_uc:
4626 case Builtin::BIstdc_has_single_bit_us:
4627 case Builtin::BIstdc_has_single_bit_ui:
4628 case Builtin::BIstdc_has_single_bit_ul:
4629 case Builtin::BIstdc_has_single_bit_ull:
4630 case Builtin::BIstdc_has_single_bit:
4631 case Builtin::BI__builtin_stdc_has_single_bit: {
4632 unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
4634 S, OpPC, Call, [ResWidth](const APSInt &Val) {
4635 return APInt(ResWidth, Val.popcount() == 1 ? 1 : 0);
4636 });
4637 }
4638
4639 case Builtin::BIstdc_bit_width_uc:
4640 case Builtin::BIstdc_bit_width_us:
4641 case Builtin::BIstdc_bit_width_ui:
4642 case Builtin::BIstdc_bit_width_ul:
4643 case Builtin::BIstdc_bit_width_ull:
4644 case Builtin::BIstdc_bit_width:
4645 case Builtin::BI__builtin_stdc_bit_width: {
4646 unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
4648 S, OpPC, Call, [ResWidth](const APSInt &Val) {
4649 unsigned BitWidth = Val.getBitWidth();
4650 return APInt(ResWidth, BitWidth - Val.countl_zero());
4651 });
4652 }
4653
4654 case Builtin::BIstdc_bit_floor_uc:
4655 case Builtin::BIstdc_bit_floor_us:
4656 case Builtin::BIstdc_bit_floor_ui:
4657 case Builtin::BIstdc_bit_floor_ul:
4658 case Builtin::BIstdc_bit_floor_ull:
4659 case Builtin::BIstdc_bit_floor:
4660 case Builtin::BI__builtin_stdc_bit_floor:
4662 S, OpPC, Call, [](const APSInt &Val) {
4663 unsigned BitWidth = Val.getBitWidth();
4664 if (Val.isZero())
4665 return APInt::getZero(BitWidth);
4666 return APInt::getOneBitSet(BitWidth,
4667 BitWidth - Val.countl_zero() - 1);
4668 });
4669
4670 case Builtin::BIstdc_bit_ceil_uc:
4671 case Builtin::BIstdc_bit_ceil_us:
4672 case Builtin::BIstdc_bit_ceil_ui:
4673 case Builtin::BIstdc_bit_ceil_ul:
4674 case Builtin::BIstdc_bit_ceil_ull:
4675 case Builtin::BIstdc_bit_ceil:
4676 case Builtin::BI__builtin_stdc_bit_ceil:
4678 S, OpPC, Call, [](const APSInt &Val) {
4679 unsigned BitWidth = Val.getBitWidth();
4680 if (Val.ule(1))
4681 return APInt(BitWidth, 1);
4682 APInt V = Val;
4683 APInt ValMinusOne = V - 1;
4684 unsigned LeadingZeros = ValMinusOne.countl_zero();
4685 if (LeadingZeros == 0)
4686 return APInt(BitWidth, 0); // overflows; wrap to 0
4687 return APInt::getOneBitSet(BitWidth, BitWidth - LeadingZeros);
4688 });
4689
4690 case Builtin::BI__builtin_ffs:
4691 case Builtin::BI__builtin_ffsl:
4692 case Builtin::BI__builtin_ffsll:
4694 S, OpPC, Call, [](const APSInt &Val) {
4695 return APInt(Val.getBitWidth(),
4696 Val.isZero() ? 0u : Val.countTrailingZeros() + 1u);
4697 });
4698
4699 case Builtin::BIaddressof:
4700 case Builtin::BI__addressof:
4701 case Builtin::BI__builtin_addressof:
4702 assert(isNoopBuiltin(BuiltinID));
4703 return interp__builtin_addressof(S, OpPC, Frame, Call);
4704
4705 case Builtin::BIas_const:
4706 case Builtin::BIforward:
4707 case Builtin::BIforward_like:
4708 case Builtin::BImove:
4709 case Builtin::BImove_if_noexcept:
4710 assert(isNoopBuiltin(BuiltinID));
4711 return interp__builtin_move(S, OpPC, Frame, Call);
4712
4713 case Builtin::BI__builtin_eh_return_data_regno:
4715
4716 case Builtin::BI__builtin_launder:
4717 assert(isNoopBuiltin(BuiltinID));
4718 return true;
4719
4720 case Builtin::BI__builtin_add_overflow:
4721 case Builtin::BI__builtin_sub_overflow:
4722 case Builtin::BI__builtin_mul_overflow:
4723 case Builtin::BI__builtin_sadd_overflow:
4724 case Builtin::BI__builtin_uadd_overflow:
4725 case Builtin::BI__builtin_uaddl_overflow:
4726 case Builtin::BI__builtin_uaddll_overflow:
4727 case Builtin::BI__builtin_usub_overflow:
4728 case Builtin::BI__builtin_usubl_overflow:
4729 case Builtin::BI__builtin_usubll_overflow:
4730 case Builtin::BI__builtin_umul_overflow:
4731 case Builtin::BI__builtin_umull_overflow:
4732 case Builtin::BI__builtin_umulll_overflow:
4733 case Builtin::BI__builtin_saddl_overflow:
4734 case Builtin::BI__builtin_saddll_overflow:
4735 case Builtin::BI__builtin_ssub_overflow:
4736 case Builtin::BI__builtin_ssubl_overflow:
4737 case Builtin::BI__builtin_ssubll_overflow:
4738 case Builtin::BI__builtin_smul_overflow:
4739 case Builtin::BI__builtin_smull_overflow:
4740 case Builtin::BI__builtin_smulll_overflow:
4741 return interp__builtin_overflowop(S, OpPC, Call, BuiltinID);
4742
4743 case Builtin::BI__builtin_addcb:
4744 case Builtin::BI__builtin_addcs:
4745 case Builtin::BI__builtin_addc:
4746 case Builtin::BI__builtin_addcl:
4747 case Builtin::BI__builtin_addcll:
4748 case Builtin::BI__builtin_subcb:
4749 case Builtin::BI__builtin_subcs:
4750 case Builtin::BI__builtin_subc:
4751 case Builtin::BI__builtin_subcl:
4752 case Builtin::BI__builtin_subcll:
4753 return interp__builtin_carryop(S, OpPC, Frame, Call, BuiltinID);
4754
4755 case Builtin::BI__builtin_clz:
4756 case Builtin::BI__builtin_clzl:
4757 case Builtin::BI__builtin_clzll:
4758 case Builtin::BI__builtin_clzs:
4759 case Builtin::BI__builtin_clzg:
4760 case Builtin::BI__lzcnt16: // Microsoft variants of count leading-zeroes
4761 case Builtin::BI__lzcnt:
4762 case Builtin::BI__lzcnt64:
4763 return interp__builtin_clz(S, OpPC, Frame, Call, BuiltinID);
4764
4765 case Builtin::BI__builtin_ctz:
4766 case Builtin::BI__builtin_ctzl:
4767 case Builtin::BI__builtin_ctzll:
4768 case Builtin::BI__builtin_ctzs:
4769 case Builtin::BI__builtin_ctzg:
4770 return interp__builtin_ctz(S, OpPC, Frame, Call, BuiltinID);
4771
4772 case Builtin::BI__builtin_elementwise_clzg:
4773 case Builtin::BI__builtin_elementwise_ctzg:
4775 BuiltinID);
4776 case Builtin::BI__builtin_bswapg:
4777 case Builtin::BI__builtin_bswap16:
4778 case Builtin::BI__builtin_bswap32:
4779 case Builtin::BI__builtin_bswap64:
4780 return interp__builtin_bswap(S, OpPC, Frame, Call);
4781
4782 case Builtin::BI__atomic_always_lock_free:
4783 case Builtin::BI__atomic_is_lock_free:
4784 return interp__builtin_atomic_lock_free(S, OpPC, Frame, Call, BuiltinID);
4785
4786 case Builtin::BI__c11_atomic_is_lock_free:
4788
4789 case Builtin::BI__builtin_complex:
4790 return interp__builtin_complex(S, OpPC, Frame, Call);
4791
4792 case Builtin::BI__builtin_is_aligned:
4793 case Builtin::BI__builtin_align_up:
4794 case Builtin::BI__builtin_align_down:
4795 return interp__builtin_is_aligned_up_down(S, OpPC, Frame, Call, BuiltinID);
4796
4797 case Builtin::BI__builtin_assume_aligned:
4798 return interp__builtin_assume_aligned(S, OpPC, Frame, Call);
4799
4800 case clang::X86::BI__builtin_ia32_crc32qi:
4801 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, 1);
4802 case clang::X86::BI__builtin_ia32_crc32hi:
4803 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, 2);
4804 case clang::X86::BI__builtin_ia32_crc32si:
4805 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, 4);
4806 case clang::X86::BI__builtin_ia32_crc32di:
4807 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, 8);
4808
4809 case clang::X86::BI__builtin_ia32_bextr_u32:
4810 case clang::X86::BI__builtin_ia32_bextr_u64:
4811 case clang::X86::BI__builtin_ia32_bextri_u32:
4812 case clang::X86::BI__builtin_ia32_bextri_u64:
4814 S, OpPC, Call, [](const APSInt &Val, const APSInt &Idx) {
4815 unsigned BitWidth = Val.getBitWidth();
4816 uint64_t Shift = Idx.extractBitsAsZExtValue(8, 0);
4817 uint64_t Length = Idx.extractBitsAsZExtValue(8, 8);
4818 if (Length > BitWidth) {
4819 Length = BitWidth;
4820 }
4821
4822 // Handle out of bounds cases.
4823 if (Length == 0 || Shift >= BitWidth)
4824 return APInt(BitWidth, 0);
4825
4826 uint64_t Result = Val.getZExtValue() >> Shift;
4827 Result &= llvm::maskTrailingOnes<uint64_t>(Length);
4828 return APInt(BitWidth, Result);
4829 });
4830
4831 case clang::X86::BI__builtin_ia32_bzhi_si:
4832 case clang::X86::BI__builtin_ia32_bzhi_di:
4834 S, OpPC, Call, [](const APSInt &Val, const APSInt &Idx) {
4835 unsigned BitWidth = Val.getBitWidth();
4836 uint64_t Index = Idx.extractBitsAsZExtValue(8, 0);
4837 APSInt Result = Val;
4838
4839 if (Index < BitWidth)
4840 Result.clearHighBits(BitWidth - Index);
4841
4842 return Result;
4843 });
4844
4845 case clang::X86::BI__builtin_ia32_ktestcqi:
4846 case clang::X86::BI__builtin_ia32_ktestchi:
4847 case clang::X86::BI__builtin_ia32_ktestcsi:
4848 case clang::X86::BI__builtin_ia32_ktestcdi:
4850 S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
4851 return APInt(sizeof(unsigned char) * 8, (~A & B) == 0);
4852 });
4853
4854 case clang::X86::BI__builtin_ia32_ktestzqi:
4855 case clang::X86::BI__builtin_ia32_ktestzhi:
4856 case clang::X86::BI__builtin_ia32_ktestzsi:
4857 case clang::X86::BI__builtin_ia32_ktestzdi:
4859 S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
4860 return APInt(sizeof(unsigned char) * 8, (A & B) == 0);
4861 });
4862
4863 case clang::X86::BI__builtin_ia32_kortestcqi:
4864 case clang::X86::BI__builtin_ia32_kortestchi:
4865 case clang::X86::BI__builtin_ia32_kortestcsi:
4866 case clang::X86::BI__builtin_ia32_kortestcdi:
4868 S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
4869 return APInt(sizeof(unsigned char) * 8, ~(A | B) == 0);
4870 });
4871
4872 case clang::X86::BI__builtin_ia32_kortestzqi:
4873 case clang::X86::BI__builtin_ia32_kortestzhi:
4874 case clang::X86::BI__builtin_ia32_kortestzsi:
4875 case clang::X86::BI__builtin_ia32_kortestzdi:
4877 S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
4878 return APInt(sizeof(unsigned char) * 8, (A | B) == 0);
4879 });
4880
4881 case clang::X86::BI__builtin_ia32_kshiftliqi:
4882 case clang::X86::BI__builtin_ia32_kshiftlihi:
4883 case clang::X86::BI__builtin_ia32_kshiftlisi:
4884 case clang::X86::BI__builtin_ia32_kshiftlidi:
4886 S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
4887 unsigned Amt = RHS.getZExtValue() & 0xFF;
4888 if (Amt >= LHS.getBitWidth())
4889 return APInt::getZero(LHS.getBitWidth());
4890 return LHS.shl(Amt);
4891 });
4892
4893 case clang::X86::BI__builtin_ia32_kshiftriqi:
4894 case clang::X86::BI__builtin_ia32_kshiftrihi:
4895 case clang::X86::BI__builtin_ia32_kshiftrisi:
4896 case clang::X86::BI__builtin_ia32_kshiftridi:
4898 S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
4899 unsigned Amt = RHS.getZExtValue() & 0xFF;
4900 if (Amt >= LHS.getBitWidth())
4901 return APInt::getZero(LHS.getBitWidth());
4902 return LHS.lshr(Amt);
4903 });
4904
4905 case clang::X86::BI__builtin_ia32_lzcnt_u16:
4906 case clang::X86::BI__builtin_ia32_lzcnt_u32:
4907 case clang::X86::BI__builtin_ia32_lzcnt_u64:
4909 S, OpPC, Call, [](const APSInt &Src) {
4910 return APInt(Src.getBitWidth(), Src.countLeadingZeros());
4911 });
4912
4913 case clang::X86::BI__builtin_ia32_tzcnt_u16:
4914 case clang::X86::BI__builtin_ia32_tzcnt_u32:
4915 case clang::X86::BI__builtin_ia32_tzcnt_u64:
4917 S, OpPC, Call, [](const APSInt &Src) {
4918 return APInt(Src.getBitWidth(), Src.countTrailingZeros());
4919 });
4920
4921 case clang::X86::BI__builtin_ia32_pdep_si:
4922 case clang::X86::BI__builtin_ia32_pdep_di:
4924 S, OpPC, Call, [](const APSInt &Val, const APSInt &Mask) {
4925 unsigned BitWidth = Val.getBitWidth();
4926 APInt Result = APInt::getZero(BitWidth);
4927
4928 for (unsigned I = 0, P = 0; I != BitWidth; ++I) {
4929 if (Mask[I])
4930 Result.setBitVal(I, Val[P++]);
4931 }
4932
4933 return Result;
4934 });
4935
4936 case clang::X86::BI__builtin_ia32_pext_si:
4937 case clang::X86::BI__builtin_ia32_pext_di:
4939 S, OpPC, Call, [](const APSInt &Val, const APSInt &Mask) {
4940 unsigned BitWidth = Val.getBitWidth();
4941 APInt Result = APInt::getZero(BitWidth);
4942
4943 for (unsigned I = 0, P = 0; I != BitWidth; ++I) {
4944 if (Mask[I])
4945 Result.setBitVal(P++, Val[I]);
4946 }
4947
4948 return Result;
4949 });
4950
4951 case clang::X86::BI__builtin_ia32_addcarryx_u32:
4952 case clang::X86::BI__builtin_ia32_addcarryx_u64:
4953 case clang::X86::BI__builtin_ia32_subborrow_u32:
4954 case clang::X86::BI__builtin_ia32_subborrow_u64:
4956 BuiltinID);
4957
4958 case Builtin::BI__builtin_os_log_format_buffer_size:
4960
4961 case Builtin::BI__builtin_ptrauth_string_discriminator:
4963
4964 case Builtin::BI__builtin_infer_alloc_token:
4966
4967 case Builtin::BI__noop:
4968 pushInteger(S, 0, Call->getType());
4969 return true;
4970
4971 case Builtin::BI__builtin_operator_new:
4972 return interp__builtin_operator_new(S, OpPC, Frame, Call);
4973
4974 case Builtin::BI__builtin_operator_delete:
4975 return interp__builtin_operator_delete(S, OpPC, Frame, Call);
4976
4977 case Builtin::BI__arithmetic_fence:
4979
4980 case Builtin::BI__builtin_reduce_add:
4981 case Builtin::BI__builtin_reduce_mul:
4982 case Builtin::BI__builtin_reduce_and:
4983 case Builtin::BI__builtin_reduce_or:
4984 case Builtin::BI__builtin_reduce_xor:
4985 case Builtin::BI__builtin_reduce_min:
4986 case Builtin::BI__builtin_reduce_max:
4987 return interp__builtin_vector_reduce(S, OpPC, Call, BuiltinID);
4988
4989 case Builtin::BI__builtin_elementwise_popcount:
4991 S, OpPC, Call, [](const APSInt &Src) {
4992 return APInt(Src.getBitWidth(), Src.popcount());
4993 });
4994 case Builtin::BI__builtin_elementwise_bitreverse:
4996 S, OpPC, Call, [](const APSInt &Src) { return Src.reverseBits(); });
4997
4998 case Builtin::BI__builtin_elementwise_abs:
4999 return interp__builtin_elementwise_abs(S, OpPC, Frame, Call, BuiltinID);
5000
5001 case Builtin::BI__builtin_memcpy:
5002 case Builtin::BImemcpy:
5003 case Builtin::BI__builtin_wmemcpy:
5004 case Builtin::BIwmemcpy:
5005 case Builtin::BI__builtin_memmove:
5006 case Builtin::BImemmove:
5007 case Builtin::BI__builtin_wmemmove:
5008 case Builtin::BIwmemmove:
5009 return interp__builtin_memcpy(S, OpPC, Frame, Call, BuiltinID);
5010
5011 case Builtin::BI__builtin_memcmp:
5012 case Builtin::BImemcmp:
5013 case Builtin::BI__builtin_bcmp:
5014 case Builtin::BIbcmp:
5015 case Builtin::BI__builtin_wmemcmp:
5016 case Builtin::BIwmemcmp:
5017 return interp__builtin_memcmp(S, OpPC, Frame, Call, BuiltinID);
5018
5019 case Builtin::BImemchr:
5020 case Builtin::BI__builtin_memchr:
5021 case Builtin::BIstrchr:
5022 case Builtin::BI__builtin_strchr:
5023 case Builtin::BIwmemchr:
5024 case Builtin::BI__builtin_wmemchr:
5025 case Builtin::BIwcschr:
5026 case Builtin::BI__builtin_wcschr:
5027 case Builtin::BI__builtin_char_memchr:
5028 return interp__builtin_memchr(S, OpPC, Call, BuiltinID);
5029
5030 case Builtin::BI__builtin_object_size:
5031 case Builtin::BI__builtin_dynamic_object_size:
5032 return interp__builtin_object_size(S, OpPC, Frame, Call);
5033
5034 case Builtin::BI__builtin_is_within_lifetime:
5036
5037 case Builtin::BI__builtin_elementwise_add_sat:
5039 S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
5040 return LHS.isSigned() ? LHS.sadd_sat(RHS) : LHS.uadd_sat(RHS);
5041 });
5042
5043 case Builtin::BI__builtin_elementwise_sub_sat:
5045 S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
5046 return LHS.isSigned() ? LHS.ssub_sat(RHS) : LHS.usub_sat(RHS);
5047 });
5048 case X86::BI__builtin_ia32_extract128i256:
5049 case X86::BI__builtin_ia32_vextractf128_pd256:
5050 case X86::BI__builtin_ia32_vextractf128_ps256:
5051 case X86::BI__builtin_ia32_vextractf128_si256:
5052 return interp__builtin_x86_extract_vector(S, OpPC, Call, BuiltinID);
5053
5054 case X86::BI__builtin_ia32_extractf32x4_256_mask:
5055 case X86::BI__builtin_ia32_extractf32x4_mask:
5056 case X86::BI__builtin_ia32_extractf32x8_mask:
5057 case X86::BI__builtin_ia32_extractf64x2_256_mask:
5058 case X86::BI__builtin_ia32_extractf64x2_512_mask:
5059 case X86::BI__builtin_ia32_extractf64x4_mask:
5060 case X86::BI__builtin_ia32_extracti32x4_256_mask:
5061 case X86::BI__builtin_ia32_extracti32x4_mask:
5062 case X86::BI__builtin_ia32_extracti32x8_mask:
5063 case X86::BI__builtin_ia32_extracti64x2_256_mask:
5064 case X86::BI__builtin_ia32_extracti64x2_512_mask:
5065 case X86::BI__builtin_ia32_extracti64x4_mask:
5066 return interp__builtin_x86_extract_vector_masked(S, OpPC, Call, BuiltinID);
5067
5068 case clang::X86::BI__builtin_ia32_pmulhrsw128:
5069 case clang::X86::BI__builtin_ia32_pmulhrsw256:
5070 case clang::X86::BI__builtin_ia32_pmulhrsw512:
5072 S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
5073 return (llvm::APIntOps::mulsExtended(LHS, RHS).ashr(14) + 1)
5074 .extractBits(16, 1);
5075 });
5076
5077 case clang::X86::BI__builtin_ia32_movmskps:
5078 case clang::X86::BI__builtin_ia32_movmskpd:
5079 case clang::X86::BI__builtin_ia32_pmovmskb128:
5080 case clang::X86::BI__builtin_ia32_pmovmskb256:
5081 case clang::X86::BI__builtin_ia32_movmskps256:
5082 case clang::X86::BI__builtin_ia32_movmskpd256: {
5083 return interp__builtin_ia32_movmsk_op(S, OpPC, Call);
5084 }
5085
5086 case X86::BI__builtin_ia32_psignb128:
5087 case X86::BI__builtin_ia32_psignb256:
5088 case X86::BI__builtin_ia32_psignw128:
5089 case X86::BI__builtin_ia32_psignw256:
5090 case X86::BI__builtin_ia32_psignd128:
5091 case X86::BI__builtin_ia32_psignd256:
5093 S, OpPC, Call, [](const APInt &AElem, const APInt &BElem) {
5094 if (BElem.isZero())
5095 return APInt::getZero(AElem.getBitWidth());
5096 if (BElem.isNegative())
5097 return -AElem;
5098 return AElem;
5099 });
5100
5101 case clang::X86::BI__builtin_ia32_pavgb128:
5102 case clang::X86::BI__builtin_ia32_pavgw128:
5103 case clang::X86::BI__builtin_ia32_pavgb256:
5104 case clang::X86::BI__builtin_ia32_pavgw256:
5105 case clang::X86::BI__builtin_ia32_pavgb512:
5106 case clang::X86::BI__builtin_ia32_pavgw512:
5108 llvm::APIntOps::avgCeilU);
5109
5110 case clang::X86::BI__builtin_ia32_pmaddubsw128:
5111 case clang::X86::BI__builtin_ia32_pmaddubsw256:
5112 case clang::X86::BI__builtin_ia32_pmaddubsw512:
5114 S, OpPC, Call,
5115 [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5116 const APSInt &HiRHS) {
5117 unsigned BitWidth = 2 * LoLHS.getBitWidth();
5118 return (LoLHS.zext(BitWidth) * LoRHS.sext(BitWidth))
5119 .sadd_sat((HiLHS.zext(BitWidth) * HiRHS.sext(BitWidth)));
5120 });
5121
5122 case clang::X86::BI__builtin_ia32_pmaddwd128:
5123 case clang::X86::BI__builtin_ia32_pmaddwd256:
5124 case clang::X86::BI__builtin_ia32_pmaddwd512:
5126 S, OpPC, Call,
5127 [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5128 const APSInt &HiRHS) {
5129 unsigned BitWidth = 2 * LoLHS.getBitWidth();
5130 return (LoLHS.sext(BitWidth) * LoRHS.sext(BitWidth)) +
5131 (HiLHS.sext(BitWidth) * HiRHS.sext(BitWidth));
5132 });
5133
5134 case clang::X86::BI__builtin_ia32_dbpsadbw128:
5135 case clang::X86::BI__builtin_ia32_dbpsadbw256:
5136 case clang::X86::BI__builtin_ia32_dbpsadbw512:
5137 return interp__builtin_ia32_dbpsadbw(S, OpPC, Call);
5138
5139 case clang::X86::BI__builtin_ia32_pmulhuw128:
5140 case clang::X86::BI__builtin_ia32_pmulhuw256:
5141 case clang::X86::BI__builtin_ia32_pmulhuw512:
5143 llvm::APIntOps::mulhu);
5144
5145 case clang::X86::BI__builtin_ia32_pmulhw128:
5146 case clang::X86::BI__builtin_ia32_pmulhw256:
5147 case clang::X86::BI__builtin_ia32_pmulhw512:
5149 llvm::APIntOps::mulhs);
5150
5151 case clang::X86::BI__builtin_ia32_psllv2di:
5152 case clang::X86::BI__builtin_ia32_psllv4di:
5153 case clang::X86::BI__builtin_ia32_psllv4si:
5154 case clang::X86::BI__builtin_ia32_psllv8di:
5155 case clang::X86::BI__builtin_ia32_psllv8hi:
5156 case clang::X86::BI__builtin_ia32_psllv8si:
5157 case clang::X86::BI__builtin_ia32_psllv16hi:
5158 case clang::X86::BI__builtin_ia32_psllv16si:
5159 case clang::X86::BI__builtin_ia32_psllv32hi:
5160 case clang::X86::BI__builtin_ia32_psllwi128:
5161 case clang::X86::BI__builtin_ia32_psllwi256:
5162 case clang::X86::BI__builtin_ia32_psllwi512:
5163 case clang::X86::BI__builtin_ia32_pslldi128:
5164 case clang::X86::BI__builtin_ia32_pslldi256:
5165 case clang::X86::BI__builtin_ia32_pslldi512:
5166 case clang::X86::BI__builtin_ia32_psllqi128:
5167 case clang::X86::BI__builtin_ia32_psllqi256:
5168 case clang::X86::BI__builtin_ia32_psllqi512:
5170 S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
5171 if (RHS.uge(LHS.getBitWidth())) {
5172 return APInt::getZero(LHS.getBitWidth());
5173 }
5174 return LHS.shl(RHS.getZExtValue());
5175 });
5176
5177 case clang::X86::BI__builtin_ia32_psrav4si:
5178 case clang::X86::BI__builtin_ia32_psrav8di:
5179 case clang::X86::BI__builtin_ia32_psrav8hi:
5180 case clang::X86::BI__builtin_ia32_psrav8si:
5181 case clang::X86::BI__builtin_ia32_psrav16hi:
5182 case clang::X86::BI__builtin_ia32_psrav16si:
5183 case clang::X86::BI__builtin_ia32_psrav32hi:
5184 case clang::X86::BI__builtin_ia32_psravq128:
5185 case clang::X86::BI__builtin_ia32_psravq256:
5186 case clang::X86::BI__builtin_ia32_psrawi128:
5187 case clang::X86::BI__builtin_ia32_psrawi256:
5188 case clang::X86::BI__builtin_ia32_psrawi512:
5189 case clang::X86::BI__builtin_ia32_psradi128:
5190 case clang::X86::BI__builtin_ia32_psradi256:
5191 case clang::X86::BI__builtin_ia32_psradi512:
5192 case clang::X86::BI__builtin_ia32_psraqi128:
5193 case clang::X86::BI__builtin_ia32_psraqi256:
5194 case clang::X86::BI__builtin_ia32_psraqi512:
5196 S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
5197 if (RHS.uge(LHS.getBitWidth())) {
5198 return LHS.ashr(LHS.getBitWidth() - 1);
5199 }
5200 return LHS.ashr(RHS.getZExtValue());
5201 });
5202
5203 case clang::X86::BI__builtin_ia32_psrlv2di:
5204 case clang::X86::BI__builtin_ia32_psrlv4di:
5205 case clang::X86::BI__builtin_ia32_psrlv4si:
5206 case clang::X86::BI__builtin_ia32_psrlv8di:
5207 case clang::X86::BI__builtin_ia32_psrlv8hi:
5208 case clang::X86::BI__builtin_ia32_psrlv8si:
5209 case clang::X86::BI__builtin_ia32_psrlv16hi:
5210 case clang::X86::BI__builtin_ia32_psrlv16si:
5211 case clang::X86::BI__builtin_ia32_psrlv32hi:
5212 case clang::X86::BI__builtin_ia32_psrlwi128:
5213 case clang::X86::BI__builtin_ia32_psrlwi256:
5214 case clang::X86::BI__builtin_ia32_psrlwi512:
5215 case clang::X86::BI__builtin_ia32_psrldi128:
5216 case clang::X86::BI__builtin_ia32_psrldi256:
5217 case clang::X86::BI__builtin_ia32_psrldi512:
5218 case clang::X86::BI__builtin_ia32_psrlqi128:
5219 case clang::X86::BI__builtin_ia32_psrlqi256:
5220 case clang::X86::BI__builtin_ia32_psrlqi512:
5222 S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
5223 if (RHS.uge(LHS.getBitWidth())) {
5224 return APInt::getZero(LHS.getBitWidth());
5225 }
5226 return LHS.lshr(RHS.getZExtValue());
5227 });
5228 case clang::X86::BI__builtin_ia32_packsswb128:
5229 case clang::X86::BI__builtin_ia32_packsswb256:
5230 case clang::X86::BI__builtin_ia32_packsswb512:
5231 case clang::X86::BI__builtin_ia32_packssdw128:
5232 case clang::X86::BI__builtin_ia32_packssdw256:
5233 case clang::X86::BI__builtin_ia32_packssdw512:
5234 return interp__builtin_x86_pack(S, OpPC, Call, [](const APSInt &Src) {
5235 return APInt(Src).truncSSat(Src.getBitWidth() / 2);
5236 });
5237 case clang::X86::BI__builtin_ia32_packusdw128:
5238 case clang::X86::BI__builtin_ia32_packusdw256:
5239 case clang::X86::BI__builtin_ia32_packusdw512:
5240 case clang::X86::BI__builtin_ia32_packuswb128:
5241 case clang::X86::BI__builtin_ia32_packuswb256:
5242 case clang::X86::BI__builtin_ia32_packuswb512:
5243 return interp__builtin_x86_pack(S, OpPC, Call, [](const APSInt &Src) {
5244 return APInt(Src).truncSSatU(Src.getBitWidth() / 2);
5245 });
5246
5247 case clang::X86::BI__builtin_ia32_selectss_128:
5248 case clang::X86::BI__builtin_ia32_selectsd_128:
5249 case clang::X86::BI__builtin_ia32_selectsh_128:
5250 case clang::X86::BI__builtin_ia32_selectsbf_128:
5252 case clang::X86::BI__builtin_ia32_vprotbi:
5253 case clang::X86::BI__builtin_ia32_vprotdi:
5254 case clang::X86::BI__builtin_ia32_vprotqi:
5255 case clang::X86::BI__builtin_ia32_vprotwi:
5256 case clang::X86::BI__builtin_ia32_prold128:
5257 case clang::X86::BI__builtin_ia32_prold256:
5258 case clang::X86::BI__builtin_ia32_prold512:
5259 case clang::X86::BI__builtin_ia32_prolq128:
5260 case clang::X86::BI__builtin_ia32_prolq256:
5261 case clang::X86::BI__builtin_ia32_prolq512:
5263 S, OpPC, Call,
5264 [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotl(RHS); });
5265
5266 case clang::X86::BI__builtin_ia32_prord128:
5267 case clang::X86::BI__builtin_ia32_prord256:
5268 case clang::X86::BI__builtin_ia32_prord512:
5269 case clang::X86::BI__builtin_ia32_prorq128:
5270 case clang::X86::BI__builtin_ia32_prorq256:
5271 case clang::X86::BI__builtin_ia32_prorq512:
5273 S, OpPC, Call,
5274 [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotr(RHS); });
5275
5276 case Builtin::BI__builtin_elementwise_max:
5277 case Builtin::BI__builtin_elementwise_min:
5278 return interp__builtin_elementwise_maxmin(S, OpPC, Call, BuiltinID);
5279
5280 case clang::X86::BI__builtin_ia32_phaddw128:
5281 case clang::X86::BI__builtin_ia32_phaddw256:
5282 case clang::X86::BI__builtin_ia32_phaddd128:
5283 case clang::X86::BI__builtin_ia32_phaddd256:
5285 S, OpPC, Call,
5286 [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
5287 case clang::X86::BI__builtin_ia32_phaddsw128:
5288 case clang::X86::BI__builtin_ia32_phaddsw256:
5290 S, OpPC, Call,
5291 [](const APSInt &LHS, const APSInt &RHS) { return LHS.sadd_sat(RHS); });
5292 case clang::X86::BI__builtin_ia32_phsubw128:
5293 case clang::X86::BI__builtin_ia32_phsubw256:
5294 case clang::X86::BI__builtin_ia32_phsubd128:
5295 case clang::X86::BI__builtin_ia32_phsubd256:
5297 S, OpPC, Call,
5298 [](const APSInt &LHS, const APSInt &RHS) { return LHS - RHS; });
5299 case clang::X86::BI__builtin_ia32_phsubsw128:
5300 case clang::X86::BI__builtin_ia32_phsubsw256:
5302 S, OpPC, Call,
5303 [](const APSInt &LHS, const APSInt &RHS) { return LHS.ssub_sat(RHS); });
5304 case clang::X86::BI__builtin_ia32_haddpd:
5305 case clang::X86::BI__builtin_ia32_haddps:
5306 case clang::X86::BI__builtin_ia32_haddpd256:
5307 case clang::X86::BI__builtin_ia32_haddps256:
5309 S, OpPC, Call,
5310 [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
5311 APFloat F = LHS;
5312 F.add(RHS, RM);
5313 return F;
5314 });
5315 case clang::X86::BI__builtin_ia32_hsubpd:
5316 case clang::X86::BI__builtin_ia32_hsubps:
5317 case clang::X86::BI__builtin_ia32_hsubpd256:
5318 case clang::X86::BI__builtin_ia32_hsubps256:
5320 S, OpPC, Call,
5321 [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
5322 APFloat F = LHS;
5323 F.subtract(RHS, RM);
5324 return F;
5325 });
5326 case clang::X86::BI__builtin_ia32_addsubpd:
5327 case clang::X86::BI__builtin_ia32_addsubps:
5328 case clang::X86::BI__builtin_ia32_addsubpd256:
5329 case clang::X86::BI__builtin_ia32_addsubps256:
5330 return interp__builtin_ia32_addsub(S, OpPC, Call);
5331
5332 case clang::X86::BI__builtin_ia32_pmuldq128:
5333 case clang::X86::BI__builtin_ia32_pmuldq256:
5334 case clang::X86::BI__builtin_ia32_pmuldq512:
5336 S, OpPC, Call,
5337 [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5338 const APSInt &HiRHS) {
5339 return llvm::APIntOps::mulsExtended(LoLHS, LoRHS);
5340 });
5341
5342 case clang::X86::BI__builtin_ia32_pmuludq128:
5343 case clang::X86::BI__builtin_ia32_pmuludq256:
5344 case clang::X86::BI__builtin_ia32_pmuludq512:
5346 S, OpPC, Call,
5347 [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5348 const APSInt &HiRHS) {
5349 return llvm::APIntOps::muluExtended(LoLHS, LoRHS);
5350 });
5351
5352 case clang::X86::BI__builtin_ia32_pclmulqdq128:
5353 case clang::X86::BI__builtin_ia32_pclmulqdq256:
5354 case clang::X86::BI__builtin_ia32_pclmulqdq512:
5355 return interp__builtin_ia32_pclmulqdq(S, OpPC, Call);
5356
5357 case Builtin::BI__builtin_elementwise_fma:
5359 S, OpPC, Call,
5360 [](const APFloat &X, const APFloat &Y, const APFloat &Z,
5361 llvm::RoundingMode RM) {
5362 APFloat F = X;
5363 F.fusedMultiplyAdd(Y, Z, RM);
5364 return F;
5365 });
5366
5367 case X86::BI__builtin_ia32_vpmadd52luq128:
5368 case X86::BI__builtin_ia32_vpmadd52luq256:
5369 case X86::BI__builtin_ia32_vpmadd52luq512:
5371 S, OpPC, Call, [](const APSInt &A, const APSInt &B, const APSInt &C) {
5372 return A + (B.trunc(52) * C.trunc(52)).zext(64);
5373 });
5374 case X86::BI__builtin_ia32_vpmadd52huq128:
5375 case X86::BI__builtin_ia32_vpmadd52huq256:
5376 case X86::BI__builtin_ia32_vpmadd52huq512:
5378 S, OpPC, Call, [](const APSInt &A, const APSInt &B, const APSInt &C) {
5379 return A + llvm::APIntOps::mulhu(B.trunc(52), C.trunc(52)).zext(64);
5380 });
5381
5382 case X86::BI__builtin_ia32_vpshldd128:
5383 case X86::BI__builtin_ia32_vpshldd256:
5384 case X86::BI__builtin_ia32_vpshldd512:
5385 case X86::BI__builtin_ia32_vpshldq128:
5386 case X86::BI__builtin_ia32_vpshldq256:
5387 case X86::BI__builtin_ia32_vpshldq512:
5388 case X86::BI__builtin_ia32_vpshldw128:
5389 case X86::BI__builtin_ia32_vpshldw256:
5390 case X86::BI__builtin_ia32_vpshldw512:
5392 S, OpPC, Call,
5393 [](const APSInt &Hi, const APSInt &Lo, const APSInt &Amt) {
5394 return llvm::APIntOps::fshl(Hi, Lo, Amt);
5395 });
5396
5397 case X86::BI__builtin_ia32_vpshrdd128:
5398 case X86::BI__builtin_ia32_vpshrdd256:
5399 case X86::BI__builtin_ia32_vpshrdd512:
5400 case X86::BI__builtin_ia32_vpshrdq128:
5401 case X86::BI__builtin_ia32_vpshrdq256:
5402 case X86::BI__builtin_ia32_vpshrdq512:
5403 case X86::BI__builtin_ia32_vpshrdw128:
5404 case X86::BI__builtin_ia32_vpshrdw256:
5405 case X86::BI__builtin_ia32_vpshrdw512:
5406 // NOTE: Reversed Hi/Lo operands.
5408 S, OpPC, Call,
5409 [](const APSInt &Lo, const APSInt &Hi, const APSInt &Amt) {
5410 return llvm::APIntOps::fshr(Hi, Lo, Amt);
5411 });
5412 case X86::BI__builtin_ia32_vpconflictsi_128:
5413 case X86::BI__builtin_ia32_vpconflictsi_256:
5414 case X86::BI__builtin_ia32_vpconflictsi_512:
5415 case X86::BI__builtin_ia32_vpconflictdi_128:
5416 case X86::BI__builtin_ia32_vpconflictdi_256:
5417 case X86::BI__builtin_ia32_vpconflictdi_512:
5418 return interp__builtin_ia32_vpconflict(S, OpPC, Call);
5419 case X86::BI__builtin_ia32_compressdf128_mask:
5420 case X86::BI__builtin_ia32_compressdf256_mask:
5421 case X86::BI__builtin_ia32_compressdf512_mask:
5422 case X86::BI__builtin_ia32_compressdi128_mask:
5423 case X86::BI__builtin_ia32_compressdi256_mask:
5424 case X86::BI__builtin_ia32_compressdi512_mask:
5425 case X86::BI__builtin_ia32_compresshi128_mask:
5426 case X86::BI__builtin_ia32_compresshi256_mask:
5427 case X86::BI__builtin_ia32_compresshi512_mask:
5428 case X86::BI__builtin_ia32_compressqi128_mask:
5429 case X86::BI__builtin_ia32_compressqi256_mask:
5430 case X86::BI__builtin_ia32_compressqi512_mask:
5431 case X86::BI__builtin_ia32_compresssf128_mask:
5432 case X86::BI__builtin_ia32_compresssf256_mask:
5433 case X86::BI__builtin_ia32_compresssf512_mask:
5434 case X86::BI__builtin_ia32_compresssi128_mask:
5435 case X86::BI__builtin_ia32_compresssi256_mask:
5436 case X86::BI__builtin_ia32_compresssi512_mask: {
5437 unsigned NumElems =
5438 Call->getArg(0)->getType()->castAs<VectorType>()->getNumElements();
5440 S, OpPC, Call, [NumElems](unsigned DstIdx, const APInt &ShuffleMask) {
5441 APInt CompressMask = ShuffleMask.trunc(NumElems);
5442 if (DstIdx < CompressMask.popcount()) {
5443 while (DstIdx != 0) {
5444 CompressMask = CompressMask & (CompressMask - 1);
5445 DstIdx--;
5446 }
5447 return std::pair<unsigned, int>{
5448 0, static_cast<int>(CompressMask.countr_zero())};
5449 }
5450 return std::pair<unsigned, int>{1, static_cast<int>(DstIdx)};
5451 });
5452 }
5453 case X86::BI__builtin_ia32_expanddf128_mask:
5454 case X86::BI__builtin_ia32_expanddf256_mask:
5455 case X86::BI__builtin_ia32_expanddf512_mask:
5456 case X86::BI__builtin_ia32_expanddi128_mask:
5457 case X86::BI__builtin_ia32_expanddi256_mask:
5458 case X86::BI__builtin_ia32_expanddi512_mask:
5459 case X86::BI__builtin_ia32_expandhi128_mask:
5460 case X86::BI__builtin_ia32_expandhi256_mask:
5461 case X86::BI__builtin_ia32_expandhi512_mask:
5462 case X86::BI__builtin_ia32_expandqi128_mask:
5463 case X86::BI__builtin_ia32_expandqi256_mask:
5464 case X86::BI__builtin_ia32_expandqi512_mask:
5465 case X86::BI__builtin_ia32_expandsf128_mask:
5466 case X86::BI__builtin_ia32_expandsf256_mask:
5467 case X86::BI__builtin_ia32_expandsf512_mask:
5468 case X86::BI__builtin_ia32_expandsi128_mask:
5469 case X86::BI__builtin_ia32_expandsi256_mask:
5470 case X86::BI__builtin_ia32_expandsi512_mask: {
5472 S, OpPC, Call, [](unsigned DstIdx, const APInt &ShuffleMask) {
5473 // Trunc to the sub-mask for the dst index and count the number of
5474 // src elements used prior to that.
5475 APInt ExpandMask = ShuffleMask.trunc(DstIdx + 1);
5476 if (ExpandMask[DstIdx]) {
5477 int SrcIdx = ExpandMask.popcount() - 1;
5478 return std::pair<unsigned, int>{0, SrcIdx};
5479 }
5480 return std::pair<unsigned, int>{1, static_cast<int>(DstIdx)};
5481 });
5482 }
5483 case clang::X86::BI__builtin_ia32_blendpd:
5484 case clang::X86::BI__builtin_ia32_blendpd256:
5485 case clang::X86::BI__builtin_ia32_blendps:
5486 case clang::X86::BI__builtin_ia32_blendps256:
5487 case clang::X86::BI__builtin_ia32_pblendw128:
5488 case clang::X86::BI__builtin_ia32_pblendw256:
5489 case clang::X86::BI__builtin_ia32_pblendd128:
5490 case clang::X86::BI__builtin_ia32_pblendd256:
5492 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5493 // Bit index for mask.
5494 unsigned MaskBit = (ShuffleMask >> (DstIdx % 8)) & 0x1;
5495 unsigned SrcVecIdx = MaskBit ? 1 : 0; // 1 = TrueVec, 0 = FalseVec
5496 return std::pair<unsigned, int>{SrcVecIdx, static_cast<int>(DstIdx)};
5497 });
5498
5499
5500
5501 case clang::X86::BI__builtin_ia32_blendvpd:
5502 case clang::X86::BI__builtin_ia32_blendvpd256:
5503 case clang::X86::BI__builtin_ia32_blendvps:
5504 case clang::X86::BI__builtin_ia32_blendvps256:
5506 S, OpPC, Call,
5507 [](const APFloat &F, const APFloat &T, const APFloat &C,
5508 llvm::RoundingMode) { return C.isNegative() ? T : F; });
5509
5510 case clang::X86::BI__builtin_ia32_pblendvb128:
5511 case clang::X86::BI__builtin_ia32_pblendvb256:
5513 S, OpPC, Call, [](const APSInt &F, const APSInt &T, const APSInt &C) {
5514 return ((APInt)C).isNegative() ? T : F;
5515 });
5516 case X86::BI__builtin_ia32_ptestz128:
5517 case X86::BI__builtin_ia32_ptestz256:
5518 case X86::BI__builtin_ia32_vtestzps:
5519 case X86::BI__builtin_ia32_vtestzps256:
5520 case X86::BI__builtin_ia32_vtestzpd:
5521 case X86::BI__builtin_ia32_vtestzpd256:
5523 S, OpPC, Call,
5524 [](const APInt &A, const APInt &B) { return (A & B) == 0; });
5525 case X86::BI__builtin_ia32_ptestc128:
5526 case X86::BI__builtin_ia32_ptestc256:
5527 case X86::BI__builtin_ia32_vtestcps:
5528 case X86::BI__builtin_ia32_vtestcps256:
5529 case X86::BI__builtin_ia32_vtestcpd:
5530 case X86::BI__builtin_ia32_vtestcpd256:
5532 S, OpPC, Call,
5533 [](const APInt &A, const APInt &B) { return (~A & B) == 0; });
5534 case X86::BI__builtin_ia32_ptestnzc128:
5535 case X86::BI__builtin_ia32_ptestnzc256:
5536 case X86::BI__builtin_ia32_vtestnzcps:
5537 case X86::BI__builtin_ia32_vtestnzcps256:
5538 case X86::BI__builtin_ia32_vtestnzcpd:
5539 case X86::BI__builtin_ia32_vtestnzcpd256:
5541 S, OpPC, Call, [](const APInt &A, const APInt &B) {
5542 return ((A & B) != 0) && ((~A & B) != 0);
5543 });
5544 case X86::BI__builtin_ia32_selectb_128:
5545 case X86::BI__builtin_ia32_selectb_256:
5546 case X86::BI__builtin_ia32_selectb_512:
5547 case X86::BI__builtin_ia32_selectw_128:
5548 case X86::BI__builtin_ia32_selectw_256:
5549 case X86::BI__builtin_ia32_selectw_512:
5550 case X86::BI__builtin_ia32_selectd_128:
5551 case X86::BI__builtin_ia32_selectd_256:
5552 case X86::BI__builtin_ia32_selectd_512:
5553 case X86::BI__builtin_ia32_selectq_128:
5554 case X86::BI__builtin_ia32_selectq_256:
5555 case X86::BI__builtin_ia32_selectq_512:
5556 case X86::BI__builtin_ia32_selectph_128:
5557 case X86::BI__builtin_ia32_selectph_256:
5558 case X86::BI__builtin_ia32_selectph_512:
5559 case X86::BI__builtin_ia32_selectpbf_128:
5560 case X86::BI__builtin_ia32_selectpbf_256:
5561 case X86::BI__builtin_ia32_selectpbf_512:
5562 case X86::BI__builtin_ia32_selectps_128:
5563 case X86::BI__builtin_ia32_selectps_256:
5564 case X86::BI__builtin_ia32_selectps_512:
5565 case X86::BI__builtin_ia32_selectpd_128:
5566 case X86::BI__builtin_ia32_selectpd_256:
5567 case X86::BI__builtin_ia32_selectpd_512:
5568 return interp__builtin_select(S, OpPC, Call);
5569
5570 case X86::BI__builtin_ia32_shufps:
5571 case X86::BI__builtin_ia32_shufps256:
5572 case X86::BI__builtin_ia32_shufps512:
5574 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5575 unsigned NumElemPerLane = 4;
5576 unsigned NumSelectableElems = NumElemPerLane / 2;
5577 unsigned BitsPerElem = 2;
5578 unsigned IndexMask = 0x3;
5579 unsigned MaskBits = 8;
5580 unsigned Lane = DstIdx / NumElemPerLane;
5581 unsigned ElemInLane = DstIdx % NumElemPerLane;
5582 unsigned LaneOffset = Lane * NumElemPerLane;
5583 unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
5584 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
5585 unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
5586 return std::pair<unsigned, int>{SrcIdx,
5587 static_cast<int>(LaneOffset + Index)};
5588 });
5589 case X86::BI__builtin_ia32_shufpd:
5590 case X86::BI__builtin_ia32_shufpd256:
5591 case X86::BI__builtin_ia32_shufpd512:
5593 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5594 unsigned NumElemPerLane = 2;
5595 unsigned NumSelectableElems = NumElemPerLane / 2;
5596 unsigned BitsPerElem = 1;
5597 unsigned IndexMask = 0x1;
5598 unsigned MaskBits = 8;
5599 unsigned Lane = DstIdx / NumElemPerLane;
5600 unsigned ElemInLane = DstIdx % NumElemPerLane;
5601 unsigned LaneOffset = Lane * NumElemPerLane;
5602 unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
5603 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
5604 unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
5605 return std::pair<unsigned, int>{SrcIdx,
5606 static_cast<int>(LaneOffset + Index)};
5607 });
5608
5609 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v16qi:
5610 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v32qi:
5611 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v64qi:
5612 return interp_builtin_ia32_gfni_affine(S, OpPC, Call, true);
5613 case X86::BI__builtin_ia32_vgf2p8affineqb_v16qi:
5614 case X86::BI__builtin_ia32_vgf2p8affineqb_v32qi:
5615 case X86::BI__builtin_ia32_vgf2p8affineqb_v64qi:
5616 return interp_builtin_ia32_gfni_affine(S, OpPC, Call, false);
5617
5618 case X86::BI__builtin_ia32_vgf2p8mulb_v16qi:
5619 case X86::BI__builtin_ia32_vgf2p8mulb_v32qi:
5620 case X86::BI__builtin_ia32_vgf2p8mulb_v64qi:
5621 return interp__builtin_ia32_gfni_mul(S, OpPC, Call);
5622
5623 case X86::BI__builtin_ia32_insertps128:
5625 S, OpPC, Call, [](unsigned DstIdx, unsigned Mask) {
5626 // Bits [3:0]: zero mask - if bit is set, zero this element
5627 if ((Mask & (1 << DstIdx)) != 0) {
5628 return std::pair<unsigned, int>{0, -1};
5629 }
5630 // Bits [7:6]: select element from source vector Y (0-3)
5631 // Bits [5:4]: select destination position (0-3)
5632 unsigned SrcElem = (Mask >> 6) & 0x3;
5633 unsigned DstElem = (Mask >> 4) & 0x3;
5634 if (DstIdx == DstElem) {
5635 // Insert element from source vector (B) at this position
5636 return std::pair<unsigned, int>{1, static_cast<int>(SrcElem)};
5637 } else {
5638 // Copy from destination vector (A)
5639 return std::pair<unsigned, int>{0, static_cast<int>(DstIdx)};
5640 }
5641 });
5642 case X86::BI__builtin_ia32_permvarsi256:
5643 case X86::BI__builtin_ia32_permvarsf256:
5644 case X86::BI__builtin_ia32_permvardf512:
5645 case X86::BI__builtin_ia32_permvardi512:
5646 case X86::BI__builtin_ia32_permvarhi128:
5648 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5649 int Offset = ShuffleMask & 0x7;
5650 return std::pair<unsigned, int>{0, Offset};
5651 });
5652 case X86::BI__builtin_ia32_permvarqi128:
5653 case X86::BI__builtin_ia32_permvarhi256:
5654 case X86::BI__builtin_ia32_permvarsi512:
5655 case X86::BI__builtin_ia32_permvarsf512:
5657 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5658 int Offset = ShuffleMask & 0xF;
5659 return std::pair<unsigned, int>{0, Offset};
5660 });
5661 case X86::BI__builtin_ia32_permvardi256:
5662 case X86::BI__builtin_ia32_permvardf256:
5664 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5665 int Offset = ShuffleMask & 0x3;
5666 return std::pair<unsigned, int>{0, Offset};
5667 });
5668 case X86::BI__builtin_ia32_permvarqi256:
5669 case X86::BI__builtin_ia32_permvarhi512:
5671 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5672 int Offset = ShuffleMask & 0x1F;
5673 return std::pair<unsigned, int>{0, Offset};
5674 });
5675 case X86::BI__builtin_ia32_permvarqi512:
5677 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5678 int Offset = ShuffleMask & 0x3F;
5679 return std::pair<unsigned, int>{0, Offset};
5680 });
5681 case X86::BI__builtin_ia32_vpermi2varq128:
5682 case X86::BI__builtin_ia32_vpermi2varpd128:
5684 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5685 int Offset = ShuffleMask & 0x1;
5686 unsigned SrcIdx = (ShuffleMask >> 1) & 0x1;
5687 return std::pair<unsigned, int>{SrcIdx, Offset};
5688 });
5689 case X86::BI__builtin_ia32_vpermi2vard128:
5690 case X86::BI__builtin_ia32_vpermi2varps128:
5691 case X86::BI__builtin_ia32_vpermi2varq256:
5692 case X86::BI__builtin_ia32_vpermi2varpd256:
5694 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5695 int Offset = ShuffleMask & 0x3;
5696 unsigned SrcIdx = (ShuffleMask >> 2) & 0x1;
5697 return std::pair<unsigned, int>{SrcIdx, Offset};
5698 });
5699 case X86::BI__builtin_ia32_vpermi2varhi128:
5700 case X86::BI__builtin_ia32_vpermi2vard256:
5701 case X86::BI__builtin_ia32_vpermi2varps256:
5702 case X86::BI__builtin_ia32_vpermi2varq512:
5703 case X86::BI__builtin_ia32_vpermi2varpd512:
5705 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5706 int Offset = ShuffleMask & 0x7;
5707 unsigned SrcIdx = (ShuffleMask >> 3) & 0x1;
5708 return std::pair<unsigned, int>{SrcIdx, Offset};
5709 });
5710 case X86::BI__builtin_ia32_vpermi2varqi128:
5711 case X86::BI__builtin_ia32_vpermi2varhi256:
5712 case X86::BI__builtin_ia32_vpermi2vard512:
5713 case X86::BI__builtin_ia32_vpermi2varps512:
5715 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5716 int Offset = ShuffleMask & 0xF;
5717 unsigned SrcIdx = (ShuffleMask >> 4) & 0x1;
5718 return std::pair<unsigned, int>{SrcIdx, Offset};
5719 });
5720 case X86::BI__builtin_ia32_vpermi2varqi256:
5721 case X86::BI__builtin_ia32_vpermi2varhi512:
5723 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5724 int Offset = ShuffleMask & 0x1F;
5725 unsigned SrcIdx = (ShuffleMask >> 5) & 0x1;
5726 return std::pair<unsigned, int>{SrcIdx, Offset};
5727 });
5728 case X86::BI__builtin_ia32_vpermi2varqi512:
5730 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5731 int Offset = ShuffleMask & 0x3F;
5732 unsigned SrcIdx = (ShuffleMask >> 6) & 0x1;
5733 return std::pair<unsigned, int>{SrcIdx, Offset};
5734 });
5735 case X86::BI__builtin_ia32_vperm2f128_pd256:
5736 case X86::BI__builtin_ia32_vperm2f128_ps256:
5737 case X86::BI__builtin_ia32_vperm2f128_si256:
5738 case X86::BI__builtin_ia32_permti256: {
5739 unsigned NumElements =
5740 Call->getArg(0)->getType()->castAs<VectorType>()->getNumElements();
5741 unsigned PreservedBitsCnt = NumElements >> 2;
5743 S, OpPC, Call,
5744 [PreservedBitsCnt](unsigned DstIdx, unsigned ShuffleMask) {
5745 unsigned ControlBitsCnt = DstIdx >> PreservedBitsCnt << 2;
5746 unsigned ControlBits = ShuffleMask >> ControlBitsCnt;
5747
5748 if (ControlBits & 0b1000)
5749 return std::make_pair(0u, -1);
5750
5751 unsigned SrcVecIdx = (ControlBits & 0b10) >> 1;
5752 unsigned PreservedBitsMask = (1 << PreservedBitsCnt) - 1;
5753 int SrcIdx = ((ControlBits & 0b1) << PreservedBitsCnt) |
5754 (DstIdx & PreservedBitsMask);
5755 return std::make_pair(SrcVecIdx, SrcIdx);
5756 });
5757 }
5758 case X86::BI__builtin_ia32_pshufb128:
5759 case X86::BI__builtin_ia32_pshufb256:
5760 case X86::BI__builtin_ia32_pshufb512:
5762 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5763 uint8_t Ctlb = static_cast<uint8_t>(ShuffleMask);
5764 if (Ctlb & 0x80)
5765 return std::make_pair(0, -1);
5766
5767 unsigned LaneBase = (DstIdx / 16) * 16;
5768 unsigned SrcOffset = Ctlb & 0x0F;
5769 unsigned SrcIdx = LaneBase + SrcOffset;
5770 return std::make_pair(0, static_cast<int>(SrcIdx));
5771 });
5772
5773 case X86::BI__builtin_ia32_pshuflw:
5774 case X86::BI__builtin_ia32_pshuflw256:
5775 case X86::BI__builtin_ia32_pshuflw512:
5777 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5778 unsigned LaneBase = (DstIdx / 8) * 8;
5779 unsigned LaneIdx = DstIdx % 8;
5780 if (LaneIdx < 4) {
5781 unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
5782 return std::make_pair(0, static_cast<int>(LaneBase + Sel));
5783 }
5784
5785 return std::make_pair(0, static_cast<int>(DstIdx));
5786 });
5787
5788 case X86::BI__builtin_ia32_pshufhw:
5789 case X86::BI__builtin_ia32_pshufhw256:
5790 case X86::BI__builtin_ia32_pshufhw512:
5792 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5793 unsigned LaneBase = (DstIdx / 8) * 8;
5794 unsigned LaneIdx = DstIdx % 8;
5795 if (LaneIdx >= 4) {
5796 unsigned Sel = (ShuffleMask >> (2 * (LaneIdx - 4))) & 0x3;
5797 return std::make_pair(0, static_cast<int>(LaneBase + 4 + Sel));
5798 }
5799
5800 return std::make_pair(0, static_cast<int>(DstIdx));
5801 });
5802
5803 case X86::BI__builtin_ia32_pshufd:
5804 case X86::BI__builtin_ia32_pshufd256:
5805 case X86::BI__builtin_ia32_pshufd512:
5806 case X86::BI__builtin_ia32_vpermilps:
5807 case X86::BI__builtin_ia32_vpermilps256:
5808 case X86::BI__builtin_ia32_vpermilps512:
5810 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5811 unsigned LaneBase = (DstIdx / 4) * 4;
5812 unsigned LaneIdx = DstIdx % 4;
5813 unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
5814 return std::make_pair(0, static_cast<int>(LaneBase + Sel));
5815 });
5816
5817 case X86::BI__builtin_ia32_vpermilvarpd:
5818 case X86::BI__builtin_ia32_vpermilvarpd256:
5819 case X86::BI__builtin_ia32_vpermilvarpd512:
5821 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5822 unsigned NumElemPerLane = 2;
5823 unsigned Lane = DstIdx / NumElemPerLane;
5824 unsigned Offset = ShuffleMask & 0b10 ? 1 : 0;
5825 return std::make_pair(
5826 0, static_cast<int>(Lane * NumElemPerLane + Offset));
5827 });
5828
5829 case X86::BI__builtin_ia32_vpermilvarps:
5830 case X86::BI__builtin_ia32_vpermilvarps256:
5831 case X86::BI__builtin_ia32_vpermilvarps512:
5833 S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
5834 unsigned NumElemPerLane = 4;
5835 unsigned Lane = DstIdx / NumElemPerLane;
5836 unsigned Offset = ShuffleMask & 0b11;
5837 return std::make_pair(
5838 0, static_cast<int>(Lane * NumElemPerLane + Offset));
5839 });
5840
5841 case X86::BI__builtin_ia32_vpermilpd:
5842 case X86::BI__builtin_ia32_vpermilpd256:
5843 case X86::BI__builtin_ia32_vpermilpd512:
5845 S, OpPC, Call, [](unsigned DstIdx, unsigned Control) {
5846 unsigned NumElemPerLane = 2;
5847 unsigned BitsPerElem = 1;
5848 unsigned MaskBits = 8;
5849 unsigned IndexMask = 0x1;
5850 unsigned Lane = DstIdx / NumElemPerLane;
5851 unsigned LaneOffset = Lane * NumElemPerLane;
5852 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
5853 unsigned Index = (Control >> BitIndex) & IndexMask;
5854 return std::make_pair(0, static_cast<int>(LaneOffset + Index));
5855 });
5856
5857 case X86::BI__builtin_ia32_permdf256:
5858 case X86::BI__builtin_ia32_permdi256:
5860 S, OpPC, Call, [](unsigned DstIdx, unsigned Control) {
5861 // permute4x64 operates on 4 64-bit elements
5862 // For element i (0-3), extract bits [2*i+1:2*i] from Control
5863 unsigned Index = (Control >> (2 * DstIdx)) & 0x3;
5864 return std::make_pair(0, static_cast<int>(Index));
5865 });
5866
5867 case X86::BI__builtin_ia32_vpmultishiftqb128:
5868 case X86::BI__builtin_ia32_vpmultishiftqb256:
5869 case X86::BI__builtin_ia32_vpmultishiftqb512:
5870 return interp__builtin_ia32_multishiftqb(S, OpPC, Call);
5871 case X86::BI__builtin_ia32_kandqi:
5872 case X86::BI__builtin_ia32_kandhi:
5873 case X86::BI__builtin_ia32_kandsi:
5874 case X86::BI__builtin_ia32_kanddi:
5876 S, OpPC, Call,
5877 [](const APSInt &LHS, const APSInt &RHS) { return LHS & RHS; });
5878
5879 case X86::BI__builtin_ia32_kandnqi:
5880 case X86::BI__builtin_ia32_kandnhi:
5881 case X86::BI__builtin_ia32_kandnsi:
5882 case X86::BI__builtin_ia32_kandndi:
5884 S, OpPC, Call,
5885 [](const APSInt &LHS, const APSInt &RHS) { return ~LHS & RHS; });
5886
5887 case X86::BI__builtin_ia32_korqi:
5888 case X86::BI__builtin_ia32_korhi:
5889 case X86::BI__builtin_ia32_korsi:
5890 case X86::BI__builtin_ia32_kordi:
5892 S, OpPC, Call,
5893 [](const APSInt &LHS, const APSInt &RHS) { return LHS | RHS; });
5894
5895 case X86::BI__builtin_ia32_kxnorqi:
5896 case X86::BI__builtin_ia32_kxnorhi:
5897 case X86::BI__builtin_ia32_kxnorsi:
5898 case X86::BI__builtin_ia32_kxnordi:
5900 S, OpPC, Call,
5901 [](const APSInt &LHS, const APSInt &RHS) { return ~(LHS ^ RHS); });
5902
5903 case X86::BI__builtin_ia32_kxorqi:
5904 case X86::BI__builtin_ia32_kxorhi:
5905 case X86::BI__builtin_ia32_kxorsi:
5906 case X86::BI__builtin_ia32_kxordi:
5908 S, OpPC, Call,
5909 [](const APSInt &LHS, const APSInt &RHS) { return LHS ^ RHS; });
5910
5911 case X86::BI__builtin_ia32_knotqi:
5912 case X86::BI__builtin_ia32_knothi:
5913 case X86::BI__builtin_ia32_knotsi:
5914 case X86::BI__builtin_ia32_knotdi:
5916 S, OpPC, Call, [](const APSInt &Src) { return ~Src; });
5917
5918 case X86::BI__builtin_ia32_kaddqi:
5919 case X86::BI__builtin_ia32_kaddhi:
5920 case X86::BI__builtin_ia32_kaddsi:
5921 case X86::BI__builtin_ia32_kadddi:
5923 S, OpPC, Call,
5924 [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
5925
5926 case X86::BI__builtin_ia32_kmovb:
5927 case X86::BI__builtin_ia32_kmovw:
5928 case X86::BI__builtin_ia32_kmovd:
5929 case X86::BI__builtin_ia32_kmovq:
5931 S, OpPC, Call, [](const APSInt &Src) { return Src; });
5932
5933 case X86::BI__builtin_ia32_kunpckhi:
5934 case X86::BI__builtin_ia32_kunpckdi:
5935 case X86::BI__builtin_ia32_kunpcksi:
5937 S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
5938 // Generic kunpack: extract lower half of each operand and concatenate
5939 // Result = A[HalfWidth-1:0] concat B[HalfWidth-1:0]
5940 unsigned BW = A.getBitWidth();
5941 return APSInt(A.trunc(BW / 2).concat(B.trunc(BW / 2)),
5942 A.isUnsigned());
5943 });
5944
5945 case X86::BI__builtin_ia32_phminposuw128:
5946 return interp__builtin_ia32_phminposuw(S, OpPC, Call);
5947
5948 case X86::BI__builtin_ia32_psraq128:
5949 case X86::BI__builtin_ia32_psraq256:
5950 case X86::BI__builtin_ia32_psraq512:
5951 case X86::BI__builtin_ia32_psrad128:
5952 case X86::BI__builtin_ia32_psrad256:
5953 case X86::BI__builtin_ia32_psrad512:
5954 case X86::BI__builtin_ia32_psraw128:
5955 case X86::BI__builtin_ia32_psraw256:
5956 case X86::BI__builtin_ia32_psraw512:
5958 S, OpPC, Call,
5959 [](const APInt &Elt, uint64_t Count) { return Elt.ashr(Count); },
5960 [](const APInt &Elt, unsigned Width) { return Elt.ashr(Width - 1); });
5961
5962 case X86::BI__builtin_ia32_psllq128:
5963 case X86::BI__builtin_ia32_psllq256:
5964 case X86::BI__builtin_ia32_psllq512:
5965 case X86::BI__builtin_ia32_pslld128:
5966 case X86::BI__builtin_ia32_pslld256:
5967 case X86::BI__builtin_ia32_pslld512:
5968 case X86::BI__builtin_ia32_psllw128:
5969 case X86::BI__builtin_ia32_psllw256:
5970 case X86::BI__builtin_ia32_psllw512:
5972 S, OpPC, Call,
5973 [](const APInt &Elt, uint64_t Count) { return Elt.shl(Count); },
5974 [](const APInt &Elt, unsigned Width) { return APInt::getZero(Width); });
5975
5976 case X86::BI__builtin_ia32_psrlq128:
5977 case X86::BI__builtin_ia32_psrlq256:
5978 case X86::BI__builtin_ia32_psrlq512:
5979 case X86::BI__builtin_ia32_psrld128:
5980 case X86::BI__builtin_ia32_psrld256:
5981 case X86::BI__builtin_ia32_psrld512:
5982 case X86::BI__builtin_ia32_psrlw128:
5983 case X86::BI__builtin_ia32_psrlw256:
5984 case X86::BI__builtin_ia32_psrlw512:
5986 S, OpPC, Call,
5987 [](const APInt &Elt, uint64_t Count) { return Elt.lshr(Count); },
5988 [](const APInt &Elt, unsigned Width) { return APInt::getZero(Width); });
5989
5990 case X86::BI__builtin_ia32_pternlogd128_mask:
5991 case X86::BI__builtin_ia32_pternlogd256_mask:
5992 case X86::BI__builtin_ia32_pternlogd512_mask:
5993 case X86::BI__builtin_ia32_pternlogq128_mask:
5994 case X86::BI__builtin_ia32_pternlogq256_mask:
5995 case X86::BI__builtin_ia32_pternlogq512_mask:
5996 return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/false);
5997 case X86::BI__builtin_ia32_pternlogd128_maskz:
5998 case X86::BI__builtin_ia32_pternlogd256_maskz:
5999 case X86::BI__builtin_ia32_pternlogd512_maskz:
6000 case X86::BI__builtin_ia32_pternlogq128_maskz:
6001 case X86::BI__builtin_ia32_pternlogq256_maskz:
6002 case X86::BI__builtin_ia32_pternlogq512_maskz:
6003 return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/true);
6004 case Builtin::BI__builtin_elementwise_fshl:
6006 llvm::APIntOps::fshl);
6007 case Builtin::BI__builtin_elementwise_fshr:
6009 llvm::APIntOps::fshr);
6010
6011 case X86::BI__builtin_ia32_shuf_f32x4_256:
6012 case X86::BI__builtin_ia32_shuf_i32x4_256:
6013 case X86::BI__builtin_ia32_shuf_f64x2_256:
6014 case X86::BI__builtin_ia32_shuf_i64x2_256:
6015 case X86::BI__builtin_ia32_shuf_f32x4:
6016 case X86::BI__builtin_ia32_shuf_i32x4:
6017 case X86::BI__builtin_ia32_shuf_f64x2:
6018 case X86::BI__builtin_ia32_shuf_i64x2: {
6019 // Destination and sources A, B all have the same type.
6020 QualType VecQT = Call->getArg(0)->getType();
6021 const auto *VecT = VecQT->castAs<VectorType>();
6022 unsigned NumElems = VecT->getNumElements();
6023 unsigned ElemBits = S.getASTContext().getTypeSize(VecT->getElementType());
6024 unsigned LaneBits = 128u;
6025 unsigned NumLanes = (NumElems * ElemBits) / LaneBits;
6026 unsigned NumElemsPerLane = LaneBits / ElemBits;
6027
6029 S, OpPC, Call,
6030 [NumLanes, NumElemsPerLane](unsigned DstIdx, unsigned ShuffleMask) {
6031 // DstIdx determines source. ShuffleMask selects lane in source.
6032 unsigned BitsPerElem = NumLanes / 2;
6033 unsigned IndexMask = (1u << BitsPerElem) - 1;
6034 unsigned Lane = DstIdx / NumElemsPerLane;
6035 unsigned SrcIdx = (Lane < NumLanes / 2) ? 0 : 1;
6036 unsigned BitIdx = BitsPerElem * Lane;
6037 unsigned SrcLaneIdx = (ShuffleMask >> BitIdx) & IndexMask;
6038 unsigned ElemInLane = DstIdx % NumElemsPerLane;
6039 unsigned IdxToPick = SrcLaneIdx * NumElemsPerLane + ElemInLane;
6040 return std::pair<unsigned, int>{SrcIdx, IdxToPick};
6041 });
6042 }
6043
6044 case X86::BI__builtin_ia32_insertf32x4_256:
6045 case X86::BI__builtin_ia32_inserti32x4_256:
6046 case X86::BI__builtin_ia32_insertf64x2_256:
6047 case X86::BI__builtin_ia32_inserti64x2_256:
6048 case X86::BI__builtin_ia32_insertf32x4:
6049 case X86::BI__builtin_ia32_inserti32x4:
6050 case X86::BI__builtin_ia32_insertf64x2_512:
6051 case X86::BI__builtin_ia32_inserti64x2_512:
6052 case X86::BI__builtin_ia32_insertf32x8:
6053 case X86::BI__builtin_ia32_inserti32x8:
6054 case X86::BI__builtin_ia32_insertf64x4:
6055 case X86::BI__builtin_ia32_inserti64x4:
6056 case X86::BI__builtin_ia32_vinsertf128_ps256:
6057 case X86::BI__builtin_ia32_vinsertf128_pd256:
6058 case X86::BI__builtin_ia32_vinsertf128_si256:
6059 case X86::BI__builtin_ia32_insert128i256:
6060 return interp__builtin_x86_insert_subvector(S, OpPC, Call, BuiltinID);
6061
6062 case clang::X86::BI__builtin_ia32_vcvtps2ph:
6063 case clang::X86::BI__builtin_ia32_vcvtps2ph256:
6064 return interp__builtin_ia32_vcvtps2ph(S, OpPC, Call);
6065
6066 case X86::BI__builtin_ia32_vec_ext_v4hi:
6067 case X86::BI__builtin_ia32_vec_ext_v16qi:
6068 case X86::BI__builtin_ia32_vec_ext_v8hi:
6069 case X86::BI__builtin_ia32_vec_ext_v4si:
6070 case X86::BI__builtin_ia32_vec_ext_v2di:
6071 case X86::BI__builtin_ia32_vec_ext_v32qi:
6072 case X86::BI__builtin_ia32_vec_ext_v16hi:
6073 case X86::BI__builtin_ia32_vec_ext_v8si:
6074 case X86::BI__builtin_ia32_vec_ext_v4di:
6075 case X86::BI__builtin_ia32_vec_ext_v4sf:
6076 return interp__builtin_vec_ext(S, OpPC, Call, BuiltinID);
6077
6078 case X86::BI__builtin_ia32_vec_set_v4hi:
6079 case X86::BI__builtin_ia32_vec_set_v16qi:
6080 case X86::BI__builtin_ia32_vec_set_v8hi:
6081 case X86::BI__builtin_ia32_vec_set_v4si:
6082 case X86::BI__builtin_ia32_vec_set_v2di:
6083 case X86::BI__builtin_ia32_vec_set_v32qi:
6084 case X86::BI__builtin_ia32_vec_set_v16hi:
6085 case X86::BI__builtin_ia32_vec_set_v8si:
6086 case X86::BI__builtin_ia32_vec_set_v4di:
6087 return interp__builtin_vec_set(S, OpPC, Call, BuiltinID);
6088
6089 case X86::BI__builtin_ia32_cvtb2mask128:
6090 case X86::BI__builtin_ia32_cvtb2mask256:
6091 case X86::BI__builtin_ia32_cvtb2mask512:
6092 case X86::BI__builtin_ia32_cvtw2mask128:
6093 case X86::BI__builtin_ia32_cvtw2mask256:
6094 case X86::BI__builtin_ia32_cvtw2mask512:
6095 case X86::BI__builtin_ia32_cvtd2mask128:
6096 case X86::BI__builtin_ia32_cvtd2mask256:
6097 case X86::BI__builtin_ia32_cvtd2mask512:
6098 case X86::BI__builtin_ia32_cvtq2mask128:
6099 case X86::BI__builtin_ia32_cvtq2mask256:
6100 case X86::BI__builtin_ia32_cvtq2mask512:
6101 return interp__builtin_ia32_cvt_vec2mask(S, OpPC, Call, BuiltinID);
6102
6103 case X86::BI__builtin_ia32_cvtmask2b128:
6104 case X86::BI__builtin_ia32_cvtmask2b256:
6105 case X86::BI__builtin_ia32_cvtmask2b512:
6106 case X86::BI__builtin_ia32_cvtmask2w128:
6107 case X86::BI__builtin_ia32_cvtmask2w256:
6108 case X86::BI__builtin_ia32_cvtmask2w512:
6109 case X86::BI__builtin_ia32_cvtmask2d128:
6110 case X86::BI__builtin_ia32_cvtmask2d256:
6111 case X86::BI__builtin_ia32_cvtmask2d512:
6112 case X86::BI__builtin_ia32_cvtmask2q128:
6113 case X86::BI__builtin_ia32_cvtmask2q256:
6114 case X86::BI__builtin_ia32_cvtmask2q512:
6115 return interp__builtin_ia32_cvt_mask2vec(S, OpPC, Call, BuiltinID);
6116
6117 case X86::BI__builtin_ia32_cvtsd2ss:
6118 return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, false);
6119
6120 case X86::BI__builtin_ia32_cvtsd2ss_round_mask:
6121 return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, true);
6122
6123 case X86::BI__builtin_ia32_cvtpd2ps:
6124 case X86::BI__builtin_ia32_cvtpd2ps256:
6125 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, false, false);
6126 case X86::BI__builtin_ia32_cvtpd2ps_mask:
6127 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, true, false);
6128 case X86::BI__builtin_ia32_cvtpd2ps512_mask:
6129 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, true, true);
6130
6131 case X86::BI__builtin_ia32_cmpb128_mask:
6132 case X86::BI__builtin_ia32_cmpw128_mask:
6133 case X86::BI__builtin_ia32_cmpd128_mask:
6134 case X86::BI__builtin_ia32_cmpq128_mask:
6135 case X86::BI__builtin_ia32_cmpb256_mask:
6136 case X86::BI__builtin_ia32_cmpw256_mask:
6137 case X86::BI__builtin_ia32_cmpd256_mask:
6138 case X86::BI__builtin_ia32_cmpq256_mask:
6139 case X86::BI__builtin_ia32_cmpb512_mask:
6140 case X86::BI__builtin_ia32_cmpw512_mask:
6141 case X86::BI__builtin_ia32_cmpd512_mask:
6142 case X86::BI__builtin_ia32_cmpq512_mask:
6143 return interp__builtin_ia32_cmp_mask(S, OpPC, Call, BuiltinID,
6144 /*IsUnsigned=*/false);
6145
6146 case X86::BI__builtin_ia32_ucmpb128_mask:
6147 case X86::BI__builtin_ia32_ucmpw128_mask:
6148 case X86::BI__builtin_ia32_ucmpd128_mask:
6149 case X86::BI__builtin_ia32_ucmpq128_mask:
6150 case X86::BI__builtin_ia32_ucmpb256_mask:
6151 case X86::BI__builtin_ia32_ucmpw256_mask:
6152 case X86::BI__builtin_ia32_ucmpd256_mask:
6153 case X86::BI__builtin_ia32_ucmpq256_mask:
6154 case X86::BI__builtin_ia32_ucmpb512_mask:
6155 case X86::BI__builtin_ia32_ucmpw512_mask:
6156 case X86::BI__builtin_ia32_ucmpd512_mask:
6157 case X86::BI__builtin_ia32_ucmpq512_mask:
6158 return interp__builtin_ia32_cmp_mask(S, OpPC, Call, BuiltinID,
6159 /*IsUnsigned=*/true);
6160
6161 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
6162 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
6163 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
6165
6166 case X86::BI__builtin_ia32_pslldqi128_byteshift:
6167 case X86::BI__builtin_ia32_pslldqi256_byteshift:
6168 case X86::BI__builtin_ia32_pslldqi512_byteshift:
6169 // These SLLDQ intrinsics always operate on byte elements (8 bits).
6170 // The lane width is hardcoded to 16 to match the SIMD register size,
6171 // but the algorithm processes one byte per iteration,
6172 // so APInt(8, ...) is correct and intentional.
6174 S, OpPC, Call,
6175 [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
6176 unsigned LaneBase = (DstIdx / 16) * 16;
6177 unsigned LaneIdx = DstIdx % 16;
6178 if (LaneIdx < Shift)
6179 return std::make_pair(0, -1);
6180
6181 return std::make_pair(0,
6182 static_cast<int>(LaneBase + LaneIdx - Shift));
6183 });
6184
6185 case X86::BI__builtin_ia32_psrldqi128_byteshift:
6186 case X86::BI__builtin_ia32_psrldqi256_byteshift:
6187 case X86::BI__builtin_ia32_psrldqi512_byteshift:
6188 // These SRLDQ intrinsics always operate on byte elements (8 bits).
6189 // The lane width is hardcoded to 16 to match the SIMD register size,
6190 // but the algorithm processes one byte per iteration,
6191 // so APInt(8, ...) is correct and intentional.
6193 S, OpPC, Call,
6194 [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
6195 unsigned LaneBase = (DstIdx / 16) * 16;
6196 unsigned LaneIdx = DstIdx % 16;
6197 if (LaneIdx + Shift < 16)
6198 return std::make_pair(0,
6199 static_cast<int>(LaneBase + LaneIdx + Shift));
6200
6201 return std::make_pair(0, -1);
6202 });
6203
6204 case X86::BI__builtin_ia32_palignr128:
6205 case X86::BI__builtin_ia32_palignr256:
6206 case X86::BI__builtin_ia32_palignr512:
6208 S, OpPC, Call, [](unsigned DstIdx, unsigned Shift) {
6209 // Default to -1 → zero-fill this destination element
6210 unsigned VecIdx = 1;
6211 int ElemIdx = -1;
6212
6213 int Lane = DstIdx / 16;
6214 int Offset = DstIdx % 16;
6215
6216 // Elements come from VecB first, then VecA after the shift boundary
6217 unsigned ShiftedIdx = Offset + (Shift & 0xFF);
6218 if (ShiftedIdx < 16) { // from VecB
6219 ElemIdx = ShiftedIdx + (Lane * 16);
6220 } else if (ShiftedIdx < 32) { // from VecA
6221 VecIdx = 0;
6222 ElemIdx = (ShiftedIdx - 16) + (Lane * 16);
6223 }
6224
6225 return std::pair<unsigned, int>{VecIdx, ElemIdx};
6226 });
6227
6228 case X86::BI__builtin_ia32_alignd128:
6229 case X86::BI__builtin_ia32_alignd256:
6230 case X86::BI__builtin_ia32_alignd512:
6231 case X86::BI__builtin_ia32_alignq128:
6232 case X86::BI__builtin_ia32_alignq256:
6233 case X86::BI__builtin_ia32_alignq512: {
6234 unsigned NumElems = Call->getType()->castAs<VectorType>()->getNumElements();
6236 S, OpPC, Call, [NumElems](unsigned DstIdx, unsigned Shift) {
6237 unsigned Imm = Shift & 0xFF;
6238 unsigned EffectiveShift = Imm & (NumElems - 1);
6239 unsigned SourcePos = DstIdx + EffectiveShift;
6240 unsigned VecIdx = SourcePos < NumElems ? 1u : 0u;
6241 unsigned ElemIdx = SourcePos & (NumElems - 1);
6242 return std::pair<unsigned, int>{VecIdx, static_cast<int>(ElemIdx)};
6243 });
6244 }
6245
6246 case clang::X86::BI__builtin_ia32_minps:
6247 case clang::X86::BI__builtin_ia32_minpd:
6248 case clang::X86::BI__builtin_ia32_minph128:
6249 case clang::X86::BI__builtin_ia32_minph256:
6250 case clang::X86::BI__builtin_ia32_minps256:
6251 case clang::X86::BI__builtin_ia32_minpd256:
6252 case clang::X86::BI__builtin_ia32_minps512:
6253 case clang::X86::BI__builtin_ia32_minpd512:
6254 case clang::X86::BI__builtin_ia32_minph512:
6256 S, OpPC, Call,
6257 [](const APFloat &A, const APFloat &B,
6258 std::optional<APSInt>) -> std::optional<APFloat> {
6259 if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
6260 B.isInfinity() || B.isDenormal())
6261 return std::nullopt;
6262 if (A.isZero() && B.isZero())
6263 return B;
6264 return llvm::minimum(A, B);
6265 });
6266
6267 case clang::X86::BI__builtin_ia32_minss:
6268 case clang::X86::BI__builtin_ia32_minsd:
6270 S, OpPC, Call,
6271 [](const APFloat &A, const APFloat &B,
6272 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6273 return EvalScalarMinMaxFp(A, B, RoundingMode, /*IsMin=*/true);
6274 },
6275 /*IsScalar=*/true);
6276
6277 case clang::X86::BI__builtin_ia32_minsd_round_mask:
6278 case clang::X86::BI__builtin_ia32_minss_round_mask:
6279 case clang::X86::BI__builtin_ia32_minsh_round_mask:
6280 case clang::X86::BI__builtin_ia32_maxsd_round_mask:
6281 case clang::X86::BI__builtin_ia32_maxss_round_mask:
6282 case clang::X86::BI__builtin_ia32_maxsh_round_mask: {
6283 bool IsMin = BuiltinID == clang::X86::BI__builtin_ia32_minsd_round_mask ||
6284 BuiltinID == clang::X86::BI__builtin_ia32_minss_round_mask ||
6285 BuiltinID == clang::X86::BI__builtin_ia32_minsh_round_mask;
6287 S, OpPC, Call,
6288 [IsMin](const APFloat &A, const APFloat &B,
6289 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6290 return EvalScalarMinMaxFp(A, B, RoundingMode, IsMin);
6291 });
6292 }
6293
6294 case clang::X86::BI__builtin_ia32_maxps:
6295 case clang::X86::BI__builtin_ia32_maxpd:
6296 case clang::X86::BI__builtin_ia32_maxph128:
6297 case clang::X86::BI__builtin_ia32_maxph256:
6298 case clang::X86::BI__builtin_ia32_maxps256:
6299 case clang::X86::BI__builtin_ia32_maxpd256:
6300 case clang::X86::BI__builtin_ia32_maxps512:
6301 case clang::X86::BI__builtin_ia32_maxpd512:
6302 case clang::X86::BI__builtin_ia32_maxph512:
6304 S, OpPC, Call,
6305 [](const APFloat &A, const APFloat &B,
6306 std::optional<APSInt>) -> std::optional<APFloat> {
6307 if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
6308 B.isInfinity() || B.isDenormal())
6309 return std::nullopt;
6310 if (A.isZero() && B.isZero())
6311 return B;
6312 return llvm::maximum(A, B);
6313 });
6314
6315 case clang::X86::BI__builtin_ia32_maxss:
6316 case clang::X86::BI__builtin_ia32_maxsd:
6318 S, OpPC, Call,
6319 [](const APFloat &A, const APFloat &B,
6320 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6321 return EvalScalarMinMaxFp(A, B, RoundingMode, /*IsMin=*/false);
6322 },
6323 /*IsScalar=*/true);
6324
6325 default:
6326 S.FFDiag(S.Current->getLocation(OpPC),
6327 diag::note_invalid_subexpr_in_const_expr)
6328 << S.Current->getRange(OpPC);
6329
6330 return false;
6331 }
6332
6333 llvm_unreachable("Unhandled builtin ID");
6334}
6335
6337 ArrayRef<int64_t> ArrayIndices, int64_t &IntResult) {
6340 unsigned N = E->getNumComponents();
6341 assert(N > 0);
6342
6343 unsigned ArrayIndex = 0;
6344 QualType CurrentType = E->getTypeSourceInfo()->getType();
6345 for (unsigned I = 0; I != N; ++I) {
6346 const OffsetOfNode &Node = E->getComponent(I);
6347 switch (Node.getKind()) {
6348 case OffsetOfNode::Field: {
6349 const FieldDecl *MemberDecl = Node.getField();
6350 const auto *RD = CurrentType->getAsRecordDecl();
6351 if (!RD || RD->isInvalidDecl())
6352 return false;
6354 unsigned FieldIndex = MemberDecl->getFieldIndex();
6355 assert(FieldIndex < RL.getFieldCount() && "offsetof field in wrong type");
6356 Result +=
6358 CurrentType = MemberDecl->getType().getNonReferenceType();
6359 break;
6360 }
6361 case OffsetOfNode::Array: {
6362 // When generating bytecode, we put all the index expressions as Sint64 on
6363 // the stack.
6364 int64_t Index = ArrayIndices[ArrayIndex];
6365 const ArrayType *AT = S.getASTContext().getAsArrayType(CurrentType);
6366 if (!AT)
6367 return false;
6368 CurrentType = AT->getElementType();
6369 CharUnits ElementSize = S.getASTContext().getTypeSizeInChars(CurrentType);
6370 Result += Index * ElementSize;
6371 ++ArrayIndex;
6372 break;
6373 }
6374 case OffsetOfNode::Base: {
6375 const CXXBaseSpecifier *BaseSpec = Node.getBase();
6376 if (BaseSpec->isVirtual())
6377 return false;
6378
6379 // Find the layout of the class whose base we are looking into.
6380 const auto *RD = CurrentType->getAsCXXRecordDecl();
6381 if (!RD || RD->isInvalidDecl())
6382 return false;
6384
6385 // Find the base class itself.
6386 CurrentType = BaseSpec->getType();
6387 const auto *BaseRD = CurrentType->getAsCXXRecordDecl();
6388 if (!BaseRD)
6389 return false;
6390
6391 // Add the offset to the base.
6392 Result += RL.getBaseClassOffset(BaseRD);
6393 break;
6394 }
6396 llvm_unreachable("Dependent OffsetOfExpr?");
6397 }
6398 }
6399
6400 IntResult = Result.getQuantity();
6401
6402 return true;
6403}
6404
6406 const Pointer &Ptr, const APSInt &IntValue) {
6407
6408 const Record *R = Ptr.getRecord();
6409 assert(R);
6410 assert(R->getNumFields() == 1);
6411
6412 unsigned FieldOffset = R->getField(0u)->Offset;
6413 const Pointer &FieldPtr = Ptr.atField(FieldOffset);
6414 PrimType FieldT = *S.getContext().classify(FieldPtr.getType());
6415
6416 INT_TYPE_SWITCH(FieldT,
6417 FieldPtr.deref<T>() = T::from(IntValue.getSExtValue()));
6418 FieldPtr.initialize();
6419 return true;
6420}
6421
6422static void zeroAll(Pointer &Dest) {
6423 const Descriptor *Desc = Dest.getFieldDesc();
6424
6425 if (Desc->isPrimitive()) {
6426 TYPE_SWITCH(Desc->getPrimType(), {
6427 Dest.deref<T>().~T();
6428 new (&Dest.deref<T>()) T();
6429 });
6430 return;
6431 }
6432
6433 if (Desc->isRecord()) {
6434 const Record *R = Desc->ElemRecord;
6435 for (const Record::Field &F : R->fields()) {
6436 Pointer FieldPtr = Dest.atField(F.Offset);
6437 zeroAll(FieldPtr);
6438 }
6439 return;
6440 }
6441
6442 if (Desc->isPrimitiveArray()) {
6443 for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
6444 TYPE_SWITCH(Desc->getPrimType(), {
6445 Dest.deref<T>().~T();
6446 new (&Dest.deref<T>()) T();
6447 });
6448 }
6449 return;
6450 }
6451
6452 if (Desc->isCompositeArray()) {
6453 for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
6454 Pointer ElemPtr = Dest.atIndex(I).narrow();
6455 zeroAll(ElemPtr);
6456 }
6457 return;
6458 }
6459}
6460
6461static bool copyComposite(InterpState &S, CodePtr OpPC, const Pointer &Src,
6462 Pointer &Dest, bool Activate);
6463static bool copyRecord(InterpState &S, CodePtr OpPC, const Pointer &Src,
6464 Pointer &Dest, bool Activate = false) {
6465 [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
6466 const Descriptor *DestDesc = Dest.getFieldDesc();
6467
6468 auto copyField = [&](const Record::Field &F, bool Activate) -> bool {
6469 Pointer DestField = Dest.atField(F.Offset);
6470 if (OptPrimType FT = S.Ctx.classify(F.Decl->getType())) {
6471 TYPE_SWITCH(*FT, {
6472 DestField.deref<T>() = Src.atField(F.Offset).deref<T>();
6473 if (Src.atField(F.Offset).isInitialized())
6474 DestField.initialize();
6475 if (Activate)
6476 DestField.activate();
6477 });
6478 return true;
6479 }
6480 // Composite field.
6481 return copyComposite(S, OpPC, Src.atField(F.Offset), DestField, Activate);
6482 };
6483
6484 assert(SrcDesc->isRecord());
6485 assert(SrcDesc->ElemRecord == DestDesc->ElemRecord);
6486 const Record *R = DestDesc->ElemRecord;
6487 for (const Record::Field &F : R->fields()) {
6488 if (R->isUnion()) {
6489 // For unions, only copy the active field. Zero all others.
6490 const Pointer &SrcField = Src.atField(F.Offset);
6491 if (SrcField.isActive()) {
6492 if (!copyField(F, /*Activate=*/true))
6493 return false;
6494 } else {
6495 if (!CheckMutable(S, OpPC, Src.atField(F.Offset)))
6496 return false;
6497 Pointer DestField = Dest.atField(F.Offset);
6498 zeroAll(DestField);
6499 }
6500 } else {
6501 if (!copyField(F, Activate))
6502 return false;
6503 }
6504 }
6505
6506 for (const Record::Base &B : R->bases()) {
6507 Pointer DestBase = Dest.atField(B.Offset);
6508 if (!copyRecord(S, OpPC, Src.atField(B.Offset), DestBase, Activate))
6509 return false;
6510 }
6511
6512 Dest.initialize();
6513 return true;
6514}
6515
6516static bool copyComposite(InterpState &S, CodePtr OpPC, const Pointer &Src,
6517 Pointer &Dest, bool Activate = false) {
6518 assert(Src.isLive() && Dest.isLive());
6519
6520 [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
6521 const Descriptor *DestDesc = Dest.getFieldDesc();
6522
6523 assert(!DestDesc->isPrimitive() && !SrcDesc->isPrimitive());
6524
6525 if (DestDesc->isPrimitiveArray()) {
6526 assert(SrcDesc->isPrimitiveArray());
6527 assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
6528 PrimType ET = DestDesc->getPrimType();
6529 for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
6530 Pointer DestElem = Dest.atIndex(I);
6531 TYPE_SWITCH(ET, {
6532 DestElem.deref<T>() = Src.elem<T>(I);
6533 DestElem.initialize();
6534 });
6535 }
6536 return true;
6537 }
6538
6539 if (DestDesc->isCompositeArray()) {
6540 assert(SrcDesc->isCompositeArray());
6541 assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
6542 for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
6543 const Pointer &SrcElem = Src.atIndex(I).narrow();
6544 Pointer DestElem = Dest.atIndex(I).narrow();
6545 if (!copyComposite(S, OpPC, SrcElem, DestElem, Activate))
6546 return false;
6547 }
6548 return true;
6549 }
6550
6551 if (DestDesc->isRecord())
6552 return copyRecord(S, OpPC, Src, Dest, Activate);
6553 return Invalid(S, OpPC);
6554}
6555
6556bool DoMemcpy(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest) {
6557 if (!Src.isBlockPointer() || Src.getFieldDesc()->isPrimitive())
6558 return false;
6559 if (!Dest.isBlockPointer() || Dest.getFieldDesc()->isPrimitive())
6560 return false;
6561
6562 return copyComposite(S, OpPC, Src, Dest);
6563}
6564
6565} // namespace interp
6566} // namespace clang
#define V(N, I)
Defines enum values for all the target-independent builtin functions.
llvm::APSInt APSInt
Definition Compiler.cpp:24
GCCTypeClass
Values returned by __builtin_classify_type, chosen to match the values produced by GCC's builtin.
std::optional< APFloat > EvalScalarMinMaxFp(const APFloat &A, const APFloat &B, std::optional< APSInt > RoundingMode, bool IsMin)
CharUnits GetAlignOfExpr(const ASTContext &Ctx, const Expr *E, UnaryExprOrTypeTrait ExprKind)
GCCTypeClass EvaluateBuiltinClassifyType(QualType T, const LangOptions &LangOpts)
EvaluateBuiltinClassifyType - Evaluate __builtin_classify_type the same way as GCC.
static bool isOneByteCharacterType(QualType T)
static bool isUserWritingOffTheEnd(const ASTContext &Ctx, const LValue &LVal)
Attempts to detect a user writing into a piece of memory that's impossible to figure out the size of ...
uint8_t GFNIMul(uint8_t AByte, uint8_t BByte)
uint8_t GFNIAffine(uint8_t XByte, const APInt &AQword, const APSInt &Imm, bool Inverse)
APSInt NormalizeRotateAmount(const APSInt &Value, const APSInt &Amount)
TokenType getType() const
Returns the token's type, e.g.
#define X(type, name)
Definition Value.h:97
static DiagnosticBuilder Diag(DiagnosticsEngine *Diags, const LangOptions &Features, FullSourceLoc TokLoc, const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd, unsigned DiagID)
Produce a diagnostic highlighting some portion of a literal.
#define FIXED_SIZE_INT_TYPE_SWITCH(Expr, B)
Definition PrimType.h:263
#define INT_TYPE_SWITCH_NO_BOOL(Expr, B)
Definition PrimType.h:279
#define INT_TYPE_SWITCH(Expr, B)
Definition PrimType.h:244
#define TYPE_SWITCH(Expr, B)
Definition PrimType.h:223
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
static QualType getPointeeType(const MemRegion *R)
Enumerates target-specific builtins in their own namespaces within namespace clang.
APValue - This class implements a discriminated union of [uninitialized] [APSInt] [APFloat],...
Definition APValue.h:122
CharUnits & getLValueOffset()
Definition APValue.cpp:1025
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition ASTContext.h:226
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
unsigned getIntWidth(QualType T) const
const llvm::fltSemantics & getFloatTypeSemantics(QualType T) const
Return the APFloat 'semantics' for the specified scalar floating point type.
CanQualType FloatTy
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
Builtin::Context & BuiltinInfo
Definition ASTContext.h:800
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
const LangOptions & getLangOpts() const
Definition ASTContext.h:952
CanQualType CharTy
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
QualType getWCharType() const
Return the unique wchar_t type available in C++ (and available as __wchar_t as a Microsoft extension)...
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
void recordOffsetOfEvaluation(const OffsetOfExpr *E)
QualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
const TargetInfo & getTargetInfo() const
Definition ASTContext.h:917
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
CanQualType getCanonicalTagType(const TagDecl *TD) const
static bool hasSameUnqualifiedType(QualType T1, QualType T2)
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
CanQualType HalfTy
uint64_t getCharWidth() const
Return the size of the character type, in bits.
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
unsigned getFieldCount() const
getFieldCount - Get the number of fields in the layout.
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
CharUnits getBaseClassOffset(const CXXRecordDecl *Base) const
getBaseClassOffset - Get the offset, in chars, for the given base class.
CharUnits getVBaseClassOffset(const CXXRecordDecl *VBase) const
getVBaseClassOffset - Get the offset, in chars, for the given base class.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition TypeBase.h:3772
QualType getElementType() const
Definition TypeBase.h:3784
std::string getQuotedName(unsigned ID) const
Return the identifier name for the specified builtin inside single quotes for a diagnostic,...
Definition Builtins.cpp:85
bool isConstantEvaluated(unsigned ID) const
Return true if this function can be constant evaluated by Clang frontend.
Definition Builtins.h:459
Represents a base class of a C++ class.
Definition DeclCXX.h:146
bool isVirtual() const
Determines whether the base class is a virtual base class (or not).
Definition DeclCXX.h:203
QualType getType() const
Retrieves the type of the base class.
Definition DeclCXX.h:249
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Definition Expr.h:2946
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
Definition Expr.h:3150
CharUnits - This is an opaque type for sizes expressed in character units.
Definition CharUnits.h:38
CharUnits alignmentAtOffset(CharUnits offset) const
Given that this is a non-zero alignment value, what is the alignment at the given offset?
Definition CharUnits.h:207
bool isZero() const
isZero - Test whether the quantity equals zero.
Definition CharUnits.h:122
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition CharUnits.h:185
static CharUnits One()
One - Construct a CharUnits quantity of one.
Definition CharUnits.h:58
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:63
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
Definition CharUnits.h:201
static unsigned getMaxSizeBits(const ASTContext &Context)
Determine the maximum number of active bits that an array's size can require, which limits the maximu...
Definition Type.cpp:256
This represents one expression.
Definition Expr.h:112
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition Expr.cpp:277
QualType getType() const
Definition Expr.h:144
Represents a member of a struct/union/class.
Definition Decl.h:3175
unsigned getFieldIndex() const
Returns the index of this field within its record, as appropriate for passing to ASTRecordLayout::get...
Definition Decl.h:3260
const RecordDecl * getParent() const
Returns the parent of this field declaration, which is the struct in which this field is defined.
Definition Decl.h:3411
Represents a function declaration or definition.
Definition Decl.h:2015
One of these records is kept for each identifier that is lexed.
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
std::optional< llvm::AllocTokenMode > AllocTokenMode
The allocation token mode.
std::optional< uint64_t > AllocTokenMax
Maximum number of allocation tokens (0 = target SIZE_MAX), nullopt if none set (use target SIZE_MAX).
OffsetOfExpr - [C99 7.17] - This represents an expression of the form offsetof(record-type,...
Definition Expr.h:2530
const OffsetOfNode & getComponent(unsigned Idx) const
Definition Expr.h:2577
TypeSourceInfo * getTypeSourceInfo() const
Definition Expr.h:2570
unsigned getNumComponents() const
Definition Expr.h:2585
Helper class for OffsetOfExpr.
Definition Expr.h:2424
FieldDecl * getField() const
For a field offsetof node, returns the field.
Definition Expr.h:2488
@ Array
An index into an array.
Definition Expr.h:2429
@ Identifier
A field in a dependent type, known only by its name.
Definition Expr.h:2433
@ Field
A field.
Definition Expr.h:2431
@ Base
An implicit indirection through a C++ base class, when the field found is in a base class.
Definition Expr.h:2436
Kind getKind() const
Determine what kind of offsetof node this is.
Definition Expr.h:2478
CXXBaseSpecifier * getBase() const
For a base class node, returns the base specifier.
Definition Expr.h:2498
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition TypeBase.h:3378
A (possibly-)qualified type.
Definition TypeBase.h:937
bool isTriviallyCopyableType(const ASTContext &Context) const
Return true if this is a trivially copyable type (C++0x [basic.types]p9)
Definition Type.cpp:2914
bool isNull() const
Return true if this QualType doesn't point to a type yet.
Definition TypeBase.h:1004
const Type * getTypePtr() const
Retrieves a pointer to the underlying (unqualified) type.
Definition TypeBase.h:8431
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition TypeBase.h:8616
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Definition Stmt.cpp:343
unsigned getMaxAtomicInlineWidth() const
Return the maximum width lock-free atomic operation which can be inlined given the supported features...
Definition TargetInfo.h:859
bool isBigEndian() const
virtual int getEHDataRegisterNumber(unsigned RegNo) const
Return the register number that __builtin_eh_return_regno would return with the specified argument.
virtual bool isNan2008() const
Returns true if NaN encoding is IEEE 754-2008.
QualType getType() const
Return the type wrapped by this type source info.
Definition TypeBase.h:8413
bool isBooleanType() const
Definition TypeBase.h:9171
bool isSignedIntegerOrEnumerationType() const
Determines whether this is an integer type that is signed or an enumeration types whose underlying ty...
Definition Type.cpp:2254
bool isUnsignedIntegerOrEnumerationType() const
Determines whether this is an integer type that is unsigned or an enumeration types whose underlying ...
Definition Type.cpp:2308
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
Definition Type.h:26
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Definition Type.h:41
bool isPointerType() const
Definition TypeBase.h:8668
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
Definition TypeBase.h:9078
const T * castAs() const
Member-template castAs<specific type>.
Definition TypeBase.h:9328
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
Definition Type.cpp:754
bool isIncompleteType(NamedDecl **Def=nullptr) const
Types are partitioned into 3 broad categories (C99 6.2.5p1): object types, function types,...
Definition Type.cpp:2480
bool isVectorType() const
Definition TypeBase.h:8807
bool isRealFloatingType() const
Floating point categories.
Definition Type.cpp:2358
bool isFloatingType() const
Definition Type.cpp:2342
const T * getAs() const
Member-template getAs<specific type>'.
Definition TypeBase.h:9261
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Definition Decl.h:712
QualType getType() const
Definition Decl.h:723
Represents a variable declaration or definition.
Definition Decl.h:926
Represents a GCC generic vector type.
Definition TypeBase.h:4225
unsigned getNumElements() const
Definition TypeBase.h:4240
QualType getElementType() const
Definition TypeBase.h:4239
A memory block, either on the stack or in the heap.
Definition InterpBlock.h:44
const Descriptor * getDescriptor() const
Returns the block's descriptor.
Definition InterpBlock.h:73
bool isDynamic() const
Definition InterpBlock.h:83
Wrapper around boolean types.
Definition Boolean.h:25
static Boolean from(T Value)
Definition Boolean.h:96
Pointer into the code segment.
Definition Source.h:30
const LangOptions & getLangOpts() const
Returns the language options.
Definition Context.cpp:374
OptPrimType classify(QualType T) const
Classifies a type.
Definition Context.cpp:408
unsigned getEvalID() const
Definition Context.h:166
Manages dynamic memory allocations done during bytecode interpretation.
bool deallocate(const Expr *Source, const Block *BlockToDelete, InterpState &S)
Deallocate the given source+block combination.
std::optional< Form > getAllocationForm(const Expr *Source) const
Checks whether the allocation done at the given source is an array allocation.
Block * allocate(const Descriptor *D, unsigned EvalID, Form AllocForm)
Allocate ONE element of the given descriptor.
If a Floating is constructed from Memory, it DOES NOT OWN THAT MEMORY.
Definition Floating.h:35
void copy(const APFloat &F)
Definition Floating.h:123
llvm::FPClassTest classify() const
Definition Floating.h:154
bool isSignaling() const
Definition Floating.h:149
bool isNormal() const
Definition Floating.h:152
ComparisonCategoryResult compare(const Floating &RHS) const
Definition Floating.h:157
bool isZero() const
Definition Floating.h:144
bool isNegative() const
Definition Floating.h:143
bool isFinite() const
Definition Floating.h:151
bool isDenormal() const
Definition Floating.h:153
APFloat::fltCategory getCategory() const
Definition Floating.h:155
APFloat getAPFloat() const
Definition Floating.h:64
Base class for stack frames, shared between VM and walker.
Definition Frame.h:25
virtual const FunctionDecl * getCallee() const =0
Returns the called function's declaration.
If an IntegralAP is constructed from Memory, it DOES NOT OWN THAT MEMORY.
Definition IntegralAP.h:36
Frame storing local variables.
Definition InterpFrame.h:27
const Expr * getExpr(CodePtr PC) const
InterpFrame * Caller
The frame of the previous function.
Definition InterpFrame.h:30
SourceInfo getSource(CodePtr PC) const
Map a location to a source.
CodePtr getRetPC() const
Returns the return address of the frame.
SourceLocation getLocation(CodePtr PC) const
SourceRange getRange(CodePtr PC) const
unsigned getDepth() const
const FunctionDecl * getCallee() const override
Returns the caller.
Stack frame storing temporaries and parameters.
Definition InterpStack.h:25
T pop()
Returns the value from the top of the stack and removes it.
Definition InterpStack.h:39
void push(Tys &&...Args)
Constructs a value in place on the top of the stack.
Definition InterpStack.h:33
void discard()
Discards the top value from the stack.
Definition InterpStack.h:50
T & peek() const
Returns a reference to the value on the top of the stack.
Definition InterpStack.h:62
Interpreter context.
Definition InterpState.h:36
Context & getContext() const
Definition InterpState.h:74
DynamicAllocator & getAllocator()
Definition InterpState.h:78
Context & Ctx
Interpreter Context.
Floating allocFloat(const llvm::fltSemantics &Sem)
llvm::SmallVector< const Block * > InitializingBlocks
List of blocks we're currently running either constructors or destructors for.
InterpStack & Stk
Temporary stack.
const VarDecl * EvaluatingDecl
Declaration we're initializing/evaluting, if any.
InterpFrame * Current
The current frame.
T allocAP(unsigned BitWidth)
StdAllocatorCaller getStdAllocatorCaller(StringRef Name) const
Program & P
Reference to the module containing all bytecode.
PrimType value_or(PrimType PT) const
Definition PrimType.h:88
A pointer to a memory block, live or dead.
Definition Pointer.h:97
Pointer narrow() const
Restricts the scope of an array element pointer.
Definition Pointer.h:194
Pointer stripBaseCasts() const
Strip base casts from this Pointer.
Definition Pointer.h:797
bool isInitialized() const
Checks if an object was initialized.
Definition Pointer.cpp:443
bool isElementInitialized(unsigned Index) const
Like isInitialized(), but for primitive arrays.
Definition Pointer.cpp:465
Pointer atIndex(uint64_t Idx) const
Offsets a pointer inside an array.
Definition Pointer.h:162
bool isDummy() const
Checks if the pointer points to a dummy value.
Definition Pointer.h:562
int64_t getIndex() const
Returns the index into an array.
Definition Pointer.h:627
bool isActive() const
Checks if the object is active.
Definition Pointer.h:551
Pointer atField(unsigned Off) const
Creates a pointer to a field.
Definition Pointer.h:179
T & deref() const
Dereferences the pointer, if it's live.
Definition Pointer.h:687
unsigned getNumElems() const
Returns the number of elements.
Definition Pointer.h:611
Pointer getArray() const
Returns the parent array.
Definition Pointer.h:326
bool isUnknownSizeArray() const
Checks if the structure is an array of unknown size.
Definition Pointer.h:426
void activate() const
Activats a field.
Definition Pointer.cpp:623
bool isIntegralPointer() const
Definition Pointer.h:480
QualType getType() const
Returns the type of the innermost field.
Definition Pointer.h:346
bool isArrayElement() const
Checks if the pointer points to an array.
Definition Pointer.h:432
void initializeAllElements() const
Initialize all elements of a primitive array at once.
Definition Pointer.cpp:582
bool isLive() const
Checks if the pointer is live.
Definition Pointer.h:278
bool inArray() const
Checks if the innermost field is an array.
Definition Pointer.h:408
T & elem(unsigned I) const
Dereferences the element at index I.
Definition Pointer.h:703
Pointer getBase() const
Returns a pointer to the object of which this pointer is a field.
Definition Pointer.h:317
std::string toDiagnosticString(const ASTContext &Ctx) const
Converts the pointer to a string usable in diagnostics.
Definition Pointer.cpp:433
bool isZero() const
Checks if the pointer is null.
Definition Pointer.h:264
bool isRoot() const
Pointer points directly to a block.
Definition Pointer.h:448
const Descriptor * getDeclDesc() const
Accessor for information about the declaration site.
Definition Pointer.h:292
static bool pointToSameBlock(const Pointer &A, const Pointer &B)
Checks if both given pointers point to the same block.
Definition Pointer.cpp:704
APValue toAPValue(const ASTContext &ASTCtx) const
Converts the pointer to an APValue.
Definition Pointer.cpp:173
bool isOnePastEnd() const
Checks if the index is one past end.
Definition Pointer.h:644
uint64_t getIntegerRepresentation() const
Definition Pointer.h:149
const FieldDecl * getField() const
Returns the field information.
Definition Pointer.h:492
Pointer expand() const
Expands a pointer to the containing array, undoing narrowing.
Definition Pointer.h:229
bool isElementPastEnd() const
Checks if the pointer is an out-of-bounds element pointer.
Definition Pointer.h:666
bool isBlockPointer() const
Definition Pointer.h:479
const Block * block() const
Definition Pointer.h:617
const Descriptor * getFieldDesc() const
Accessors for information about the innermost field.
Definition Pointer.h:336
bool isVirtualBaseClass() const
Definition Pointer.h:558
bool isBaseClass() const
Checks if a structure is a base class.
Definition Pointer.h:557
bool canBeInitialized() const
If this pointer has an InlineDescriptor we can use to initialize.
Definition Pointer.h:455
Lifetime getLifetime() const
Definition Pointer.h:761
void initialize() const
Initializes a field.
Definition Pointer.cpp:535
bool isField() const
Checks if the item is a field in an object.
Definition Pointer.h:284
const Record * getRecord() const
Returns the record descriptor of a class.
Definition Pointer.h:485
Descriptor * createDescriptor(const DeclTy &D, PrimType T, const Type *SourceTy=nullptr, Descriptor::MetadataSize MDSize=std::nullopt, bool IsConst=false, bool IsTemporary=false, bool IsMutable=false, bool IsVolatile=false)
Creates a descriptor for a primitive type.
Definition Program.h:121
Structure/Class descriptor.
Definition Record.h:25
const RecordDecl * getDecl() const
Returns the underlying declaration.
Definition Record.h:65
unsigned getNumFields() const
Definition Record.h:94
Describes the statement/declaration an opcode was generated from.
Definition Source.h:74
OptionalDiagnostic Note(SourceLocation Loc, diag::kind DiagId)
Add a note to a prior diagnostic.
Definition State.cpp:63
Expr::EvalStatus & getEvalStatus() const
Definition State.h:92
DiagnosticBuilder report(SourceLocation Loc, diag::kind DiagId)
Directly reports a diagnostic message.
Definition State.cpp:74
OptionalDiagnostic FFDiag(SourceLocation Loc, diag::kind DiagId=diag::note_invalid_subexpr_in_const_expr, unsigned ExtraNotes=0)
Diagnose that the evaluation could not be folded (FF => FoldFailure)
Definition State.cpp:21
ASTContext & getASTContext() const
Definition State.h:93
OptionalDiagnostic CCEDiag(SourceLocation Loc, diag::kind DiagId=diag::note_invalid_subexpr_in_const_expr, unsigned ExtraNotes=0)
Diagnose that the evaluation does not produce a C++11 core constant expression.
Definition State.cpp:42
const LangOptions & getLangOpts() const
Definition State.h:94
bool checkingPotentialConstantExpression() const
Are we checking whether the expression is a potential constant expression?
Definition State.h:120
Defines the clang::TargetInfo interface.
bool computeOSLogBufferLayout(clang::ASTContext &Ctx, const clang::CallExpr *E, OSLogBufferLayout &layout)
Definition OSLog.cpp:192
std::optional< llvm::AllocTokenMetadata > getAllocTokenMetadata(QualType T, const ASTContext &Ctx)
Get the information required for construction of an allocation token ID.
QualType inferPossibleType(const CallExpr *E, const ASTContext &Ctx, const CastExpr *CastE)
Infer the possible allocated type from an allocation call expression.
static bool isNoopBuiltin(unsigned ID)
static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC, const CallExpr *Call)
static bool interp__builtin_ia32_shuffle_generic(InterpState &S, CodePtr OpPC, const CallExpr *Call, llvm::function_ref< std::pair< unsigned, int >(unsigned, const APInt &)> GetSourceIndex)
static bool interp__builtin_ia32_phminposuw(InterpState &S, CodePtr OpPC, const CallExpr *Call)
static void assignInteger(InterpState &S, const Pointer &Dest, PrimType ValueT, const APSInt &Value)
static bool interp_builtin_ia32_gfni_affine(InterpState &S, CodePtr OpPC, const CallExpr *Call, bool Inverse)
bool readPointerToBuffer(const Context &Ctx, const Pointer &FromPtr, BitcastBuffer &Buffer, bool ReturnOnUninit)
static Floating abs(InterpState &S, const Floating &In)
static bool interp__builtin_x86_extract_vector(InterpState &S, CodePtr OpPC, const CallExpr *Call, unsigned ID)
static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, bool IsNumBuiltin)
static bool interp__builtin_elementwise_maxmin(InterpState &S, CodePtr OpPC, const CallExpr *Call, unsigned BuiltinID)
static bool interp__builtin_bswap(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_elementwise_triop(InterpState &S, CodePtr OpPC, const CallExpr *Call, llvm::function_ref< APInt(const APSInt &, const APSInt &, const APSInt &)> Fn)
static bool interp__builtin_assume(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
bool CheckNewDeleteForms(InterpState &S, CodePtr OpPC, DynamicAllocator::Form AllocForm, DynamicAllocator::Form DeleteForm, const Descriptor *D, const Expr *NewExpr)
Diagnose mismatched new[]/delete or new/delete[] pairs.
Definition Interp.cpp:1116
static bool interp__builtin_ia32_shift_with_count(InterpState &S, CodePtr OpPC, const CallExpr *Call, llvm::function_ref< APInt(const APInt &, uint64_t)> ShiftOp, llvm::function_ref< APInt(const APInt &, unsigned)> OverflowOp)
static bool interp__builtin_isnan(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
Defined as __builtin_isnan(...), to accommodate the fact that it can take a float,...
static llvm::RoundingMode getRoundingMode(FPOptions FPO)
static bool interp__builtin_ia32_crc32(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call, unsigned DataBytes)
static bool interp__builtin_elementwise_countzeroes(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call, unsigned BuiltinID)
Can be called with an integer or vector as the first and only parameter.
bool Call(InterpState &S, CodePtr OpPC, const Function *Func, uint32_t VarArgSize)
Definition Interp.cpp:1667
static bool interp__builtin_classify_type(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, bool IsNumBuiltin)
bool SetThreeWayComparisonField(InterpState &S, CodePtr OpPC, const Pointer &Ptr, const APSInt &IntValue)
Sets the given integral value to the pointer, which is of a std::{weak,partial,strong}...
static bool interp__builtin_elementwise_fp_binop(InterpState &S, CodePtr OpPC, const CallExpr *Call, llvm::function_ref< std::optional< APFloat >(const APFloat &, const APFloat &, std::optional< APSInt > RoundingMode)> Fn, bool IsScalar=false)
static bool interp__builtin_operator_delete(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC, const InterpFrame *Frame)
static uint64_t popToUInt64(const InterpState &S, const Expr *E)
static bool interp__builtin_ia32_vpconflict(InterpState &S, CodePtr OpPC, const CallExpr *Call)
static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call, unsigned ID)
static bool interp__builtin_atomic_lock_free(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call, unsigned BuiltinOp)
bool __atomic_always_lock_free(size_t, void const volatile*) bool __atomic_is_lock_free(size_t,...
static llvm::APSInt convertBoolVectorToInt(const Pointer &Val)
static bool interp__builtin_move(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_clz(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call, unsigned BuiltinOp)
bool CheckMutable(InterpState &S, CodePtr OpPC, const Pointer &Ptr)
Checks if a pointer points to a mutable field.
Definition Interp.cpp:590
static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call, unsigned BuiltinOp)
__builtin_is_aligned() __builtin_align_up() __builtin_align_down() The first parameter is either an i...
static bool interp__builtin_ia32_addsub(InterpState &S, CodePtr OpPC, const CallExpr *Call)
static bool Activate(InterpState &S, CodePtr OpPC)
Definition Interp.h:2213
static bool interp__builtin_ia32_addcarry_subborrow(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call, unsigned BuiltinOp)
(CarryIn, LHS, RHS, Result)
static bool isOneByteCharacterType(QualType T)
Determine if T is a character type for which we guarantee that sizeof(T) == 1.
static bool convertDoubleToFloatStrict(APFloat Src, Floating &Dst, InterpState &S, const Expr *DiagExpr)
static unsigned computePointerOffset(const ASTContext &ASTCtx, const Pointer &Ptr)
Compute the byte offset of Ptr in the full declaration.
static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call, unsigned ID)
bool CheckLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr, AccessKinds AK)
Checks if a value can be loaded from a block.
Definition Interp.cpp:791
static bool interp__builtin_ia32_cmp_mask(InterpState &S, CodePtr OpPC, const CallExpr *Call, unsigned ID, bool IsUnsigned)
static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC, const CallExpr *Call, unsigned BuiltinOp)
static bool isReadable(const Pointer &P)
Check for common reasons a pointer can't be read from, which are usually not diagnosed in a builtin f...
static bool interp__builtin_inf(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_vec_ext(InterpState &S, CodePtr OpPC, const CallExpr *Call, unsigned ID)
static bool interp__builtin_ia32_dbpsadbw(InterpState &S, CodePtr OpPC, const CallExpr *Call)
static bool interp__builtin_ia32_test_op(InterpState &S, CodePtr OpPC, const CallExpr *Call, llvm::function_ref< bool(const APInt &A, const APInt &B)> Fn)
static bool interp__builtin_isinf(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, bool CheckSign, const CallExpr *Call)
static bool interp__builtin_os_log_format_buffer_size(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E, ArrayRef< int64_t > ArrayIndices, int64_t &IntResult)
Interpret an offsetof operation.
static bool interp__builtin_x86_insert_subvector(InterpState &S, CodePtr OpPC, const CallExpr *Call, unsigned ID)
bool CheckRange(InterpState &S, CodePtr OpPC, const Pointer &Ptr, AccessKinds AK)
Checks if a pointer is in range.
Definition Interp.cpp:513
static bool pointsToLastObject(const Pointer &Ptr)
Does Ptr point to the last subobject?
static bool interp__builtin_select(InterpState &S, CodePtr OpPC, const CallExpr *Call)
AVX512 predicated move: "Result = Mask[] ? LHS[] : RHS[]".
llvm::APFloat APFloat
Definition Floating.h:27
static void discard(InterpStack &Stk, PrimType T)
static bool interp__builtin_select_scalar(InterpState &S, const CallExpr *Call)
Scalar variant of AVX512 predicated select: Result[i] = (Mask bit 0) ?
bool CheckLive(InterpState &S, CodePtr OpPC, const Pointer &Ptr, AccessKinds AK)
Checks if a pointer is live and accessible.
Definition Interp.cpp:412
static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
Five int values followed by one floating value.
static bool interp__builtin_abs(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp_floating_comparison(InterpState &S, CodePtr OpPC, const CallExpr *Call, unsigned ID)
static bool handleOverflow(InterpState &S, CodePtr OpPC, const T &SrcValue)
llvm::APInt APInt
Definition FixedPoint.h:19
static bool copyComposite(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest, bool Activate)
static bool copyRecord(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest, bool Activate=false)
static bool interp__builtin_c11_atomic_is_lock_free(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
bool __c11_atomic_is_lock_free(size_t)
static void zeroAll(Pointer &Dest)
static bool interp__builtin_elementwise_int_binop(InterpState &S, CodePtr OpPC, const CallExpr *Call, llvm::function_ref< APInt(const APSInt &, const APSInt &)> Fn)
static bool interp__builtin_issubnormal(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_arithmetic_fence(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_x86_extract_vector_masked(InterpState &S, CodePtr OpPC, const CallExpr *Call, unsigned ID)
static bool interp__builtin_ia32_cvt_mask2vec(InterpState &S, CodePtr OpPC, const CallExpr *Call, unsigned ID)
PrimType
Enumeration of the primitive types of the VM.
Definition PrimType.h:34
static bool interp__builtin_isfinite(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call, uint32_t BuiltinID)
Interpret a builtin function.
static bool interp__builtin_expect(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_vec_set(InterpState &S, CodePtr OpPC, const CallExpr *Call, unsigned ID)
static bool interp__builtin_complex(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
__builtin_complex(Float A, float B);
static bool evalICmpImm(uint8_t Imm, const APSInt &A, const APSInt &B, bool IsUnsigned)
bool CheckDummy(InterpState &S, CodePtr OpPC, const Block *B, AccessKinds AK)
Checks if a pointer is a dummy pointer.
Definition Interp.cpp:1178
static bool interp__builtin_assume_aligned(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
__builtin_assume_aligned(Ptr, Alignment[, ExtraOffset])
static bool interp__builtin_ia32_cvt_vec2mask(InterpState &S, CodePtr OpPC, const CallExpr *Call, unsigned ID)
static bool interp__builtin_ptrauth_string_discriminator(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC, const CallExpr *Call, unsigned ID)
static bool interp__builtin_ia32_pmul(InterpState &S, CodePtr OpPC, const CallExpr *Call, llvm::function_ref< APInt(const APSInt &, const APSInt &, const APSInt &, const APSInt &)> Fn)
static bool interp__builtin_x86_pack(InterpState &S, CodePtr, const CallExpr *E, llvm::function_ref< APInt(const APSInt &)> PackFn)
static void pushInteger(InterpState &S, const APSInt &Val, QualType QT)
Pushes Val on the stack as the type given by QT.
static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call, unsigned ID)
bool CheckArray(InterpState &S, CodePtr OpPC, const Pointer &Ptr)
Checks if the array is offsetable.
Definition Interp.cpp:404
static bool interp__builtin_elementwise_abs(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call, unsigned BuiltinID)
static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC, const InterpFrame *Frame)
static bool interp__builtin_iszero(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_addressof(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_signbit(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
bool Error(InterpState &S, CodePtr OpPC)
Do nothing and just abort execution.
Definition Interp.h:3655
static bool interp__builtin_vector_reduce(InterpState &S, CodePtr OpPC, const CallExpr *Call, unsigned ID)
static bool interp__builtin_ia32_movmsk_op(InterpState &S, CodePtr OpPC, const CallExpr *Call)
static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call, unsigned ID)
static bool interp_builtin_horizontal_fp_binop(InterpState &S, CodePtr OpPC, const CallExpr *Call, llvm::function_ref< APFloat(const APFloat &, const APFloat &, llvm::RoundingMode)> Fn)
static bool interp__builtin_ia32_pclmulqdq(InterpState &S, CodePtr OpPC, const CallExpr *Call)
static bool interp__builtin_elementwise_triop_fp(InterpState &S, CodePtr OpPC, const CallExpr *Call, llvm::function_ref< APFloat(const APFloat &, const APFloat &, const APFloat &, llvm::RoundingMode)> Fn)
static bool interp__builtin_popcount(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_carryop(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call, unsigned BuiltinOp)
Three integral values followed by a pointer (lhs, rhs, carry, carryOut).
bool CheckArraySize(InterpState &S, CodePtr OpPC, uint64_t NumElems)
static bool interp__builtin_scalar_fp_round_mask_binop(InterpState &S, CodePtr OpPC, const CallExpr *Call, llvm::function_ref< std::optional< APFloat >(const APFloat &, const APFloat &, std::optional< APSInt >)> Fn)
static bool interp__builtin_ctz(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call, unsigned BuiltinID)
static bool interp__builtin_is_constant_evaluated(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static APSInt popToAPSInt(InterpStack &Stk, PrimType T)
static std::optional< unsigned > computeFullDescSize(const ASTContext &ASTCtx, const Descriptor *Desc)
static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
First parameter to __builtin_isfpclass is the floating value, the second one is an integral value.
static bool interp__builtin_ia32_vcvtps2ph(InterpState &S, CodePtr OpPC, const CallExpr *Call)
static bool interp__builtin_issignaling(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_ia32_multishiftqb(InterpState &S, CodePtr OpPC, const CallExpr *Call)
static bool interp__builtin_ia32_shufbitqmb_mask(InterpState &S, CodePtr OpPC, const CallExpr *Call)
static bool interp__builtin_nan(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call, bool Signaling)
bool DoMemcpy(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest)
Copy the contents of Src into Dest.
static bool interp__builtin_elementwise_int_unaryop(InterpState &S, CodePtr OpPC, const CallExpr *Call, llvm::function_ref< APInt(const APSInt &)> Fn)
constexpr bool isIntegerType(PrimType T)
Definition PrimType.h:53
static bool interp__builtin_eh_return_data_regno(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static bool interp__builtin_infer_alloc_token(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
UnsignedOrNone evaluateBuiltinObjectSize(const ASTContext &ASTCtx, unsigned Kind, Pointer &Ptr)
static bool interp_builtin_horizontal_int_binop(InterpState &S, CodePtr OpPC, const CallExpr *Call, llvm::function_ref< APInt(const APSInt &, const APSInt &)> Fn)
static bool interp__builtin_ia32_cvtsd2ss(InterpState &S, CodePtr OpPC, const CallExpr *Call, bool HasRoundingMask)
static void diagnoseNonConstexprBuiltin(InterpState &S, CodePtr OpPC, unsigned ID)
llvm::APSInt APSInt
Definition FixedPoint.h:20
static bool interp__builtin_ia32_gfni_mul(InterpState &S, CodePtr OpPC, const CallExpr *Call)
static QualType getElemType(const Pointer &P)
static bool interp__builtin_ia32_pternlog(InterpState &S, CodePtr OpPC, const CallExpr *Call, bool MaskZ)
static bool interp__builtin_isnormal(InterpState &S, CodePtr OpPC, const InterpFrame *Frame, const CallExpr *Call)
static void swapBytes(std::byte *M, size_t N)
static bool interp__builtin_ia32_cvtpd2ps(InterpState &S, CodePtr OpPC, const CallExpr *Call, bool IsMasked, bool HasRounding)
The JSON file list parser is used to communicate input to InstallAPI.
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
ComparisonCategoryResult
An enumeration representing the possible results of a three-way comparison.
@ Result
The result type of a method or function.
Definition TypeBase.h:905
@ AK_Read
Definition State.h:29
OptionalUnsigned< unsigned > UnsignedOrNone
U cast(CodeGen::Address addr)
Definition Address.h:327
SmallVectorImpl< PartialDiagnosticAt > * Diag
Diag - If this is non-null, it will be filled in with a stack of notes indicating why evaluation fail...
Definition Expr.h:636
Track what bits have been initialized to known values and which ones have indeterminate value.
T deref(Bytes Offset) const
Dereferences the value at the given offset.
std::unique_ptr< std::byte[]> Data
A quantity in bits.
A quantity in bytes.
size_t getQuantity() const
Describes a memory block created by an allocation site.
Definition Descriptor.h:122
unsigned getNumElems() const
Returns the number of elements stored in the block.
Definition Descriptor.h:259
bool isPrimitive() const
Checks if the descriptor is of a primitive.
Definition Descriptor.h:273
QualType getElemQualType() const
bool isCompositeArray() const
Checks if the descriptor is of an array of composites.
Definition Descriptor.h:266
const ValueDecl * asValueDecl() const
Definition Descriptor.h:215
static constexpr unsigned MaxArrayElemBytes
Maximum number of bytes to be used for array elements.
Definition Descriptor.h:148
QualType getType() const
const Decl * asDecl() const
Definition Descriptor.h:211
static constexpr MetadataSize InlineDescMD
Definition Descriptor.h:144
unsigned getElemDataSize() const
Returns the element data size, i.e.
bool isPrimitiveArray() const
Checks if the descriptor is of an array of primitives.
Definition Descriptor.h:264
const VarDecl * asVarDecl() const
Definition Descriptor.h:219
PrimType getPrimType() const
Definition Descriptor.h:241
bool isRecord() const
Checks if the descriptor is of a record.
Definition Descriptor.h:278
const Record *const ElemRecord
Pointer to the record, if block contains records.
Definition Descriptor.h:153
const Expr * asExpr() const
Definition Descriptor.h:212
bool isArray() const
Checks if the descriptor is of an array.
Definition Descriptor.h:276
Mapping from primitive types to their representation.
Definition PrimType.h:150