clang  7.0.0svn
CGBuiltin.cpp
Go to the documentation of this file.
1 //===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This contains code to emit Builtin calls as LLVM code.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGCXXABI.h"
15 #include "CGObjCRuntime.h"
16 #include "CGOpenCLRuntime.h"
17 #include "CGRecordLayout.h"
18 #include "CodeGenFunction.h"
19 #include "CodeGenModule.h"
20 #include "ConstantEmitter.h"
21 #include "TargetInfo.h"
22 #include "clang/AST/ASTContext.h"
23 #include "clang/AST/Decl.h"
26 #include "clang/Basic/TargetInfo.h"
28 #include "llvm/ADT/StringExtras.h"
29 #include "llvm/IR/CallSite.h"
30 #include "llvm/IR/DataLayout.h"
31 #include "llvm/IR/InlineAsm.h"
32 #include "llvm/IR/Intrinsics.h"
33 #include "llvm/IR/MDBuilder.h"
34 #include "llvm/Support/ConvertUTF.h"
35 #include "llvm/Support/ScopedPrinter.h"
36 #include "llvm/Support/TargetParser.h"
37 #include <sstream>
38 
39 using namespace clang;
40 using namespace CodeGen;
41 using namespace llvm;
42 
43 static
44 int64_t clamp(int64_t Value, int64_t Low, int64_t High) {
45  return std::min(High, std::max(Low, Value));
46 }
47 
48 /// getBuiltinLibFunction - Given a builtin id for a function like
49 /// "__builtin_fabsf", return a Function* for "fabsf".
51  unsigned BuiltinID) {
52  assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
53 
54  // Get the name, skip over the __builtin_ prefix (if necessary).
55  StringRef Name;
56  GlobalDecl D(FD);
57 
58  // If the builtin has been declared explicitly with an assembler label,
59  // use the mangled name. This differs from the plain label on platforms
60  // that prefix labels.
61  if (FD->hasAttr<AsmLabelAttr>())
62  Name = getMangledName(D);
63  else
64  Name = Context.BuiltinInfo.getName(BuiltinID) + 10;
65 
66  llvm::FunctionType *Ty =
67  cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
68 
69  return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);
70 }
71 
72 /// Emit the conversions required to turn the given value into an
73 /// integer of the given size.
75  QualType T, llvm::IntegerType *IntType) {
76  V = CGF.EmitToMemory(V, T);
77 
78  if (V->getType()->isPointerTy())
79  return CGF.Builder.CreatePtrToInt(V, IntType);
80 
81  assert(V->getType() == IntType);
82  return V;
83 }
84 
86  QualType T, llvm::Type *ResultType) {
87  V = CGF.EmitFromMemory(V, T);
88 
89  if (ResultType->isPointerTy())
90  return CGF.Builder.CreateIntToPtr(V, ResultType);
91 
92  assert(V->getType() == ResultType);
93  return V;
94 }
95 
96 /// Utility to insert an atomic instruction based on Instrinsic::ID
97 /// and the expression node.
99  llvm::AtomicRMWInst::BinOp Kind,
100  const CallExpr *E) {
101  QualType T = E->getType();
102  assert(E->getArg(0)->getType()->isPointerType());
103  assert(CGF.getContext().hasSameUnqualifiedType(T,
104  E->getArg(0)->getType()->getPointeeType()));
105  assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
106 
107  llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
108  unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
109 
110  llvm::IntegerType *IntType =
111  llvm::IntegerType::get(CGF.getLLVMContext(),
112  CGF.getContext().getTypeSize(T));
113  llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
114 
115  llvm::Value *Args[2];
116  Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
117  Args[1] = CGF.EmitScalarExpr(E->getArg(1));
118  llvm::Type *ValueType = Args[1]->getType();
119  Args[1] = EmitToInt(CGF, Args[1], T, IntType);
120 
121  llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
122  Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
123  return EmitFromInt(CGF, Result, T, ValueType);
124 }
125 
127  Value *Val = CGF.EmitScalarExpr(E->getArg(0));
128  Value *Address = CGF.EmitScalarExpr(E->getArg(1));
129 
130  // Convert the type of the pointer to a pointer to the stored type.
131  Val = CGF.EmitToMemory(Val, E->getArg(0)->getType());
132  Value *BC = CGF.Builder.CreateBitCast(
133  Address, llvm::PointerType::getUnqual(Val->getType()), "cast");
134  LValue LV = CGF.MakeNaturalAlignAddrLValue(BC, E->getArg(0)->getType());
135  LV.setNontemporal(true);
136  CGF.EmitStoreOfScalar(Val, LV, false);
137  return nullptr;
138 }
139 
141  Value *Address = CGF.EmitScalarExpr(E->getArg(0));
142 
143  LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType());
144  LV.setNontemporal(true);
145  return CGF.EmitLoadOfScalar(LV, E->getExprLoc());
146 }
147 
149  llvm::AtomicRMWInst::BinOp Kind,
150  const CallExpr *E) {
151  return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E));
152 }
153 
154 /// Utility to insert an atomic instruction based Instrinsic::ID and
155 /// the expression node, where the return value is the result of the
156 /// operation.
158  llvm::AtomicRMWInst::BinOp Kind,
159  const CallExpr *E,
160  Instruction::BinaryOps Op,
161  bool Invert = false) {
162  QualType T = E->getType();
163  assert(E->getArg(0)->getType()->isPointerType());
164  assert(CGF.getContext().hasSameUnqualifiedType(T,
165  E->getArg(0)->getType()->getPointeeType()));
166  assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
167 
168  llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
169  unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
170 
171  llvm::IntegerType *IntType =
172  llvm::IntegerType::get(CGF.getLLVMContext(),
173  CGF.getContext().getTypeSize(T));
174  llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
175 
176  llvm::Value *Args[2];
177  Args[1] = CGF.EmitScalarExpr(E->getArg(1));
178  llvm::Type *ValueType = Args[1]->getType();
179  Args[1] = EmitToInt(CGF, Args[1], T, IntType);
180  Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
181 
182  llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
183  Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
184  Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]);
185  if (Invert)
186  Result = CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,
187  llvm::ConstantInt::get(IntType, -1));
188  Result = EmitFromInt(CGF, Result, T, ValueType);
189  return RValue::get(Result);
190 }
191 
192 /// Utility to insert an atomic cmpxchg instruction.
193 ///
194 /// @param CGF The current codegen function.
195 /// @param E Builtin call expression to convert to cmpxchg.
196 /// arg0 - address to operate on
197 /// arg1 - value to compare with
198 /// arg2 - new value
199 /// @param ReturnBool Specifies whether to return success flag of
200 /// cmpxchg result or the old value.
201 ///
202 /// @returns result of cmpxchg, according to ReturnBool
204  bool ReturnBool) {
205  QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType();
206  llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
207  unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
208 
209  llvm::IntegerType *IntType = llvm::IntegerType::get(
210  CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
211  llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
212 
213  Value *Args[3];
214  Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
215  Args[1] = CGF.EmitScalarExpr(E->getArg(1));
216  llvm::Type *ValueType = Args[1]->getType();
217  Args[1] = EmitToInt(CGF, Args[1], T, IntType);
218  Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);
219 
220  Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
221  Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,
222  llvm::AtomicOrdering::SequentiallyConsistent);
223  if (ReturnBool)
224  // Extract boolean success flag and zext it to int.
225  return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1),
226  CGF.ConvertType(E->getType()));
227  else
228  // Extract old value and emit it using the same type as compare value.
229  return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T,
230  ValueType);
231 }
232 
233 // Emit a simple mangled intrinsic that has 1 argument and a return type
234 // matching the argument type.
236  const CallExpr *E,
237  unsigned IntrinsicID) {
238  llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
239 
240  Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
241  return CGF.Builder.CreateCall(F, Src0);
242 }
243 
244 // Emit an intrinsic that has 2 operands of the same type as its result.
246  const CallExpr *E,
247  unsigned IntrinsicID) {
248  llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
249  llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
250 
251  Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
252  return CGF.Builder.CreateCall(F, { Src0, Src1 });
253 }
254 
255 // Emit an intrinsic that has 3 operands of the same type as its result.
257  const CallExpr *E,
258  unsigned IntrinsicID) {
259  llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
260  llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
261  llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
262 
263  Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
264  return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });
265 }
266 
267 // Emit an intrinsic that has 1 float or double operand, and 1 integer.
269  const CallExpr *E,
270  unsigned IntrinsicID) {
271  llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
272  llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
273 
274  Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
275  return CGF.Builder.CreateCall(F, {Src0, Src1});
276 }
277 
278 /// EmitFAbs - Emit a call to @llvm.fabs().
279 static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {
280  Value *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType());
281  llvm::CallInst *Call = CGF.Builder.CreateCall(F, V);
282  Call->setDoesNotAccessMemory();
283  return Call;
284 }
285 
286 /// Emit the computation of the sign bit for a floating point value. Returns
287 /// the i1 sign bit value.
289  LLVMContext &C = CGF.CGM.getLLVMContext();
290 
291  llvm::Type *Ty = V->getType();
292  int Width = Ty->getPrimitiveSizeInBits();
293  llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
294  V = CGF.Builder.CreateBitCast(V, IntTy);
295  if (Ty->isPPC_FP128Ty()) {
296  // We want the sign bit of the higher-order double. The bitcast we just
297  // did works as if the double-double was stored to memory and then
298  // read as an i128. The "store" will put the higher-order double in the
299  // lower address in both little- and big-Endian modes, but the "load"
300  // will treat those bits as a different part of the i128: the low bits in
301  // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
302  // we need to shift the high bits down to the low before truncating.
303  Width >>= 1;
304  if (CGF.getTarget().isBigEndian()) {
305  Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
306  V = CGF.Builder.CreateLShr(V, ShiftCst);
307  }
308  // We are truncating value in order to extract the higher-order
309  // double, which we will be using to extract the sign from.
310  IntTy = llvm::IntegerType::get(C, Width);
311  V = CGF.Builder.CreateTrunc(V, IntTy);
312  }
313  Value *Zero = llvm::Constant::getNullValue(IntTy);
314  return CGF.Builder.CreateICmpSLT(V, Zero);
315 }
316 
318  const CallExpr *E, llvm::Constant *calleeValue) {
319  CGCallee callee = CGCallee::forDirect(calleeValue, FD);
320  return CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot());
321 }
322 
323 /// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
324 /// depending on IntrinsicID.
325 ///
326 /// \arg CGF The current codegen function.
327 /// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
328 /// \arg X The first argument to the llvm.*.with.overflow.*.
329 /// \arg Y The second argument to the llvm.*.with.overflow.*.
330 /// \arg Carry The carry returned by the llvm.*.with.overflow.*.
331 /// \returns The result (i.e. sum/product) returned by the intrinsic.
333  const llvm::Intrinsic::ID IntrinsicID,
335  llvm::Value *&Carry) {
336  // Make sure we have integers of the same width.
337  assert(X->getType() == Y->getType() &&
338  "Arguments must be the same type. (Did you forget to make sure both "
339  "arguments have the same integer width?)");
340 
341  llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType());
342  llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y});
343  Carry = CGF.Builder.CreateExtractValue(Tmp, 1);
344  return CGF.Builder.CreateExtractValue(Tmp, 0);
345 }
346 
348  unsigned IntrinsicID,
349  int low, int high) {
350  llvm::MDBuilder MDHelper(CGF.getLLVMContext());
351  llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
352  Value *F = CGF.CGM.getIntrinsic(IntrinsicID, {});
353  llvm::Instruction *Call = CGF.Builder.CreateCall(F);
354  Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
355  return Call;
356 }
357 
358 namespace {
359  struct WidthAndSignedness {
360  unsigned Width;
361  bool Signed;
362  };
363 }
364 
365 static WidthAndSignedness
367  const clang::QualType Type) {
368  assert(Type->isIntegerType() && "Given type is not an integer.");
369  unsigned Width = Type->isBooleanType() ? 1 : context.getTypeInfo(Type).Width;
370  bool Signed = Type->isSignedIntegerType();
371  return {Width, Signed};
372 }
373 
374 // Given one or more integer types, this function produces an integer type that
375 // encompasses them: any value in one of the given types could be expressed in
376 // the encompassing type.
377 static struct WidthAndSignedness
378 EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
379  assert(Types.size() > 0 && "Empty list of types.");
380 
381  // If any of the given types is signed, we must return a signed type.
382  bool Signed = false;
383  for (const auto &Type : Types) {
384  Signed |= Type.Signed;
385  }
386 
387  // The encompassing type must have a width greater than or equal to the width
388  // of the specified types. Additionally, if the encompassing type is signed,
389  // its width must be strictly greater than the width of any unsigned types
390  // given.
391  unsigned Width = 0;
392  for (const auto &Type : Types) {
393  unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
394  if (Width < MinWidth) {
395  Width = MinWidth;
396  }
397  }
398 
399  return {Width, Signed};
400 }
401 
402 Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
403  llvm::Type *DestType = Int8PtrTy;
404  if (ArgValue->getType() != DestType)
405  ArgValue =
406  Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());
407 
408  Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
409  return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue);
410 }
411 
412 /// Checks if using the result of __builtin_object_size(p, @p From) in place of
413 /// __builtin_object_size(p, @p To) is correct
414 static bool areBOSTypesCompatible(int From, int To) {
415  // Note: Our __builtin_object_size implementation currently treats Type=0 and
416  // Type=2 identically. Encoding this implementation detail here may make
417  // improving __builtin_object_size difficult in the future, so it's omitted.
418  return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
419 }
420 
421 static llvm::Value *
422 getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
423  return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true);
424 }
425 
426 llvm::Value *
427 CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
428  llvm::IntegerType *ResType,
429  llvm::Value *EmittedE) {
430  uint64_t ObjectSize;
431  if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type))
432  return emitBuiltinObjectSize(E, Type, ResType, EmittedE);
433  return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true);
434 }
435 
436 /// Returns a Value corresponding to the size of the given expression.
437 /// This Value may be either of the following:
438 /// - A llvm::Argument (if E is a param with the pass_object_size attribute on
439 /// it)
440 /// - A call to the @llvm.objectsize intrinsic
441 ///
442 /// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
443 /// and we wouldn't otherwise try to reference a pass_object_size parameter,
444 /// we'll call @llvm.objectsize on EmittedE, rather than emitting E.
445 llvm::Value *
446 CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
447  llvm::IntegerType *ResType,
448  llvm::Value *EmittedE) {
449  // We need to reference an argument if the pointer is a parameter with the
450  // pass_object_size attribute.
451  if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) {
452  auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
453  auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
454  if (Param != nullptr && PS != nullptr &&
455  areBOSTypesCompatible(PS->getType(), Type)) {
456  auto Iter = SizeArguments.find(Param);
457  assert(Iter != SizeArguments.end());
458 
459  const ImplicitParamDecl *D = Iter->second;
460  auto DIter = LocalDeclMap.find(D);
461  assert(DIter != LocalDeclMap.end());
462 
463  return EmitLoadOfScalar(DIter->second, /*volatile=*/false,
464  getContext().getSizeType(), E->getLocStart());
465  }
466  }
467 
468  // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
469  // evaluate E for side-effects. In either case, we shouldn't lower to
470  // @llvm.objectsize.
471  if (Type == 3 || (!EmittedE && E->HasSideEffects(getContext())))
472  return getDefaultBuiltinObjectSizeResult(Type, ResType);
473 
474  Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
475  assert(Ptr->getType()->isPointerTy() &&
476  "Non-pointer passed to __builtin_object_size?");
477 
478  Value *F = CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
479 
480  // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.
481  Value *Min = Builder.getInt1((Type & 2) != 0);
482  // For GCC compatibility, __builtin_object_size treat NULL as unknown size.
483  Value *NullIsUnknown = Builder.getTrue();
484  return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown});
485 }
486 
487 // Many of MSVC builtins are on both x64 and ARM; to avoid repeating code, we
488 // handle them here.
490  _BitScanForward,
491  _BitScanReverse,
492  _InterlockedAnd,
493  _InterlockedDecrement,
494  _InterlockedExchange,
495  _InterlockedExchangeAdd,
496  _InterlockedExchangeSub,
497  _InterlockedIncrement,
498  _InterlockedOr,
499  _InterlockedXor,
500  _interlockedbittestandset,
501  __fastfail,
502 };
503 
505  const CallExpr *E) {
506  switch (BuiltinID) {
507  case MSVCIntrin::_BitScanForward:
508  case MSVCIntrin::_BitScanReverse: {
509  Value *ArgValue = EmitScalarExpr(E->getArg(1));
510 
511  llvm::Type *ArgType = ArgValue->getType();
512  llvm::Type *IndexType =
513  EmitScalarExpr(E->getArg(0))->getType()->getPointerElementType();
514  llvm::Type *ResultType = ConvertType(E->getType());
515 
516  Value *ArgZero = llvm::Constant::getNullValue(ArgType);
517  Value *ResZero = llvm::Constant::getNullValue(ResultType);
518  Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
519 
520  BasicBlock *Begin = Builder.GetInsertBlock();
521  BasicBlock *End = createBasicBlock("bitscan_end", this->CurFn);
522  Builder.SetInsertPoint(End);
523  PHINode *Result = Builder.CreatePHI(ResultType, 2, "bitscan_result");
524 
525  Builder.SetInsertPoint(Begin);
526  Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
527  BasicBlock *NotZero = createBasicBlock("bitscan_not_zero", this->CurFn);
528  Builder.CreateCondBr(IsZero, End, NotZero);
529  Result->addIncoming(ResZero, Begin);
530 
531  Builder.SetInsertPoint(NotZero);
532  Address IndexAddress = EmitPointerWithAlignment(E->getArg(0));
533 
534  if (BuiltinID == MSVCIntrin::_BitScanForward) {
535  Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
536  Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
537  ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
538  Builder.CreateStore(ZeroCount, IndexAddress, false);
539  } else {
540  unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
541  Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
542 
543  Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
544  Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
545  ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
546  Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
547  Builder.CreateStore(Index, IndexAddress, false);
548  }
549  Builder.CreateBr(End);
550  Result->addIncoming(ResOne, NotZero);
551 
552  Builder.SetInsertPoint(End);
553  return Result;
554  }
555  case MSVCIntrin::_InterlockedAnd:
556  return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E);
557  case MSVCIntrin::_InterlockedExchange:
558  return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E);
559  case MSVCIntrin::_InterlockedExchangeAdd:
560  return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E);
561  case MSVCIntrin::_InterlockedExchangeSub:
562  return MakeBinaryAtomicValue(*this, AtomicRMWInst::Sub, E);
563  case MSVCIntrin::_InterlockedOr:
564  return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E);
565  case MSVCIntrin::_InterlockedXor:
566  return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E);
567 
568  case MSVCIntrin::_interlockedbittestandset: {
569  llvm::Value *Addr = EmitScalarExpr(E->getArg(0));
570  llvm::Value *Bit = EmitScalarExpr(E->getArg(1));
571  AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
572  AtomicRMWInst::Or, Addr,
573  Builder.CreateShl(ConstantInt::get(Bit->getType(), 1), Bit),
574  llvm::AtomicOrdering::SequentiallyConsistent);
575  // Shift the relevant bit to the least significant position, truncate to
576  // the result type, and test the low bit.
577  llvm::Value *Shifted = Builder.CreateLShr(RMWI, Bit);
578  llvm::Value *Truncated =
579  Builder.CreateTrunc(Shifted, ConvertType(E->getType()));
580  return Builder.CreateAnd(Truncated,
581  ConstantInt::get(Truncated->getType(), 1));
582  }
583 
584  case MSVCIntrin::_InterlockedDecrement: {
585  llvm::Type *IntTy = ConvertType(E->getType());
586  AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
587  AtomicRMWInst::Sub,
588  EmitScalarExpr(E->getArg(0)),
589  ConstantInt::get(IntTy, 1),
590  llvm::AtomicOrdering::SequentiallyConsistent);
591  return Builder.CreateSub(RMWI, ConstantInt::get(IntTy, 1));
592  }
593  case MSVCIntrin::_InterlockedIncrement: {
594  llvm::Type *IntTy = ConvertType(E->getType());
595  AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
596  AtomicRMWInst::Add,
597  EmitScalarExpr(E->getArg(0)),
598  ConstantInt::get(IntTy, 1),
599  llvm::AtomicOrdering::SequentiallyConsistent);
600  return Builder.CreateAdd(RMWI, ConstantInt::get(IntTy, 1));
601  }
602 
603  case MSVCIntrin::__fastfail: {
604  // Request immediate process termination from the kernel. The instruction
605  // sequences to do this are documented on MSDN:
606  // https://msdn.microsoft.com/en-us/library/dn774154.aspx
607  llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
608  StringRef Asm, Constraints;
609  switch (ISA) {
610  default:
611  ErrorUnsupported(E, "__fastfail call for this architecture");
612  break;
613  case llvm::Triple::x86:
614  case llvm::Triple::x86_64:
615  Asm = "int $$0x29";
616  Constraints = "{cx}";
617  break;
618  case llvm::Triple::thumb:
619  Asm = "udf #251";
620  Constraints = "{r0}";
621  break;
622  }
623  llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, {Int32Ty}, false);
624  llvm::InlineAsm *IA =
625  llvm::InlineAsm::get(FTy, Asm, Constraints, /*SideEffects=*/true);
626  llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
627  getLLVMContext(), llvm::AttributeList::FunctionIndex,
628  llvm::Attribute::NoReturn);
629  CallSite CS = Builder.CreateCall(IA, EmitScalarExpr(E->getArg(0)));
630  CS.setAttributes(NoReturnAttr);
631  return CS.getInstruction();
632  }
633  }
634  llvm_unreachable("Incorrect MSVC intrinsic!");
635 }
636 
637 namespace {
638 // ARC cleanup for __builtin_os_log_format
639 struct CallObjCArcUse final : EHScopeStack::Cleanup {
640  CallObjCArcUse(llvm::Value *object) : object(object) {}
641  llvm::Value *object;
642 
643  void Emit(CodeGenFunction &CGF, Flags flags) override {
644  CGF.EmitARCIntrinsicUse(object);
645  }
646 };
647 }
648 
651  assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero)
652  && "Unsupported builtin check kind");
653 
654  Value *ArgValue = EmitScalarExpr(E);
655  if (!SanOpts.has(SanitizerKind::Builtin) || !getTarget().isCLZForZeroUndef())
656  return ArgValue;
657 
658  SanitizerScope SanScope(this);
659  Value *Cond = Builder.CreateICmpNE(
660  ArgValue, llvm::Constant::getNullValue(ArgValue->getType()));
661  EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
662  SanitizerHandler::InvalidBuiltin,
663  {EmitCheckSourceLocation(E->getExprLoc()),
664  llvm::ConstantInt::get(Builder.getInt8Ty(), Kind)},
665  None);
666  return ArgValue;
667 }
668 
669 /// Get the argument type for arguments to os_log_helper.
670 static CanQualType getOSLogArgType(ASTContext &C, int Size) {
671  QualType UnsignedTy = C.getIntTypeForBitwidth(Size * 8, /*Signed=*/false);
672  return C.getCanonicalType(UnsignedTy);
673 }
674 
676  const analyze_os_log::OSLogBufferLayout &Layout,
677  CharUnits BufferAlignment) {
678  ASTContext &Ctx = getContext();
679 
681  {
682  raw_svector_ostream OS(Name);
683  OS << "__os_log_helper";
684  OS << "_" << BufferAlignment.getQuantity();
685  OS << "_" << int(Layout.getSummaryByte());
686  OS << "_" << int(Layout.getNumArgsByte());
687  for (const auto &Item : Layout.Items)
688  OS << "_" << int(Item.getSizeByte()) << "_"
689  << int(Item.getDescriptorByte());
690  }
691 
692  if (llvm::Function *F = CGM.getModule().getFunction(Name))
693  return F;
694 
696  Params.emplace_back(Ctx, nullptr, SourceLocation(), &Ctx.Idents.get("buffer"),
698 
699  for (unsigned int I = 0, E = Layout.Items.size(); I < E; ++I) {
700  char Size = Layout.Items[I].getSizeByte();
701  if (!Size)
702  continue;
703 
704  Params.emplace_back(
705  Ctx, nullptr, SourceLocation(),
706  &Ctx.Idents.get(std::string("arg") + llvm::to_string(I)),
708  }
709 
710  FunctionArgList Args;
711  for (auto &P : Params)
712  Args.push_back(&P);
713 
714  // The helper function has linkonce_odr linkage to enable the linker to merge
715  // identical functions. To ensure the merging always happens, 'noinline' is
716  // attached to the function when compiling with -Oz.
717  const CGFunctionInfo &FI =
718  CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
719  llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
720  llvm::Function *Fn = llvm::Function::Create(
721  FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &CGM.getModule());
722  Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
723  CGM.SetLLVMFunctionAttributes(nullptr, FI, Fn);
724  CGM.SetLLVMFunctionAttributesForDefinition(nullptr, Fn);
725 
726  // Attach 'noinline' at -Oz.
727  if (CGM.getCodeGenOpts().OptimizeSize == 2)
728  Fn->addFnAttr(llvm::Attribute::NoInline);
729 
730  auto NL = ApplyDebugLocation::CreateEmpty(*this);
731  IdentifierInfo *II = &Ctx.Idents.get(Name);
734  Ctx.VoidTy, nullptr, SC_PrivateExtern, false, false);
735 
736  StartFunction(FD, Ctx.VoidTy, Fn, FI, Args);
737 
738  // Create a scope with an artificial location for the body of this function.
739  auto AL = ApplyDebugLocation::CreateArtificial(*this);
740 
742  Address BufAddr(Builder.CreateLoad(GetAddrOfLocalVar(&Params[0]), "buf"),
743  BufferAlignment);
744  Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()),
745  Builder.CreateConstByteGEP(BufAddr, Offset++, "summary"));
746  Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()),
747  Builder.CreateConstByteGEP(BufAddr, Offset++, "numArgs"));
748 
749  unsigned I = 1;
750  for (const auto &Item : Layout.Items) {
751  Builder.CreateStore(
752  Builder.getInt8(Item.getDescriptorByte()),
753  Builder.CreateConstByteGEP(BufAddr, Offset++, "argDescriptor"));
754  Builder.CreateStore(
755  Builder.getInt8(Item.getSizeByte()),
756  Builder.CreateConstByteGEP(BufAddr, Offset++, "argSize"));
757 
758  CharUnits Size = Item.size();
759  if (!Size.getQuantity())
760  continue;
761 
762  Address Arg = GetAddrOfLocalVar(&Params[I]);
763  Address Addr = Builder.CreateConstByteGEP(BufAddr, Offset, "argData");
764  Addr = Builder.CreateBitCast(Addr, Arg.getPointer()->getType(),
765  "argDataCast");
766  Builder.CreateStore(Builder.CreateLoad(Arg), Addr);
767  Offset += Size;
768  ++I;
769  }
770 
771  FinishFunction();
772 
773  return Fn;
774 }
775 
777  assert(E.getNumArgs() >= 2 &&
778  "__builtin_os_log_format takes at least 2 arguments");
779  ASTContext &Ctx = getContext();
782  Address BufAddr = EmitPointerWithAlignment(E.getArg(0));
783  llvm::SmallVector<llvm::Value *, 4> RetainableOperands;
784 
785  // Ignore argument 1, the format string. It is not currently used.
786  CallArgList Args;
787  Args.add(RValue::get(BufAddr.getPointer()), Ctx.VoidPtrTy);
788 
789  for (const auto &Item : Layout.Items) {
790  int Size = Item.getSizeByte();
791  if (!Size)
792  continue;
793 
794  llvm::Value *ArgVal;
795 
796  if (const Expr *TheExpr = Item.getExpr()) {
797  ArgVal = EmitScalarExpr(TheExpr, /*Ignore*/ false);
798 
799  // Check if this is a retainable type.
800  if (TheExpr->getType()->isObjCRetainableType()) {
801  assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
802  "Only scalar can be a ObjC retainable type");
803  // Check if the object is constant, if not, save it in
804  // RetainableOperands.
805  if (!isa<Constant>(ArgVal))
806  RetainableOperands.push_back(ArgVal);
807  }
808  } else {
809  ArgVal = Builder.getInt32(Item.getConstValue().getQuantity());
810  }
811 
812  unsigned ArgValSize =
813  CGM.getDataLayout().getTypeSizeInBits(ArgVal->getType());
814  llvm::IntegerType *IntTy = llvm::Type::getIntNTy(getLLVMContext(),
815  ArgValSize);
816  ArgVal = Builder.CreateBitOrPointerCast(ArgVal, IntTy);
817  CanQualType ArgTy = getOSLogArgType(Ctx, Size);
818  // If ArgVal has type x86_fp80, zero-extend ArgVal.
819  ArgVal = Builder.CreateZExtOrBitCast(ArgVal, ConvertType(ArgTy));
820  Args.add(RValue::get(ArgVal), ArgTy);
821  }
822 
823  const CGFunctionInfo &FI =
824  CGM.getTypes().arrangeBuiltinFunctionCall(Ctx.VoidTy, Args);
825  llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction(
826  Layout, BufAddr.getAlignment());
827  EmitCall(FI, CGCallee::forDirect(F), ReturnValueSlot(), Args);
828 
829  // Push a clang.arc.use cleanup for each object in RetainableOperands. The
830  // cleanup will cause the use to appear after the final log call, keeping
831  // the object valid while it’s held in the log buffer. Note that if there’s
832  // a release cleanup on the object, it will already be active; since
833  // cleanups are emitted in reverse order, the use will occur before the
834  // object is released.
835  if (!RetainableOperands.empty() && getLangOpts().ObjCAutoRefCount &&
836  CGM.getCodeGenOpts().OptimizationLevel != 0)
837  for (llvm::Value *Object : RetainableOperands)
838  pushFullExprCleanup<CallObjCArcUse>(getARCCleanupKind(), Object);
839 
840  return RValue::get(BufAddr.getPointer());
841 }
842 
843 /// Determine if a binop is a checked mixed-sign multiply we can specialize.
844 static bool isSpecialMixedSignMultiply(unsigned BuiltinID,
845  WidthAndSignedness Op1Info,
846  WidthAndSignedness Op2Info,
847  WidthAndSignedness ResultInfo) {
848  return BuiltinID == Builtin::BI__builtin_mul_overflow &&
849  Op1Info.Width == Op2Info.Width && Op1Info.Width >= ResultInfo.Width &&
850  Op1Info.Signed != Op2Info.Signed;
851 }
852 
853 /// Emit a checked mixed-sign multiply. This is a cheaper specialization of
854 /// the generic checked-binop irgen.
855 static RValue
857  WidthAndSignedness Op1Info, const clang::Expr *Op2,
858  WidthAndSignedness Op2Info,
859  const clang::Expr *ResultArg, QualType ResultQTy,
860  WidthAndSignedness ResultInfo) {
861  assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info,
862  Op2Info, ResultInfo) &&
863  "Not a mixed-sign multipliction we can specialize");
864 
865  // Emit the signed and unsigned operands.
866  const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
867  const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
868  llvm::Value *Signed = CGF.EmitScalarExpr(SignedOp);
869  llvm::Value *Unsigned = CGF.EmitScalarExpr(UnsignedOp);
870 
871  llvm::Type *OpTy = Signed->getType();
872  llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
873  Address ResultPtr = CGF.EmitPointerWithAlignment(ResultArg);
874  llvm::Type *ResTy = ResultPtr.getElementType();
875 
876  // Take the absolute value of the signed operand.
877  llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(Signed, Zero);
878  llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(Zero, Signed);
879  llvm::Value *AbsSigned =
880  CGF.Builder.CreateSelect(IsNegative, AbsOfNegative, Signed);
881 
882  // Perform a checked unsigned multiplication.
883  llvm::Value *UnsignedOverflow;
884  llvm::Value *UnsignedResult =
885  EmitOverflowIntrinsic(CGF, llvm::Intrinsic::umul_with_overflow, AbsSigned,
886  Unsigned, UnsignedOverflow);
887 
888  llvm::Value *Overflow, *Result;
889  if (ResultInfo.Signed) {
890  // Signed overflow occurs if the result is greater than INT_MAX or lesser
891  // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
892  auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width)
893  .zextOrSelf(Op1Info.Width);
894  llvm::Value *MaxResult =
895  CGF.Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
896  CGF.Builder.CreateZExt(IsNegative, OpTy));
897  llvm::Value *SignedOverflow =
898  CGF.Builder.CreateICmpUGT(UnsignedResult, MaxResult);
899  Overflow = CGF.Builder.CreateOr(UnsignedOverflow, SignedOverflow);
900 
901  // Prepare the signed result (possibly by negating it).
902  llvm::Value *NegativeResult = CGF.Builder.CreateNeg(UnsignedResult);
903  llvm::Value *SignedResult =
904  CGF.Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
905  Result = CGF.Builder.CreateTrunc(SignedResult, ResTy);
906  } else {
907  // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX.
908  llvm::Value *Underflow = CGF.Builder.CreateAnd(
909  IsNegative, CGF.Builder.CreateIsNotNull(UnsignedResult));
910  Overflow = CGF.Builder.CreateOr(UnsignedOverflow, Underflow);
911  if (ResultInfo.Width < Op1Info.Width) {
912  auto IntMax =
913  llvm::APInt::getMaxValue(ResultInfo.Width).zext(Op1Info.Width);
914  llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
915  UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
916  Overflow = CGF.Builder.CreateOr(Overflow, TruncOverflow);
917  }
918 
919  // Negate the product if it would be negative in infinite precision.
920  Result = CGF.Builder.CreateSelect(
921  IsNegative, CGF.Builder.CreateNeg(UnsignedResult), UnsignedResult);
922 
923  Result = CGF.Builder.CreateTrunc(Result, ResTy);
924  }
925  assert(Overflow && Result && "Missing overflow or result");
926 
927  bool isVolatile =
928  ResultArg->getType()->getPointeeType().isVolatileQualified();
929  CGF.Builder.CreateStore(CGF.EmitToMemory(Result, ResultQTy), ResultPtr,
930  isVolatile);
931  return RValue::get(Overflow);
932 }
933 
935  Value *&RecordPtr, CharUnits Align, Value *Func,
936  int Lvl) {
937  const auto *RT = RType->getAs<RecordType>();
938  ASTContext &Context = CGF.getContext();
939  RecordDecl *RD = RT->getDecl()->getDefinition();
940  ASTContext &Ctx = RD->getASTContext();
941  const ASTRecordLayout &RL = Ctx.getASTRecordLayout(RD);
942  std::string Pad = std::string(Lvl * 4, ' ');
943 
944  Value *GString =
945  CGF.Builder.CreateGlobalStringPtr(RType.getAsString() + " {\n");
946  Value *Res = CGF.Builder.CreateCall(Func, {GString});
947 
948  static llvm::DenseMap<QualType, const char *> Types;
949  if (Types.empty()) {
950  Types[Context.CharTy] = "%c";
951  Types[Context.BoolTy] = "%d";
952  Types[Context.SignedCharTy] = "%hhd";
953  Types[Context.UnsignedCharTy] = "%hhu";
954  Types[Context.IntTy] = "%d";
955  Types[Context.UnsignedIntTy] = "%u";
956  Types[Context.LongTy] = "%ld";
957  Types[Context.UnsignedLongTy] = "%lu";
958  Types[Context.LongLongTy] = "%lld";
959  Types[Context.UnsignedLongLongTy] = "%llu";
960  Types[Context.ShortTy] = "%hd";
961  Types[Context.UnsignedShortTy] = "%hu";
962  Types[Context.VoidPtrTy] = "%p";
963  Types[Context.FloatTy] = "%f";
964  Types[Context.DoubleTy] = "%f";
965  Types[Context.LongDoubleTy] = "%Lf";
966  Types[Context.getPointerType(Context.CharTy)] = "%s";
967  Types[Context.getPointerType(Context.getConstType(Context.CharTy))] = "%s";
968  }
969 
970  for (const auto *FD : RD->fields()) {
971  uint64_t Off = RL.getFieldOffset(FD->getFieldIndex());
972  Off = Ctx.toCharUnitsFromBits(Off).getQuantity();
973 
974  Value *FieldPtr = RecordPtr;
975  if (RD->isUnion())
976  FieldPtr = CGF.Builder.CreatePointerCast(
977  FieldPtr, CGF.ConvertType(Context.getPointerType(FD->getType())));
978  else
979  FieldPtr = CGF.Builder.CreateStructGEP(CGF.ConvertType(RType), FieldPtr,
980  FD->getFieldIndex());
981 
982  GString = CGF.Builder.CreateGlobalStringPtr(
983  llvm::Twine(Pad)
984  .concat(FD->getType().getAsString())
985  .concat(llvm::Twine(' '))
986  .concat(FD->getNameAsString())
987  .concat(" : ")
988  .str());
989  Value *TmpRes = CGF.Builder.CreateCall(Func, {GString});
990  Res = CGF.Builder.CreateAdd(Res, TmpRes);
991 
992  QualType CanonicalType =
993  FD->getType().getUnqualifiedType().getCanonicalType();
994 
995  // We check whether we are in a recursive type
996  if (CanonicalType->isRecordType()) {
997  Value *TmpRes =
998  dumpRecord(CGF, CanonicalType, FieldPtr, Align, Func, Lvl + 1);
999  Res = CGF.Builder.CreateAdd(TmpRes, Res);
1000  continue;
1001  }
1002 
1003  // We try to determine the best format to print the current field
1004  llvm::Twine Format = Types.find(CanonicalType) == Types.end()
1005  ? Types[Context.VoidPtrTy]
1006  : Types[CanonicalType];
1007 
1008  Address FieldAddress = Address(FieldPtr, Align);
1009  FieldPtr = CGF.Builder.CreateLoad(FieldAddress);
1010 
1011  // FIXME Need to handle bitfield here
1012  GString = CGF.Builder.CreateGlobalStringPtr(
1013  Format.concat(llvm::Twine('\n')).str());
1014  TmpRes = CGF.Builder.CreateCall(Func, {GString, FieldPtr});
1015  Res = CGF.Builder.CreateAdd(Res, TmpRes);
1016  }
1017 
1018  GString = CGF.Builder.CreateGlobalStringPtr(Pad + "}\n");
1019  Value *TmpRes = CGF.Builder.CreateCall(Func, {GString});
1020  Res = CGF.Builder.CreateAdd(Res, TmpRes);
1021  return Res;
1022 }
1023 
1025  unsigned BuiltinID, const CallExpr *E,
1026  ReturnValueSlot ReturnValue) {
1027  // See if we can constant fold this builtin. If so, don't emit it at all.
1028  Expr::EvalResult Result;
1029  if (E->EvaluateAsRValue(Result, CGM.getContext()) &&
1030  !Result.hasSideEffects()) {
1031  if (Result.Val.isInt())
1032  return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
1033  Result.Val.getInt()));
1034  if (Result.Val.isFloat())
1035  return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
1036  Result.Val.getFloat()));
1037  }
1038 
1039  // There are LLVM math intrinsics/instructions corresponding to math library
1040  // functions except the LLVM op will never set errno while the math library
1041  // might. Also, math builtins have the same semantics as their math library
1042  // twins. Thus, we can transform math library and builtin calls to their
1043  // LLVM counterparts if the call is marked 'const' (known to never set errno).
1044  if (FD->hasAttr<ConstAttr>()) {
1045  switch (BuiltinID) {
1046  case Builtin::BIceil:
1047  case Builtin::BIceilf:
1048  case Builtin::BIceill:
1049  case Builtin::BI__builtin_ceil:
1050  case Builtin::BI__builtin_ceilf:
1051  case Builtin::BI__builtin_ceill:
1052  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::ceil));
1053 
1054  case Builtin::BIcopysign:
1055  case Builtin::BIcopysignf:
1056  case Builtin::BIcopysignl:
1057  case Builtin::BI__builtin_copysign:
1058  case Builtin::BI__builtin_copysignf:
1059  case Builtin::BI__builtin_copysignl:
1060  case Builtin::BI__builtin_copysignf128:
1062 
1063  case Builtin::BIcos:
1064  case Builtin::BIcosf:
1065  case Builtin::BIcosl:
1066  case Builtin::BI__builtin_cos:
1067  case Builtin::BI__builtin_cosf:
1068  case Builtin::BI__builtin_cosl:
1069  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::cos));
1070 
1071  case Builtin::BIexp:
1072  case Builtin::BIexpf:
1073  case Builtin::BIexpl:
1074  case Builtin::BI__builtin_exp:
1075  case Builtin::BI__builtin_expf:
1076  case Builtin::BI__builtin_expl:
1077  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::exp));
1078 
1079  case Builtin::BIexp2:
1080  case Builtin::BIexp2f:
1081  case Builtin::BIexp2l:
1082  case Builtin::BI__builtin_exp2:
1083  case Builtin::BI__builtin_exp2f:
1084  case Builtin::BI__builtin_exp2l:
1085  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::exp2));
1086 
1087  case Builtin::BIfabs:
1088  case Builtin::BIfabsf:
1089  case Builtin::BIfabsl:
1090  case Builtin::BI__builtin_fabs:
1091  case Builtin::BI__builtin_fabsf:
1092  case Builtin::BI__builtin_fabsl:
1093  case Builtin::BI__builtin_fabsf128:
1094  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::fabs));
1095 
1096  case Builtin::BIfloor:
1097  case Builtin::BIfloorf:
1098  case Builtin::BIfloorl:
1099  case Builtin::BI__builtin_floor:
1100  case Builtin::BI__builtin_floorf:
1101  case Builtin::BI__builtin_floorl:
1102  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::floor));
1103 
1104  case Builtin::BIfma:
1105  case Builtin::BIfmaf:
1106  case Builtin::BIfmal:
1107  case Builtin::BI__builtin_fma:
1108  case Builtin::BI__builtin_fmaf:
1109  case Builtin::BI__builtin_fmal:
1110  return RValue::get(emitTernaryBuiltin(*this, E, Intrinsic::fma));
1111 
1112  case Builtin::BIfmax:
1113  case Builtin::BIfmaxf:
1114  case Builtin::BIfmaxl:
1115  case Builtin::BI__builtin_fmax:
1116  case Builtin::BI__builtin_fmaxf:
1117  case Builtin::BI__builtin_fmaxl:
1118  return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::maxnum));
1119 
1120  case Builtin::BIfmin:
1121  case Builtin::BIfminf:
1122  case Builtin::BIfminl:
1123  case Builtin::BI__builtin_fmin:
1124  case Builtin::BI__builtin_fminf:
1125  case Builtin::BI__builtin_fminl:
1126  return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::minnum));
1127 
1128  // fmod() is a special-case. It maps to the frem instruction rather than an
1129  // LLVM intrinsic.
1130  case Builtin::BIfmod:
1131  case Builtin::BIfmodf:
1132  case Builtin::BIfmodl:
1133  case Builtin::BI__builtin_fmod:
1134  case Builtin::BI__builtin_fmodf:
1135  case Builtin::BI__builtin_fmodl: {
1136  Value *Arg1 = EmitScalarExpr(E->getArg(0));
1137  Value *Arg2 = EmitScalarExpr(E->getArg(1));
1138  return RValue::get(Builder.CreateFRem(Arg1, Arg2, "fmod"));
1139  }
1140 
1141  case Builtin::BIlog:
1142  case Builtin::BIlogf:
1143  case Builtin::BIlogl:
1144  case Builtin::BI__builtin_log:
1145  case Builtin::BI__builtin_logf:
1146  case Builtin::BI__builtin_logl:
1147  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::log));
1148 
1149  case Builtin::BIlog10:
1150  case Builtin::BIlog10f:
1151  case Builtin::BIlog10l:
1152  case Builtin::BI__builtin_log10:
1153  case Builtin::BI__builtin_log10f:
1154  case Builtin::BI__builtin_log10l:
1155  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::log10));
1156 
1157  case Builtin::BIlog2:
1158  case Builtin::BIlog2f:
1159  case Builtin::BIlog2l:
1160  case Builtin::BI__builtin_log2:
1161  case Builtin::BI__builtin_log2f:
1162  case Builtin::BI__builtin_log2l:
1163  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::log2));
1164 
1165  case Builtin::BInearbyint:
1166  case Builtin::BInearbyintf:
1167  case Builtin::BInearbyintl:
1168  case Builtin::BI__builtin_nearbyint:
1169  case Builtin::BI__builtin_nearbyintf:
1170  case Builtin::BI__builtin_nearbyintl:
1172 
1173  case Builtin::BIpow:
1174  case Builtin::BIpowf:
1175  case Builtin::BIpowl:
1176  case Builtin::BI__builtin_pow:
1177  case Builtin::BI__builtin_powf:
1178  case Builtin::BI__builtin_powl:
1179  return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::pow));
1180 
1181  case Builtin::BIrint:
1182  case Builtin::BIrintf:
1183  case Builtin::BIrintl:
1184  case Builtin::BI__builtin_rint:
1185  case Builtin::BI__builtin_rintf:
1186  case Builtin::BI__builtin_rintl:
1187  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::rint));
1188 
1189  case Builtin::BIround:
1190  case Builtin::BIroundf:
1191  case Builtin::BIroundl:
1192  case Builtin::BI__builtin_round:
1193  case Builtin::BI__builtin_roundf:
1194  case Builtin::BI__builtin_roundl:
1195  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::round));
1196 
1197  case Builtin::BIsin:
1198  case Builtin::BIsinf:
1199  case Builtin::BIsinl:
1200  case Builtin::BI__builtin_sin:
1201  case Builtin::BI__builtin_sinf:
1202  case Builtin::BI__builtin_sinl:
1203  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::sin));
1204 
1205  case Builtin::BIsqrt:
1206  case Builtin::BIsqrtf:
1207  case Builtin::BIsqrtl:
1208  case Builtin::BI__builtin_sqrt:
1209  case Builtin::BI__builtin_sqrtf:
1210  case Builtin::BI__builtin_sqrtl:
1211  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::sqrt));
1212 
1213  case Builtin::BItrunc:
1214  case Builtin::BItruncf:
1215  case Builtin::BItruncl:
1216  case Builtin::BI__builtin_trunc:
1217  case Builtin::BI__builtin_truncf:
1218  case Builtin::BI__builtin_truncl:
1219  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::trunc));
1220 
1221  default:
1222  break;
1223  }
1224  }
1225 
1226  switch (BuiltinID) {
1227  default: break;
1228  case Builtin::BI__builtin___CFStringMakeConstantString:
1229  case Builtin::BI__builtin___NSStringMakeConstantString:
1230  return RValue::get(ConstantEmitter(*this).emitAbstract(E, E->getType()));
1231  case Builtin::BI__builtin_stdarg_start:
1232  case Builtin::BI__builtin_va_start:
1233  case Builtin::BI__va_start:
1234  case Builtin::BI__builtin_va_end:
1235  return RValue::get(
1236  EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
1237  ? EmitScalarExpr(E->getArg(0))
1238  : EmitVAListRef(E->getArg(0)).getPointer(),
1239  BuiltinID != Builtin::BI__builtin_va_end));
1240  case Builtin::BI__builtin_va_copy: {
1241  Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer();
1242  Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer();
1243 
1244  llvm::Type *Type = Int8PtrTy;
1245 
1246  DstPtr = Builder.CreateBitCast(DstPtr, Type);
1247  SrcPtr = Builder.CreateBitCast(SrcPtr, Type);
1248  return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy),
1249  {DstPtr, SrcPtr}));
1250  }
1251  case Builtin::BI__builtin_abs:
1252  case Builtin::BI__builtin_labs:
1253  case Builtin::BI__builtin_llabs: {
1254  // X < 0 ? -X : X
1255  // The negation has 'nsw' because abs of INT_MIN is undefined.
1256  Value *ArgValue = EmitScalarExpr(E->getArg(0));
1257  Value *NegOp = Builder.CreateNSWNeg(ArgValue, "neg");
1258  Constant *Zero = llvm::Constant::getNullValue(ArgValue->getType());
1259  Value *CmpResult = Builder.CreateICmpSLT(ArgValue, Zero, "abscond");
1260  Value *Result = Builder.CreateSelect(CmpResult, NegOp, ArgValue, "abs");
1261  return RValue::get(Result);
1262  }
1263  case Builtin::BI__builtin_conj:
1264  case Builtin::BI__builtin_conjf:
1265  case Builtin::BI__builtin_conjl: {
1266  ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
1267  Value *Real = ComplexVal.first;
1268  Value *Imag = ComplexVal.second;
1269  Value *Zero =
1270  Imag->getType()->isFPOrFPVectorTy()
1271  ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType())
1272  : llvm::Constant::getNullValue(Imag->getType());
1273 
1274  Imag = Builder.CreateFSub(Zero, Imag, "sub");
1275  return RValue::getComplex(std::make_pair(Real, Imag));
1276  }
1277  case Builtin::BI__builtin_creal:
1278  case Builtin::BI__builtin_crealf:
1279  case Builtin::BI__builtin_creall:
1280  case Builtin::BIcreal:
1281  case Builtin::BIcrealf:
1282  case Builtin::BIcreall: {
1283  ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
1284  return RValue::get(ComplexVal.first);
1285  }
1286 
1287  case Builtin::BI__builtin_dump_struct: {
1288  Value *Func = EmitScalarExpr(E->getArg(1)->IgnoreImpCasts());
1289  CharUnits Arg0Align = EmitPointerWithAlignment(E->getArg(0)).getAlignment();
1290 
1291  const Expr *Arg0 = E->getArg(0)->IgnoreImpCasts();
1292  QualType Arg0Type = Arg0->getType()->getPointeeType();
1293 
1294  Value *RecordPtr = EmitScalarExpr(Arg0);
1295  Value *Res = dumpRecord(*this, Arg0Type, RecordPtr, Arg0Align, Func, 0);
1296  return RValue::get(Res);
1297  }
1298 
1299  case Builtin::BI__builtin_cimag:
1300  case Builtin::BI__builtin_cimagf:
1301  case Builtin::BI__builtin_cimagl:
1302  case Builtin::BIcimag:
1303  case Builtin::BIcimagf:
1304  case Builtin::BIcimagl: {
1305  ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
1306  return RValue::get(ComplexVal.second);
1307  }
1308 
1309  case Builtin::BI__builtin_ctzs:
1310  case Builtin::BI__builtin_ctz:
1311  case Builtin::BI__builtin_ctzl:
1312  case Builtin::BI__builtin_ctzll: {
1313  Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CTZPassedZero);
1314 
1315  llvm::Type *ArgType = ArgValue->getType();
1316  Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1317 
1318  llvm::Type *ResultType = ConvertType(E->getType());
1319  Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
1320  Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
1321  if (Result->getType() != ResultType)
1322  Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1323  "cast");
1324  return RValue::get(Result);
1325  }
1326  case Builtin::BI__builtin_clzs:
1327  case Builtin::BI__builtin_clz:
1328  case Builtin::BI__builtin_clzl:
1329  case Builtin::BI__builtin_clzll: {
1330  Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CLZPassedZero);
1331 
1332  llvm::Type *ArgType = ArgValue->getType();
1333  Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
1334 
1335  llvm::Type *ResultType = ConvertType(E->getType());
1336  Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
1337  Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
1338  if (Result->getType() != ResultType)
1339  Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1340  "cast");
1341  return RValue::get(Result);
1342  }
1343  case Builtin::BI__builtin_ffs:
1344  case Builtin::BI__builtin_ffsl:
1345  case Builtin::BI__builtin_ffsll: {
1346  // ffs(x) -> x ? cttz(x) + 1 : 0
1347  Value *ArgValue = EmitScalarExpr(E->getArg(0));
1348 
1349  llvm::Type *ArgType = ArgValue->getType();
1350  Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1351 
1352  llvm::Type *ResultType = ConvertType(E->getType());
1353  Value *Tmp =
1354  Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
1355  llvm::ConstantInt::get(ArgType, 1));
1356  Value *Zero = llvm::Constant::getNullValue(ArgType);
1357  Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
1358  Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
1359  if (Result->getType() != ResultType)
1360  Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1361  "cast");
1362  return RValue::get(Result);
1363  }
1364  case Builtin::BI__builtin_parity:
1365  case Builtin::BI__builtin_parityl:
1366  case Builtin::BI__builtin_parityll: {
1367  // parity(x) -> ctpop(x) & 1
1368  Value *ArgValue = EmitScalarExpr(E->getArg(0));
1369 
1370  llvm::Type *ArgType = ArgValue->getType();
1371  Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
1372 
1373  llvm::Type *ResultType = ConvertType(E->getType());
1374  Value *Tmp = Builder.CreateCall(F, ArgValue);
1375  Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
1376  if (Result->getType() != ResultType)
1377  Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1378  "cast");
1379  return RValue::get(Result);
1380  }
1381  case Builtin::BI__popcnt16:
1382  case Builtin::BI__popcnt:
1383  case Builtin::BI__popcnt64:
1384  case Builtin::BI__builtin_popcount:
1385  case Builtin::BI__builtin_popcountl:
1386  case Builtin::BI__builtin_popcountll: {
1387  Value *ArgValue = EmitScalarExpr(E->getArg(0));
1388 
1389  llvm::Type *ArgType = ArgValue->getType();
1390  Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
1391 
1392  llvm::Type *ResultType = ConvertType(E->getType());
1393  Value *Result = Builder.CreateCall(F, ArgValue);
1394  if (Result->getType() != ResultType)
1395  Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1396  "cast");
1397  return RValue::get(Result);
1398  }
1399  case Builtin::BI_rotr8:
1400  case Builtin::BI_rotr16:
1401  case Builtin::BI_rotr:
1402  case Builtin::BI_lrotr:
1403  case Builtin::BI_rotr64: {
1404  Value *Val = EmitScalarExpr(E->getArg(0));
1405  Value *Shift = EmitScalarExpr(E->getArg(1));
1406 
1407  llvm::Type *ArgType = Val->getType();
1408  Shift = Builder.CreateIntCast(Shift, ArgType, false);
1409  unsigned ArgWidth = ArgType->getIntegerBitWidth();
1410  Value *Mask = llvm::ConstantInt::get(ArgType, ArgWidth - 1);
1411 
1412  Value *RightShiftAmt = Builder.CreateAnd(Shift, Mask);
1413  Value *RightShifted = Builder.CreateLShr(Val, RightShiftAmt);
1414  Value *LeftShiftAmt = Builder.CreateAnd(Builder.CreateNeg(Shift), Mask);
1415  Value *LeftShifted = Builder.CreateShl(Val, LeftShiftAmt);
1416  Value *Result = Builder.CreateOr(LeftShifted, RightShifted);
1417  return RValue::get(Result);
1418  }
1419  case Builtin::BI_rotl8:
1420  case Builtin::BI_rotl16:
1421  case Builtin::BI_rotl:
1422  case Builtin::BI_lrotl:
1423  case Builtin::BI_rotl64: {
1424  Value *Val = EmitScalarExpr(E->getArg(0));
1425  Value *Shift = EmitScalarExpr(E->getArg(1));
1426 
1427  llvm::Type *ArgType = Val->getType();
1428  Shift = Builder.CreateIntCast(Shift, ArgType, false);
1429  unsigned ArgWidth = ArgType->getIntegerBitWidth();
1430  Value *Mask = llvm::ConstantInt::get(ArgType, ArgWidth - 1);
1431 
1432  Value *LeftShiftAmt = Builder.CreateAnd(Shift, Mask);
1433  Value *LeftShifted = Builder.CreateShl(Val, LeftShiftAmt);
1434  Value *RightShiftAmt = Builder.CreateAnd(Builder.CreateNeg(Shift), Mask);
1435  Value *RightShifted = Builder.CreateLShr(Val, RightShiftAmt);
1436  Value *Result = Builder.CreateOr(LeftShifted, RightShifted);
1437  return RValue::get(Result);
1438  }
1439  case Builtin::BI__builtin_unpredictable: {
1440  // Always return the argument of __builtin_unpredictable. LLVM does not
1441  // handle this builtin. Metadata for this builtin should be added directly
1442  // to instructions such as branches or switches that use it.
1443  return RValue::get(EmitScalarExpr(E->getArg(0)));
1444  }
1445  case Builtin::BI__builtin_expect: {
1446  Value *ArgValue = EmitScalarExpr(E->getArg(0));
1447  llvm::Type *ArgType = ArgValue->getType();
1448 
1449  Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
1450  // Don't generate llvm.expect on -O0 as the backend won't use it for
1451  // anything.
1452  // Note, we still IRGen ExpectedValue because it could have side-effects.
1453  if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1454  return RValue::get(ArgValue);
1455 
1456  Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
1457  Value *Result =
1458  Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval");
1459  return RValue::get(Result);
1460  }
1461  case Builtin::BI__builtin_assume_aligned: {
1462  Value *PtrValue = EmitScalarExpr(E->getArg(0));
1463  Value *OffsetValue =
1464  (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr;
1465 
1466  Value *AlignmentValue = EmitScalarExpr(E->getArg(1));
1467  ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
1468  unsigned Alignment = (unsigned) AlignmentCI->getZExtValue();
1469 
1470  EmitAlignmentAssumption(PtrValue, Alignment, OffsetValue);
1471  return RValue::get(PtrValue);
1472  }
1473  case Builtin::BI__assume:
1474  case Builtin::BI__builtin_assume: {
1475  if (E->getArg(0)->HasSideEffects(getContext()))
1476  return RValue::get(nullptr);
1477 
1478  Value *ArgValue = EmitScalarExpr(E->getArg(0));
1479  Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
1480  return RValue::get(Builder.CreateCall(FnAssume, ArgValue));
1481  }
1482  case Builtin::BI__builtin_bswap16:
1483  case Builtin::BI__builtin_bswap32:
1484  case Builtin::BI__builtin_bswap64: {
1485  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bswap));
1486  }
1487  case Builtin::BI__builtin_bitreverse8:
1488  case Builtin::BI__builtin_bitreverse16:
1489  case Builtin::BI__builtin_bitreverse32:
1490  case Builtin::BI__builtin_bitreverse64: {
1491  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bitreverse));
1492  }
1493  case Builtin::BI__builtin_object_size: {
1494  unsigned Type =
1495  E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue();
1496  auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType()));
1497 
1498  // We pass this builtin onto the optimizer so that it can figure out the
1499  // object size in more complex cases.
1500  return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType,
1501  /*EmittedE=*/nullptr));
1502  }
1503  case Builtin::BI__builtin_prefetch: {
1504  Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
1505  // FIXME: Technically these constants should of type 'int', yes?
1506  RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) :
1507  llvm::ConstantInt::get(Int32Ty, 0);
1508  Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) :
1509  llvm::ConstantInt::get(Int32Ty, 3);
1510  Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
1511  Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
1512  return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data}));
1513  }
1514  case Builtin::BI__builtin_readcyclecounter: {
1515  Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
1516  return RValue::get(Builder.CreateCall(F));
1517  }
1518  case Builtin::BI__builtin___clear_cache: {
1519  Value *Begin = EmitScalarExpr(E->getArg(0));
1520  Value *End = EmitScalarExpr(E->getArg(1));
1521  Value *F = CGM.getIntrinsic(Intrinsic::clear_cache);
1522  return RValue::get(Builder.CreateCall(F, {Begin, End}));
1523  }
1524  case Builtin::BI__builtin_trap:
1525  return RValue::get(EmitTrapCall(Intrinsic::trap));
1526  case Builtin::BI__debugbreak:
1527  return RValue::get(EmitTrapCall(Intrinsic::debugtrap));
1528  case Builtin::BI__builtin_unreachable: {
1529  EmitUnreachable(E->getExprLoc());
1530 
1531  // We do need to preserve an insertion point.
1532  EmitBlock(createBasicBlock("unreachable.cont"));
1533 
1534  return RValue::get(nullptr);
1535  }
1536 
1537  case Builtin::BI__builtin_powi:
1538  case Builtin::BI__builtin_powif:
1539  case Builtin::BI__builtin_powil: {
1540  Value *Base = EmitScalarExpr(E->getArg(0));
1541  Value *Exponent = EmitScalarExpr(E->getArg(1));
1542  llvm::Type *ArgType = Base->getType();
1543  Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType);
1544  return RValue::get(Builder.CreateCall(F, {Base, Exponent}));
1545  }
1546 
1547  case Builtin::BI__builtin_isgreater:
1548  case Builtin::BI__builtin_isgreaterequal:
1549  case Builtin::BI__builtin_isless:
1550  case Builtin::BI__builtin_islessequal:
1551  case Builtin::BI__builtin_islessgreater:
1552  case Builtin::BI__builtin_isunordered: {
1553  // Ordered comparisons: we know the arguments to these are matching scalar
1554  // floating point values.
1555  Value *LHS = EmitScalarExpr(E->getArg(0));
1556  Value *RHS = EmitScalarExpr(E->getArg(1));
1557 
1558  switch (BuiltinID) {
1559  default: llvm_unreachable("Unknown ordered comparison");
1560  case Builtin::BI__builtin_isgreater:
1561  LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp");
1562  break;
1563  case Builtin::BI__builtin_isgreaterequal:
1564  LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp");
1565  break;
1566  case Builtin::BI__builtin_isless:
1567  LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp");
1568  break;
1569  case Builtin::BI__builtin_islessequal:
1570  LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp");
1571  break;
1572  case Builtin::BI__builtin_islessgreater:
1573  LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp");
1574  break;
1575  case Builtin::BI__builtin_isunordered:
1576  LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp");
1577  break;
1578  }
1579  // ZExt bool to int type.
1580  return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
1581  }
1582  case Builtin::BI__builtin_isnan: {
1583  Value *V = EmitScalarExpr(E->getArg(0));
1584  V = Builder.CreateFCmpUNO(V, V, "cmp");
1585  return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
1586  }
1587 
1588  case Builtin::BIfinite:
1589  case Builtin::BI__finite:
1590  case Builtin::BIfinitef:
1591  case Builtin::BI__finitef:
1592  case Builtin::BIfinitel:
1593  case Builtin::BI__finitel:
1594  case Builtin::BI__builtin_isinf:
1595  case Builtin::BI__builtin_isfinite: {
1596  // isinf(x) --> fabs(x) == infinity
1597  // isfinite(x) --> fabs(x) != infinity
1598  // x != NaN via the ordered compare in either case.
1599  Value *V = EmitScalarExpr(E->getArg(0));
1600  Value *Fabs = EmitFAbs(*this, V);
1601  Constant *Infinity = ConstantFP::getInfinity(V->getType());
1602  CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf)
1603  ? CmpInst::FCMP_OEQ
1604  : CmpInst::FCMP_ONE;
1605  Value *FCmp = Builder.CreateFCmp(Pred, Fabs, Infinity, "cmpinf");
1606  return RValue::get(Builder.CreateZExt(FCmp, ConvertType(E->getType())));
1607  }
1608 
1609  case Builtin::BI__builtin_isinf_sign: {
1610  // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
1611  Value *Arg = EmitScalarExpr(E->getArg(0));
1612  Value *AbsArg = EmitFAbs(*this, Arg);
1613  Value *IsInf = Builder.CreateFCmpOEQ(
1614  AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf");
1615  Value *IsNeg = EmitSignBit(*this, Arg);
1616 
1617  llvm::Type *IntTy = ConvertType(E->getType());
1618  Value *Zero = Constant::getNullValue(IntTy);
1619  Value *One = ConstantInt::get(IntTy, 1);
1620  Value *NegativeOne = ConstantInt::get(IntTy, -1);
1621  Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
1622  Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero);
1623  return RValue::get(Result);
1624  }
1625 
1626  case Builtin::BI__builtin_isnormal: {
1627  // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min
1628  Value *V = EmitScalarExpr(E->getArg(0));
1629  Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
1630 
1631  Value *Abs = EmitFAbs(*this, V);
1632  Value *IsLessThanInf =
1633  Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf");
1634  APFloat Smallest = APFloat::getSmallestNormalized(
1635  getContext().getFloatTypeSemantics(E->getArg(0)->getType()));
1636  Value *IsNormal =
1637  Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest),
1638  "isnormal");
1639  V = Builder.CreateAnd(Eq, IsLessThanInf, "and");
1640  V = Builder.CreateAnd(V, IsNormal, "and");
1641  return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
1642  }
1643 
1644  case Builtin::BI__builtin_fpclassify: {
1645  Value *V = EmitScalarExpr(E->getArg(5));
1646  llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
1647 
1648  // Create Result
1649  BasicBlock *Begin = Builder.GetInsertBlock();
1650  BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn);
1651  Builder.SetInsertPoint(End);
1652  PHINode *Result =
1653  Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4,
1654  "fpclassify_result");
1655 
1656  // if (V==0) return FP_ZERO
1657  Builder.SetInsertPoint(Begin);
1658  Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
1659  "iszero");
1660  Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
1661  BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn);
1662  Builder.CreateCondBr(IsZero, End, NotZero);
1663  Result->addIncoming(ZeroLiteral, Begin);
1664 
1665  // if (V != V) return FP_NAN
1666  Builder.SetInsertPoint(NotZero);
1667  Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp");
1668  Value *NanLiteral = EmitScalarExpr(E->getArg(0));
1669  BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn);
1670  Builder.CreateCondBr(IsNan, End, NotNan);
1671  Result->addIncoming(NanLiteral, NotZero);
1672 
1673  // if (fabs(V) == infinity) return FP_INFINITY
1674  Builder.SetInsertPoint(NotNan);
1675  Value *VAbs = EmitFAbs(*this, V);
1676  Value *IsInf =
1677  Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
1678  "isinf");
1679  Value *InfLiteral = EmitScalarExpr(E->getArg(1));
1680  BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn);
1681  Builder.CreateCondBr(IsInf, End, NotInf);
1682  Result->addIncoming(InfLiteral, NotNan);
1683 
1684  // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
1685  Builder.SetInsertPoint(NotInf);
1686  APFloat Smallest = APFloat::getSmallestNormalized(
1687  getContext().getFloatTypeSemantics(E->getArg(5)->getType()));
1688  Value *IsNormal =
1689  Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
1690  "isnormal");
1691  Value *NormalResult =
1692  Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
1693  EmitScalarExpr(E->getArg(3)));
1694  Builder.CreateBr(End);
1695  Result->addIncoming(NormalResult, NotInf);
1696 
1697  // return Result
1698  Builder.SetInsertPoint(End);
1699  return RValue::get(Result);
1700  }
1701 
1702  case Builtin::BIalloca:
1703  case Builtin::BI_alloca:
1704  case Builtin::BI__builtin_alloca: {
1705  Value *Size = EmitScalarExpr(E->getArg(0));
1706  const TargetInfo &TI = getContext().getTargetInfo();
1707  // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
1708  unsigned SuitableAlignmentInBytes =
1709  CGM.getContext()
1710  .toCharUnitsFromBits(TI.getSuitableAlign())
1711  .getQuantity();
1712  AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
1713  AI->setAlignment(SuitableAlignmentInBytes);
1714  return RValue::get(AI);
1715  }
1716 
1717  case Builtin::BI__builtin_alloca_with_align: {
1718  Value *Size = EmitScalarExpr(E->getArg(0));
1719  Value *AlignmentInBitsValue = EmitScalarExpr(E->getArg(1));
1720  auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
1721  unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
1722  unsigned AlignmentInBytes =
1723  CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getQuantity();
1724  AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
1725  AI->setAlignment(AlignmentInBytes);
1726  return RValue::get(AI);
1727  }
1728 
1729  case Builtin::BIbzero:
1730  case Builtin::BI__builtin_bzero: {
1731  Address Dest = EmitPointerWithAlignment(E->getArg(0));
1732  Value *SizeVal = EmitScalarExpr(E->getArg(1));
1733  EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
1734  E->getArg(0)->getExprLoc(), FD, 0);
1735  Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false);
1736  return RValue::get(nullptr);
1737  }
1738  case Builtin::BImemcpy:
1739  case Builtin::BI__builtin_memcpy: {
1740  Address Dest = EmitPointerWithAlignment(E->getArg(0));
1741  Address Src = EmitPointerWithAlignment(E->getArg(1));
1742  Value *SizeVal = EmitScalarExpr(E->getArg(2));
1743  EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
1744  E->getArg(0)->getExprLoc(), FD, 0);
1745  EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
1746  E->getArg(1)->getExprLoc(), FD, 1);
1747  Builder.CreateMemCpy(Dest, Src, SizeVal, false);
1748  return RValue::get(Dest.getPointer());
1749  }
1750 
1751  case Builtin::BI__builtin_char_memchr:
1752  BuiltinID = Builtin::BI__builtin_memchr;
1753  break;
1754 
1755  case Builtin::BI__builtin___memcpy_chk: {
1756  // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
1757  llvm::APSInt Size, DstSize;
1758  if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
1759  !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
1760  break;
1761  if (Size.ugt(DstSize))
1762  break;
1763  Address Dest = EmitPointerWithAlignment(E->getArg(0));
1764  Address Src = EmitPointerWithAlignment(E->getArg(1));
1765  Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
1766  Builder.CreateMemCpy(Dest, Src, SizeVal, false);
1767  return RValue::get(Dest.getPointer());
1768  }
1769 
1770  case Builtin::BI__builtin_objc_memmove_collectable: {
1771  Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
1772  Address SrcAddr = EmitPointerWithAlignment(E->getArg(1));
1773  Value *SizeVal = EmitScalarExpr(E->getArg(2));
1774  CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this,
1775  DestAddr, SrcAddr, SizeVal);
1776  return RValue::get(DestAddr.getPointer());
1777  }
1778 
1779  case Builtin::BI__builtin___memmove_chk: {
1780  // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
1781  llvm::APSInt Size, DstSize;
1782  if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
1783  !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
1784  break;
1785  if (Size.ugt(DstSize))
1786  break;
1787  Address Dest = EmitPointerWithAlignment(E->getArg(0));
1788  Address Src = EmitPointerWithAlignment(E->getArg(1));
1789  Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
1790  Builder.CreateMemMove(Dest, Src, SizeVal, false);
1791  return RValue::get(Dest.getPointer());
1792  }
1793 
1794  case Builtin::BImemmove:
1795  case Builtin::BI__builtin_memmove: {
1796  Address Dest = EmitPointerWithAlignment(E->getArg(0));
1797  Address Src = EmitPointerWithAlignment(E->getArg(1));
1798  Value *SizeVal = EmitScalarExpr(E->getArg(2));
1799  EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
1800  E->getArg(0)->getExprLoc(), FD, 0);
1801  EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
1802  E->getArg(1)->getExprLoc(), FD, 1);
1803  Builder.CreateMemMove(Dest, Src, SizeVal, false);
1804  return RValue::get(Dest.getPointer());
1805  }
1806  case Builtin::BImemset:
1807  case Builtin::BI__builtin_memset: {
1808  Address Dest = EmitPointerWithAlignment(E->getArg(0));
1809  Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
1810  Builder.getInt8Ty());
1811  Value *SizeVal = EmitScalarExpr(E->getArg(2));
1812  EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
1813  E->getArg(0)->getExprLoc(), FD, 0);
1814  Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
1815  return RValue::get(Dest.getPointer());
1816  }
1817  case Builtin::BI__builtin___memset_chk: {
1818  // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
1819  llvm::APSInt Size, DstSize;
1820  if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
1821  !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
1822  break;
1823  if (Size.ugt(DstSize))
1824  break;
1825  Address Dest = EmitPointerWithAlignment(E->getArg(0));
1826  Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
1827  Builder.getInt8Ty());
1828  Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
1829  Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
1830  return RValue::get(Dest.getPointer());
1831  }
1832  case Builtin::BI__builtin_wmemcmp: {
1833  // The MSVC runtime library does not provide a definition of wmemcmp, so we
1834  // need an inline implementation.
1835  if (!getTarget().getTriple().isOSMSVCRT())
1836  break;
1837 
1838  llvm::Type *WCharTy = ConvertType(getContext().WCharTy);
1839 
1840  Value *Dst = EmitScalarExpr(E->getArg(0));
1841  Value *Src = EmitScalarExpr(E->getArg(1));
1842  Value *Size = EmitScalarExpr(E->getArg(2));
1843 
1844  BasicBlock *Entry = Builder.GetInsertBlock();
1845  BasicBlock *CmpGT = createBasicBlock("wmemcmp.gt");
1846  BasicBlock *CmpLT = createBasicBlock("wmemcmp.lt");
1847  BasicBlock *Next = createBasicBlock("wmemcmp.next");
1848  BasicBlock *Exit = createBasicBlock("wmemcmp.exit");
1849  Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(SizeTy, 0));
1850  Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
1851 
1852  EmitBlock(CmpGT);
1853  PHINode *DstPhi = Builder.CreatePHI(Dst->getType(), 2);
1854  DstPhi->addIncoming(Dst, Entry);
1855  PHINode *SrcPhi = Builder.CreatePHI(Src->getType(), 2);
1856  SrcPhi->addIncoming(Src, Entry);
1857  PHINode *SizePhi = Builder.CreatePHI(SizeTy, 2);
1858  SizePhi->addIncoming(Size, Entry);
1859  CharUnits WCharAlign =
1860  getContext().getTypeAlignInChars(getContext().WCharTy);
1861  Value *DstCh = Builder.CreateAlignedLoad(WCharTy, DstPhi, WCharAlign);
1862  Value *SrcCh = Builder.CreateAlignedLoad(WCharTy, SrcPhi, WCharAlign);
1863  Value *DstGtSrc = Builder.CreateICmpUGT(DstCh, SrcCh);
1864  Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
1865 
1866  EmitBlock(CmpLT);
1867  Value *DstLtSrc = Builder.CreateICmpULT(DstCh, SrcCh);
1868  Builder.CreateCondBr(DstLtSrc, Exit, Next);
1869 
1870  EmitBlock(Next);
1871  Value *NextDst = Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
1872  Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
1873  Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(SizeTy, 1));
1874  Value *NextSizeEq0 =
1875  Builder.CreateICmpEQ(NextSize, ConstantInt::get(SizeTy, 0));
1876  Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
1877  DstPhi->addIncoming(NextDst, Next);
1878  SrcPhi->addIncoming(NextSrc, Next);
1879  SizePhi->addIncoming(NextSize, Next);
1880 
1881  EmitBlock(Exit);
1882  PHINode *Ret = Builder.CreatePHI(IntTy, 4);
1883  Ret->addIncoming(ConstantInt::get(IntTy, 0), Entry);
1884  Ret->addIncoming(ConstantInt::get(IntTy, 1), CmpGT);
1885  Ret->addIncoming(ConstantInt::get(IntTy, -1), CmpLT);
1886  Ret->addIncoming(ConstantInt::get(IntTy, 0), Next);
1887  return RValue::get(Ret);
1888  }
1889  case Builtin::BI__builtin_dwarf_cfa: {
1890  // The offset in bytes from the first argument to the CFA.
1891  //
1892  // Why on earth is this in the frontend? Is there any reason at
1893  // all that the backend can't reasonably determine this while
1894  // lowering llvm.eh.dwarf.cfa()?
1895  //
1896  // TODO: If there's a satisfactory reason, add a target hook for
1897  // this instead of hard-coding 0, which is correct for most targets.
1898  int32_t Offset = 0;
1899 
1900  Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
1901  return RValue::get(Builder.CreateCall(F,
1902  llvm::ConstantInt::get(Int32Ty, Offset)));
1903  }
1904  case Builtin::BI__builtin_return_address: {
1905  Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
1906  getContext().UnsignedIntTy);
1907  Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
1908  return RValue::get(Builder.CreateCall(F, Depth));
1909  }
1910  case Builtin::BI_ReturnAddress: {
1911  Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
1912  return RValue::get(Builder.CreateCall(F, Builder.getInt32(0)));
1913  }
1914  case Builtin::BI__builtin_frame_address: {
1915  Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
1916  getContext().UnsignedIntTy);
1917  Value *F = CGM.getIntrinsic(Intrinsic::frameaddress);
1918  return RValue::get(Builder.CreateCall(F, Depth));
1919  }
1920  case Builtin::BI__builtin_extract_return_addr: {
1921  Value *Address = EmitScalarExpr(E->getArg(0));
1922  Value *Result = getTargetHooks().decodeReturnAddress(*this, Address);
1923  return RValue::get(Result);
1924  }
1925  case Builtin::BI__builtin_frob_return_addr: {
1926  Value *Address = EmitScalarExpr(E->getArg(0));
1927  Value *Result = getTargetHooks().encodeReturnAddress(*this, Address);
1928  return RValue::get(Result);
1929  }
1930  case Builtin::BI__builtin_dwarf_sp_column: {
1931  llvm::IntegerType *Ty
1932  = cast<llvm::IntegerType>(ConvertType(E->getType()));
1933  int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
1934  if (Column == -1) {
1935  CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column");
1936  return RValue::get(llvm::UndefValue::get(Ty));
1937  }
1938  return RValue::get(llvm::ConstantInt::get(Ty, Column, true));
1939  }
1940  case Builtin::BI__builtin_init_dwarf_reg_size_table: {
1941  Value *Address = EmitScalarExpr(E->getArg(0));
1942  if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address))
1943  CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table");
1944  return RValue::get(llvm::UndefValue::get(ConvertType(E->getType())));
1945  }
1946  case Builtin::BI__builtin_eh_return: {
1947  Value *Int = EmitScalarExpr(E->getArg(0));
1948  Value *Ptr = EmitScalarExpr(E->getArg(1));
1949 
1950  llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
1951  assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
1952  "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
1953  Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32
1954  ? Intrinsic::eh_return_i32
1955  : Intrinsic::eh_return_i64);
1956  Builder.CreateCall(F, {Int, Ptr});
1957  Builder.CreateUnreachable();
1958 
1959  // We do need to preserve an insertion point.
1960  EmitBlock(createBasicBlock("builtin_eh_return.cont"));
1961 
1962  return RValue::get(nullptr);
1963  }
1964  case Builtin::BI__builtin_unwind_init: {
1965  Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
1966  return RValue::get(Builder.CreateCall(F));
1967  }
1968  case Builtin::BI__builtin_extend_pointer: {
1969  // Extends a pointer to the size of an _Unwind_Word, which is
1970  // uint64_t on all platforms. Generally this gets poked into a
1971  // register and eventually used as an address, so if the
1972  // addressing registers are wider than pointers and the platform
1973  // doesn't implicitly ignore high-order bits when doing
1974  // addressing, we need to make sure we zext / sext based on
1975  // the platform's expectations.
1976  //
1977  // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
1978 
1979  // Cast the pointer to intptr_t.
1980  Value *Ptr = EmitScalarExpr(E->getArg(0));
1981  Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast");
1982 
1983  // If that's 64 bits, we're done.
1984  if (IntPtrTy->getBitWidth() == 64)
1985  return RValue::get(Result);
1986 
1987  // Otherwise, ask the codegen data what to do.
1988  if (getTargetHooks().extendPointerWithSExt())
1989  return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext"));
1990  else
1991  return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext"));
1992  }
1993  case Builtin::BI__builtin_setjmp: {
1994  // Buffer is a void**.
1995  Address Buf = EmitPointerWithAlignment(E->getArg(0));
1996 
1997  // Store the frame pointer to the setjmp buffer.
1998  Value *FrameAddr =
1999  Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
2000  ConstantInt::get(Int32Ty, 0));
2001  Builder.CreateStore(FrameAddr, Buf);
2002 
2003  // Store the stack pointer to the setjmp buffer.
2004  Value *StackAddr =
2005  Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
2006  Address StackSaveSlot =
2007  Builder.CreateConstInBoundsGEP(Buf, 2, getPointerSize());
2008  Builder.CreateStore(StackAddr, StackSaveSlot);
2009 
2010  // Call LLVM's EH setjmp, which is lightweight.
2011  Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
2012  Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
2013  return RValue::get(Builder.CreateCall(F, Buf.getPointer()));
2014  }
2015  case Builtin::BI__builtin_longjmp: {
2016  Value *Buf = EmitScalarExpr(E->getArg(0));
2017  Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
2018 
2019  // Call LLVM's EH longjmp, which is lightweight.
2020  Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
2021 
2022  // longjmp doesn't return; mark this as unreachable.
2023  Builder.CreateUnreachable();
2024 
2025  // We do need to preserve an insertion point.
2026  EmitBlock(createBasicBlock("longjmp.cont"));
2027 
2028  return RValue::get(nullptr);
2029  }
2030  case Builtin::BI__sync_fetch_and_add:
2031  case Builtin::BI__sync_fetch_and_sub:
2032  case Builtin::BI__sync_fetch_and_or:
2033  case Builtin::BI__sync_fetch_and_and:
2034  case Builtin::BI__sync_fetch_and_xor:
2035  case Builtin::BI__sync_fetch_and_nand:
2036  case Builtin::BI__sync_add_and_fetch:
2037  case Builtin::BI__sync_sub_and_fetch:
2038  case Builtin::BI__sync_and_and_fetch:
2039  case Builtin::BI__sync_or_and_fetch:
2040  case Builtin::BI__sync_xor_and_fetch:
2041  case Builtin::BI__sync_nand_and_fetch:
2042  case Builtin::BI__sync_val_compare_and_swap:
2043  case Builtin::BI__sync_bool_compare_and_swap:
2044  case Builtin::BI__sync_lock_test_and_set:
2045  case Builtin::BI__sync_lock_release:
2046  case Builtin::BI__sync_swap:
2047  llvm_unreachable("Shouldn't make it through sema");
2048  case Builtin::BI__sync_fetch_and_add_1:
2049  case Builtin::BI__sync_fetch_and_add_2:
2050  case Builtin::BI__sync_fetch_and_add_4:
2051  case Builtin::BI__sync_fetch_and_add_8:
2052  case Builtin::BI__sync_fetch_and_add_16:
2053  return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E);
2054  case Builtin::BI__sync_fetch_and_sub_1:
2055  case Builtin::BI__sync_fetch_and_sub_2:
2056  case Builtin::BI__sync_fetch_and_sub_4:
2057  case Builtin::BI__sync_fetch_and_sub_8:
2058  case Builtin::BI__sync_fetch_and_sub_16:
2059  return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E);
2060  case Builtin::BI__sync_fetch_and_or_1:
2061  case Builtin::BI__sync_fetch_and_or_2:
2062  case Builtin::BI__sync_fetch_and_or_4:
2063  case Builtin::BI__sync_fetch_and_or_8:
2064  case Builtin::BI__sync_fetch_and_or_16:
2065  return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E);
2066  case Builtin::BI__sync_fetch_and_and_1:
2067  case Builtin::BI__sync_fetch_and_and_2:
2068  case Builtin::BI__sync_fetch_and_and_4:
2069  case Builtin::BI__sync_fetch_and_and_8:
2070  case Builtin::BI__sync_fetch_and_and_16:
2071  return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E);
2072  case Builtin::BI__sync_fetch_and_xor_1:
2073  case Builtin::BI__sync_fetch_and_xor_2:
2074  case Builtin::BI__sync_fetch_and_xor_4:
2075  case Builtin::BI__sync_fetch_and_xor_8:
2076  case Builtin::BI__sync_fetch_and_xor_16:
2077  return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E);
2078  case Builtin::BI__sync_fetch_and_nand_1:
2079  case Builtin::BI__sync_fetch_and_nand_2:
2080  case Builtin::BI__sync_fetch_and_nand_4:
2081  case Builtin::BI__sync_fetch_and_nand_8:
2082  case Builtin::BI__sync_fetch_and_nand_16:
2083  return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Nand, E);
2084 
2085  // Clang extensions: not overloaded yet.
2086  case Builtin::BI__sync_fetch_and_min:
2087  return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E);
2088  case Builtin::BI__sync_fetch_and_max:
2089  return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E);
2090  case Builtin::BI__sync_fetch_and_umin:
2091  return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E);
2092  case Builtin::BI__sync_fetch_and_umax:
2093  return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E);
2094 
2095  case Builtin::BI__sync_add_and_fetch_1:
2096  case Builtin::BI__sync_add_and_fetch_2:
2097  case Builtin::BI__sync_add_and_fetch_4:
2098  case Builtin::BI__sync_add_and_fetch_8:
2099  case Builtin::BI__sync_add_and_fetch_16:
2100  return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E,
2101  llvm::Instruction::Add);
2102  case Builtin::BI__sync_sub_and_fetch_1:
2103  case Builtin::BI__sync_sub_and_fetch_2:
2104  case Builtin::BI__sync_sub_and_fetch_4:
2105  case Builtin::BI__sync_sub_and_fetch_8:
2106  case Builtin::BI__sync_sub_and_fetch_16:
2107  return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E,
2108  llvm::Instruction::Sub);
2109  case Builtin::BI__sync_and_and_fetch_1:
2110  case Builtin::BI__sync_and_and_fetch_2:
2111  case Builtin::BI__sync_and_and_fetch_4:
2112  case Builtin::BI__sync_and_and_fetch_8:
2113  case Builtin::BI__sync_and_and_fetch_16:
2116  case Builtin::BI__sync_or_and_fetch_1:
2117  case Builtin::BI__sync_or_and_fetch_2:
2118  case Builtin::BI__sync_or_and_fetch_4:
2119  case Builtin::BI__sync_or_and_fetch_8:
2120  case Builtin::BI__sync_or_and_fetch_16:
2121  return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E,
2122  llvm::Instruction::Or);
2123  case Builtin::BI__sync_xor_and_fetch_1:
2124  case Builtin::BI__sync_xor_and_fetch_2:
2125  case Builtin::BI__sync_xor_and_fetch_4:
2126  case Builtin::BI__sync_xor_and_fetch_8:
2127  case Builtin::BI__sync_xor_and_fetch_16:
2128  return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E,
2129  llvm::Instruction::Xor);
2130  case Builtin::BI__sync_nand_and_fetch_1:
2131  case Builtin::BI__sync_nand_and_fetch_2:
2132  case Builtin::BI__sync_nand_and_fetch_4:
2133  case Builtin::BI__sync_nand_and_fetch_8:
2134  case Builtin::BI__sync_nand_and_fetch_16:
2135  return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Nand, E,
2136  llvm::Instruction::And, true);
2137 
2138  case Builtin::BI__sync_val_compare_and_swap_1:
2139  case Builtin::BI__sync_val_compare_and_swap_2:
2140  case Builtin::BI__sync_val_compare_and_swap_4:
2141  case Builtin::BI__sync_val_compare_and_swap_8:
2142  case Builtin::BI__sync_val_compare_and_swap_16:
2143  return RValue::get(MakeAtomicCmpXchgValue(*this, E, false));
2144 
2145  case Builtin::BI__sync_bool_compare_and_swap_1:
2146  case Builtin::BI__sync_bool_compare_and_swap_2:
2147  case Builtin::BI__sync_bool_compare_and_swap_4:
2148  case Builtin::BI__sync_bool_compare_and_swap_8:
2149  case Builtin::BI__sync_bool_compare_and_swap_16:
2150  return RValue::get(MakeAtomicCmpXchgValue(*this, E, true));
2151 
2152  case Builtin::BI__sync_swap_1:
2153  case Builtin::BI__sync_swap_2:
2154  case Builtin::BI__sync_swap_4:
2155  case Builtin::BI__sync_swap_8:
2156  case Builtin::BI__sync_swap_16:
2157  return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
2158 
2159  case Builtin::BI__sync_lock_test_and_set_1:
2160  case Builtin::BI__sync_lock_test_and_set_2:
2161  case Builtin::BI__sync_lock_test_and_set_4:
2162  case Builtin::BI__sync_lock_test_and_set_8:
2163  case Builtin::BI__sync_lock_test_and_set_16:
2164  return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
2165 
2166  case Builtin::BI__sync_lock_release_1:
2167  case Builtin::BI__sync_lock_release_2:
2168  case Builtin::BI__sync_lock_release_4:
2169  case Builtin::BI__sync_lock_release_8:
2170  case Builtin::BI__sync_lock_release_16: {
2171  Value *Ptr = EmitScalarExpr(E->getArg(0));
2172  QualType ElTy = E->getArg(0)->getType()->getPointeeType();
2173  CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
2174  llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
2175  StoreSize.getQuantity() * 8);
2176  Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
2177  llvm::StoreInst *Store =
2178  Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
2179  StoreSize);
2180  Store->setAtomic(llvm::AtomicOrdering::Release);
2181  return RValue::get(nullptr);
2182  }
2183 
2184  case Builtin::BI__sync_synchronize: {
2185  // We assume this is supposed to correspond to a C++0x-style
2186  // sequentially-consistent fence (i.e. this is only usable for
2187  // synchronization, not device I/O or anything like that). This intrinsic
2188  // is really badly designed in the sense that in theory, there isn't
2189  // any way to safely use it... but in practice, it mostly works
2190  // to use it with non-atomic loads and stores to get acquire/release
2191  // semantics.
2192  Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
2193  return RValue::get(nullptr);
2194  }
2195 
2196  case Builtin::BI__builtin_nontemporal_load:
2197  return RValue::get(EmitNontemporalLoad(*this, E));
2198  case Builtin::BI__builtin_nontemporal_store:
2199  return RValue::get(EmitNontemporalStore(*this, E));
2200  case Builtin::BI__c11_atomic_is_lock_free:
2201  case Builtin::BI__atomic_is_lock_free: {
2202  // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
2203  // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
2204  // _Atomic(T) is always properly-aligned.
2205  const char *LibCallName = "__atomic_is_lock_free";
2206  CallArgList Args;
2207  Args.add(RValue::get(EmitScalarExpr(E->getArg(0))),
2208  getContext().getSizeType());
2209  if (BuiltinID == Builtin::BI__atomic_is_lock_free)
2210  Args.add(RValue::get(EmitScalarExpr(E->getArg(1))),
2211  getContext().VoidPtrTy);
2212  else
2213  Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)),
2214  getContext().VoidPtrTy);
2215  const CGFunctionInfo &FuncInfo =
2216  CGM.getTypes().arrangeBuiltinFunctionCall(E->getType(), Args);
2217  llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
2218  llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
2219  return EmitCall(FuncInfo, CGCallee::forDirect(Func),
2220  ReturnValueSlot(), Args);
2221  }
2222 
2223  case Builtin::BI__atomic_test_and_set: {
2224  // Look at the argument type to determine whether this is a volatile
2225  // operation. The parameter type is always volatile.
2226  QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
2227  bool Volatile =
2228  PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
2229 
2230  Value *Ptr = EmitScalarExpr(E->getArg(0));
2231  unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
2232  Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
2233  Value *NewVal = Builder.getInt8(1);
2234  Value *Order = EmitScalarExpr(E->getArg(1));
2235  if (isa<llvm::ConstantInt>(Order)) {
2236  int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
2237  AtomicRMWInst *Result = nullptr;
2238  switch (ord) {
2239  case 0: // memory_order_relaxed
2240  default: // invalid order
2241  Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2242  llvm::AtomicOrdering::Monotonic);
2243  break;
2244  case 1: // memory_order_consume
2245  case 2: // memory_order_acquire
2246  Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2247  llvm::AtomicOrdering::Acquire);
2248  break;
2249  case 3: // memory_order_release
2250  Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2251  llvm::AtomicOrdering::Release);
2252  break;
2253  case 4: // memory_order_acq_rel
2254 
2255  Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2256  llvm::AtomicOrdering::AcquireRelease);
2257  break;
2258  case 5: // memory_order_seq_cst
2259  Result = Builder.CreateAtomicRMW(
2260  llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2261  llvm::AtomicOrdering::SequentiallyConsistent);
2262  break;
2263  }
2264  Result->setVolatile(Volatile);
2265  return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
2266  }
2267 
2268  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
2269 
2270  llvm::BasicBlock *BBs[5] = {
2271  createBasicBlock("monotonic", CurFn),
2272  createBasicBlock("acquire", CurFn),
2273  createBasicBlock("release", CurFn),
2274  createBasicBlock("acqrel", CurFn),
2275  createBasicBlock("seqcst", CurFn)
2276  };
2277  llvm::AtomicOrdering Orders[5] = {
2278  llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
2279  llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
2280  llvm::AtomicOrdering::SequentiallyConsistent};
2281 
2282  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
2283  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
2284 
2285  Builder.SetInsertPoint(ContBB);
2286  PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set");
2287 
2288  for (unsigned i = 0; i < 5; ++i) {
2289  Builder.SetInsertPoint(BBs[i]);
2290  AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
2291  Ptr, NewVal, Orders[i]);
2292  RMW->setVolatile(Volatile);
2293  Result->addIncoming(RMW, BBs[i]);
2294  Builder.CreateBr(ContBB);
2295  }
2296 
2297  SI->addCase(Builder.getInt32(0), BBs[0]);
2298  SI->addCase(Builder.getInt32(1), BBs[1]);
2299  SI->addCase(Builder.getInt32(2), BBs[1]);
2300  SI->addCase(Builder.getInt32(3), BBs[2]);
2301  SI->addCase(Builder.getInt32(4), BBs[3]);
2302  SI->addCase(Builder.getInt32(5), BBs[4]);
2303 
2304  Builder.SetInsertPoint(ContBB);
2305  return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
2306  }
2307 
2308  case Builtin::BI__atomic_clear: {
2309  QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
2310  bool Volatile =
2311  PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
2312 
2313  Address Ptr = EmitPointerWithAlignment(E->getArg(0));
2314  unsigned AddrSpace = Ptr.getPointer()->getType()->getPointerAddressSpace();
2315  Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
2316  Value *NewVal = Builder.getInt8(0);
2317  Value *Order = EmitScalarExpr(E->getArg(1));
2318  if (isa<llvm::ConstantInt>(Order)) {
2319  int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
2320  StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
2321  switch (ord) {
2322  case 0: // memory_order_relaxed
2323  default: // invalid order
2324  Store->setOrdering(llvm::AtomicOrdering::Monotonic);
2325  break;
2326  case 3: // memory_order_release
2327  Store->setOrdering(llvm::AtomicOrdering::Release);
2328  break;
2329  case 5: // memory_order_seq_cst
2330  Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
2331  break;
2332  }
2333  return RValue::get(nullptr);
2334  }
2335 
2336  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
2337 
2338  llvm::BasicBlock *BBs[3] = {
2339  createBasicBlock("monotonic", CurFn),
2340  createBasicBlock("release", CurFn),
2341  createBasicBlock("seqcst", CurFn)
2342  };
2343  llvm::AtomicOrdering Orders[3] = {
2344  llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
2345  llvm::AtomicOrdering::SequentiallyConsistent};
2346 
2347  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
2348  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
2349 
2350  for (unsigned i = 0; i < 3; ++i) {
2351  Builder.SetInsertPoint(BBs[i]);
2352  StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
2353  Store->setOrdering(Orders[i]);
2354  Builder.CreateBr(ContBB);
2355  }
2356 
2357  SI->addCase(Builder.getInt32(0), BBs[0]);
2358  SI->addCase(Builder.getInt32(3), BBs[1]);
2359  SI->addCase(Builder.getInt32(5), BBs[2]);
2360 
2361  Builder.SetInsertPoint(ContBB);
2362  return RValue::get(nullptr);
2363  }
2364 
2365  case Builtin::BI__atomic_thread_fence:
2366  case Builtin::BI__atomic_signal_fence:
2367  case Builtin::BI__c11_atomic_thread_fence:
2368  case Builtin::BI__c11_atomic_signal_fence: {
2369  llvm::SyncScope::ID SSID;
2370  if (BuiltinID == Builtin::BI__atomic_signal_fence ||
2371  BuiltinID == Builtin::BI__c11_atomic_signal_fence)
2372  SSID = llvm::SyncScope::SingleThread;
2373  else
2374  SSID = llvm::SyncScope::System;
2375  Value *Order = EmitScalarExpr(E->getArg(0));
2376  if (isa<llvm::ConstantInt>(Order)) {
2377  int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
2378  switch (ord) {
2379  case 0: // memory_order_relaxed
2380  default: // invalid order
2381  break;
2382  case 1: // memory_order_consume
2383  case 2: // memory_order_acquire
2384  Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
2385  break;
2386  case 3: // memory_order_release
2387  Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
2388  break;
2389  case 4: // memory_order_acq_rel
2390  Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
2391  break;
2392  case 5: // memory_order_seq_cst
2393  Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
2394  break;
2395  }
2396  return RValue::get(nullptr);
2397  }
2398 
2399  llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
2400  AcquireBB = createBasicBlock("acquire", CurFn);
2401  ReleaseBB = createBasicBlock("release", CurFn);
2402  AcqRelBB = createBasicBlock("acqrel", CurFn);
2403  SeqCstBB = createBasicBlock("seqcst", CurFn);
2404  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
2405 
2406  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
2407  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
2408 
2409  Builder.SetInsertPoint(AcquireBB);
2410  Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
2411  Builder.CreateBr(ContBB);
2412  SI->addCase(Builder.getInt32(1), AcquireBB);
2413  SI->addCase(Builder.getInt32(2), AcquireBB);
2414 
2415  Builder.SetInsertPoint(ReleaseBB);
2416  Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
2417  Builder.CreateBr(ContBB);
2418  SI->addCase(Builder.getInt32(3), ReleaseBB);
2419 
2420  Builder.SetInsertPoint(AcqRelBB);
2421  Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
2422  Builder.CreateBr(ContBB);
2423  SI->addCase(Builder.getInt32(4), AcqRelBB);
2424 
2425  Builder.SetInsertPoint(SeqCstBB);
2426  Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
2427  Builder.CreateBr(ContBB);
2428  SI->addCase(Builder.getInt32(5), SeqCstBB);
2429 
2430  Builder.SetInsertPoint(ContBB);
2431  return RValue::get(nullptr);
2432  }
2433 
2434  case Builtin::BI__builtin_signbit:
2435  case Builtin::BI__builtin_signbitf:
2436  case Builtin::BI__builtin_signbitl: {
2437  return RValue::get(
2438  Builder.CreateZExt(EmitSignBit(*this, EmitScalarExpr(E->getArg(0))),
2439  ConvertType(E->getType())));
2440  }
2441  case Builtin::BI__annotation: {
2442  // Re-encode each wide string to UTF8 and make an MDString.
2444  for (const Expr *Arg : E->arguments()) {
2445  const auto *Str = cast<StringLiteral>(Arg->IgnoreParenCasts());
2446  assert(Str->getCharByteWidth() == 2);
2447  StringRef WideBytes = Str->getBytes();
2448  std::string StrUtf8;
2449  if (!convertUTF16ToUTF8String(
2450  makeArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
2451  CGM.ErrorUnsupported(E, "non-UTF16 __annotation argument");
2452  continue;
2453  }
2454  Strings.push_back(llvm::MDString::get(getLLVMContext(), StrUtf8));
2455  }
2456 
2457  // Build and MDTuple of MDStrings and emit the intrinsic call.
2458  llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::codeview_annotation, {});
2459  MDTuple *StrTuple = MDTuple::get(getLLVMContext(), Strings);
2460  Builder.CreateCall(F, MetadataAsValue::get(getLLVMContext(), StrTuple));
2461  return RValue::getIgnored();
2462  }
2463  case Builtin::BI__builtin_annotation: {
2464  llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0));
2465  llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation,
2466  AnnVal->getType());
2467 
2468  // Get the annotation string, go through casts. Sema requires this to be a
2469  // non-wide string literal, potentially casted, so the cast<> is safe.
2470  const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts();
2471  StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
2472  return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc()));
2473  }
2474  case Builtin::BI__builtin_addcb:
2475  case Builtin::BI__builtin_addcs:
2476  case Builtin::BI__builtin_addc:
2477  case Builtin::BI__builtin_addcl:
2478  case Builtin::BI__builtin_addcll:
2479  case Builtin::BI__builtin_subcb:
2480  case Builtin::BI__builtin_subcs:
2481  case Builtin::BI__builtin_subc:
2482  case Builtin::BI__builtin_subcl:
2483  case Builtin::BI__builtin_subcll: {
2484 
2485  // We translate all of these builtins from expressions of the form:
2486  // int x = ..., y = ..., carryin = ..., carryout, result;
2487  // result = __builtin_addc(x, y, carryin, &carryout);
2488  //
2489  // to LLVM IR of the form:
2490  //
2491  // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
2492  // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0
2493  // %carry1 = extractvalue {i32, i1} %tmp1, 1
2494  // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1,
2495  // i32 %carryin)
2496  // %result = extractvalue {i32, i1} %tmp2, 0
2497  // %carry2 = extractvalue {i32, i1} %tmp2, 1
2498  // %tmp3 = or i1 %carry1, %carry2
2499  // %tmp4 = zext i1 %tmp3 to i32
2500  // store i32 %tmp4, i32* %carryout
2501 
2502  // Scalarize our inputs.
2503  llvm::Value *X = EmitScalarExpr(E->getArg(0));
2504  llvm::Value *Y = EmitScalarExpr(E->getArg(1));
2505  llvm::Value *Carryin = EmitScalarExpr(E->getArg(2));
2506  Address CarryOutPtr = EmitPointerWithAlignment(E->getArg(3));
2507 
2508  // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow.
2509  llvm::Intrinsic::ID IntrinsicId;
2510  switch (BuiltinID) {
2511  default: llvm_unreachable("Unknown multiprecision builtin id.");
2512  case Builtin::BI__builtin_addcb:
2513  case Builtin::BI__builtin_addcs:
2514  case Builtin::BI__builtin_addc:
2515  case Builtin::BI__builtin_addcl:
2516  case Builtin::BI__builtin_addcll:
2517  IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
2518  break;
2519  case Builtin::BI__builtin_subcb:
2520  case Builtin::BI__builtin_subcs:
2521  case Builtin::BI__builtin_subc:
2522  case Builtin::BI__builtin_subcl:
2523  case Builtin::BI__builtin_subcll:
2524  IntrinsicId = llvm::Intrinsic::usub_with_overflow;
2525  break;
2526  }
2527 
2528  // Construct our resulting LLVM IR expression.
2529  llvm::Value *Carry1;
2530  llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId,
2531  X, Y, Carry1);
2532  llvm::Value *Carry2;
2533  llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId,
2534  Sum1, Carryin, Carry2);
2535  llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2),
2536  X->getType());
2537  Builder.CreateStore(CarryOut, CarryOutPtr);
2538  return RValue::get(Sum2);
2539  }
2540 
2541  case Builtin::BI__builtin_add_overflow:
2542  case Builtin::BI__builtin_sub_overflow:
2543  case Builtin::BI__builtin_mul_overflow: {
2544  const clang::Expr *LeftArg = E->getArg(0);
2545  const clang::Expr *RightArg = E->getArg(1);
2546  const clang::Expr *ResultArg = E->getArg(2);
2547 
2548  clang::QualType ResultQTy =
2549  ResultArg->getType()->castAs<PointerType>()->getPointeeType();
2550 
2551  WidthAndSignedness LeftInfo =
2552  getIntegerWidthAndSignedness(CGM.getContext(), LeftArg->getType());
2553  WidthAndSignedness RightInfo =
2554  getIntegerWidthAndSignedness(CGM.getContext(), RightArg->getType());
2555  WidthAndSignedness ResultInfo =
2556  getIntegerWidthAndSignedness(CGM.getContext(), ResultQTy);
2557 
2558  // Handle mixed-sign multiplication as a special case, because adding
2559  // runtime or backend support for our generic irgen would be too expensive.
2560  if (isSpecialMixedSignMultiply(BuiltinID, LeftInfo, RightInfo, ResultInfo))
2561  return EmitCheckedMixedSignMultiply(*this, LeftArg, LeftInfo, RightArg,
2562  RightInfo, ResultArg, ResultQTy,
2563  ResultInfo);
2564 
2565  WidthAndSignedness EncompassingInfo =
2566  EncompassingIntegerType({LeftInfo, RightInfo, ResultInfo});
2567 
2568  llvm::Type *EncompassingLLVMTy =
2569  llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width);
2570 
2571  llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy);
2572 
2573  llvm::Intrinsic::ID IntrinsicId;
2574  switch (BuiltinID) {
2575  default:
2576  llvm_unreachable("Unknown overflow builtin id.");
2577  case Builtin::BI__builtin_add_overflow:
2578  IntrinsicId = EncompassingInfo.Signed
2579  ? llvm::Intrinsic::sadd_with_overflow
2580  : llvm::Intrinsic::uadd_with_overflow;
2581  break;
2582  case Builtin::BI__builtin_sub_overflow:
2583  IntrinsicId = EncompassingInfo.Signed
2584  ? llvm::Intrinsic::ssub_with_overflow
2585  : llvm::Intrinsic::usub_with_overflow;
2586  break;
2587  case Builtin::BI__builtin_mul_overflow:
2588  IntrinsicId = EncompassingInfo.Signed
2589  ? llvm::Intrinsic::smul_with_overflow
2590  : llvm::Intrinsic::umul_with_overflow;
2591  break;
2592  }
2593 
2594  llvm::Value *Left = EmitScalarExpr(LeftArg);
2595  llvm::Value *Right = EmitScalarExpr(RightArg);
2596  Address ResultPtr = EmitPointerWithAlignment(ResultArg);
2597 
2598  // Extend each operand to the encompassing type.
2599  Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
2600  Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
2601 
2602  // Perform the operation on the extended values.
2603  llvm::Value *Overflow, *Result;
2604  Result = EmitOverflowIntrinsic(*this, IntrinsicId, Left, Right, Overflow);
2605 
2606  if (EncompassingInfo.Width > ResultInfo.Width) {
2607  // The encompassing type is wider than the result type, so we need to
2608  // truncate it.
2609  llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy);
2610 
2611  // To see if the truncation caused an overflow, we will extend
2612  // the result and then compare it to the original result.
2613  llvm::Value *ResultTruncExt = Builder.CreateIntCast(
2614  ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
2615  llvm::Value *TruncationOverflow =
2616  Builder.CreateICmpNE(Result, ResultTruncExt);
2617 
2618  Overflow = Builder.CreateOr(Overflow, TruncationOverflow);
2619  Result = ResultTrunc;
2620  }
2621 
2622  // Finally, store the result using the pointer.
2623  bool isVolatile =
2624  ResultArg->getType()->getPointeeType().isVolatileQualified();
2625  Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile);
2626 
2627  return RValue::get(Overflow);
2628  }
2629 
2630  case Builtin::BI__builtin_uadd_overflow:
2631  case Builtin::BI__builtin_uaddl_overflow:
2632  case Builtin::BI__builtin_uaddll_overflow:
2633  case Builtin::BI__builtin_usub_overflow:
2634  case Builtin::BI__builtin_usubl_overflow:
2635  case Builtin::BI__builtin_usubll_overflow:
2636  case Builtin::BI__builtin_umul_overflow:
2637  case Builtin::BI__builtin_umull_overflow:
2638  case Builtin::BI__builtin_umulll_overflow:
2639  case Builtin::BI__builtin_sadd_overflow:
2640  case Builtin::BI__builtin_saddl_overflow:
2641  case Builtin::BI__builtin_saddll_overflow:
2642  case Builtin::BI__builtin_ssub_overflow:
2643  case Builtin::BI__builtin_ssubl_overflow:
2644  case Builtin::BI__builtin_ssubll_overflow:
2645  case Builtin::BI__builtin_smul_overflow:
2646  case Builtin::BI__builtin_smull_overflow:
2647  case Builtin::BI__builtin_smulll_overflow: {
2648 
2649  // We translate all of these builtins directly to the relevant llvm IR node.
2650 
2651  // Scalarize our inputs.
2652  llvm::Value *X = EmitScalarExpr(E->getArg(0));
2653  llvm::Value *Y = EmitScalarExpr(E->getArg(1));
2654  Address SumOutPtr = EmitPointerWithAlignment(E->getArg(2));
2655 
2656  // Decide which of the overflow intrinsics we are lowering to:
2657  llvm::Intrinsic::ID IntrinsicId;
2658  switch (BuiltinID) {
2659  default: llvm_unreachable("Unknown overflow builtin id.");
2660  case Builtin::BI__builtin_uadd_overflow:
2661  case Builtin::BI__builtin_uaddl_overflow:
2662  case Builtin::BI__builtin_uaddll_overflow:
2663  IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
2664  break;
2665  case Builtin::BI__builtin_usub_overflow:
2666  case Builtin::BI__builtin_usubl_overflow:
2667  case Builtin::BI__builtin_usubll_overflow:
2668  IntrinsicId = llvm::Intrinsic::usub_with_overflow;
2669  break;
2670  case Builtin::BI__builtin_umul_overflow:
2671  case Builtin::BI__builtin_umull_overflow:
2672  case Builtin::BI__builtin_umulll_overflow:
2673  IntrinsicId = llvm::Intrinsic::umul_with_overflow;
2674  break;
2675  case Builtin::BI__builtin_sadd_overflow:
2676  case Builtin::BI__builtin_saddl_overflow:
2677  case Builtin::BI__builtin_saddll_overflow:
2678  IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
2679  break;
2680  case Builtin::BI__builtin_ssub_overflow:
2681  case Builtin::BI__builtin_ssubl_overflow:
2682  case Builtin::BI__builtin_ssubll_overflow:
2683  IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
2684  break;
2685  case Builtin::BI__builtin_smul_overflow:
2686  case Builtin::BI__builtin_smull_overflow:
2687  case Builtin::BI__builtin_smulll_overflow:
2688  IntrinsicId = llvm::Intrinsic::smul_with_overflow;
2689  break;
2690  }
2691 
2692 
2693  llvm::Value *Carry;
2694  llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry);
2695  Builder.CreateStore(Sum, SumOutPtr);
2696 
2697  return RValue::get(Carry);
2698  }
2699  case Builtin::BI__builtin_addressof:
2700  return RValue::get(EmitLValue(E->getArg(0)).getPointer());
2701  case Builtin::BI__builtin_operator_new:
2702  return EmitBuiltinNewDeleteCall(
2703  E->getCallee()->getType()->castAs<FunctionProtoType>(), E, false);
2704  case Builtin::BI__builtin_operator_delete:
2705  return EmitBuiltinNewDeleteCall(
2706  E->getCallee()->getType()->castAs<FunctionProtoType>(), E, true);
2707 
2708  case Builtin::BI__noop:
2709  // __noop always evaluates to an integer literal zero.
2710  return RValue::get(ConstantInt::get(IntTy, 0));
2711  case Builtin::BI__builtin_call_with_static_chain: {
2712  const CallExpr *Call = cast<CallExpr>(E->getArg(0));
2713  const Expr *Chain = E->getArg(1);
2714  return EmitCall(Call->getCallee()->getType(),
2715  EmitCallee(Call->getCallee()), Call, ReturnValue,
2716  EmitScalarExpr(Chain));
2717  }
2718  case Builtin::BI_InterlockedExchange8:
2719  case Builtin::BI_InterlockedExchange16:
2720  case Builtin::BI_InterlockedExchange:
2721  case Builtin::BI_InterlockedExchangePointer:
2722  return RValue::get(
2723  EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
2724  case Builtin::BI_InterlockedCompareExchangePointer: {
2725  llvm::Type *RTy;
2726  llvm::IntegerType *IntType =
2727  IntegerType::get(getLLVMContext(),
2728  getContext().getTypeSize(E->getType()));
2729  llvm::Type *IntPtrType = IntType->getPointerTo();
2730 
2731  llvm::Value *Destination =
2732  Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType);
2733 
2734  llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
2735  RTy = Exchange->getType();
2736  Exchange = Builder.CreatePtrToInt(Exchange, IntType);
2737 
2738  llvm::Value *Comparand =
2739  Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
2740 
2741  auto Result =
2742  Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
2743  AtomicOrdering::SequentiallyConsistent,
2744  AtomicOrdering::SequentiallyConsistent);
2745  Result->setVolatile(true);
2746 
2747  return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
2748  0),
2749  RTy));
2750  }
2751  case Builtin::BI_InterlockedCompareExchange8:
2752  case Builtin::BI_InterlockedCompareExchange16:
2753  case Builtin::BI_InterlockedCompareExchange:
2754  case Builtin::BI_InterlockedCompareExchange64: {
2755  AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg(
2756  EmitScalarExpr(E->getArg(0)),
2757  EmitScalarExpr(E->getArg(2)),
2758  EmitScalarExpr(E->getArg(1)),
2759  AtomicOrdering::SequentiallyConsistent,
2760  AtomicOrdering::SequentiallyConsistent);
2761  CXI->setVolatile(true);
2762  return RValue::get(Builder.CreateExtractValue(CXI, 0));
2763  }
2764  case Builtin::BI_InterlockedIncrement16:
2765  case Builtin::BI_InterlockedIncrement:
2766  return RValue::get(
2767  EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E));
2768  case Builtin::BI_InterlockedDecrement16:
2769  case Builtin::BI_InterlockedDecrement:
2770  return RValue::get(
2771  EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E));
2772  case Builtin::BI_InterlockedAnd8:
2773  case Builtin::BI_InterlockedAnd16:
2774  case Builtin::BI_InterlockedAnd:
2775  return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E));
2776  case Builtin::BI_InterlockedExchangeAdd8:
2777  case Builtin::BI_InterlockedExchangeAdd16:
2778  case Builtin::BI_InterlockedExchangeAdd:
2779  return RValue::get(
2780  EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E));
2781  case Builtin::BI_InterlockedExchangeSub8:
2782  case Builtin::BI_InterlockedExchangeSub16:
2783  case Builtin::BI_InterlockedExchangeSub:
2784  return RValue::get(
2785  EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E));
2786  case Builtin::BI_InterlockedOr8:
2787  case Builtin::BI_InterlockedOr16:
2788  case Builtin::BI_InterlockedOr:
2789  return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E));
2790  case Builtin::BI_InterlockedXor8:
2791  case Builtin::BI_InterlockedXor16:
2792  case Builtin::BI_InterlockedXor:
2793  return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E));
2794  case Builtin::BI_interlockedbittestandset:
2795  return RValue::get(
2796  EmitMSVCBuiltinExpr(MSVCIntrin::_interlockedbittestandset, E));
2797 
2798  case Builtin::BI__exception_code:
2799  case Builtin::BI_exception_code:
2800  return RValue::get(EmitSEHExceptionCode());
2801  case Builtin::BI__exception_info:
2802  case Builtin::BI_exception_info:
2803  return RValue::get(EmitSEHExceptionInfo());
2804  case Builtin::BI__abnormal_termination:
2805  case Builtin::BI_abnormal_termination:
2806  return RValue::get(EmitSEHAbnormalTermination());
2807  case Builtin::BI_setjmpex: {
2808  if (getTarget().getTriple().isOSMSVCRT()) {
2809  llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy};
2810  llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
2811  getLLVMContext(), llvm::AttributeList::FunctionIndex,
2812  llvm::Attribute::ReturnsTwice);
2813  llvm::Constant *SetJmpEx = CGM.CreateRuntimeFunction(
2814  llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false),
2815  "_setjmpex", ReturnsTwiceAttr, /*Local=*/true);
2816  llvm::Value *Buf = Builder.CreateBitOrPointerCast(
2817  EmitScalarExpr(E->getArg(0)), Int8PtrTy);
2818  llvm::Value *FrameAddr =
2819  Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
2820  ConstantInt::get(Int32Ty, 0));
2821  llvm::Value *Args[] = {Buf, FrameAddr};
2822  llvm::CallSite CS = EmitRuntimeCallOrInvoke(SetJmpEx, Args);
2823  CS.setAttributes(ReturnsTwiceAttr);
2824  return RValue::get(CS.getInstruction());
2825  }
2826  break;
2827  }
2828  case Builtin::BI_setjmp: {
2829  if (getTarget().getTriple().isOSMSVCRT()) {
2830  llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
2831  getLLVMContext(), llvm::AttributeList::FunctionIndex,
2832  llvm::Attribute::ReturnsTwice);
2833  llvm::Value *Buf = Builder.CreateBitOrPointerCast(
2834  EmitScalarExpr(E->getArg(0)), Int8PtrTy);
2835  llvm::CallSite CS;
2836  if (getTarget().getTriple().getArch() == llvm::Triple::x86) {
2837  llvm::Type *ArgTypes[] = {Int8PtrTy, IntTy};
2838  llvm::Constant *SetJmp3 = CGM.CreateRuntimeFunction(
2839  llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/true),
2840  "_setjmp3", ReturnsTwiceAttr, /*Local=*/true);
2841  llvm::Value *Count = ConstantInt::get(IntTy, 0);
2842  llvm::Value *Args[] = {Buf, Count};
2843  CS = EmitRuntimeCallOrInvoke(SetJmp3, Args);
2844  } else {
2845  llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy};
2846  llvm::Constant *SetJmp = CGM.CreateRuntimeFunction(
2847  llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false),
2848  "_setjmp", ReturnsTwiceAttr, /*Local=*/true);
2849  llvm::Value *FrameAddr =
2850  Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
2851  ConstantInt::get(Int32Ty, 0));
2852  llvm::Value *Args[] = {Buf, FrameAddr};
2853  CS = EmitRuntimeCallOrInvoke(SetJmp, Args);
2854  }
2855  CS.setAttributes(ReturnsTwiceAttr);
2856  return RValue::get(CS.getInstruction());
2857  }
2858  break;
2859  }
2860 
2861  case Builtin::BI__GetExceptionInfo: {
2862  if (llvm::GlobalVariable *GV =
2863  CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType()))
2864  return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy));
2865  break;
2866  }
2867 
2868  case Builtin::BI__fastfail:
2869  return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::__fastfail, E));
2870 
2871  case Builtin::BI__builtin_coro_size: {
2872  auto & Context = getContext();
2873  auto SizeTy = Context.getSizeType();
2874  auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
2875  Value *F = CGM.getIntrinsic(Intrinsic::coro_size, T);
2876  return RValue::get(Builder.CreateCall(F));
2877  }
2878 
2879  case Builtin::BI__builtin_coro_id:
2880  return EmitCoroutineIntrinsic(E, Intrinsic::coro_id);
2881  case Builtin::BI__builtin_coro_promise:
2882  return EmitCoroutineIntrinsic(E, Intrinsic::coro_promise);
2883  case Builtin::BI__builtin_coro_resume:
2884  return EmitCoroutineIntrinsic(E, Intrinsic::coro_resume);
2885  case Builtin::BI__builtin_coro_frame:
2886  return EmitCoroutineIntrinsic(E, Intrinsic::coro_frame);
2887  case Builtin::BI__builtin_coro_noop:
2888  return EmitCoroutineIntrinsic(E, Intrinsic::coro_noop);
2889  case Builtin::BI__builtin_coro_free:
2890  return EmitCoroutineIntrinsic(E, Intrinsic::coro_free);
2891  case Builtin::BI__builtin_coro_destroy:
2892  return EmitCoroutineIntrinsic(E, Intrinsic::coro_destroy);
2893  case Builtin::BI__builtin_coro_done:
2894  return EmitCoroutineIntrinsic(E, Intrinsic::coro_done);
2895  case Builtin::BI__builtin_coro_alloc:
2896  return EmitCoroutineIntrinsic(E, Intrinsic::coro_alloc);
2897  case Builtin::BI__builtin_coro_begin:
2898  return EmitCoroutineIntrinsic(E, Intrinsic::coro_begin);
2899  case Builtin::BI__builtin_coro_end:
2900  return EmitCoroutineIntrinsic(E, Intrinsic::coro_end);
2901  case Builtin::BI__builtin_coro_suspend:
2902  return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend);
2903  case Builtin::BI__builtin_coro_param:
2904  return EmitCoroutineIntrinsic(E, Intrinsic::coro_param);
2905 
2906  // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions
2907  case Builtin::BIread_pipe:
2908  case Builtin::BIwrite_pipe: {
2909  Value *Arg0 = EmitScalarExpr(E->getArg(0)),
2910  *Arg1 = EmitScalarExpr(E->getArg(1));
2911  CGOpenCLRuntime OpenCLRT(CGM);
2912  Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
2913  Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
2914 
2915  // Type of the generic packet parameter.
2916  unsigned GenericAS =
2917  getContext().getTargetAddressSpace(LangAS::opencl_generic);
2918  llvm::Type *I8PTy = llvm::PointerType::get(
2919  llvm::Type::getInt8Ty(getLLVMContext()), GenericAS);
2920 
2921  // Testing which overloaded version we should generate the call for.
2922  if (2U == E->getNumArgs()) {
2923  const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2"
2924  : "__write_pipe_2";
2925  // Creating a generic function type to be able to call with any builtin or
2926  // user defined type.
2927  llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
2928  llvm::FunctionType *FTy = llvm::FunctionType::get(
2929  Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2930  Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy);
2931  return RValue::get(
2932  Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2933  {Arg0, BCast, PacketSize, PacketAlign}));
2934  } else {
2935  assert(4 == E->getNumArgs() &&
2936  "Illegal number of parameters to pipe function");
2937  const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4"
2938  : "__write_pipe_4";
2939 
2940  llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
2941  Int32Ty, Int32Ty};
2942  Value *Arg2 = EmitScalarExpr(E->getArg(2)),
2943  *Arg3 = EmitScalarExpr(E->getArg(3));
2944  llvm::FunctionType *FTy = llvm::FunctionType::get(
2945  Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2946  Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy);
2947  // We know the third argument is an integer type, but we may need to cast
2948  // it to i32.
2949  if (Arg2->getType() != Int32Ty)
2950  Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty);
2951  return RValue::get(Builder.CreateCall(
2952  CGM.CreateRuntimeFunction(FTy, Name),
2953  {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
2954  }
2955  }
2956  // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write
2957  // functions
2958  case Builtin::BIreserve_read_pipe:
2959  case Builtin::BIreserve_write_pipe:
2960  case Builtin::BIwork_group_reserve_read_pipe:
2961  case Builtin::BIwork_group_reserve_write_pipe:
2962  case Builtin::BIsub_group_reserve_read_pipe:
2963  case Builtin::BIsub_group_reserve_write_pipe: {
2964  // Composing the mangled name for the function.
2965  const char *Name;
2966  if (BuiltinID == Builtin::BIreserve_read_pipe)
2967  Name = "__reserve_read_pipe";
2968  else if (BuiltinID == Builtin::BIreserve_write_pipe)
2969  Name = "__reserve_write_pipe";
2970  else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
2971  Name = "__work_group_reserve_read_pipe";
2972  else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
2973  Name = "__work_group_reserve_write_pipe";
2974  else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
2975  Name = "__sub_group_reserve_read_pipe";
2976  else
2977  Name = "__sub_group_reserve_write_pipe";
2978 
2979  Value *Arg0 = EmitScalarExpr(E->getArg(0)),
2980  *Arg1 = EmitScalarExpr(E->getArg(1));
2981  llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy);
2982  CGOpenCLRuntime OpenCLRT(CGM);
2983  Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
2984  Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
2985 
2986  // Building the generic function prototype.
2987  llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
2988  llvm::FunctionType *FTy = llvm::FunctionType::get(
2989  ReservedIDTy, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2990  // We know the second argument is an integer type, but we may need to cast
2991  // it to i32.
2992  if (Arg1->getType() != Int32Ty)
2993  Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty);
2994  return RValue::get(
2995  Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2996  {Arg0, Arg1, PacketSize, PacketAlign}));
2997  }
2998  // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write
2999  // functions
3000  case Builtin::BIcommit_read_pipe:
3001  case Builtin::BIcommit_write_pipe:
3002  case Builtin::BIwork_group_commit_read_pipe:
3003  case Builtin::BIwork_group_commit_write_pipe:
3004  case Builtin::BIsub_group_commit_read_pipe:
3005  case Builtin::BIsub_group_commit_write_pipe: {
3006  const char *Name;
3007  if (BuiltinID == Builtin::BIcommit_read_pipe)
3008  Name = "__commit_read_pipe";
3009  else if (BuiltinID == Builtin::BIcommit_write_pipe)
3010  Name = "__commit_write_pipe";
3011  else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
3012  Name = "__work_group_commit_read_pipe";
3013  else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
3014  Name = "__work_group_commit_write_pipe";
3015  else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
3016  Name = "__sub_group_commit_read_pipe";
3017  else
3018  Name = "__sub_group_commit_write_pipe";
3019 
3020  Value *Arg0 = EmitScalarExpr(E->getArg(0)),
3021  *Arg1 = EmitScalarExpr(E->getArg(1));
3022  CGOpenCLRuntime OpenCLRT(CGM);
3023  Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
3024  Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
3025 
3026  // Building the generic function prototype.
3027  llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
3028  llvm::FunctionType *FTy =
3029  llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
3030  llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3031 
3032  return RValue::get(
3033  Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3034  {Arg0, Arg1, PacketSize, PacketAlign}));
3035  }
3036  // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions
3037  case Builtin::BIget_pipe_num_packets:
3038  case Builtin::BIget_pipe_max_packets: {
3039  const char *BaseName;
3040  const PipeType *PipeTy = E->getArg(0)->getType()->getAs<PipeType>();
3041  if (BuiltinID == Builtin::BIget_pipe_num_packets)
3042  BaseName = "__get_pipe_num_packets";
3043  else
3044  BaseName = "__get_pipe_max_packets";
3045  auto Name = std::string(BaseName) +
3046  std::string(PipeTy->isReadOnly() ? "_ro" : "_wo");
3047 
3048  // Building the generic function prototype.
3049  Value *Arg0 = EmitScalarExpr(E->getArg(0));
3050  CGOpenCLRuntime OpenCLRT(CGM);
3051  Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
3052  Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
3053  llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
3054  llvm::FunctionType *FTy = llvm::FunctionType::get(
3055  Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3056 
3057  return RValue::get(Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3058  {Arg0, PacketSize, PacketAlign}));
3059  }
3060 
3061  // OpenCL v2.0 s6.13.9 - Address space qualifier functions.
3062  case Builtin::BIto_global:
3063  case Builtin::BIto_local:
3064  case Builtin::BIto_private: {
3065  auto Arg0 = EmitScalarExpr(E->getArg(0));
3066  auto NewArgT = llvm::PointerType::get(Int8Ty,
3067  CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic));
3068  auto NewRetT = llvm::PointerType::get(Int8Ty,
3069  CGM.getContext().getTargetAddressSpace(
3071  auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT}, false);
3072  llvm::Value *NewArg;
3073  if (Arg0->getType()->getPointerAddressSpace() !=
3074  NewArgT->getPointerAddressSpace())
3075  NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT);
3076  else
3077  NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT);
3078  auto NewName = std::string("__") + E->getDirectCallee()->getName().str();
3079  auto NewCall =
3080  Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, NewName), {NewArg});
3081  return RValue::get(Builder.CreateBitOrPointerCast(NewCall,
3082  ConvertType(E->getType())));
3083  }
3084 
3085  // OpenCL v2.0, s6.13.17 - Enqueue kernel function.
3086  // It contains four different overload formats specified in Table 6.13.17.1.
3087  case Builtin::BIenqueue_kernel: {
3088  StringRef Name; // Generated function call name
3089  unsigned NumArgs = E->getNumArgs();
3090 
3091  llvm::Type *QueueTy = ConvertType(getContext().OCLQueueTy);
3092  llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3093  getContext().getTargetAddressSpace(LangAS::opencl_generic));
3094 
3095  llvm::Value *Queue = EmitScalarExpr(E->getArg(0));
3096  llvm::Value *Flags = EmitScalarExpr(E->getArg(1));
3097  LValue NDRangeL = EmitAggExprToLValue(E->getArg(2));
3098  llvm::Value *Range = NDRangeL.getAddress().getPointer();
3099  llvm::Type *RangeTy = NDRangeL.getAddress().getType();
3100 
3101  if (NumArgs == 4) {
3102  // The most basic form of the call with parameters:
3103  // queue_t, kernel_enqueue_flags_t, ndrange_t, block(void)
3104  Name = "__enqueue_kernel_basic";
3105  llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangeTy, GenericVoidPtrTy,
3106  GenericVoidPtrTy};
3107  llvm::FunctionType *FTy = llvm::FunctionType::get(
3108  Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3109 
3110  auto Info =
3111  CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(3));
3112  llvm::Value *Kernel =
3113  Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3114  llvm::Value *Block =
3115  Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3116 
3117  AttrBuilder B;
3118  B.addAttribute(Attribute::ByVal);
3119  llvm::AttributeList ByValAttrSet =
3120  llvm::AttributeList::get(CGM.getModule().getContext(), 3U, B);
3121 
3122  auto RTCall =
3123  Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name, ByValAttrSet),
3124  {Queue, Flags, Range, Kernel, Block});
3125  RTCall->setAttributes(ByValAttrSet);
3126  return RValue::get(RTCall);
3127  }
3128  assert(NumArgs >= 5 && "Invalid enqueue_kernel signature");
3129 
3130  // Create a temporary array to hold the sizes of local pointer arguments
3131  // for the block. \p First is the position of the first size argument.
3132  auto CreateArrayForSizeVar = [=](unsigned First) {
3133  auto *AT = llvm::ArrayType::get(SizeTy, NumArgs - First);
3134  auto *Arr = Builder.CreateAlloca(AT);
3135  llvm::Value *Ptr;
3136  // Each of the following arguments specifies the size of the corresponding
3137  // argument passed to the enqueued block.
3138  auto *Zero = llvm::ConstantInt::get(IntTy, 0);
3139  for (unsigned I = First; I < NumArgs; ++I) {
3140  auto *Index = llvm::ConstantInt::get(IntTy, I - First);
3141  auto *GEP = Builder.CreateGEP(Arr, {Zero, Index});
3142  if (I == First)
3143  Ptr = GEP;
3144  auto *V =
3145  Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(I)), SizeTy);
3146  Builder.CreateAlignedStore(
3147  V, GEP, CGM.getDataLayout().getPrefTypeAlignment(SizeTy));
3148  }
3149  return Ptr;
3150  };
3151 
3152  // Could have events and/or varargs.
3153  if (E->getArg(3)->getType()->isBlockPointerType()) {
3154  // No events passed, but has variadic arguments.
3155  Name = "__enqueue_kernel_varargs";
3156  auto Info =
3157  CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(3));
3158  llvm::Value *Kernel =
3159  Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3160  auto *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3161  auto *PtrToSizeArray = CreateArrayForSizeVar(4);
3162 
3163  // Create a vector of the arguments, as well as a constant value to
3164  // express to the runtime the number of variadic arguments.
3165  std::vector<llvm::Value *> Args = {
3166  Queue, Flags, Range,
3167  Kernel, Block, ConstantInt::get(IntTy, NumArgs - 4),
3168  PtrToSizeArray};
3169  std::vector<llvm::Type *> ArgTys = {
3170  QueueTy, IntTy, RangeTy,
3171  GenericVoidPtrTy, GenericVoidPtrTy, IntTy,
3172  PtrToSizeArray->getType()};
3173 
3174  llvm::FunctionType *FTy = llvm::FunctionType::get(
3175  Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3176  return RValue::get(
3177  Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3179  }
3180  // Any calls now have event arguments passed.
3181  if (NumArgs >= 7) {
3182  llvm::Type *EventTy = ConvertType(getContext().OCLClkEventTy);
3183  llvm::Type *EventPtrTy = EventTy->getPointerTo(
3184  CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic));
3185 
3186  llvm::Value *NumEvents =
3187  Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(3)), Int32Ty);
3188  llvm::Value *EventList =
3189  E->getArg(4)->getType()->isArrayType()
3190  ? EmitArrayToPointerDecay(E->getArg(4)).getPointer()
3191  : EmitScalarExpr(E->getArg(4));
3192  llvm::Value *ClkEvent = EmitScalarExpr(E->getArg(5));
3193  // Convert to generic address space.
3194  EventList = Builder.CreatePointerCast(EventList, EventPtrTy);
3195  ClkEvent = Builder.CreatePointerCast(ClkEvent, EventPtrTy);
3196  auto Info =
3197  CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(6));
3198  llvm::Value *Kernel =
3199  Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3200  llvm::Value *Block =
3201  Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3202 
3203  std::vector<llvm::Type *> ArgTys = {
3204  QueueTy, Int32Ty, RangeTy, Int32Ty,
3205  EventPtrTy, EventPtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
3206 
3207  std::vector<llvm::Value *> Args = {Queue, Flags, Range, NumEvents,
3208  EventList, ClkEvent, Kernel, Block};
3209 
3210  if (NumArgs == 7) {
3211  // Has events but no variadics.
3212  Name = "__enqueue_kernel_basic_events";
3213  llvm::FunctionType *FTy = llvm::FunctionType::get(
3214  Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3215  return RValue::get(
3216  Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3218  }
3219  // Has event info and variadics
3220  // Pass the number of variadics to the runtime function too.
3221  Args.push_back(ConstantInt::get(Int32Ty, NumArgs - 7));
3222  ArgTys.push_back(Int32Ty);
3223  Name = "__enqueue_kernel_events_varargs";
3224 
3225  auto *PtrToSizeArray = CreateArrayForSizeVar(7);
3226  Args.push_back(PtrToSizeArray);
3227  ArgTys.push_back(PtrToSizeArray->getType());
3228 
3229  llvm::FunctionType *FTy = llvm::FunctionType::get(
3230  Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3231  return RValue::get(
3232  Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3234  }
3235  LLVM_FALLTHROUGH;
3236  }
3237  // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block
3238  // parameter.
3239  case Builtin::BIget_kernel_work_group_size: {
3240  llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3241  getContext().getTargetAddressSpace(LangAS::opencl_generic));
3242  auto Info =
3243  CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(0));
3244  Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3245  Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3246  return RValue::get(Builder.CreateCall(
3247  CGM.CreateRuntimeFunction(
3248  llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
3249  false),
3250  "__get_kernel_work_group_size_impl"),
3251  {Kernel, Arg}));
3252  }
3253  case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
3254  llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3255  getContext().getTargetAddressSpace(LangAS::opencl_generic));
3256  auto Info =
3257  CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(0));
3258  Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3259  Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3260  return RValue::get(Builder.CreateCall(
3261  CGM.CreateRuntimeFunction(
3262  llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
3263  false),
3264  "__get_kernel_preferred_work_group_size_multiple_impl"),
3265  {Kernel, Arg}));
3266  }
3267  case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
3268  case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
3269  llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3270  getContext().getTargetAddressSpace(LangAS::opencl_generic));
3271  LValue NDRangeL = EmitAggExprToLValue(E->getArg(0));
3272  llvm::Value *NDRange = NDRangeL.getAddress().getPointer();
3273  auto Info =
3274  CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(1));
3275  Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3276  Value *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3277  const char *Name =
3278  BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
3279  ? "__get_kernel_max_sub_group_size_for_ndrange_impl"
3280  : "__get_kernel_sub_group_count_for_ndrange_impl";
3281  return RValue::get(Builder.CreateCall(
3282  CGM.CreateRuntimeFunction(
3283  llvm::FunctionType::get(
3284  IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
3285  false),
3286  Name),
3287  {NDRange, Kernel, Block}));
3288  }
3289 
3290  case Builtin::BI__builtin_store_half:
3291  case Builtin::BI__builtin_store_halff: {
3292  Value *Val = EmitScalarExpr(E->getArg(0));
3293  Address Address = EmitPointerWithAlignment(E->getArg(1));
3294  Value *HalfVal = Builder.CreateFPTrunc(Val, Builder.getHalfTy());
3295  return RValue::get(Builder.CreateStore(HalfVal, Address));
3296  }
3297  case Builtin::BI__builtin_load_half: {
3298  Address Address = EmitPointerWithAlignment(E->getArg(0));
3299  Value *HalfVal = Builder.CreateLoad(Address);
3300  return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getDoubleTy()));
3301  }
3302  case Builtin::BI__builtin_load_halff: {
3303  Address Address = EmitPointerWithAlignment(E->getArg(0));
3304  Value *HalfVal = Builder.CreateLoad(Address);
3305  return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getFloatTy()));
3306  }
3307  case Builtin::BIprintf:
3308  if (getTarget().getTriple().isNVPTX())
3309  return EmitNVPTXDevicePrintfCallExpr(E, ReturnValue);
3310  break;
3311  case Builtin::BI__builtin_canonicalize:
3312  case Builtin::BI__builtin_canonicalizef:
3313  case Builtin::BI__builtin_canonicalizel:
3314  return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::canonicalize));
3315 
3316  case Builtin::BI__builtin_thread_pointer: {
3317  if (!getContext().getTargetInfo().isTLSSupported())
3318  CGM.ErrorUnsupported(E, "__builtin_thread_pointer");
3319  // Fall through - it's already mapped to the intrinsic by GCCBuiltin.
3320  break;
3321  }
3322  case Builtin::BI__builtin_os_log_format:
3323  return emitBuiltinOSLogFormat(*E);
3324 
3325  case Builtin::BI__builtin_os_log_format_buffer_size: {
3327  analyze_os_log::computeOSLogBufferLayout(CGM.getContext(), E, Layout);
3328  return RValue::get(ConstantInt::get(ConvertType(E->getType()),
3329  Layout.size().getQuantity()));
3330  }
3331 
3332  case Builtin::BI__xray_customevent: {
3333  if (!ShouldXRayInstrumentFunction())
3334  return RValue::getIgnored();
3335 
3336  if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
3338  return RValue::getIgnored();
3339 
3340  if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
3341  if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayCustomEvents())
3342  return RValue::getIgnored();
3343 
3344  Function *F = CGM.getIntrinsic(Intrinsic::xray_customevent);
3345  auto FTy = F->getFunctionType();
3346  auto Arg0 = E->getArg(0);
3347  auto Arg0Val = EmitScalarExpr(Arg0);
3348  auto Arg0Ty = Arg0->getType();
3349  auto PTy0 = FTy->getParamType(0);
3350  if (PTy0 != Arg0Val->getType()) {
3351  if (Arg0Ty->isArrayType())
3352  Arg0Val = EmitArrayToPointerDecay(Arg0).getPointer();
3353  else
3354  Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0);
3355  }
3356  auto Arg1 = EmitScalarExpr(E->getArg(1));
3357  auto PTy1 = FTy->getParamType(1);
3358  if (PTy1 != Arg1->getType())
3359  Arg1 = Builder.CreateTruncOrBitCast(Arg1, PTy1);
3360  return RValue::get(Builder.CreateCall(F, {Arg0Val, Arg1}));
3361  }
3362 
3363  case Builtin::BI__xray_typedevent: {
3364  // TODO: There should be a way to always emit events even if the current
3365  // function is not instrumented. Losing events in a stream can cripple
3366  // a trace.
3367  if (!ShouldXRayInstrumentFunction())
3368  return RValue::getIgnored();
3369 
3370  if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
3372  return RValue::getIgnored();
3373 
3374  if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
3375  if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayTypedEvents())
3376  return RValue::getIgnored();
3377 
3378  Function *F = CGM.getIntrinsic(Intrinsic::xray_typedevent);
3379  auto FTy = F->getFunctionType();
3380  auto Arg0 = EmitScalarExpr(E->getArg(0));
3381  auto PTy0 = FTy->getParamType(0);
3382  if (PTy0 != Arg0->getType())
3383  Arg0 = Builder.CreateTruncOrBitCast(Arg0, PTy0);
3384  auto Arg1 = E->getArg(1);
3385  auto Arg1Val = EmitScalarExpr(Arg1);
3386  auto Arg1Ty = Arg1->getType();
3387  auto PTy1 = FTy->getParamType(1);
3388  if (PTy1 != Arg1Val->getType()) {
3389  if (Arg1Ty->isArrayType())
3390  Arg1Val = EmitArrayToPointerDecay(Arg1).getPointer();
3391  else
3392  Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1);
3393  }
3394  auto Arg2 = EmitScalarExpr(E->getArg(2));
3395  auto PTy2 = FTy->getParamType(2);
3396  if (PTy2 != Arg2->getType())
3397  Arg2 = Builder.CreateTruncOrBitCast(Arg2, PTy2);
3398  return RValue::get(Builder.CreateCall(F, {Arg0, Arg1Val, Arg2}));
3399  }
3400 
3401  case Builtin::BI__builtin_ms_va_start:
3402  case Builtin::BI__builtin_ms_va_end:
3403  return RValue::get(
3404  EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(),
3405  BuiltinID == Builtin::BI__builtin_ms_va_start));
3406 
3407  case Builtin::BI__builtin_ms_va_copy: {
3408  // Lower this manually. We can't reliably determine whether or not any
3409  // given va_copy() is for a Win64 va_list from the calling convention
3410  // alone, because it's legal to do this from a System V ABI function.
3411  // With opaque pointer types, we won't have enough information in LLVM
3412  // IR to determine this from the argument types, either. Best to do it
3413  // now, while we have enough information.
3414  Address DestAddr = EmitMSVAListRef(E->getArg(0));
3415  Address SrcAddr = EmitMSVAListRef(E->getArg(1));
3416 
3417  llvm::Type *BPP = Int8PtrPtrTy;
3418 
3419  DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), BPP, "cp"),
3420  DestAddr.getAlignment());
3421  SrcAddr = Address(Builder.CreateBitCast(SrcAddr.getPointer(), BPP, "ap"),
3422  SrcAddr.getAlignment());
3423 
3424  Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val");
3425  return RValue::get(Builder.CreateStore(ArgPtr, DestAddr));
3426  }
3427  }
3428 
3429  // If this is an alias for a lib function (e.g. __builtin_sin), emit
3430  // the call using the normal call path, but using the unmangled
3431  // version of the function name.
3432  if (getContext().BuiltinInfo.isLibFunction(BuiltinID))
3433  return emitLibraryCall(*this, FD, E,
3434  CGM.getBuiltinLibFunction(FD, BuiltinID));
3435 
3436  // If this is a predefined lib function (e.g. malloc), emit the call
3437  // using exactly the normal call path.
3438  if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
3439  return emitLibraryCall(*this, FD, E,
3440  cast<llvm::Constant>(EmitScalarExpr(E->getCallee())));
3441 
3442  // Check that a call to a target specific builtin has the correct target
3443  // features.
3444  // This is down here to avoid non-target specific builtins, however, if
3445  // generic builtins start to require generic target features then we
3446  // can move this up to the beginning of the function.
3447  checkTargetFeatures(E, FD);
3448 
3449  // See if we have a target specific intrinsic.
3450  const char *Name = getContext().BuiltinInfo.getName(BuiltinID);
3451  Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
3452  StringRef Prefix =
3453  llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch());
3454  if (!Prefix.empty()) {
3455  IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix.data(), Name);
3456  // NOTE we don't need to perform a compatibility flag check here since the
3457  // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
3458  // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
3459  if (IntrinsicID == Intrinsic::not_intrinsic)
3460  IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
3461  }
3462 
3463  if (IntrinsicID != Intrinsic::not_intrinsic) {
3465 
3466  // Find out if any arguments are required to be integer constant
3467  // expressions.
3468  unsigned ICEArguments = 0;
3470  getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
3471  assert(Error == ASTContext::GE_None && "Should not codegen an error");
3472 
3473  Function *F = CGM.getIntrinsic(IntrinsicID);
3474  llvm::FunctionType *FTy = F->getFunctionType();
3475 
3476  for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
3477  Value *ArgValue;
3478  // If this is a normal argument, just emit it as a scalar.
3479  if ((ICEArguments & (1 << i)) == 0) {
3480  ArgValue = EmitScalarExpr(E->getArg(i));
3481  } else {
3482  // If this is required to be a constant, constant fold it so that we
3483  // know that the generated intrinsic gets a ConstantInt.
3484  llvm::APSInt Result;
3485  bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext());
3486  assert(IsConst && "Constant arg isn't actually constant?");
3487  (void)IsConst;
3488  ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result);
3489  }
3490 
3491  // If the intrinsic arg type is different from the builtin arg type
3492  // we need to do a bit cast.
3493  llvm::Type *PTy = FTy->getParamType(i);
3494  if (PTy != ArgValue->getType()) {
3495  assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
3496  "Must be able to losslessly bit cast to param");
3497  ArgValue = Builder.CreateBitCast(ArgValue, PTy);
3498  }
3499 
3500  Args.push_back(ArgValue);
3501  }
3502 
3503  Value *V = Builder.CreateCall(F, Args);
3504  QualType BuiltinRetType = E->getType();
3505 
3506  llvm::Type *RetTy = VoidTy;
3507  if (!BuiltinRetType->isVoidType())
3508  RetTy = ConvertType(BuiltinRetType);
3509 
3510  if (RetTy != V->getType()) {
3511  assert(V->getType()->canLosslesslyBitCastTo(RetTy) &&
3512  "Must be able to losslessly bit cast result type");
3513  V = Builder.CreateBitCast(V, RetTy);
3514  }
3515 
3516  return RValue::get(V);
3517  }
3518 
3519  // See if we have a target specific builtin that needs to be lowered.
3520  if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E))
3521  return RValue::get(V);
3522 
3523  ErrorUnsupported(E, "builtin function");
3524 
3525  // Unknown builtin, for now just dump it out and return undef.
3526  return GetUndefRValue(E->getType());
3527 }
3528 
3530  unsigned BuiltinID, const CallExpr *E,
3531  llvm::Triple::ArchType Arch) {
3532  switch (Arch) {
3533  case llvm::Triple::arm:
3534  case llvm::Triple::armeb:
3535  case llvm::Triple::thumb:
3536  case llvm::Triple::thumbeb:
3537  return CGF->EmitARMBuiltinExpr(BuiltinID, E, Arch);
3538  case llvm::Triple::aarch64:
3539  case llvm::Triple::aarch64_be:
3540  return CGF->EmitAArch64BuiltinExpr(BuiltinID, E, Arch);
3541  case llvm::Triple::x86:
3542  case llvm::Triple::x86_64:
3543  return CGF->EmitX86BuiltinExpr(BuiltinID, E);
3544  case llvm::Triple::ppc:
3545  case llvm::Triple::ppc64:
3546  case llvm::Triple::ppc64le:
3547  return CGF->EmitPPCBuiltinExpr(BuiltinID, E);
3548  case llvm::Triple::r600:
3549  case llvm::Triple::amdgcn:
3550  return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
3551  case llvm::Triple::systemz:
3552  return CGF->EmitSystemZBuiltinExpr(BuiltinID, E);
3553  case llvm::Triple::nvptx:
3554  case llvm::Triple::nvptx64:
3555  return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E);
3556  case llvm::Triple::wasm32:
3557  case llvm::Triple::wasm64:
3558  return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E);
3559  case llvm::Triple::hexagon:
3560  return CGF->EmitHexagonBuiltinExpr(BuiltinID, E);
3561  default:
3562  return nullptr;
3563  }
3564 }
3565 
3567  const CallExpr *E) {
3568  if (getContext().BuiltinInfo.isAuxBuiltinID(BuiltinID)) {
3569  assert(getContext().getAuxTargetInfo() && "Missing aux target info");
3571  this, getContext().BuiltinInfo.getAuxBuiltinID(BuiltinID), E,
3572  getContext().getAuxTargetInfo()->getTriple().getArch());
3573  }
3574 
3575  return EmitTargetArchBuiltinExpr(this, BuiltinID, E,
3576  getTarget().getTriple().getArch());
3577 }
3578 
3579 static llvm::VectorType *GetNeonType(CodeGenFunction *CGF,
3580  NeonTypeFlags TypeFlags,
3581  bool HasLegalHalfType=true,
3582  bool V1Ty=false) {
3583  int IsQuad = TypeFlags.isQuad();
3584  switch (TypeFlags.getEltType()) {
3585  case NeonTypeFlags::Int8:
3586  case NeonTypeFlags::Poly8:
3587  return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad));
3588  case NeonTypeFlags::Int16:
3589  case NeonTypeFlags::Poly16:
3590  return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
3592  if (HasLegalHalfType)
3593  return llvm::VectorType::get(CGF->HalfTy, V1Ty ? 1 : (4 << IsQuad));
3594  else
3595  return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
3596  case NeonTypeFlags::Int32:
3597  return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad));
3598  case NeonTypeFlags::Int64:
3599  case NeonTypeFlags::Poly64:
3600  return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad));
3602  // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
3603  // There is a lot of i128 and f128 API missing.
3604  // so we use v16i8 to represent poly128 and get pattern matched.
3605  return llvm::VectorType::get(CGF->Int8Ty, 16);
3607  return llvm::VectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad));
3609  return llvm::VectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad));
3610  }
3611  llvm_unreachable("Unknown vector element type!");
3612 }
3613 
3614 static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF,
3615  NeonTypeFlags IntTypeFlags) {
3616  int IsQuad = IntTypeFlags.isQuad();
3617  switch (IntTypeFlags.getEltType()) {
3618  case NeonTypeFlags::Int16:
3619  return llvm::VectorType::get(CGF->HalfTy, (4 << IsQuad));
3620  case NeonTypeFlags::Int32:
3621  return llvm::VectorType::get(CGF->FloatTy, (2 << IsQuad));
3622  case NeonTypeFlags::Int64:
3623  return llvm::VectorType::get(CGF->DoubleTy, (1 << IsQuad));
3624  default:
3625  llvm_unreachable("Type can't be converted to floating-point!");
3626  }
3627 }
3628 
3630  unsigned nElts = V->getType()->getVectorNumElements();
3631  Value* SV = llvm::ConstantVector::getSplat(nElts, C);
3632  return Builder.CreateShuffleVector(V, V, SV, "lane");
3633 }
3634 
3636  const char *name,
3637  unsigned shift, bool rightshift) {
3638  unsigned j = 0;
3639  for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
3640  ai != ae; ++ai, ++j)
3641  if (shift > 0 && shift == j)
3642  Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
3643  else
3644  Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
3645 
3646  return Builder.CreateCall(F, Ops, name);
3647 }
3648 
3650  bool neg) {
3651  int SV = cast<ConstantInt>(V)->getSExtValue();
3652  return ConstantInt::get(Ty, neg ? -SV : SV);
3653 }
3654 
3655 // Right-shift a vector by a constant.
3657  llvm::Type *Ty, bool usgn,
3658  const char *name) {
3659  llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
3660 
3661  int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
3662  int EltSize = VTy->getScalarSizeInBits();
3663 
3664  Vec = Builder.CreateBitCast(Vec, Ty);
3665 
3666  // lshr/ashr are undefined when the shift amount is equal to the vector
3667  // element size.
3668  if (ShiftAmt == EltSize) {
3669  if (usgn) {
3670  // Right-shifting an unsigned value by its size yields 0.
3671  return llvm::ConstantAggregateZero::get(VTy);
3672  } else {
3673  // Right-shifting a signed value by its size is equivalent
3674  // to a shift of size-1.
3675  --ShiftAmt;
3676  Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
3677  }
3678  }
3679 
3680  Shift = EmitNeonShiftVector(Shift, Ty, false);
3681  if (usgn)
3682  return Builder.CreateLShr(Vec, Shift, name);
3683  else
3684  return Builder.CreateAShr(Vec, Shift, name);
3685 }
3686 
3687 enum {
3688  AddRetType = (1 << 0),
3689  Add1ArgType = (1 << 1),
3690  Add2ArgTypes = (1 << 2),
3691 
3692  VectorizeRetType = (1 << 3),
3693  VectorizeArgTypes = (1 << 4),
3694 
3695  InventFloatType = (1 << 5),
3696  UnsignedAlts = (1 << 6),
3697 
3698  Use64BitVectors = (1 << 7),
3699  Use128BitVectors = (1 << 8),
3700 
3707 };
3708 
3709 namespace {
3710 struct NeonIntrinsicInfo {
3711  const char *NameHint;
3712  unsigned BuiltinID;
3713  unsigned LLVMIntrinsic;
3714  unsigned AltLLVMIntrinsic;
3715  unsigned TypeModifier;
3716 
3717  bool operator<(unsigned RHSBuiltinID) const {
3718  return BuiltinID < RHSBuiltinID;
3719  }
3720  bool operator<(const NeonIntrinsicInfo &TE) const {
3721  return BuiltinID < TE.BuiltinID;
3722  }
3723 };
3724 } // end anonymous namespace
3725 
3726 #define NEONMAP0(NameBase) \
3727  { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
3728 
3729 #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
3730  { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
3731  Intrinsic::LLVMIntrinsic, 0, TypeModifier }
3732 
3733 #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
3734  { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
3735  Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
3736  TypeModifier }
3737 
3738 static const NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = {
3739  NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
3740  NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
3741  NEONMAP1(vabs_v, arm_neon_vabs, 0),
3742  NEONMAP1(vabsq_v, arm_neon_vabs, 0),
3743  NEONMAP0(vaddhn_v),
3744  NEONMAP1(vaesdq_v, arm_neon_aesd, 0),
3745  NEONMAP1(vaeseq_v, arm_neon_aese, 0),
3746  NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0),
3747  NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0),
3748  NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType),
3749  NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType),
3750  NEONMAP1(vcage_v, arm_neon_vacge, 0),
3751  NEONMAP1(vcageq_v, arm_neon_vacge, 0),
3752  NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
3753  NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
3754  NEONMAP1(vcale_v, arm_neon_vacge, 0),
3755  NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
3756  NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
3757  NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
3758  NEONMAP0(vceqz_v),
3759  NEONMAP0(vceqzq_v),
3760  NEONMAP0(vcgez_v),
3761  NEONMAP0(vcgezq_v),
3762  NEONMAP0(vcgtz_v),
3763  NEONMAP0(vcgtzq_v),
3764  NEONMAP0(vclez_v),
3765  NEONMAP0(vclezq_v),
3766  NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType),
3767  NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType),
3768  NEONMAP0(vcltz_v),
3769  NEONMAP0(vcltzq_v),
3770  NEONMAP1(vclz_v, ctlz, Add1ArgType),
3771  NEONMAP1(vclzq_v, ctlz, Add1ArgType),
3772  NEONMAP1(vcnt_v, ctpop, Add1ArgType),
3773  NEONMAP1(vcntq_v, ctpop, Add1ArgType),
3774  NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
3775  NEONMAP0(vcvt_f16_v),
3776  NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
3777  NEONMAP0(vcvt_f32_v),
3778  NEONMAP2(vcvt_n_f16_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
3779  NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
3780  NEONMAP1(vcvt_n_s16_v, arm_neon_vcvtfp2fxs, 0),
3781  NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
3782  NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
3783  NEONMAP1(vcvt_n_u16_v, arm_neon_vcvtfp2fxu, 0),
3784  NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
3785  NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
3786  NEONMAP0(vcvt_s16_v),
3787  NEONMAP0(vcvt_s32_v),
3788  NEONMAP0(vcvt_s64_v),
3789  NEONMAP0(vcvt_u16_v),
3790  NEONMAP0(vcvt_u32_v),
3791  NEONMAP0(vcvt_u64_v),
3792  NEONMAP1(vcvta_s16_v, arm_neon_vcvtas, 0),
3793  NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
3794  NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
3795  NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
3796  NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
3797  NEONMAP1(vcvtaq_s16_v, arm_neon_vcvtas, 0),
3798  NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
3799  NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
3800  NEONMAP1(vcvtaq_u16_v, arm_neon_vcvtau, 0),
3801  NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
3802  NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
3803  NEONMAP1(vcvtm_s16_v, arm_neon_vcvtms, 0),
3804  NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
3805  NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
3806  NEONMAP1(vcvtm_u16_v, arm_neon_vcvtmu, 0),
3807  NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
3808  NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
3809  NEONMAP1(vcvtmq_s16_v, arm_neon_vcvtms, 0),
3810  NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
3811  NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
3812  NEONMAP1(vcvtmq_u16_v, arm_neon_vcvtmu, 0),
3813  NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
3814  NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
3815  NEONMAP1(vcvtn_s16_v, arm_neon_vcvtns, 0),
3816  NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
3817  NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
3818  NEONMAP1(vcvtn_u16_v, arm_neon_vcvtnu, 0),
3819  NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
3820  NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
3821  NEONMAP1(vcvtnq_s16_v, arm_neon_vcvtns, 0),
3822  NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
3823  NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
3824  NEONMAP1(vcvtnq_u16_v, arm_neon_vcvtnu, 0),
3825  NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
3826  NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
3827  NEONMAP1(vcvtp_s16_v, arm_neon_vcvtps, 0),
3828  NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
3829  NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
3830  NEONMAP1(vcvtp_u16_v, arm_neon_vcvtpu, 0),
3831  NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
3832  NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
3833  NEONMAP1(vcvtpq_s16_v, arm_neon_vcvtps, 0),
3834  NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
3835  NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
3836  NEONMAP1(vcvtpq_u16_v, arm_neon_vcvtpu, 0),
3837  NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
3838  NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
3839  NEONMAP0(vcvtq_f16_v),
3840  NEONMAP0(vcvtq_f32_v),
3841  NEONMAP2(vcvtq_n_f16_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
3842  NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
3843  NEONMAP1(vcvtq_n_s16_v, arm_neon_vcvtfp2fxs, 0),
3844  NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
3845  NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
3846  NEONMAP1(vcvtq_n_u16_v, arm_neon_vcvtfp2fxu, 0),
3847  NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
3848  NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
3849  NEONMAP0(vcvtq_s16_v),
3850  NEONMAP0(vcvtq_s32_v),
3851  NEONMAP0(vcvtq_s64_v),
3852  NEONMAP0(vcvtq_u16_v),
3853  NEONMAP0(vcvtq_u32_v),
3854  NEONMAP0(vcvtq_u64_v),
3855  NEONMAP2(vdot_v, arm_neon_udot, arm_neon_sdot, 0),
3856  NEONMAP2(vdotq_v, arm_neon_udot, arm_neon_sdot, 0),
3857  NEONMAP0(vext_v),
3858  NEONMAP0(vextq_v),
3859  NEONMAP0(vfma_v),
3860  NEONMAP0(vfmaq_v),
3861  NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
3862  NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
3863  NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
3864  NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
3865  NEONMAP0(vld1_dup_v),
3866  NEONMAP1(vld1_v, arm_neon_vld1, 0),
3867  NEONMAP0(vld1q_dup_v),
3868  NEONMAP1(vld1q_v, arm_neon_vld1, 0),
3869  NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
3870  NEONMAP1(vld2_v, arm_neon_vld2, 0),
3871  NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
3872  NEONMAP1(vld2q_v, arm_neon_vld2, 0),
3873  NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
3874  NEONMAP1(vld3_v, arm_neon_vld3, 0),
3875  NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
3876  NEONMAP1(vld3q_v, arm_neon_vld3, 0),
3877  NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
3878  NEONMAP1(vld4_v, arm_neon_vld4, 0),
3879  NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
3880  NEONMAP1(vld4q_v, arm_neon_vld4, 0),
3881  NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
3882  NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType),
3883  NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType),
3884  NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
3885  NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
3886  NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType),
3887  NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType),
3888  NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
3889  NEONMAP0(vmovl_v),
3890  NEONMAP0(vmovn_v),
3891  NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType),
3892  NEONMAP0(vmull_v),
3893  NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType),
3894  NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
3895  NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
3896  NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType),
3897  NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
3898  NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
3899  NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType),
3900  NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts),
3901  NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts),
3902  NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType),
3903  NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType),
3904  NEONMAP2(vqadd_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts),
3905  NEONMAP2(vqaddq_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts),
3906  NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0),
3907  NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0),
3908  NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType),
3909  NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType),
3910  NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType),
3911  NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts),
3912  NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType),
3913  NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType),
3914  NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType),
3915  NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType),
3916  NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType),
3917  NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
3918  NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
3919  NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
3920  NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
3921  NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
3922  NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
3923  NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
3924  NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
3925  NEONMAP2(vqsub_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts),
3926  NEONMAP2(vqsubq_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts),
3927  NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType),
3928  NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
3929  NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
3930  NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType),
3931  NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType),
3932  NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
3933  NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
3934  NEONMAP1(vrnd_v, arm_neon_vrintz, Add1ArgType),
3935  NEONMAP1(vrnda_v, arm_neon_vrinta, Add1ArgType),
3936  NEONMAP1(vrndaq_v, arm_neon_vrinta, Add1ArgType),
3937  NEONMAP1(vrndm_v, arm_neon_vrintm, Add1ArgType),
3938  NEONMAP1(vrndmq_v, arm_neon_vrintm, Add1ArgType),
3939  NEONMAP1(vrndn_v, arm_neon_vrintn, Add1ArgType),
3940  NEONMAP1(vrndnq_v, arm_neon_vrintn, Add1ArgType),
3941  NEONMAP1(vrndp_v, arm_neon_vrintp, Add1ArgType),
3942  NEONMAP1(vrndpq_v, arm_neon_vrintp, Add1ArgType),
3943  NEONMAP1(vrndq_v, arm_neon_vrintz, Add1ArgType),
3944  NEONMAP1(vrndx_v, arm_neon_vrintx, Add1ArgType),
3945  NEONMAP1(vrndxq_v, arm_neon_vrintx, Add1ArgType),
3946  NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
3947  NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
3948  NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
3949  NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
3950  NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
3951  NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
3952  NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType),
3953  NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType),
3954  NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType),
3955  NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0),
3956  NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0),
3957  NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0),
3958  NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0),
3959  NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0),
3960  NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0),
3961  NEONMAP0(vshl_n_v),
3962  NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
3963  NEONMAP0(vshll_n_v),
3964  NEONMAP0(vshlq_n_v),
3965  NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
3966  NEONMAP0(vshr_n_v),
3967  NEONMAP0(vshrn_n_v),
3968  NEONMAP0(vshrq_n_v),
3969  NEONMAP1(vst1_v, arm_neon_vst1, 0),
3970  NEONMAP1(vst1q_v, arm_neon_vst1, 0),
3971  NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
3972  NEONMAP1(vst2_v, arm_neon_vst2, 0),
3973  NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
3974  NEONMAP1(vst2q_v, arm_neon_vst2, 0),
3975  NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
3976  NEONMAP1(vst3_v, arm_neon_vst3, 0),
3977  NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
3978  NEONMAP1(vst3q_v, arm_neon_vst3, 0),
3979  NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
3980  NEONMAP1(vst4_v, arm_neon_vst4, 0),
3981  NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
3982  NEONMAP1(vst4q_v, arm_neon_vst4, 0),
3983  NEONMAP0(vsubhn_v),
3984  NEONMAP0(vtrn_v),
3985  NEONMAP0(vtrnq_v),
3986  NEONMAP0(vtst_v),
3987  NEONMAP0(vtstq_v),
3988  NEONMAP0(vuzp_v),
3989  NEONMAP0(vuzpq_v),
3990  NEONMAP0(vzip_v),
3991  NEONMAP0(vzipq_v)
3992 };
3993 
3994 static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = {
3995  NEONMAP1(vabs_v, aarch64_neon_abs, 0),
3996  NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
3997  NEONMAP0(vaddhn_v),
3998  NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0),
3999  NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0),
4000  NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0),
4001  NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0),
4002  NEONMAP1(vcage_v, aarch64_neon_facge, 0),
4003  NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
4004  NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
4005  NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
4006  NEONMAP1(vcale_v, aarch64_neon_facge, 0),
4007  NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
4008  NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
4009  NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
4010  NEONMAP0(vceqz_v),
4011  NEONMAP0(vceqzq_v),
4012  NEONMAP0(vcgez_v),
4013  NEONMAP0(vcgezq_v),
4014  NEONMAP0(vcgtz_v),
4015  NEONMAP0(vcgtzq_v),
4016  NEONMAP0(vclez_v),
4017  NEONMAP0(vclezq_v),
4018  NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType),
4019  NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType),
4020  NEONMAP0(vcltz_v),
4021  NEONMAP0(vcltzq_v),
4022  NEONMAP1(vclz_v, ctlz, Add1ArgType),
4023  NEONMAP1(vclzq_v, ctlz, Add1ArgType),
4024  NEONMAP1(vcnt_v, ctpop, Add1ArgType),
4025  NEONMAP1(vcntq_v, ctpop, Add1ArgType),
4026  NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
4027  NEONMAP0(vcvt_f16_v),
4028  NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
4029  NEONMAP0(vcvt_f32_v),
4030  NEONMAP2(vcvt_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4031  NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4032  NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4033  NEONMAP1(vcvt_n_s16_v, aarch64_neon_vcvtfp2fxs, 0),
4034  NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
4035  NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
4036  NEONMAP1(vcvt_n_u16_v, aarch64_neon_vcvtfp2fxu, 0),
4037  NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
4038  NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
4039  NEONMAP0(vcvtq_f16_v),
4040  NEONMAP0(vcvtq_f32_v),
4041  NEONMAP2(vcvtq_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4042  NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4043  NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4044  NEONMAP1(vcvtq_n_s16_v, aarch64_neon_vcvtfp2fxs, 0),
4045  NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
4046  NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
4047  NEONMAP1(vcvtq_n_u16_v, aarch64_neon_vcvtfp2fxu, 0),
4048  NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
4049  NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
4050  NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType),
4051  NEONMAP2(vdot_v, aarch64_neon_udot, aarch64_neon_sdot, 0),
4052  NEONMAP2(vdotq_v, aarch64_neon_udot, aarch64_neon_sdot, 0),
4053  NEONMAP0(vext_v),
4054  NEONMAP0(vextq_v),
4055  NEONMAP0(vfma_v),
4056  NEONMAP0(vfmaq_v),
4057  NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
4058  NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
4059  NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
4060  NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
4061  NEONMAP0(vmovl_v),
4062  NEONMAP0(vmovn_v),
4063  NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType),
4064  NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType),
4065  NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType),
4066  NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
4067  NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
4068  NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType),
4069  NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType),
4070  NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType),
4071  NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
4072  NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
4073  NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
4074  NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
4075  NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType),
4076  NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType),
4077  NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType),
4078  NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts),
4079  NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType),
4080  NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType),
4081  NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType),
4082  NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType),
4083  NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType),
4084  NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
4085  NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
4086  NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts),
4087  NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
4088  NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts),
4089  NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
4090  NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
4091  NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
4092  NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
4093  NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
4094  NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType),
4095  NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
4096  NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
4097  NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType),
4098  NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType),
4099  NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
4100  NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
4101  NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
4102  NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
4103  NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
4104  NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
4105  NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
4106  NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
4107  NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType),
4108  NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType),
4109  NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType),
4110  NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0),
4111  NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0),
4112  NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0),
4113  NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0),
4114  NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0),
4115  NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0),
4116  NEONMAP0(vshl_n_v),
4117  NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
4118  NEONMAP0(vshll_n_v),
4119  NEONMAP0(vshlq_n_v),
4120  NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
4121  NEONMAP0(vshr_n_v),
4122  NEONMAP0(vshrn_n_v),
4123  NEONMAP0(vshrq_n_v),
4124  NEONMAP0(vsubhn_v),
4125  NEONMAP0(vtst_v),
4126  NEONMAP0(vtstq_v),
4127 };
4128 
4129 static const NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = {
4130  NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType),
4131  NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType),
4132  NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType),
4133  NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
4134  NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
4135  NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
4136  NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
4137  NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
4138  NEONMAP1(vaddv_s32, aarch64_neon_saddv, AddRetType | Add1ArgType),
4139  NEONMAP1(vaddv_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType),
4140  NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
4141  NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType),
4142  NEONMAP1(vaddvq_s32, aarch64_neon_saddv, AddRetType | Add1ArgType),
4143  NEONMAP1(vaddvq_s64, aarch64_neon_saddv, AddRetType | Add1ArgType),
4144  NEONMAP1(vaddvq_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType),
4145  NEONMAP1(vaddvq_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
4146  NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
4147  NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
4148  NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
4149  NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
4150  NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
4151  NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
4152  NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
4153  NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
4154  NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
4155  NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
4156  NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
4157  NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
4158  NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
4159  NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
4160  NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
4161  NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
4162  NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
4163  NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
4164  NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
4165  NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
4166  NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
4167  NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
4168  NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
4169  NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
4170  NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
4171  NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
4172  NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
4173  NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
4174  NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
4175  NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
4176  NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
4177  NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
4178  NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
4179  NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
4180  NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
4181  NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
4182  NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
4183  NEONMAP1(vmaxv_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType),
4184  NEONMAP1(vmaxv_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType),
4185  NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
4186  NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
4187  NEONMAP1(vmaxvq_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType),
4188  NEONMAP1(vmaxvq_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType),
4189  NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
4190  NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
4191  NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
4192  NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
4193  NEONMAP1(vminv_s32, aarch64_neon_sminv, AddRetType | Add1ArgType),
4194  NEONMAP1(vminv_u32, aarch64_neon_uminv, AddRetType | Add1ArgType),
4195  NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
4196  NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
4197  NEONMAP1(vminvq_s32, aarch64_neon_sminv, AddRetType | Add1ArgType),
4198  NEONMAP1(vminvq_u32, aarch64_neon_uminv, AddRetType | Add1ArgType),
4199  NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
4200  NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType),
4201  NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType),
4202  NEONMAP1(vpaddd_s64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
4203  NEONMAP1(vpaddd_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
4204  NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
4205  NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
4206  NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
4207  NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
4208  NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
4209  NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
4210  NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
4211  NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
4212  NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
4213  NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType),
4214  NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
4215  NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType),
4216  NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
4217  NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
4218  NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType),
4219  NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType),
4220  NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
4221  NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
4222  NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType),
4223  NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType),
4224  NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors),
4225  NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType),
4226  NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors),
4227  NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
4228  NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType),
4229  NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType),
4230  NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
4231  NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
4232  NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
4233  NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
4234  NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType),
4235  NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
4236  NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
4237  NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
4238  NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType),
4239  NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
4240  NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType),
4241  NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors),
4242  NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType),
4243  NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
4244  NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
4245  NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType),
4246  NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType),
4247  NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
4248  NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
4249  NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType),
4250  NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType),
4251  NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType),
4252  NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType),
4253  NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors),
4254  NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors),
4255  NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors),
4256  NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors),
4257  NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType),
4258  NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors),
4259  NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors),
4260  NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
4261  NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
4262  NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
4263  NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
4264  NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType),
4265  NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType),
4266  NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
4267  NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
4268  NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
4269  NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
4270  NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType),
4271  NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType),
4272  NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType),
4273  NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType),
4274  NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors),
4275  NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors),
4276  NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType),
4277  NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType),
4278  NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType),
4279  NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors),
4280  NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors),
4281  NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors),
4282  NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors),
4283  NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType),
4284  NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors),
4285  NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors),
4286  NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
4287  NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
4288  NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType),
4289  NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType),
4290  NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
4291  NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
4292  NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType),
4293  NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType),
4294  NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType),
4295  NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType),
4296  NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType),
4297  NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType),
4298  NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType),
4299  NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType),
4300  NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType),
4301  NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType),
4302  NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType),
4303  NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType),
4304  NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
4305  NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
4306  NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
4307  NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
4308  NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType),
4309  NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType),
4310  NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType),
4311  NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType),
4312  NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
4313  NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType),
4314  NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
4315  NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType),
4316  NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType),
4317  NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType),
4318  NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
4319  NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType),
4320  NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
4321  NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType),
4322  // FP16 scalar intrinisics go here.
4323  NEONMAP1(vabdh_f16, aarch64_sisd_fabd, Add1ArgType),
4324  NEONMAP1(vcvtah_s32_f16, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
4325  NEONMAP1(vcvtah_s64_f16, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
4326  NEONMAP1(vcvtah_u32_f16, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
4327  NEONMAP1(vcvtah_u64_f16, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
4328  NEONMAP1(vcvth_n_f16_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
4329  NEONMAP1(vcvth_n_f16_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
4330  NEONMAP1(vcvth_n_f16_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
4331  NEONMAP1(vcvth_n_f16_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
4332  NEONMAP1(vcvth_n_s32_f16, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
4333  NEONMAP1(vcvth_n_s64_f16, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
4334  NEONMAP1(vcvth_n_u32_f16, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
4335  NEONMAP1(vcvth_n_u64_f16, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
4336  NEONMAP1(vcvtmh_s32_f16, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
4337  NEONMAP1(vcvtmh_s64_f16, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
4338  NEONMAP1(vcvtmh_u32_f16, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
4339  NEONMAP1(vcvtmh_u64_f16, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
4340  NEONMAP1(vcvtnh_s32_f16, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
4341  NEONMAP1(vcvtnh_s64_f16, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
4342  NEONMAP1(vcvtnh_u32_f16, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
4343  NEONMAP1(vcvtnh_u64_f16, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
4344  NEONMAP1(vcvtph_s32_f16, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
4345  NEONMAP1(vcvtph_s64_f16, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
4346  NEONMAP1(vcvtph_u32_f16, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
4347  NEONMAP1(vcvtph_u64_f16, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
4348  NEONMAP1(vmulxh_f16, aarch64_neon_fmulx, Add1ArgType),
4349  NEONMAP1(vrecpeh_f16, aarch64_neon_frecpe, Add1ArgType),
4350  NEONMAP1(vrecpxh_f16, aarch64_neon_frecpx, Add1ArgType),
4351  NEONMAP1(vrsqrteh_f16, aarch64_neon_frsqrte, Add1ArgType),
4352  NEONMAP1(vrsqrtsh_f16, aarch64_neon_frsqrts, Add1ArgType),
4353 };
4354 
4355 #undef NEONMAP0
4356 #undef NEONMAP1
4357 #undef NEONMAP2
4358 
4360 
4363 
4364 
4365 static const NeonIntrinsicInfo *
4367  unsigned BuiltinID, bool &MapProvenSorted) {
4368 
4369 #ifndef NDEBUG
4370  if (!MapProvenSorted) {
4371  assert(std::is_sorted(std::begin(IntrinsicMap), std::end(IntrinsicMap)));
4372  MapProvenSorted = true;
4373  }
4374 #endif
4375 
4376  const NeonIntrinsicInfo *Builtin =
4377  std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID);
4378 
4379  if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
4380  return Builtin;
4381 
4382  return nullptr;
4383 }
4384 
4385 Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
4386  unsigned Modifier,
4387  llvm::Type *ArgType,
4388  const CallExpr *E) {
4389  int VectorSize = 0;
4390  if (Modifier & Use64BitVectors)
4391  VectorSize = 64;
4392  else if (Modifier & Use128BitVectors)
4393  VectorSize = 128;
4394 
4395  // Return type.
4397  if (Modifier & AddRetType) {
4398  llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
4399  if (Modifier & VectorizeRetType)
4400  Ty = llvm::VectorType::get(
4401  Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
4402 
4403  Tys.push_back(Ty);
4404  }
4405 
4406  // Arguments.
4407  if (Modifier & VectorizeArgTypes) {
4408  int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
4409  ArgType = llvm::VectorType::get(ArgType, Elts);
4410  }
4411 
4412  if (Modifier & (Add1ArgType | Add2ArgTypes))
4413  Tys.push_back(ArgType);
4414 
4415  if (Modifier & Add2ArgTypes)
4416  Tys.push_back(ArgType);
4417 
4418  if (Modifier & InventFloatType)
4419  Tys.push_back(FloatTy);
4420 
4421  return CGM.getIntrinsic(IntrinsicID, Tys);
4422 }
4423 
4425  const NeonIntrinsicInfo &SISDInfo,
4427  const CallExpr *E) {
4428  unsigned BuiltinID = SISDInfo.BuiltinID;
4429  unsigned int Int = SISDInfo.LLVMIntrinsic;
4430  unsigned Modifier = SISDInfo.TypeModifier;
4431  const char *s = SISDInfo.NameHint;
4432 
4433  switch (BuiltinID) {
4434  case NEON::BI__builtin_neon_vcled_s64:
4435  case NEON::BI__builtin_neon_vcled_u64:
4436  case NEON::BI__builtin_neon_vcles_f32:
4437  case NEON::BI__builtin_neon_vcled_f64:
4438  case NEON::BI__builtin_neon_vcltd_s64:
4439  case NEON::BI__builtin_neon_vcltd_u64:
4440  case NEON::BI__builtin_neon_vclts_f32:
4441  case NEON::BI__builtin_neon_vcltd_f64:
4442  case NEON::BI__builtin_neon_vcales_f32:
4443  case NEON::BI__builtin_neon_vcaled_f64:
4444  case NEON::BI__builtin_neon_vcalts_f32:
4445  case NEON::BI__builtin_neon_vcaltd_f64:
4446  // Only one direction of comparisons actually exist, cmle is actually a cmge
4447  // with swapped operands. The table gives us the right intrinsic but we
4448  // still need to do the swap.
4449  std::swap(Ops[0], Ops[1]);
4450  break;
4451  }
4452 
4453  assert(Int && "Generic code assumes a valid intrinsic");
4454 
4455  // Determine the type(s) of this overloaded AArch64 intrinsic.
4456  const Expr *Arg = E->getArg(0);
4457  llvm::Type *ArgTy = CGF.ConvertType(Arg->getType());
4458  Function *F = CGF.LookupNeonLLVMIntrinsic(Int, Modifier, ArgTy, E);
4459 
4460  int j = 0;
4461  ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0);
4462  for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
4463  ai != ae; ++ai, ++j) {
4464  llvm::Type *ArgTy = ai->getType();
4465  if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
4466  ArgTy->getPrimitiveSizeInBits())
4467  continue;
4468 
4469  assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
4470  // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate
4471  // it before inserting.
4472  Ops[j] =
4473  CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType());
4474  Ops[j] =
4475  CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0);
4476  }
4477 
4478  Value *Result = CGF.EmitNeonCall(F, Ops, s);
4479  llvm::Type *ResultType = CGF.ConvertType(E->getType());
4480  if (ResultType->getPrimitiveSizeInBits() <
4481  Result->getType()->getPrimitiveSizeInBits())
4482  return CGF.Builder.CreateExtractElement(Result, C0);
4483 
4484  return CGF.Builder.CreateBitCast(Result, ResultType, s);
4485 }
4486 
4488  unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
4489  const char *NameHint, unsigned Modifier, const CallExpr *E,
4490  SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1,
4491  llvm::Triple::ArchType Arch) {
4492  // Get the last argument, which specifies the vector type.
4493  llvm::APSInt NeonTypeConst;
4494  const Expr *Arg = E->getArg(E->getNumArgs() - 1);
4495  if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext()))
4496  return nullptr;
4497 
4498  // Determine the type of this overloaded NEON intrinsic.
4499  NeonTypeFlags Type(NeonTypeConst.getZExtValue());
4500  bool Usgn = Type.isUnsigned();
4501  bool Quad = Type.isQuad();
4502  const bool HasLegalHalfType = getTarget().hasLegalHalfType();
4503 
4504  llvm::VectorType *VTy = GetNeonType(this, Type, HasLegalHalfType);
4505  llvm::Type *Ty = VTy;
4506  if (!Ty)
4507  return nullptr;
4508 
4509  auto getAlignmentValue32 = [&](Address addr) -> Value* {
4510  return Builder.getInt32(addr.getAlignment().getQuantity());
4511  };
4512 
4513  unsigned Int = LLVMIntrinsic;
4514  if ((Modifier & UnsignedAlts) && !Usgn)
4515  Int = AltLLVMIntrinsic;
4516 
4517  switch (BuiltinID) {
4518  default: break;
4519  case NEON::BI__builtin_neon_vabs_v:
4520  case NEON::BI__builtin_neon_vabsq_v:
4521  if (VTy->getElementType()->isFloatingPointTy())
4522  return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs");
4523  return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs");
4524  case NEON::BI__builtin_neon_vaddhn_v: {
4525  llvm::VectorType *SrcTy =
4526  llvm::VectorType::getExtendedElementVectorType(VTy);
4527 
4528  // %sum = add <4 x i32> %lhs, %rhs
4529  Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
4530  Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
4531  Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn");
4532 
4533  // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
4534  Constant *ShiftAmt =
4535  ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
4536  Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn");
4537 
4538  // %res = trunc <4 x i32> %high to <4 x i16>
4539  return Builder.CreateTrunc(Ops[0], VTy, "vaddhn");
4540  }
4541  case NEON::BI__builtin_neon_vcale_v:
4542  case NEON::BI__builtin_neon_vcaleq_v:
4543  case NEON::BI__builtin_neon_vcalt_v:
4544  case NEON::BI__builtin_neon_vcaltq_v:
4545  std::swap(Ops[0], Ops[1]);
4546  LLVM_FALLTHROUGH;
4547  case NEON::BI__builtin_neon_vcage_v:
4548  case NEON::BI__builtin_neon_vcageq_v:
4549  case NEON::BI__builtin_neon_vcagt_v:
4550  case NEON::BI__builtin_neon_vcagtq_v: {
4551  llvm::Type *Ty;
4552  switch (VTy->getScalarSizeInBits()) {
4553  default: llvm_unreachable("unexpected type");
4554  case 32:
4555  Ty = FloatTy;
4556  break;
4557  case 64:
4558  Ty = DoubleTy;
4559  break;
4560  case 16:
4561  Ty = HalfTy;
4562  break;
4563  }
4564  llvm::Type *VecFlt = llvm::VectorType::get(Ty, VTy->getNumElements());
4565  llvm::Type *Tys[] = { VTy, VecFlt };
4566  Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4567  return EmitNeonCall(F, Ops, NameHint);
4568  }
4569  case NEON::BI__builtin_neon_vceqz_v:
4570  case NEON::BI__builtin_neon_vceqzq_v:
4571  return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ,
4572  ICmpInst::ICMP_EQ, "vceqz");
4573  case NEON::BI__builtin_neon_vcgez_v:
4574  case NEON::BI__builtin_neon_vcgezq_v:
4575  return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE,
4576  ICmpInst::ICMP_SGE, "vcgez");
4577  case NEON::BI__builtin_neon_vclez_v:
4578  case NEON::BI__builtin_neon_vclezq_v:
4579  return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE,
4580  ICmpInst::ICMP_SLE, "vclez");
4581  case NEON::BI__builtin_neon_vcgtz_v:
4582  case NEON::BI__builtin_neon_vcgtzq_v:
4583  return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT,
4584  ICmpInst::ICMP_SGT, "vcgtz");
4585  case NEON::BI__builtin_neon_vcltz_v:
4586  case NEON::BI__builtin_neon_vcltzq_v:
4587  return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT,
4588  ICmpInst::ICMP_SLT, "vcltz");
4589  case NEON::BI__builtin_neon_vclz_v:
4590  case NEON::BI__builtin_neon_vclzq_v:
4591  // We generate target-independent intrinsic, which needs a second argument
4592  // for whether or not clz of zero is undefined; on ARM it isn't.
4593  Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
4594  break;
4595  case NEON::BI__builtin_neon_vcvt_f32_v:
4596  case NEON::BI__builtin_neon_vcvtq_f32_v:
4597  Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4598  Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad),
4599  HasLegalHalfType);
4600  return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
4601  : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
4602  case NEON::BI__builtin_neon_vcvt_f16_v:
4603  case NEON::BI__builtin_neon_vcvtq_f16_v:
4604  Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4605  Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float16, false, Quad),
4606  HasLegalHalfType);
4607  return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
4608  : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
4609  case NEON::BI__builtin_neon_vcvt_n_f16_v:
4610  case NEON::BI__builtin_neon_vcvt_n_f32_v:
4611  case NEON::BI__builtin_neon_vcvt_n_f64_v:
4612  case NEON::BI__builtin_neon_vcvtq_n_f16_v:
4613  case NEON::BI__builtin_neon_vcvtq_n_f32_v:
4614  case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
4615  llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty };
4616  Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
4617  Function *F = CGM.getIntrinsic(Int, Tys);
4618  return EmitNeonCall(F, Ops, "vcvt_n");
4619  }
4620  case NEON::BI__builtin_neon_vcvt_n_s16_v:
4621  case NEON::BI__builtin_neon_vcvt_n_s32_v:
4622  case NEON::BI__builtin_neon_vcvt_n_u16_v:
4623  case NEON::BI__builtin_neon_vcvt_n_u32_v:
4624  case NEON::BI__builtin_neon_vcvt_n_s64_v:
4625  case NEON::BI__builtin_neon_vcvt_n_u64_v:
4626  case NEON::BI__builtin_neon_vcvtq_n_s16_v:
4627  case NEON::BI__builtin_neon_vcvtq_n_s32_v:
4628  case NEON::BI__builtin_neon_vcvtq_n_u16_v:
4629  case NEON::BI__builtin_neon_vcvtq_n_u32_v:
4630  case NEON::BI__builtin_neon_vcvtq_n_s64_v:
4631  case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
4632  llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
4633  Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4634  return EmitNeonCall(F, Ops, "vcvt_n");
4635  }
4636  case NEON::BI__builtin_neon_vcvt_s32_v:
4637  case NEON::BI__builtin_neon_vcvt_u32_v:
4638  case NEON::BI__builtin_neon_vcvt_s64_v:
4639  case NEON::BI__builtin_neon_vcvt_u64_v:
4640  case NEON::BI__builtin_neon_vcvt_s16_v:
4641  case NEON::BI__builtin_neon_vcvt_u16_v:
4642  case NEON::BI__builtin_neon_vcvtq_s32_v:
4643  case NEON::BI__builtin_neon_vcvtq_u32_v:
4644  case NEON::BI__builtin_neon_vcvtq_s64_v:
4645  case NEON::BI__builtin_neon_vcvtq_u64_v:
4646  case NEON::BI__builtin_neon_vcvtq_s16_v:
4647  case NEON::BI__builtin_neon_vcvtq_u16_v: {
4648  Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
4649  return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
4650  : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
4651  }
4652  case NEON::BI__builtin_neon_vcvta_s16_v:
4653  case NEON::BI__builtin_neon_vcvta_s32_v:
4654  case NEON::BI__builtin_neon_vcvta_s64_v:
4655  case NEON::BI__builtin_neon_vcvta_u32_v:
4656  case NEON::BI__builtin_neon_vcvta_u64_v:
4657  case NEON::BI__builtin_neon_vcvtaq_s16_v:
4658  case NEON::BI__builtin_neon_vcvtaq_s32_v:
4659  case NEON::BI__builtin_neon_vcvtaq_s64_v:
4660  case NEON::BI__builtin_neon_vcvtaq_u16_v:
4661  case NEON::BI__builtin_neon_vcvtaq_u32_v:
4662  case NEON::BI__builtin_neon_vcvtaq_u64_v:
4663  case NEON::BI__builtin_neon_vcvtn_s16_v:
4664  case NEON::BI__builtin_neon_vcvtn_s32_v:
4665  case NEON::BI__builtin_neon_vcvtn_s64_v:
4666  case NEON::BI__builtin_neon_vcvtn_u16_v:
4667  case NEON::BI__builtin_neon_vcvtn_u32_v:
4668  case NEON::BI__builtin_neon_vcvtn_u64_v:
4669  case NEON::BI__builtin_neon_vcvtnq_s16_v:
4670  case NEON::BI__builtin_neon_vcvtnq_s32_v:
4671  case NEON::BI__builtin_neon_vcvtnq_s64_v:
4672  case NEON::BI__builtin_neon_vcvtnq_u16_v:
4673  case NEON::BI__builtin_neon_vcvtnq_u32_v:
4674  case NEON::BI__builtin_neon_vcvtnq_u64_v:
4675  case NEON::BI__builtin_neon_vcvtp_s16_v:
4676  case NEON::BI__builtin_neon_vcvtp_s32_v:
4677  case NEON::BI__builtin_neon_vcvtp_s64_v:
4678  case NEON::BI__builtin_neon_vcvtp_u16_v:
4679  case NEON::BI__builtin_neon_vcvtp_u32_v:
4680  case NEON::BI__builtin_neon_vcvtp_u64_v:
4681  case NEON::BI__builtin_neon_vcvtpq_s16_v:
4682  case NEON::BI__builtin_neon_vcvtpq_s32_v:
4683  case NEON::BI__builtin_neon_vcvtpq_s64_v:
4684  case NEON::BI__builtin_neon_vcvtpq_u16_v:
4685  case NEON::BI__builtin_neon_vcvtpq_u32_v:
4686  case NEON::BI__builtin_neon_vcvtpq_u64_v:
4687  case NEON::BI__builtin_neon_vcvtm_s16_v:
4688  case NEON::BI__builtin_neon_vcvtm_s32_v:
4689  case NEON::BI__builtin_neon_vcvtm_s64_v:
4690  case NEON::BI__builtin_neon_vcvtm_u16_v:
4691  case NEON::BI__builtin_neon_vcvtm_u32_v:
4692  case NEON::BI__builtin_neon_vcvtm_u64_v:
4693  case NEON::BI__builtin_neon_vcvtmq_s16_v:
4694  case NEON::BI__builtin_neon_vcvtmq_s32_v:
4695  case NEON::BI__builtin_neon_vcvtmq_s64_v:
4696  case NEON::BI__builtin_neon_vcvtmq_u16_v:
4697  case NEON::BI__builtin_neon_vcvtmq_u32_v:
4698  case NEON::BI__builtin_neon_vcvtmq_u64_v: {
4699  llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
4700  return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
4701  }
4702  case NEON::BI__builtin_neon_vext_v:
4703  case NEON::BI__builtin_neon_vextq_v: {
4704  int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
4705  SmallVector<uint32_t, 16> Indices;
4706  for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
4707  Indices.push_back(i+CV);
4708 
4709  Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4710  Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4711  return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext");
4712  }
4713  case NEON::BI__builtin_neon_vfma_v:
4714  case NEON::BI__builtin_neon_vfmaq_v: {
4715  Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
4716  Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4717  Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4718  Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4719 
4720  // NEON intrinsic puts accumulator first, unlike the LLVM fma.
4721  return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
4722  }
4723  case NEON::BI__builtin_neon_vld1_v:
4724  case NEON::BI__builtin_neon_vld1q_v: {
4725  llvm::Type *Tys[] = {Ty, Int8PtrTy};
4726  Ops.push_back(getAlignmentValue32(PtrOp0));
4727  return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1");
4728  }
4729  case NEON::BI__builtin_neon_vld2_v:
4730  case NEON::BI__builtin_neon_vld2q_v:
4731  case NEON::BI__builtin_neon_vld3_v:
4732  case NEON::BI__builtin_neon_vld3q_v:
4733  case NEON::BI__builtin_neon_vld4_v:
4734  case NEON::BI__builtin_neon_vld4q_v: {
4735  llvm::Type *Tys[] = {Ty, Int8PtrTy};
4736  Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4737  Value *Align = getAlignmentValue32(PtrOp1);
4738  Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
4739  Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4740  Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4741  return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
4742  }
4743  case NEON::BI__builtin_neon_vld1_dup_v:
4744  case NEON::BI__builtin_neon_vld1q_dup_v: {
4745  Value *V = UndefValue::get(Ty);
4746  Ty = llvm::PointerType::getUnqual(VTy->getElementType());
4747  PtrOp0 = Builder.CreateBitCast(PtrOp0, Ty);
4748  LoadInst *Ld = Builder.CreateLoad(PtrOp0);
4749  llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
4750  Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
4751  return EmitNeonSplat(Ops[0], CI);
4752  }
4753  case NEON::BI__builtin_neon_vld2_lane_v:
4754  case NEON::BI__builtin_neon_vld2q_lane_v:
4755  case NEON::BI__builtin_neon_vld3_lane_v:
4756  case NEON::BI__builtin_neon_vld3q_lane_v:
4757  case NEON::BI__builtin_neon_vld4_lane_v:
4758  case NEON::BI__builtin_neon_vld4q_lane_v: {
4759  llvm::Type *Tys[] = {Ty, Int8PtrTy};
4760  Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4761  for (unsigned I = 2; I < Ops.size() - 1; ++I)
4762  Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
4763  Ops.push_back(getAlignmentValue32(PtrOp1));
4764  Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint);
4765  Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4766  Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4767  return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
4768  }
4769  case NEON::BI__builtin_neon_vmovl_v: {
4770  llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
4771  Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
4772  if (Usgn)
4773  return Builder.CreateZExt(Ops[0], Ty, "vmovl");
4774  return Builder.CreateSExt(Ops[0], Ty, "vmovl");
4775  }
4776  case NEON::BI__builtin_neon_vmovn_v: {
4777  llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
4778  Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
4779  return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
4780  }
4781  case NEON::BI__builtin_neon_vmull_v:
4782  // FIXME: the integer vmull operations could be emitted in terms of pure
4783  // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of
4784  // hoisting the exts outside loops. Until global ISel comes along that can
4785  // see through such movement this leads to bad CodeGen. So we need an
4786  // intrinsic for now.
4787  Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
4788  Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
4789  return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
4790  case NEON::BI__builtin_neon_vpadal_v:
4791  case NEON::BI__builtin_neon_vpadalq_v: {
4792  // The source operand type has twice as many elements of half the size.
4793  unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
4794  llvm::Type *EltTy =
4795  llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
4796  llvm::Type *NarrowTy =
4797  llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
4798  llvm::Type *Tys[2] = { Ty, NarrowTy };
4799  return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
4800  }
4801  case NEON::BI__builtin_neon_vpaddl_v:
4802  case NEON::BI__builtin_neon_vpaddlq_v: {
4803  // The source operand type has twice as many elements of half the size.
4804  unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
4805  llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
4806  llvm::Type *NarrowTy =
4807  llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
4808  llvm::Type *Tys[2] = { Ty, NarrowTy };
4809  return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl");
4810  }
4811  case NEON::BI__builtin_neon_vqdmlal_v:
4812  case NEON::BI__builtin_neon_vqdmlsl_v: {
4813  SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
4814  Ops[1] =
4815  EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps, "vqdmlal");
4816  Ops.resize(2);
4817  return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint);
4818  }
4819  case NEON::BI__builtin_neon_vqshl_n_v:
4820  case NEON::BI__builtin_neon_vqshlq_n_v:
4821  return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n",
4822  1, false);
4823  case NEON::BI__builtin_neon_vqshlu_n_v:
4824  case NEON::BI__builtin_neon_vqshluq_n_v:
4825  return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n",
4826  1, false);
4827  case NEON::BI__builtin_neon_vrecpe_v:
4828  case NEON::BI__builtin_neon_vrecpeq_v:
4829  case NEON::BI__builtin_neon_vrsqrte_v:
4830  case NEON::BI__builtin_neon_vrsqrteq_v:
4831  Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
4832  return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
4833 
4834  case NEON::BI__builtin_neon_vrshr_n_v:
4835  case NEON::BI__builtin_neon_vrshrq_n_v:
4836  return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n",
4837  1, true);
4838  case NEON::BI__builtin_neon_vshl_n_v:
4839  case NEON::BI__builtin_neon_vshlq_n_v:
4840  Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false);
4841  return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
4842  "vshl_n");
4843  case NEON::BI__builtin_neon_vshll_n_v: {
4844  llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy);
4845  Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
4846  if (Usgn)
4847  Ops[0] = Builder.CreateZExt(Ops[0], VTy);
4848  else
4849  Ops[0] = Builder.CreateSExt(Ops[0], VTy);
4850  Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false);
4851  return Builder.CreateShl(Ops[0], Ops[1], "vshll_n");
4852  }
4853  case NEON::BI__builtin_neon_vshrn_n_v: {
4854  llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy);
4855  Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
4856  Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false);
4857  if (Usgn)
4858  Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
4859  else
4860  Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
4861  return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n");
4862  }
4863  case NEON::BI__builtin_neon_vshr_n_v:
4864  case NEON::BI__builtin_neon_vshrq_n_v:
4865  return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n");
4866  case NEON::BI__builtin_neon_vst1_v:
4867  case NEON::BI__builtin_neon_vst1q_v:
4868  case NEON::BI__builtin_neon_vst2_v:
4869  case NEON::BI__builtin_neon_vst2q_v:
4870  case NEON::BI__builtin_neon_vst3_v:
4871  case NEON::BI__builtin_neon_vst3q_v:
4872  case NEON::BI__builtin_neon_vst4_v:
4873  case NEON::BI__builtin_neon_vst4q_v:
4874  case NEON::BI__builtin_neon_vst2_lane_v:
4875  case NEON::BI__builtin_neon_vst2q_lane_v:
4876  case NEON::BI__builtin_neon_vst3_lane_v:
4877  case NEON::BI__builtin_neon_vst3q_lane_v:
4878  case NEON::BI__builtin_neon_vst4_lane_v:
4879  case NEON::BI__builtin_neon_vst4q_lane_v: {
4880  llvm::Type *Tys[] = {Int8PtrTy, Ty};
4881  Ops.push_back(getAlignmentValue32(PtrOp0));
4882  return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "");
4883  }
4884  case NEON::BI__builtin_neon_vsubhn_v: {
4885  llvm::VectorType *SrcTy =
4886  llvm::VectorType::getExtendedElementVectorType(VTy);
4887 
4888  // %sum = add <4 x i32> %lhs, %rhs
4889  Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
4890  Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
4891  Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn");
4892 
4893  // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
4894  Constant *ShiftAmt =
4895  ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
4896  Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn");
4897 
4898  // %res = trunc <4 x i32> %high to <4 x i16>
4899  return Builder.CreateTrunc(Ops[0], VTy, "vsubhn");
4900  }
4901  case NEON::BI__builtin_neon_vtrn_v:
4902  case NEON::BI__builtin_neon_vtrnq_v: {
4903  Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
4904  Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4905  Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4906  Value *SV = nullptr;
4907 
4908  for (unsigned vi = 0; vi != 2; ++vi) {
4909  SmallVector<uint32_t, 16> Indices;
4910  for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
4911  Indices.push_back(i+vi);
4912  Indices.push_back(i+e+vi);
4913  }
4914  Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
4915  SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
4916  SV = Builder.CreateDefaultAlignedStore(SV, Addr);
4917  }
4918  return SV;
4919  }
4920  case NEON::BI__builtin_neon_vtst_v:
4921  case NEON::BI__builtin_neon_vtstq_v: {
4922  Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4923  Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4924  Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
4925  Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
4926  ConstantAggregateZero::get(Ty));
4927  return Builder.CreateSExt(Ops[0], Ty, "vtst");
4928  }
4929  case NEON::BI__builtin_neon_vuzp_v:
4930  case NEON::BI__builtin_neon_vuzpq_v: {
4931  Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
4932  Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4933  Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4934  Value *SV = nullptr;
4935 
4936  for (unsigned vi = 0; vi != 2; ++vi) {
4937  SmallVector<uint32_t, 16> Indices;
4938  for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
4939  Indices.push_back(2*i+vi);
4940 
4941  Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
4942  SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
4943  SV = Builder.CreateDefaultAlignedStore(SV, Addr);
4944  }
4945  return SV;
4946  }
4947  case NEON::BI__builtin_neon_vzip_v:
4948  case NEON::BI__builtin_neon_vzipq_v: {
4949  Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
4950  Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4951  Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4952  Value *SV = nullptr;
4953 
4954  for (unsigned vi = 0; vi != 2; ++vi) {
4955  SmallVector<uint32_t, 16> Indices;
4956  for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
4957  Indices.push_back((i + vi*e) >> 1);
4958  Indices.push_back(((i + vi*e) >> 1)+e);
4959  }
4960  Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
4961  SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
4962  SV = Builder.CreateDefaultAlignedStore(SV, Addr);
4963  }
4964  return SV;
4965  }
4966  case NEON::BI__builtin_neon_vdot_v:
4967  case NEON::BI__builtin_neon_vdotq_v: {
4968  llvm::Type *InputTy =
4969  llvm::VectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
4970  llvm::Type *Tys[2] = { Ty, InputTy };
4971  Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
4972  return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vdot");
4973  }
4974  }
4975 
4976  assert(Int && "Expected valid intrinsic number");
4977 
4978  // Determine the type(s) of this overloaded AArch64 intrinsic.
4979  Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E);
4980 
4981  Value *Result = EmitNeonCall(F, Ops, NameHint);
4982  llvm::Type *ResultType = ConvertType(E->getType());
4983  // AArch64 intrinsic one-element vector type cast to
4984  // scalar type expected by the builtin
4985  return Builder.CreateBitCast(Result, ResultType, NameHint);
4986 }
4987 
4989  Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp,
4990  const CmpInst::Predicate Ip, const Twine &Name) {
4991  llvm::Type *OTy = Op->getType();
4992 
4993  // FIXME: this is utterly horrific. We should not be looking at previous
4994  // codegen context to find out what needs doing. Unfortunately TableGen
4995  // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32
4996  // (etc).
4997  if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
4998  OTy = BI->getOperand(0)->getType();
4999 
5000  Op = Builder.CreateBitCast(Op, OTy);
5001  if (OTy->getScalarType()->isFloatingPointTy()) {
5002  Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
5003  } else {
5004  Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
5005  }
5006  return Builder.CreateSExt(Op, Ty, Name);
5007 }
5008 
5010  Value *ExtOp, Value *IndexOp,
5011  llvm::Type *ResTy, unsigned IntID,
5012  const char *Name) {
5013  SmallVector<Value *, 2> TblOps;
5014  if (ExtOp)
5015  TblOps.push_back(ExtOp);
5016 
5017  // Build a vector containing sequential number like (0, 1, 2, ..., 15)
5018  SmallVector<uint32_t, 16> Indices;
5019  llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType());
5020  for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
5021  Indices.push_back(2*i);
5022  Indices.push_back(2*i+1);
5023  }
5024 
5025  int PairPos = 0, End = Ops.size() - 1;
5026  while (PairPos < End) {
5027  TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
5028  Ops[PairPos+1], Indices,
5029  Name));
5030  PairPos += 2;
5031  }
5032 
5033  // If there's an odd number of 64-bit lookup table, fill the high 64-bit
5034  // of the 128-bit lookup table with zero.
5035  if (PairPos == End) {
5036  Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
5037  TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
5038  ZeroTbl, Indices, Name));
5039  }
5040 
5041  Function *TblF;
5042  TblOps.push_back(IndexOp);
5043  TblF = CGF.CGM.getIntrinsic(IntID, ResTy);
5044 
5045  return CGF.EmitNeonCall(TblF, TblOps, Name);
5046 }
5047 
5048 Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) {
5049  unsigned Value;
5050  switch (BuiltinID) {
5051  default:
5052  return nullptr;
5053  case ARM::BI__builtin_arm_nop:
5054  Value = 0;
5055  break;
5056  case ARM::BI__builtin_arm_yield:
5057  case ARM::BI__yield:
5058  Value = 1;
5059  break;
5060  case ARM::BI__builtin_arm_wfe:
5061  case ARM::BI__wfe:
5062  Value = 2;
5063  break;
5064  case ARM::BI__builtin_arm_wfi:
5065  case ARM::BI__wfi:
5066  Value = 3;
5067  break;
5068  case ARM::BI__builtin_arm_sev:
5069  case ARM::BI__sev:
5070  Value = 4;
5071  break;
5072  case ARM::BI__builtin_arm_sevl:
5073  case ARM::BI__sevl:
5074  Value = 5;
5075  break;
5076  }
5077 
5078  return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
5079  llvm::ConstantInt::get(Int32Ty, Value));
5080 }
5081 
5082 // Generates the IR for the read/write special register builtin,
5083 // ValueType is the type of the value that is to be written or read,
5084 // RegisterType is the type of the register being written to or read from.
5086  const CallExpr *E,
5087  llvm::Type *RegisterType,
5088  llvm::Type *ValueType,
5089  bool IsRead,
5090  StringRef SysReg = "") {
5091  // write and register intrinsics only support 32 and 64 bit operations.
5092  assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64))
5093  && "Unsupported size for register.");
5094 
5095  CodeGen::CGBuilderTy &Builder = CGF.Builder;
5096  CodeGen::CodeGenModule &CGM = CGF.CGM;
5097  LLVMContext &Context = CGM.getLLVMContext();
5098 
5099  if (SysReg.empty()) {
5100  const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts();
5101  SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
5102  }
5103 
5104  llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
5105  llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
5106  llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
5107 
5108  llvm::Type *Types[] = { RegisterType };
5109 
5110  bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
5111  assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
5112  && "Can't fit 64-bit value in 32-bit register");
5113 
5114  if (IsRead) {
5115  llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
5116  llvm::Value *Call = Builder.CreateCall(F, Metadata);
5117 
5118  if (MixedTypes)
5119  // Read into 64 bit register and then truncate result to 32 bit.
5120  return Builder.CreateTrunc(Call, ValueType);
5121 
5122  if (ValueType->isPointerTy())
5123  // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*).
5124  return Builder.CreateIntToPtr(Call, ValueType);
5125 
5126  return Call;
5127  }
5128 
5129  llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
5130  llvm::Value *ArgValue = CGF.EmitScalarExpr(E->getArg(1));
5131  if (MixedTypes) {
5132  // Extend 32 bit write value to 64 bit to pass to write.
5133  ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
5134  return Builder.CreateCall(F, { Metadata, ArgValue });
5135  }
5136 
5137  if (ValueType->isPointerTy()) {
5138  // Have VoidPtrTy ArgValue but want to return an i32/i64.
5139  ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
5140  return Builder.CreateCall(F, { Metadata, ArgValue });
5141  }
5142 
5143  return Builder.CreateCall(F, { Metadata, ArgValue });
5144 }
5145 
5146 /// Return true if BuiltinID is an overloaded Neon intrinsic with an extra
5147 /// argument that specifies the vector type.
5148 static bool HasExtraNeonArgument(unsigned BuiltinID) {
5149  switch (BuiltinID) {
5150  default: break;
5151  case NEON::BI__builtin_neon_vget_lane_i8:
5152  case NEON::BI__builtin_neon_vget_lane_i16:
5153  case NEON::BI__builtin_neon_vget_lane_i32:
5154  case NEON::BI__builtin_neon_vget_lane_i64:
5155  case NEON::BI__builtin_neon_vget_lane_f32:
5156  case NEON::BI__builtin_neon_vgetq_lane_i8:
5157  case NEON::BI__builtin_neon_vgetq_lane_i16:
5158  case NEON::BI__builtin_neon_vgetq_lane_i32:
5159  case NEON::BI__builtin_neon_vgetq_lane_i64:
5160  case NEON::BI__builtin_neon_vgetq_lane_f32:
5161  case NEON::BI__builtin_neon_vset_lane_i8:
5162  case NEON::BI__builtin_neon_vset_lane_i16:
5163  case NEON::BI__builtin_neon_vset_lane_i32:
5164  case NEON::BI__builtin_neon_vset_lane_i64:
5165  case NEON::BI__builtin_neon_vset_lane_f32:
5166  case NEON::BI__builtin_neon_vsetq_lane_i8:
5167  case NEON::BI__builtin_neon_vsetq_lane_i16:
5168  case NEON::BI__builtin_neon_vsetq_lane_i32:
5169  case NEON::BI__builtin_neon_vsetq_lane_i64:
5170  case NEON::BI__builtin_neon_vsetq_lane_f32:
5171  case NEON::BI__builtin_neon_vsha1h_u32:
5172  case NEON::BI__builtin_neon_vsha1cq_u32:
5173  case NEON::BI__builtin_neon_vsha1pq_u32:
5174  case NEON::BI__builtin_neon_vsha1mq_u32:
5175  case clang::ARM::BI_MoveToCoprocessor:
5176  case clang::ARM::BI_MoveToCoprocessor2:
5177  return false;
5178  }
5179  return true;
5180 }
5181 
5183  const CallExpr *E,
5184  llvm::Triple::ArchType Arch) {
5185  if (auto Hint = GetValueForARMHint(BuiltinID))
5186  return Hint;
5187 
5188  if (BuiltinID == ARM::BI__emit) {
5189  bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
5190  llvm::FunctionType *FTy =
5191  llvm::FunctionType::get(VoidTy, /*Variadic=*/false);
5192 
5193  APSInt Value;
5194  if (!E->getArg(0)->EvaluateAsInt(Value, CGM.getContext()))
5195  llvm_unreachable("Sema will ensure that the parameter is constant");
5196 
5197  uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
5198 
5199  llvm::InlineAsm *Emit =
5200  IsThumb ? InlineAsm::get(FTy, ".inst.n 0x" + utohexstr(ZExtValue), "",
5201  /*SideEffects=*/true)
5202  : InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "",
5203  /*SideEffects=*/true);
5204 
5205  return Builder.CreateCall(Emit);
5206  }
5207 
5208  if (BuiltinID == ARM::BI__builtin_arm_dbg) {
5209  Value *Option = EmitScalarExpr(E->getArg(0));
5210  return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
5211  }
5212 
5213  if (BuiltinID == ARM::BI__builtin_arm_prefetch) {
5214  Value *Address = EmitScalarExpr(E->getArg(0));
5215  Value *RW = EmitScalarExpr(E->getArg(1));
5216  Value *IsData = EmitScalarExpr(E->getArg(2));
5217 
5218  // Locality is not supported on ARM target
5219  Value *Locality = llvm::ConstantInt::get(Int32Ty, 3);
5220 
5221  Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
5222  return Builder.CreateCall(F, {Address, RW, Locality, IsData});
5223  }
5224 
5225  if (BuiltinID == ARM::BI__builtin_arm_rbit) {
5226  llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
5227  return Builder.CreateCall(
5228  CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
5229  }
5230 
5231  if (BuiltinID == ARM::BI__clear_cache) {
5232  assert(E->