clang 20.0.0git
CGBuiltin.cpp
Go to the documentation of this file.
1//===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfo.h"
14#include "CGCUDARuntime.h"
15#include "CGCXXABI.h"
16#include "CGHLSLRuntime.h"
17#include "CGObjCRuntime.h"
18#include "CGOpenCLRuntime.h"
19#include "CGRecordLayout.h"
20#include "CGValue.h"
21#include "CodeGenFunction.h"
22#include "CodeGenModule.h"
23#include "ConstantEmitter.h"
24#include "PatternInit.h"
25#include "TargetInfo.h"
27#include "clang/AST/Attr.h"
28#include "clang/AST/Decl.h"
29#include "clang/AST/Expr.h"
30#include "clang/AST/OSLog.h"
32#include "clang/AST/Type.h"
38#include "llvm/ADT/APFloat.h"
39#include "llvm/ADT/APInt.h"
40#include "llvm/ADT/FloatingPointMode.h"
41#include "llvm/ADT/SmallPtrSet.h"
42#include "llvm/ADT/StringExtras.h"
43#include "llvm/Analysis/ValueTracking.h"
44#include "llvm/IR/DataLayout.h"
45#include "llvm/IR/InlineAsm.h"
46#include "llvm/IR/Intrinsics.h"
47#include "llvm/IR/IntrinsicsAArch64.h"
48#include "llvm/IR/IntrinsicsAMDGPU.h"
49#include "llvm/IR/IntrinsicsARM.h"
50#include "llvm/IR/IntrinsicsBPF.h"
51#include "llvm/IR/IntrinsicsDirectX.h"
52#include "llvm/IR/IntrinsicsHexagon.h"
53#include "llvm/IR/IntrinsicsNVPTX.h"
54#include "llvm/IR/IntrinsicsPowerPC.h"
55#include "llvm/IR/IntrinsicsR600.h"
56#include "llvm/IR/IntrinsicsRISCV.h"
57#include "llvm/IR/IntrinsicsS390.h"
58#include "llvm/IR/IntrinsicsWebAssembly.h"
59#include "llvm/IR/IntrinsicsX86.h"
60#include "llvm/IR/MDBuilder.h"
61#include "llvm/IR/MatrixBuilder.h"
62#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
63#include "llvm/Support/AMDGPUAddrSpace.h"
64#include "llvm/Support/ConvertUTF.h"
65#include "llvm/Support/MathExtras.h"
66#include "llvm/Support/ScopedPrinter.h"
67#include "llvm/TargetParser/AArch64TargetParser.h"
68#include "llvm/TargetParser/RISCVISAInfo.h"
69#include "llvm/TargetParser/RISCVTargetParser.h"
70#include "llvm/TargetParser/X86TargetParser.h"
71#include <optional>
72#include <utility>
73
74using namespace clang;
75using namespace CodeGen;
76using namespace llvm;
77
78static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size,
79 Align AlignmentInBytes) {
80 ConstantInt *Byte;
81 switch (CGF.getLangOpts().getTrivialAutoVarInit()) {
82 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
83 // Nothing to initialize.
84 return;
85 case LangOptions::TrivialAutoVarInitKind::Zero:
86 Byte = CGF.Builder.getInt8(0x00);
87 break;
88 case LangOptions::TrivialAutoVarInitKind::Pattern: {
89 llvm::Type *Int8 = llvm::IntegerType::getInt8Ty(CGF.CGM.getLLVMContext());
90 Byte = llvm::dyn_cast<llvm::ConstantInt>(
91 initializationPatternFor(CGF.CGM, Int8));
92 break;
93 }
94 }
95 if (CGF.CGM.stopAutoInit())
96 return;
97 auto *I = CGF.Builder.CreateMemSet(AI, Byte, Size, AlignmentInBytes);
98 I->addAnnotationMetadata("auto-init");
99}
100
102 Value *Op0 = CGF->EmitScalarExpr(E->getArg(0));
103
104 Constant *FZeroConst = ConstantFP::getZero(CGF->FloatTy);
105 Value *CMP;
106 Value *LastInstr;
107
108 if (const auto *VecTy = E->getArg(0)->getType()->getAs<clang::VectorType>()) {
109 FZeroConst = ConstantVector::getSplat(
110 ElementCount::getFixed(VecTy->getNumElements()), FZeroConst);
111 auto *FCompInst = CGF->Builder.CreateFCmpOLT(Op0, FZeroConst);
112 CMP = CGF->Builder.CreateIntrinsic(
113 CGF->Builder.getInt1Ty(), CGF->CGM.getHLSLRuntime().getAnyIntrinsic(),
114 {FCompInst}, nullptr);
115 } else
116 CMP = CGF->Builder.CreateFCmpOLT(Op0, FZeroConst);
117
118 if (CGF->CGM.getTarget().getTriple().isDXIL())
119 LastInstr = CGF->Builder.CreateIntrinsic(
120 CGF->VoidTy, llvm::Intrinsic::dx_discard, {CMP}, nullptr);
121 else if (CGF->CGM.getTarget().getTriple().isSPIRV()) {
122 BasicBlock *LT0 = CGF->createBasicBlock("lt0", CGF->CurFn);
123 BasicBlock *End = CGF->createBasicBlock("end", CGF->CurFn);
124
125 CGF->Builder.CreateCondBr(CMP, LT0, End);
126
127 CGF->Builder.SetInsertPoint(LT0);
128
129 CGF->Builder.CreateIntrinsic(CGF->VoidTy, llvm::Intrinsic::spv_discard, {},
130 nullptr);
131
132 LastInstr = CGF->Builder.CreateBr(End);
133
134 CGF->Builder.SetInsertPoint(End);
135 } else {
136 llvm_unreachable("Backend Codegen not supported.");
137 }
138
139 return LastInstr;
140}
141
143 Value *Op0 = CGF->EmitScalarExpr(E->getArg(0));
144 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(E->getArg(1));
145 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(E->getArg(2));
146
147 CallArgList Args;
148 LValue Op1TmpLValue =
149 CGF->EmitHLSLOutArgExpr(OutArg1, Args, OutArg1->getType());
150 LValue Op2TmpLValue =
151 CGF->EmitHLSLOutArgExpr(OutArg2, Args, OutArg2->getType());
152
154 Args.reverseWritebacks();
155
156 Value *LowBits = nullptr;
157 Value *HighBits = nullptr;
158
159 if (CGF->CGM.getTarget().getTriple().isDXIL()) {
160
161 llvm::Type *RetElementTy = CGF->Int32Ty;
162 if (auto *Op0VecTy = E->getArg(0)->getType()->getAs<clang::VectorType>())
163 RetElementTy = llvm::VectorType::get(
164 CGF->Int32Ty, ElementCount::getFixed(Op0VecTy->getNumElements()));
165 auto *RetTy = llvm::StructType::get(RetElementTy, RetElementTy);
166
167 CallInst *CI = CGF->Builder.CreateIntrinsic(
168 RetTy, Intrinsic::dx_splitdouble, {Op0}, nullptr, "hlsl.splitdouble");
169
170 LowBits = CGF->Builder.CreateExtractValue(CI, 0);
171 HighBits = CGF->Builder.CreateExtractValue(CI, 1);
172
173 } else {
174 // For Non DXIL targets we generate the instructions.
175
176 if (!Op0->getType()->isVectorTy()) {
177 FixedVectorType *DestTy = FixedVectorType::get(CGF->Int32Ty, 2);
178 Value *Bitcast = CGF->Builder.CreateBitCast(Op0, DestTy);
179
180 LowBits = CGF->Builder.CreateExtractElement(Bitcast, (uint64_t)0);
181 HighBits = CGF->Builder.CreateExtractElement(Bitcast, 1);
182 } else {
183 int NumElements = 1;
184 if (const auto *VecTy =
185 E->getArg(0)->getType()->getAs<clang::VectorType>())
186 NumElements = VecTy->getNumElements();
187
188 FixedVectorType *Uint32VecTy =
189 FixedVectorType::get(CGF->Int32Ty, NumElements * 2);
190 Value *Uint32Vec = CGF->Builder.CreateBitCast(Op0, Uint32VecTy);
191 if (NumElements == 1) {
192 LowBits = CGF->Builder.CreateExtractElement(Uint32Vec, (uint64_t)0);
193 HighBits = CGF->Builder.CreateExtractElement(Uint32Vec, 1);
194 } else {
195 SmallVector<int> EvenMask, OddMask;
196 for (int I = 0, E = NumElements; I != E; ++I) {
197 EvenMask.push_back(I * 2);
198 OddMask.push_back(I * 2 + 1);
199 }
200 LowBits = CGF->Builder.CreateShuffleVector(Uint32Vec, EvenMask);
201 HighBits = CGF->Builder.CreateShuffleVector(Uint32Vec, OddMask);
202 }
203 }
204 }
205 CGF->Builder.CreateStore(LowBits, Op1TmpLValue.getAddress());
206 auto *LastInst =
207 CGF->Builder.CreateStore(HighBits, Op2TmpLValue.getAddress());
208 CGF->EmitWritebacks(Args);
209 return LastInst;
210}
211
213 assert((E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
214 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation()) &&
215 "asdouble operands types mismatch");
216 Value *OpLowBits = CGF.EmitScalarExpr(E->getArg(0));
217 Value *OpHighBits = CGF.EmitScalarExpr(E->getArg(1));
218
219 llvm::Type *ResultType = CGF.DoubleTy;
220 int N = 1;
221 if (auto *VTy = E->getArg(0)->getType()->getAs<clang::VectorType>()) {
222 N = VTy->getNumElements();
223 ResultType = llvm::FixedVectorType::get(CGF.DoubleTy, N);
224 }
225
226 if (CGF.CGM.getTarget().getTriple().isDXIL())
227 return CGF.Builder.CreateIntrinsic(
228 /*ReturnType=*/ResultType, Intrinsic::dx_asdouble,
229 ArrayRef<Value *>{OpLowBits, OpHighBits}, nullptr, "hlsl.asdouble");
230
231 if (!E->getArg(0)->getType()->isVectorType()) {
232 OpLowBits = CGF.Builder.CreateVectorSplat(1, OpLowBits);
233 OpHighBits = CGF.Builder.CreateVectorSplat(1, OpHighBits);
234 }
235
237 for (int i = 0; i < N; i++) {
238 Mask.push_back(i);
239 Mask.push_back(i + N);
240 }
241
242 Value *BitVec = CGF.Builder.CreateShuffleVector(OpLowBits, OpHighBits, Mask);
243
244 return CGF.Builder.CreateBitCast(BitVec, ResultType);
245}
246
247/// Helper for the read/write/add/inc X18 builtins: read the X18 register and
248/// return it as an i8 pointer.
250 LLVMContext &Context = CGF.CGM.getLLVMContext();
251 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")};
252 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
253 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
254 llvm::Function *F =
255 CGF.CGM.getIntrinsic(llvm::Intrinsic::read_register, {CGF.Int64Ty});
256 llvm::Value *X18 = CGF.Builder.CreateCall(F, Metadata);
257 return CGF.Builder.CreateIntToPtr(X18, CGF.Int8PtrTy);
258}
259
260/// getBuiltinLibFunction - Given a builtin id for a function like
261/// "__builtin_fabsf", return a Function* for "fabsf".
263 unsigned BuiltinID) {
264 assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
265
266 // Get the name, skip over the __builtin_ prefix (if necessary).
267 StringRef Name;
268 GlobalDecl D(FD);
269
270 // TODO: This list should be expanded or refactored after all GCC-compatible
271 // std libcall builtins are implemented.
272 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
273 {Builtin::BI__builtin___fprintf_chk, "__fprintf_chkieee128"},
274 {Builtin::BI__builtin___printf_chk, "__printf_chkieee128"},
275 {Builtin::BI__builtin___snprintf_chk, "__snprintf_chkieee128"},
276 {Builtin::BI__builtin___sprintf_chk, "__sprintf_chkieee128"},
277 {Builtin::BI__builtin___vfprintf_chk, "__vfprintf_chkieee128"},
278 {Builtin::BI__builtin___vprintf_chk, "__vprintf_chkieee128"},
279 {Builtin::BI__builtin___vsnprintf_chk, "__vsnprintf_chkieee128"},
280 {Builtin::BI__builtin___vsprintf_chk, "__vsprintf_chkieee128"},
281 {Builtin::BI__builtin_fprintf, "__fprintfieee128"},
282 {Builtin::BI__builtin_printf, "__printfieee128"},
283 {Builtin::BI__builtin_snprintf, "__snprintfieee128"},
284 {Builtin::BI__builtin_sprintf, "__sprintfieee128"},
285 {Builtin::BI__builtin_vfprintf, "__vfprintfieee128"},
286 {Builtin::BI__builtin_vprintf, "__vprintfieee128"},
287 {Builtin::BI__builtin_vsnprintf, "__vsnprintfieee128"},
288 {Builtin::BI__builtin_vsprintf, "__vsprintfieee128"},
289 {Builtin::BI__builtin_fscanf, "__fscanfieee128"},
290 {Builtin::BI__builtin_scanf, "__scanfieee128"},
291 {Builtin::BI__builtin_sscanf, "__sscanfieee128"},
292 {Builtin::BI__builtin_vfscanf, "__vfscanfieee128"},
293 {Builtin::BI__builtin_vscanf, "__vscanfieee128"},
294 {Builtin::BI__builtin_vsscanf, "__vsscanfieee128"},
295 {Builtin::BI__builtin_nexttowardf128, "__nexttowardieee128"},
296 };
297
298 // The AIX library functions frexpl, ldexpl, and modfl are for 128-bit
299 // IBM 'long double' (i.e. __ibm128). Map to the 'double' versions
300 // if it is 64-bit 'long double' mode.
301 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
302 {Builtin::BI__builtin_frexpl, "frexp"},
303 {Builtin::BI__builtin_ldexpl, "ldexp"},
304 {Builtin::BI__builtin_modfl, "modf"},
305 };
306
307 // If the builtin has been declared explicitly with an assembler label,
308 // use the mangled name. This differs from the plain label on platforms
309 // that prefix labels.
310 if (FD->hasAttr<AsmLabelAttr>())
311 Name = getMangledName(D);
312 else {
313 // TODO: This mutation should also be applied to other targets other than
314 // PPC, after backend supports IEEE 128-bit style libcalls.
315 if (getTriple().isPPC64() &&
316 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
317 F128Builtins.contains(BuiltinID))
318 Name = F128Builtins[BuiltinID];
319 else if (getTriple().isOSAIX() &&
320 &getTarget().getLongDoubleFormat() ==
321 &llvm::APFloat::IEEEdouble() &&
322 AIXLongDouble64Builtins.contains(BuiltinID))
323 Name = AIXLongDouble64Builtins[BuiltinID];
324 else
325 Name = Context.BuiltinInfo.getName(BuiltinID).substr(10);
326 }
327
328 llvm::FunctionType *Ty =
329 cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
330
331 return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);
332}
333
334/// Emit the conversions required to turn the given value into an
335/// integer of the given size.
336static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
337 QualType T, llvm::IntegerType *IntType) {
338 V = CGF.EmitToMemory(V, T);
339
340 if (V->getType()->isPointerTy())
341 return CGF.Builder.CreatePtrToInt(V, IntType);
342
343 assert(V->getType() == IntType);
344 return V;
345}
346
347static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
348 QualType T, llvm::Type *ResultType) {
349 V = CGF.EmitFromMemory(V, T);
350
351 if (ResultType->isPointerTy())
352 return CGF.Builder.CreateIntToPtr(V, ResultType);
353
354 assert(V->getType() == ResultType);
355 return V;
356}
357
359 ASTContext &Ctx = CGF.getContext();
360 Address Ptr = CGF.EmitPointerWithAlignment(E->getArg(0));
361 unsigned Bytes = Ptr.getElementType()->isPointerTy()
363 : Ptr.getElementType()->getScalarSizeInBits() / 8;
364 unsigned Align = Ptr.getAlignment().getQuantity();
365 if (Align % Bytes != 0) {
366 DiagnosticsEngine &Diags = CGF.CGM.getDiags();
367 Diags.Report(E->getBeginLoc(), diag::warn_sync_op_misaligned);
368 // Force address to be at least naturally-aligned.
369 return Ptr.withAlignment(CharUnits::fromQuantity(Bytes));
370 }
371 return Ptr;
372}
373
374/// Utility to insert an atomic instruction based on Intrinsic::ID
375/// and the expression node.
377 CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E,
378 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
379
380 QualType T = E->getType();
381 assert(E->getArg(0)->getType()->isPointerType());
383 E->getArg(0)->getType()->getPointeeType()));
384 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
385
386 Address DestAddr = CheckAtomicAlignment(CGF, E);
387
388 llvm::IntegerType *IntType = llvm::IntegerType::get(
389 CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
390
391 llvm::Value *Val = CGF.EmitScalarExpr(E->getArg(1));
392 llvm::Type *ValueType = Val->getType();
393 Val = EmitToInt(CGF, Val, T, IntType);
394
395 llvm::Value *Result =
396 CGF.Builder.CreateAtomicRMW(Kind, DestAddr, Val, Ordering);
397 return EmitFromInt(CGF, Result, T, ValueType);
398}
399
401 Value *Val = CGF.EmitScalarExpr(E->getArg(0));
402 Address Addr = CGF.EmitPointerWithAlignment(E->getArg(1));
403
404 Val = CGF.EmitToMemory(Val, E->getArg(0)->getType());
405 LValue LV = CGF.MakeAddrLValue(Addr, E->getArg(0)->getType());
406 LV.setNontemporal(true);
407 CGF.EmitStoreOfScalar(Val, LV, false);
408 return nullptr;
409}
410
412 Address Addr = CGF.EmitPointerWithAlignment(E->getArg(0));
413
414 LValue LV = CGF.MakeAddrLValue(Addr, E->getType());
415 LV.setNontemporal(true);
416 return CGF.EmitLoadOfScalar(LV, E->getExprLoc());
417}
418
420 llvm::AtomicRMWInst::BinOp Kind,
421 const CallExpr *E) {
422 return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E));
423}
424
425/// Utility to insert an atomic instruction based Intrinsic::ID and
426/// the expression node, where the return value is the result of the
427/// operation.
429 llvm::AtomicRMWInst::BinOp Kind,
430 const CallExpr *E,
431 Instruction::BinaryOps Op,
432 bool Invert = false) {
433 QualType T = E->getType();
434 assert(E->getArg(0)->getType()->isPointerType());
436 E->getArg(0)->getType()->getPointeeType()));
437 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
438
439 Address DestAddr = CheckAtomicAlignment(CGF, E);
440
441 llvm::IntegerType *IntType = llvm::IntegerType::get(
442 CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
443
444 llvm::Value *Val = CGF.EmitScalarExpr(E->getArg(1));
445 llvm::Type *ValueType = Val->getType();
446 Val = EmitToInt(CGF, Val, T, IntType);
447
448 llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
449 Kind, DestAddr, Val, llvm::AtomicOrdering::SequentiallyConsistent);
450 Result = CGF.Builder.CreateBinOp(Op, Result, Val);
451 if (Invert)
452 Result =
453 CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,
454 llvm::ConstantInt::getAllOnesValue(IntType));
455 Result = EmitFromInt(CGF, Result, T, ValueType);
456 return RValue::get(Result);
457}
458
459/// Utility to insert an atomic cmpxchg instruction.
460///
461/// @param CGF The current codegen function.
462/// @param E Builtin call expression to convert to cmpxchg.
463/// arg0 - address to operate on
464/// arg1 - value to compare with
465/// arg2 - new value
466/// @param ReturnBool Specifies whether to return success flag of
467/// cmpxchg result or the old value.
468///
469/// @returns result of cmpxchg, according to ReturnBool
470///
471/// Note: In order to lower Microsoft's _InterlockedCompareExchange* intrinsics
472/// invoke the function EmitAtomicCmpXchgForMSIntrin.
474 bool ReturnBool) {
475 QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType();
476 Address DestAddr = CheckAtomicAlignment(CGF, E);
477
478 llvm::IntegerType *IntType = llvm::IntegerType::get(
479 CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
480
481 Value *Cmp = CGF.EmitScalarExpr(E->getArg(1));
482 llvm::Type *ValueType = Cmp->getType();
483 Cmp = EmitToInt(CGF, Cmp, T, IntType);
484 Value *New = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);
485
487 DestAddr, Cmp, New, llvm::AtomicOrdering::SequentiallyConsistent,
488 llvm::AtomicOrdering::SequentiallyConsistent);
489 if (ReturnBool)
490 // Extract boolean success flag and zext it to int.
491 return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1),
492 CGF.ConvertType(E->getType()));
493 else
494 // Extract old value and emit it using the same type as compare value.
495 return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T,
496 ValueType);
497}
498
499/// This function should be invoked to emit atomic cmpxchg for Microsoft's
500/// _InterlockedCompareExchange* intrinsics which have the following signature:
501/// T _InterlockedCompareExchange(T volatile *Destination,
502/// T Exchange,
503/// T Comparand);
504///
505/// Whereas the llvm 'cmpxchg' instruction has the following syntax:
506/// cmpxchg *Destination, Comparand, Exchange.
507/// So we need to swap Comparand and Exchange when invoking
508/// CreateAtomicCmpXchg. That is the reason we could not use the above utility
509/// function MakeAtomicCmpXchgValue since it expects the arguments to be
510/// already swapped.
511
512static
514 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
515 assert(E->getArg(0)->getType()->isPointerType());
517 E->getType(), E->getArg(0)->getType()->getPointeeType()));
519 E->getArg(1)->getType()));
521 E->getArg(2)->getType()));
522
523 Address DestAddr = CheckAtomicAlignment(CGF, E);
524
525 auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));
526 auto *RTy = Exchange->getType();
527
528 auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
529
530 if (RTy->isPointerTy()) {
531 Exchange = CGF.Builder.CreatePtrToInt(Exchange, CGF.IntPtrTy);
532 Comparand = CGF.Builder.CreatePtrToInt(Comparand, CGF.IntPtrTy);
533 }
534
535 // For Release ordering, the failure ordering should be Monotonic.
536 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
537 AtomicOrdering::Monotonic :
538 SuccessOrdering;
539
540 // The atomic instruction is marked volatile for consistency with MSVC. This
541 // blocks the few atomics optimizations that LLVM has. If we want to optimize
542 // _Interlocked* operations in the future, we will have to remove the volatile
543 // marker.
544 auto *CmpXchg = CGF.Builder.CreateAtomicCmpXchg(
545 DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
546 CmpXchg->setVolatile(true);
547
548 auto *Result = CGF.Builder.CreateExtractValue(CmpXchg, 0);
549 if (RTy->isPointerTy()) {
550 Result = CGF.Builder.CreateIntToPtr(Result, RTy);
551 }
552
553 return Result;
554}
555
556// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
557// prototyped like this:
558//
559// unsigned char _InterlockedCompareExchange128...(
560// __int64 volatile * _Destination,
561// __int64 _ExchangeHigh,
562// __int64 _ExchangeLow,
563// __int64 * _ComparandResult);
564//
565// Note that Destination is assumed to be at least 16-byte aligned, despite
566// being typed int64.
567
569 const CallExpr *E,
570 AtomicOrdering SuccessOrdering) {
571 assert(E->getNumArgs() == 4);
572 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
573 llvm::Value *ExchangeHigh = CGF.EmitScalarExpr(E->getArg(1));
574 llvm::Value *ExchangeLow = CGF.EmitScalarExpr(E->getArg(2));
575 Address ComparandAddr = CGF.EmitPointerWithAlignment(E->getArg(3));
576
577 assert(DestPtr->getType()->isPointerTy());
578 assert(!ExchangeHigh->getType()->isPointerTy());
579 assert(!ExchangeLow->getType()->isPointerTy());
580
581 // For Release ordering, the failure ordering should be Monotonic.
582 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
583 ? AtomicOrdering::Monotonic
584 : SuccessOrdering;
585
586 // Convert to i128 pointers and values. Alignment is also overridden for
587 // destination pointer.
588 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.getLLVMContext(), 128);
589 Address DestAddr(DestPtr, Int128Ty,
591 ComparandAddr = ComparandAddr.withElementType(Int128Ty);
592
593 // (((i128)hi) << 64) | ((i128)lo)
594 ExchangeHigh = CGF.Builder.CreateZExt(ExchangeHigh, Int128Ty);
595 ExchangeLow = CGF.Builder.CreateZExt(ExchangeLow, Int128Ty);
596 ExchangeHigh =
597 CGF.Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
598 llvm::Value *Exchange = CGF.Builder.CreateOr(ExchangeHigh, ExchangeLow);
599
600 // Load the comparand for the instruction.
601 llvm::Value *Comparand = CGF.Builder.CreateLoad(ComparandAddr);
602
603 auto *CXI = CGF.Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
604 SuccessOrdering, FailureOrdering);
605
606 // The atomic instruction is marked volatile for consistency with MSVC. This
607 // blocks the few atomics optimizations that LLVM has. If we want to optimize
608 // _Interlocked* operations in the future, we will have to remove the volatile
609 // marker.
610 CXI->setVolatile(true);
611
612 // Store the result as an outparameter.
613 CGF.Builder.CreateStore(CGF.Builder.CreateExtractValue(CXI, 0),
614 ComparandAddr);
615
616 // Get the success boolean and zero extend it to i8.
617 Value *Success = CGF.Builder.CreateExtractValue(CXI, 1);
618 return CGF.Builder.CreateZExt(Success, CGF.Int8Ty);
619}
620
622 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
623 assert(E->getArg(0)->getType()->isPointerType());
624
625 auto *IntTy = CGF.ConvertType(E->getType());
626 Address DestAddr = CheckAtomicAlignment(CGF, E);
627 auto *Result = CGF.Builder.CreateAtomicRMW(
628 AtomicRMWInst::Add, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
629 return CGF.Builder.CreateAdd(Result, ConstantInt::get(IntTy, 1));
630}
631
633 CodeGenFunction &CGF, const CallExpr *E,
634 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
635 assert(E->getArg(0)->getType()->isPointerType());
636
637 auto *IntTy = CGF.ConvertType(E->getType());
638 Address DestAddr = CheckAtomicAlignment(CGF, E);
639 auto *Result = CGF.Builder.CreateAtomicRMW(
640 AtomicRMWInst::Sub, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
641 return CGF.Builder.CreateSub(Result, ConstantInt::get(IntTy, 1));
642}
643
644// Build a plain volatile load.
646 Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
647 QualType ElTy = E->getArg(0)->getType()->getPointeeType();
648 CharUnits LoadSize = CGF.getContext().getTypeSizeInChars(ElTy);
649 llvm::Type *ITy =
650 llvm::IntegerType::get(CGF.getLLVMContext(), LoadSize.getQuantity() * 8);
651 llvm::LoadInst *Load = CGF.Builder.CreateAlignedLoad(ITy, Ptr, LoadSize);
652 Load->setVolatile(true);
653 return Load;
654}
655
656// Build a plain volatile store.
658 Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
659 Value *Value = CGF.EmitScalarExpr(E->getArg(1));
660 QualType ElTy = E->getArg(0)->getType()->getPointeeType();
661 CharUnits StoreSize = CGF.getContext().getTypeSizeInChars(ElTy);
662 llvm::StoreInst *Store =
663 CGF.Builder.CreateAlignedStore(Value, Ptr, StoreSize);
664 Store->setVolatile(true);
665 return Store;
666}
667
668// Emit a simple mangled intrinsic that has 1 argument and a return type
669// matching the argument type. Depending on mode, this may be a constrained
670// floating-point intrinsic.
672 const CallExpr *E, unsigned IntrinsicID,
673 unsigned ConstrainedIntrinsicID) {
674 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
675
676 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
677 if (CGF.Builder.getIsFPConstrained()) {
678 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
679 return CGF.Builder.CreateConstrainedFPCall(F, { Src0 });
680 } else {
681 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
682 return CGF.Builder.CreateCall(F, Src0);
683 }
684}
685
686// Emit an intrinsic that has 2 operands of the same type as its result.
687// Depending on mode, this may be a constrained floating-point intrinsic.
689 const CallExpr *E, unsigned IntrinsicID,
690 unsigned ConstrainedIntrinsicID) {
691 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
692 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
693
694 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
695 if (CGF.Builder.getIsFPConstrained()) {
696 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
697 return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
698 } else {
699 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
700 return CGF.Builder.CreateCall(F, { Src0, Src1 });
701 }
702}
703
704// Has second type mangled argument.
706 CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID,
707 llvm::Intrinsic::ID ConstrainedIntrinsicID) {
708 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
709 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
710
711 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
712 if (CGF.Builder.getIsFPConstrained()) {
713 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID,
714 {Src0->getType(), Src1->getType()});
715 return CGF.Builder.CreateConstrainedFPCall(F, {Src0, Src1});
716 }
717
718 Function *F =
719 CGF.CGM.getIntrinsic(IntrinsicID, {Src0->getType(), Src1->getType()});
720 return CGF.Builder.CreateCall(F, {Src0, Src1});
721}
722
723// Emit an intrinsic that has 3 operands of the same type as its result.
724// Depending on mode, this may be a constrained floating-point intrinsic.
726 const CallExpr *E, unsigned IntrinsicID,
727 unsigned ConstrainedIntrinsicID) {
728 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
729 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
730 llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
731
732 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
733 if (CGF.Builder.getIsFPConstrained()) {
734 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
735 return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
736 } else {
737 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
738 return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });
739 }
740}
741
742// Emit an intrinsic where all operands are of the same type as the result.
743// Depending on mode, this may be a constrained floating-point intrinsic.
745 unsigned IntrinsicID,
746 unsigned ConstrainedIntrinsicID,
747 llvm::Type *Ty,
748 ArrayRef<Value *> Args) {
749 Function *F;
750 if (CGF.Builder.getIsFPConstrained())
751 F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Ty);
752 else
753 F = CGF.CGM.getIntrinsic(IntrinsicID, Ty);
754
755 if (CGF.Builder.getIsFPConstrained())
756 return CGF.Builder.CreateConstrainedFPCall(F, Args);
757 else
758 return CGF.Builder.CreateCall(F, Args);
759}
760
761// Emit a simple intrinsic that has N scalar arguments and a return type
762// matching the argument type. It is assumed that only the first argument is
763// overloaded.
764template <unsigned N>
766 const CallExpr *E,
767 unsigned IntrinsicID,
768 llvm::StringRef Name = "") {
769 static_assert(N, "expect non-empty argument");
771 for (unsigned I = 0; I < N; ++I)
772 Args.push_back(CGF.EmitScalarExpr(E->getArg(I)));
773 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Args[0]->getType());
774 return CGF.Builder.CreateCall(F, Args, Name);
775}
776
777// Emit an intrinsic that has 4 operands of the same type as its result.
779 unsigned IntrinsicID) {
780 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
781 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
782 llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
783 llvm::Value *Src3 = CGF.EmitScalarExpr(E->getArg(3));
784
785 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
786 return CGF.Builder.CreateCall(F, {Src0, Src1, Src2, Src3});
787}
788
789// Emit an intrinsic that has 1 float or double operand, and 1 integer.
791 const CallExpr *E,
792 unsigned IntrinsicID) {
793 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
794 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
795
796 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
797 return CGF.Builder.CreateCall(F, {Src0, Src1});
798}
799
800// Emit an intrinsic that has overloaded integer result and fp operand.
801static Value *
803 unsigned IntrinsicID,
804 unsigned ConstrainedIntrinsicID) {
805 llvm::Type *ResultType = CGF.ConvertType(E->getType());
806 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
807
808 if (CGF.Builder.getIsFPConstrained()) {
809 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
810 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID,
811 {ResultType, Src0->getType()});
812 return CGF.Builder.CreateConstrainedFPCall(F, {Src0});
813 } else {
814 Function *F =
815 CGF.CGM.getIntrinsic(IntrinsicID, {ResultType, Src0->getType()});
816 return CGF.Builder.CreateCall(F, Src0);
817 }
818}
819
821 llvm::Intrinsic::ID IntrinsicID) {
822 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
823 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
824
825 QualType IntPtrTy = E->getArg(1)->getType()->getPointeeType();
826 llvm::Type *IntTy = CGF.ConvertType(IntPtrTy);
827 llvm::Function *F =
828 CGF.CGM.getIntrinsic(IntrinsicID, {Src0->getType(), IntTy});
829 llvm::Value *Call = CGF.Builder.CreateCall(F, Src0);
830
831 llvm::Value *Exp = CGF.Builder.CreateExtractValue(Call, 1);
832 LValue LV = CGF.MakeNaturalAlignAddrLValue(Src1, IntPtrTy);
833 CGF.EmitStoreOfScalar(Exp, LV);
834
835 return CGF.Builder.CreateExtractValue(Call, 0);
836}
837
839 llvm::Intrinsic::ID IntrinsicID) {
840 llvm::Value *Val = CGF.EmitScalarExpr(E->getArg(0));
841 llvm::Value *Dest0 = CGF.EmitScalarExpr(E->getArg(1));
842 llvm::Value *Dest1 = CGF.EmitScalarExpr(E->getArg(2));
843
844 llvm::Function *F = CGF.CGM.getIntrinsic(IntrinsicID, {Val->getType()});
845 llvm::Value *Call = CGF.Builder.CreateCall(F, Val);
846
847 llvm::Value *SinResult = CGF.Builder.CreateExtractValue(Call, 0);
848 llvm::Value *CosResult = CGF.Builder.CreateExtractValue(Call, 1);
849
850 QualType DestPtrType = E->getArg(1)->getType()->getPointeeType();
851 LValue SinLV = CGF.MakeNaturalAlignAddrLValue(Dest0, DestPtrType);
852 LValue CosLV = CGF.MakeNaturalAlignAddrLValue(Dest1, DestPtrType);
853
854 llvm::StoreInst *StoreSin =
855 CGF.Builder.CreateStore(SinResult, SinLV.getAddress());
856 llvm::StoreInst *StoreCos =
857 CGF.Builder.CreateStore(CosResult, CosLV.getAddress());
858
859 // Mark the two stores as non-aliasing with each other. The order of stores
860 // emitted by this builtin is arbitrary, enforcing a particular order will
861 // prevent optimizations later on.
862 llvm::MDBuilder MDHelper(CGF.getLLVMContext());
863 MDNode *Domain = MDHelper.createAnonymousAliasScopeDomain();
864 MDNode *AliasScope = MDHelper.createAnonymousAliasScope(Domain);
865 MDNode *AliasScopeList = MDNode::get(Call->getContext(), AliasScope);
866 StoreSin->setMetadata(LLVMContext::MD_alias_scope, AliasScopeList);
867 StoreCos->setMetadata(LLVMContext::MD_noalias, AliasScopeList);
868}
869
870/// EmitFAbs - Emit a call to @llvm.fabs().
872 Function *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType());
873 llvm::CallInst *Call = CGF.Builder.CreateCall(F, V);
874 Call->setDoesNotAccessMemory();
875 return Call;
876}
877
878/// Emit the computation of the sign bit for a floating point value. Returns
879/// the i1 sign bit value.
881 LLVMContext &C = CGF.CGM.getLLVMContext();
882
883 llvm::Type *Ty = V->getType();
884 int Width = Ty->getPrimitiveSizeInBits();
885 llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
886 V = CGF.Builder.CreateBitCast(V, IntTy);
887 if (Ty->isPPC_FP128Ty()) {
888 // We want the sign bit of the higher-order double. The bitcast we just
889 // did works as if the double-double was stored to memory and then
890 // read as an i128. The "store" will put the higher-order double in the
891 // lower address in both little- and big-Endian modes, but the "load"
892 // will treat those bits as a different part of the i128: the low bits in
893 // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
894 // we need to shift the high bits down to the low before truncating.
895 Width >>= 1;
896 if (CGF.getTarget().isBigEndian()) {
897 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
898 V = CGF.Builder.CreateLShr(V, ShiftCst);
899 }
900 // We are truncating value in order to extract the higher-order
901 // double, which we will be using to extract the sign from.
902 IntTy = llvm::IntegerType::get(C, Width);
903 V = CGF.Builder.CreateTrunc(V, IntTy);
904 }
905 Value *Zero = llvm::Constant::getNullValue(IntTy);
906 return CGF.Builder.CreateICmpSLT(V, Zero);
907}
908
909/// Checks no arguments or results are passed indirectly in the ABI (i.e. via a
910/// hidden pointer). This is used to check annotating FP libcalls (that could
911/// set `errno`) with "int" TBAA metadata is safe. If any floating-point
912/// arguments are passed indirectly, setup for the call could be incorrectly
913/// optimized out.
915 auto IsIndirect = [&](ABIArgInfo const &info) {
916 return info.isIndirect() || info.isIndirectAliased() || info.isInAlloca();
917 };
918 return !IsIndirect(FnInfo.getReturnInfo()) &&
919 llvm::none_of(FnInfo.arguments(),
920 [&](CGFunctionInfoArgInfo const &ArgInfo) {
921 return IsIndirect(ArgInfo.info);
922 });
923}
924
926 const CallExpr *E, llvm::Constant *calleeValue) {
927 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
928 CGCallee callee = CGCallee::forDirect(calleeValue, GlobalDecl(FD));
929 llvm::CallBase *callOrInvoke = nullptr;
930 CGFunctionInfo const *FnInfo = nullptr;
931 RValue Call =
932 CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot(),
933 /*Chain=*/nullptr, &callOrInvoke, &FnInfo);
934
935 if (unsigned BuiltinID = FD->getBuiltinID()) {
936 // Check whether a FP math builtin function, such as BI__builtin_expf
937 ASTContext &Context = CGF.getContext();
938 bool ConstWithoutErrnoAndExceptions =
940 // Restrict to target with errno, for example, MacOS doesn't set errno.
941 // TODO: Support builtin function with complex type returned, eg: cacosh
942 if (ConstWithoutErrnoAndExceptions && CGF.CGM.getLangOpts().MathErrno &&
943 !CGF.Builder.getIsFPConstrained() && Call.isScalar() &&
945 // Emit "int" TBAA metadata on FP math libcalls.
946 clang::QualType IntTy = Context.IntTy;
947 TBAAAccessInfo TBAAInfo = CGF.CGM.getTBAAAccessInfo(IntTy);
948 CGF.CGM.DecorateInstructionWithTBAA(callOrInvoke, TBAAInfo);
949 }
950 }
951 return Call;
952}
953
954/// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
955/// depending on IntrinsicID.
956///
957/// \arg CGF The current codegen function.
958/// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
959/// \arg X The first argument to the llvm.*.with.overflow.*.
960/// \arg Y The second argument to the llvm.*.with.overflow.*.
961/// \arg Carry The carry returned by the llvm.*.with.overflow.*.
962/// \returns The result (i.e. sum/product) returned by the intrinsic.
963static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
964 const llvm::Intrinsic::ID IntrinsicID,
965 llvm::Value *X, llvm::Value *Y,
966 llvm::Value *&Carry) {
967 // Make sure we have integers of the same width.
968 assert(X->getType() == Y->getType() &&
969 "Arguments must be the same type. (Did you forget to make sure both "
970 "arguments have the same integer width?)");
971
972 Function *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType());
973 llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y});
974 Carry = CGF.Builder.CreateExtractValue(Tmp, 1);
975 return CGF.Builder.CreateExtractValue(Tmp, 0);
976}
977
978static Value *emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID,
979 int low, int high) {
980 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, {});
981 llvm::CallInst *Call = CGF.Builder.CreateCall(F);
982 llvm::ConstantRange CR(APInt(32, low), APInt(32, high));
983 Call->addRangeRetAttr(CR);
984 Call->addRetAttr(llvm::Attribute::AttrKind::NoUndef);
985 return Call;
986}
987
988namespace {
989 struct WidthAndSignedness {
990 unsigned Width;
991 bool Signed;
992 };
993}
994
995static WidthAndSignedness
997 const clang::QualType Type) {
998 assert(Type->isIntegerType() && "Given type is not an integer.");
999 unsigned Width = context.getIntWidth(Type);
1001 return {Width, Signed};
1002}
1003
1004// Given one or more integer types, this function produces an integer type that
1005// encompasses them: any value in one of the given types could be expressed in
1006// the encompassing type.
1007static struct WidthAndSignedness
1008EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
1009 assert(Types.size() > 0 && "Empty list of types.");
1010
1011 // If any of the given types is signed, we must return a signed type.
1012 bool Signed = false;
1013 for (const auto &Type : Types) {
1014 Signed |= Type.Signed;
1015 }
1016
1017 // The encompassing type must have a width greater than or equal to the width
1018 // of the specified types. Additionally, if the encompassing type is signed,
1019 // its width must be strictly greater than the width of any unsigned types
1020 // given.
1021 unsigned Width = 0;
1022 for (const auto &Type : Types) {
1023 unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
1024 if (Width < MinWidth) {
1025 Width = MinWidth;
1026 }
1027 }
1028
1029 return {Width, Signed};
1030}
1031
1032Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
1033 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
1034 return Builder.CreateCall(CGM.getIntrinsic(inst, {ArgValue->getType()}),
1035 ArgValue);
1036}
1037
1038/// Checks if using the result of __builtin_object_size(p, @p From) in place of
1039/// __builtin_object_size(p, @p To) is correct
1040static bool areBOSTypesCompatible(int From, int To) {
1041 // Note: Our __builtin_object_size implementation currently treats Type=0 and
1042 // Type=2 identically. Encoding this implementation detail here may make
1043 // improving __builtin_object_size difficult in the future, so it's omitted.
1044 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
1045}
1046
1047static llvm::Value *
1048getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
1049 return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true);
1050}
1051
1052llvm::Value *
1053CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
1054 llvm::IntegerType *ResType,
1055 llvm::Value *EmittedE,
1056 bool IsDynamic) {
1057 uint64_t ObjectSize;
1058 if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type))
1059 return emitBuiltinObjectSize(E, Type, ResType, EmittedE, IsDynamic);
1060 return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true);
1061}
1062
1064 ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FAMDecl,
1065 uint64_t &Offset) {
1066 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
1067 getLangOpts().getStrictFlexArraysLevel();
1068 uint32_t FieldNo = 0;
1069
1070 if (RD->isImplicit())
1071 return nullptr;
1072
1073 for (const FieldDecl *FD : RD->fields()) {
1074 if ((!FAMDecl || FD == FAMDecl) &&
1076 Ctx, FD, FD->getType(), StrictFlexArraysLevel,
1077 /*IgnoreTemplateOrMacroSubstitution=*/true)) {
1078 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
1079 Offset += Layout.getFieldOffset(FieldNo);
1080 return FD;
1081 }
1082
1083 QualType Ty = FD->getType();
1084 if (Ty->isRecordType()) {
1086 Ctx, Ty->getAsRecordDecl(), FAMDecl, Offset)) {
1087 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
1088 Offset += Layout.getFieldOffset(FieldNo);
1089 return Field;
1090 }
1091 }
1092
1093 if (!RD->isUnion())
1094 ++FieldNo;
1095 }
1096
1097 return nullptr;
1098}
1099
1100static unsigned CountCountedByAttrs(const RecordDecl *RD) {
1101 unsigned Num = 0;
1102
1103 for (const FieldDecl *FD : RD->fields()) {
1104 if (FD->getType()->isCountAttributedType())
1105 return ++Num;
1106
1107 QualType Ty = FD->getType();
1108 if (Ty->isRecordType())
1110 }
1111
1112 return Num;
1113}
1114
1115llvm::Value *
1116CodeGenFunction::emitFlexibleArrayMemberSize(const Expr *E, unsigned Type,
1117 llvm::IntegerType *ResType) {
1118 // The code generated here calculates the size of a struct with a flexible
1119 // array member that uses the counted_by attribute. There are two instances
1120 // we handle:
1121 //
1122 // struct s {
1123 // unsigned long flags;
1124 // int count;
1125 // int array[] __attribute__((counted_by(count)));
1126 // }
1127 //
1128 // 1) bdos of the flexible array itself:
1129 //
1130 // __builtin_dynamic_object_size(p->array, 1) ==
1131 // p->count * sizeof(*p->array)
1132 //
1133 // 2) bdos of a pointer into the flexible array:
1134 //
1135 // __builtin_dynamic_object_size(&p->array[42], 1) ==
1136 // (p->count - 42) * sizeof(*p->array)
1137 //
1138 // 2) bdos of the whole struct, including the flexible array:
1139 //
1140 // __builtin_dynamic_object_size(p, 1) ==
1141 // max(sizeof(struct s),
1142 // offsetof(struct s, array) + p->count * sizeof(*p->array))
1143 //
1144 ASTContext &Ctx = getContext();
1145 const Expr *Base = E->IgnoreParenImpCasts();
1146 const Expr *Idx = nullptr;
1147
1148 if (const auto *UO = dyn_cast<UnaryOperator>(Base);
1149 UO && UO->getOpcode() == UO_AddrOf) {
1150 Expr *SubExpr = UO->getSubExpr()->IgnoreParenImpCasts();
1151 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) {
1152 Base = ASE->getBase()->IgnoreParenImpCasts();
1153 Idx = ASE->getIdx()->IgnoreParenImpCasts();
1154
1155 if (const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
1156 int64_t Val = IL->getValue().getSExtValue();
1157 if (Val < 0)
1158 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1159
1160 if (Val == 0)
1161 // The index is 0, so we don't need to take it into account.
1162 Idx = nullptr;
1163 }
1164 } else {
1165 // Potential pointer to another element in the struct.
1166 Base = SubExpr;
1167 }
1168 }
1169
1170 // Get the flexible array member Decl.
1171 const RecordDecl *OuterRD = nullptr;
1172 const FieldDecl *FAMDecl = nullptr;
1173 if (const auto *ME = dyn_cast<MemberExpr>(Base)) {
1174 // Check if \p Base is referencing the FAM itself.
1175 const ValueDecl *VD = ME->getMemberDecl();
1177 FAMDecl = dyn_cast<FieldDecl>(VD);
1178 if (!FAMDecl)
1179 return nullptr;
1180 } else if (const auto *DRE = dyn_cast<DeclRefExpr>(Base)) {
1181 // Check if we're pointing to the whole struct.
1182 QualType Ty = DRE->getDecl()->getType();
1183 if (Ty->isPointerType())
1184 Ty = Ty->getPointeeType();
1185 OuterRD = Ty->getAsRecordDecl();
1186
1187 // If we have a situation like this:
1188 //
1189 // struct union_of_fams {
1190 // int flags;
1191 // union {
1192 // signed char normal_field;
1193 // struct {
1194 // int count1;
1195 // int arr1[] __counted_by(count1);
1196 // };
1197 // struct {
1198 // signed char count2;
1199 // int arr2[] __counted_by(count2);
1200 // };
1201 // };
1202 // };
1203 //
1204 // We don't know which 'count' to use in this scenario:
1205 //
1206 // size_t get_size(struct union_of_fams *p) {
1207 // return __builtin_dynamic_object_size(p, 1);
1208 // }
1209 //
1210 // Instead of calculating a wrong number, we give up.
1211 if (OuterRD && CountCountedByAttrs(OuterRD) > 1)
1212 return nullptr;
1213 }
1214
1215 if (!OuterRD)
1216 return nullptr;
1217
1218 // We call FindFlexibleArrayMemberAndOffset even if FAMDecl is non-null to
1219 // get its offset.
1220 uint64_t Offset = 0;
1221 FAMDecl =
1222 FindFlexibleArrayMemberFieldAndOffset(Ctx, OuterRD, FAMDecl, Offset);
1223 Offset = Ctx.toCharUnitsFromBits(Offset).getQuantity();
1224
1225 if (!FAMDecl || !FAMDecl->getType()->isCountAttributedType())
1226 // No flexible array member found or it doesn't have the "counted_by"
1227 // attribute.
1228 return nullptr;
1229
1230 const FieldDecl *CountedByFD = FAMDecl->findCountedByField();
1231 if (!CountedByFD)
1232 // Can't find the field referenced by the "counted_by" attribute.
1233 return nullptr;
1234
1235 if (isa<DeclRefExpr>(Base))
1236 // The whole struct is specificed in the __bdos. The calculation of the
1237 // whole size of the structure can be done in two ways:
1238 //
1239 // 1) sizeof(struct S) + count * sizeof(typeof(fam))
1240 // 2) offsetof(struct S, fam) + count * sizeof(typeof(fam))
1241 //
1242 // The first will add additional padding after the end of the array,
1243 // allocation while the second method is more precise, but not quite
1244 // expected from programmers. See
1245 // https://lore.kernel.org/lkml/ZvV6X5FPBBW7CO1f@archlinux/ for a
1246 // discussion of the topic.
1247 //
1248 // GCC isn't (currently) able to calculate __bdos on a pointer to the whole
1249 // structure. Therefore, because of the above issue, we'll choose to match
1250 // what GCC does for consistency's sake.
1251 return nullptr;
1252
1253 // Build a load of the counted_by field.
1254 bool IsSigned = CountedByFD->getType()->isSignedIntegerType();
1255 Value *CountedByInst = EmitLoadOfCountedByField(Base, FAMDecl, CountedByFD);
1256 if (!CountedByInst)
1257 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1258
1259 CountedByInst = Builder.CreateIntCast(CountedByInst, ResType, IsSigned);
1260
1261 // Build a load of the index and subtract it from the count.
1262 Value *IdxInst = nullptr;
1263 if (Idx) {
1264 if (Idx->HasSideEffects(getContext()))
1265 // We can't have side-effects.
1266 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1267
1268 bool IdxSigned = Idx->getType()->isSignedIntegerType();
1269 IdxInst = EmitAnyExprToTemp(Idx).getScalarVal();
1270 IdxInst = Builder.CreateIntCast(IdxInst, ResType, IdxSigned);
1271
1272 // We go ahead with the calculation here. If the index turns out to be
1273 // negative, we'll catch it at the end.
1274 CountedByInst =
1275 Builder.CreateSub(CountedByInst, IdxInst, "", !IsSigned, IsSigned);
1276 }
1277
1278 // Calculate how large the flexible array member is in bytes.
1279 const ArrayType *ArrayTy = Ctx.getAsArrayType(FAMDecl->getType());
1281 llvm::Constant *ElemSize =
1282 llvm::ConstantInt::get(ResType, Size.getQuantity(), IsSigned);
1283 Value *Res =
1284 Builder.CreateMul(CountedByInst, ElemSize, "", !IsSigned, IsSigned);
1285 Res = Builder.CreateIntCast(Res, ResType, IsSigned);
1286
1287 // A negative \p IdxInst or \p CountedByInst means that the index lands
1288 // outside of the flexible array member. If that's the case, we want to
1289 // return 0.
1290 Value *Cmp = Builder.CreateIsNotNeg(CountedByInst);
1291 if (IdxInst)
1292 Cmp = Builder.CreateAnd(Builder.CreateIsNotNeg(IdxInst), Cmp);
1293
1294 return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
1295}
1296
1297/// Returns a Value corresponding to the size of the given expression.
1298/// This Value may be either of the following:
1299/// - A llvm::Argument (if E is a param with the pass_object_size attribute on
1300/// it)
1301/// - A call to the @llvm.objectsize intrinsic
1302///
1303/// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
1304/// and we wouldn't otherwise try to reference a pass_object_size parameter,
1305/// we'll call @llvm.objectsize on EmittedE, rather than emitting E.
1306llvm::Value *
1307CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
1308 llvm::IntegerType *ResType,
1309 llvm::Value *EmittedE, bool IsDynamic) {
1310 // We need to reference an argument if the pointer is a parameter with the
1311 // pass_object_size attribute.
1312 if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) {
1313 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
1314 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
1315 if (Param != nullptr && PS != nullptr &&
1316 areBOSTypesCompatible(PS->getType(), Type)) {
1317 auto Iter = SizeArguments.find(Param);
1318 assert(Iter != SizeArguments.end());
1319
1320 const ImplicitParamDecl *D = Iter->second;
1321 auto DIter = LocalDeclMap.find(D);
1322 assert(DIter != LocalDeclMap.end());
1323
1324 return EmitLoadOfScalar(DIter->second, /*Volatile=*/false,
1325 getContext().getSizeType(), E->getBeginLoc());
1326 }
1327 }
1328
1329 if (IsDynamic) {
1330 // Emit special code for a flexible array member with the "counted_by"
1331 // attribute.
1332 if (Value *V = emitFlexibleArrayMemberSize(E, Type, ResType))
1333 return V;
1334 }
1335
1336 // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
1337 // evaluate E for side-effects. In either case, we shouldn't lower to
1338 // @llvm.objectsize.
1339 if (Type == 3 || (!EmittedE && E->HasSideEffects(getContext())))
1340 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1341
1342 Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
1343 assert(Ptr->getType()->isPointerTy() &&
1344 "Non-pointer passed to __builtin_object_size?");
1345
1346 Function *F =
1347 CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
1348
1349 // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.
1350 Value *Min = Builder.getInt1((Type & 2) != 0);
1351 // For GCC compatibility, __builtin_object_size treat NULL as unknown size.
1352 Value *NullIsUnknown = Builder.getTrue();
1353 Value *Dynamic = Builder.getInt1(IsDynamic);
1354 return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic});
1355}
1356
1357namespace {
1358/// A struct to generically describe a bit test intrinsic.
1359struct BitTest {
1360 enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
1361 enum InterlockingKind : uint8_t {
1362 Unlocked,
1363 Sequential,
1364 Acquire,
1365 Release,
1366 NoFence
1367 };
1368
1369 ActionKind Action;
1370 InterlockingKind Interlocking;
1371 bool Is64Bit;
1372
1373 static BitTest decodeBitTestBuiltin(unsigned BuiltinID);
1374};
1375
1376} // namespace
1377
1378BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
1379 switch (BuiltinID) {
1380 // Main portable variants.
1381 case Builtin::BI_bittest:
1382 return {TestOnly, Unlocked, false};
1383 case Builtin::BI_bittestandcomplement:
1384 return {Complement, Unlocked, false};
1385 case Builtin::BI_bittestandreset:
1386 return {Reset, Unlocked, false};
1387 case Builtin::BI_bittestandset:
1388 return {Set, Unlocked, false};
1389 case Builtin::BI_interlockedbittestandreset:
1390 return {Reset, Sequential, false};
1391 case Builtin::BI_interlockedbittestandset:
1392 return {Set, Sequential, false};
1393
1394 // X86-specific 64-bit variants.
1395 case Builtin::BI_bittest64:
1396 return {TestOnly, Unlocked, true};
1397 case Builtin::BI_bittestandcomplement64:
1398 return {Complement, Unlocked, true};
1399 case Builtin::BI_bittestandreset64:
1400 return {Reset, Unlocked, true};
1401 case Builtin::BI_bittestandset64:
1402 return {Set, Unlocked, true};
1403 case Builtin::BI_interlockedbittestandreset64:
1404 return {Reset, Sequential, true};
1405 case Builtin::BI_interlockedbittestandset64:
1406 return {Set, Sequential, true};
1407
1408 // ARM/AArch64-specific ordering variants.
1409 case Builtin::BI_interlockedbittestandset_acq:
1410 return {Set, Acquire, false};
1411 case Builtin::BI_interlockedbittestandset_rel:
1412 return {Set, Release, false};
1413 case Builtin::BI_interlockedbittestandset_nf:
1414 return {Set, NoFence, false};
1415 case Builtin::BI_interlockedbittestandreset_acq:
1416 return {Reset, Acquire, false};
1417 case Builtin::BI_interlockedbittestandreset_rel:
1418 return {Reset, Release, false};
1419 case Builtin::BI_interlockedbittestandreset_nf:
1420 return {Reset, NoFence, false};
1421 }
1422 llvm_unreachable("expected only bittest intrinsics");
1423}
1424
1425static char bitActionToX86BTCode(BitTest::ActionKind A) {
1426 switch (A) {
1427 case BitTest::TestOnly: return '\0';
1428 case BitTest::Complement: return 'c';
1429 case BitTest::Reset: return 'r';
1430 case BitTest::Set: return 's';
1431 }
1432 llvm_unreachable("invalid action");
1433}
1434
1436 BitTest BT,
1437 const CallExpr *E, Value *BitBase,
1438 Value *BitPos) {
1439 char Action = bitActionToX86BTCode(BT.Action);
1440 char SizeSuffix = BT.Is64Bit ? 'q' : 'l';
1441
1442 // Build the assembly.
1444 raw_svector_ostream AsmOS(Asm);
1445 if (BT.Interlocking != BitTest::Unlocked)
1446 AsmOS << "lock ";
1447 AsmOS << "bt";
1448 if (Action)
1449 AsmOS << Action;
1450 AsmOS << SizeSuffix << " $2, ($1)";
1451
1452 // Build the constraints. FIXME: We should support immediates when possible.
1453 std::string Constraints = "={@ccc},r,r,~{cc},~{memory}";
1454 std::string_view MachineClobbers = CGF.getTarget().getClobbers();
1455 if (!MachineClobbers.empty()) {
1456 Constraints += ',';
1457 Constraints += MachineClobbers;
1458 }
1459 llvm::IntegerType *IntType = llvm::IntegerType::get(
1460 CGF.getLLVMContext(),
1461 CGF.getContext().getTypeSize(E->getArg(1)->getType()));
1462 llvm::FunctionType *FTy =
1463 llvm::FunctionType::get(CGF.Int8Ty, {CGF.UnqualPtrTy, IntType}, false);
1464
1465 llvm::InlineAsm *IA =
1466 llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);
1467 return CGF.Builder.CreateCall(IA, {BitBase, BitPos});
1468}
1469
1470static llvm::AtomicOrdering
1471getBitTestAtomicOrdering(BitTest::InterlockingKind I) {
1472 switch (I) {
1473 case BitTest::Unlocked: return llvm::AtomicOrdering::NotAtomic;
1474 case BitTest::Sequential: return llvm::AtomicOrdering::SequentiallyConsistent;
1475 case BitTest::Acquire: return llvm::AtomicOrdering::Acquire;
1476 case BitTest::Release: return llvm::AtomicOrdering::Release;
1477 case BitTest::NoFence: return llvm::AtomicOrdering::Monotonic;
1478 }
1479 llvm_unreachable("invalid interlocking");
1480}
1481
1482/// Emit a _bittest* intrinsic. These intrinsics take a pointer to an array of
1483/// bits and a bit position and read and optionally modify the bit at that
1484/// position. The position index can be arbitrarily large, i.e. it can be larger
1485/// than 31 or 63, so we need an indexed load in the general case.
1486static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
1487 unsigned BuiltinID,
1488 const CallExpr *E) {
1489 Value *BitBase = CGF.EmitScalarExpr(E->getArg(0));
1490 Value *BitPos = CGF.EmitScalarExpr(E->getArg(1));
1491
1492 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1493
1494 // X86 has special BT, BTC, BTR, and BTS instructions that handle the array
1495 // indexing operation internally. Use them if possible.
1496 if (CGF.getTarget().getTriple().isX86())
1497 return EmitX86BitTestIntrinsic(CGF, BT, E, BitBase, BitPos);
1498
1499 // Otherwise, use generic code to load one byte and test the bit. Use all but
1500 // the bottom three bits as the array index, and the bottom three bits to form
1501 // a mask.
1502 // Bit = BitBaseI8[BitPos >> 3] & (1 << (BitPos & 0x7)) != 0;
1503 Value *ByteIndex = CGF.Builder.CreateAShr(
1504 BitPos, llvm::ConstantInt::get(BitPos->getType(), 3), "bittest.byteidx");
1505 Address ByteAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, BitBase, ByteIndex,
1506 "bittest.byteaddr"),
1507 CGF.Int8Ty, CharUnits::One());
1508 Value *PosLow =
1509 CGF.Builder.CreateAnd(CGF.Builder.CreateTrunc(BitPos, CGF.Int8Ty),
1510 llvm::ConstantInt::get(CGF.Int8Ty, 0x7));
1511
1512 // The updating instructions will need a mask.
1513 Value *Mask = nullptr;
1514 if (BT.Action != BitTest::TestOnly) {
1515 Mask = CGF.Builder.CreateShl(llvm::ConstantInt::get(CGF.Int8Ty, 1), PosLow,
1516 "bittest.mask");
1517 }
1518
1519 // Check the action and ordering of the interlocked intrinsics.
1520 llvm::AtomicOrdering Ordering = getBitTestAtomicOrdering(BT.Interlocking);
1521
1522 Value *OldByte = nullptr;
1523 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1524 // Emit a combined atomicrmw load/store operation for the interlocked
1525 // intrinsics.
1526 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1527 if (BT.Action == BitTest::Reset) {
1528 Mask = CGF.Builder.CreateNot(Mask);
1529 RMWOp = llvm::AtomicRMWInst::And;
1530 }
1531 OldByte = CGF.Builder.CreateAtomicRMW(RMWOp, ByteAddr, Mask, Ordering);
1532 } else {
1533 // Emit a plain load for the non-interlocked intrinsics.
1534 OldByte = CGF.Builder.CreateLoad(ByteAddr, "bittest.byte");
1535 Value *NewByte = nullptr;
1536 switch (BT.Action) {
1537 case BitTest::TestOnly:
1538 // Don't store anything.
1539 break;
1540 case BitTest::Complement:
1541 NewByte = CGF.Builder.CreateXor(OldByte, Mask);
1542 break;
1543 case BitTest::Reset:
1544 NewByte = CGF.Builder.CreateAnd(OldByte, CGF.Builder.CreateNot(Mask));
1545 break;
1546 case BitTest::Set:
1547 NewByte = CGF.Builder.CreateOr(OldByte, Mask);
1548 break;
1549 }
1550 if (NewByte)
1551 CGF.Builder.CreateStore(NewByte, ByteAddr);
1552 }
1553
1554 // However we loaded the old byte, either by plain load or atomicrmw, shift
1555 // the bit into the low position and mask it to 0 or 1.
1556 Value *ShiftedByte = CGF.Builder.CreateLShr(OldByte, PosLow, "bittest.shr");
1557 return CGF.Builder.CreateAnd(
1558 ShiftedByte, llvm::ConstantInt::get(CGF.Int8Ty, 1), "bittest.res");
1559}
1560
1562 unsigned BuiltinID,
1563 const CallExpr *E) {
1564 Value *Addr = CGF.EmitScalarExpr(E->getArg(0));
1565
1567 raw_svector_ostream AsmOS(Asm);
1568 llvm::IntegerType *RetType = CGF.Int32Ty;
1569
1570 switch (BuiltinID) {
1571 case clang::PPC::BI__builtin_ppc_ldarx:
1572 AsmOS << "ldarx ";
1573 RetType = CGF.Int64Ty;
1574 break;
1575 case clang::PPC::BI__builtin_ppc_lwarx:
1576 AsmOS << "lwarx ";
1577 RetType = CGF.Int32Ty;
1578 break;
1579 case clang::PPC::BI__builtin_ppc_lharx:
1580 AsmOS << "lharx ";
1581 RetType = CGF.Int16Ty;
1582 break;
1583 case clang::PPC::BI__builtin_ppc_lbarx:
1584 AsmOS << "lbarx ";
1585 RetType = CGF.Int8Ty;
1586 break;
1587 default:
1588 llvm_unreachable("Expected only PowerPC load reserve intrinsics");
1589 }
1590
1591 AsmOS << "$0, ${1:y}";
1592
1593 std::string Constraints = "=r,*Z,~{memory}";
1594 std::string_view MachineClobbers = CGF.getTarget().getClobbers();
1595 if (!MachineClobbers.empty()) {
1596 Constraints += ',';
1597 Constraints += MachineClobbers;
1598 }
1599
1600 llvm::Type *PtrType = CGF.UnqualPtrTy;
1601 llvm::FunctionType *FTy = llvm::FunctionType::get(RetType, {PtrType}, false);
1602
1603 llvm::InlineAsm *IA =
1604 llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);
1605 llvm::CallInst *CI = CGF.Builder.CreateCall(IA, {Addr});
1606 CI->addParamAttr(
1607 0, Attribute::get(CGF.getLLVMContext(), Attribute::ElementType, RetType));
1608 return CI;
1609}
1610
1611namespace {
1612enum class MSVCSetJmpKind {
1613 _setjmpex,
1614 _setjmp3,
1615 _setjmp
1616};
1617}
1618
1619/// MSVC handles setjmp a bit differently on different platforms. On every
1620/// architecture except 32-bit x86, the frame address is passed. On x86, extra
1621/// parameters can be passed as variadic arguments, but we always pass none.
1622static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind,
1623 const CallExpr *E) {
1624 llvm::Value *Arg1 = nullptr;
1625 llvm::Type *Arg1Ty = nullptr;
1626 StringRef Name;
1627 bool IsVarArg = false;
1628 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1629 Name = "_setjmp3";
1630 Arg1Ty = CGF.Int32Ty;
1631 Arg1 = llvm::ConstantInt::get(CGF.IntTy, 0);
1632 IsVarArg = true;
1633 } else {
1634 Name = SJKind == MSVCSetJmpKind::_setjmp ? "_setjmp" : "_setjmpex";
1635 Arg1Ty = CGF.Int8PtrTy;
1636 if (CGF.getTarget().getTriple().getArch() == llvm::Triple::aarch64) {
1637 Arg1 = CGF.Builder.CreateCall(
1638 CGF.CGM.getIntrinsic(Intrinsic::sponentry, CGF.AllocaInt8PtrTy));
1639 } else
1640 Arg1 = CGF.Builder.CreateCall(
1641 CGF.CGM.getIntrinsic(Intrinsic::frameaddress, CGF.AllocaInt8PtrTy),
1642 llvm::ConstantInt::get(CGF.Int32Ty, 0));
1643 }
1644
1645 // Mark the call site and declaration with ReturnsTwice.
1646 llvm::Type *ArgTypes[2] = {CGF.Int8PtrTy, Arg1Ty};
1647 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1648 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex,
1649 llvm::Attribute::ReturnsTwice);
1650 llvm::FunctionCallee SetJmpFn = CGF.CGM.CreateRuntimeFunction(
1651 llvm::FunctionType::get(CGF.IntTy, ArgTypes, IsVarArg), Name,
1652 ReturnsTwiceAttr, /*Local=*/true);
1653
1654 llvm::Value *Buf = CGF.Builder.CreateBitOrPointerCast(
1655 CGF.EmitScalarExpr(E->getArg(0)), CGF.Int8PtrTy);
1656 llvm::Value *Args[] = {Buf, Arg1};
1657 llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(SetJmpFn, Args);
1658 CB->setAttributes(ReturnsTwiceAttr);
1659 return RValue::get(CB);
1660}
1661
1662// Many of MSVC builtins are on x64, ARM and AArch64; to avoid repeating code,
1663// we handle them here.
1704 __fastfail,
1705};
1706
1707static std::optional<CodeGenFunction::MSVCIntrin>
1708translateArmToMsvcIntrin(unsigned BuiltinID) {
1709 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
1710 switch (BuiltinID) {
1711 default:
1712 return std::nullopt;
1713 case clang::ARM::BI_BitScanForward:
1714 case clang::ARM::BI_BitScanForward64:
1715 return MSVCIntrin::_BitScanForward;
1716 case clang::ARM::BI_BitScanReverse:
1717 case clang::ARM::BI_BitScanReverse64:
1718 return MSVCIntrin::_BitScanReverse;
1719 case clang::ARM::BI_InterlockedAnd64:
1720 return MSVCIntrin::_InterlockedAnd;
1721 case clang::ARM::BI_InterlockedExchange64:
1722 return MSVCIntrin::_InterlockedExchange;
1723 case clang::ARM::BI_InterlockedExchangeAdd64:
1724 return MSVCIntrin::_InterlockedExchangeAdd;
1725 case clang::ARM::BI_InterlockedExchangeSub64:
1726 return MSVCIntrin::_InterlockedExchangeSub;
1727 case clang::ARM::BI_InterlockedOr64:
1728 return MSVCIntrin::_InterlockedOr;
1729 case clang::ARM::BI_InterlockedXor64:
1730 return MSVCIntrin::_InterlockedXor;
1731 case clang::ARM::BI_InterlockedDecrement64:
1732 return MSVCIntrin::_InterlockedDecrement;
1733 case clang::ARM::BI_InterlockedIncrement64:
1734 return MSVCIntrin::_InterlockedIncrement;
1735 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1736 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1737 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1738 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1739 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1740 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1741 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1742 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1743 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1744 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1745 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1746 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1747 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1748 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1749 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1750 case clang::ARM::BI_InterlockedExchange8_acq:
1751 case clang::ARM::BI_InterlockedExchange16_acq:
1752 case clang::ARM::BI_InterlockedExchange_acq:
1753 case clang::ARM::BI_InterlockedExchange64_acq:
1754 case clang::ARM::BI_InterlockedExchangePointer_acq:
1755 return MSVCIntrin::_InterlockedExchange_acq;
1756 case clang::ARM::BI_InterlockedExchange8_rel:
1757 case clang::ARM::BI_InterlockedExchange16_rel:
1758 case clang::ARM::BI_InterlockedExchange_rel:
1759 case clang::ARM::BI_InterlockedExchange64_rel:
1760 case clang::ARM::BI_InterlockedExchangePointer_rel:
1761 return MSVCIntrin::_InterlockedExchange_rel;
1762 case clang::ARM::BI_InterlockedExchange8_nf:
1763 case clang::ARM::BI_InterlockedExchange16_nf:
1764 case clang::ARM::BI_InterlockedExchange_nf:
1765 case clang::ARM::BI_InterlockedExchange64_nf:
1766 case clang::ARM::BI_InterlockedExchangePointer_nf:
1767 return MSVCIntrin::_InterlockedExchange_nf;
1768 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1769 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1770 case clang::ARM::BI_InterlockedCompareExchange_acq:
1771 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1772 case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
1773 return MSVCIntrin::_InterlockedCompareExchange_acq;
1774 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1775 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1776 case clang::ARM::BI_InterlockedCompareExchange_rel:
1777 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1778 case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
1779 return MSVCIntrin::_InterlockedCompareExchange_rel;
1780 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1781 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1782 case clang::ARM::BI_InterlockedCompareExchange_nf:
1783 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1784 return MSVCIntrin::_InterlockedCompareExchange_nf;
1785 case clang::ARM::BI_InterlockedOr8_acq:
1786 case clang::ARM::BI_InterlockedOr16_acq:
1787 case clang::ARM::BI_InterlockedOr_acq:
1788 case clang::ARM::BI_InterlockedOr64_acq:
1789 return MSVCIntrin::_InterlockedOr_acq;
1790 case clang::ARM::BI_InterlockedOr8_rel:
1791 case clang::ARM::BI_InterlockedOr16_rel:
1792 case clang::ARM::BI_InterlockedOr_rel:
1793 case clang::ARM::BI_InterlockedOr64_rel:
1794 return MSVCIntrin::_InterlockedOr_rel;
1795 case clang::ARM::BI_InterlockedOr8_nf:
1796 case clang::ARM::BI_InterlockedOr16_nf:
1797 case clang::ARM::BI_InterlockedOr_nf:
1798 case clang::ARM::BI_InterlockedOr64_nf:
1799 return MSVCIntrin::_InterlockedOr_nf;
1800 case clang::ARM::BI_InterlockedXor8_acq:
1801 case clang::ARM::BI_InterlockedXor16_acq:
1802 case clang::ARM::BI_InterlockedXor_acq:
1803 case clang::ARM::BI_InterlockedXor64_acq:
1804 return MSVCIntrin::_InterlockedXor_acq;
1805 case clang::ARM::BI_InterlockedXor8_rel:
1806 case clang::ARM::BI_InterlockedXor16_rel:
1807 case clang::ARM::BI_InterlockedXor_rel:
1808 case clang::ARM::BI_InterlockedXor64_rel:
1809 return MSVCIntrin::_InterlockedXor_rel;
1810 case clang::ARM::BI_InterlockedXor8_nf:
1811 case clang::ARM::BI_InterlockedXor16_nf:
1812 case clang::ARM::BI_InterlockedXor_nf:
1813 case clang::ARM::BI_InterlockedXor64_nf:
1814 return MSVCIntrin::_InterlockedXor_nf;
1815 case clang::ARM::BI_InterlockedAnd8_acq:
1816 case clang::ARM::BI_InterlockedAnd16_acq:
1817 case clang::ARM::BI_InterlockedAnd_acq:
1818 case clang::ARM::BI_InterlockedAnd64_acq:
1819 return MSVCIntrin::_InterlockedAnd_acq;
1820 case clang::ARM::BI_InterlockedAnd8_rel:
1821 case clang::ARM::BI_InterlockedAnd16_rel:
1822 case clang::ARM::BI_InterlockedAnd_rel:
1823 case clang::ARM::BI_InterlockedAnd64_rel:
1824 return MSVCIntrin::_InterlockedAnd_rel;
1825 case clang::ARM::BI_InterlockedAnd8_nf:
1826 case clang::ARM::BI_InterlockedAnd16_nf:
1827 case clang::ARM::BI_InterlockedAnd_nf:
1828 case clang::ARM::BI_InterlockedAnd64_nf:
1829 return MSVCIntrin::_InterlockedAnd_nf;
1830 case clang::ARM::BI_InterlockedIncrement16_acq:
1831 case clang::ARM::BI_InterlockedIncrement_acq:
1832 case clang::ARM::BI_InterlockedIncrement64_acq:
1833 return MSVCIntrin::_InterlockedIncrement_acq;
1834 case clang::ARM::BI_InterlockedIncrement16_rel:
1835 case clang::ARM::BI_InterlockedIncrement_rel:
1836 case clang::ARM::BI_InterlockedIncrement64_rel:
1837 return MSVCIntrin::_InterlockedIncrement_rel;
1838 case clang::ARM::BI_InterlockedIncrement16_nf:
1839 case clang::ARM::BI_InterlockedIncrement_nf:
1840 case clang::ARM::BI_InterlockedIncrement64_nf:
1841 return MSVCIntrin::_InterlockedIncrement_nf;
1842 case clang::ARM::BI_InterlockedDecrement16_acq:
1843 case clang::ARM::BI_InterlockedDecrement_acq:
1844 case clang::ARM::BI_InterlockedDecrement64_acq:
1845 return MSVCIntrin::_InterlockedDecrement_acq;
1846 case clang::ARM::BI_InterlockedDecrement16_rel:
1847 case clang::ARM::BI_InterlockedDecrement_rel:
1848 case clang::ARM::BI_InterlockedDecrement64_rel:
1849 return MSVCIntrin::_InterlockedDecrement_rel;
1850 case clang::ARM::BI_InterlockedDecrement16_nf:
1851 case clang::ARM::BI_InterlockedDecrement_nf:
1852 case clang::ARM::BI_InterlockedDecrement64_nf:
1853 return MSVCIntrin::_InterlockedDecrement_nf;
1854 }
1855 llvm_unreachable("must return from switch");
1856}
1857
1858static std::optional<CodeGenFunction::MSVCIntrin>
1859translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
1860 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
1861 switch (BuiltinID) {
1862 default:
1863 return std::nullopt;
1864 case clang::AArch64::BI_BitScanForward:
1865 case clang::AArch64::BI_BitScanForward64:
1866 return MSVCIntrin::_BitScanForward;
1867 case clang::AArch64::BI_BitScanReverse:
1868 case clang::AArch64::BI_BitScanReverse64:
1869 return MSVCIntrin::_BitScanReverse;
1870 case clang::AArch64::BI_InterlockedAnd64:
1871 return MSVCIntrin::_InterlockedAnd;
1872 case clang::AArch64::BI_InterlockedExchange64:
1873 return MSVCIntrin::_InterlockedExchange;
1874 case clang::AArch64::BI_InterlockedExchangeAdd64:
1875 return MSVCIntrin::_InterlockedExchangeAdd;
1876 case clang::AArch64::BI_InterlockedExchangeSub64:
1877 return MSVCIntrin::_InterlockedExchangeSub;
1878 case clang::AArch64::BI_InterlockedOr64:
1879 return MSVCIntrin::_InterlockedOr;
1880 case clang::AArch64::BI_InterlockedXor64:
1881 return MSVCIntrin::_InterlockedXor;
1882 case clang::AArch64::BI_InterlockedDecrement64:
1883 return MSVCIntrin::_InterlockedDecrement;
1884 case clang::AArch64::BI_InterlockedIncrement64:
1885 return MSVCIntrin::_InterlockedIncrement;
1886 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1887 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1888 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1889 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1890 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1891 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1892 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1893 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1894 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1895 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1896 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1897 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1898 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1899 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1900 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1901 case clang::AArch64::BI_InterlockedExchange8_acq:
1902 case clang::AArch64::BI_InterlockedExchange16_acq:
1903 case clang::AArch64::BI_InterlockedExchange_acq:
1904 case clang::AArch64::BI_InterlockedExchange64_acq:
1905 case clang::AArch64::BI_InterlockedExchangePointer_acq:
1906 return MSVCIntrin::_InterlockedExchange_acq;
1907 case clang::AArch64::BI_InterlockedExchange8_rel:
1908 case clang::AArch64::BI_InterlockedExchange16_rel:
1909 case clang::AArch64::BI_InterlockedExchange_rel:
1910 case clang::AArch64::BI_InterlockedExchange64_rel:
1911 case clang::AArch64::BI_InterlockedExchangePointer_rel:
1912 return MSVCIntrin::_InterlockedExchange_rel;
1913 case clang::AArch64::BI_InterlockedExchange8_nf:
1914 case clang::AArch64::BI_InterlockedExchange16_nf:
1915 case clang::AArch64::BI_InterlockedExchange_nf:
1916 case clang::AArch64::BI_InterlockedExchange64_nf:
1917 case clang::AArch64::BI_InterlockedExchangePointer_nf:
1918 return MSVCIntrin::_InterlockedExchange_nf;
1919 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1920 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1921 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1922 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1923 case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
1924 return MSVCIntrin::_InterlockedCompareExchange_acq;
1925 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1926 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1927 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1928 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1929 case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
1930 return MSVCIntrin::_InterlockedCompareExchange_rel;
1931 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1932 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1933 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1934 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1935 return MSVCIntrin::_InterlockedCompareExchange_nf;
1936 case clang::AArch64::BI_InterlockedCompareExchange128:
1937 return MSVCIntrin::_InterlockedCompareExchange128;
1938 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1939 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1940 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1941 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1942 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1943 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1944 case clang::AArch64::BI_InterlockedOr8_acq:
1945 case clang::AArch64::BI_InterlockedOr16_acq:
1946 case clang::AArch64::BI_InterlockedOr_acq:
1947 case clang::AArch64::BI_InterlockedOr64_acq:
1948 return MSVCIntrin::_InterlockedOr_acq;
1949 case clang::AArch64::BI_InterlockedOr8_rel:
1950 case clang::AArch64::BI_InterlockedOr16_rel:
1951 case clang::AArch64::BI_InterlockedOr_rel:
1952 case clang::AArch64::BI_InterlockedOr64_rel:
1953 return MSVCIntrin::_InterlockedOr_rel;
1954 case clang::AArch64::BI_InterlockedOr8_nf:
1955 case clang::AArch64::BI_InterlockedOr16_nf:
1956 case clang::AArch64::BI_InterlockedOr_nf:
1957 case clang::AArch64::BI_InterlockedOr64_nf:
1958 return MSVCIntrin::_InterlockedOr_nf;
1959 case clang::AArch64::BI_InterlockedXor8_acq:
1960 case clang::AArch64::BI_InterlockedXor16_acq:
1961 case clang::AArch64::BI_InterlockedXor_acq:
1962 case clang::AArch64::BI_InterlockedXor64_acq:
1963 return MSVCIntrin::_InterlockedXor_acq;
1964 case clang::AArch64::BI_InterlockedXor8_rel:
1965 case clang::AArch64::BI_InterlockedXor16_rel:
1966 case clang::AArch64::BI_InterlockedXor_rel:
1967 case clang::AArch64::BI_InterlockedXor64_rel:
1968 return MSVCIntrin::_InterlockedXor_rel;
1969 case clang::AArch64::BI_InterlockedXor8_nf:
1970 case clang::AArch64::BI_InterlockedXor16_nf:
1971 case clang::AArch64::BI_InterlockedXor_nf:
1972 case clang::AArch64::BI_InterlockedXor64_nf:
1973 return MSVCIntrin::_InterlockedXor_nf;
1974 case clang::AArch64::BI_InterlockedAnd8_acq:
1975 case clang::AArch64::BI_InterlockedAnd16_acq:
1976 case clang::AArch64::BI_InterlockedAnd_acq:
1977 case clang::AArch64::BI_InterlockedAnd64_acq:
1978 return MSVCIntrin::_InterlockedAnd_acq;
1979 case clang::AArch64::BI_InterlockedAnd8_rel:
1980 case clang::AArch64::BI_InterlockedAnd16_rel:
1981 case clang::AArch64::BI_InterlockedAnd_rel:
1982 case clang::AArch64::BI_InterlockedAnd64_rel:
1983 return MSVCIntrin::_InterlockedAnd_rel;
1984 case clang::AArch64::BI_InterlockedAnd8_nf:
1985 case clang::AArch64::BI_InterlockedAnd16_nf:
1986 case clang::AArch64::BI_InterlockedAnd_nf:
1987 case clang::AArch64::BI_InterlockedAnd64_nf:
1988 return MSVCIntrin::_InterlockedAnd_nf;
1989 case clang::AArch64::BI_InterlockedIncrement16_acq:
1990 case clang::AArch64::BI_InterlockedIncrement_acq:
1991 case clang::AArch64::BI_InterlockedIncrement64_acq:
1992 return MSVCIntrin::_InterlockedIncrement_acq;
1993 case clang::AArch64::BI_InterlockedIncrement16_rel:
1994 case clang::AArch64::BI_InterlockedIncrement_rel:
1995 case clang::AArch64::BI_InterlockedIncrement64_rel:
1996 return MSVCIntrin::_InterlockedIncrement_rel;
1997 case clang::AArch64::BI_InterlockedIncrement16_nf:
1998 case clang::AArch64::BI_InterlockedIncrement_nf:
1999 case clang::AArch64::BI_InterlockedIncrement64_nf:
2000 return MSVCIntrin::_InterlockedIncrement_nf;
2001 case clang::AArch64::BI_InterlockedDecrement16_acq:
2002 case clang::AArch64::BI_InterlockedDecrement_acq:
2003 case clang::AArch64::BI_InterlockedDecrement64_acq:
2004 return MSVCIntrin::_InterlockedDecrement_acq;
2005 case clang::AArch64::BI_InterlockedDecrement16_rel:
2006 case clang::AArch64::BI_InterlockedDecrement_rel:
2007 case clang::AArch64::BI_InterlockedDecrement64_rel:
2008 return MSVCIntrin::_InterlockedDecrement_rel;
2009 case clang::AArch64::BI_InterlockedDecrement16_nf:
2010 case clang::AArch64::BI_InterlockedDecrement_nf:
2011 case clang::AArch64::BI_InterlockedDecrement64_nf:
2012 return MSVCIntrin::_InterlockedDecrement_nf;
2013 }
2014 llvm_unreachable("must return from switch");
2015}
2016
2017static std::optional<CodeGenFunction::MSVCIntrin>
2018translateX86ToMsvcIntrin(unsigned BuiltinID) {
2019 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
2020 switch (BuiltinID) {
2021 default:
2022 return std::nullopt;
2023 case clang::X86::BI_BitScanForward:
2024 case clang::X86::BI_BitScanForward64:
2025 return MSVCIntrin::_BitScanForward;
2026 case clang::X86::BI_BitScanReverse:
2027 case clang::X86::BI_BitScanReverse64:
2028 return MSVCIntrin::_BitScanReverse;
2029 case clang::X86::BI_InterlockedAnd64:
2030 return MSVCIntrin::_InterlockedAnd;
2031 case clang::X86::BI_InterlockedCompareExchange128:
2032 return MSVCIntrin::_InterlockedCompareExchange128;
2033 case clang::X86::BI_InterlockedExchange64:
2034 return MSVCIntrin::_InterlockedExchange;
2035 case clang::X86::BI_InterlockedExchangeAdd64:
2036 return MSVCIntrin::_InterlockedExchangeAdd;
2037 case clang::X86::BI_InterlockedExchangeSub64:
2038 return MSVCIntrin::_InterlockedExchangeSub;
2039 case clang::X86::BI_InterlockedOr64:
2040 return MSVCIntrin::_InterlockedOr;
2041 case clang::X86::BI_InterlockedXor64:
2042 return MSVCIntrin::_InterlockedXor;
2043 case clang::X86::BI_InterlockedDecrement64:
2044 return MSVCIntrin::_InterlockedDecrement;
2045 case clang::X86::BI_InterlockedIncrement64:
2046 return MSVCIntrin::_InterlockedIncrement;
2047 }
2048 llvm_unreachable("must return from switch");
2049}
2050
2051// Emit an MSVC intrinsic. Assumes that arguments have *not* been evaluated.
2052Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
2053 const CallExpr *E) {
2054 switch (BuiltinID) {
2055 case MSVCIntrin::_BitScanForward:
2056 case MSVCIntrin::_BitScanReverse: {
2057 Address IndexAddress(EmitPointerWithAlignment(E->getArg(0)));
2058 Value *ArgValue = EmitScalarExpr(E->getArg(1));
2059
2060 llvm::Type *ArgType = ArgValue->getType();
2061 llvm::Type *IndexType = IndexAddress.getElementType();
2062 llvm::Type *ResultType = ConvertType(E->getType());
2063
2064 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
2065 Value *ResZero = llvm::Constant::getNullValue(ResultType);
2066 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
2067
2068 BasicBlock *Begin = Builder.GetInsertBlock();
2069 BasicBlock *End = createBasicBlock("bitscan_end", this->CurFn);
2070 Builder.SetInsertPoint(End);
2071 PHINode *Result = Builder.CreatePHI(ResultType, 2, "bitscan_result");
2072
2073 Builder.SetInsertPoint(Begin);
2074 Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
2075 BasicBlock *NotZero = createBasicBlock("bitscan_not_zero", this->CurFn);
2076 Builder.CreateCondBr(IsZero, End, NotZero);
2077 Result->addIncoming(ResZero, Begin);
2078
2079 Builder.SetInsertPoint(NotZero);
2080
2081 if (BuiltinID == MSVCIntrin::_BitScanForward) {
2082 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
2083 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
2084 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
2085 Builder.CreateStore(ZeroCount, IndexAddress, false);
2086 } else {
2087 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
2088 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
2089
2090 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
2091 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
2092 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
2093 Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
2094 Builder.CreateStore(Index, IndexAddress, false);
2095 }
2096 Builder.CreateBr(End);
2097 Result->addIncoming(ResOne, NotZero);
2098
2099 Builder.SetInsertPoint(End);
2100 return Result;
2101 }
2102 case MSVCIntrin::_InterlockedAnd:
2103 return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E);
2104 case MSVCIntrin::_InterlockedExchange:
2105 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E);
2106 case MSVCIntrin::_InterlockedExchangeAdd:
2107 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E);
2108 case MSVCIntrin::_InterlockedExchangeSub:
2109 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Sub, E);
2110 case MSVCIntrin::_InterlockedOr:
2111 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E);
2112 case MSVCIntrin::_InterlockedXor:
2113 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E);
2114 case MSVCIntrin::_InterlockedExchangeAdd_acq:
2115 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
2116 AtomicOrdering::Acquire);
2117 case MSVCIntrin::_InterlockedExchangeAdd_rel:
2118 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
2119 AtomicOrdering::Release);
2120 case MSVCIntrin::_InterlockedExchangeAdd_nf:
2121 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
2122 AtomicOrdering::Monotonic);
2123 case MSVCIntrin::_InterlockedExchange_acq:
2124 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
2125 AtomicOrdering::Acquire);
2126 case MSVCIntrin::_InterlockedExchange_rel:
2127 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
2128 AtomicOrdering::Release);
2129 case MSVCIntrin::_InterlockedExchange_nf:
2130 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
2131 AtomicOrdering::Monotonic);
2132 case MSVCIntrin::_InterlockedCompareExchange:
2133 return EmitAtomicCmpXchgForMSIntrin(*this, E);
2134 case MSVCIntrin::_InterlockedCompareExchange_acq:
2135 return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Acquire);
2136 case MSVCIntrin::_InterlockedCompareExchange_rel:
2137 return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Release);
2138 case MSVCIntrin::_InterlockedCompareExchange_nf:
2139 return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Monotonic);
2140 case MSVCIntrin::_InterlockedCompareExchange128:
2142 *this, E, AtomicOrdering::SequentiallyConsistent);
2143 case MSVCIntrin::_InterlockedCompareExchange128_acq:
2144 return EmitAtomicCmpXchg128ForMSIntrin(*this, E, AtomicOrdering::Acquire);
2145 case MSVCIntrin::_InterlockedCompareExchange128_rel:
2146 return EmitAtomicCmpXchg128ForMSIntrin(*this, E, AtomicOrdering::Release);
2147 case MSVCIntrin::_InterlockedCompareExchange128_nf:
2148 return EmitAtomicCmpXchg128ForMSIntrin(*this, E, AtomicOrdering::Monotonic);
2149 case MSVCIntrin::_InterlockedOr_acq:
2150 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
2151 AtomicOrdering::Acquire);
2152 case MSVCIntrin::_InterlockedOr_rel:
2153 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
2154 AtomicOrdering::Release);
2155 case MSVCIntrin::_InterlockedOr_nf:
2156 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
2157 AtomicOrdering::Monotonic);
2158 case MSVCIntrin::_InterlockedXor_acq:
2159 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
2160 AtomicOrdering::Acquire);
2161 case MSVCIntrin::_InterlockedXor_rel:
2162 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
2163 AtomicOrdering::Release);
2164 case MSVCIntrin::_InterlockedXor_nf:
2165 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
2166 AtomicOrdering::Monotonic);
2167 case MSVCIntrin::_InterlockedAnd_acq:
2168 return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
2169 AtomicOrdering::Acquire);
2170 case MSVCIntrin::_InterlockedAnd_rel:
2171 return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
2172 AtomicOrdering::Release);
2173 case MSVCIntrin::_InterlockedAnd_nf:
2174 return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
2175 AtomicOrdering::Monotonic);
2176 case MSVCIntrin::_InterlockedIncrement_acq:
2177 return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Acquire);
2178 case MSVCIntrin::_InterlockedIncrement_rel:
2179 return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Release);
2180 case MSVCIntrin::_InterlockedIncrement_nf:
2181 return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Monotonic);
2182 case MSVCIntrin::_InterlockedDecrement_acq:
2183 return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Acquire);
2184 case MSVCIntrin::_InterlockedDecrement_rel:
2185 return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Release);
2186 case MSVCIntrin::_InterlockedDecrement_nf:
2187 return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Monotonic);
2188
2189 case MSVCIntrin::_InterlockedDecrement:
2190 return EmitAtomicDecrementValue(*this, E);
2191 case MSVCIntrin::_InterlockedIncrement:
2192 return EmitAtomicIncrementValue(*this, E);
2193
2194 case MSVCIntrin::__fastfail: {
2195 // Request immediate process termination from the kernel. The instruction
2196 // sequences to do this are documented on MSDN:
2197 // https://msdn.microsoft.com/en-us/library/dn774154.aspx
2198 llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
2199 StringRef Asm, Constraints;
2200 switch (ISA) {
2201 default:
2202 ErrorUnsupported(E, "__fastfail call for this architecture");
2203 break;
2204 case llvm::Triple::x86:
2205 case llvm::Triple::x86_64:
2206 Asm = "int $$0x29";
2207 Constraints = "{cx}";
2208 break;
2209 case llvm::Triple::thumb:
2210 Asm = "udf #251";
2211 Constraints = "{r0}";
2212 break;
2213 case llvm::Triple::aarch64:
2214 Asm = "brk #0xF003";
2215 Constraints = "{w0}";
2216 }
2217 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, {Int32Ty}, false);
2218 llvm::InlineAsm *IA =
2219 llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);
2220 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
2221 getLLVMContext(), llvm::AttributeList::FunctionIndex,
2222 llvm::Attribute::NoReturn);
2223 llvm::CallInst *CI = Builder.CreateCall(IA, EmitScalarExpr(E->getArg(0)));
2224 CI->setAttributes(NoReturnAttr);
2225 return CI;
2226 }
2227 }
2228 llvm_unreachable("Incorrect MSVC intrinsic!");
2229}
2230
2231namespace {
2232// ARC cleanup for __builtin_os_log_format
2233struct CallObjCArcUse final : EHScopeStack::Cleanup {
2234 CallObjCArcUse(llvm::Value *object) : object(object) {}
2235 llvm::Value *object;
2236
2237 void Emit(CodeGenFunction &CGF, Flags flags) override {
2238 CGF.EmitARCIntrinsicUse(object);
2239 }
2240};
2241}
2242
2244 BuiltinCheckKind Kind) {
2245 assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero) &&
2246 "Unsupported builtin check kind");
2247
2248 Value *ArgValue = EmitScalarExpr(E);
2249 if (!SanOpts.has(SanitizerKind::Builtin))
2250 return ArgValue;
2251
2252 SanitizerScope SanScope(this);
2253 Value *Cond = Builder.CreateICmpNE(
2254 ArgValue, llvm::Constant::getNullValue(ArgValue->getType()));
2255 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
2256 SanitizerHandler::InvalidBuiltin,
2258 llvm::ConstantInt::get(Builder.getInt8Ty(), Kind)},
2259 {});
2260 return ArgValue;
2261}
2262
2264 Value *ArgValue = EvaluateExprAsBool(E);
2265 if (!SanOpts.has(SanitizerKind::Builtin))
2266 return ArgValue;
2267
2268 SanitizerScope SanScope(this);
2269 EmitCheck(
2270 std::make_pair(ArgValue, SanitizerKind::Builtin),
2271 SanitizerHandler::InvalidBuiltin,
2273 llvm::ConstantInt::get(Builder.getInt8Ty(), BCK_AssumePassedFalse)},
2274 std::nullopt);
2275 return ArgValue;
2276}
2277
2278static Value *EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW) {
2279 return CGF.Builder.CreateBinaryIntrinsic(
2280 Intrinsic::abs, ArgValue,
2281 ConstantInt::get(CGF.Builder.getInt1Ty(), HasNSW));
2282}
2283
2285 bool SanitizeOverflow) {
2286 Value *ArgValue = CGF.EmitScalarExpr(E->getArg(0));
2287
2288 // Try to eliminate overflow check.
2289 if (const auto *VCI = dyn_cast<llvm::ConstantInt>(ArgValue)) {
2290 if (!VCI->isMinSignedValue())
2291 return EmitAbs(CGF, ArgValue, true);
2292 }
2293
2294 CodeGenFunction::SanitizerScope SanScope(&CGF);
2295
2296 Constant *Zero = Constant::getNullValue(ArgValue->getType());
2297 Value *ResultAndOverflow = CGF.Builder.CreateBinaryIntrinsic(
2298 Intrinsic::ssub_with_overflow, Zero, ArgValue);
2299 Value *Result = CGF.Builder.CreateExtractValue(ResultAndOverflow, 0);
2300 Value *NotOverflow = CGF.Builder.CreateNot(
2301 CGF.Builder.CreateExtractValue(ResultAndOverflow, 1));
2302
2303 // TODO: support -ftrapv-handler.
2304 if (SanitizeOverflow) {
2305 CGF.EmitCheck({{NotOverflow, SanitizerKind::SignedIntegerOverflow}},
2306 SanitizerHandler::NegateOverflow,
2307 {CGF.EmitCheckSourceLocation(E->getArg(0)->getExprLoc()),
2309 {ArgValue});
2310 } else
2311 CGF.EmitTrapCheck(NotOverflow, SanitizerHandler::SubOverflow);
2312
2313 Value *CmpResult = CGF.Builder.CreateICmpSLT(ArgValue, Zero, "abscond");
2314 return CGF.Builder.CreateSelect(CmpResult, Result, ArgValue, "abs");
2315}
2316
2317/// Get the argument type for arguments to os_log_helper.
2319 QualType UnsignedTy = C.getIntTypeForBitwidth(Size * 8, /*Signed=*/false);
2320 return C.getCanonicalType(UnsignedTy);
2321}
2322
2325 CharUnits BufferAlignment) {
2326 ASTContext &Ctx = getContext();
2327
2329 {
2330 raw_svector_ostream OS(Name);
2331 OS << "__os_log_helper";
2332 OS << "_" << BufferAlignment.getQuantity();
2333 OS << "_" << int(Layout.getSummaryByte());
2334 OS << "_" << int(Layout.getNumArgsByte());
2335 for (const auto &Item : Layout.Items)
2336 OS << "_" << int(Item.getSizeByte()) << "_"
2337 << int(Item.getDescriptorByte());
2338 }
2339
2340 if (llvm::Function *F = CGM.getModule().getFunction(Name))
2341 return F;
2342
2344 FunctionArgList Args;
2345 Args.push_back(ImplicitParamDecl::Create(
2346 Ctx, nullptr, SourceLocation(), &Ctx.Idents.get("buffer"), Ctx.VoidPtrTy,
2348 ArgTys.emplace_back(Ctx.VoidPtrTy);
2349
2350 for (unsigned int I = 0, E = Layout.Items.size(); I < E; ++I) {
2351 char Size = Layout.Items[I].getSizeByte();
2352 if (!Size)
2353 continue;
2354
2355 QualType ArgTy = getOSLogArgType(Ctx, Size);
2356 Args.push_back(ImplicitParamDecl::Create(
2357 Ctx, nullptr, SourceLocation(),
2358 &Ctx.Idents.get(std::string("arg") + llvm::to_string(I)), ArgTy,
2360 ArgTys.emplace_back(ArgTy);
2361 }
2362
2363 QualType ReturnTy = Ctx.VoidTy;
2364
2365 // The helper function has linkonce_odr linkage to enable the linker to merge
2366 // identical functions. To ensure the merging always happens, 'noinline' is
2367 // attached to the function when compiling with -Oz.
2368 const CGFunctionInfo &FI =
2370 llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
2371 llvm::Function *Fn = llvm::Function::Create(
2372 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &CGM.getModule());
2373 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2374 CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, Fn, /*IsThunk=*/false);
2376 Fn->setDoesNotThrow();
2377
2378 // Attach 'noinline' at -Oz.
2379 if (CGM.getCodeGenOpts().OptimizeSize == 2)
2380 Fn->addFnAttr(llvm::Attribute::NoInline);
2381
2382 auto NL = ApplyDebugLocation::CreateEmpty(*this);
2383 StartFunction(GlobalDecl(), ReturnTy, Fn, FI, Args);
2384
2385 // Create a scope with an artificial location for the body of this function.
2386 auto AL = ApplyDebugLocation::CreateArtificial(*this);
2387
2388 CharUnits Offset;
2390 Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Ctx.VoidTy,
2391 BufferAlignment);
2392 Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()),
2393 Builder.CreateConstByteGEP(BufAddr, Offset++, "summary"));
2394 Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()),
2395 Builder.CreateConstByteGEP(BufAddr, Offset++, "numArgs"));
2396
2397 unsigned I = 1;
2398 for (const auto &Item : Layout.Items) {
2400 Builder.getInt8(Item.getDescriptorByte()),
2401 Builder.CreateConstByteGEP(BufAddr, Offset++, "argDescriptor"));
2403 Builder.getInt8(Item.getSizeByte()),
2404 Builder.CreateConstByteGEP(BufAddr, Offset++, "argSize"));
2405
2406 CharUnits Size = Item.size();
2407 if (!Size.getQuantity())
2408 continue;
2409
2410 Address Arg = GetAddrOfLocalVar(Args[I]);
2411 Address Addr = Builder.CreateConstByteGEP(BufAddr, Offset, "argData");
2412 Addr = Addr.withElementType(Arg.getElementType());
2414 Offset += Size;
2415 ++I;
2416 }
2417
2419
2420 return Fn;
2421}
2422
2424 assert(E.getNumArgs() >= 2 &&
2425 "__builtin_os_log_format takes at least 2 arguments");
2426 ASTContext &Ctx = getContext();
2429 Address BufAddr = EmitPointerWithAlignment(E.getArg(0));
2430 llvm::SmallVector<llvm::Value *, 4> RetainableOperands;
2431
2432 // Ignore argument 1, the format string. It is not currently used.
2433 CallArgList Args;
2434 Args.add(RValue::get(BufAddr.emitRawPointer(*this)), Ctx.VoidPtrTy);
2435
2436 for (const auto &Item : Layout.Items) {
2437 int Size = Item.getSizeByte();
2438 if (!Size)
2439 continue;
2440
2441 llvm::Value *ArgVal;
2442
2443 if (Item.getKind() == analyze_os_log::OSLogBufferItem::MaskKind) {
2444 uint64_t Val = 0;
2445 for (unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
2446 Val |= ((uint64_t)Item.getMaskType()[I]) << I * 8;
2447 ArgVal = llvm::Constant::getIntegerValue(Int64Ty, llvm::APInt(64, Val));
2448 } else if (const Expr *TheExpr = Item.getExpr()) {
2449 ArgVal = EmitScalarExpr(TheExpr, /*Ignore*/ false);
2450
2451 // If a temporary object that requires destruction after the full
2452 // expression is passed, push a lifetime-extended cleanup to extend its
2453 // lifetime to the end of the enclosing block scope.
2454 auto LifetimeExtendObject = [&](const Expr *E) {
2455 E = E->IgnoreParenCasts();
2456 // Extend lifetimes of objects returned by function calls and message
2457 // sends.
2458
2459 // FIXME: We should do this in other cases in which temporaries are
2460 // created including arguments of non-ARC types (e.g., C++
2461 // temporaries).
2462 if (isa<CallExpr>(E) || isa<ObjCMessageExpr>(E))
2463 return true;
2464 return false;
2465 };
2466
2467 if (TheExpr->getType()->isObjCRetainableType() &&
2468 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2469 assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
2470 "Only scalar can be a ObjC retainable type");
2471 if (!isa<Constant>(ArgVal)) {
2472 CleanupKind Cleanup = getARCCleanupKind();
2473 QualType Ty = TheExpr->getType();
2475 RawAddress Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca);
2476 ArgVal = EmitARCRetain(Ty, ArgVal);
2477 Builder.CreateStore(ArgVal, Addr);
2478 pushLifetimeExtendedDestroy(Cleanup, Alloca, Ty,
2480 Cleanup & EHCleanup);
2481
2482 // Push a clang.arc.use call to ensure ARC optimizer knows that the
2483 // argument has to be alive.
2484 if (CGM.getCodeGenOpts().OptimizationLevel != 0)
2485 pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
2486 }
2487 }
2488 } else {
2489 ArgVal = Builder.getInt32(Item.getConstValue().getQuantity());
2490 }
2491
2492 unsigned ArgValSize =
2493 CGM.getDataLayout().getTypeSizeInBits(ArgVal->getType());
2494 llvm::IntegerType *IntTy = llvm::Type::getIntNTy(getLLVMContext(),
2495 ArgValSize);
2496 ArgVal = Builder.CreateBitOrPointerCast(ArgVal, IntTy);
2497 CanQualType ArgTy = getOSLogArgType(Ctx, Size);
2498 // If ArgVal has type x86_fp80, zero-extend ArgVal.
2499 ArgVal = Builder.CreateZExtOrBitCast(ArgVal, ConvertType(ArgTy));
2500 Args.add(RValue::get(ArgVal), ArgTy);
2501 }
2502
2503 const CGFunctionInfo &FI =
2506 Layout, BufAddr.getAlignment());
2508 return RValue::get(BufAddr, *this);
2509}
2510
2512 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2513 WidthAndSignedness ResultInfo) {
2514 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2515 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2516 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2517}
2518
2520 CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info,
2521 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2522 const clang::Expr *ResultArg, QualType ResultQTy,
2523 WidthAndSignedness ResultInfo) {
2525 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2526 "Cannot specialize this multiply");
2527
2528 llvm::Value *V1 = CGF.EmitScalarExpr(Op1);
2529 llvm::Value *V2 = CGF.EmitScalarExpr(Op2);
2530
2531 llvm::Value *HasOverflow;
2532 llvm::Value *Result = EmitOverflowIntrinsic(
2533 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2534
2535 // The intrinsic call will detect overflow when the value is > UINT_MAX,
2536 // however, since the original builtin had a signed result, we need to report
2537 // an overflow when the result is greater than INT_MAX.
2538 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
2539 llvm::Value *IntMaxValue = llvm::ConstantInt::get(Result->getType(), IntMax);
2540
2541 llvm::Value *IntMaxOverflow = CGF.Builder.CreateICmpUGT(Result, IntMaxValue);
2542 HasOverflow = CGF.Builder.CreateOr(HasOverflow, IntMaxOverflow);
2543
2544 bool isVolatile =
2545 ResultArg->getType()->getPointeeType().isVolatileQualified();
2546 Address ResultPtr = CGF.EmitPointerWithAlignment(ResultArg);
2547 CGF.Builder.CreateStore(CGF.EmitToMemory(Result, ResultQTy), ResultPtr,
2548 isVolatile);
2549 return RValue::get(HasOverflow);
2550}
2551
2552/// Determine if a binop is a checked mixed-sign multiply we can specialize.
2553static bool isSpecialMixedSignMultiply(unsigned BuiltinID,
2554 WidthAndSignedness Op1Info,
2555 WidthAndSignedness Op2Info,
2556 WidthAndSignedness ResultInfo) {
2557 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2558 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2559 Op1Info.Signed != Op2Info.Signed;
2560}
2561
2562/// Emit a checked mixed-sign multiply. This is a cheaper specialization of
2563/// the generic checked-binop irgen.
2564static RValue
2566 WidthAndSignedness Op1Info, const clang::Expr *Op2,
2567 WidthAndSignedness Op2Info,
2568 const clang::Expr *ResultArg, QualType ResultQTy,
2569 WidthAndSignedness ResultInfo) {
2570 assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info,
2571 Op2Info, ResultInfo) &&
2572 "Not a mixed-sign multipliction we can specialize");
2573
2574 // Emit the signed and unsigned operands.
2575 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2576 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2577 llvm::Value *Signed = CGF.EmitScalarExpr(SignedOp);
2578 llvm::Value *Unsigned = CGF.EmitScalarExpr(UnsignedOp);
2579 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2580 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2581
2582 // One of the operands may be smaller than the other. If so, [s|z]ext it.
2583 if (SignedOpWidth < UnsignedOpWidth)
2584 Signed = CGF.Builder.CreateSExt(Signed, Unsigned->getType(), "op.sext");
2585 if (UnsignedOpWidth < SignedOpWidth)
2586 Unsigned = CGF.Builder.CreateZExt(Unsigned, Signed->getType(), "op.zext");
2587
2588 llvm::Type *OpTy = Signed->getType();
2589 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2590 Address ResultPtr = CGF.EmitPointerWithAlignment(ResultArg);
2591 llvm::Type *ResTy = ResultPtr.getElementType();
2592 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
2593
2594 // Take the absolute value of the signed operand.
2595 llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(Signed, Zero);
2596 llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(Zero, Signed);
2597 llvm::Value *AbsSigned =
2598 CGF.Builder.CreateSelect(IsNegative, AbsOfNegative, Signed);
2599
2600 // Perform a checked unsigned multiplication.
2601 llvm::Value *UnsignedOverflow;
2602 llvm::Value *UnsignedResult =
2603 EmitOverflowIntrinsic(CGF, llvm::Intrinsic::umul_with_overflow, AbsSigned,
2604 Unsigned, UnsignedOverflow);
2605
2606 llvm::Value *Overflow, *Result;
2607 if (ResultInfo.Signed) {
2608 // Signed overflow occurs if the result is greater than INT_MAX or lesser
2609 // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
2610 auto IntMax =
2611 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2612 llvm::Value *MaxResult =
2613 CGF.Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2614 CGF.Builder.CreateZExt(IsNegative, OpTy));
2615 llvm::Value *SignedOverflow =
2616 CGF.Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2617 Overflow = CGF.Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2618
2619 // Prepare the signed result (possibly by negating it).
2620 llvm::Value *NegativeResult = CGF.Builder.CreateNeg(UnsignedResult);
2621 llvm::Value *SignedResult =
2622 CGF.Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2623 Result = CGF.Builder.CreateTrunc(SignedResult, ResTy);
2624 } else {
2625 // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX.
2626 llvm::Value *Underflow = CGF.Builder.CreateAnd(
2627 IsNegative, CGF.Builder.CreateIsNotNull(UnsignedResult));
2628 Overflow = CGF.Builder.CreateOr(UnsignedOverflow, Underflow);
2629 if (ResultInfo.Width < OpWidth) {
2630 auto IntMax =
2631 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2632 llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
2633 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2634 Overflow = CGF.Builder.CreateOr(Overflow, TruncOverflow);
2635 }
2636
2637 // Negate the product if it would be negative in infinite precision.
2638 Result = CGF.Builder.CreateSelect(
2639 IsNegative, CGF.Builder.CreateNeg(UnsignedResult), UnsignedResult);
2640
2641 Result = CGF.Builder.CreateTrunc(Result, ResTy);
2642 }
2643 assert(Overflow && Result && "Missing overflow or result");
2644
2645 bool isVolatile =
2646 ResultArg->getType()->getPointeeType().isVolatileQualified();
2647 CGF.Builder.CreateStore(CGF.EmitToMemory(Result, ResultQTy), ResultPtr,
2648 isVolatile);
2649 return RValue::get(Overflow);
2650}
2651
2652static bool
2654 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2655 if (const auto *Arr = Ctx.getAsArrayType(Ty))
2656 Ty = Ctx.getBaseElementType(Arr);
2657
2658 const auto *Record = Ty->getAsCXXRecordDecl();
2659 if (!Record)
2660 return false;
2661
2662 // We've already checked this type, or are in the process of checking it.
2663 if (!Seen.insert(Record).second)
2664 return false;
2665
2666 assert(Record->hasDefinition() &&
2667 "Incomplete types should already be diagnosed");
2668
2669 if (Record->isDynamicClass())
2670 return true;
2671
2672 for (FieldDecl *F : Record->fields()) {
2673 if (TypeRequiresBuiltinLaunderImp(Ctx, F->getType(), Seen))
2674 return true;
2675 }
2676 return false;
2677}
2678
2679/// Determine if the specified type requires laundering by checking if it is a
2680/// dynamic class type or contains a subobject which is a dynamic class type.
2682 if (!CGM.getCodeGenOpts().StrictVTablePointers)
2683 return false;
2685 return TypeRequiresBuiltinLaunderImp(CGM.getContext(), Ty, Seen);
2686}
2687
2688RValue CodeGenFunction::emitRotate(const CallExpr *E, bool IsRotateRight) {
2689 llvm::Value *Src = EmitScalarExpr(E->getArg(0));
2690 llvm::Value *ShiftAmt = EmitScalarExpr(E->getArg(1));
2691
2692 // The builtin's shift arg may have a different type than the source arg and
2693 // result, but the LLVM intrinsic uses the same type for all values.
2694 llvm::Type *Ty = Src->getType();
2695 ShiftAmt = Builder.CreateIntCast(ShiftAmt, Ty, false);
2696
2697 // Rotate is a special case of LLVM funnel shift - 1st 2 args are the same.
2698 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2699 Function *F = CGM.getIntrinsic(IID, Ty);
2700 return RValue::get(Builder.CreateCall(F, { Src, Src, ShiftAmt }));
2701}
2702
2703// Map math builtins for long-double to f128 version.
2704static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID) {
2705 switch (BuiltinID) {
2706#define MUTATE_LDBL(func) \
2707 case Builtin::BI__builtin_##func##l: \
2708 return Builtin::BI__builtin_##func##f128;
2739 MUTATE_LDBL(nans)
2740 MUTATE_LDBL(inf)
2759 MUTATE_LDBL(huge_val)
2769#undef MUTATE_LDBL
2770 default:
2771 return BuiltinID;
2772 }
2773}
2774
2775static Value *tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID,
2776 Value *V) {
2777 if (CGF.Builder.getIsFPConstrained() &&
2778 CGF.Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2779 if (Value *Result =
2780 CGF.getTargetHooks().testFPKind(V, BuiltinID, CGF.Builder, CGF.CGM))
2781 return Result;
2782 }
2783 return nullptr;
2784}
2785
2787 const FunctionDecl *FD) {
2788 auto Name = FD->getNameAsString() + "__hipstdpar_unsupported";
2789 auto FnTy = CGF->CGM.getTypes().GetFunctionType(FD);
2790 auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, FnTy);
2791
2793 for (auto &&FormalTy : FnTy->params())
2794 Args.push_back(llvm::PoisonValue::get(FormalTy));
2795
2796 return RValue::get(CGF->Builder.CreateCall(UBF, Args));
2797}
2798
2799RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
2800 const CallExpr *E,
2801 ReturnValueSlot ReturnValue) {
2802 assert(!getContext().BuiltinInfo.isImmediate(BuiltinID) &&
2803 "Should not codegen for consteval builtins");
2804
2805 const FunctionDecl *FD = GD.getDecl()->getAsFunction();
2806 // See if we can constant fold this builtin. If so, don't emit it at all.
2807 // TODO: Extend this handling to all builtin calls that we can constant-fold.
2810 !Result.hasSideEffects()) {
2811 if (Result.Val.isInt())
2812 return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
2813 Result.Val.getInt()));
2814 if (Result.Val.isFloat())
2815 return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
2816 Result.Val.getFloat()));
2817 }
2818
2819 // If current long-double semantics is IEEE 128-bit, replace math builtins
2820 // of long-double with f128 equivalent.
2821 // TODO: This mutation should also be applied to other targets other than PPC,
2822 // after backend supports IEEE 128-bit style libcalls.
2823 if (getTarget().getTriple().isPPC64() &&
2824 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2825 BuiltinID = mutateLongDoubleBuiltin(BuiltinID);
2826
2827 // If the builtin has been declared explicitly with an assembler label,
2828 // disable the specialized emitting below. Ideally we should communicate the
2829 // rename in IR, or at least avoid generating the intrinsic calls that are
2830 // likely to get lowered to the renamed library functions.
2831 const unsigned BuiltinIDIfNoAsmLabel =
2832 FD->hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2833
2834 std::optional<bool> ErrnoOverriden;
2835 // ErrnoOverriden is true if math-errno is overriden via the
2836 // '#pragma float_control(precise, on)'. This pragma disables fast-math,
2837 // which implies math-errno.
2838 if (E->hasStoredFPFeatures()) {
2839 FPOptionsOverride OP = E->getFPFeatures();
2840 if (OP.hasMathErrnoOverride())
2841 ErrnoOverriden = OP.getMathErrnoOverride();
2842 }
2843 // True if 'attribute__((optnone))' is used. This attribute overrides
2844 // fast-math which implies math-errno.
2845 bool OptNone = CurFuncDecl && CurFuncDecl->hasAttr<OptimizeNoneAttr>();
2846
2847 // True if we are compiling at -O2 and errno has been disabled
2848 // using the '#pragma float_control(precise, off)', and
2849 // attribute opt-none hasn't been seen.
2850 bool ErrnoOverridenToFalseWithOpt =
2851 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2852 CGM.getCodeGenOpts().OptimizationLevel != 0;
2853
2854 // There are LLVM math intrinsics/instructions corresponding to math library
2855 // functions except the LLVM op will never set errno while the math library
2856 // might. Also, math builtins have the same semantics as their math library
2857 // twins. Thus, we can transform math library and builtin calls to their
2858 // LLVM counterparts if the call is marked 'const' (known to never set errno).
2859 // In case FP exceptions are enabled, the experimental versions of the
2860 // intrinsics model those.
2861 bool ConstAlways =
2862 getContext().BuiltinInfo.isConst(BuiltinID);
2863
2864 // There's a special case with the fma builtins where they are always const
2865 // if the target environment is GNU or the target is OS is Windows and we're
2866 // targeting the MSVCRT.dll environment.
2867 // FIXME: This list can be become outdated. Need to find a way to get it some
2868 // other way.
2869 switch (BuiltinID) {
2870 case Builtin::BI__builtin_fma:
2871 case Builtin::BI__builtin_fmaf:
2872 case Builtin::BI__builtin_fmal:
2873 case Builtin::BI__builtin_fmaf16:
2874 case Builtin::BIfma:
2875 case Builtin::BIfmaf:
2876 case Builtin::BIfmal: {
2877 auto &Trip = CGM.getTriple();
2878 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2879 ConstAlways = true;
2880 break;
2881 }
2882 default:
2883 break;
2884 }
2885
2886 bool ConstWithoutErrnoAndExceptions =
2888 bool ConstWithoutExceptions =
2890
2891 // ConstAttr is enabled in fast-math mode. In fast-math mode, math-errno is
2892 // disabled.
2893 // Math intrinsics are generated only when math-errno is disabled. Any pragmas
2894 // or attributes that affect math-errno should prevent or allow math
2895 // intrincs to be generated. Intrinsics are generated:
2896 // 1- In fast math mode, unless math-errno is overriden
2897 // via '#pragma float_control(precise, on)', or via an
2898 // 'attribute__((optnone))'.
2899 // 2- If math-errno was enabled on command line but overriden
2900 // to false via '#pragma float_control(precise, off))' and
2901 // 'attribute__((optnone))' hasn't been used.
2902 // 3- If we are compiling with optimization and errno has been disabled
2903 // via '#pragma float_control(precise, off)', and
2904 // 'attribute__((optnone))' hasn't been used.
2905
2906 bool ConstWithoutErrnoOrExceptions =
2907 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2908 bool GenerateIntrinsics =
2909 (ConstAlways && !OptNone) ||
2910 (!getLangOpts().MathErrno &&
2911 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2912 if (!GenerateIntrinsics) {
2913 GenerateIntrinsics =
2914 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2915 if (!GenerateIntrinsics)
2916 GenerateIntrinsics =
2917 ConstWithoutErrnoOrExceptions &&
2918 (!getLangOpts().MathErrno &&
2919 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2920 if (!GenerateIntrinsics)
2921 GenerateIntrinsics =
2922 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2923 }
2924 if (GenerateIntrinsics) {
2925 switch (BuiltinIDIfNoAsmLabel) {
2926 case Builtin::BIacos:
2927 case Builtin::BIacosf:
2928 case Builtin::BIacosl:
2929 case Builtin::BI__builtin_acos:
2930 case Builtin::BI__builtin_acosf:
2931 case Builtin::BI__builtin_acosf16:
2932 case Builtin::BI__builtin_acosl:
2933 case Builtin::BI__builtin_acosf128:
2935 *this, E, Intrinsic::acos, Intrinsic::experimental_constrained_acos));
2936
2937 case Builtin::BIasin:
2938 case Builtin::BIasinf:
2939 case Builtin::BIasinl:
2940 case Builtin::BI__builtin_asin:
2941 case Builtin::BI__builtin_asinf:
2942 case Builtin::BI__builtin_asinf16:
2943 case Builtin::BI__builtin_asinl:
2944 case Builtin::BI__builtin_asinf128:
2946 *this, E, Intrinsic::asin, Intrinsic::experimental_constrained_asin));
2947
2948 case Builtin::BIatan:
2949 case Builtin::BIatanf:
2950 case Builtin::BIatanl:
2951 case Builtin::BI__builtin_atan:
2952 case Builtin::BI__builtin_atanf:
2953 case Builtin::BI__builtin_atanf16:
2954 case Builtin::BI__builtin_atanl:
2955 case Builtin::BI__builtin_atanf128:
2957 *this, E, Intrinsic::atan, Intrinsic::experimental_constrained_atan));
2958
2959 case Builtin::BIatan2:
2960 case Builtin::BIatan2f:
2961 case Builtin::BIatan2l:
2962 case Builtin::BI__builtin_atan2:
2963 case Builtin::BI__builtin_atan2f:
2964 case Builtin::BI__builtin_atan2f16:
2965 case Builtin::BI__builtin_atan2l:
2966 case Builtin::BI__builtin_atan2f128:
2968 *this, E, Intrinsic::atan2,
2969 Intrinsic::experimental_constrained_atan2));
2970
2971 case Builtin::BIceil:
2972 case Builtin::BIceilf:
2973 case Builtin::BIceill:
2974 case Builtin::BI__builtin_ceil:
2975 case Builtin::BI__builtin_ceilf:
2976 case Builtin::BI__builtin_ceilf16:
2977 case Builtin::BI__builtin_ceill:
2978 case Builtin::BI__builtin_ceilf128:
2980 Intrinsic::ceil,
2981 Intrinsic::experimental_constrained_ceil));
2982
2983 case Builtin::BIcopysign:
2984 case Builtin::BIcopysignf:
2985 case Builtin::BIcopysignl:
2986 case Builtin::BI__builtin_copysign:
2987 case Builtin::BI__builtin_copysignf:
2988 case Builtin::BI__builtin_copysignf16:
2989 case Builtin::BI__builtin_copysignl:
2990 case Builtin::BI__builtin_copysignf128:
2991 return RValue::get(
2992 emitBuiltinWithOneOverloadedType<2>(*this, E, Intrinsic::copysign));
2993
2994 case Builtin::BIcos:
2995 case Builtin::BIcosf:
2996 case Builtin::BIcosl:
2997 case Builtin::BI__builtin_cos:
2998 case Builtin::BI__builtin_cosf:
2999 case Builtin::BI__builtin_cosf16:
3000 case Builtin::BI__builtin_cosl:
3001 case Builtin::BI__builtin_cosf128:
3003 Intrinsic::cos,
3004 Intrinsic::experimental_constrained_cos));
3005
3006 case Builtin::BIcosh:
3007 case Builtin::BIcoshf:
3008 case Builtin::BIcoshl:
3009 case Builtin::BI__builtin_cosh:
3010 case Builtin::BI__builtin_coshf:
3011 case Builtin::BI__builtin_coshf16:
3012 case Builtin::BI__builtin_coshl:
3013 case Builtin::BI__builtin_coshf128:
3015 *this, E, Intrinsic::cosh, Intrinsic::experimental_constrained_cosh));
3016
3017 case Builtin::BIexp:
3018 case Builtin::BIexpf:
3019 case Builtin::BIexpl:
3020 case Builtin::BI__builtin_exp:
3021 case Builtin::BI__builtin_expf:
3022 case Builtin::BI__builtin_expf16:
3023 case Builtin::BI__builtin_expl:
3024 case Builtin::BI__builtin_expf128:
3026 Intrinsic::exp,
3027 Intrinsic::experimental_constrained_exp));
3028
3029 case Builtin::BIexp2:
3030 case Builtin::BIexp2f:
3031 case Builtin::BIexp2l:
3032 case Builtin::BI__builtin_exp2:
3033 case Builtin::BI__builtin_exp2f:
3034 case Builtin::BI__builtin_exp2f16:
3035 case Builtin::BI__builtin_exp2l:
3036 case Builtin::BI__builtin_exp2f128:
3038 Intrinsic::exp2,
3039 Intrinsic::experimental_constrained_exp2));
3040 case Builtin::BI__builtin_exp10:
3041 case Builtin::BI__builtin_exp10f:
3042 case Builtin::BI__builtin_exp10f16:
3043 case Builtin::BI__builtin_exp10l:
3044 case Builtin::BI__builtin_exp10f128: {
3045 // TODO: strictfp support
3046 if (Builder.getIsFPConstrained())
3047 break;
3048 return RValue::get(
3049 emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::exp10));
3050 }
3051 case Builtin::BIfabs:
3052 case Builtin::BIfabsf:
3053 case Builtin::BIfabsl:
3054 case Builtin::BI__builtin_fabs:
3055 case Builtin::BI__builtin_fabsf:
3056 case Builtin::BI__builtin_fabsf16:
3057 case Builtin::BI__builtin_fabsl:
3058 case Builtin::BI__builtin_fabsf128:
3059 return RValue::get(
3060 emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::fabs));
3061
3062 case Builtin::BIfloor:
3063 case Builtin::BIfloorf:
3064 case Builtin::BIfloorl:
3065 case Builtin::BI__builtin_floor:
3066 case Builtin::BI__builtin_floorf:
3067 case Builtin::BI__builtin_floorf16:
3068 case Builtin::BI__builtin_floorl:
3069 case Builtin::BI__builtin_floorf128:
3071 Intrinsic::floor,
3072 Intrinsic::experimental_constrained_floor));
3073
3074 case Builtin::BIfma:
3075 case Builtin::BIfmaf:
3076 case Builtin::BIfmal:
3077 case Builtin::BI__builtin_fma:
3078 case Builtin::BI__builtin_fmaf:
3079 case Builtin::BI__builtin_fmaf16:
3080 case Builtin::BI__builtin_fmal:
3081 case Builtin::BI__builtin_fmaf128:
3083 Intrinsic::fma,
3084 Intrinsic::experimental_constrained_fma));
3085
3086 case Builtin::BIfmax:
3087 case Builtin::BIfmaxf:
3088 case Builtin::BIfmaxl:
3089 case Builtin::BI__builtin_fmax:
3090 case Builtin::BI__builtin_fmaxf:
3091 case Builtin::BI__builtin_fmaxf16:
3092 case Builtin::BI__builtin_fmaxl:
3093 case Builtin::BI__builtin_fmaxf128:
3095 Intrinsic::maxnum,
3096 Intrinsic::experimental_constrained_maxnum));
3097
3098 case Builtin::BIfmin:
3099 case Builtin::BIfminf:
3100 case Builtin::BIfminl:
3101 case Builtin::BI__builtin_fmin:
3102 case Builtin::BI__builtin_fminf:
3103 case Builtin::BI__builtin_fminf16:
3104 case Builtin::BI__builtin_fminl:
3105 case Builtin::BI__builtin_fminf128:
3107 Intrinsic::minnum,
3108 Intrinsic::experimental_constrained_minnum));
3109
3110 case Builtin::BIfmaximum_num:
3111 case Builtin::BIfmaximum_numf:
3112 case Builtin::BIfmaximum_numl:
3113 case Builtin::BI__builtin_fmaximum_num:
3114 case Builtin::BI__builtin_fmaximum_numf:
3115 case Builtin::BI__builtin_fmaximum_numf16:
3116 case Builtin::BI__builtin_fmaximum_numl:
3117 case Builtin::BI__builtin_fmaximum_numf128:
3118 return RValue::get(
3119 emitBuiltinWithOneOverloadedType<2>(*this, E, Intrinsic::maximumnum));
3120
3121 case Builtin::BIfminimum_num:
3122 case Builtin::BIfminimum_numf:
3123 case Builtin::BIfminimum_numl:
3124 case Builtin::BI__builtin_fminimum_num:
3125 case Builtin::BI__builtin_fminimum_numf:
3126 case Builtin::BI__builtin_fminimum_numf16:
3127 case Builtin::BI__builtin_fminimum_numl:
3128 case Builtin::BI__builtin_fminimum_numf128:
3129 return RValue::get(
3130 emitBuiltinWithOneOverloadedType<2>(*this, E, Intrinsic::minimumnum));
3131
3132 // fmod() is a special-case. It maps to the frem instruction rather than an
3133 // LLVM intrinsic.
3134 case Builtin::BIfmod:
3135 case Builtin::BIfmodf:
3136 case Builtin::BIfmodl:
3137 case Builtin::BI__builtin_fmod:
3138 case Builtin::BI__builtin_fmodf:
3139 case Builtin::BI__builtin_fmodf16:
3140 case Builtin::BI__builtin_fmodl:
3141 case Builtin::BI__builtin_fmodf128:
3142 case Builtin::BI__builtin_elementwise_fmod: {
3143 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3144 Value *Arg1 = EmitScalarExpr(E->getArg(0));
3145 Value *Arg2 = EmitScalarExpr(E->getArg(1));
3146 return RValue::get(Builder.CreateFRem(Arg1, Arg2, "fmod"));
3147 }
3148
3149 case Builtin::BIlog:
3150 case Builtin::BIlogf:
3151 case Builtin::BIlogl:
3152 case Builtin::BI__builtin_log:
3153 case Builtin::BI__builtin_logf:
3154 case Builtin::BI__builtin_logf16:
3155 case Builtin::BI__builtin_logl:
3156 case Builtin::BI__builtin_logf128:
3158 Intrinsic::log,
3159 Intrinsic::experimental_constrained_log));
3160
3161 case Builtin::BIlog10:
3162 case Builtin::BIlog10f:
3163 case Builtin::BIlog10l:
3164 case Builtin::BI__builtin_log10:
3165 case Builtin::BI__builtin_log10f:
3166 case Builtin::BI__builtin_log10f16:
3167 case Builtin::BI__builtin_log10l:
3168 case Builtin::BI__builtin_log10f128:
3170 Intrinsic::log10,
3171 Intrinsic::experimental_constrained_log10));
3172
3173 case Builtin::BIlog2:
3174 case Builtin::BIlog2f:
3175 case Builtin::BIlog2l:
3176 case Builtin::BI__builtin_log2:
3177 case Builtin::BI__builtin_log2f:
3178 case Builtin::BI__builtin_log2f16:
3179 case Builtin::BI__builtin_log2l:
3180 case Builtin::BI__builtin_log2f128:
3182 Intrinsic::log2,
3183 Intrinsic::experimental_constrained_log2));
3184
3185 case Builtin::BInearbyint:
3186 case Builtin::BInearbyintf:
3187 case Builtin::BInearbyintl:
3188 case Builtin::BI__builtin_nearbyint:
3189 case Builtin::BI__builtin_nearbyintf:
3190 case Builtin::BI__builtin_nearbyintl:
3191 case Builtin::BI__builtin_nearbyintf128:
3193 Intrinsic::nearbyint,
3194 Intrinsic::experimental_constrained_nearbyint));
3195
3196 case Builtin::BIpow:
3197 case Builtin::BIpowf:
3198 case Builtin::BIpowl:
3199 case Builtin::BI__builtin_pow:
3200 case Builtin::BI__builtin_powf:
3201 case Builtin::BI__builtin_powf16:
3202 case Builtin::BI__builtin_powl:
3203 case Builtin::BI__builtin_powf128:
3205 Intrinsic::pow,
3206 Intrinsic::experimental_constrained_pow));
3207
3208 case Builtin::BIrint:
3209 case Builtin::BIrintf:
3210 case Builtin::BIrintl:
3211 case Builtin::BI__builtin_rint:
3212 case Builtin::BI__builtin_rintf:
3213 case Builtin::BI__builtin_rintf16:
3214 case Builtin::BI__builtin_rintl:
3215 case Builtin::BI__builtin_rintf128:
3217 Intrinsic::rint,
3218 Intrinsic::experimental_constrained_rint));
3219
3220 case Builtin::BIround:
3221 case Builtin::BIroundf:
3222 case Builtin::BIroundl:
3223 case Builtin::BI__builtin_round:
3224 case Builtin::BI__builtin_roundf:
3225 case Builtin::BI__builtin_roundf16:
3226 case Builtin::BI__builtin_roundl:
3227 case Builtin::BI__builtin_roundf128:
3229 Intrinsic::round,
3230 Intrinsic::experimental_constrained_round));
3231
3232 case Builtin::BIroundeven:
3233 case Builtin::BIroundevenf:
3234 case Builtin::BIroundevenl:
3235 case Builtin::BI__builtin_roundeven:
3236 case Builtin::BI__builtin_roundevenf:
3237 case Builtin::BI__builtin_roundevenf16:
3238 case Builtin::BI__builtin_roundevenl:
3239 case Builtin::BI__builtin_roundevenf128:
3241 Intrinsic::roundeven,
3242 Intrinsic::experimental_constrained_roundeven));
3243
3244 case Builtin::BIsin:
3245 case Builtin::BIsinf:
3246 case Builtin::BIsinl:
3247 case Builtin::BI__builtin_sin:
3248 case Builtin::BI__builtin_sinf:
3249 case Builtin::BI__builtin_sinf16:
3250 case Builtin::BI__builtin_sinl:
3251 case Builtin::BI__builtin_sinf128:
3253 Intrinsic::sin,
3254 Intrinsic::experimental_constrained_sin));
3255
3256 case Builtin::BIsinh:
3257 case Builtin::BIsinhf:
3258 case Builtin::BIsinhl:
3259 case Builtin::BI__builtin_sinh:
3260 case Builtin::BI__builtin_sinhf:
3261 case Builtin::BI__builtin_sinhf16:
3262 case Builtin::BI__builtin_sinhl:
3263 case Builtin::BI__builtin_sinhf128:
3265 *this, E, Intrinsic::sinh, Intrinsic::experimental_constrained_sinh));
3266
3267 case Builtin::BI__builtin_sincos:
3268 case Builtin::BI__builtin_sincosf:
3269 case Builtin::BI__builtin_sincosf16:
3270 case Builtin::BI__builtin_sincosl:
3271 case Builtin::BI__builtin_sincosf128:
3272 emitSincosBuiltin(*this, E, Intrinsic::sincos);
3273 return RValue::get(nullptr);
3274
3275 case Builtin::BIsqrt:
3276 case Builtin::BIsqrtf:
3277 case Builtin::BIsqrtl:
3278 case Builtin::BI__builtin_sqrt:
3279 case Builtin::BI__builtin_sqrtf:
3280 case Builtin::BI__builtin_sqrtf16:
3281 case Builtin::BI__builtin_sqrtl:
3282 case Builtin::BI__builtin_sqrtf128:
3283 case Builtin::BI__builtin_elementwise_sqrt: {
3285 *this, E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
3287 return RValue::get(Call);
3288 }
3289
3290 case Builtin::BItan:
3291 case Builtin::BItanf:
3292 case Builtin::BItanl:
3293 case Builtin::BI__builtin_tan:
3294 case Builtin::BI__builtin_tanf:
3295 case Builtin::BI__builtin_tanf16:
3296 case Builtin::BI__builtin_tanl:
3297 case Builtin::BI__builtin_tanf128:
3299 *this, E, Intrinsic::tan, Intrinsic::experimental_constrained_tan));
3300
3301 case Builtin::BItanh:
3302 case Builtin::BItanhf:
3303 case Builtin::BItanhl:
3304 case Builtin::BI__builtin_tanh:
3305 case Builtin::BI__builtin_tanhf:
3306 case Builtin::BI__builtin_tanhf16:
3307 case Builtin::BI__builtin_tanhl:
3308 case Builtin::BI__builtin_tanhf128:
3310 *this, E, Intrinsic::tanh, Intrinsic::experimental_constrained_tanh));
3311
3312 case Builtin::BItrunc:
3313 case Builtin::BItruncf:
3314 case Builtin::BItruncl:
3315 case Builtin::BI__builtin_trunc:
3316 case Builtin::BI__builtin_truncf:
3317 case Builtin::BI__builtin_truncf16:
3318 case Builtin::BI__builtin_truncl:
3319 case Builtin::BI__builtin_truncf128:
3321 Intrinsic::trunc,
3322 Intrinsic::experimental_constrained_trunc));
3323
3324 case Builtin::BIlround:
3325 case Builtin::BIlroundf:
3326 case Builtin::BIlroundl:
3327 case Builtin::BI__builtin_lround:
3328 case Builtin::BI__builtin_lroundf:
3329 case Builtin::BI__builtin_lroundl:
3330 case Builtin::BI__builtin_lroundf128:
3332 *this, E, Intrinsic::lround,
3333 Intrinsic::experimental_constrained_lround));
3334
3335 case Builtin::BIllround:
3336 case Builtin::BIllroundf:
3337 case Builtin::BIllroundl:
3338 case Builtin::BI__builtin_llround:
3339 case Builtin::BI__builtin_llroundf:
3340 case Builtin::BI__builtin_llroundl:
3341 case Builtin::BI__builtin_llroundf128:
3343 *this, E, Intrinsic::llround,
3344 Intrinsic::experimental_constrained_llround));
3345
3346 case Builtin::BIlrint:
3347 case Builtin::BIlrintf:
3348 case Builtin::BIlrintl:
3349 case Builtin::BI__builtin_lrint:
3350 case Builtin::BI__builtin_lrintf:
3351 case Builtin::BI__builtin_lrintl:
3352 case Builtin::BI__builtin_lrintf128:
3354 *this, E, Intrinsic::lrint,
3355 Intrinsic::experimental_constrained_lrint));
3356
3357 case Builtin::BIllrint:
3358 case Builtin::BIllrintf:
3359 case Builtin::BIllrintl:
3360 case Builtin::BI__builtin_llrint:
3361 case Builtin::BI__builtin_llrintf:
3362 case Builtin::BI__builtin_llrintl:
3363 case Builtin::BI__builtin_llrintf128:
3365 *this, E, Intrinsic::llrint,
3366 Intrinsic::experimental_constrained_llrint));
3367 case Builtin::BI__builtin_ldexp:
3368 case Builtin::BI__builtin_ldexpf:
3369 case Builtin::BI__builtin_ldexpl:
3370 case Builtin::BI__builtin_ldexpf16:
3371 case Builtin::BI__builtin_ldexpf128: {
3373 *this, E, Intrinsic::ldexp,
3374 Intrinsic::experimental_constrained_ldexp));
3375 }
3376 default:
3377 break;
3378 }
3379 }
3380
3381 // Check NonnullAttribute/NullabilityArg and Alignment.
3382 auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg,
3383 unsigned ParmNum) {
3384 Value *Val = A.emitRawPointer(*this);
3385 EmitNonNullArgCheck(RValue::get(Val), Arg->getType(), Arg->getExprLoc(), FD,
3386 ParmNum);
3387
3388 if (SanOpts.has(SanitizerKind::Alignment)) {
3389 SanitizerSet SkippedChecks;
3390 SkippedChecks.set(SanitizerKind::All);
3391 SkippedChecks.clear(SanitizerKind::Alignment);
3392 SourceLocation Loc = Arg->getExprLoc();
3393 // Strip an implicit cast.
3394 if (auto *CE = dyn_cast<ImplicitCastExpr>(Arg))
3395 if (CE->getCastKind() == CK_BitCast)
3396 Arg = CE->getSubExpr();
3397 EmitTypeCheck(Kind, Loc, Val, Arg->getType(), A.getAlignment(),
3398 SkippedChecks);
3399 }
3400 };
3401
3402 switch (BuiltinIDIfNoAsmLabel) {
3403 default: break;
3404 case Builtin::BI__builtin___CFStringMakeConstantString:
3405 case Builtin::BI__builtin___NSStringMakeConstantString:
3406 return RValue::get(ConstantEmitter(*this).emitAbstract(E, E->getType()));
3407 case Builtin::BI__builtin_stdarg_start:
3408 case Builtin::BI__builtin_va_start:
3409 case Builtin::BI__va_start:
3410 case Builtin::BI__builtin_va_end:
3411 EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
3412 ? EmitScalarExpr(E->getArg(0))
3413 : EmitVAListRef(E->getArg(0)).emitRawPointer(*this),
3414 BuiltinID != Builtin::BI__builtin_va_end);
3415 return RValue::get(nullptr);
3416 case Builtin::BI__builtin_va_copy: {
3417 Value *DstPtr = EmitVAListRef(E->getArg(0)).emitRawPointer(*this);
3418 Value *SrcPtr = EmitVAListRef(E->getArg(1)).emitRawPointer(*this);
3419 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy, {DstPtr->getType()}),
3420 {DstPtr, SrcPtr});
3421 return RValue::get(nullptr);
3422 }
3423 case Builtin::BIabs:
3424 case Builtin::BIlabs:
3425 case Builtin::BIllabs:
3426 case Builtin::BI__builtin_abs:
3427 case Builtin::BI__builtin_labs:
3428 case Builtin::BI__builtin_llabs: {
3429 bool SanitizeOverflow = SanOpts.has(SanitizerKind::SignedIntegerOverflow);
3430
3431 Value *Result;
3432 switch (getLangOpts().getSignedOverflowBehavior()) {
3434 Result = EmitAbs(*this, EmitScalarExpr(E->getArg(0)), false);
3435 break;
3437 if (!SanitizeOverflow) {
3438 Result = EmitAbs(*this, EmitScalarExpr(E->getArg(0)), true);
3439 break;
3440 }
3441 [[fallthrough]];
3443 // TODO: Somehow handle the corner case when the address of abs is taken.
3444 Result = EmitOverflowCheckedAbs(*this, E, SanitizeOverflow);
3445 break;
3446 }
3447 return RValue::get(Result);
3448 }
3449 case Builtin::BI__builtin_complex: {
3450 Value *Real = EmitScalarExpr(E->getArg(0));
3451 Value *Imag = EmitScalarExpr(E->getArg(1));
3452 return RValue::getComplex({Real, Imag});
3453 }
3454 case Builtin::BI__builtin_conj:
3455 case Builtin::BI__builtin_conjf:
3456 case Builtin::BI__builtin_conjl:
3457 case Builtin::BIconj:
3458 case Builtin::BIconjf:
3459 case Builtin::BIconjl: {
3460 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3461 Value *Real = ComplexVal.first;
3462 Value *Imag = ComplexVal.second;
3463 Imag = Builder.CreateFNeg(Imag, "neg");
3464 return RValue::getComplex(std::make_pair(Real, Imag));
3465 }
3466 case Builtin::BI__builtin_creal:
3467 case Builtin::BI__builtin_crealf:
3468 case Builtin::BI__builtin_creall:
3469 case Builtin::BIcreal:
3470 case Builtin::BIcrealf:
3471 case Builtin::BIcreall: {
3472 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3473 return RValue::get(ComplexVal.first);
3474 }
3475
3476 case Builtin::BI__builtin_preserve_access_index: {
3477 // Only enabled preserved access index region when debuginfo
3478 // is available as debuginfo is needed to preserve user-level
3479 // access pattern.
3480 if (!getDebugInfo()) {
3481 CGM.Error(E->getExprLoc(), "using builtin_preserve_access_index() without -g");
3482 return RValue::get(EmitScalarExpr(E->getArg(0)));
3483 }
3484
3485 // Nested builtin_preserve_access_index() not supported
3487 CGM.Error(E->getExprLoc(), "nested builtin_preserve_access_index() not supported");
3488 return RValue::get(EmitScalarExpr(E->getArg(0)));
3489 }
3490
3491 IsInPreservedAIRegion = true;
3492 Value *Res = EmitScalarExpr(E->getArg(0));
3493 IsInPreservedAIRegion = false;
3494 return RValue::get(Res);
3495 }
3496
3497 case Builtin::BI__builtin_cimag:
3498 case Builtin::BI__builtin_cimagf:
3499 case Builtin::BI__builtin_cimagl:
3500 case Builtin::BIcimag:
3501 case Builtin::BIcimagf:
3502 case Builtin::BIcimagl: {
3503 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3504 return RValue::get(ComplexVal.second);
3505 }
3506
3507 case Builtin::BI__builtin_clrsb:
3508 case Builtin::BI__builtin_clrsbl:
3509 case Builtin::BI__builtin_clrsbll: {
3510 // clrsb(x) -> clz(x < 0 ? ~x : x) - 1 or
3511 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3512
3513 llvm::Type *ArgType = ArgValue->getType();
3514 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3515
3516 llvm::Type *ResultType = ConvertType(E->getType());
3517 Value *Zero = llvm::Constant::getNullValue(ArgType);
3518 Value *IsNeg = Builder.CreateICmpSLT(ArgValue, Zero, "isneg");
3519 Value *Inverse = Builder.CreateNot(ArgValue, "not");
3520 Value *Tmp = Builder.CreateSelect(IsNeg, Inverse, ArgValue);
3521 Value *Ctlz = Builder.CreateCall(F, {Tmp, Builder.getFalse()});
3522 Value *Result = Builder.CreateSub(Ctlz, llvm::ConstantInt::get(ArgType, 1));
3523 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3524 "cast");
3525 return RValue::get(Result);
3526 }
3527 case Builtin::BI__builtin_ctzs:
3528 case Builtin::BI__builtin_ctz:
3529 case Builtin::BI__builtin_ctzl:
3530 case Builtin::BI__builtin_ctzll:
3531 case Builtin::BI__builtin_ctzg: {
3532 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
3533 E->getNumArgs() > 1;
3534
3535 Value *ArgValue =
3536 HasFallback ? EmitScalarExpr(E->getArg(0))
3538
3539 llvm::Type *ArgType = ArgValue->getType();
3540 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
3541
3542 llvm::Type *ResultType = ConvertType(E->getType());
3543 Value *ZeroUndef =
3544 Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef());
3545 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
3546 if (Result->getType() != ResultType)
3547 Result =
3548 Builder.CreateIntCast(Result, ResultType, /*isSigned*/ false, "cast");
3549 if (!HasFallback)
3550 return RValue::get(Result);
3551
3552 Value *Zero = Constant::getNullValue(ArgType);
3553 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
3554 Value *FallbackValue = EmitScalarExpr(E->getArg(1));
3555 Value *ResultOrFallback =
3556 Builder.CreateSelect(IsZero, FallbackValue, Result, "ctzg");
3557 return RValue::get(ResultOrFallback);
3558 }
3559 case Builtin::BI__builtin_clzs:
3560 case Builtin::BI__builtin_clz:
3561 case Builtin::BI__builtin_clzl:
3562 case Builtin::BI__builtin_clzll:
3563 case Builtin::BI__builtin_clzg: {
3564 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
3565 E->getNumArgs() > 1;
3566
3567 Value *ArgValue =
3568 HasFallback ? EmitScalarExpr(E->getArg(0))
3570
3571 llvm::Type *ArgType = ArgValue->getType();
3572 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3573
3574 llvm::Type *ResultType = ConvertType(E->getType());
3575 Value *ZeroUndef =
3576 Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef());
3577 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
3578 if (Result->getType() != ResultType)
3579 Result =
3580 Builder.CreateIntCast(Result, ResultType, /*isSigned*/ false, "cast");
3581 if (!HasFallback)
3582 return RValue::get(Result);
3583
3584 Value *Zero = Constant::getNullValue(ArgType);
3585 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
3586 Value *FallbackValue = EmitScalarExpr(E->getArg(1));
3587 Value *ResultOrFallback =
3588 Builder.CreateSelect(IsZero, FallbackValue, Result, "clzg");
3589 return RValue::get(ResultOrFallback);
3590 }
3591 case Builtin::BI__builtin_ffs:
3592 case Builtin::BI__builtin_ffsl:
3593 case Builtin::BI__builtin_ffsll: {
3594 // ffs(x) -> x ? cttz(x) + 1 : 0
3595 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3596
3597 llvm::Type *ArgType = ArgValue->getType();
3598 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
3599
3600 llvm::Type *ResultType = ConvertType(E->getType());
3601 Value *Tmp =
3602 Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
3603 llvm::ConstantInt::get(ArgType, 1));
3604 Value *Zero = llvm::Constant::getNullValue(ArgType);
3605 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
3606 Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
3607 if (Result->getType() != ResultType)
3608 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3609 "cast");
3610 return RValue::get(Result);
3611 }
3612 case Builtin::BI__builtin_parity:
3613 case Builtin::BI__builtin_parityl:
3614 case Builtin::BI__builtin_parityll: {
3615 // parity(x) -> ctpop(x) & 1
3616 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3617
3618 llvm::Type *ArgType = ArgValue->getType();
3619 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
3620
3621 llvm::Type *ResultType = ConvertType(E->getType());
3622 Value *Tmp = Builder.CreateCall(F, ArgValue);
3623 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
3624 if (Result->getType() != ResultType)
3625 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3626 "cast");
3627 return RValue::get(Result);
3628 }
3629 case Builtin::BI__lzcnt16:
3630 case Builtin::BI__lzcnt:
3631 case Builtin::BI__lzcnt64: {
3632 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3633
3634 llvm::Type *ArgType = ArgValue->getType();
3635 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3636
3637 llvm::Type *ResultType = ConvertType(E->getType());
3638 Value *Result = Builder.CreateCall(F, {ArgValue, Builder.getFalse()});
3639 if (Result->getType() != ResultType)
3640 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3641 "cast");
3642 return RValue::get(Result);
3643 }
3644 case Builtin::BI__popcnt16:
3645 case Builtin::BI__popcnt:
3646 case Builtin::BI__popcnt64:
3647 case Builtin::BI__builtin_popcount:
3648 case Builtin::BI__builtin_popcountl:
3649 case Builtin::BI__builtin_popcountll:
3650 case Builtin::BI__builtin_popcountg: {
3651 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3652
3653 llvm::Type *ArgType = ArgValue->getType();
3654 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
3655
3656 llvm::Type *ResultType = ConvertType(E->getType());
3657 Value *Result = Builder.CreateCall(F, ArgValue);
3658 if (Result->getType() != ResultType)
3659 Result =
3660 Builder.CreateIntCast(Result, ResultType, /*isSigned*/ false, "cast");
3661 return RValue::get(Result);
3662 }
3663 case Builtin::BI__builtin_unpredictable: {
3664 // Always return the argument of __builtin_unpredictable. LLVM does not
3665 // handle this builtin. Metadata for this builtin should be added directly
3666 // to instructions such as branches or switches that use it.
3667 return RValue::get(EmitScalarExpr(E->getArg(0)));
3668 }
3669 case Builtin::BI__builtin_expect: {
3670 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3671 llvm::Type *ArgType = ArgValue->getType();
3672
3673 Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
3674 // Don't generate llvm.expect on -O0 as the backend won't use it for
3675 // anything.
3676 // Note, we still IRGen ExpectedValue because it could have side-effects.
3677 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3678 return RValue::get(ArgValue);
3679
3680 Function *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
3681 Value *Result =
3682 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval");
3683 return RValue::get(Result);
3684 }
3685 case Builtin::BI__builtin_expect_with_probability: {
3686 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3687 llvm::Type *ArgType = ArgValue->getType();
3688
3689 Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
3690 llvm::APFloat Probability(0.0);
3691 const Expr *ProbArg = E->getArg(2);
3692 bool EvalSucceed = ProbArg->EvaluateAsFloat(Probability, CGM.getContext());
3693 assert(EvalSucceed && "probability should be able to evaluate as float");
3694 (void)EvalSucceed;
3695 bool LoseInfo = false;
3696 Probability.convert(llvm::APFloat::IEEEdouble(),
3697 llvm::RoundingMode::Dynamic, &LoseInfo);
3698 llvm::Type *Ty = ConvertType(ProbArg->getType());
3699 Constant *Confidence = ConstantFP::get(Ty, Probability);
3700 // Don't generate llvm.expect.with.probability on -O0 as the backend
3701 // won't use it for anything.
3702 // Note, we still IRGen ExpectedValue because it could have side-effects.
3703 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3704 return RValue::get(ArgValue);
3705
3706 Function *FnExpect =
3707 CGM.getIntrinsic(Intrinsic::expect_with_probability, ArgType);
3708 Value *Result = Builder.CreateCall(
3709 FnExpect, {ArgValue, ExpectedValue, Confidence}, "expval");
3710 return RValue::get(Result);
3711 }
3712 case Builtin::BI__builtin_assume_aligned: {
3713 const Expr *Ptr = E->getArg(0);
3714 Value *PtrValue = EmitScalarExpr(Ptr);
3715 Value *OffsetValue =
3716 (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr;
3717
3718 Value *AlignmentValue = EmitScalarExpr(E->getArg(1));
3719 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
3720 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
3721 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
3722 llvm::Value::MaximumAlignment);
3723
3724 emitAlignmentAssumption(PtrValue, Ptr,
3725 /*The expr loc is sufficient.*/ SourceLocation(),
3726 AlignmentCI, OffsetValue);
3727 return RValue::get(PtrValue);
3728 }
3729 case Builtin::BI__assume:
3730 case Builtin::BI__builtin_assume: {
3731 if (E->getArg(0)->HasSideEffects(getContext()))
3732 return RValue::get(nullptr);
3733
3734 Value *ArgValue = EmitCheckedArgForAssume(E->getArg(0));
3735 Function *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
3736 Builder.CreateCall(FnAssume, ArgValue);
3737 return RValue::get(nullptr);
3738 }
3739 case Builtin::BI__builtin_assume_separate_storage: {
3740 const Expr *Arg0 = E->getArg(0);
3741 const Expr *Arg1 = E->getArg(1);
3742
3743 Value *Value0 = EmitScalarExpr(Arg0);
3744 Value *Value1 = EmitScalarExpr(Arg1);
3745
3746 Value *Values[] = {Value0, Value1};
3747 OperandBundleDefT<Value *> OBD("separate_storage", Values);
3748 Builder.CreateAssumption(ConstantInt::getTrue(getLLVMContext()), {OBD});
3749 return RValue::get(nullptr);
3750 }
3751 case Builtin::BI__builtin_allow_runtime_check: {
3752 StringRef Kind =
3753 cast<StringLiteral>(E->getArg(0)->IgnoreParenCasts())->getString();
3754 LLVMContext &Ctx = CGM.getLLVMContext();
3755 llvm::Value *Allow = Builder.CreateCall(
3756 CGM.getIntrinsic(llvm::Intrinsic::allow_runtime_check),
3757 llvm::MetadataAsValue::get(Ctx, llvm::MDString::get(Ctx, Kind)));
3758 return RValue::get(Allow);
3759 }
3760 case Builtin::BI__arithmetic_fence: {
3761 // Create the builtin call if FastMath is selected, and the target
3762 // supports the builtin, otherwise just return the argument.
3763 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3764 llvm::FastMathFlags FMF = Builder.getFastMathFlags();
3765 bool isArithmeticFenceEnabled =
3766 FMF.allowReassoc() &&
3768 QualType ArgType = E->getArg(0)->getType();
3769 if (ArgType->isComplexType()) {
3770 if (isArithmeticFenceEnabled) {
3771 QualType ElementType = ArgType->castAs<ComplexType>()->getElementType();
3772 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3773 Value *Real = Builder.CreateArithmeticFence(ComplexVal.first,
3774 ConvertType(ElementType));
3775 Value *Imag = Builder.CreateArithmeticFence(ComplexVal.second,
3776 ConvertType(ElementType));
3777 return RValue::getComplex(std::make_pair(Real, Imag));
3778 }
3779 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3780 Value *Real = ComplexVal.first;
3781 Value *Imag = ComplexVal.second;
3782 return RValue::getComplex(std::make_pair(Real, Imag));
3783 }
3784 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3785 if (isArithmeticFenceEnabled)
3786 return RValue::get(
3787 Builder.CreateArithmeticFence(ArgValue, ConvertType(ArgType)));
3788 return RValue::get(ArgValue);
3789 }
3790 case Builtin::BI__builtin_bswap16:
3791 case Builtin::BI__builtin_bswap32:
3792 case Builtin::BI__builtin_bswap64:
3793 case Builtin::BI_byteswap_ushort:
3794 case Builtin::BI_byteswap_ulong:
3795 case Builtin::BI_byteswap_uint64: {
3796 return RValue::get(
3797 emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::bswap));
3798 }
3799 case Builtin::BI__builtin_bitreverse8:
3800 case Builtin::BI__builtin_bitreverse16:
3801 case Builtin::BI__builtin_bitreverse32:
3802 case Builtin::BI__builtin_bitreverse64: {
3803 return RValue::get(
3804 emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::bitreverse));
3805 }
3806 case Builtin::BI__builtin_rotateleft8:
3807 case Builtin::BI__builtin_rotateleft16:
3808 case Builtin::BI__builtin_rotateleft32:
3809 case Builtin::BI__builtin_rotateleft64:
3810 case Builtin::BI_rotl8: // Microsoft variants of rotate left
3811 case Builtin::BI_rotl16:
3812 case Builtin::BI_rotl:
3813 case Builtin::BI_lrotl:
3814 case Builtin::BI_rotl64:
3815 return emitRotate(E, false);
3816
3817 case Builtin::BI__builtin_rotateright8:
3818 case Builtin::BI__builtin_rotateright16:
3819 case Builtin::BI__builtin_rotateright32:
3820 case Builtin::BI__builtin_rotateright64:
3821 case Builtin::BI_rotr8: // Microsoft variants of rotate right
3822 case Builtin::BI_rotr16:
3823 case Builtin::BI_rotr:
3824 case Builtin::BI_lrotr:
3825 case Builtin::BI_rotr64:
3826 return emitRotate(E, true);
3827
3828 case Builtin::BI__builtin_constant_p: {
3829 llvm::Type *ResultType = ConvertType(E->getType());
3830
3831 const Expr *Arg = E->getArg(0);
3832 QualType ArgType = Arg->getType();
3833 // FIXME: The allowance for Obj-C pointers and block pointers is historical
3834 // and likely a mistake.
3835 if (!ArgType->isIntegralOrEnumerationType() && !ArgType->isFloatingType() &&
3836 !ArgType->isObjCObjectPointerType() && !ArgType->isBlockPointerType())
3837 // Per the GCC documentation, only numeric constants are recognized after
3838 // inlining.
3839 return RValue::get(ConstantInt::get(ResultType, 0));
3840
3841 if (Arg->HasSideEffects(getContext()))
3842 // The argument is unevaluated, so be conservative if it might have
3843 // side-effects.
3844 return RValue::get(ConstantInt::get(ResultType, 0));
3845
3846 Value *ArgValue = EmitScalarExpr(Arg);
3847 if (ArgType->isObjCObjectPointerType()) {
3848 // Convert Objective-C objects to id because we cannot distinguish between
3849 // LLVM types for Obj-C classes as they are opaque.
3850 ArgType = CGM.getContext().getObjCIdType();
3851 ArgValue = Builder.CreateBitCast(ArgValue, ConvertType(ArgType));
3852 }
3853 Function *F =
3854 CGM.getIntrinsic(Intrinsic::is_constant, ConvertType(ArgType));
3855 Value *Result = Builder.CreateCall(F, ArgValue);
3856 if (Result->getType() != ResultType)
3857 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/false);
3858 return RValue::get(Result);
3859 }
3860 case Builtin::BI__builtin_dynamic_object_size:
3861 case Builtin::BI__builtin_object_size: {
3862 unsigned Type =
3863 E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue();
3864 auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType()));
3865
3866 // We pass this builtin onto the optimizer so that it can figure out the
3867 // object size in more complex cases.
3868 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3869 return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType,
3870 /*EmittedE=*/nullptr, IsDynamic));
3871 }
3872 case Builtin::BI__builtin_counted_by_ref: {
3873 // Default to returning '(void *) 0'.
3874 llvm::Value *Result = llvm::ConstantPointerNull::get(
3875 llvm::PointerType::getUnqual(getLLVMContext()));
3876
3877 const Expr *Arg = E->getArg(0)->IgnoreParenImpCasts();
3878
3879 if (auto *UO = dyn_cast<UnaryOperator>(Arg);
3880 UO && UO->getOpcode() == UO_AddrOf) {
3881 Arg = UO->getSubExpr()->IgnoreParenImpCasts();
3882
3883 if (auto *ASE = dyn_cast<ArraySubscriptExpr>(Arg))
3884 Arg = ASE->getBase()->IgnoreParenImpCasts();
3885 }
3886
3887 if (const MemberExpr *ME = dyn_cast_if_present<MemberExpr>(Arg)) {
3888 if (auto *CATy =
3889 ME->getMemberDecl()->getType()->getAs<CountAttributedType>();
3890 CATy && CATy->getKind() == CountAttributedType::CountedBy) {
3891 const auto *FAMDecl = cast<FieldDecl>(ME->getMemberDecl());
3892 if (const FieldDecl *CountFD = FAMDecl->findCountedByField())
3893 Result = GetCountedByFieldExprGEP(Arg, FAMDecl, CountFD);
3894 else
3895 llvm::report_fatal_error("Cannot find the counted_by 'count' field");
3896 }
3897 }
3898
3899 return RValue::get(Result);
3900 }
3901 case Builtin::BI__builtin_prefetch: {
3902 Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
3903 // FIXME: Technically these constants should of type 'int', yes?
3904 RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) :
3905 llvm::ConstantInt::get(Int32Ty, 0);
3906 Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) :
3907 llvm::ConstantInt::get(Int32Ty, 3);
3908 Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
3909 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
3910 Builder.CreateCall(F, {Address, RW, Locality, Data});
3911 return RValue::get(nullptr);
3912 }
3913 case Builtin::BI__builtin_readcyclecounter: {
3914 Function *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
3915 return RValue::get(Builder.CreateCall(F));
3916 }
3917 case Builtin::BI__builtin_readsteadycounter: {
3918 Function *F = CGM.getIntrinsic(Intrinsic::readsteadycounter);
3919 return RValue::get(Builder.CreateCall(F));
3920 }
3921 case Builtin::BI__builtin___clear_cache: {
3922 Value *Begin = EmitScalarExpr(E->getArg(0));
3923 Value *End = EmitScalarExpr(E->getArg(1));
3924 Function *F = CGM.getIntrinsic(Intrinsic::clear_cache);
3925 return RValue::get(Builder.CreateCall(F, {Begin, End}));
3926 }
3927 case Builtin::BI__builtin_trap:
3928 EmitTrapCall(Intrinsic::trap);
3929 return RValue::get(nullptr);
3930 case Builtin::BI__builtin_verbose_trap: {
3931 llvm::DILocation *TrapLocation = Builder.getCurrentDebugLocation();
3932 if (getDebugInfo()) {
3933 TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor(
3934 TrapLocation, *E->getArg(0)->tryEvaluateString(getContext()),
3935 *E->getArg(1)->tryEvaluateString(getContext()));
3936 }
3937 ApplyDebugLocation ApplyTrapDI(*this, TrapLocation);
3938 // Currently no attempt is made to prevent traps from being merged.
3939 EmitTrapCall(Intrinsic::trap);
3940 return RValue::get(nullptr);
3941 }
3942 case Builtin::BI__debugbreak:
3943 EmitTrapCall(Intrinsic::debugtrap);
3944 return RValue::get(nullptr);
3945 case Builtin::BI__builtin_unreachable: {
3947
3948 // We do need to preserve an insertion point.
3949 EmitBlock(createBasicBlock("unreachable.cont"));
3950
3951 return RValue::get(nullptr);
3952 }
3953
3954 case Builtin::BI__builtin_powi:
3955 case Builtin::BI__builtin_powif:
3956 case Builtin::BI__builtin_powil: {
3957 llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
3958 llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
3959
3960 if (Builder.getIsFPConstrained()) {
3961 // FIXME: llvm.powi has 2 mangling types,
3962 // llvm.experimental.constrained.powi has one.
3963 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3964 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_powi,
3965 Src0->getType());
3966 return RValue::get(Builder.CreateConstrainedFPCall(F, { Src0, Src1 }));
3967 }
3968
3969 Function *F = CGM.getIntrinsic(Intrinsic::powi,
3970 { Src0->getType(), Src1->getType() });
3971 return RValue::get(Builder.CreateCall(F, { Src0, Src1 }));
3972 }
3973 case Builtin::BI__builtin_frexpl: {
3974 // Linux PPC will not be adding additional PPCDoubleDouble support.
3975 // WIP to switch default to IEEE long double. Will emit libcall for
3976 // frexpl instead of legalizing this type in the BE.
3977 if (&getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
3978 break;
3979 [[fallthrough]];
3980 }
3981 case Builtin::BI__builtin_frexp:
3982 case Builtin::BI__builtin_frexpf:
3983 case Builtin::BI__builtin_frexpf128:
3984 case Builtin::BI__builtin_frexpf16:
3985 return RValue::get(emitFrexpBuiltin(*this, E, Intrinsic::frexp));
3986 case Builtin::BI__builtin_isgreater:
3987 case Builtin::BI__builtin_isgreaterequal:
3988 case Builtin::BI__builtin_isless:
3989 case Builtin::BI__builtin_islessequal:
3990 case Builtin::BI__builtin_islessgreater:
3991 case Builtin::BI__builtin_isunordered: {
3992 // Ordered comparisons: we know the arguments to these are matching scalar
3993 // floating point values.
3994 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3995 Value *LHS = EmitScalarExpr(E->getArg(0));
3996 Value *RHS = EmitScalarExpr(E->getArg(1));
3997
3998 switch (BuiltinID) {
3999 default: llvm_unreachable("Unknown ordered comparison");
4000 case Builtin::BI__builtin_isgreater:
4001 LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp");
4002 break;
4003 case Builtin::BI__builtin_isgreaterequal:
4004 LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp");
4005 break;
4006 case Builtin::BI__builtin_isless:
4007 LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp");
4008 break;
4009 case Builtin::BI__builtin_islessequal:
4010 LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp");
4011 break;
4012 case Builtin::BI__builtin_islessgreater:
4013 LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp");
4014 break;
4015 case Builtin::BI__builtin_isunordered:
4016 LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp");
4017 break;
4018 }
4019 // ZExt bool to int type.
4020 return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
4021 }
4022
4023 case Builtin::BI__builtin_isnan: {
4024 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4025 Value *V = EmitScalarExpr(E->getArg(0));
4026 if (Value *Result = tryUseTestFPKind(*this, BuiltinID, V))
4027 return RValue::get(Result);
4028 return RValue::get(
4029 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcNan),
4030 ConvertType(E->getType())));
4031 }
4032
4033 case Builtin::BI__builtin_issignaling: {
4034 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4035 Value *V = EmitScalarExpr(E->getArg(0));
4036 return RValue::get(
4037 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcSNan),
4038 ConvertType(E->getType())));
4039 }
4040
4041 case Builtin::BI__builtin_isinf: {
4042 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4043 Value *V = EmitScalarExpr(E->getArg(0));
4044 if (Value *Result = tryUseTestFPKind(*this, BuiltinID, V))
4045 return RValue::get(Result);
4046 return RValue::get(
4047 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcInf),
4048 ConvertType(E->getType())));
4049 }
4050
4051 case Builtin::BIfinite:
4052 case Builtin::BI__finite:
4053 case Builtin::BIfinitef:
4054 case Builtin::BI__finitef:
4055 case Builtin::BIfinitel:
4056 case Builtin::BI__finitel:
4057 case Builtin::BI__builtin_isfinite: {
4058 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4059 Value *V = EmitScalarExpr(E->getArg(0));
4060 if (Value *Result = tryUseTestFPKind(*this, BuiltinID, V))
4061 return RValue::get(Result);
4062 return RValue::get(
4063 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcFinite),
4064 ConvertType(E->getType())));
4065 }
4066
4067 case Builtin::BI__builtin_isnormal: {
4068 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4069 Value *V = EmitScalarExpr(E->getArg(0));
4070 return RValue::get(
4071 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcNormal),
4072 ConvertType(E->getType())));
4073 }
4074
4075 case Builtin::BI__builtin_issubnormal: {
4076 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4077 Value *V = EmitScalarExpr(E->getArg(0));
4078 return RValue::get(
4079 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcSubnormal),
4080 ConvertType(E->getType())));
4081 }
4082
4083 case Builtin::BI__builtin_iszero: {
4084 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4085 Value *V = EmitScalarExpr(E->getArg(0));
4086 return RValue::get(
4087 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcZero),
4088 ConvertType(E->getType())));
4089 }
4090
4091 case Builtin::BI__builtin_isfpclass: {
4093 if (!E->getArg(1)->EvaluateAsInt(Result, CGM.getContext()))
4094 break;
4095 uint64_t Test = Result.Val.getInt().getLimitedValue();
4096 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4097 Value *V = EmitScalarExpr(E->getArg(0));
4098 return RValue::get(Builder.CreateZExt(Builder.createIsFPClass(V, Test),
4099 ConvertType(E->getType())));
4100 }
4101
4102 case Builtin::BI__builtin_nondeterministic_value: {
4103 llvm::Type *Ty = ConvertType(E->getArg(0)->getType());
4104
4105 Value *Result = PoisonValue::get(Ty);
4106 Result = Builder.CreateFreeze(Result);
4107
4108 return RValue::get(Result);
4109 }
4110
4111 case Builtin::BI__builtin_elementwise_abs: {
4112 Value *Result;
4113 QualType QT = E->getArg(0)->getType();
4114
4115 if (auto *VecTy = QT->getAs<VectorType>())
4116 QT = VecTy->getElementType();
4117 if (QT->isIntegerType())
4118 Result = Builder.CreateBinaryIntrinsic(
4119 llvm::Intrinsic::abs, EmitScalarExpr(E->getArg(0)),
4120 Builder.getFalse(), nullptr, "elt.abs");
4121 else
4122 Result = emitBuiltinWithOneOverloadedType<1>(
4123 *this, E, llvm::Intrinsic::fabs, "elt.abs");
4124
4125 return RValue::get(Result);
4126 }
4127 case Builtin::BI__builtin_elementwise_acos:
4128 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4129 *this, E, llvm::Intrinsic::acos, "elt.acos"));
4130 case Builtin::BI__builtin_elementwise_asin:
4131 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4132 *this, E, llvm::Intrinsic::asin, "elt.asin"));
4133 case Builtin::BI__builtin_elementwise_atan:
4134 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4135 *this, E, llvm::Intrinsic::atan, "elt.atan"));
4136 case Builtin::BI__builtin_elementwise_atan2:
4137 return RValue::get(emitBuiltinWithOneOverloadedType<2>(
4138 *this, E, llvm::Intrinsic::atan2, "elt.atan2"));
4139 case Builtin::BI__builtin_elementwise_ceil:
4140 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4141 *this, E, llvm::Intrinsic::ceil, "elt.ceil"));
4142 case Builtin::BI__builtin_elementwise_exp:
4143 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4144 *this, E, llvm::Intrinsic::exp, "elt.exp"));
4145 case Builtin::BI__builtin_elementwise_exp2:
4146 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4147 *this, E, llvm::Intrinsic::exp2, "elt.exp2"));
4148 case Builtin::BI__builtin_elementwise_log:
4149 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4150 *this, E, llvm::Intrinsic::log, "elt.log"));
4151 case Builtin::BI__builtin_elementwise_log2:
4152 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4153 *this, E, llvm::Intrinsic::log2, "elt.log2"));
4154 case Builtin::BI__builtin_elementwise_log10:
4155 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4156 *this, E, llvm::Intrinsic::log10, "elt.log10"));
4157 case Builtin::BI__builtin_elementwise_pow: {
4158 return RValue::get(
4159 emitBuiltinWithOneOverloadedType<2>(*this, E, llvm::Intrinsic::pow));
4160 }
4161 case Builtin::BI__builtin_elementwise_bitreverse:
4162 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4163 *this, E, llvm::Intrinsic::bitreverse, "elt.bitreverse"));
4164 case Builtin::BI__builtin_elementwise_cos:
4165 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4166 *this, E, llvm::Intrinsic::cos, "elt.cos"));
4167 case Builtin::BI__builtin_elementwise_cosh:
4168 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4169 *this, E, llvm::Intrinsic::cosh, "elt.cosh"));
4170 case Builtin::BI__builtin_elementwise_floor:
4171 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4172 *this, E, llvm::Intrinsic::floor, "elt.floor"));
4173 case Builtin::BI__builtin_elementwise_popcount:
4174 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4175 *this, E, llvm::Intrinsic::ctpop, "elt.ctpop"));
4176 case Builtin::BI__builtin_elementwise_roundeven:
4177 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4178 *this, E, llvm::Intrinsic::roundeven, "elt.roundeven"));
4179 case Builtin::BI__builtin_elementwise_round:
4180 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4181 *this, E, llvm::Intrinsic::round, "elt.round"));
4182 case Builtin::BI__builtin_elementwise_rint:
4183 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4184 *this, E, llvm::Intrinsic::rint, "elt.rint"));
4185 case Builtin::BI__builtin_elementwise_nearbyint:
4186 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4187 *this, E, llvm::Intrinsic::nearbyint, "elt.nearbyint"));
4188 case Builtin::BI__builtin_elementwise_sin:
4189 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4190 *this, E, llvm::Intrinsic::sin, "elt.sin"));
4191 case Builtin::BI__builtin_elementwise_sinh:
4192 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4193 *this, E, llvm::Intrinsic::sinh, "elt.sinh"));
4194 case Builtin::BI__builtin_elementwise_tan:
4195 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4196 *this, E, llvm::Intrinsic::tan, "elt.tan"));
4197 case Builtin::BI__builtin_elementwise_tanh:
4198 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4199 *this, E, llvm::Intrinsic::tanh, "elt.tanh"));
4200 case Builtin::BI__builtin_elementwise_trunc:
4201 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4202 *this, E, llvm::Intrinsic::trunc, "elt.trunc"));
4203 case Builtin::BI__builtin_elementwise_canonicalize:
4204 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4205 *this, E, llvm::Intrinsic::canonicalize, "elt.canonicalize"));
4206 case Builtin::BI__builtin_elementwise_copysign:
4207 return RValue::get(emitBuiltinWithOneOverloadedType<2>(
4208 *this, E, llvm::Intrinsic::copysign));
4209 case Builtin::BI__builtin_elementwise_fma:
4210 return RValue::get(
4211 emitBuiltinWithOneOverloadedType<3>(*this, E, llvm::Intrinsic::fma));
4212 case Builtin::BI__builtin_elementwise_add_sat:
4213 case Builtin::BI__builtin_elementwise_sub_sat: {
4214 Value *Op0 = EmitScalarExpr(E->getArg(0));
4215 Value *Op1 = EmitScalarExpr(E->getArg(1));
4216 Value *Result;
4217 assert(Op0->getType()->isIntOrIntVectorTy() && "integer type expected");
4218 QualType Ty = E->getArg(0)->getType();
4219 if (auto *VecTy = Ty->getAs<VectorType>())
4220 Ty = VecTy->getElementType();
4221 bool IsSigned = Ty->isSignedIntegerType();
4222 unsigned Opc;
4223 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
4224 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
4225 else
4226 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
4227 Result = Builder.CreateBinaryIntrinsic(Opc, Op0, Op1, nullptr, "elt.sat");
4228 return RValue::get(Result);
4229 }
4230
4231 case Builtin::BI__builtin_elementwise_max: {
4232 Value *Op0 = EmitScalarExpr(E->getArg(0));
4233 Value *Op1 = EmitScalarExpr(E->getArg(1));
4234 Value *Result;
4235 if (Op0->getType()->isIntOrIntVectorTy()) {
4236 QualType Ty = E->getArg(0)->getType();
4237 if (auto *VecTy = Ty->getAs<VectorType>())
4238 Ty = VecTy->getElementType();
4239 Result = Builder.CreateBinaryIntrinsic(Ty->isSignedIntegerType()
4240 ? llvm::Intrinsic::smax
4241 : llvm::Intrinsic::umax,
4242 Op0, Op1, nullptr, "elt.max");
4243 } else
4244 Result = Builder.CreateMaxNum(Op0, Op1, "elt.max");
4245 return RValue::get(Result);
4246 }
4247 case Builtin::BI__builtin_elementwise_min: {
4248 Value *Op0 = EmitScalarExpr(E->getArg(0));
4249 Value *Op1 = EmitScalarExpr(E->getArg(1));
4250 Value *Result;
4251 if (Op0->getType()->isIntOrIntVectorTy()) {
4252 QualType Ty = E->getArg(0)->getType();
4253 if (auto *VecTy = Ty->getAs<VectorType>())
4254 Ty = VecTy->getElementType();
4255 Result = Builder.CreateBinaryIntrinsic(Ty->isSignedIntegerType()
4256 ? llvm::Intrinsic::smin
4257 : llvm::Intrinsic::umin,
4258 Op0, Op1, nullptr, "elt.min");
4259 } else
4260 Result = Builder.CreateMinNum(Op0, Op1, "elt.min");
4261 return RValue::get(Result);
4262 }
4263
4264 case Builtin::BI__builtin_elementwise_maximum: {
4265 Value *Op0 = EmitScalarExpr(E->getArg(0));
4266 Value *Op1 = EmitScalarExpr(E->getArg(1));
4267 Value *Result = Builder.CreateBinaryIntrinsic(llvm::Intrinsic::maximum, Op0,
4268 Op1, nullptr, "elt.maximum");
4269 return RValue::get(Result);
4270 }
4271
4272 case Builtin::BI__builtin_elementwise_minimum: {
4273 Value *Op0 = EmitScalarExpr(E->getArg(0));
4274 Value *Op1 = EmitScalarExpr(E->getArg(1));
4275 Value *Result = Builder.CreateBinaryIntrinsic(llvm::Intrinsic::minimum, Op0,
4276 Op1, nullptr, "elt.minimum");
4277 return RValue::get(Result);
4278 }
4279
4280 case Builtin::BI__builtin_reduce_max: {
4281 auto GetIntrinsicID = [this](QualType QT) {
4282 if (auto *VecTy = QT->getAs<VectorType>())
4283 QT = VecTy->getElementType();
4284 else if (QT->isSizelessVectorType())
4286
4287 if (QT->isSignedIntegerType())
4288 return llvm::Intrinsic::vector_reduce_smax;
4289 if (QT->isUnsignedIntegerType())
4290 return llvm::Intrinsic::vector_reduce_umax;
4291 assert(QT->isFloatingType() && "must have a float here");
4292 return llvm::Intrinsic::vector_reduce_fmax;
4293 };
4294 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4295 *this, E, GetIntrinsicID(E->getArg(0)->getType()), "rdx.min"));
4296 }
4297
4298 case Builtin::BI__builtin_reduce_min: {
4299 auto GetIntrinsicID = [this](QualType QT) {
4300 if (auto *VecTy = QT->getAs<VectorType>())
4301 QT = VecTy->getElementType();
4302 else if (QT->isSizelessVectorType())
4304
4305 if (QT->isSignedIntegerType())
4306 return llvm::Intrinsic::vector_reduce_smin;
4307 if (QT->isUnsignedIntegerType())
4308 return llvm::Intrinsic::vector_reduce_umin;
4309 assert(QT->isFloatingType() && "must have a float here");
4310 return llvm::Intrinsic::vector_reduce_fmin;
4311 };
4312
4313 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4314 *this, E, GetIntrinsicID(E->getArg(0)->getType()), "rdx.min"));
4315 }
4316
4317 case Builtin::BI__builtin_reduce_add:
4318 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4319 *this, E, llvm::Intrinsic::vector_reduce_add, "rdx.add"));
4320 case Builtin::BI__builtin_reduce_mul:
4321 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4322 *this, E, llvm::Intrinsic::vector_reduce_mul, "rdx.mul"));
4323 case Builtin::BI__builtin_reduce_xor:
4324 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4325 *this, E, llvm::Intrinsic::vector_reduce_xor, "rdx.xor"));
4326 case Builtin::BI__builtin_reduce_or:
4327 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4328 *this, E, llvm::Intrinsic::vector_reduce_or, "rdx.or"));
4329 case Builtin::BI__builtin_reduce_and:
4330 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4331 *this, E, llvm::Intrinsic::vector_reduce_and, "rdx.and"));
4332 case Builtin::BI__builtin_reduce_maximum:
4333 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4334 *this, E, llvm::Intrinsic::vector_reduce_fmaximum, "rdx.maximum"));
4335 case Builtin::BI__builtin_reduce_minimum:
4336 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4337 *this, E, llvm::Intrinsic::vector_reduce_fminimum, "rdx.minimum"));
4338
4339 case Builtin::BI__builtin_matrix_transpose: {
4340 auto *MatrixTy = E->getArg(0)->getType()->castAs<ConstantMatrixType>();
4341 Value *MatValue = EmitScalarExpr(E->getArg(0));
4342 MatrixBuilder MB(Builder);
4343 Value *Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
4344 MatrixTy->getNumColumns());
4345 return RValue::get(Result);
4346 }
4347
4348 case Builtin::BI__builtin_matrix_column_major_load: {
4349 MatrixBuilder MB(Builder);
4350 // Emit everything that isn't dependent on the first parameter type
4351 Value *Stride = EmitScalarExpr(E->getArg(3));
4352 const auto *ResultTy = E->getType()->getAs<ConstantMatrixType>();
4353 auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
4354 assert(PtrTy && "arg0 must be of pointer type");
4355 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4356
4357 Address Src = EmitPointerWithAlignment(E->getArg(0));
4359 E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD,
4360 0);
4361 Value *Result = MB.CreateColumnMajorLoad(
4362 Src.getElementType(), Src.emitRawPointer(*this),
4363 Align(Src.getAlignment().getQuantity()), Stride, IsVolatile,
4364 ResultTy->getNumRows(), ResultTy->getNumColumns(), "matrix");
4365 return RValue::get(Result);
4366 }
4367
4368 case Builtin::BI__builtin_matrix_column_major_store: {
4369 MatrixBuilder MB(Builder);
4370 Value *Matrix = EmitScalarExpr(E->getArg(0));
4371 Address Dst = EmitPointerWithAlignment(E->getArg(1));
4372 Value *Stride = EmitScalarExpr(E->getArg(2));
4373
4374 const auto *MatrixTy = E->getArg(0)->getType()->getAs<ConstantMatrixType>();
4375 auto *PtrTy = E->getArg(1)->getType()->getAs<PointerType>();
4376 assert(PtrTy && "arg1 must be of pointer type");
4377 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4378
4380 E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD,
4381 0);
4382 Value *Result = MB.CreateColumnMajorStore(
4383 Matrix, Dst.emitRawPointer(*this),
4384 Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile,
4385 MatrixTy->getNumRows(), MatrixTy->getNumColumns());
4386 return RValue::get(Result);
4387 }
4388
4389 case Builtin::BI__builtin_isinf_sign: {
4390 // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
4391 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4392 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4393 Value *Arg = EmitScalarExpr(E->getArg(0));
4394 Value *AbsArg = EmitFAbs(*this, Arg);
4395 Value *IsInf = Builder.CreateFCmpOEQ(
4396 AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf");
4397 Value *IsNeg = EmitSignBit(*this, Arg);
4398
4399 llvm::Type *IntTy = ConvertType(E->getType());
4400 Value *Zero = Constant::getNullValue(IntTy);
4401 Value *One = ConstantInt::get(IntTy, 1);
4402 Value *NegativeOne = ConstantInt::get(IntTy, -1);
4403 Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
4404 Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero);
4405 return RValue::get(Result);
4406 }
4407
4408 case Builtin::BI__builtin_flt_rounds: {
4409 Function *F = CGM.getIntrinsic(Intrinsic::get_rounding);
4410
4411 llvm::Type *ResultType = ConvertType(E->getType());
4412 Value *Result = Builder.CreateCall(F);
4413 if (Result->getType() != ResultType)
4414 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
4415 "cast");
4416 return RValue::get(Result);
4417 }
4418
4419 case Builtin::BI__builtin_set_flt_rounds: {
4420 Function *F = CGM.getIntrinsic(Intrinsic::set_rounding);
4421
4422 Value *V = EmitScalarExpr(E->getArg(0));
4423 Builder.CreateCall(F, V);
4424 return RValue::get(nullptr);
4425 }
4426
4427 case Builtin::BI__builtin_fpclassify: {
4428 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4429 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4430 Value *V = EmitScalarExpr(E->getArg(5));
4431 llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
4432
4433 // Create Result
4434 BasicBlock *Begin = Builder.GetInsertBlock();
4435 BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn);
4436 Builder.SetInsertPoint(End);
4437 PHINode *Result =
4438 Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4,
4439 "fpclassify_result");
4440
4441 // if (V==0) return FP_ZERO
4442 Builder.SetInsertPoint(Begin);
4443 Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
4444 "iszero");
4445 Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
4446 BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn);
4447 Builder.CreateCondBr(IsZero, End, NotZero);
4448 Result->addIncoming(ZeroLiteral, Begin);
4449
4450 // if (V != V) return FP_NAN
4451 Builder.SetInsertPoint(NotZero);
4452 Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp");
4453 Value *NanLiteral = EmitScalarExpr(E->getArg(0));
4454 BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn);
4455 Builder.CreateCondBr(IsNan, End, NotNan);
4456 Result->addIncoming(NanLiteral, NotZero);
4457
4458 // if (fabs(V) == infinity) return FP_INFINITY
4459 Builder.SetInsertPoint(NotNan);
4460 Value *VAbs = EmitFAbs(*this, V);
4461 Value *IsInf =
4462 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
4463 "isinf");
4464 Value *InfLiteral = EmitScalarExpr(E->getArg(1));
4465 BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn);
4466 Builder.CreateCondBr(IsInf, End, NotInf);
4467 Result->addIncoming(InfLiteral, NotNan);
4468
4469 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
4470 Builder.SetInsertPoint(NotInf);
4471 APFloat Smallest = APFloat::getSmallestNormalized(
4472 getContext().getFloatTypeSemantics(E->getArg(5)->getType()));
4473 Value *IsNormal =
4474 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
4475 "isnormal");
4476 Value *NormalResult =
4477 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
4478 EmitScalarExpr(E->getArg(3)));
4479 Builder.CreateBr(End);
4480 Result->addIncoming(NormalResult, NotInf);
4481
4482 // return Result
4483 Builder.SetInsertPoint(End);
4484 return RValue::get(Result);
4485 }
4486
4487 // An alloca will always return a pointer to the alloca (stack) address
4488 // space. This address space need not be the same as the AST / Language
4489 // default (e.g. in C / C++ auto vars are in the generic address space). At
4490 // the AST level this is handled within CreateTempAlloca et al., but for the
4491 // builtin / dynamic alloca we have to handle it here. We use an explicit cast
4492 // instead of passing an AS to CreateAlloca so as to not inhibit optimisation.
4493 case Builtin::BIalloca:
4494 case Builtin::BI_alloca:
4495 case Builtin::BI__builtin_alloca_uninitialized:
4496 case Builtin::BI__builtin_alloca: {
4497 Value *Size = EmitScalarExpr(E->getArg(0));
4498 const TargetInfo &TI = getContext().getTargetInfo();
4499 // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
4500 const Align SuitableAlignmentInBytes =
4501 CGM.getContext()
4503 .getAsAlign();
4504 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
4505 AI->setAlignment(SuitableAlignmentInBytes);
4506 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4507 initializeAlloca(*this, AI, Size, SuitableAlignmentInBytes);
4510 if (AAS != EAS) {
4511 llvm::Type *Ty = CGM.getTypes().ConvertType(E->getType());
4512 return RValue::get(getTargetHooks().performAddrSpaceCast(*this, AI, AAS,
4513 EAS, Ty));
4514 }
4515 return RValue::get(AI);
4516 }
4517
4518 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4519 case Builtin::BI__builtin_alloca_with_align: {
4520 Value *Size = EmitScalarExpr(E->getArg(0));
4521 Value *AlignmentInBitsValue = EmitScalarExpr(E->getArg(1));
4522 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
4523 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4524 const Align AlignmentInBytes =
4525 CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getAsAlign();
4526 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
4527 AI->setAlignment(AlignmentInBytes);
4528 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4529 initializeAlloca(*this, AI, Size, AlignmentInBytes);
4532 if (AAS != EAS) {
4533 llvm::Type *Ty = CGM.getTypes().ConvertType(E->getType());
4534 return RValue::get(getTargetHooks().performAddrSpaceCast(*this, AI, AAS,
4535 EAS, Ty));
4536 }
4537 return RValue::get(AI);
4538 }
4539
4540 case Builtin::BIbzero:
4541 case Builtin::BI__builtin_bzero: {