clang 18.0.0git
CGBuiltin.cpp
Go to the documentation of this file.
1//===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfo.h"
14#include "CGCUDARuntime.h"
15#include "CGCXXABI.h"
16#include "CGObjCRuntime.h"
17#include "CGOpenCLRuntime.h"
18#include "CGRecordLayout.h"
19#include "CodeGenFunction.h"
20#include "CodeGenModule.h"
21#include "ConstantEmitter.h"
22#include "PatternInit.h"
23#include "TargetInfo.h"
25#include "clang/AST/Attr.h"
26#include "clang/AST/Decl.h"
27#include "clang/AST/OSLog.h"
34#include "llvm/ADT/APFloat.h"
35#include "llvm/ADT/APInt.h"
36#include "llvm/ADT/FloatingPointMode.h"
37#include "llvm/ADT/SmallPtrSet.h"
38#include "llvm/ADT/StringExtras.h"
39#include "llvm/Analysis/ValueTracking.h"
40#include "llvm/IR/DataLayout.h"
41#include "llvm/IR/InlineAsm.h"
42#include "llvm/IR/Intrinsics.h"
43#include "llvm/IR/IntrinsicsAArch64.h"
44#include "llvm/IR/IntrinsicsAMDGPU.h"
45#include "llvm/IR/IntrinsicsARM.h"
46#include "llvm/IR/IntrinsicsBPF.h"
47#include "llvm/IR/IntrinsicsHexagon.h"
48#include "llvm/IR/IntrinsicsNVPTX.h"
49#include "llvm/IR/IntrinsicsPowerPC.h"
50#include "llvm/IR/IntrinsicsR600.h"
51#include "llvm/IR/IntrinsicsRISCV.h"
52#include "llvm/IR/IntrinsicsS390.h"
53#include "llvm/IR/IntrinsicsVE.h"
54#include "llvm/IR/IntrinsicsWebAssembly.h"
55#include "llvm/IR/IntrinsicsX86.h"
56#include "llvm/IR/MDBuilder.h"
57#include "llvm/IR/MatrixBuilder.h"
58#include "llvm/Support/ConvertUTF.h"
59#include "llvm/Support/MathExtras.h"
60#include "llvm/Support/ScopedPrinter.h"
61#include "llvm/TargetParser/AArch64TargetParser.h"
62#include "llvm/TargetParser/X86TargetParser.h"
63#include <optional>
64#include <sstream>
65
66using namespace clang;
67using namespace CodeGen;
68using namespace llvm;
69
70static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size,
71 Align AlignmentInBytes) {
72 ConstantInt *Byte;
73 switch (CGF.getLangOpts().getTrivialAutoVarInit()) {
74 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
75 // Nothing to initialize.
76 return;
77 case LangOptions::TrivialAutoVarInitKind::Zero:
78 Byte = CGF.Builder.getInt8(0x00);
79 break;
80 case LangOptions::TrivialAutoVarInitKind::Pattern: {
81 llvm::Type *Int8 = llvm::IntegerType::getInt8Ty(CGF.CGM.getLLVMContext());
82 Byte = llvm::dyn_cast<llvm::ConstantInt>(
83 initializationPatternFor(CGF.CGM, Int8));
84 break;
85 }
86 }
87 if (CGF.CGM.stopAutoInit())
88 return;
89 auto *I = CGF.Builder.CreateMemSet(AI, Byte, Size, AlignmentInBytes);
90 I->addAnnotationMetadata("auto-init");
91}
92
93/// getBuiltinLibFunction - Given a builtin id for a function like
94/// "__builtin_fabsf", return a Function* for "fabsf".
96 unsigned BuiltinID) {
97 assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
98
99 // Get the name, skip over the __builtin_ prefix (if necessary).
100 StringRef Name;
101 GlobalDecl D(FD);
102
103 // TODO: This list should be expanded or refactored after all GCC-compatible
104 // std libcall builtins are implemented.
105 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
106 {Builtin::BI__builtin___fprintf_chk, "__fprintf_chkieee128"},
107 {Builtin::BI__builtin___printf_chk, "__printf_chkieee128"},
108 {Builtin::BI__builtin___snprintf_chk, "__snprintf_chkieee128"},
109 {Builtin::BI__builtin___sprintf_chk, "__sprintf_chkieee128"},
110 {Builtin::BI__builtin___vfprintf_chk, "__vfprintf_chkieee128"},
111 {Builtin::BI__builtin___vprintf_chk, "__vprintf_chkieee128"},
112 {Builtin::BI__builtin___vsnprintf_chk, "__vsnprintf_chkieee128"},
113 {Builtin::BI__builtin___vsprintf_chk, "__vsprintf_chkieee128"},
114 {Builtin::BI__builtin_fprintf, "__fprintfieee128"},
115 {Builtin::BI__builtin_printf, "__printfieee128"},
116 {Builtin::BI__builtin_snprintf, "__snprintfieee128"},
117 {Builtin::BI__builtin_sprintf, "__sprintfieee128"},
118 {Builtin::BI__builtin_vfprintf, "__vfprintfieee128"},
119 {Builtin::BI__builtin_vprintf, "__vprintfieee128"},
120 {Builtin::BI__builtin_vsnprintf, "__vsnprintfieee128"},
121 {Builtin::BI__builtin_vsprintf, "__vsprintfieee128"},
122 {Builtin::BI__builtin_fscanf, "__fscanfieee128"},
123 {Builtin::BI__builtin_scanf, "__scanfieee128"},
124 {Builtin::BI__builtin_sscanf, "__sscanfieee128"},
125 {Builtin::BI__builtin_vfscanf, "__vfscanfieee128"},
126 {Builtin::BI__builtin_vscanf, "__vscanfieee128"},
127 {Builtin::BI__builtin_vsscanf, "__vsscanfieee128"},
128 {Builtin::BI__builtin_nexttowardf128, "__nexttowardieee128"},
129 };
130
131 // The AIX library functions frexpl, ldexpl, and modfl are for 128-bit
132 // IBM 'long double' (i.e. __ibm128). Map to the 'double' versions
133 // if it is 64-bit 'long double' mode.
134 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
135 {Builtin::BI__builtin_frexpl, "frexp"},
136 {Builtin::BI__builtin_ldexpl, "ldexp"},
137 {Builtin::BI__builtin_modfl, "modf"},
138 };
139
140 // If the builtin has been declared explicitly with an assembler label,
141 // use the mangled name. This differs from the plain label on platforms
142 // that prefix labels.
143 if (FD->hasAttr<AsmLabelAttr>())
144 Name = getMangledName(D);
145 else {
146 // TODO: This mutation should also be applied to other targets other than
147 // PPC, after backend supports IEEE 128-bit style libcalls.
148 if (getTriple().isPPC64() &&
149 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
150 F128Builtins.contains(BuiltinID))
151 Name = F128Builtins[BuiltinID];
152 else if (getTriple().isOSAIX() &&
153 &getTarget().getLongDoubleFormat() ==
154 &llvm::APFloat::IEEEdouble() &&
155 AIXLongDouble64Builtins.contains(BuiltinID))
156 Name = AIXLongDouble64Builtins[BuiltinID];
157 else
158 Name = Context.BuiltinInfo.getName(BuiltinID).substr(10);
159 }
160
161 llvm::FunctionType *Ty =
162 cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
163
164 return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);
165}
166
167/// Emit the conversions required to turn the given value into an
168/// integer of the given size.
169static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
170 QualType T, llvm::IntegerType *IntType) {
171 V = CGF.EmitToMemory(V, T);
172
173 if (V->getType()->isPointerTy())
174 return CGF.Builder.CreatePtrToInt(V, IntType);
175
176 assert(V->getType() == IntType);
177 return V;
178}
179
180static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
181 QualType T, llvm::Type *ResultType) {
182 V = CGF.EmitFromMemory(V, T);
183
184 if (ResultType->isPointerTy())
185 return CGF.Builder.CreateIntToPtr(V, ResultType);
186
187 assert(V->getType() == ResultType);
188 return V;
189}
190
192 ASTContext &Ctx = CGF.getContext();
193 Address Ptr = CGF.EmitPointerWithAlignment(E->getArg(0));
194 unsigned Bytes = Ptr.getElementType()->isPointerTy()
196 : Ptr.getElementType()->getScalarSizeInBits() / 8;
197 unsigned Align = Ptr.getAlignment().getQuantity();
198 if (Align % Bytes != 0) {
199 DiagnosticsEngine &Diags = CGF.CGM.getDiags();
200 Diags.Report(E->getBeginLoc(), diag::warn_sync_op_misaligned);
201 // Force address to be at least naturally-aligned.
202 return Ptr.withAlignment(CharUnits::fromQuantity(Bytes));
203 }
204 return Ptr;
205}
206
207/// Utility to insert an atomic instruction based on Intrinsic::ID
208/// and the expression node.
210 CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E,
211 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
212
213 QualType T = E->getType();
214 assert(E->getArg(0)->getType()->isPointerType());
215 assert(CGF.getContext().hasSameUnqualifiedType(T,
216 E->getArg(0)->getType()->getPointeeType()));
217 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
218
219 Address DestAddr = CheckAtomicAlignment(CGF, E);
220
221 llvm::IntegerType *IntType = llvm::IntegerType::get(
222 CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
223
224 llvm::Value *Val = CGF.EmitScalarExpr(E->getArg(1));
225 llvm::Type *ValueType = Val->getType();
226 Val = EmitToInt(CGF, Val, T, IntType);
227
228 llvm::Value *Result =
229 CGF.Builder.CreateAtomicRMW(Kind, DestAddr, Val, Ordering);
230 return EmitFromInt(CGF, Result, T, ValueType);
231}
232
234 Value *Val = CGF.EmitScalarExpr(E->getArg(0));
235 Value *Address = CGF.EmitScalarExpr(E->getArg(1));
236
237 Val = CGF.EmitToMemory(Val, E->getArg(0)->getType());
239 LV.setNontemporal(true);
240 CGF.EmitStoreOfScalar(Val, LV, false);
241 return nullptr;
242}
243
245 Value *Address = CGF.EmitScalarExpr(E->getArg(0));
246
248 LV.setNontemporal(true);
249 return CGF.EmitLoadOfScalar(LV, E->getExprLoc());
250}
251
253 llvm::AtomicRMWInst::BinOp Kind,
254 const CallExpr *E) {
255 return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E));
256}
257
258/// Utility to insert an atomic instruction based Intrinsic::ID and
259/// the expression node, where the return value is the result of the
260/// operation.
262 llvm::AtomicRMWInst::BinOp Kind,
263 const CallExpr *E,
264 Instruction::BinaryOps Op,
265 bool Invert = false) {
266 QualType T = E->getType();
267 assert(E->getArg(0)->getType()->isPointerType());
268 assert(CGF.getContext().hasSameUnqualifiedType(T,
269 E->getArg(0)->getType()->getPointeeType()));
270 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
271
272 Address DestAddr = CheckAtomicAlignment(CGF, E);
273
274 llvm::IntegerType *IntType = llvm::IntegerType::get(
275 CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
276
277 llvm::Value *Val = CGF.EmitScalarExpr(E->getArg(1));
278 llvm::Type *ValueType = Val->getType();
279 Val = EmitToInt(CGF, Val, T, IntType);
280
281 llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
282 Kind, DestAddr, Val, llvm::AtomicOrdering::SequentiallyConsistent);
283 Result = CGF.Builder.CreateBinOp(Op, Result, Val);
284 if (Invert)
285 Result =
286 CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,
287 llvm::ConstantInt::getAllOnesValue(IntType));
288 Result = EmitFromInt(CGF, Result, T, ValueType);
289 return RValue::get(Result);
290}
291
292/// Utility to insert an atomic cmpxchg instruction.
293///
294/// @param CGF The current codegen function.
295/// @param E Builtin call expression to convert to cmpxchg.
296/// arg0 - address to operate on
297/// arg1 - value to compare with
298/// arg2 - new value
299/// @param ReturnBool Specifies whether to return success flag of
300/// cmpxchg result or the old value.
301///
302/// @returns result of cmpxchg, according to ReturnBool
303///
304/// Note: In order to lower Microsoft's _InterlockedCompareExchange* intrinsics
305/// invoke the function EmitAtomicCmpXchgForMSIntrin.
307 bool ReturnBool) {
308 QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType();
309 Address DestAddr = CheckAtomicAlignment(CGF, E);
310
311 llvm::IntegerType *IntType = llvm::IntegerType::get(
312 CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
313
314 Value *Cmp = CGF.EmitScalarExpr(E->getArg(1));
315 llvm::Type *ValueType = Cmp->getType();
316 Cmp = EmitToInt(CGF, Cmp, T, IntType);
317 Value *New = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);
318
320 DestAddr, Cmp, New, llvm::AtomicOrdering::SequentiallyConsistent,
321 llvm::AtomicOrdering::SequentiallyConsistent);
322 if (ReturnBool)
323 // Extract boolean success flag and zext it to int.
324 return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1),
325 CGF.ConvertType(E->getType()));
326 else
327 // Extract old value and emit it using the same type as compare value.
328 return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T,
329 ValueType);
330}
331
332/// This function should be invoked to emit atomic cmpxchg for Microsoft's
333/// _InterlockedCompareExchange* intrinsics which have the following signature:
334/// T _InterlockedCompareExchange(T volatile *Destination,
335/// T Exchange,
336/// T Comparand);
337///
338/// Whereas the llvm 'cmpxchg' instruction has the following syntax:
339/// cmpxchg *Destination, Comparand, Exchange.
340/// So we need to swap Comparand and Exchange when invoking
341/// CreateAtomicCmpXchg. That is the reason we could not use the above utility
342/// function MakeAtomicCmpXchgValue since it expects the arguments to be
343/// already swapped.
344
345static
347 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
348 assert(E->getArg(0)->getType()->isPointerType());
350 E->getType(), E->getArg(0)->getType()->getPointeeType()));
351 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
352 E->getArg(1)->getType()));
353 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
354 E->getArg(2)->getType()));
355
356 Address DestAddr = CheckAtomicAlignment(CGF, E);
357
358 auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
359 auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));
360
361 // For Release ordering, the failure ordering should be Monotonic.
362 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
363 AtomicOrdering::Monotonic :
364 SuccessOrdering;
365
366 // The atomic instruction is marked volatile for consistency with MSVC. This
367 // blocks the few atomics optimizations that LLVM has. If we want to optimize
368 // _Interlocked* operations in the future, we will have to remove the volatile
369 // marker.
371 DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
372 Result->setVolatile(true);
373 return CGF.Builder.CreateExtractValue(Result, 0);
374}
375
376// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
377// prototyped like this:
378//
379// unsigned char _InterlockedCompareExchange128...(
380// __int64 volatile * _Destination,
381// __int64 _ExchangeHigh,
382// __int64 _ExchangeLow,
383// __int64 * _ComparandResult);
384//
385// Note that Destination is assumed to be at least 16-byte aligned, despite
386// being typed int64.
387
389 const CallExpr *E,
390 AtomicOrdering SuccessOrdering) {
391 assert(E->getNumArgs() == 4);
392 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
393 llvm::Value *ExchangeHigh = CGF.EmitScalarExpr(E->getArg(1));
394 llvm::Value *ExchangeLow = CGF.EmitScalarExpr(E->getArg(2));
395 Address ComparandAddr = CGF.EmitPointerWithAlignment(E->getArg(3));
396
397 assert(DestPtr->getType()->isPointerTy());
398 assert(!ExchangeHigh->getType()->isPointerTy());
399 assert(!ExchangeLow->getType()->isPointerTy());
400
401 // For Release ordering, the failure ordering should be Monotonic.
402 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
403 ? AtomicOrdering::Monotonic
404 : SuccessOrdering;
405
406 // Convert to i128 pointers and values. Alignment is also overridden for
407 // destination pointer.
408 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.getLLVMContext(), 128);
409 Address DestAddr(DestPtr, Int128Ty,
411 ComparandAddr = ComparandAddr.withElementType(Int128Ty);
412
413 // (((i128)hi) << 64) | ((i128)lo)
414 ExchangeHigh = CGF.Builder.CreateZExt(ExchangeHigh, Int128Ty);
415 ExchangeLow = CGF.Builder.CreateZExt(ExchangeLow, Int128Ty);
416 ExchangeHigh =
417 CGF.Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
418 llvm::Value *Exchange = CGF.Builder.CreateOr(ExchangeHigh, ExchangeLow);
419
420 // Load the comparand for the instruction.
421 llvm::Value *Comparand = CGF.Builder.CreateLoad(ComparandAddr);
422
423 auto *CXI = CGF.Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
424 SuccessOrdering, FailureOrdering);
425
426 // The atomic instruction is marked volatile for consistency with MSVC. This
427 // blocks the few atomics optimizations that LLVM has. If we want to optimize
428 // _Interlocked* operations in the future, we will have to remove the volatile
429 // marker.
430 CXI->setVolatile(true);
431
432 // Store the result as an outparameter.
433 CGF.Builder.CreateStore(CGF.Builder.CreateExtractValue(CXI, 0),
434 ComparandAddr);
435
436 // Get the success boolean and zero extend it to i8.
437 Value *Success = CGF.Builder.CreateExtractValue(CXI, 1);
438 return CGF.Builder.CreateZExt(Success, CGF.Int8Ty);
439}
440
442 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
443 assert(E->getArg(0)->getType()->isPointerType());
444
445 auto *IntTy = CGF.ConvertType(E->getType());
446 Address DestAddr = CheckAtomicAlignment(CGF, E);
447 auto *Result = CGF.Builder.CreateAtomicRMW(
448 AtomicRMWInst::Add, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
449 return CGF.Builder.CreateAdd(Result, ConstantInt::get(IntTy, 1));
450}
451
453 CodeGenFunction &CGF, const CallExpr *E,
454 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
455 assert(E->getArg(0)->getType()->isPointerType());
456
457 auto *IntTy = CGF.ConvertType(E->getType());
458 Address DestAddr = CheckAtomicAlignment(CGF, E);
459 auto *Result = CGF.Builder.CreateAtomicRMW(
460 AtomicRMWInst::Sub, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
461 return CGF.Builder.CreateSub(Result, ConstantInt::get(IntTy, 1));
462}
463
464// Build a plain volatile load.
466 Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
467 QualType ElTy = E->getArg(0)->getType()->getPointeeType();
468 CharUnits LoadSize = CGF.getContext().getTypeSizeInChars(ElTy);
469 llvm::Type *ITy =
470 llvm::IntegerType::get(CGF.getLLVMContext(), LoadSize.getQuantity() * 8);
471 llvm::LoadInst *Load = CGF.Builder.CreateAlignedLoad(ITy, Ptr, LoadSize);
472 Load->setVolatile(true);
473 return Load;
474}
475
476// Build a plain volatile store.
478 Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
479 Value *Value = CGF.EmitScalarExpr(E->getArg(1));
480 QualType ElTy = E->getArg(0)->getType()->getPointeeType();
481 CharUnits StoreSize = CGF.getContext().getTypeSizeInChars(ElTy);
482 llvm::StoreInst *Store =
483 CGF.Builder.CreateAlignedStore(Value, Ptr, StoreSize);
484 Store->setVolatile(true);
485 return Store;
486}
487
488// Emit a simple mangled intrinsic that has 1 argument and a return type
489// matching the argument type. Depending on mode, this may be a constrained
490// floating-point intrinsic.
492 const CallExpr *E, unsigned IntrinsicID,
493 unsigned ConstrainedIntrinsicID) {
494 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
495
496 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
497 if (CGF.Builder.getIsFPConstrained()) {
498 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
499 return CGF.Builder.CreateConstrainedFPCall(F, { Src0 });
500 } else {
501 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
502 return CGF.Builder.CreateCall(F, Src0);
503 }
504}
505
506// Emit an intrinsic that has 2 operands of the same type as its result.
507// Depending on mode, this may be a constrained floating-point intrinsic.
509 const CallExpr *E, unsigned IntrinsicID,
510 unsigned ConstrainedIntrinsicID) {
511 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
512 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
513
514 if (CGF.Builder.getIsFPConstrained()) {
515 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
516 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
517 return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
518 } else {
519 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
520 return CGF.Builder.CreateCall(F, { Src0, Src1 });
521 }
522}
523
524// Has second type mangled argument.
526 CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID,
527 llvm::Intrinsic::ID ConstrainedIntrinsicID) {
528 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
529 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
530
531 if (CGF.Builder.getIsFPConstrained()) {
532 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
533 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID,
534 {Src0->getType(), Src1->getType()});
535 return CGF.Builder.CreateConstrainedFPCall(F, {Src0, Src1});
536 }
537
538 Function *F =
539 CGF.CGM.getIntrinsic(IntrinsicID, {Src0->getType(), Src1->getType()});
540 return CGF.Builder.CreateCall(F, {Src0, Src1});
541}
542
543// Emit an intrinsic that has 3 operands of the same type as its result.
544// Depending on mode, this may be a constrained floating-point intrinsic.
546 const CallExpr *E, unsigned IntrinsicID,
547 unsigned ConstrainedIntrinsicID) {
548 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
549 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
550 llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
551
552 if (CGF.Builder.getIsFPConstrained()) {
553 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
554 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
555 return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
556 } else {
557 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
558 return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });
559 }
560}
561
562// Emit an intrinsic where all operands are of the same type as the result.
563// Depending on mode, this may be a constrained floating-point intrinsic.
565 unsigned IntrinsicID,
566 unsigned ConstrainedIntrinsicID,
567 llvm::Type *Ty,
568 ArrayRef<Value *> Args) {
569 Function *F;
570 if (CGF.Builder.getIsFPConstrained())
571 F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Ty);
572 else
573 F = CGF.CGM.getIntrinsic(IntrinsicID, Ty);
574
575 if (CGF.Builder.getIsFPConstrained())
576 return CGF.Builder.CreateConstrainedFPCall(F, Args);
577 else
578 return CGF.Builder.CreateCall(F, Args);
579}
580
581// Emit a simple mangled intrinsic that has 1 argument and a return type
582// matching the argument type.
584 unsigned IntrinsicID,
585 llvm::StringRef Name = "") {
586 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
587
588 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
589 return CGF.Builder.CreateCall(F, Src0, Name);
590}
591
592// Emit an intrinsic that has 2 operands of the same type as its result.
594 const CallExpr *E,
595 unsigned IntrinsicID) {
596 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
597 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
598
599 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
600 return CGF.Builder.CreateCall(F, { Src0, Src1 });
601}
602
603// Emit an intrinsic that has 3 operands of the same type as its result.
605 const CallExpr *E,
606 unsigned IntrinsicID) {
607 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
608 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
609 llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
610
611 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
612 return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });
613}
614
615// Emit an intrinsic that has 1 float or double operand, and 1 integer.
617 const CallExpr *E,
618 unsigned IntrinsicID) {
619 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
620 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
621
622 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
623 return CGF.Builder.CreateCall(F, {Src0, Src1});
624}
625
626// Emit an intrinsic that has overloaded integer result and fp operand.
627static Value *
629 unsigned IntrinsicID,
630 unsigned ConstrainedIntrinsicID) {
631 llvm::Type *ResultType = CGF.ConvertType(E->getType());
632 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
633
634 if (CGF.Builder.getIsFPConstrained()) {
635 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
636 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID,
637 {ResultType, Src0->getType()});
638 return CGF.Builder.CreateConstrainedFPCall(F, {Src0});
639 } else {
640 Function *F =
641 CGF.CGM.getIntrinsic(IntrinsicID, {ResultType, Src0->getType()});
642 return CGF.Builder.CreateCall(F, Src0);
643 }
644}
645
647 llvm::Intrinsic::ID IntrinsicID) {
648 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
649 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
650
651 QualType IntPtrTy = E->getArg(1)->getType()->getPointeeType();
652 llvm::Type *IntTy = CGF.ConvertType(IntPtrTy);
653 llvm::Function *F =
654 CGF.CGM.getIntrinsic(IntrinsicID, {Src0->getType(), IntTy});
655 llvm::Value *Call = CGF.Builder.CreateCall(F, Src0);
656
657 llvm::Value *Exp = CGF.Builder.CreateExtractValue(Call, 1);
658 LValue LV = CGF.MakeNaturalAlignAddrLValue(Src1, IntPtrTy);
659 CGF.EmitStoreOfScalar(Exp, LV);
660
661 return CGF.Builder.CreateExtractValue(Call, 0);
662}
663
664/// EmitFAbs - Emit a call to @llvm.fabs().
666 Function *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType());
667 llvm::CallInst *Call = CGF.Builder.CreateCall(F, V);
668 Call->setDoesNotAccessMemory();
669 return Call;
670}
671
672/// Emit the computation of the sign bit for a floating point value. Returns
673/// the i1 sign bit value.
675 LLVMContext &C = CGF.CGM.getLLVMContext();
676
677 llvm::Type *Ty = V->getType();
678 int Width = Ty->getPrimitiveSizeInBits();
679 llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
680 V = CGF.Builder.CreateBitCast(V, IntTy);
681 if (Ty->isPPC_FP128Ty()) {
682 // We want the sign bit of the higher-order double. The bitcast we just
683 // did works as if the double-double was stored to memory and then
684 // read as an i128. The "store" will put the higher-order double in the
685 // lower address in both little- and big-Endian modes, but the "load"
686 // will treat those bits as a different part of the i128: the low bits in
687 // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
688 // we need to shift the high bits down to the low before truncating.
689 Width >>= 1;
690 if (CGF.getTarget().isBigEndian()) {
691 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
692 V = CGF.Builder.CreateLShr(V, ShiftCst);
693 }
694 // We are truncating value in order to extract the higher-order
695 // double, which we will be using to extract the sign from.
696 IntTy = llvm::IntegerType::get(C, Width);
697 V = CGF.Builder.CreateTrunc(V, IntTy);
698 }
699 Value *Zero = llvm::Constant::getNullValue(IntTy);
700 return CGF.Builder.CreateICmpSLT(V, Zero);
701}
702
704 const CallExpr *E, llvm::Constant *calleeValue) {
705 CGCallee callee = CGCallee::forDirect(calleeValue, GlobalDecl(FD));
706 return CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot());
707}
708
709/// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
710/// depending on IntrinsicID.
711///
712/// \arg CGF The current codegen function.
713/// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
714/// \arg X The first argument to the llvm.*.with.overflow.*.
715/// \arg Y The second argument to the llvm.*.with.overflow.*.
716/// \arg Carry The carry returned by the llvm.*.with.overflow.*.
717/// \returns The result (i.e. sum/product) returned by the intrinsic.
718static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
719 const llvm::Intrinsic::ID IntrinsicID,
720 llvm::Value *X, llvm::Value *Y,
721 llvm::Value *&Carry) {
722 // Make sure we have integers of the same width.
723 assert(X->getType() == Y->getType() &&
724 "Arguments must be the same type. (Did you forget to make sure both "
725 "arguments have the same integer width?)");
726
727 Function *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType());
728 llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y});
729 Carry = CGF.Builder.CreateExtractValue(Tmp, 1);
730 return CGF.Builder.CreateExtractValue(Tmp, 0);
731}
732
734 unsigned IntrinsicID,
735 int low, int high) {
736 llvm::MDBuilder MDHelper(CGF.getLLVMContext());
737 llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
738 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, {});
739 llvm::Instruction *Call = CGF.Builder.CreateCall(F);
740 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
741 Call->setMetadata(llvm::LLVMContext::MD_noundef,
742 llvm::MDNode::get(CGF.getLLVMContext(), std::nullopt));
743 return Call;
744}
745
746namespace {
747 struct WidthAndSignedness {
748 unsigned Width;
749 bool Signed;
750 };
751}
752
753static WidthAndSignedness
755 const clang::QualType Type) {
756 assert(Type->isIntegerType() && "Given type is not an integer.");
757 unsigned Width = Type->isBooleanType() ? 1
758 : Type->isBitIntType() ? context.getIntWidth(Type)
759 : context.getTypeInfo(Type).Width;
761 return {Width, Signed};
762}
763
764// Given one or more integer types, this function produces an integer type that
765// encompasses them: any value in one of the given types could be expressed in
766// the encompassing type.
767static struct WidthAndSignedness
768EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
769 assert(Types.size() > 0 && "Empty list of types.");
770
771 // If any of the given types is signed, we must return a signed type.
772 bool Signed = false;
773 for (const auto &Type : Types) {
774 Signed |= Type.Signed;
775 }
776
777 // The encompassing type must have a width greater than or equal to the width
778 // of the specified types. Additionally, if the encompassing type is signed,
779 // its width must be strictly greater than the width of any unsigned types
780 // given.
781 unsigned Width = 0;
782 for (const auto &Type : Types) {
783 unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
784 if (Width < MinWidth) {
785 Width = MinWidth;
786 }
787 }
788
789 return {Width, Signed};
790}
791
792Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
793 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
794 return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue);
795}
796
797/// Checks if using the result of __builtin_object_size(p, @p From) in place of
798/// __builtin_object_size(p, @p To) is correct
799static bool areBOSTypesCompatible(int From, int To) {
800 // Note: Our __builtin_object_size implementation currently treats Type=0 and
801 // Type=2 identically. Encoding this implementation detail here may make
802 // improving __builtin_object_size difficult in the future, so it's omitted.
803 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
804}
805
806static llvm::Value *
807getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
808 return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true);
809}
810
811llvm::Value *
812CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
813 llvm::IntegerType *ResType,
814 llvm::Value *EmittedE,
815 bool IsDynamic) {
816 uint64_t ObjectSize;
817 if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type))
818 return emitBuiltinObjectSize(E, Type, ResType, EmittedE, IsDynamic);
819 return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true);
820}
821
822llvm::Value *
823CodeGenFunction::emitFlexibleArrayMemberSize(const Expr *E, unsigned Type,
824 llvm::IntegerType *ResType) {
825 // The code generated here calculates the size of a struct with a flexible
826 // array member that uses the counted_by attribute. There are two instances
827 // we handle:
828 //
829 // struct s {
830 // unsigned long flags;
831 // int count;
832 // int array[] __attribute__((counted_by(count)));
833 // }
834 //
835 // 1) bdos of the flexible array itself:
836 //
837 // __builtin_dynamic_object_size(p->array, 1) ==
838 // p->count * sizeof(*p->array)
839 //
840 // 2) bdos of a pointer into the flexible array:
841 //
842 // __builtin_dynamic_object_size(&p->array[42], 1) ==
843 // (p->count - 42) * sizeof(*p->array)
844 //
845 // 2) bdos of the whole struct, including the flexible array:
846 //
847 // __builtin_dynamic_object_size(p, 1) ==
848 // max(sizeof(struct s),
849 // offsetof(struct s, array) + p->count * sizeof(*p->array))
850 //
851 ASTContext &Ctx = getContext();
852 const Expr *Base = E->IgnoreParenImpCasts();
853 const Expr *Idx = nullptr;
854
855 if (const auto *UO = dyn_cast<UnaryOperator>(Base);
856 UO && UO->getOpcode() == UO_AddrOf) {
857 Expr *SubExpr = UO->getSubExpr()->IgnoreParenImpCasts();
858 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) {
859 Base = ASE->getBase()->IgnoreParenImpCasts();
860 Idx = ASE->getIdx()->IgnoreParenImpCasts();
861
862 if (const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
863 int64_t Val = IL->getValue().getSExtValue();
864 if (Val < 0)
865 // __bdos returns 0 for negative indexes into an array in a struct.
867
868 if (Val == 0)
869 // The index is 0, so we don't need to take it into account.
870 Idx = nullptr;
871 }
872 } else {
873 // Potential pointer to another element in the struct.
874 Base = SubExpr;
875 }
876 }
877
878 // Get the flexible array member Decl.
879 const ValueDecl *FAMDecl = nullptr;
880 if (const auto *ME = dyn_cast<MemberExpr>(Base)) {
881 // Check if \p Base is referencing the FAM itself.
882 if (const ValueDecl *MD = ME->getMemberDecl()) {
883 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
884 getLangOpts().getStrictFlexArraysLevel();
886 Ctx, MD, MD->getType(), StrictFlexArraysLevel,
887 /*IgnoreTemplateOrMacroSubstitution=*/true))
888 return nullptr;
889
890 FAMDecl = MD;
891 }
892 } else if (const auto *DRE = dyn_cast<DeclRefExpr>(Base)) {
893 // Check if we're pointing to the whole struct.
894 QualType Ty = DRE->getDecl()->getType();
895 if (Ty->isPointerType())
896 Ty = Ty->getPointeeType();
897
898 if (const auto *RD = Ty->getAsRecordDecl())
899 // Don't use the outer lexical record because the FAM might be in a
900 // different RecordDecl.
901 FAMDecl = FindFlexibleArrayMemberField(Ctx, RD);
902 }
903
904 if (!FAMDecl || !FAMDecl->hasAttr<CountedByAttr>())
905 // No flexible array member found or it doesn't have the "counted_by"
906 // attribute.
907 return nullptr;
908
909 const ValueDecl *CountedByFD = FindCountedByField(Base);
910 if (!CountedByFD)
911 // Can't find the field referenced by the "counted_by" attribute.
912 return nullptr;
913
914 // Build a load of the counted_by field.
915 bool IsSigned = CountedByFD->getType()->isSignedIntegerType();
916 const Expr *CountedByExpr = BuildCountedByFieldExpr(Base, CountedByFD);
917 Value *CountedByInst = EmitAnyExprToTemp(CountedByExpr).getScalarVal();
918 llvm::Type *CountedByTy = CountedByInst->getType();
919
920 // Build a load of the index and subtract it from the count.
921 Value *IdxInst = nullptr;
922 if (Idx) {
923 bool IdxSigned = Idx->getType()->isSignedIntegerType();
924 IdxInst = EmitAnyExprToTemp(Idx).getScalarVal();
925 IdxInst = IdxSigned ? Builder.CreateSExtOrTrunc(IdxInst, CountedByTy)
926 : Builder.CreateZExtOrTrunc(IdxInst, CountedByTy);
927
928 // We go ahead with the calculation here. If the index turns out to be
929 // negative, we'll catch it at the end.
930 CountedByInst =
931 Builder.CreateSub(CountedByInst, IdxInst, "", !IsSigned, IsSigned);
932 }
933
934 // Calculate how large the flexible array member is in bytes.
935 const ArrayType *ArrayTy = Ctx.getAsArrayType(FAMDecl->getType());
937 llvm::Constant *ElemSize =
938 llvm::ConstantInt::get(CountedByTy, Size.getQuantity(), IsSigned);
939 Value *FAMSize =
940 Builder.CreateMul(CountedByInst, ElemSize, "", !IsSigned, IsSigned);
941 FAMSize = IsSigned ? Builder.CreateSExtOrTrunc(FAMSize, ResType)
942 : Builder.CreateZExtOrTrunc(FAMSize, ResType);
943 Value *Res = FAMSize;
944
945 if (const auto *DRE = dyn_cast<DeclRefExpr>(Base)) {
946 // The whole struct is specificed in the __bdos.
947 const RecordDecl *OuterRD =
949 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(OuterRD);
950
951 // Get the offset of the FAM.
952 CharUnits Offset = Ctx.toCharUnitsFromBits(Ctx.getFieldOffset(FAMDecl));
953 llvm::Constant *FAMOffset =
954 ConstantInt::get(ResType, Offset.getQuantity(), IsSigned);
955 Value *OffsetAndFAMSize =
956 Builder.CreateAdd(FAMOffset, Res, "", !IsSigned, IsSigned);
957
958 // Get the full size of the struct.
959 llvm::Constant *SizeofStruct =
960 ConstantInt::get(ResType, Layout.getSize().getQuantity(), IsSigned);
961
962 // max(sizeof(struct s),
963 // offsetof(struct s, array) + p->count * sizeof(*p->array))
964 Res = IsSigned
965 ? Builder.CreateBinaryIntrinsic(llvm::Intrinsic::smax,
966 OffsetAndFAMSize, SizeofStruct)
967 : Builder.CreateBinaryIntrinsic(llvm::Intrinsic::umax,
968 OffsetAndFAMSize, SizeofStruct);
969 }
970
971 // A negative \p IdxInst or \p CountedByInst means that the index lands
972 // outside of the flexible array member. If that's the case, we want to
973 // return 0.
974 Value *Cmp = Builder.CreateIsNotNeg(CountedByInst);
975 if (IdxInst)
976 Cmp = Builder.CreateAnd(Builder.CreateIsNotNeg(IdxInst), Cmp);
977
978 return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
979}
980
981/// Returns a Value corresponding to the size of the given expression.
982/// This Value may be either of the following:
983/// - A llvm::Argument (if E is a param with the pass_object_size attribute on
984/// it)
985/// - A call to the @llvm.objectsize intrinsic
986///
987/// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
988/// and we wouldn't otherwise try to reference a pass_object_size parameter,
989/// we'll call @llvm.objectsize on EmittedE, rather than emitting E.
990llvm::Value *
991CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
992 llvm::IntegerType *ResType,
993 llvm::Value *EmittedE, bool IsDynamic) {
994 // We need to reference an argument if the pointer is a parameter with the
995 // pass_object_size attribute.
996 if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) {
997 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
998 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
999 if (Param != nullptr && PS != nullptr &&
1000 areBOSTypesCompatible(PS->getType(), Type)) {
1001 auto Iter = SizeArguments.find(Param);
1002 assert(Iter != SizeArguments.end());
1003
1004 const ImplicitParamDecl *D = Iter->second;
1005 auto DIter = LocalDeclMap.find(D);
1006 assert(DIter != LocalDeclMap.end());
1007
1008 return EmitLoadOfScalar(DIter->second, /*Volatile=*/false,
1009 getContext().getSizeType(), E->getBeginLoc());
1010 }
1011 }
1012
1013 if (IsDynamic) {
1014 // Emit special code for a flexible array member with the "counted_by"
1015 // attribute.
1016 if (Value *V = emitFlexibleArrayMemberSize(E, Type, ResType))
1017 return V;
1018 }
1019
1020 // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
1021 // evaluate E for side-effects. In either case, we shouldn't lower to
1022 // @llvm.objectsize.
1023 if (Type == 3 || (!EmittedE && E->HasSideEffects(getContext())))
1024 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1025
1026 Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
1027 assert(Ptr->getType()->isPointerTy() &&
1028 "Non-pointer passed to __builtin_object_size?");
1029
1030 Function *F =
1031 CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
1032
1033 // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.
1034 Value *Min = Builder.getInt1((Type & 2) != 0);
1035 // For GCC compatibility, __builtin_object_size treat NULL as unknown size.
1036 Value *NullIsUnknown = Builder.getTrue();
1037 Value *Dynamic = Builder.getInt1(IsDynamic);
1038 return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic});
1039}
1040
1041namespace {
1042/// A struct to generically describe a bit test intrinsic.
1043struct BitTest {
1044 enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
1045 enum InterlockingKind : uint8_t {
1046 Unlocked,
1047 Sequential,
1048 Acquire,
1049 Release,
1050 NoFence
1051 };
1052
1053 ActionKind Action;
1054 InterlockingKind Interlocking;
1055 bool Is64Bit;
1056
1057 static BitTest decodeBitTestBuiltin(unsigned BuiltinID);
1058};
1059} // namespace
1060
1061BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
1062 switch (BuiltinID) {
1063 // Main portable variants.
1064 case Builtin::BI_bittest:
1065 return {TestOnly, Unlocked, false};
1066 case Builtin::BI_bittestandcomplement:
1067 return {Complement, Unlocked, false};
1068 case Builtin::BI_bittestandreset:
1069 return {Reset, Unlocked, false};
1070 case Builtin::BI_bittestandset:
1071 return {Set, Unlocked, false};
1072 case Builtin::BI_interlockedbittestandreset:
1073 return {Reset, Sequential, false};
1074 case Builtin::BI_interlockedbittestandset:
1075 return {Set, Sequential, false};
1076
1077 // X86-specific 64-bit variants.
1078 case Builtin::BI_bittest64:
1079 return {TestOnly, Unlocked, true};
1080 case Builtin::BI_bittestandcomplement64:
1081 return {Complement, Unlocked, true};
1082 case Builtin::BI_bittestandreset64:
1083 return {Reset, Unlocked, true};
1084 case Builtin::BI_bittestandset64:
1085 return {Set, Unlocked, true};
1086 case Builtin::BI_interlockedbittestandreset64:
1087 return {Reset, Sequential, true};
1088 case Builtin::BI_interlockedbittestandset64:
1089 return {Set, Sequential, true};
1090
1091 // ARM/AArch64-specific ordering variants.
1092 case Builtin::BI_interlockedbittestandset_acq:
1093 return {Set, Acquire, false};
1094 case Builtin::BI_interlockedbittestandset_rel:
1095 return {Set, Release, false};
1096 case Builtin::BI_interlockedbittestandset_nf:
1097 return {Set, NoFence, false};
1098 case Builtin::BI_interlockedbittestandreset_acq:
1099 return {Reset, Acquire, false};
1100 case Builtin::BI_interlockedbittestandreset_rel:
1101 return {Reset, Release, false};
1102 case Builtin::BI_interlockedbittestandreset_nf:
1103 return {Reset, NoFence, false};
1104 }
1105 llvm_unreachable("expected only bittest intrinsics");
1106}
1107
1108static char bitActionToX86BTCode(BitTest::ActionKind A) {
1109 switch (A) {
1110 case BitTest::TestOnly: return '\0';
1111 case BitTest::Complement: return 'c';
1112 case BitTest::Reset: return 'r';
1113 case BitTest::Set: return 's';
1114 }
1115 llvm_unreachable("invalid action");
1116}
1117
1119 BitTest BT,
1120 const CallExpr *E, Value *BitBase,
1121 Value *BitPos) {
1122 char Action = bitActionToX86BTCode(BT.Action);
1123 char SizeSuffix = BT.Is64Bit ? 'q' : 'l';
1124
1125 // Build the assembly.
1127 raw_svector_ostream AsmOS(Asm);
1128 if (BT.Interlocking != BitTest::Unlocked)
1129 AsmOS << "lock ";
1130 AsmOS << "bt";
1131 if (Action)
1132 AsmOS << Action;
1133 AsmOS << SizeSuffix << " $2, ($1)";
1134
1135 // Build the constraints. FIXME: We should support immediates when possible.
1136 std::string Constraints = "={@ccc},r,r,~{cc},~{memory}";
1137 std::string_view MachineClobbers = CGF.getTarget().getClobbers();
1138 if (!MachineClobbers.empty()) {
1139 Constraints += ',';
1140 Constraints += MachineClobbers;
1141 }
1142 llvm::IntegerType *IntType = llvm::IntegerType::get(
1143 CGF.getLLVMContext(),
1144 CGF.getContext().getTypeSize(E->getArg(1)->getType()));
1145 llvm::FunctionType *FTy =
1146 llvm::FunctionType::get(CGF.Int8Ty, {CGF.UnqualPtrTy, IntType}, false);
1147
1148 llvm::InlineAsm *IA =
1149 llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);
1150 return CGF.Builder.CreateCall(IA, {BitBase, BitPos});
1151}
1152
1153static llvm::AtomicOrdering
1154getBitTestAtomicOrdering(BitTest::InterlockingKind I) {
1155 switch (I) {
1156 case BitTest::Unlocked: return llvm::AtomicOrdering::NotAtomic;
1157 case BitTest::Sequential: return llvm::AtomicOrdering::SequentiallyConsistent;
1158 case BitTest::Acquire: return llvm::AtomicOrdering::Acquire;
1159 case BitTest::Release: return llvm::AtomicOrdering::Release;
1160 case BitTest::NoFence: return llvm::AtomicOrdering::Monotonic;
1161 }
1162 llvm_unreachable("invalid interlocking");
1163}
1164
1165/// Emit a _bittest* intrinsic. These intrinsics take a pointer to an array of
1166/// bits and a bit position and read and optionally modify the bit at that
1167/// position. The position index can be arbitrarily large, i.e. it can be larger
1168/// than 31 or 63, so we need an indexed load in the general case.
1169static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
1170 unsigned BuiltinID,
1171 const CallExpr *E) {
1172 Value *BitBase = CGF.EmitScalarExpr(E->getArg(0));
1173 Value *BitPos = CGF.EmitScalarExpr(E->getArg(1));
1174
1175 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1176
1177 // X86 has special BT, BTC, BTR, and BTS instructions that handle the array
1178 // indexing operation internally. Use them if possible.
1179 if (CGF.getTarget().getTriple().isX86())
1180 return EmitX86BitTestIntrinsic(CGF, BT, E, BitBase, BitPos);
1181
1182 // Otherwise, use generic code to load one byte and test the bit. Use all but
1183 // the bottom three bits as the array index, and the bottom three bits to form
1184 // a mask.
1185 // Bit = BitBaseI8[BitPos >> 3] & (1 << (BitPos & 0x7)) != 0;
1186 Value *ByteIndex = CGF.Builder.CreateAShr(
1187 BitPos, llvm::ConstantInt::get(BitPos->getType(), 3), "bittest.byteidx");
1188 Value *BitBaseI8 = CGF.Builder.CreatePointerCast(BitBase, CGF.Int8PtrTy);
1189 Address ByteAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, BitBaseI8,
1190 ByteIndex, "bittest.byteaddr"),
1191 CGF.Int8Ty, CharUnits::One());
1192 Value *PosLow =
1193 CGF.Builder.CreateAnd(CGF.Builder.CreateTrunc(BitPos, CGF.Int8Ty),
1194 llvm::ConstantInt::get(CGF.Int8Ty, 0x7));
1195
1196 // The updating instructions will need a mask.
1197 Value *Mask = nullptr;
1198 if (BT.Action != BitTest::TestOnly) {
1199 Mask = CGF.Builder.CreateShl(llvm::ConstantInt::get(CGF.Int8Ty, 1), PosLow,
1200 "bittest.mask");
1201 }
1202
1203 // Check the action and ordering of the interlocked intrinsics.
1204 llvm::AtomicOrdering Ordering = getBitTestAtomicOrdering(BT.Interlocking);
1205
1206 Value *OldByte = nullptr;
1207 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1208 // Emit a combined atomicrmw load/store operation for the interlocked
1209 // intrinsics.
1210 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1211 if (BT.Action == BitTest::Reset) {
1212 Mask = CGF.Builder.CreateNot(Mask);
1213 RMWOp = llvm::AtomicRMWInst::And;
1214 }
1215 OldByte = CGF.Builder.CreateAtomicRMW(RMWOp, ByteAddr, Mask, Ordering);
1216 } else {
1217 // Emit a plain load for the non-interlocked intrinsics.
1218 OldByte = CGF.Builder.CreateLoad(ByteAddr, "bittest.byte");
1219 Value *NewByte = nullptr;
1220 switch (BT.Action) {
1221 case BitTest::TestOnly:
1222 // Don't store anything.
1223 break;
1224 case BitTest::Complement:
1225 NewByte = CGF.Builder.CreateXor(OldByte, Mask);
1226 break;
1227 case BitTest::Reset:
1228 NewByte = CGF.Builder.CreateAnd(OldByte, CGF.Builder.CreateNot(Mask));
1229 break;
1230 case BitTest::Set:
1231 NewByte = CGF.Builder.CreateOr(OldByte, Mask);
1232 break;
1233 }
1234 if (NewByte)
1235 CGF.Builder.CreateStore(NewByte, ByteAddr);
1236 }
1237
1238 // However we loaded the old byte, either by plain load or atomicrmw, shift
1239 // the bit into the low position and mask it to 0 or 1.
1240 Value *ShiftedByte = CGF.Builder.CreateLShr(OldByte, PosLow, "bittest.shr");
1241 return CGF.Builder.CreateAnd(
1242 ShiftedByte, llvm::ConstantInt::get(CGF.Int8Ty, 1), "bittest.res");
1243}
1244
1246 unsigned BuiltinID,
1247 const CallExpr *E) {
1248 Value *Addr = CGF.EmitScalarExpr(E->getArg(0));
1249
1251 raw_svector_ostream AsmOS(Asm);
1252 llvm::IntegerType *RetType = CGF.Int32Ty;
1253
1254 switch (BuiltinID) {
1255 case clang::PPC::BI__builtin_ppc_ldarx:
1256 AsmOS << "ldarx ";
1257 RetType = CGF.Int64Ty;
1258 break;
1259 case clang::PPC::BI__builtin_ppc_lwarx:
1260 AsmOS << "lwarx ";
1261 RetType = CGF.Int32Ty;
1262 break;
1263 case clang::PPC::BI__builtin_ppc_lharx:
1264 AsmOS << "lharx ";
1265 RetType = CGF.Int16Ty;
1266 break;
1267 case clang::PPC::BI__builtin_ppc_lbarx:
1268 AsmOS << "lbarx ";
1269 RetType = CGF.Int8Ty;
1270 break;
1271 default:
1272 llvm_unreachable("Expected only PowerPC load reserve intrinsics");
1273 }
1274
1275 AsmOS << "$0, ${1:y}";
1276
1277 std::string Constraints = "=r,*Z,~{memory}";
1278 std::string_view MachineClobbers = CGF.getTarget().getClobbers();
1279 if (!MachineClobbers.empty()) {
1280 Constraints += ',';
1281 Constraints += MachineClobbers;
1282 }
1283
1284 llvm::Type *PtrType = CGF.UnqualPtrTy;
1285 llvm::FunctionType *FTy = llvm::FunctionType::get(RetType, {PtrType}, false);
1286
1287 llvm::InlineAsm *IA =
1288 llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);
1289 llvm::CallInst *CI = CGF.Builder.CreateCall(IA, {Addr});
1290 CI->addParamAttr(
1291 0, Attribute::get(CGF.getLLVMContext(), Attribute::ElementType, RetType));
1292 return CI;
1293}
1294
1295namespace {
1296enum class MSVCSetJmpKind {
1297 _setjmpex,
1298 _setjmp3,
1299 _setjmp
1300};
1301}
1302
1303/// MSVC handles setjmp a bit differently on different platforms. On every
1304/// architecture except 32-bit x86, the frame address is passed. On x86, extra
1305/// parameters can be passed as variadic arguments, but we always pass none.
1306static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind,
1307 const CallExpr *E) {
1308 llvm::Value *Arg1 = nullptr;
1309 llvm::Type *Arg1Ty = nullptr;
1310 StringRef Name;
1311 bool IsVarArg = false;
1312 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1313 Name = "_setjmp3";
1314 Arg1Ty = CGF.Int32Ty;
1315 Arg1 = llvm::ConstantInt::get(CGF.IntTy, 0);
1316 IsVarArg = true;
1317 } else {
1318 Name = SJKind == MSVCSetJmpKind::_setjmp ? "_setjmp" : "_setjmpex";
1319 Arg1Ty = CGF.Int8PtrTy;
1320 if (CGF.getTarget().getTriple().getArch() == llvm::Triple::aarch64) {
1321 Arg1 = CGF.Builder.CreateCall(
1322 CGF.CGM.getIntrinsic(Intrinsic::sponentry, CGF.AllocaInt8PtrTy));
1323 } else
1324 Arg1 = CGF.Builder.CreateCall(
1325 CGF.CGM.getIntrinsic(Intrinsic::frameaddress, CGF.AllocaInt8PtrTy),
1326 llvm::ConstantInt::get(CGF.Int32Ty, 0));
1327 }
1328
1329 // Mark the call site and declaration with ReturnsTwice.
1330 llvm::Type *ArgTypes[2] = {CGF.Int8PtrTy, Arg1Ty};
1331 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1332 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex,
1333 llvm::Attribute::ReturnsTwice);
1334 llvm::FunctionCallee SetJmpFn = CGF.CGM.CreateRuntimeFunction(
1335 llvm::FunctionType::get(CGF.IntTy, ArgTypes, IsVarArg), Name,
1336 ReturnsTwiceAttr, /*Local=*/true);
1337
1338 llvm::Value *Buf = CGF.Builder.CreateBitOrPointerCast(
1339 CGF.EmitScalarExpr(E->getArg(0)), CGF.Int8PtrTy);
1340 llvm::Value *Args[] = {Buf, Arg1};
1341 llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(SetJmpFn, Args);
1342 CB->setAttributes(ReturnsTwiceAttr);
1343 return RValue::get(CB);
1344}
1345
1346// Many of MSVC builtins are on x64, ARM and AArch64; to avoid repeating code,
1347// we handle them here.
1387 __fastfail,
1388};
1389
1390static std::optional<CodeGenFunction::MSVCIntrin>
1391translateArmToMsvcIntrin(unsigned BuiltinID) {
1392 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
1393 switch (BuiltinID) {
1394 default:
1395 return std::nullopt;
1396 case clang::ARM::BI_BitScanForward:
1397 case clang::ARM::BI_BitScanForward64:
1398 return MSVCIntrin::_BitScanForward;
1399 case clang::ARM::BI_BitScanReverse:
1400 case clang::ARM::BI_BitScanReverse64:
1401 return MSVCIntrin::_BitScanReverse;
1402 case clang::ARM::BI_InterlockedAnd64:
1403 return MSVCIntrin::_InterlockedAnd;
1404 case clang::ARM::BI_InterlockedExchange64:
1405 return MSVCIntrin::_InterlockedExchange;
1406 case clang::ARM::BI_InterlockedExchangeAdd64:
1407 return MSVCIntrin::_InterlockedExchangeAdd;
1408 case clang::ARM::BI_InterlockedExchangeSub64:
1409 return MSVCIntrin::_InterlockedExchangeSub;
1410 case clang::ARM::BI_InterlockedOr64:
1411 return MSVCIntrin::_InterlockedOr;
1412 case clang::ARM::BI_InterlockedXor64:
1413 return MSVCIntrin::_InterlockedXor;
1414 case clang::ARM::BI_InterlockedDecrement64:
1415 return MSVCIntrin::_InterlockedDecrement;
1416 case clang::ARM::BI_InterlockedIncrement64:
1417 return MSVCIntrin::_InterlockedIncrement;
1418 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1419 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1420 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1421 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1422 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1423 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1424 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1425 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1426 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1427 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1428 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1429 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1430 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1431 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1432 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1433 case clang::ARM::BI_InterlockedExchange8_acq:
1434 case clang::ARM::BI_InterlockedExchange16_acq:
1435 case clang::ARM::BI_InterlockedExchange_acq:
1436 case clang::ARM::BI_InterlockedExchange64_acq:
1437 return MSVCIntrin::_InterlockedExchange_acq;
1438 case clang::ARM::BI_InterlockedExchange8_rel:
1439 case clang::ARM::BI_InterlockedExchange16_rel:
1440 case clang::ARM::BI_InterlockedExchange_rel:
1441 case clang::ARM::BI_InterlockedExchange64_rel:
1442 return MSVCIntrin::_InterlockedExchange_rel;
1443 case clang::ARM::BI_InterlockedExchange8_nf:
1444 case clang::ARM::BI_InterlockedExchange16_nf:
1445 case clang::ARM::BI_InterlockedExchange_nf:
1446 case clang::ARM::BI_InterlockedExchange64_nf:
1447 return MSVCIntrin::_InterlockedExchange_nf;
1448 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1449 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1450 case clang::ARM::BI_InterlockedCompareExchange_acq:
1451 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1452 return MSVCIntrin::_InterlockedCompareExchange_acq;
1453 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1454 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1455 case clang::ARM::BI_InterlockedCompareExchange_rel:
1456 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1457 return MSVCIntrin::_InterlockedCompareExchange_rel;
1458 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1459 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1460 case clang::ARM::BI_InterlockedCompareExchange_nf:
1461 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1462 return MSVCIntrin::_InterlockedCompareExchange_nf;
1463 case clang::ARM::BI_InterlockedOr8_acq:
1464 case clang::ARM::BI_InterlockedOr16_acq:
1465 case clang::ARM::BI_InterlockedOr_acq:
1466 case clang::ARM::BI_InterlockedOr64_acq:
1467 return MSVCIntrin::_InterlockedOr_acq;
1468 case clang::ARM::BI_InterlockedOr8_rel:
1469 case clang::ARM::BI_InterlockedOr16_rel:
1470 case clang::ARM::BI_InterlockedOr_rel:
1471 case clang::ARM::BI_InterlockedOr64_rel:
1472 return MSVCIntrin::_InterlockedOr_rel;
1473 case clang::ARM::BI_InterlockedOr8_nf:
1474 case clang::ARM::BI_InterlockedOr16_nf:
1475 case clang::ARM::BI_InterlockedOr_nf:
1476 case clang::ARM::BI_InterlockedOr64_nf:
1477 return MSVCIntrin::_InterlockedOr_nf;
1478 case clang::ARM::BI_InterlockedXor8_acq:
1479 case clang::ARM::BI_InterlockedXor16_acq:
1480 case clang::ARM::BI_InterlockedXor_acq:
1481 case clang::ARM::BI_InterlockedXor64_acq:
1482 return MSVCIntrin::_InterlockedXor_acq;
1483 case clang::ARM::BI_InterlockedXor8_rel:
1484 case clang::ARM::BI_InterlockedXor16_rel:
1485 case clang::ARM::BI_InterlockedXor_rel:
1486 case clang::ARM::BI_InterlockedXor64_rel:
1487 return MSVCIntrin::_InterlockedXor_rel;
1488 case clang::ARM::BI_InterlockedXor8_nf:
1489 case clang::ARM::BI_InterlockedXor16_nf:
1490 case clang::ARM::BI_InterlockedXor_nf:
1491 case clang::ARM::BI_InterlockedXor64_nf:
1492 return MSVCIntrin::_InterlockedXor_nf;
1493 case clang::ARM::BI_InterlockedAnd8_acq:
1494 case clang::ARM::BI_InterlockedAnd16_acq:
1495 case clang::ARM::BI_InterlockedAnd_acq:
1496 case clang::ARM::BI_InterlockedAnd64_acq:
1497 return MSVCIntrin::_InterlockedAnd_acq;
1498 case clang::ARM::BI_InterlockedAnd8_rel:
1499 case clang::ARM::BI_InterlockedAnd16_rel:
1500 case clang::ARM::BI_InterlockedAnd_rel:
1501 case clang::ARM::BI_InterlockedAnd64_rel:
1502 return MSVCIntrin::_InterlockedAnd_rel;
1503 case clang::ARM::BI_InterlockedAnd8_nf:
1504 case clang::ARM::BI_InterlockedAnd16_nf:
1505 case clang::ARM::BI_InterlockedAnd_nf:
1506 case clang::ARM::BI_InterlockedAnd64_nf:
1507 return MSVCIntrin::_InterlockedAnd_nf;
1508 case clang::ARM::BI_InterlockedIncrement16_acq:
1509 case clang::ARM::BI_InterlockedIncrement_acq:
1510 case clang::ARM::BI_InterlockedIncrement64_acq:
1511 return MSVCIntrin::_InterlockedIncrement_acq;
1512 case clang::ARM::BI_InterlockedIncrement16_rel:
1513 case clang::ARM::BI_InterlockedIncrement_rel:
1514 case clang::ARM::BI_InterlockedIncrement64_rel:
1515 return MSVCIntrin::_InterlockedIncrement_rel;
1516 case clang::ARM::BI_InterlockedIncrement16_nf:
1517 case clang::ARM::BI_InterlockedIncrement_nf:
1518 case clang::ARM::BI_InterlockedIncrement64_nf:
1519 return MSVCIntrin::_InterlockedIncrement_nf;
1520 case clang::ARM::BI_InterlockedDecrement16_acq:
1521 case clang::ARM::BI_InterlockedDecrement_acq:
1522 case clang::ARM::BI_InterlockedDecrement64_acq:
1523 return MSVCIntrin::_InterlockedDecrement_acq;
1524 case clang::ARM::BI_InterlockedDecrement16_rel:
1525 case clang::ARM::BI_InterlockedDecrement_rel:
1526 case clang::ARM::BI_InterlockedDecrement64_rel:
1527 return MSVCIntrin::_InterlockedDecrement_rel;
1528 case clang::ARM::BI_InterlockedDecrement16_nf:
1529 case clang::ARM::BI_InterlockedDecrement_nf:
1530 case clang::ARM::BI_InterlockedDecrement64_nf:
1531 return MSVCIntrin::_InterlockedDecrement_nf;
1532 }
1533 llvm_unreachable("must return from switch");
1534}
1535
1536static std::optional<CodeGenFunction::MSVCIntrin>
1537translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
1538 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
1539 switch (BuiltinID) {
1540 default:
1541 return std::nullopt;
1542 case clang::AArch64::BI_BitScanForward:
1543 case clang::AArch64::BI_BitScanForward64:
1544 return MSVCIntrin::_BitScanForward;
1545 case clang::AArch64::BI_BitScanReverse:
1546 case clang::AArch64::BI_BitScanReverse64:
1547 return MSVCIntrin::_BitScanReverse;
1548 case clang::AArch64::BI_InterlockedAnd64:
1549 return MSVCIntrin::_InterlockedAnd;
1550 case clang::AArch64::BI_InterlockedExchange64:
1551 return MSVCIntrin::_InterlockedExchange;
1552 case clang::AArch64::BI_InterlockedExchangeAdd64:
1553 return MSVCIntrin::_InterlockedExchangeAdd;
1554 case clang::AArch64::BI_InterlockedExchangeSub64:
1555 return MSVCIntrin::_InterlockedExchangeSub;
1556 case clang::AArch64::BI_InterlockedOr64:
1557 return MSVCIntrin::_InterlockedOr;
1558 case clang::AArch64::BI_InterlockedXor64:
1559 return MSVCIntrin::_InterlockedXor;
1560 case clang::AArch64::BI_InterlockedDecrement64:
1561 return MSVCIntrin::_InterlockedDecrement;
1562 case clang::AArch64::BI_InterlockedIncrement64:
1563 return MSVCIntrin::_InterlockedIncrement;
1564 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1565 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1566 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1567 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1568 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1569 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1570 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1571 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1572 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1573 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1574 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1575 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1576 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1577 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1578 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1579 case clang::AArch64::BI_InterlockedExchange8_acq:
1580 case clang::AArch64::BI_InterlockedExchange16_acq:
1581 case clang::AArch64::BI_InterlockedExchange_acq:
1582 case clang::AArch64::BI_InterlockedExchange64_acq:
1583 return MSVCIntrin::_InterlockedExchange_acq;
1584 case clang::AArch64::BI_InterlockedExchange8_rel:
1585 case clang::AArch64::BI_InterlockedExchange16_rel:
1586 case clang::AArch64::BI_InterlockedExchange_rel:
1587 case clang::AArch64::BI_InterlockedExchange64_rel:
1588 return MSVCIntrin::_InterlockedExchange_rel;
1589 case clang::AArch64::BI_InterlockedExchange8_nf:
1590 case clang::AArch64::BI_InterlockedExchange16_nf:
1591 case clang::AArch64::BI_InterlockedExchange_nf:
1592 case clang::AArch64::BI_InterlockedExchange64_nf:
1593 return MSVCIntrin::_InterlockedExchange_nf;
1594 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1595 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1596 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1597 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1598 return MSVCIntrin::_InterlockedCompareExchange_acq;
1599 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1600 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1601 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1602 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1603 return MSVCIntrin::_InterlockedCompareExchange_rel;
1604 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1605 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1606 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1607 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1608 return MSVCIntrin::_InterlockedCompareExchange_nf;
1609 case clang::AArch64::BI_InterlockedCompareExchange128:
1610 return MSVCIntrin::_InterlockedCompareExchange128;
1611 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1612 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1613 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1614 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1615 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1616 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1617 case clang::AArch64::BI_InterlockedOr8_acq:
1618 case clang::AArch64::BI_InterlockedOr16_acq:
1619 case clang::AArch64::BI_InterlockedOr_acq:
1620 case clang::AArch64::BI_InterlockedOr64_acq:
1621 return MSVCIntrin::_InterlockedOr_acq;
1622 case clang::AArch64::BI_InterlockedOr8_rel:
1623 case clang::AArch64::BI_InterlockedOr16_rel:
1624 case clang::AArch64::BI_InterlockedOr_rel:
1625 case clang::AArch64::BI_InterlockedOr64_rel:
1626 return MSVCIntrin::_InterlockedOr_rel;
1627 case clang::AArch64::BI_InterlockedOr8_nf:
1628 case clang::AArch64::BI_InterlockedOr16_nf:
1629 case clang::AArch64::BI_InterlockedOr_nf:
1630 case clang::AArch64::BI_InterlockedOr64_nf:
1631 return MSVCIntrin::_InterlockedOr_nf;
1632 case clang::AArch64::BI_InterlockedXor8_acq:
1633 case clang::AArch64::BI_InterlockedXor16_acq:
1634 case clang::AArch64::BI_InterlockedXor_acq:
1635 case clang::AArch64::BI_InterlockedXor64_acq:
1636 return MSVCIntrin::_InterlockedXor_acq;
1637 case clang::AArch64::BI_InterlockedXor8_rel:
1638 case clang::AArch64::BI_InterlockedXor16_rel:
1639 case clang::AArch64::BI_InterlockedXor_rel:
1640 case clang::AArch64::BI_InterlockedXor64_rel:
1641 return MSVCIntrin::_InterlockedXor_rel;
1642 case clang::AArch64::BI_InterlockedXor8_nf:
1643 case clang::AArch64::BI_InterlockedXor16_nf:
1644 case clang::AArch64::BI_InterlockedXor_nf:
1645 case clang::AArch64::BI_InterlockedXor64_nf:
1646 return MSVCIntrin::_InterlockedXor_nf;
1647 case clang::AArch64::BI_InterlockedAnd8_acq:
1648 case clang::AArch64::BI_InterlockedAnd16_acq:
1649 case clang::AArch64::BI_InterlockedAnd_acq:
1650 case clang::AArch64::BI_InterlockedAnd64_acq:
1651 return MSVCIntrin::_InterlockedAnd_acq;
1652 case clang::AArch64::BI_InterlockedAnd8_rel:
1653 case clang::AArch64::BI_InterlockedAnd16_rel:
1654 case clang::AArch64::BI_InterlockedAnd_rel:
1655 case clang::AArch64::BI_InterlockedAnd64_rel:
1656 return MSVCIntrin::_InterlockedAnd_rel;
1657 case clang::AArch64::BI_InterlockedAnd8_nf:
1658 case clang::AArch64::BI_InterlockedAnd16_nf:
1659 case clang::AArch64::BI_InterlockedAnd_nf:
1660 case clang::AArch64::BI_InterlockedAnd64_nf:
1661 return MSVCIntrin::_InterlockedAnd_nf;
1662 case clang::AArch64::BI_InterlockedIncrement16_acq:
1663 case clang::AArch64::BI_InterlockedIncrement_acq:
1664 case clang::AArch64::BI_InterlockedIncrement64_acq:
1665 return MSVCIntrin::_InterlockedIncrement_acq;
1666 case clang::AArch64::BI_InterlockedIncrement16_rel:
1667 case clang::AArch64::BI_InterlockedIncrement_rel:
1668 case clang::AArch64::BI_InterlockedIncrement64_rel:
1669 return MSVCIntrin::_InterlockedIncrement_rel;
1670 case clang::AArch64::BI_InterlockedIncrement16_nf:
1671 case clang::AArch64::BI_InterlockedIncrement_nf:
1672 case clang::AArch64::BI_InterlockedIncrement64_nf:
1673 return MSVCIntrin::_InterlockedIncrement_nf;
1674 case clang::AArch64::BI_InterlockedDecrement16_acq:
1675 case clang::AArch64::BI_InterlockedDecrement_acq:
1676 case clang::AArch64::BI_InterlockedDecrement64_acq:
1677 return MSVCIntrin::_InterlockedDecrement_acq;
1678 case clang::AArch64::BI_InterlockedDecrement16_rel:
1679 case clang::AArch64::BI_InterlockedDecrement_rel:
1680 case clang::AArch64::BI_InterlockedDecrement64_rel:
1681 return MSVCIntrin::_InterlockedDecrement_rel;
1682 case clang::AArch64::BI_InterlockedDecrement16_nf:
1683 case clang::AArch64::BI_InterlockedDecrement_nf:
1684 case clang::AArch64::BI_InterlockedDecrement64_nf:
1685 return MSVCIntrin::_InterlockedDecrement_nf;
1686 }
1687 llvm_unreachable("must return from switch");
1688}
1689
1690static std::optional<CodeGenFunction::MSVCIntrin>
1691translateX86ToMsvcIntrin(unsigned BuiltinID) {
1692 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
1693 switch (BuiltinID) {
1694 default:
1695 return std::nullopt;
1696 case clang::X86::BI_BitScanForward:
1697 case clang::X86::BI_BitScanForward64:
1698 return MSVCIntrin::_BitScanForward;
1699 case clang::X86::BI_BitScanReverse:
1700 case clang::X86::BI_BitScanReverse64:
1701 return MSVCIntrin::_BitScanReverse;
1702 case clang::X86::BI_InterlockedAnd64:
1703 return MSVCIntrin::_InterlockedAnd;
1704 case clang::X86::BI_InterlockedCompareExchange128:
1705 return MSVCIntrin::_InterlockedCompareExchange128;
1706 case clang::X86::BI_InterlockedExchange64:
1707 return MSVCIntrin::_InterlockedExchange;
1708 case clang::X86::BI_InterlockedExchangeAdd64:
1709 return MSVCIntrin::_InterlockedExchangeAdd;
1710 case clang::X86::BI_InterlockedExchangeSub64:
1711 return MSVCIntrin::_InterlockedExchangeSub;
1712 case clang::X86::BI_InterlockedOr64:
1713 return MSVCIntrin::_InterlockedOr;
1714 case clang::X86::BI_InterlockedXor64:
1715 return MSVCIntrin::_InterlockedXor;
1716 case clang::X86::BI_InterlockedDecrement64:
1717 return MSVCIntrin::_InterlockedDecrement;
1718 case clang::X86::BI_InterlockedIncrement64:
1719 return MSVCIntrin::_InterlockedIncrement;
1720 }
1721 llvm_unreachable("must return from switch");
1722}
1723
1724// Emit an MSVC intrinsic. Assumes that arguments have *not* been evaluated.
1725Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
1726 const CallExpr *E) {
1727 switch (BuiltinID) {
1728 case MSVCIntrin::_BitScanForward:
1729 case MSVCIntrin::_BitScanReverse: {
1730 Address IndexAddress(EmitPointerWithAlignment(E->getArg(0)));
1731 Value *ArgValue = EmitScalarExpr(E->getArg(1));
1732
1733 llvm::Type *ArgType = ArgValue->getType();
1734 llvm::Type *IndexType = IndexAddress.getElementType();
1735 llvm::Type *ResultType = ConvertType(E->getType());
1736
1737 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1738 Value *ResZero = llvm::Constant::getNullValue(ResultType);
1739 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
1740
1741 BasicBlock *Begin = Builder.GetInsertBlock();
1742 BasicBlock *End = createBasicBlock("bitscan_end", this->CurFn);
1743 Builder.SetInsertPoint(End);
1744 PHINode *Result = Builder.CreatePHI(ResultType, 2, "bitscan_result");
1745
1746 Builder.SetInsertPoint(Begin);
1747 Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
1748 BasicBlock *NotZero = createBasicBlock("bitscan_not_zero", this->CurFn);
1749 Builder.CreateCondBr(IsZero, End, NotZero);
1750 Result->addIncoming(ResZero, Begin);
1751
1752 Builder.SetInsertPoint(NotZero);
1753
1754 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1755 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1756 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1757 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
1758 Builder.CreateStore(ZeroCount, IndexAddress, false);
1759 } else {
1760 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
1761 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
1762
1763 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
1764 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1765 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
1766 Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
1767 Builder.CreateStore(Index, IndexAddress, false);
1768 }
1769 Builder.CreateBr(End);
1770 Result->addIncoming(ResOne, NotZero);
1771
1772 Builder.SetInsertPoint(End);
1773 return Result;
1774 }
1775 case MSVCIntrin::_InterlockedAnd:
1776 return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E);
1777 case MSVCIntrin::_InterlockedExchange:
1778 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E);
1779 case MSVCIntrin::_InterlockedExchangeAdd:
1780 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E);
1781 case MSVCIntrin::_InterlockedExchangeSub:
1782 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Sub, E);
1783 case MSVCIntrin::_InterlockedOr:
1784 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E);
1785 case MSVCIntrin::_InterlockedXor:
1786 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E);
1787 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1788 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
1789 AtomicOrdering::Acquire);
1790 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1791 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
1792 AtomicOrdering::Release);
1793 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1794 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
1795 AtomicOrdering::Monotonic);
1796 case MSVCIntrin::_InterlockedExchange_acq:
1797 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
1798 AtomicOrdering::Acquire);
1799 case MSVCIntrin::_InterlockedExchange_rel:
1800 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
1801 AtomicOrdering::Release);
1802 case MSVCIntrin::_InterlockedExchange_nf:
1803 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
1804 AtomicOrdering::Monotonic);
1805 case MSVCIntrin::_InterlockedCompareExchange_acq:
1806 return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Acquire);
1807 case MSVCIntrin::_InterlockedCompareExchange_rel:
1808 return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Release);
1809 case MSVCIntrin::_InterlockedCompareExchange_nf:
1810 return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Monotonic);
1811 case MSVCIntrin::_InterlockedCompareExchange128:
1813 *this, E, AtomicOrdering::SequentiallyConsistent);
1814 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1815 return EmitAtomicCmpXchg128ForMSIntrin(*this, E, AtomicOrdering::Acquire);
1816 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1817 return EmitAtomicCmpXchg128ForMSIntrin(*this, E, AtomicOrdering::Release);
1818 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1819 return EmitAtomicCmpXchg128ForMSIntrin(*this, E, AtomicOrdering::Monotonic);
1820 case MSVCIntrin::_InterlockedOr_acq:
1821 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
1822 AtomicOrdering::Acquire);
1823 case MSVCIntrin::_InterlockedOr_rel:
1824 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
1825 AtomicOrdering::Release);
1826 case MSVCIntrin::_InterlockedOr_nf:
1827 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
1828 AtomicOrdering::Monotonic);
1829 case MSVCIntrin::_InterlockedXor_acq:
1830 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
1831 AtomicOrdering::Acquire);
1832 case MSVCIntrin::_InterlockedXor_rel:
1833 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
1834 AtomicOrdering::Release);
1835 case MSVCIntrin::_InterlockedXor_nf:
1836 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
1837 AtomicOrdering::Monotonic);
1838 case MSVCIntrin::_InterlockedAnd_acq:
1839 return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
1840 AtomicOrdering::Acquire);
1841 case MSVCIntrin::_InterlockedAnd_rel:
1842 return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
1843 AtomicOrdering::Release);
1844 case MSVCIntrin::_InterlockedAnd_nf:
1845 return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
1846 AtomicOrdering::Monotonic);
1847 case MSVCIntrin::_InterlockedIncrement_acq:
1848 return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Acquire);
1849 case MSVCIntrin::_InterlockedIncrement_rel:
1850 return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Release);
1851 case MSVCIntrin::_InterlockedIncrement_nf:
1852 return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Monotonic);
1853 case MSVCIntrin::_InterlockedDecrement_acq:
1854 return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Acquire);
1855 case MSVCIntrin::_InterlockedDecrement_rel:
1856 return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Release);
1857 case MSVCIntrin::_InterlockedDecrement_nf:
1858 return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Monotonic);
1859
1860 case MSVCIntrin::_InterlockedDecrement:
1861 return EmitAtomicDecrementValue(*this, E);
1862 case MSVCIntrin::_InterlockedIncrement:
1863 return EmitAtomicIncrementValue(*this, E);
1864
1865 case MSVCIntrin::__fastfail: {
1866 // Request immediate process termination from the kernel. The instruction
1867 // sequences to do this are documented on MSDN:
1868 // https://msdn.microsoft.com/en-us/library/dn774154.aspx
1869 llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
1870 StringRef Asm, Constraints;
1871 switch (ISA) {
1872 default:
1873 ErrorUnsupported(E, "__fastfail call for this architecture");
1874 break;
1875 case llvm::Triple::x86:
1876 case llvm::Triple::x86_64:
1877 Asm = "int $$0x29";
1878 Constraints = "{cx}";
1879 break;
1880 case llvm::Triple::thumb:
1881 Asm = "udf #251";
1882 Constraints = "{r0}";
1883 break;
1884 case llvm::Triple::aarch64:
1885 Asm = "brk #0xF003";
1886 Constraints = "{w0}";
1887 }
1888 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, {Int32Ty}, false);
1889 llvm::InlineAsm *IA =
1890 llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);
1891 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1892 getLLVMContext(), llvm::AttributeList::FunctionIndex,
1893 llvm::Attribute::NoReturn);
1894 llvm::CallInst *CI = Builder.CreateCall(IA, EmitScalarExpr(E->getArg(0)));
1895 CI->setAttributes(NoReturnAttr);
1896 return CI;
1897 }
1898 }
1899 llvm_unreachable("Incorrect MSVC intrinsic!");
1900}
1901
1902namespace {
1903// ARC cleanup for __builtin_os_log_format
1904struct CallObjCArcUse final : EHScopeStack::Cleanup {
1905 CallObjCArcUse(llvm::Value *object) : object(object) {}
1906 llvm::Value *object;
1907
1908 void Emit(CodeGenFunction &CGF, Flags flags) override {
1909 CGF.EmitARCIntrinsicUse(object);
1910 }
1911};
1912}
1913
1915 BuiltinCheckKind Kind) {
1916 assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero)
1917 && "Unsupported builtin check kind");
1918
1919 Value *ArgValue = EmitScalarExpr(E);
1920 if (!SanOpts.has(SanitizerKind::Builtin))
1921 return ArgValue;
1922
1923 SanitizerScope SanScope(this);
1924 Value *Cond = Builder.CreateICmpNE(
1925 ArgValue, llvm::Constant::getNullValue(ArgValue->getType()));
1926 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
1927 SanitizerHandler::InvalidBuiltin,
1929 llvm::ConstantInt::get(Builder.getInt8Ty(), Kind)},
1930 std::nullopt);
1931 return ArgValue;
1932}
1933
1934static Value *EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW) {
1935 return CGF.Builder.CreateBinaryIntrinsic(
1936 Intrinsic::abs, ArgValue,
1937 ConstantInt::get(CGF.Builder.getInt1Ty(), HasNSW));
1938}
1939
1941 bool SanitizeOverflow) {
1942 Value *ArgValue = CGF.EmitScalarExpr(E->getArg(0));
1943
1944 // Try to eliminate overflow check.
1945 if (const auto *VCI = dyn_cast<llvm::ConstantInt>(ArgValue)) {
1946 if (!VCI->isMinSignedValue())
1947 return EmitAbs(CGF, ArgValue, true);
1948 }
1949
1950 CodeGenFunction::SanitizerScope SanScope(&CGF);
1951
1952 Constant *Zero = Constant::getNullValue(ArgValue->getType());
1953 Value *ResultAndOverflow = CGF.Builder.CreateBinaryIntrinsic(
1954 Intrinsic::ssub_with_overflow, Zero, ArgValue);
1955 Value *Result = CGF.Builder.CreateExtractValue(ResultAndOverflow, 0);
1956 Value *NotOverflow = CGF.Builder.CreateNot(
1957 CGF.Builder.CreateExtractValue(ResultAndOverflow, 1));
1958
1959 // TODO: support -ftrapv-handler.
1960 if (SanitizeOverflow) {
1961 CGF.EmitCheck({{NotOverflow, SanitizerKind::SignedIntegerOverflow}},
1962 SanitizerHandler::NegateOverflow,
1965 {ArgValue});
1966 } else
1967 CGF.EmitTrapCheck(NotOverflow, SanitizerHandler::SubOverflow);
1968
1969 Value *CmpResult = CGF.Builder.CreateICmpSLT(ArgValue, Zero, "abscond");
1970 return CGF.Builder.CreateSelect(CmpResult, Result, ArgValue, "abs");
1971}
1972
1973/// Get the argument type for arguments to os_log_helper.
1975 QualType UnsignedTy = C.getIntTypeForBitwidth(Size * 8, /*Signed=*/false);
1976 return C.getCanonicalType(UnsignedTy);
1977}
1978
1981 CharUnits BufferAlignment) {
1982 ASTContext &Ctx = getContext();
1983
1985 {
1986 raw_svector_ostream OS(Name);
1987 OS << "__os_log_helper";
1988 OS << "_" << BufferAlignment.getQuantity();
1989 OS << "_" << int(Layout.getSummaryByte());
1990 OS << "_" << int(Layout.getNumArgsByte());
1991 for (const auto &Item : Layout.Items)
1992 OS << "_" << int(Item.getSizeByte()) << "_"
1993 << int(Item.getDescriptorByte());
1994 }
1995
1996 if (llvm::Function *F = CGM.getModule().getFunction(Name))
1997 return F;
1998
2000 FunctionArgList Args;
2001 Args.push_back(ImplicitParamDecl::Create(
2002 Ctx, nullptr, SourceLocation(), &Ctx.Idents.get("buffer"), Ctx.VoidPtrTy,
2004 ArgTys.emplace_back(Ctx.VoidPtrTy);
2005
2006 for (unsigned int I = 0, E = Layout.Items.size(); I < E; ++I) {
2007 char Size = Layout.Items[I].getSizeByte();
2008 if (!Size)
2009 continue;
2010
2011 QualType ArgTy = getOSLogArgType(Ctx, Size);
2012 Args.push_back(ImplicitParamDecl::Create(
2013 Ctx, nullptr, SourceLocation(),
2014 &Ctx.Idents.get(std::string("arg") + llvm::to_string(I)), ArgTy,
2016 ArgTys.emplace_back(ArgTy);
2017 }
2018
2019 QualType ReturnTy = Ctx.VoidTy;
2020
2021 // The helper function has linkonce_odr linkage to enable the linker to merge
2022 // identical functions. To ensure the merging always happens, 'noinline' is
2023 // attached to the function when compiling with -Oz.
2024 const CGFunctionInfo &FI =
2026 llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
2027 llvm::Function *Fn = llvm::Function::Create(
2028 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &CGM.getModule());
2029 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2030 CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, Fn, /*IsThunk=*/false);
2032 Fn->setDoesNotThrow();
2033
2034 // Attach 'noinline' at -Oz.
2035 if (CGM.getCodeGenOpts().OptimizeSize == 2)
2036 Fn->addFnAttr(llvm::Attribute::NoInline);
2037
2038 auto NL = ApplyDebugLocation::CreateEmpty(*this);
2039 StartFunction(GlobalDecl(), ReturnTy, Fn, FI, Args);
2040
2041 // Create a scope with an artificial location for the body of this function.
2042 auto AL = ApplyDebugLocation::CreateArtificial(*this);
2043
2044 CharUnits Offset;
2045 Address BufAddr =
2047 BufferAlignment);
2048 Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()),
2049 Builder.CreateConstByteGEP(BufAddr, Offset++, "summary"));
2050 Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()),
2051 Builder.CreateConstByteGEP(BufAddr, Offset++, "numArgs"));
2052
2053 unsigned I = 1;
2054 for (const auto &Item : Layout.Items) {
2056 Builder.getInt8(Item.getDescriptorByte()),
2057 Builder.CreateConstByteGEP(BufAddr, Offset++, "argDescriptor"));
2059 Builder.getInt8(Item.getSizeByte()),
2060 Builder.CreateConstByteGEP(BufAddr, Offset++, "argSize"));
2061
2062 CharUnits Size = Item.size();
2063 if (!Size.getQuantity())
2064 continue;
2065
2066 Address Arg = GetAddrOfLocalVar(Args[I]);
2067 Address Addr = Builder.CreateConstByteGEP(BufAddr, Offset, "argData");
2068 Addr = Addr.withElementType(Arg.getElementType());
2070 Offset += Size;
2071 ++I;
2072 }
2073
2075
2076 return Fn;
2077}
2078
2080 assert(E.getNumArgs() >= 2 &&
2081 "__builtin_os_log_format takes at least 2 arguments");
2082 ASTContext &Ctx = getContext();
2085 Address BufAddr = EmitPointerWithAlignment(E.getArg(0));
2086 llvm::SmallVector<llvm::Value *, 4> RetainableOperands;
2087
2088 // Ignore argument 1, the format string. It is not currently used.
2089 CallArgList Args;
2090 Args.add(RValue::get(BufAddr.getPointer()), Ctx.VoidPtrTy);
2091
2092 for (const auto &Item : Layout.Items) {
2093 int Size = Item.getSizeByte();
2094 if (!Size)
2095 continue;
2096
2097 llvm::Value *ArgVal;
2098
2099 if (Item.getKind() == analyze_os_log::OSLogBufferItem::MaskKind) {
2100 uint64_t Val = 0;
2101 for (unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
2102 Val |= ((uint64_t)Item.getMaskType()[I]) << I * 8;
2103 ArgVal = llvm::Constant::getIntegerValue(Int64Ty, llvm::APInt(64, Val));
2104 } else if (const Expr *TheExpr = Item.getExpr()) {
2105 ArgVal = EmitScalarExpr(TheExpr, /*Ignore*/ false);
2106
2107 // If a temporary object that requires destruction after the full
2108 // expression is passed, push a lifetime-extended cleanup to extend its
2109 // lifetime to the end of the enclosing block scope.
2110 auto LifetimeExtendObject = [&](const Expr *E) {
2111 E = E->IgnoreParenCasts();
2112 // Extend lifetimes of objects returned by function calls and message
2113 // sends.
2114
2115 // FIXME: We should do this in other cases in which temporaries are
2116 // created including arguments of non-ARC types (e.g., C++
2117 // temporaries).
2118 if (isa<CallExpr>(E) || isa<ObjCMessageExpr>(E))
2119 return true;
2120 return false;
2121 };
2122
2123 if (TheExpr->getType()->isObjCRetainableType() &&
2124 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2125 assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
2126 "Only scalar can be a ObjC retainable type");
2127 if (!isa<Constant>(ArgVal)) {
2128 CleanupKind Cleanup = getARCCleanupKind();
2129 QualType Ty = TheExpr->getType();
2130 Address Alloca = Address::invalid();
2131 Address Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca);
2132 ArgVal = EmitARCRetain(Ty, ArgVal);
2133 Builder.CreateStore(ArgVal, Addr);
2134 pushLifetimeExtendedDestroy(Cleanup, Alloca, Ty,
2136 Cleanup & EHCleanup);
2137
2138 // Push a clang.arc.use call to ensure ARC optimizer knows that the
2139 // argument has to be alive.
2140 if (CGM.getCodeGenOpts().OptimizationLevel != 0)
2141 pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
2142 }
2143 }
2144 } else {
2145 ArgVal = Builder.getInt32(Item.getConstValue().getQuantity());
2146 }
2147
2148 unsigned ArgValSize =
2149 CGM.getDataLayout().getTypeSizeInBits(ArgVal->getType());
2150 llvm::IntegerType *IntTy = llvm::Type::getIntNTy(getLLVMContext(),
2151 ArgValSize);
2152 ArgVal = Builder.CreateBitOrPointerCast(ArgVal, IntTy);
2153 CanQualType ArgTy = getOSLogArgType(Ctx, Size);
2154 // If ArgVal has type x86_fp80, zero-extend ArgVal.
2155 ArgVal = Builder.CreateZExtOrBitCast(ArgVal, ConvertType(ArgTy));
2156 Args.add(RValue::get(ArgVal), ArgTy);
2157 }
2158
2159 const CGFunctionInfo &FI =
2162 Layout, BufAddr.getAlignment());
2164 return RValue::get(BufAddr.getPointer());
2165}
2166
2168 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2169 WidthAndSignedness ResultInfo) {
2170 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2171 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2172 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2173}
2174
2176 CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info,
2177 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2178 const clang::Expr *ResultArg, QualType ResultQTy,
2179 WidthAndSignedness ResultInfo) {
2181 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2182 "Cannot specialize this multiply");
2183
2184 llvm::Value *V1 = CGF.EmitScalarExpr(Op1);
2185 llvm::Value *V2 = CGF.EmitScalarExpr(Op2);
2186
2187 llvm::Value *HasOverflow;
2188 llvm::Value *Result = EmitOverflowIntrinsic(
2189 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2190
2191 // The intrinsic call will detect overflow when the value is > UINT_MAX,
2192 // however, since the original builtin had a signed result, we need to report
2193 // an overflow when the result is greater than INT_MAX.
2194 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
2195 llvm::Value *IntMaxValue = llvm::ConstantInt::get(Result->getType(), IntMax);
2196
2197 llvm::Value *IntMaxOverflow = CGF.Builder.CreateICmpUGT(Result, IntMaxValue);
2198 HasOverflow = CGF.Builder.CreateOr(HasOverflow, IntMaxOverflow);
2199
2200 bool isVolatile =
2201 ResultArg->getType()->getPointeeType().isVolatileQualified();
2202 Address ResultPtr = CGF.EmitPointerWithAlignment(ResultArg);
2203 CGF.Builder.CreateStore(CGF.EmitToMemory(Result, ResultQTy), ResultPtr,
2204 isVolatile);
2205 return RValue::get(HasOverflow);
2206}
2207
2208/// Determine if a binop is a checked mixed-sign multiply we can specialize.
2209static bool isSpecialMixedSignMultiply(unsigned BuiltinID,
2210 WidthAndSignedness Op1Info,
2211 WidthAndSignedness Op2Info,
2212 WidthAndSignedness ResultInfo) {
2213 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2214 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2215 Op1Info.Signed != Op2Info.Signed;
2216}
2217
2218/// Emit a checked mixed-sign multiply. This is a cheaper specialization of
2219/// the generic checked-binop irgen.
2220static RValue
2222 WidthAndSignedness Op1Info, const clang::Expr *Op2,
2223 WidthAndSignedness Op2Info,
2224 const clang::Expr *ResultArg, QualType ResultQTy,
2225 WidthAndSignedness ResultInfo) {
2226 assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info,
2227 Op2Info, ResultInfo) &&
2228 "Not a mixed-sign multipliction we can specialize");
2229
2230 // Emit the signed and unsigned operands.
2231 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2232 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2233 llvm::Value *Signed = CGF.EmitScalarExpr(SignedOp);
2234 llvm::Value *Unsigned = CGF.EmitScalarExpr(UnsignedOp);
2235 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2236 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2237
2238 // One of the operands may be smaller than the other. If so, [s|z]ext it.
2239 if (SignedOpWidth < UnsignedOpWidth)
2240 Signed = CGF.Builder.CreateSExt(Signed, Unsigned->getType(), "op.sext");
2241 if (UnsignedOpWidth < SignedOpWidth)
2242 Unsigned = CGF.Builder.CreateZExt(Unsigned, Signed->getType(), "op.zext");
2243
2244 llvm::Type *OpTy = Signed->getType();
2245 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2246 Address ResultPtr = CGF.EmitPointerWithAlignment(ResultArg);
2247 llvm::Type *ResTy = ResultPtr.getElementType();
2248 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
2249
2250 // Take the absolute value of the signed operand.
2251 llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(Signed, Zero);
2252 llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(Zero, Signed);
2253 llvm::Value *AbsSigned =
2254 CGF.Builder.CreateSelect(IsNegative, AbsOfNegative, Signed);
2255
2256 // Perform a checked unsigned multiplication.
2257 llvm::Value *UnsignedOverflow;
2258 llvm::Value *UnsignedResult =
2259 EmitOverflowIntrinsic(CGF, llvm::Intrinsic::umul_with_overflow, AbsSigned,
2260 Unsigned, UnsignedOverflow);
2261
2262 llvm::Value *Overflow, *Result;
2263 if (ResultInfo.Signed) {
2264 // Signed overflow occurs if the result is greater than INT_MAX or lesser
2265 // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
2266 auto IntMax =
2267 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2268 llvm::Value *MaxResult =
2269 CGF.Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2270 CGF.Builder.CreateZExt(IsNegative, OpTy));
2271 llvm::Value *SignedOverflow =
2272 CGF.Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2273 Overflow = CGF.Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2274
2275 // Prepare the signed result (possibly by negating it).
2276 llvm::Value *NegativeResult = CGF.Builder.CreateNeg(UnsignedResult);
2277 llvm::Value *SignedResult =
2278 CGF.Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2279 Result = CGF.Builder.CreateTrunc(SignedResult, ResTy);
2280 } else {
2281 // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX.
2282 llvm::Value *Underflow = CGF.Builder.CreateAnd(
2283 IsNegative, CGF.Builder.CreateIsNotNull(UnsignedResult));
2284 Overflow = CGF.Builder.CreateOr(UnsignedOverflow, Underflow);
2285 if (ResultInfo.Width < OpWidth) {
2286 auto IntMax =
2287 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2288 llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
2289 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2290 Overflow = CGF.Builder.CreateOr(Overflow, TruncOverflow);
2291 }
2292
2293 // Negate the product if it would be negative in infinite precision.
2294 Result = CGF.Builder.CreateSelect(
2295 IsNegative, CGF.Builder.CreateNeg(UnsignedResult), UnsignedResult);
2296
2297 Result = CGF.Builder.CreateTrunc(Result, ResTy);
2298 }
2299 assert(Overflow && Result && "Missing overflow or result");
2300
2301 bool isVolatile =
2302 ResultArg->getType()->getPointeeType().isVolatileQualified();
2303 CGF.Builder.CreateStore(CGF.EmitToMemory(Result, ResultQTy), ResultPtr,
2304 isVolatile);
2305 return RValue::get(Overflow);
2306}
2307
2308static bool
2310 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2311 if (const auto *Arr = Ctx.getAsArrayType(Ty))
2312 Ty = Ctx.getBaseElementType(Arr);
2313
2314 const auto *Record = Ty->getAsCXXRecordDecl();
2315 if (!Record)
2316 return false;
2317
2318 // We've already checked this type, or are in the process of checking it.
2319 if (!Seen.insert(Record).second)
2320 return false;
2321
2322 assert(Record->hasDefinition() &&
2323 "Incomplete types should already be diagnosed");
2324
2325 if (Record->isDynamicClass())
2326 return true;
2327
2328 for (FieldDecl *F : Record->fields()) {
2329 if (TypeRequiresBuiltinLaunderImp(Ctx, F->getType(), Seen))
2330 return true;
2331 }
2332 return false;
2333}
2334
2335/// Determine if the specified type requires laundering by checking if it is a
2336/// dynamic class type or contains a subobject which is a dynamic class type.
2338 if (!CGM.getCodeGenOpts().StrictVTablePointers)
2339 return false;
2341 return TypeRequiresBuiltinLaunderImp(CGM.getContext(), Ty, Seen);
2342}
2343
2344RValue CodeGenFunction::emitRotate(const CallExpr *E, bool IsRotateRight) {
2345 llvm::Value *Src = EmitScalarExpr(E->getArg(0));
2346 llvm::Value *ShiftAmt = EmitScalarExpr(E->getArg(1));
2347
2348 // The builtin's shift arg may have a different type than the source arg and
2349 // result, but the LLVM intrinsic uses the same type for all values.
2350 llvm::Type *Ty = Src->getType();
2351 ShiftAmt = Builder.CreateIntCast(ShiftAmt, Ty, false);
2352
2353 // Rotate is a special case of LLVM funnel shift - 1st 2 args are the same.
2354 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2355 Function *F = CGM.getIntrinsic(IID, Ty);
2356 return RValue::get(Builder.CreateCall(F, { Src, Src, ShiftAmt }));
2357}
2358
2359// Map math builtins for long-double to f128 version.
2360static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID) {
2361 switch (BuiltinID) {
2362#define MUTATE_LDBL(func) \
2363 case Builtin::BI__builtin_##func##l: \
2364 return Builtin::BI__builtin_##func##f128;
2395 MUTATE_LDBL(nans)
2396 MUTATE_LDBL(inf)
2415 MUTATE_LDBL(huge_val)
2425#undef MUTATE_LDBL
2426 default:
2427 return BuiltinID;
2428 }
2429}
2430
2431static Value *tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID,
2432 Value *V) {
2433 if (CGF.Builder.getIsFPConstrained() &&
2434 CGF.Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2435 if (Value *Result =
2436 CGF.getTargetHooks().testFPKind(V, BuiltinID, CGF.Builder, CGF.CGM))
2437 return Result;
2438 }
2439 return nullptr;
2440}
2441
2443 const FunctionDecl *FD) {
2444 auto Name = FD->getNameAsString() + "__hipstdpar_unsupported";
2445 auto FnTy = CGF->CGM.getTypes().GetFunctionType(FD);
2446 auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, FnTy);
2447
2449 for (auto &&FormalTy : FnTy->params())
2450 Args.push_back(llvm::PoisonValue::get(FormalTy));
2451
2452 return RValue::get(CGF->Builder.CreateCall(UBF, Args));
2453}
2454
2455RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
2456 const CallExpr *E,
2457 ReturnValueSlot ReturnValue) {
2458 const FunctionDecl *FD = GD.getDecl()->getAsFunction();
2459 // See if we can constant fold this builtin. If so, don't emit it at all.
2460 // TODO: Extend this handling to all builtin calls that we can constant-fold.
2462 if (E->isPRValue() && E->EvaluateAsRValue(Result, CGM.getContext()) &&
2463 !Result.hasSideEffects()) {
2464 if (Result.Val.isInt())
2465 return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
2466 Result.Val.getInt()));
2467 if (Result.Val.isFloat())
2468 return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
2469 Result.Val.getFloat()));
2470 }
2471
2472 // If current long-double semantics is IEEE 128-bit, replace math builtins
2473 // of long-double with f128 equivalent.
2474 // TODO: This mutation should also be applied to other targets other than PPC,
2475 // after backend supports IEEE 128-bit style libcalls.
2476 if (getTarget().getTriple().isPPC64() &&
2477 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2478 BuiltinID = mutateLongDoubleBuiltin(BuiltinID);
2479
2480 // If the builtin has been declared explicitly with an assembler label,
2481 // disable the specialized emitting below. Ideally we should communicate the
2482 // rename in IR, or at least avoid generating the intrinsic calls that are
2483 // likely to get lowered to the renamed library functions.
2484 const unsigned BuiltinIDIfNoAsmLabel =
2485 FD->hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2486
2487 std::optional<bool> ErrnoOverriden;
2488 // ErrnoOverriden is true if math-errno is overriden via the
2489 // '#pragma float_control(precise, on)'. This pragma disables fast-math,
2490 // which implies math-errno.
2491 if (E->hasStoredFPFeatures()) {
2493 if (OP.hasMathErrnoOverride())
2494 ErrnoOverriden = OP.getMathErrnoOverride();
2495 }
2496 // True if 'atttibute__((optnone)) is used. This attibute overrides
2497 // fast-math which implies math-errno.
2498 bool OptNone = CurFuncDecl && CurFuncDecl->hasAttr<OptimizeNoneAttr>();
2499
2500 // True if we are compiling at -O2 and errno has been disabled
2501 // using the '#pragma float_control(precise, off)', and
2502 // attribute opt-none hasn't been seen.
2503 bool ErrnoOverridenToFalseWithOpt =
2504 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2505 CGM.getCodeGenOpts().OptimizationLevel != 0;
2506
2507 // There are LLVM math intrinsics/instructions corresponding to math library
2508 // functions except the LLVM op will never set errno while the math library
2509 // might. Also, math builtins have the same semantics as their math library
2510 // twins. Thus, we can transform math library and builtin calls to their
2511 // LLVM counterparts if the call is marked 'const' (known to never set errno).
2512 // In case FP exceptions are enabled, the experimental versions of the
2513 // intrinsics model those.
2514 bool ConstAlways =
2515 getContext().BuiltinInfo.isConst(BuiltinID);
2516
2517 // There's a special case with the fma builtins where they are always const
2518 // if the target environment is GNU or the target is OS is Windows and we're
2519 // targeting the MSVCRT.dll environment.
2520 // FIXME: This list can be become outdated. Need to find a way to get it some
2521 // other way.
2522 switch (BuiltinID) {
2523 case Builtin::BI__builtin_fma:
2524 case Builtin::BI__builtin_fmaf:
2525 case Builtin::BI__builtin_fmal:
2526 case Builtin::BIfma:
2527 case Builtin::BIfmaf:
2528 case Builtin::BIfmal: {
2529 auto &Trip = CGM.getTriple();
2530 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2531 ConstAlways = true;
2532 break;
2533 }
2534 default:
2535 break;
2536 }
2537
2538 bool ConstWithoutErrnoAndExceptions =
2540 bool ConstWithoutExceptions =
2542
2543 // ConstAttr is enabled in fast-math mode. In fast-math mode, math-errno is
2544 // disabled.
2545 // Math intrinsics are generated only when math-errno is disabled. Any pragmas
2546 // or attributes that affect math-errno should prevent or allow math
2547 // intrincs to be generated. Intrinsics are generated:
2548 // 1- In fast math mode, unless math-errno is overriden
2549 // via '#pragma float_control(precise, on)', or via an
2550 // 'attribute__((optnone))'.
2551 // 2- If math-errno was enabled on command line but overriden
2552 // to false via '#pragma float_control(precise, off))' and
2553 // 'attribute__((optnone))' hasn't been used.
2554 // 3- If we are compiling with optimization and errno has been disabled
2555 // via '#pragma float_control(precise, off)', and
2556 // 'attribute__((optnone))' hasn't been used.
2557
2558 bool ConstWithoutErrnoOrExceptions =
2559 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2560 bool GenerateIntrinsics =
2561 (ConstAlways && !OptNone) ||
2562 (!getLangOpts().MathErrno &&
2563 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2564 if (!GenerateIntrinsics) {
2565 GenerateIntrinsics =
2566 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2567 if (!GenerateIntrinsics)
2568 GenerateIntrinsics =
2569 ConstWithoutErrnoOrExceptions &&
2570 (!getLangOpts().MathErrno &&
2571 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2572 if (!GenerateIntrinsics)
2573 GenerateIntrinsics =
2574 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2575 }
2576 if (GenerateIntrinsics) {
2577 switch (BuiltinIDIfNoAsmLabel) {
2578 case Builtin::BIceil:
2579 case Builtin::BIceilf:
2580 case Builtin::BIceill:
2581 case Builtin::BI__builtin_ceil:
2582 case Builtin::BI__builtin_ceilf:
2583 case Builtin::BI__builtin_ceilf16:
2584 case Builtin::BI__builtin_ceill:
2585 case Builtin::BI__builtin_ceilf128:
2587 Intrinsic::ceil,
2588 Intrinsic::experimental_constrained_ceil));
2589
2590 case Builtin::BIcopysign:
2591 case Builtin::BIcopysignf:
2592 case Builtin::BIcopysignl:
2593 case Builtin::BI__builtin_copysign:
2594 case Builtin::BI__builtin_copysignf:
2595 case Builtin::BI__builtin_copysignf16:
2596 case Builtin::BI__builtin_copysignl:
2597 case Builtin::BI__builtin_copysignf128:
2598 return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::copysign));
2599
2600 case Builtin::BIcos:
2601 case Builtin::BIcosf:
2602 case Builtin::BIcosl:
2603 case Builtin::BI__builtin_cos:
2604 case Builtin::BI__builtin_cosf:
2605 case Builtin::BI__builtin_cosf16:
2606 case Builtin::BI__builtin_cosl:
2607 case Builtin::BI__builtin_cosf128:
2609 Intrinsic::cos,
2610 Intrinsic::experimental_constrained_cos));
2611
2612 case Builtin::BIexp:
2613 case Builtin::BIexpf:
2614 case Builtin::BIexpl:
2615 case Builtin::BI__builtin_exp:
2616 case Builtin::BI__builtin_expf:
2617 case Builtin::BI__builtin_expf16:
2618 case Builtin::BI__builtin_expl:
2619 case Builtin::BI__builtin_expf128:
2621 Intrinsic::exp,
2622 Intrinsic::experimental_constrained_exp));
2623
2624 case Builtin::BIexp2:
2625 case Builtin::BIexp2f:
2626 case Builtin::BIexp2l:
2627 case Builtin::BI__builtin_exp2:
2628 case Builtin::BI__builtin_exp2f:
2629 case Builtin::BI__builtin_exp2f16:
2630 case Builtin::BI__builtin_exp2l:
2631 case Builtin::BI__builtin_exp2f128:
2633 Intrinsic::exp2,
2634 Intrinsic::experimental_constrained_exp2));
2635 case Builtin::BI__builtin_exp10:
2636 case Builtin::BI__builtin_exp10f:
2637 case Builtin::BI__builtin_exp10f16:
2638 case Builtin::BI__builtin_exp10l:
2639 case Builtin::BI__builtin_exp10f128: {
2640 // TODO: strictfp support
2641 if (Builder.getIsFPConstrained())
2642 break;
2643 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::exp10));
2644 }
2645 case Builtin::BIfabs:
2646 case Builtin::BIfabsf:
2647 case Builtin::BIfabsl:
2648 case Builtin::BI__builtin_fabs:
2649 case Builtin::BI__builtin_fabsf:
2650 case Builtin::BI__builtin_fabsf16:
2651 case Builtin::BI__builtin_fabsl:
2652 case Builtin::BI__builtin_fabsf128:
2653 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::fabs));
2654
2655 case Builtin::BIfloor:
2656 case Builtin::BIfloorf:
2657 case Builtin::BIfloorl:
2658 case Builtin::BI__builtin_floor:
2659 case Builtin::BI__builtin_floorf:
2660 case Builtin::BI__builtin_floorf16:
2661 case Builtin::BI__builtin_floorl:
2662 case Builtin::BI__builtin_floorf128:
2664 Intrinsic::floor,
2665 Intrinsic::experimental_constrained_floor));
2666
2667 case Builtin::BIfma:
2668 case Builtin::BIfmaf:
2669 case Builtin::BIfmal:
2670 case Builtin::BI__builtin_fma:
2671 case Builtin::BI__builtin_fmaf:
2672 case Builtin::BI__builtin_fmaf16:
2673 case Builtin::BI__builtin_fmal:
2674 case Builtin::BI__builtin_fmaf128:
2676 Intrinsic::fma,
2677 Intrinsic::experimental_constrained_fma));
2678
2679 case Builtin::BIfmax:
2680 case Builtin::BIfmaxf:
2681 case Builtin::BIfmaxl:
2682 case Builtin::BI__builtin_fmax:
2683 case Builtin::BI__builtin_fmaxf:
2684 case Builtin::BI__builtin_fmaxf16:
2685 case Builtin::BI__builtin_fmaxl:
2686 case Builtin::BI__builtin_fmaxf128:
2688 Intrinsic::maxnum,
2689 Intrinsic::experimental_constrained_maxnum));
2690
2691 case Builtin::BIfmin:
2692 case Builtin::BIfminf:
2693 case Builtin::BIfminl:
2694 case Builtin::BI__builtin_fmin:
2695 case Builtin::BI__builtin_fminf:
2696 case Builtin::BI__builtin_fminf16:
2697 case Builtin::BI__builtin_fminl:
2698 case Builtin::BI__builtin_fminf128:
2700 Intrinsic::minnum,
2701 Intrinsic::experimental_constrained_minnum));
2702
2703 // fmod() is a special-case. It maps to the frem instruction rather than an
2704 // LLVM intrinsic.
2705 case Builtin::BIfmod:
2706 case Builtin::BIfmodf:
2707 case Builtin::BIfmodl:
2708 case Builtin::BI__builtin_fmod:
2709 case Builtin::BI__builtin_fmodf:
2710 case Builtin::BI__builtin_fmodf16:
2711 case Builtin::BI__builtin_fmodl:
2712 case Builtin::BI__builtin_fmodf128: {
2713 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
2714 Value *Arg1 = EmitScalarExpr(E->getArg(0));
2715 Value *Arg2 = EmitScalarExpr(E->getArg(1));
2716 return RValue::get(Builder.CreateFRem(Arg1, Arg2, "fmod"));
2717 }
2718
2719 case Builtin::BIlog:
2720 case Builtin::BIlogf:
2721 case Builtin::BIlogl:
2722 case Builtin::BI__builtin_log:
2723 case Builtin::BI__builtin_logf:
2724 case Builtin::BI__builtin_logf16:
2725 case Builtin::BI__builtin_logl:
2726 case Builtin::BI__builtin_logf128:
2728 Intrinsic::log,
2729 Intrinsic::experimental_constrained_log));
2730
2731 case Builtin::BIlog10:
2732 case Builtin::BIlog10f:
2733 case Builtin::BIlog10l:
2734 case Builtin::BI__builtin_log10:
2735 case Builtin::BI__builtin_log10f:
2736 case Builtin::BI__builtin_log10f16:
2737 case Builtin::BI__builtin_log10l:
2738 case Builtin::BI__builtin_log10f128:
2740 Intrinsic::log10,
2741 Intrinsic::experimental_constrained_log10));
2742
2743 case Builtin::BIlog2:
2744 case Builtin::BIlog2f:
2745 case Builtin::BIlog2l:
2746 case Builtin::BI__builtin_log2:
2747 case Builtin::BI__builtin_log2f:
2748 case Builtin::BI__builtin_log2f16:
2749 case Builtin::BI__builtin_log2l:
2750 case Builtin::BI__builtin_log2f128:
2752 Intrinsic::log2,
2753 Intrinsic::experimental_constrained_log2));
2754
2755 case Builtin::BInearbyint:
2756 case Builtin::BInearbyintf:
2757 case Builtin::BInearbyintl:
2758 case Builtin::BI__builtin_nearbyint:
2759 case Builtin::BI__builtin_nearbyintf:
2760 case Builtin::BI__builtin_nearbyintl:
2761 case Builtin::BI__builtin_nearbyintf128:
2763 Intrinsic::nearbyint,
2764 Intrinsic::experimental_constrained_nearbyint));
2765
2766 case Builtin::BIpow:
2767 case Builtin::BIpowf:
2768 case Builtin::BIpowl:
2769 case Builtin::BI__builtin_pow:
2770 case Builtin::BI__builtin_powf:
2771 case Builtin::BI__builtin_powf16:
2772 case Builtin::BI__builtin_powl:
2773 case Builtin::BI__builtin_powf128:
2775 Intrinsic::pow,
2776 Intrinsic::experimental_constrained_pow));
2777
2778 case Builtin::BIrint:
2779 case Builtin::BIrintf:
2780 case Builtin::BIrintl:
2781 case Builtin::BI__builtin_rint:
2782 case Builtin::BI__builtin_rintf:
2783 case Builtin::BI__builtin_rintf16:
2784 case Builtin::BI__builtin_rintl:
2785 case Builtin::BI__builtin_rintf128:
2787 Intrinsic::rint,
2788 Intrinsic::experimental_constrained_rint));
2789
2790 case Builtin::BIround:
2791 case Builtin::BIroundf:
2792 case Builtin::BIroundl:
2793 case Builtin::BI__builtin_round:
2794 case Builtin::BI__builtin_roundf:
2795 case Builtin::BI__builtin_roundf16:
2796 case Builtin::BI__builtin_roundl:
2797 case Builtin::BI__builtin_roundf128:
2799 Intrinsic::round,
2800 Intrinsic::experimental_constrained_round));
2801
2802 case Builtin::BIroundeven:
2803 case Builtin::BIroundevenf:
2804 case Builtin::BIroundevenl:
2805 case Builtin::BI__builtin_roundeven:
2806 case Builtin::BI__builtin_roundevenf:
2807 case Builtin::BI__builtin_roundevenf16:
2808 case Builtin::BI__builtin_roundevenl:
2809 case Builtin::BI__builtin_roundevenf128:
2811 Intrinsic::roundeven,
2812 Intrinsic::experimental_constrained_roundeven));
2813
2814 case Builtin::BIsin:
2815 case Builtin::BIsinf:
2816 case Builtin::BIsinl:
2817 case Builtin::BI__builtin_sin:
2818 case Builtin::BI__builtin_sinf:
2819 case Builtin::BI__builtin_sinf16:
2820 case Builtin::BI__builtin_sinl:
2821 case Builtin::BI__builtin_sinf128:
2823 Intrinsic::sin,
2824 Intrinsic::experimental_constrained_sin));
2825
2826 case Builtin::BIsqrt:
2827 case Builtin::BIsqrtf:
2828 case Builtin::BIsqrtl:
2829 case Builtin::BI__builtin_sqrt:
2830 case Builtin::BI__builtin_sqrtf:
2831 case Builtin::BI__builtin_sqrtf16:
2832 case Builtin::BI__builtin_sqrtl:
2833 case Builtin::BI__builtin_sqrtf128:
2834 case Builtin::BI__builtin_elementwise_sqrt: {
2836 *this, E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
2838 return RValue::get(Call);
2839 }
2840 case Builtin::BItrunc:
2841 case Builtin::BItruncf:
2842 case Builtin::BItruncl:
2843 case Builtin::BI__builtin_trunc:
2844 case Builtin::BI__builtin_truncf:
2845 case Builtin::BI__builtin_truncf16:
2846 case Builtin::BI__builtin_truncl:
2847 case Builtin::BI__builtin_truncf128:
2849 Intrinsic::trunc,
2850 Intrinsic::experimental_constrained_trunc));
2851
2852 case Builtin::BIlround:
2853 case Builtin::BIlroundf:
2854 case Builtin::BIlroundl:
2855 case Builtin::BI__builtin_lround:
2856 case Builtin::BI__builtin_lroundf:
2857 case Builtin::BI__builtin_lroundl:
2858 case Builtin::BI__builtin_lroundf128:
2860 *this, E, Intrinsic::lround,
2861 Intrinsic::experimental_constrained_lround));
2862
2863 case Builtin::BIllround:
2864 case Builtin::BIllroundf:
2865 case Builtin::BIllroundl:
2866 case Builtin::BI__builtin_llround:
2867 case Builtin::BI__builtin_llroundf:
2868 case Builtin::BI__builtin_llroundl:
2869 case Builtin::BI__builtin_llroundf128:
2871 *this, E, Intrinsic::llround,
2872 Intrinsic::experimental_constrained_llround));
2873
2874 case Builtin::BIlrint:
2875 case Builtin::BIlrintf:
2876 case Builtin::BIlrintl:
2877 case Builtin::BI__builtin_lrint:
2878 case Builtin::BI__builtin_lrintf:
2879 case Builtin::BI__builtin_lrintl:
2880 case Builtin::BI__builtin_lrintf128:
2882 *this, E, Intrinsic::lrint,
2883 Intrinsic::experimental_constrained_lrint));
2884
2885 case Builtin::BIllrint:
2886 case Builtin::BIllrintf:
2887 case Builtin::BIllrintl:
2888 case Builtin::BI__builtin_llrint:
2889 case Builtin::BI__builtin_llrintf:
2890 case Builtin::BI__builtin_llrintl:
2891 case Builtin::BI__builtin_llrintf128:
2893 *this, E, Intrinsic::llrint,
2894 Intrinsic::experimental_constrained_llrint));
2895 case Builtin::BI__builtin_ldexp:
2896 case Builtin::BI__builtin_ldexpf:
2897 case Builtin::BI__builtin_ldexpl:
2898 case Builtin::BI__builtin_ldexpf16:
2899 case Builtin::BI__builtin_ldexpf128: {
2901 *this, E, Intrinsic::ldexp,
2902 Intrinsic::experimental_constrained_ldexp));
2903 }
2904 default:
2905 break;
2906 }
2907 }
2908
2909 // Check NonnullAttribute/NullabilityArg and Alignment.
2910 auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg,
2911 unsigned ParmNum) {
2912 Value *Val = A.getPointer();
2913 EmitNonNullArgCheck(RValue::get(Val), Arg->getType(), Arg->getExprLoc(), FD,
2914 ParmNum);
2915
2916 if (SanOpts.has(SanitizerKind::Alignment)) {
2917 SanitizerSet SkippedChecks;
2918 SkippedChecks.set(SanitizerKind::All);
2919 SkippedChecks.clear(SanitizerKind::Alignment);
2920 SourceLocation Loc = Arg->getExprLoc();
2921 // Strip an implicit cast.
2922 if (auto *CE = dyn_cast<ImplicitCastExpr>(Arg))
2923 if (CE->getCastKind() == CK_BitCast)
2924 Arg = CE->getSubExpr();
2925 EmitTypeCheck(Kind, Loc, Val, Arg->getType(), A.getAlignment(),
2926 SkippedChecks);
2927 }
2928 };
2929
2930 switch (BuiltinIDIfNoAsmLabel) {
2931 default: break;
2932 case Builtin::BI__builtin___CFStringMakeConstantString:
2933 case Builtin::BI__builtin___NSStringMakeConstantString:
2934 return RValue::get(ConstantEmitter(*this).emitAbstract(E, E->getType()));
2935 case Builtin::BI__builtin_stdarg_start:
2936 case Builtin::BI__builtin_va_start:
2937 case Builtin::BI__va_start:
2938 case Builtin::BI__builtin_va_end:
2939 EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
2940 ? EmitScalarExpr(E->getArg(0))
2941 : EmitVAListRef(E->getArg(0)).getPointer(),
2942 BuiltinID != Builtin::BI__builtin_va_end);
2943 return RValue::get(nullptr);
2944 case Builtin::BI__builtin_va_copy: {
2945 Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer();
2946 Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer();
2947 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy), {DstPtr, SrcPtr});
2948 return RValue::get(nullptr);
2949 }
2950 case Builtin::BIabs:
2951 case Builtin::BIlabs:
2952 case Builtin::BIllabs:
2953 case Builtin::BI__builtin_abs:
2954 case Builtin::BI__builtin_labs:
2955 case Builtin::BI__builtin_llabs: {
2956 bool SanitizeOverflow = SanOpts.has(SanitizerKind::SignedIntegerOverflow);
2957
2958 Value *Result;
2959 switch (getLangOpts().getSignedOverflowBehavior()) {
2961 Result = EmitAbs(*this, EmitScalarExpr(E->getArg(0)), false);
2962 break;
2964 if (!SanitizeOverflow) {
2965 Result = EmitAbs(*this, EmitScalarExpr(E->getArg(0)), true);
2966 break;
2967 }
2968 [[fallthrough]];
2970 // TODO: Somehow handle the corner case when the address of abs is taken.
2971 Result = EmitOverflowCheckedAbs(*this, E, SanitizeOverflow);
2972 break;
2973 }
2974 return RValue::get(Result);
2975 }
2976 case Builtin::BI__builtin_complex: {
2977 Value *Real = EmitScalarExpr(E->getArg(0));
2978 Value *Imag = EmitScalarExpr(E->getArg(1));
2979 return RValue::getComplex({Real, Imag});
2980 }
2981 case Builtin::BI__builtin_conj:
2982 case Builtin::BI__builtin_conjf:
2983 case Builtin::BI__builtin_conjl:
2984 case Builtin::BIconj:
2985 case Builtin::BIconjf:
2986 case Builtin::BIconjl: {
2987 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
2988 Value *Real = ComplexVal.first;
2989 Value *Imag = ComplexVal.second;
2990 Imag = Builder.CreateFNeg(Imag, "neg");
2991 return RValue::getComplex(std::make_pair(Real, Imag));
2992 }
2993 case Builtin::BI__builtin_creal:
2994 case Builtin::BI__builtin_crealf:
2995 case Builtin::BI__builtin_creall:
2996 case Builtin::BIcreal:
2997 case Builtin::BIcrealf:
2998 case Builtin::BIcreall: {
2999 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3000 return RValue::get(ComplexVal.first);
3001 }
3002
3003 case Builtin::BI__builtin_preserve_access_index: {
3004 // Only enabled preserved access index region when debuginfo
3005 // is available as debuginfo is needed to preserve user-level
3006 // access pattern.
3007 if (!getDebugInfo()) {
3008 CGM.Error(E->getExprLoc(), "using builtin_preserve_access_index() without -g");
3009 return RValue::get(EmitScalarExpr(E->getArg(0)));
3010 }
3011
3012 // Nested builtin_preserve_access_index() not supported
3014 CGM.Error(E->getExprLoc(), "nested builtin_preserve_access_index() not supported");
3015 return RValue::get(EmitScalarExpr(E->getArg(0)));
3016 }
3017
3018 IsInPreservedAIRegion = true;
3019 Value *Res = EmitScalarExpr(E->getArg(0));
3020 IsInPreservedAIRegion = false;
3021 return RValue::get(Res);
3022 }
3023
3024 case Builtin::BI__builtin_cimag:
3025 case Builtin::BI__builtin_cimagf:
3026 case Builtin::BI__builtin_cimagl:
3027 case Builtin::BIcimag:
3028 case Builtin::BIcimagf:
3029 case Builtin::BIcimagl: {
3030 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3031 return RValue::get(ComplexVal.second);
3032 }
3033
3034 case Builtin::BI__builtin_clrsb:
3035 case Builtin::BI__builtin_clrsbl:
3036 case Builtin::BI__builtin_clrsbll: {
3037 // clrsb(x) -> clz(x < 0 ? ~x : x) - 1 or
3038 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3039
3040 llvm::Type *ArgType = ArgValue->getType();
3041 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3042
3043 llvm::Type *ResultType = ConvertType(E->getType());
3044 Value *Zero = llvm::Constant::getNullValue(ArgType);
3045 Value *IsNeg = Builder.CreateICmpSLT(ArgValue, Zero, "isneg");
3046 Value *Inverse = Builder.CreateNot(ArgValue, "not");
3047 Value *Tmp = Builder.CreateSelect(IsNeg, Inverse, ArgValue);
3048 Value *Ctlz = Builder.CreateCall(F, {Tmp, Builder.getFalse()});
3049 Value *Result = Builder.CreateSub(Ctlz, llvm::ConstantInt::get(ArgType, 1));
3050 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3051 "cast");
3052 return RValue::get(Result);
3053 }
3054 case Builtin::BI__builtin_ctzs:
3055 case Builtin::BI__builtin_ctz:
3056 case Builtin::BI__builtin_ctzl:
3057 case Builtin::BI__builtin_ctzll: {
3059
3060 llvm::Type *ArgType = ArgValue->getType();
3061 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
3062
3063 llvm::Type *ResultType = ConvertType(E->getType());
3064 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
3065 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
3066 if (Result->getType() != ResultType)
3067 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3068 "cast");
3069 return RValue::get(Result);
3070 }
3071 case Builtin::BI__builtin_clzs:
3072 case Builtin::BI__builtin_clz:
3073 case Builtin::BI__builtin_clzl:
3074 case Builtin::BI__builtin_clzll: {
3076
3077 llvm::Type *ArgType = ArgValue->getType();
3078 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3079
3080 llvm::Type *ResultType = ConvertType(E->getType());
3081 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
3082 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
3083 if (Result->getType() != ResultType)
3084 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3085 "cast");
3086 return RValue::get(Result);
3087 }
3088 case Builtin::BI__builtin_ffs:
3089 case Builtin::BI__builtin_ffsl:
3090 case Builtin::BI__builtin_ffsll: {
3091 // ffs(x) -> x ? cttz(x) + 1 : 0
3092 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3093
3094 llvm::Type *ArgType = ArgValue->getType();
3095 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
3096
3097 llvm::Type *ResultType = ConvertType(E->getType());
3098 Value *Tmp =
3099 Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
3100 llvm::ConstantInt::get(ArgType, 1));
3101 Value *Zero = llvm::Constant::getNullValue(ArgType);
3102 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
3103 Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
3104 if (Result->getType() != ResultType)
3105 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3106 "cast");
3107 return RValue::get(Result);
3108 }
3109 case Builtin::BI__builtin_parity:
3110 case Builtin::BI__builtin_parityl:
3111 case Builtin::BI__builtin_parityll: {
3112 // parity(x) -> ctpop(x) & 1
3113 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3114
3115 llvm::Type *ArgType = ArgValue->getType();
3116 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
3117
3118 llvm::Type *ResultType = ConvertType(E->getType());
3119 Value *Tmp = Builder.CreateCall(F, ArgValue);
3120 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
3121 if (Result->getType() != ResultType)
3122 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3123 "cast");
3124 return RValue::get(Result);
3125 }
3126 case Builtin::BI__lzcnt16:
3127 case Builtin::BI__lzcnt:
3128 case Builtin::BI__lzcnt64: {
3129 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3130
3131 llvm::Type *ArgType = ArgValue->getType();
3132 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3133
3134 llvm::Type *ResultType = ConvertType(E->getType());
3135 Value *Result = Builder.CreateCall(F, {ArgValue, Builder.getFalse()});
3136 if (Result->getType() != ResultType)
3137 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3138 "cast");
3139 return RValue::get(Result);
3140 }
3141 case Builtin::BI__popcnt16:
3142 case Builtin::BI__popcnt:
3143 case Builtin::BI__popcnt64:
3144 case Builtin::BI__builtin_popcount:
3145 case Builtin::BI__builtin_popcountl:
3146 case Builtin::BI__builtin_popcountll: {
3147 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3148
3149 llvm::Type *ArgType = ArgValue->getType();
3150 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
3151
3152 llvm::Type *ResultType = ConvertType(E->getType());
3153 Value *Result = Builder.CreateCall(F, ArgValue);
3154 if (Result->getType() != ResultType)
3155 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3156 "cast");
3157 return RValue::get(Result);
3158 }
3159 case Builtin::BI__builtin_unpredictable: {
3160 // Always return the argument of __builtin_unpredictable. LLVM does not
3161 // handle this builtin. Metadata for this builtin should be added directly
3162 // to instructions such as branches or switches that use it.
3163 return RValue::get(EmitScalarExpr(E->getArg(0)));
3164 }
3165 case Builtin::BI__builtin_expect: {
3166 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3167 llvm::Type *ArgType = ArgValue->getType();
3168
3169 Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
3170 // Don't generate llvm.expect on -O0 as the backend won't use it for
3171 // anything.
3172 // Note, we still IRGen ExpectedValue because it could have side-effects.
3173 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3174 return RValue::get(ArgValue);
3175
3176 Function *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
3177 Value *Result =
3178 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval");
3179 return RValue::get(Result);
3180 }
3181 case Builtin::BI__builtin_expect_with_probability: {
3182 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3183 llvm::Type *ArgType = ArgValue->getType();
3184
3185 Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
3186 llvm::APFloat Probability(0.0);
3187 const Expr *ProbArg = E->getArg(2);
3188 bool EvalSucceed = ProbArg->EvaluateAsFloat(Probability, CGM.getContext());
3189 assert(EvalSucceed && "probability should be able to evaluate as float");
3190 (void)EvalSucceed;
3191 bool LoseInfo = false;
3192 Probability.convert(llvm::APFloat::IEEEdouble(),
3193 llvm::RoundingMode::Dynamic, &LoseInfo);
3194 llvm::Type *Ty = ConvertType(ProbArg->getType());
3195 Constant *Confidence = ConstantFP::get(Ty, Probability);
3196 // Don't generate llvm.expect.with.probability on -O0 as the backend
3197 // won't use it for anything.
3198 // Note, we still IRGen ExpectedValue because it could have side-effects.
3199 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3200 return RValue::get(ArgValue);
3201
3202 Function *FnExpect =
3203 CGM.getIntrinsic(Intrinsic::expect_with_probability, ArgType);
3204 Value *Result = Builder.CreateCall(
3205 FnExpect, {ArgValue, ExpectedValue, Confidence}, "expval");
3206 return RValue::get(Result);
3207 }
3208 case Builtin::BI__builtin_assume_aligned: {
3209 const Expr *Ptr = E->getArg(0);
3210 Value *PtrValue = EmitScalarExpr(Ptr);
3211 Value *OffsetValue =
3212 (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr;
3213
3214 Value *AlignmentValue = EmitScalarExpr(E->getArg(1));
3215 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
3216 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
3217 AlignmentCI = ConstantInt::get(AlignmentCI->getType(),
3218 llvm::Value::MaximumAlignment);
3219
3220 emitAlignmentAssumption(PtrValue, Ptr,
3221 /*The expr loc is sufficient.*/ SourceLocation(),
3222 AlignmentCI, OffsetValue);
3223 return RValue::get(PtrValue);
3224 }
3225 case Builtin::BI__assume:
3226 case Builtin::BI__builtin_assume: {
3227 if (E->getArg(0)->HasSideEffects(getContext()))
3228 return RValue::get(nullptr);
3229
3230 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3231 Function *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
3232 Builder.CreateCall(FnAssume, ArgValue);
3233 return RValue::get(nullptr);
3234 }
3235 case Builtin::BI__builtin_assume_separate_storage: {
3236 const Expr *Arg0 = E->getArg(0);
3237 const Expr *Arg1 = E->getArg(1);
3238
3239 Value *Value0 = EmitScalarExpr(Arg0);
3240 Value *Value1 = EmitScalarExpr(Arg1);
3241
3242 Value *Values[] = {Value0, Value1};
3243 OperandBundleDefT<Value *> OBD("separate_storage", Values);
3244 Builder.CreateAssumption(ConstantInt::getTrue(getLLVMContext()), {OBD});
3245 return RValue::get(nullptr);
3246 }
3247 case Builtin::BI__arithmetic_fence: {
3248 // Create the builtin call if FastMath is selected, and the target
3249 // supports the builtin, otherwise just return the argument.
3250 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3251 llvm::FastMathFlags FMF = Builder.getFastMathFlags();
3252 bool isArithmeticFenceEnabled =
3253 FMF.allowReassoc() &&
3255 QualType ArgType = E->getArg(0)->getType();
3256 if (ArgType->isComplexType()) {
3257 if (isArithmeticFenceEnabled) {
3258 QualType ElementType = ArgType->castAs<ComplexType>()->getElementType();
3259 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3260 Value *Real = Builder.CreateArithmeticFence(ComplexVal.first,
3261 ConvertType(ElementType));
3262 Value *Imag = Builder.CreateArithmeticFence(ComplexVal.second,
3263 ConvertType(ElementType));
3264 return RValue::getComplex(std::make_pair(Real, Imag));
3265 }
3266 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3267 Value *Real = ComplexVal.first;
3268 Value *Imag = ComplexVal.second;
3269 return RValue::getComplex(std::make_pair(Real, Imag));
3270 }
3271 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3272 if (isArithmeticFenceEnabled)
3273 return RValue::get(
3274 Builder.CreateArithmeticFence(ArgValue, ConvertType(ArgType)));
3275 return RValue::get(ArgValue);
3276 }
3277 case Builtin::BI__builtin_bswap16:
3278 case Builtin::BI__builtin_bswap32:
3279 case Builtin::BI__builtin_bswap64:
3280 case Builtin::BI_byteswap_ushort:
3281 case Builtin::BI_byteswap_ulong:
3282 case Builtin::BI_byteswap_uint64: {
3283 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bswap));
3284 }
3285 case Builtin::BI__builtin_bitreverse8:
3286 case Builtin::BI__builtin_bitreverse16:
3287 case Builtin::BI__builtin_bitreverse32:
3288 case Builtin::BI__builtin_bitreverse64: {
3289 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bitreverse));
3290 }
3291 case Builtin::BI__builtin_rotateleft8:
3292 case Builtin::BI__builtin_rotateleft16:
3293 case Builtin::BI__builtin_rotateleft32:
3294 case Builtin::BI__builtin_rotateleft64:
3295 case Builtin::BI_rotl8: // Microsoft variants of rotate left
3296 case Builtin::BI_rotl16:
3297 case Builtin::BI_rotl:
3298 case Builtin::BI_lrotl:
3299 case Builtin::BI_rotl64:
3300 return emitRotate(E, false);
3301
3302 case Builtin::BI__builtin_rotateright8:
3303 case Builtin::BI__builtin_rotateright16:
3304 case Builtin::BI__builtin_rotateright32:
3305 case Builtin::BI__builtin_rotateright64:
3306 case Builtin::BI_rotr8: // Microsoft variants of rotate right
3307 case Builtin::BI_rotr16:
3308 case Builtin::BI_rotr:
3309 case Builtin::BI_lrotr:
3310 case Builtin::BI_rotr64:
3311 return emitRotate(E, true);
3312
3313 case Builtin::BI__builtin_constant_p: {
3314 llvm::Type *ResultType = ConvertType(E->getType());
3315
3316 const Expr *Arg = E->getArg(0);
3317 QualType ArgType = Arg->getType();
3318 // FIXME: The allowance for Obj-C pointers and block pointers is historical
3319 // and likely a mistake.
3320 if (!ArgType->isIntegralOrEnumerationType() && !ArgType->isFloatingType() &&
3321 !ArgType->isObjCObjectPointerType() && !ArgType->isBlockPointerType())
3322 // Per the GCC documentation, only numeric constants are recognized after
3323 // inlining.
3324 return RValue::get(ConstantInt::get(ResultType, 0));
3325
3326 if (Arg->HasSideEffects(getContext()))
3327 // The argument is unevaluated, so be conservative if it might have
3328 // side-effects.
3329 return RValue::get(ConstantInt::get(ResultType, 0));
3330
3331 Value *ArgValue = EmitScalarExpr(Arg);
3332 if (ArgType->isObjCObjectPointerType()) {
3333 // Convert Objective-C objects to id because we cannot distinguish between
3334 // LLVM types for Obj-C classes as they are opaque.
3335 ArgType = CGM.getContext().getObjCIdType();
3336 ArgValue = Builder.CreateBitCast(ArgValue, ConvertType(ArgType));
3337 }
3338 Function *F =
3339 CGM.getIntrinsic(Intrinsic::is_constant, ConvertType(ArgType));
3340 Value *Result = Builder.CreateCall(F, ArgValue);
3341 if (Result->getType() != ResultType)
3342 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/false);
3343 return RValue::get(Result);
3344 }
3345 case Builtin::BI__builtin_dynamic_object_size:
3346 case Builtin::BI__builtin_object_size: {
3347 unsigned Type =
3348 E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue();
3349 auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType()));
3350
3351 // We pass this builtin onto the optimizer so that it can figure out the
3352 // object size in more complex cases.
3353 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3354 return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType,
3355 /*EmittedE=*/nullptr, IsDynamic));
3356 }
3357 case Builtin::BI__builtin_prefetch: {
3358 Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
3359 // FIXME: Technically these constants should of type 'int', yes?
3360 RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) :
3361 llvm::ConstantInt::get(Int32Ty, 0);
3362 Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) :
3363 llvm::ConstantInt::get(Int32Ty, 3);
3364 Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
3365 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
3366 Builder.CreateCall(F, {Address, RW, Locality, Data});
3367 return RValue::get(nullptr);
3368 }
3369 case Builtin::BI__builtin_readcyclecounter: {
3370 Function *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
3371 return RValue::get(Builder.CreateCall(F));
3372 }
3373 case Builtin::BI__builtin___clear_cache: {
3374 Value *Begin = EmitScalarExpr(E->getArg(0));
3375 Value *End = EmitScalarExpr(E->getArg(1));
3376 Function *F = CGM.getIntrinsic(Intrinsic::clear_cache);
3377 return RValue::get(Builder.CreateCall(F, {Begin, End}));
3378 }
3379 case Builtin::BI__builtin_trap:
3380 EmitTrapCall(Intrinsic::trap);
3381 return RValue::get(nullptr);
3382 case Builtin::BI__debugbreak:
3383 EmitTrapCall(Intrinsic::debugtrap);
3384 return RValue::get(nullptr);
3385 case Builtin::BI__builtin_unreachable: {
3387
3388 // We do need to preserve an insertion point.
3389 EmitBlock(createBasicBlock("unreachable.cont"));
3390
3391 return RValue::get(nullptr);
3392 }
3393
3394 case Builtin::BI__builtin_powi:
3395 case Builtin::BI__builtin_powif:
3396 case Builtin::BI__builtin_powil: {
3397 llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
3398 llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
3399
3400 if (Builder.getIsFPConstrained()) {
3401 // FIXME: llvm.powi has 2 mangling types,
3402 // llvm.experimental.constrained.powi has one.
3403 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3404 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_powi,
3405 Src0->getType());
3406 return RValue::get(Builder.CreateConstrainedFPCall(F, { Src0, Src1 }));
3407 }
3408
3409 Function *F = CGM.getIntrinsic(Intrinsic::powi,
3410 { Src0->getType(), Src1->getType() });
3411 return RValue::get(Builder.CreateCall(F, { Src0, Src1 }));
3412 }
3413 case Builtin::BI__builtin_frexp:
3414 case Builtin::BI__builtin_frexpf:
3415 case Builtin::BI__builtin_frexpl:
3416 case Builtin::BI__builtin_frexpf128:
3417 case Builtin::BI__builtin_frexpf16:
3418 return RValue::get(emitFrexpBuiltin(*this, E, Intrinsic::frexp));
3419 case Builtin::BI__builtin_isgreater:
3420 case Builtin::BI__builtin_isgreaterequal:
3421 case Builtin::BI__builtin_isless:
3422 case Builtin::BI__builtin_islessequal:
3423 case Builtin::BI__builtin_islessgreater:
3424 case Builtin::BI__builtin_isunordered: {
3425 // Ordered comparisons: we know the arguments to these are matching scalar
3426 // floating point values.
3427 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3428 Value *LHS = EmitScalarExpr(E->getArg(0));
3429 Value *RHS = EmitScalarExpr(E->getArg(1));
3430
3431 switch (BuiltinID) {
3432 default: llvm_unreachable("Unknown ordered comparison");
3433 case Builtin::BI__builtin_isgreater:
3434 LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp");
3435 break;
3436 case Builtin::BI__builtin_isgreaterequal:
3437 LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp");
3438 break;
3439 case Builtin::BI__builtin_isless:
3440 LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp");
3441 break;
3442 case Builtin::BI__builtin_islessequal:
3443 LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp");
3444 break;
3445 case Builtin::BI__builtin_islessgreater:
3446 LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp");
3447 break;
3448 case Builtin::BI__builtin_isunordered:
3449 LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp");
3450 break;
3451 }
3452 // ZExt bool to int type.
3453 return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
3454 }
3455
3456 case Builtin::BI__builtin_isnan: {
3457 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3458 Value *V = EmitScalarExpr(E->getArg(0));
3459 if (Value *Result = tryUseTestFPKind(*this, BuiltinID, V))
3460 return RValue::get(Result);
3461 return RValue::get(
3462 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcNan),
3463 ConvertType(E->getType())));
3464 }
3465
3466 case Builtin::BI__builtin_issignaling: {
3467 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3468 Value *V = EmitScalarExpr(E->getArg(0));
3469 return RValue::get(
3470 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcSNan),
3471 ConvertType(E->getType())));
3472 }
3473
3474 case Builtin::BI__builtin_isinf: {
3475 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3476 Value *V = EmitScalarExpr(E->getArg(0));
3477 if (Value *Result = tryUseTestFPKind(*this, BuiltinID, V))
3478 return RValue::get(Result);
3479 return RValue::get(
3480 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcInf),
3481 ConvertType(E->getType())));
3482 }
3483
3484 case Builtin::BIfinite:
3485 case Builtin::BI__finite:
3486 case Builtin::BIfinitef:
3487 case Builtin::BI__finitef:
3488 case Builtin::BIfinitel:
3489 case Builtin::BI__finitel:
3490 case Builtin::BI__builtin_isfinite: {
3491 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3492 Value *V = EmitScalarExpr(E->getArg(0));
3493 if (Value *Result = tryUseTestFPKind(*this, BuiltinID, V))
3494 return RValue::get(Result);
3495 return RValue::get(
3496 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcFinite),
3497 ConvertType(E->getType())));
3498 }
3499
3500 case Builtin::BI__builtin_isnormal: {
3501 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3502 Value *V = EmitScalarExpr(E->getArg(0));
3503 return RValue::get(
3504 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcNormal),
3505 ConvertType(E->getType())));
3506 }
3507
3508 case Builtin::BI__builtin_issubnormal: {
3509 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3510 Value *V = EmitScalarExpr(E->getArg(0));
3511 return RValue::get(
3512 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcSubnormal),
3513 ConvertType(E->getType())));
3514 }
3515
3516 case Builtin::BI__builtin_iszero: {
3517 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3518 Value *V = EmitScalarExpr(E->getArg(0));
3519 return RValue::get(
3520 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcZero),
3521 ConvertType(E->getType())));
3522 }
3523
3524 case Builtin::BI__builtin_isfpclass: {
3526 if (!E->getArg(1)->EvaluateAsInt(Result, CGM.getContext()))
3527 break;
3528 uint64_t Test = Result.Val.getInt().getLimitedValue();
3529 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3530 Value *V = EmitScalarExpr(E->getArg(0));
3531 return RValue::get(Builder.CreateZExt(Builder.createIsFPClass(V, Test),
3532 ConvertType(E->getType())));
3533 }
3534
3535 case Builtin::BI__builtin_nondeterministic_value: {
3536 llvm::Type *Ty = ConvertType(E->getArg(0)->getType());
3537
3538 Value *Result = PoisonValue::get(Ty);
3539 Result = Builder.CreateFreeze(Result);
3540
3541 return RValue::get(Result);
3542 }
3543
3544 case Builtin::BI__builtin_elementwise_abs: {
3545 Value *Result;
3546 QualType QT = E->getArg(0)->getType();
3547
3548 if (auto *VecTy = QT->getAs<VectorType>())
3549 QT = VecTy->getElementType();
3550 if (QT->isIntegerType())
3551 Result = Builder.CreateBinaryIntrinsic(
3552 llvm::Intrinsic::abs, EmitScalarExpr(E->getArg(0)),
3553 Builder.getFalse(), nullptr, "elt.abs");
3554 else
3555 Result = emitUnaryBuiltin(*this, E, llvm::Intrinsic::fabs, "elt.abs");
3556
3557 return RValue::get(Result);
3558 }
3559
3560 case Builtin::BI__builtin_elementwise_ceil:
3561 return RValue::get(
3562 emitUnaryBuiltin(*this, E, llvm::Intrinsic::ceil, "elt.ceil"));
3563 case Builtin::BI__builtin_elementwise_exp:
3564 return RValue::get(
3565 emitUnaryBuiltin(*this, E, llvm::Intrinsic::exp, "elt.exp"));
3566 case Builtin::BI__builtin_elementwise_exp2:
3567 return RValue::get(
3568 emitUnaryBuiltin(*this, E, llvm::Intrinsic::exp2, "elt.exp2"));
3569 case Builtin::BI__builtin_elementwise_log:
3570 return RValue::get(
3571 emitUnaryBuiltin(*this, E, llvm::Intrinsic::log, "elt.log"));
3572 case Builtin::BI__builtin_elementwise_log2:
3573 return RValue::get(
3574 emitUnaryBuiltin(*this, E, llvm::Intrinsic::log2, "elt.log2"));
3575 case Builtin::BI__builtin_elementwise_log10:
3576 return RValue::get(
3577 emitUnaryBuiltin(*this, E, llvm::Intrinsic::log10, "elt.log10"));
3578 case Builtin::BI__builtin_elementwise_pow: {
3579 return RValue::get(emitBinaryBuiltin(*this, E, llvm::Intrinsic::pow));
3580 }
3581 case Builtin::BI__builtin_elementwise_bitreverse:
3582 return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::bitreverse,
3583 "elt.bitreverse"));
3584 case Builtin::BI__builtin_elementwise_cos:
3585 return RValue::get(
3586 emitUnaryBuiltin(*this, E, llvm::Intrinsic::cos, "elt.cos"));
3587 case Builtin::BI__builtin_elementwise_floor:
3588 return RValue::get(
3589 emitUnaryBuiltin(*this, E, llvm::Intrinsic::floor, "elt.floor"));
3590 case Builtin::BI__builtin_elementwise_roundeven:
3591 return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::roundeven,
3592 "elt.roundeven"));
3593 case Builtin::BI__builtin_elementwise_round:
3594 return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::round,
3595 "elt.round"));
3596 case Builtin::BI__builtin_elementwise_rint:
3597 return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::rint,
3598 "elt.rint"));
3599 case Builtin::BI__builtin_elementwise_nearbyint:
3600 return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::nearbyint,
3601 "elt.nearbyint"));
3602 case Builtin::BI__builtin_elementwise_sin:
3603 return RValue::get(
3604 emitUnaryBuiltin(*this, E, llvm::Intrinsic::sin, "elt.sin"));
3605
3606 case Builtin::BI__builtin_elementwise_trunc:
3607 return RValue::get(
3608 emitUnaryBuiltin(*this, E, llvm::Intrinsic::trunc, "elt.trunc"));
3609 case Builtin::BI__builtin_elementwise_canonicalize:
3610 return RValue::get(
3611 emitUnaryBuiltin(*this, E, llvm::Intrinsic::canonicalize, "elt.canonicalize"));
3612 case Builtin::BI__builtin_elementwise_copysign:
3613 return RValue::get(emitBinaryBuiltin(*this, E, llvm::Intrinsic::copysign));
3614 case Builtin::BI__builtin_elementwise_fma:
3615 return RValue::get(emitTernaryBuiltin(*this, E, llvm::Intrinsic::fma));
3616 case Builtin::BI__builtin_elementwise_add_sat:
3617 case Builtin::BI__builtin_elementwise_sub_sat: {
3618 Value *Op0 = EmitScalarExpr(E->getArg(0));
3619 Value *Op1 = EmitScalarExpr(E->getArg(1));
3620 Value *Result;
3621 assert(Op0->getType()->isIntOrIntVectorTy() && "integer type expected");
3622 QualType Ty = E->getArg(0)->getType();
3623 if (auto *VecTy = Ty->getAs<VectorType>())
3624 Ty = VecTy->getElementType();
3625 bool IsSigned = Ty->isSignedIntegerType();
3626 unsigned Opc;
3627 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
3628 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
3629 else
3630 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
3631 Result = Builder.CreateBinaryIntrinsic(Opc, Op0, Op1, nullptr, "elt.sat");
3632 return RValue::get(Result);
3633 }
3634
3635 case Builtin::BI__builtin_elementwise_max: {
3636 Value *Op0 = EmitScalarExpr(E->getArg(0));
3637 Value *Op1 = EmitScalarExpr(E->getArg(1));
3638 Value *Result;
3639 if (Op0->getType()->isIntOrIntVectorTy()) {
3640 QualType Ty = E->getArg(0)->getType();
3641 if (auto *VecTy = Ty->getAs<VectorType>())
3642 Ty = VecTy->getElementType();
3643 Result = Builder.CreateBinaryIntrinsic(Ty->isSignedIntegerType()
3644 ? llvm::Intrinsic::smax
3645 : llvm::Intrinsic::umax,
3646 Op0, Op1, nullptr, "elt.max");
3647 } else
3648 Result = Builder.CreateMaxNum(Op0, Op1, "elt.max");
3649 return RValue::get(Result);
3650 }
3651 case Builtin::BI__builtin_elementwise_min: {
3652 Value *Op0 = EmitScalarExpr(E->getArg(0));
3653 Value *Op1 = EmitScalarExpr(E->getArg(1));
3654 Value *Result;
3655 if (Op0->getType()->isIntOrIntVectorTy()) {
3656 QualType Ty = E->getArg(0)->getType();
3657 if (auto *VecTy = Ty->getAs<VectorType>())
3658 Ty = VecTy->getElementType();
3659 Result = Builder.CreateBinaryIntrinsic(Ty->isSignedIntegerType()
3660 ? llvm::Intrinsic::smin
3661 : llvm::Intrinsic::umin,
3662 Op0, Op1, nullptr, "elt.min");
3663 } else
3664 Result = Builder.CreateMinNum(Op0, Op1, "elt.min");
3665 return RValue::get(Result);
3666 }
3667
3668 case Builtin::BI__builtin_reduce_max: {
3669 auto GetIntrinsicID = [](QualType QT) {
3670 if (auto *VecTy = QT->getAs<VectorType>())
3671 QT = VecTy->getElementType();
3672 if (QT->isSignedIntegerType())
3673 return llvm::Intrinsic::vector_reduce_smax;
3674 if (QT->isUnsignedIntegerType())
3675 return llvm::Intrinsic::vector_reduce_umax;
3676 assert(QT->isFloatingType() && "must have a float here");
3677 return llvm::Intrinsic::vector_reduce_fmax;
3678 };
3680 *this, E, GetIntrinsicID(E->getArg(0)->getType()), "rdx.min"));
3681 }
3682
3683 case Builtin::BI__builtin_reduce_min: {
3684 auto GetIntrinsicID = [](QualType QT) {
3685 if (auto *VecTy = QT->getAs<VectorType>())
3686 QT = VecTy->getElementType();
3687 if (QT->isSignedIntegerType())
3688 return llvm::Intrinsic::vector_reduce_smin;
3689 if (QT->isUnsignedIntegerType())
3690 return llvm::Intrinsic::vector_reduce_umin;
3691 assert(QT->isFloatingType() && "must have a float here");
3692 return llvm::Intrinsic::vector_reduce_fmin;
3693 };
3694
3696 *this, E, GetIntrinsicID(E->getArg(0)->getType()), "rdx.min"));
3697 }
3698
3699 case Builtin::BI__builtin_reduce_add:
3701 *this, E, llvm::Intrinsic::vector_reduce_add, "rdx.add"));
3702 case Builtin::BI__builtin_reduce_mul:
3704 *this, E, llvm::Intrinsic::vector_reduce_mul, "rdx.mul"));
3705 case Builtin::BI__builtin_reduce_xor:
3707 *this, E, llvm::Intrinsic::vector_reduce_xor, "rdx.xor"));
3708 case Builtin::BI__builtin_reduce_or:
3710 *this, E, llvm::Intrinsic::vector_reduce_or, "rdx.or"));
3711 case Builtin::BI__builtin_reduce_and:
3713 *this, E, llvm::Intrinsic::vector_reduce_and, "rdx.and"));
3714
3715 case Builtin::BI__builtin_matrix_transpose: {
3716 auto *MatrixTy = E->getArg(0)->getType()->castAs<ConstantMatrixType>();
3717 Value *MatValue = EmitScalarExpr(E->getArg(0));
3718 MatrixBuilder MB(Builder);
3719 Value *Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
3720 MatrixTy->getNumColumns());
3721 return RValue::get(Result);
3722 }
3723
3724 case Builtin::BI__builtin_matrix_column_major_load: {
3725 MatrixBuilder MB(Builder);
3726 // Emit everything that isn't dependent on the first parameter type
3727 Value *Stride = EmitScalarExpr(E->getArg(3));
3728 const auto *ResultTy = E->getType()->getAs<ConstantMatrixType>();
3729 auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
3730 assert(PtrTy && "arg0 must be of pointer type");
3731 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
3732
3735 E->getArg(0)->getExprLoc(), FD, 0);
3736 Value *Result = MB.CreateColumnMajorLoad(
3737 Src.getElementType(), Src.getPointer(),
3738 Align(Src.getAlignment().getQuantity()), Stride, IsVolatile,
3739 ResultTy->getNumRows(), ResultTy->getNumColumns(),
3740 "matrix");
3741 return RValue::get(Result);
3742 }
3743
3744 case Builtin::BI__builtin_matrix_column_major_store: {
3745 MatrixBuilder MB(Builder);
3746 Value *Matrix = EmitScalarExpr(E->getArg(0));
3748 Value *Stride = EmitScalarExpr(E->getArg(2));
3749
3750 const auto *MatrixTy = E->getArg(0)->getType()->getAs<ConstantMatrixType>();
3751 auto *PtrTy = E->getArg(1)->getType()->getAs<PointerType>();
3752 assert(PtrTy && "arg1 must be of pointer type");
3753 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
3754
3756 E->getArg(1)->getExprLoc(), FD, 0);
3757 Value *Result = MB.CreateColumnMajorStore(
3758 Matrix, Dst.getPointer(), Align(Dst.getAlignment().getQuantity()),
3759 Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns());
3760 return RValue::get(Result);
3761 }
3762
3763 case Builtin::BI__builtin_isinf_sign: {
3764 // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
3765 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3766 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
3767 Value *Arg = EmitScalarExpr(E->getArg(0));
3768 Value *AbsArg = EmitFAbs(*this, Arg);
3769 Value *IsInf = Builder.CreateFCmpOEQ(
3770 AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf");
3771 Value *IsNeg = EmitSignBit(*this, Arg);
3772
3773 llvm::Type *IntTy = ConvertType(E->getType());
3774 Value *Zero = Constant::getNullValue(IntTy);
3775 Value *One = ConstantInt::get(IntTy, 1);
3776 Value *NegativeOne = ConstantInt::get(IntTy, -1);
3777 Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
3778 Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero);
3779 return RValue::get(Result);
3780 }
3781
3782 case Builtin::BI__builtin_flt_rounds: {
3783 Function *F = CGM.getIntrinsic(Intrinsic::get_rounding);
3784
3785 llvm::Type *ResultType = ConvertType(E->getType());
3786 Value *Result = Builder.CreateCall(F);
3787 if (Result->getType() != ResultType)
3788 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3789 "cast");
3790 return RValue::get(Result);
3791 }
3792
3793 case Builtin::BI__builtin_set_flt_rounds: {
3794 Function *F = CGM.getIntrinsic(Intrinsic::set_rounding);
3795
3796 Value *V = EmitScalarExpr(E->getArg(0));
3797 Builder.CreateCall(F, V);
3798 return RValue::get(nullptr);
3799 }
3800
3801 case Builtin::BI__builtin_fpclassify: {
3802 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3803 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
3804 Value *V = EmitScalarExpr(E->getArg(5));
3805 llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
3806
3807 // Create Result
3808 BasicBlock *Begin = Builder.GetInsertBlock();
3809 BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn);
3810 Builder.SetInsertPoint(End);
3811 PHINode *Result =
3812 Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4,
3813 "fpclassify_result");
3814
3815 // if (V==0) return FP_ZERO
3816 Builder.SetInsertPoint(Begin);
3817 Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
3818 "iszero");
3819 Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
3820 BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn);
3821 Builder.CreateCondBr(IsZero, End, NotZero);
3822 Result->addIncoming(ZeroLiteral, Begin);
3823
3824 // if (V != V) return FP_NAN
3825 Builder.SetInsertPoint(NotZero);
3826 Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp");
3827 Value *NanLiteral = EmitScalarExpr(E->getArg(0));
3828 BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn);
3829 Builder.CreateCondBr(IsNan, End, NotNan);
3830 Result->addIncoming(NanLiteral, NotZero);
3831
3832 // if (fabs(V) == infinity) return FP_INFINITY
3833 Builder.SetInsertPoint(NotNan);
3834 Value *VAbs = EmitFAbs(*this, V);
3835 Value *IsInf =
3836 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
3837 "isinf");
3838 Value *InfLiteral = EmitScalarExpr(E->getArg(1));
3839 BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn);
3840 Builder.CreateCondBr(IsInf, End, NotInf);
3841 Result->addIncoming(InfLiteral, NotNan);
3842
3843 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
3844 Builder.SetInsertPoint(NotInf);
3845 APFloat Smallest = APFloat::getSmallestNormalized(
3846 getContext().getFloatTypeSemantics(E->getArg(5)->getType()));
3847 Value *IsNormal =
3848 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
3849 "isnormal");
3850 Value *NormalResult =
3851 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
3852 EmitScalarExpr(E->getArg(3)));
3853 Builder.CreateBr(End);
3854 Result->addIncoming(NormalResult, NotInf);
3855
3856 // return Result
3857 Builder.SetInsertPoint(End);
3858 return RValue::get(Result);
3859 }
3860
3861 // An alloca will always return a pointer to the alloca (stack) address
3862 // space. This address space need not be the same as the AST / Language
3863 // default (e.g. in C / C++ auto vars are in the generic address space). At
3864 // the AST level this is handled within CreateTempAlloca et al., but for the
3865 // builtin / dynamic alloca we have to handle it here. We use an explicit cast
3866 // instead of passing an AS to CreateAlloca so as to not inhibit optimisation.
3867 case Builtin::BIalloca:
3868 case Builtin::BI_alloca:
3869 case Builtin::BI__builtin_alloca_uninitialized:
3870 case Builtin::BI__builtin_alloca: {
3871 Value *Size = EmitScalarExpr(E->getArg(0));
3872 const TargetInfo &TI = getContext().getTargetInfo();
3873 // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
3874 const Align SuitableAlignmentInBytes =
3875 CGM.getContext()
3877 .getAsAlign();
3878 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
3879 AI->setAlignment(SuitableAlignmentInBytes);
3880 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
3881 initializeAlloca(*this, AI, Size, SuitableAlignmentInBytes);
3884 if (AAS != EAS) {
3885 llvm::Type *Ty = CGM.getTypes().ConvertType(E->getType());
3886 return RValue::get(getTargetHooks().performAddrSpaceCast(*this, AI, AAS,
3887 EAS, Ty));
3888 }
3889 return RValue::get(AI);
3890 }
3891
3892 case Builtin::BI__builtin_alloca_with_align_uninitialized:
3893 case Builtin::BI__builtin_alloca_with_align: {
3894 Value *Size = EmitScalarExpr(E->getArg(0));
3895 Value *AlignmentInBitsValue = EmitScalarExpr(E->getArg(1));
3896 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
3897 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
3898 const Align AlignmentInBytes =
3899 CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getAsAlign();
3900 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
3901 AI->setAlignment(AlignmentInBytes);
3902 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
3903 initializeAlloca(*this, AI, Size, AlignmentInBytes);
3906 if (AAS != EAS) {
3907 llvm::Type *Ty = CGM.getTypes().ConvertType(E->getType());
3908 return RValue::get(getTargetHooks().performAddrSpaceCast(*this, AI, AAS,
3909 EAS, Ty));
3910 }
3911 return RValue::get(AI);
3912 }
3913
3914 case Builtin::BIbzero:
3915 case Builtin::BI__builtin_bzero: {
3917 Value *SizeVal = EmitScalarExpr(E->getArg(1));
3919 E->getArg(0)->getExprLoc(), FD, 0);
3920 Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false);
3921 return RValue::get(nullptr);
3922 }
3923
3924 case Builtin::BIbcopy:
3925 case Builtin::BI__builtin_bcopy: {
3928 Value *SizeVal = EmitScalarExpr(E->getArg(2));
3930 E->getArg(0)->getExprLoc(), FD, 0);
3932 E->getArg(1)->getExprLoc(), FD, 0);
3933 Builder.CreateMemMove(Dest, Src, SizeVal, false);
3934 return RValue::get(Dest.getPointer());
3935 }
3936
3937 case Builtin::BImemcpy:
3938 case Builtin::BI__builtin_memcpy:
3939 case Builtin::BImempcpy:
3940 case Builtin::BI__builtin_mempcpy: {
3943 Value *SizeVal = EmitScalarExpr(E->getArg(2));
3944 EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0);
3945 EmitArgCheck(TCK_Load, Src, E->getArg(1), 1);
3946 Builder.CreateMemCpy(Dest, Src, SizeVal, false);
3947 if (BuiltinID == Builtin::BImempcpy ||
3948 BuiltinID == Builtin::BI__builtin_mempcpy)
3949 return RValue::get(Builder.CreateInBoundsGEP(Dest.getElementType(),
3950 Dest.getPointer(), SizeVal));
3951 else
3952 return RValue::get(Dest.getPointer());
3953 }
3954
3955 case Builtin::BI__builtin_memcpy_inline: {
3958 uint64_t Size =
3959 E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue();
3960 EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0);
3961 EmitArgCheck(TCK_Load, Src, E->getArg(1), 1);
3962 Builder.CreateMemCpyInline(Dest, Src, Size);
3963 return RValue::get(nullptr);
3964 }
3965
3966 case Builtin::BI__builtin_char_memchr:
3967 BuiltinID = Builtin::BI__builtin_memchr;
3968 break;
3969
3970 case Builtin::BI__builtin___memcpy_chk: {
3971 // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
3972 Expr::EvalResult SizeResult, DstSizeResult;
3973 if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
3974 !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
3975 break;
3976 llvm::APSInt Size = SizeResult.Val.getInt();
3977 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
3978 if (Size.ugt(DstSize))
3979 break;
3982 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
3983 Builder.CreateMemCpy(Dest, Src, SizeVal, false);
3984 return RValue::get(Dest.getPointer());
3985 }
3986
3987 case Builtin::BI__builtin_objc_memmove_collectable: {
3988 Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
3989 Address SrcAddr = EmitPointerWithAlignment(E->getArg(1));
3990 Value *SizeVal = EmitScalarExpr(E->getArg(2));
3992 DestAddr, SrcAddr, SizeVal);
3993 return RValue::get(DestAddr.getPointer());
3994 }
3995
3996 case Builtin::BI__builtin___memmove_chk: {
3997 // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
3998 Expr::EvalResult SizeResult, DstSizeResult;
3999 if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
4000 !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
4001 break;
4002 llvm::APSInt Size = SizeResult.Val.getInt();
4003 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4004 if (Size.ugt(DstSize))
4005 break;
4008 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
4009 Builder.CreateMemMove(Dest, Src, SizeVal, false);
4010 return RValue::get(Dest.getPointer());
4011 }
4012
4013 case Builtin::BImemmove:
4014 case Builtin::BI__builtin_memmove: {
4017 Value *SizeVal = EmitScalarExpr(E->getArg(2));
4018 EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0);
4019 EmitArgCheck(TCK_Load, Src, E->getArg(1), 1);
4020 Builder.CreateMemMove(Dest, Src, SizeVal, false);
4021 return RValue::get(Dest.getPointer());
4022 }
4023 case Builtin::BImemset:
4024 case Builtin::BI__builtin_memset: {
4026 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
4027 Builder.getInt8Ty());
4028 Value *SizeVal = EmitScalarExpr(E->getArg(2));
4030 E->getArg(0)->getExprLoc(), FD, 0);
4031 Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
4032 return RValue::get(Dest.getPointer());
4033 }
4034 case Builtin::BI__builtin_memset_inline: {
4036 Value *ByteVal =
4037 Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty());
4038 uint64_t Size =
4039 E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue();
4041 E->getArg(0)->getExprLoc(), FD, 0);
4042 Builder.CreateMemSetInline(Dest, ByteVal, Size);
4043 return RValue::get(nullptr);
4044 }
4045 case Builtin::BI__builtin___memset_chk: {
4046 // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
4047 Expr::EvalResult SizeResult, DstSizeResult;
4048 if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
4049 !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
4050 break;
4051 llvm::APSInt Size = SizeResult.Val.getInt();
4052 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4053 if (Size.ugt(DstSize))
4054 break;
4056 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
4057 Builder.getInt8Ty());
4058 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
4059 Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
4060 return RValue::get(Dest.getPointer());
4061 }
4062 case Builtin::BI__builtin_wmemchr: {
4063 // The MSVC runtime library does not provide a definition of wmemchr, so we
4064 // need an inline implementation.
4065 if (!getTarget().getTriple().isOSMSVCRT())
4066 break;
4067
4068 llvm::Type *WCharTy = ConvertType(getContext().WCharTy);
4069 Value *Str = EmitScalarExpr(E->getArg(0));
4070 Value *Chr = EmitScalarExpr(E->getArg(1));
4071 Value *Size = EmitScalarExpr(E->getArg(2));
4072
4073 BasicBlock *Entry = Builder.GetInsertBlock();
4074 BasicBlock *CmpEq = createBasicBlock("wmemchr.eq");
4075 BasicBlock *Next = createBasicBlock("wmemchr.next");
4076 BasicBlock *Exit = createBasicBlock("wmemchr.exit");
4077 Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(SizeTy, 0));
4078 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
4079
4080 EmitBlock(CmpEq);
4081 PHINode *StrPhi = Builder.CreatePHI(Str->getType(), 2);
4082 StrPhi->addIncoming(Str, Entry);
4083 PHINode *SizePhi = Builder.CreatePHI(SizeTy, 2);
4084 SizePhi->addIncoming(Size, Entry);
4085 CharUnits WCharAlign =
4087 Value *StrCh = Builder.CreateAlignedLoad(WCharTy, StrPhi, WCharAlign);
4088 Value *FoundChr = Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
4089 Value *StrEqChr = Builder.CreateICmpEQ(StrCh, Chr);
4090 Builder.CreateCondBr(StrEqChr, Exit, Next);
4091
4092 EmitBlock(Next);
4093 Value *NextStr = Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
4094 Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(SizeTy, 1));
4095 Value *NextSizeEq0 =
4096 Builder.CreateICmpEQ(NextSize, ConstantInt::get(SizeTy, 0));
4097 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
4098 StrPhi->addIncoming(NextStr, Next);
4099 SizePhi->addIncoming(NextSize, Next);
4100
4101 EmitBlock(Exit);
4102 PHINode *Ret = Builder.CreatePHI(Str->getType(), 3);
4103 Ret->addIncoming(llvm::Constant::getNullValue(Str->getType()), Entry);
4104 Ret->addIncoming(llvm::Constant::getNullValue(Str->getType()), Next);
4105 Ret->addIncoming(FoundChr, CmpEq);
4106 return RValue::get(Ret);
4107 }
4108 case Builtin::BI__builtin_wmemcmp: {
4109 // The MSVC runtime library does not provide a definition of wmemcmp, so we
4110 // need an inline implementation.
4111 if (!getTarget().getTriple().isOSMSVCRT())
4112 break;
4113
4114 llvm::Type *WCharTy = ConvertType(getContext().WCharTy);
4115
4116 Value *Dst = EmitScalarExpr(E->getArg(0));
4117 Value *Src = EmitScalarExpr(E->getArg(1));
4118 Value *Size = EmitScalarExpr(E->getArg(2));
4119
4120 BasicBlock *Entry = Builder.GetInsertBlock();
4121 BasicBlock *CmpGT = createBasicBlock("wmemcmp.gt");
4122 BasicBlock *CmpLT = createBasicBlock("wmemcmp.lt");
4123 BasicBlock *Next = createBasicBlock("wmemcmp.next");
4124 BasicBlock *Exit = createBasicBlock("wmemcmp.exit");
4125 Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(SizeTy, 0));
4126 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
4127
4128 EmitBlock(CmpGT);
4129 PHINode *DstPhi = Builder.CreatePHI(Dst->getType(), 2);
4130 DstPhi->addIncoming(Dst, Entry);
4131 PHINode *SrcPhi = Builder.CreatePHI(Src->getType(), 2);
4132 SrcPhi->addIncoming(Src, Entry);
4133 PHINode *SizePhi = Builder.CreatePHI(SizeTy, 2);
4134 SizePhi->addIncoming(Size, Entry);
4135 CharUnits WCharAlign =
4137 Value *DstCh = Builder.CreateAlignedLoad(WCharTy, DstPhi, WCharAlign);
4138 Value *SrcCh = Builder.CreateAlignedLoad(WCharTy, SrcPhi, WCharAlign);
4139 Value *DstGtSrc = Builder.CreateICmpUGT(DstCh, SrcCh);
4140 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
4141
4142 EmitBlock(CmpLT);
4143 Value *DstLtSrc = Builder.CreateICmpULT(DstCh, SrcCh);
4144 Builder.CreateCondBr(DstLtSrc, Exit, Next);
4145
4146 EmitBlock(Next);
4147 Value *NextDst = Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
4148 Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
4149 Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(SizeTy, 1));
4150 Value *NextSizeEq0 =
4151 Builder.CreateICmpEQ(NextSize, ConstantInt::get(SizeTy, 0));
4152 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
4153 DstPhi->addIncoming(NextDst, Next);
4154 SrcPhi->addIncoming(NextSrc, Next);
4155 SizePhi->addIncoming(NextSize, Next);
4156
4157 EmitBlock(Exit);
4158 PHINode *Ret = Builder.CreatePHI(IntTy, 4);
4159 Ret->addIncoming(ConstantInt::get(IntTy, 0), Entry);
4160 Ret->addIncoming(ConstantInt::get(IntTy, 1), CmpGT);
4161 Ret->addIncoming(ConstantInt::get(IntTy, -1), CmpLT);
4162 Ret->addIncoming(ConstantInt::get(IntTy, 0), Next);
4163 return RValue::get(Ret);
4164 }
4165 case Builtin::BI__builtin_dwarf_cfa: {
4166 // The offset in bytes from the first argument to the CFA.
4167 //
4168 // Why on earth is this in the frontend? Is there any reason at
4169 // all that the backend can't reasonably determine this while
4170 // lowering llvm.eh.dwarf.cfa()?
4171 //
4172 // TODO: If there's a satisfactory reason, add a target hook for
4173 // this instead of hard-coding 0, which is correct for most targets.
4174 int32_t Offset = 0;
4175
4176 Function *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
4177 return RValue::get(Builder.CreateCall(F,
4178 llvm::ConstantInt::get(Int32Ty, Offset)));
4179 }
4180 case Builtin::BI__builtin_return_address: {
4181 Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
4182 getContext().UnsignedIntTy);
4183 Function *F = CGM.getIntrinsic(Intrinsic::returnaddress);
4184 return RValue::get(Builder.CreateCall(F, Depth));
4185 }
4186 case Builtin::BI_ReturnAddress: {
4187 Function *F = CGM.getIntrinsic(Intrinsic::returnaddress);
4188 return RValue::get(Builder.CreateCall(F, Builder.getInt32(0)));
4189 }
4190 case Builtin::BI__builtin_frame_address: {
4191 Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
4192 getContext().UnsignedIntTy);
4193 Function *F = CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy);
4194 return RValue::get(Builder.CreateCall(F, Depth));
4195 }
4196 case Builtin::BI__builtin_extract_return_addr: {
4199 return RValue::get(Result);
4200 }
4201 case Builtin::BI__builtin_frob_return_addr: {
4204 return RValue::get(Result);
4205 }
4206 case Builtin::BI__builtin_dwarf_sp_column: {
4207 llvm::IntegerType *Ty
4208 = cast<llvm::IntegerType>(ConvertType(E->getType()));
4210 if (Column == -1) {
4211 CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column");
4212 return RValue::get(llvm::UndefValue::get(Ty));
4213 }
4214 return RValue::get(llvm::ConstantInt::get(Ty, Column, true));
4215 }
4216 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
4218 if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address))
4219 CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table");
4220 return RValue::get(llvm::UndefValue::get(ConvertType(E->getType())));
4221 }
4222 case Builtin::BI__builtin_eh_return: {
4223 Value *Int = EmitScalarExpr(E->getArg(0));
4224 Value *Ptr = EmitScalarExpr(E->getArg(1));
4225
4226 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
4227 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
4228 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
4229 Function *F =
4230 CGM.getIntrinsic(IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32
4231 : Intrinsic::eh_return_i64);
4232 Builder.CreateCall(F, {Int, Ptr});
4233 Builder.CreateUnreachable();
4234
4235 // We do need to preserve an insertion point.
4236 EmitBlock(createBasicBlock("builtin_eh_return.cont"));
4237
4238 return RValue::get(nullptr);
4239 }
4240 case Builtin::BI__builtin_unwind_init: {
4241 Function *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
4242 Builder.CreateCall(F);
4243 return RValue::get(nullptr);
4244 }
4245 case Builtin::BI__builtin_extend_pointer: {
4246 // Extends a pointer to the size of an _Unwind_Word, which is
4247 // uint64_t on all platforms. Generally this gets poked into a
4248 // register and eventually used as an address, so if the
4249 // addressing registers are wider than pointers and the platform
4250 // doesn't implicitly ignore high-order bits when doing
4251 // addressing, we need to make sure we zext / sext based on
4252 // the platform's expectations.
4253 //
4254 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
4255
4256 // Cast the pointer to intptr_t.
4257 Value *Ptr = EmitScalarExpr(E->getArg(0));
4258 Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast");
4259
4260 // If that's 64 bits, we're done.
4261 if (IntPtrTy->getBitWidth() == 64)
4262 return RValue::get(Result);
4263
4264 // Otherwise, ask the codegen data what to do.
4265 if (getTargetHooks().extendPointerWithSExt())
4266 return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext"));
4267 else
4268 return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext"));
4269 }
4270 case Builtin::BI__builtin_setjmp: {
4271 // Buffer is a void**.
4273
4274 // Store the frame pointer to the setjmp buffer.
4275 Value *FrameAddr = Builder.CreateCall(
4276 CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy),
4277 ConstantInt::get(Int32Ty, 0));
4278 Builder.CreateStore(FrameAddr, Buf);
4279
4280 // Store the stack pointer to the setjmp buffer.
4281 Value *StackAddr = Builder.CreateStackSave();
4282 assert(Buf.getPointer()->getType() == StackAddr->getType());
4283
4284 Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Buf, 2);
4285 Builder.CreateStore(StackAddr, StackSaveSlot);
4286
4287 // Call LLVM's EH setjmp, which is lightweight.
4288 Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
4289 return RValue::get(Builder.CreateCall(F, Buf.getPointer()));
4290 }
4291 case Builtin::BI__builtin_longjmp: {
4292 Value *Buf = EmitScalarExpr(E->getArg(0));
4293
4294 // Call LLVM's EH longjmp, which is lightweight.
4295 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
4296
4297 // longjmp doesn't return; mark this as unreachable.
4298 Builder.CreateUnreachable();
4299
4300 // We do need to preserve an insertion point.
4301 EmitBlock(createBasicBlock("longjmp.cont"));
4302
4303 return RValue::get(nullptr);
4304 }
4305 case Builtin::BI__builtin_launder: {
4306 const Expr *Arg = E->getArg(0);
4307 QualType ArgTy = Arg->getType()->getPointeeType();
4308 Value *Ptr = EmitScalarExpr(Arg);
4309 if (TypeRequiresBuiltinLaunder(CGM, ArgTy))
4311
4312 return RValue::get(Ptr);
4313 }
4314 case Builtin::BI__sync_fetch_and_add:
4315 case Builtin::BI__sync_fetch_and_sub:
4316 case Builtin::BI__sync_fetch_and_or:
4317 case Builtin::BI__sync_fetch_and_and:
4318 case Builtin::BI__sync_fetch_and_xor:
4319 case Builtin::BI__sync_fetch_and_nand:
4320 case Builtin::BI__sync_add_and_fetch:
4321 case Builtin::BI__sync_sub_and_fetch:
4322 case Builtin::BI__sync_and_and_fetch:
4323 case Builtin::BI__sync_or_and_fetch:
4324 case Builtin::BI__sync_xor_and_fetch:
4325 case Builtin::BI__sync_nand_and_fetch:
4326 case Builtin::BI__sync_val_compare_and_swap:
4327 case Builtin::BI__sync_bool_compare_and_swap:
4328 case Builtin::BI__sync_lock_test_and_set:
4329 case Builtin::BI__sync_lock_release:
4330 case Builtin::BI__sync_swap:
4331 llvm_unreachable("Shouldn't make it through sema");
4332 case Builtin::BI__sync_fetch_and_add_1:
4333 case Builtin::BI__sync_fetch_and_add_2:
4334 case Builtin::BI__sync_fetch_and_add_4:
4335 case Builtin::BI__sync_fetch_and_add_8:
4336 case Builtin::BI__sync_fetch_and_add_16:
4337 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E);
4338 case Builtin::BI__sync_fetch_and_sub_1:
4339 case Builtin::BI__sync_fetch_and_sub_2:
4340 case Builtin::BI__sync_fetch_and_sub_4:
4341 case Builtin::BI__sync_fetch_and_sub_8:
4342 case Builtin::BI__sync_fetch_and_sub_16:
4343 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E);
4344 case Builtin::BI__sync_fetch_and_or_1:
4345 case Builtin::BI__sync_fetch_and_or_2:
4346 case Builtin::BI__sync_fetch_and_or_4:
4347 case Builtin::BI__sync_fetch_and_or_8:
4348 case Builtin::BI__sync_fetch_and_or_16:
4349 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E);
4350 case Builtin::BI__sync_fetch_and_and_1:
4351 case Builtin::BI__sync_fetch_and_and_2:
4352 case Builtin::BI__sync_fetch_and_and_4:
4353 case Builtin::BI__sync_fetch_and_and_8:
4354 case Builtin::BI__sync_fetch_and_and_16:
4355 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E);
4356 case Builtin::BI__sync_fetch_and_xor_1:
4357 case Builtin::BI__sync_fetch_and_xor_2:
4358 case Builtin::BI__sync_fetch_and_xor_4:
4359 case Builtin::BI__sync_fetch_and_xor_8:
4360 case Builtin::BI__sync_fetch_and_xor_16:
4361 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E);
4362 case Builtin::BI__sync_fetch_and_nand_1:
4363 case Builtin::BI__sync_fetch_and_nand_2:
4364 case Builtin::BI__sync_fetch_and_nand_4:
4365 case Builtin::BI__sync_fetch_and_nand_8:
4366 case Builtin::BI__sync_fetch_and_nand_16:
4367 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Nand, E);
4368
4369 // Clang extensions: not overloaded yet.
4370 case Builtin::BI__sync_fetch_and_min:
4371 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E);
4372 case Builtin::BI__sync_fetch_and_max:
4373 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E);
4374 case Builtin::BI__sync_fetch_and_umin:
4375 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E);
4376 case Builtin::BI__sync_fetch_and_umax:
4377 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E);
4378
4379 case Builtin::BI__sync_add_and_fetch_1:
4380 case Builtin::BI__sync_add_and_fetch_2:
4381 case Builtin::BI__sync_add_and_fetch_4:
4382 case Builtin::BI__sync_add_and_fetch_8:
4383 case Builtin::BI__sync_add_and_fetch_16:
4384 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E,
4385 llvm::Instruction::Add);
4386 case Builtin::BI__sync_sub_and_fetch_1:
4387 case Builtin::BI__sync_sub_and_fetch_2:
4388 case Builtin::BI__sync_sub_and_fetch_4:
4389 case Builtin::BI__sync_sub_and_fetch_8:
4390 case Builtin::BI__sync_sub_and_fetch_16:
4391 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E,
4392 llvm::Instruction::Sub);
4393 case Builtin::BI__sync_and_and_fetch_1:
4394 case Builtin::BI__sync_and_and_fetch_2:
4395 case Builtin::BI__sync_and_and_fetch_4:
4396 case Builtin::BI__sync_and_and_fetch_8:
4397 case Builtin::BI__sync_and_and_fetch_16:
4398 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E,
4399 llvm::Instruction::And);
4400 case Builtin::BI__sync_or_and_fetch_1:
4401 case Builtin::BI__sync_or_and_fetch_2:
4402 case Builtin::BI__sync_or_and_fetch_4:
4403 case Builtin::BI__sync_or_and_fetch_8:
4404 case Builtin::BI__sync_or_and_fetch_16:
4405 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E,
4406 llvm::Instruction::Or);
4407 case Builtin::BI__sync_xor_and_fetch_1:
4408 case Builtin::BI__sync_xor_and_fetch_2:
4409 case Builtin::BI__sync_xor_and_fetch_4:
4410 case Builtin::BI__sync_xor_and_fetch_8:
4411 case Builtin::BI__sync_xor_and_fetch_16:
4412 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E,
4413 llvm::Instruction::Xor);
4414 case Builtin::BI__sync_nand_and_fetch_1:
4415 case Builtin::BI__sync_nand_and_fetch_2:
4416 case Builtin::BI__sync_nand_and_fetch_4:
4417 case Builtin::BI__sync_nand_and_fetch_8:
4418 case Builtin::BI__sync_nand_and_fetch_16:
4419 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Nand, E,
4420 llvm::Instruction::And, true);
4421
4422 case Builtin::BI__sync_val_compare_and_swap_1:
4423 case Builtin::BI__sync_val_compare_and_swap_2:
4424 case Builtin::BI__sync_val_compare_and_swap_4:
4425 case Builtin::BI__sync_val_compare_and_swap_8:
4426 case Builtin::BI__sync_val_compare_and_swap_16:
4427 return RValue::get(MakeAtomicCmpXchgValue(*this, E, false));
4428
4429 case Builtin::BI__sync_bool_compare_and_swap_1:
4430 case Builtin::BI__sync_bool_compare_and_swap_2:
4431 case Builtin::BI__sync_bool_compare_and_swap_4:
4432 case Builtin::BI__sync_bool_compare_and_swap_8:
4433 case Builtin::BI__sync_bool_compare_and_swap_16:
4434 return RValue::get(MakeAtomicCmpXchgValue(*this, E, true));
4435
4436 case Builtin::BI__sync_swap_1:
4437 case Builtin::BI__sync_swap_2:
4438 case Builtin::BI__sync_swap_4:
4439 case Builtin::BI__sync_swap_8:
4440 case Builtin::BI__sync_swap_16:
4441 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
4442
4443 case Builtin::BI__sync_lock_test_and_set_1:
4444 case Builtin::BI__sync_lock_test_and_set_2:
4445 case Builtin::BI__sync_lock_test_and_set_4:
4446 case Builtin::BI__sync_lock_test_and_set_8:
4447 case Builtin::BI__sync_lock_test_and_set_16:
4448 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
4449
4450 case Builtin::BI__sync_lock_release_1:
4451 case Builtin::BI__sync_lock_release_2:
4452 case Builtin::BI__sync_lock_release_4:
4453 case Builtin::BI__sync_lock_release_8:
4454 case Builtin::BI__sync_lock_release_16: {
4455 Address Ptr = CheckAtomicAlignment(*this, E);
4456 QualType ElTy = E->getArg(0)->getType()->getPointeeType();
4457
4458 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
4459 getContext().getTypeSize(ElTy));
4460 llvm::StoreInst *Store =
4461 Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr);
4462 Store->setAtomic(llvm::AtomicOrdering::Release);
4463 return RValue::get(nullptr);
4464 }
4465
4466 case Builtin::BI__sync_synchronize: {
4467 // We assume this is supposed to correspond to a C++0x-style
4468 // sequentially-consistent fence (i.e. this is only usable for
4469 // synchronization, not device I/O or anything like that). This intrinsic
4470 // is really badly designed in the sense that in theory, there isn't
4471 // any way to safely use it... but in practice, it mostly works
4472 // to use it with non-atomic loads and stores to get acquire/release
4473 // semantics.
4474 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
4475 return RValue::get(nullptr);
4476 }
4477
4478 case Builtin::BI__builtin_nontemporal_load:
4479 return RValue::get(EmitNontemporalLoad(*this, E));
4480 case Builtin::BI__builtin_nontemporal_store:
4481 return RValue::get(EmitNontemporalStore(*this, E));
4482 case Builtin::BI__c11_atomic_is_lock_free:
4483 case Builtin::BI__atomic_is_lock_free: {
4484 // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
4485 // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
4486 // _Atomic(T) is always properly-aligned.
4487 const char *LibCallName = "__atomic_is_lock_free";
4488 CallArgList Args;
4489 Args.add(RValue::get(EmitScalarExpr(E->getArg(0))),
4490 getContext().getSizeType());
4491 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
4492 Args.add(RValue::get(EmitScalarExpr(E->getArg(1))),
4494 else
4495 Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)),
4497 const CGFunctionInfo &FuncInfo =
4499 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
4500 llvm::FunctionCallee Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
4501 return EmitCall(FuncInfo, CGCallee::forDirect(Func),
4502 ReturnValueSlot(), Args);
4503 }
4504
4505 case Builtin::BI__atomic_test_and_set: {
4506 // Look at the argument type to determine whether this is a volatile
4507 // operation. The parameter type is always volatile.
4508 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
4509 bool Volatile =
4510 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
4511
4512 Address Ptr =
4514
4515 Value *NewVal = Builder.getInt8(1);
4516 Value *Order = EmitScalarExpr(E->getArg(1));
4517 if (isa<llvm::ConstantInt>(Order)) {
4518 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4519 AtomicRMWInst *Result = nullptr;
4520 switch (ord) {
4521 case 0: // memory_order_relaxed
4522 default: // invalid order
4523 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4524 llvm::AtomicOrdering::Monotonic);
4525 break;
4526 case 1: // memory_order_consume
4527 case 2: // memory_order_acquire
4528 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,