clang 19.0.0git
CGBuiltin.cpp
Go to the documentation of this file.
1//===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfo.h"
14#include "CGCUDARuntime.h"
15#include "CGCXXABI.h"
16#include "CGHLSLRuntime.h"
17#include "CGObjCRuntime.h"
18#include "CGOpenCLRuntime.h"
19#include "CGRecordLayout.h"
20#include "CodeGenFunction.h"
21#include "CodeGenModule.h"
22#include "ConstantEmitter.h"
23#include "PatternInit.h"
24#include "TargetInfo.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/Decl.h"
28#include "clang/AST/OSLog.h"
35#include "llvm/ADT/APFloat.h"
36#include "llvm/ADT/APInt.h"
37#include "llvm/ADT/FloatingPointMode.h"
38#include "llvm/ADT/SmallPtrSet.h"
39#include "llvm/ADT/StringExtras.h"
40#include "llvm/Analysis/ValueTracking.h"
41#include "llvm/IR/DataLayout.h"
42#include "llvm/IR/InlineAsm.h"
43#include "llvm/IR/Intrinsics.h"
44#include "llvm/IR/IntrinsicsAArch64.h"
45#include "llvm/IR/IntrinsicsAMDGPU.h"
46#include "llvm/IR/IntrinsicsARM.h"
47#include "llvm/IR/IntrinsicsBPF.h"
48#include "llvm/IR/IntrinsicsDirectX.h"
49#include "llvm/IR/IntrinsicsHexagon.h"
50#include "llvm/IR/IntrinsicsNVPTX.h"
51#include "llvm/IR/IntrinsicsPowerPC.h"
52#include "llvm/IR/IntrinsicsR600.h"
53#include "llvm/IR/IntrinsicsRISCV.h"
54#include "llvm/IR/IntrinsicsS390.h"
55#include "llvm/IR/IntrinsicsVE.h"
56#include "llvm/IR/IntrinsicsWebAssembly.h"
57#include "llvm/IR/IntrinsicsX86.h"
58#include "llvm/IR/MDBuilder.h"
59#include "llvm/IR/MatrixBuilder.h"
60#include "llvm/Support/ConvertUTF.h"
61#include "llvm/Support/MathExtras.h"
62#include "llvm/Support/ScopedPrinter.h"
63#include "llvm/TargetParser/AArch64TargetParser.h"
64#include "llvm/TargetParser/X86TargetParser.h"
65#include <optional>
66#include <sstream>
67
68using namespace clang;
69using namespace CodeGen;
70using namespace llvm;
71
72static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size,
73 Align AlignmentInBytes) {
74 ConstantInt *Byte;
75 switch (CGF.getLangOpts().getTrivialAutoVarInit()) {
76 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
77 // Nothing to initialize.
78 return;
79 case LangOptions::TrivialAutoVarInitKind::Zero:
80 Byte = CGF.Builder.getInt8(0x00);
81 break;
82 case LangOptions::TrivialAutoVarInitKind::Pattern: {
83 llvm::Type *Int8 = llvm::IntegerType::getInt8Ty(CGF.CGM.getLLVMContext());
84 Byte = llvm::dyn_cast<llvm::ConstantInt>(
85 initializationPatternFor(CGF.CGM, Int8));
86 break;
87 }
88 }
89 if (CGF.CGM.stopAutoInit())
90 return;
91 auto *I = CGF.Builder.CreateMemSet(AI, Byte, Size, AlignmentInBytes);
92 I->addAnnotationMetadata("auto-init");
93}
94
95/// getBuiltinLibFunction - Given a builtin id for a function like
96/// "__builtin_fabsf", return a Function* for "fabsf".
98 unsigned BuiltinID) {
99 assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
100
101 // Get the name, skip over the __builtin_ prefix (if necessary).
102 StringRef Name;
103 GlobalDecl D(FD);
104
105 // TODO: This list should be expanded or refactored after all GCC-compatible
106 // std libcall builtins are implemented.
107 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
108 {Builtin::BI__builtin___fprintf_chk, "__fprintf_chkieee128"},
109 {Builtin::BI__builtin___printf_chk, "__printf_chkieee128"},
110 {Builtin::BI__builtin___snprintf_chk, "__snprintf_chkieee128"},
111 {Builtin::BI__builtin___sprintf_chk, "__sprintf_chkieee128"},
112 {Builtin::BI__builtin___vfprintf_chk, "__vfprintf_chkieee128"},
113 {Builtin::BI__builtin___vprintf_chk, "__vprintf_chkieee128"},
114 {Builtin::BI__builtin___vsnprintf_chk, "__vsnprintf_chkieee128"},
115 {Builtin::BI__builtin___vsprintf_chk, "__vsprintf_chkieee128"},
116 {Builtin::BI__builtin_fprintf, "__fprintfieee128"},
117 {Builtin::BI__builtin_printf, "__printfieee128"},
118 {Builtin::BI__builtin_snprintf, "__snprintfieee128"},
119 {Builtin::BI__builtin_sprintf, "__sprintfieee128"},
120 {Builtin::BI__builtin_vfprintf, "__vfprintfieee128"},
121 {Builtin::BI__builtin_vprintf, "__vprintfieee128"},
122 {Builtin::BI__builtin_vsnprintf, "__vsnprintfieee128"},
123 {Builtin::BI__builtin_vsprintf, "__vsprintfieee128"},
124 {Builtin::BI__builtin_fscanf, "__fscanfieee128"},
125 {Builtin::BI__builtin_scanf, "__scanfieee128"},
126 {Builtin::BI__builtin_sscanf, "__sscanfieee128"},
127 {Builtin::BI__builtin_vfscanf, "__vfscanfieee128"},
128 {Builtin::BI__builtin_vscanf, "__vscanfieee128"},
129 {Builtin::BI__builtin_vsscanf, "__vsscanfieee128"},
130 {Builtin::BI__builtin_nexttowardf128, "__nexttowardieee128"},
131 };
132
133 // The AIX library functions frexpl, ldexpl, and modfl are for 128-bit
134 // IBM 'long double' (i.e. __ibm128). Map to the 'double' versions
135 // if it is 64-bit 'long double' mode.
136 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
137 {Builtin::BI__builtin_frexpl, "frexp"},
138 {Builtin::BI__builtin_ldexpl, "ldexp"},
139 {Builtin::BI__builtin_modfl, "modf"},
140 };
141
142 // If the builtin has been declared explicitly with an assembler label,
143 // use the mangled name. This differs from the plain label on platforms
144 // that prefix labels.
145 if (FD->hasAttr<AsmLabelAttr>())
146 Name = getMangledName(D);
147 else {
148 // TODO: This mutation should also be applied to other targets other than
149 // PPC, after backend supports IEEE 128-bit style libcalls.
150 if (getTriple().isPPC64() &&
151 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
152 F128Builtins.contains(BuiltinID))
153 Name = F128Builtins[BuiltinID];
154 else if (getTriple().isOSAIX() &&
155 &getTarget().getLongDoubleFormat() ==
156 &llvm::APFloat::IEEEdouble() &&
157 AIXLongDouble64Builtins.contains(BuiltinID))
158 Name = AIXLongDouble64Builtins[BuiltinID];
159 else
160 Name = Context.BuiltinInfo.getName(BuiltinID).substr(10);
161 }
162
163 llvm::FunctionType *Ty =
164 cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
165
166 return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);
167}
168
169/// Emit the conversions required to turn the given value into an
170/// integer of the given size.
171static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
172 QualType T, llvm::IntegerType *IntType) {
173 V = CGF.EmitToMemory(V, T);
174
175 if (V->getType()->isPointerTy())
176 return CGF.Builder.CreatePtrToInt(V, IntType);
177
178 assert(V->getType() == IntType);
179 return V;
180}
181
182static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
183 QualType T, llvm::Type *ResultType) {
184 V = CGF.EmitFromMemory(V, T);
185
186 if (ResultType->isPointerTy())
187 return CGF.Builder.CreateIntToPtr(V, ResultType);
188
189 assert(V->getType() == ResultType);
190 return V;
191}
192
194 ASTContext &Ctx = CGF.getContext();
195 Address Ptr = CGF.EmitPointerWithAlignment(E->getArg(0));
196 unsigned Bytes = Ptr.getElementType()->isPointerTy()
198 : Ptr.getElementType()->getScalarSizeInBits() / 8;
199 unsigned Align = Ptr.getAlignment().getQuantity();
200 if (Align % Bytes != 0) {
201 DiagnosticsEngine &Diags = CGF.CGM.getDiags();
202 Diags.Report(E->getBeginLoc(), diag::warn_sync_op_misaligned);
203 // Force address to be at least naturally-aligned.
204 return Ptr.withAlignment(CharUnits::fromQuantity(Bytes));
205 }
206 return Ptr;
207}
208
209/// Utility to insert an atomic instruction based on Intrinsic::ID
210/// and the expression node.
212 CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E,
213 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
214
215 QualType T = E->getType();
216 assert(E->getArg(0)->getType()->isPointerType());
218 E->getArg(0)->getType()->getPointeeType()));
219 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
220
221 Address DestAddr = CheckAtomicAlignment(CGF, E);
222
223 llvm::IntegerType *IntType = llvm::IntegerType::get(
224 CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
225
226 llvm::Value *Val = CGF.EmitScalarExpr(E->getArg(1));
227 llvm::Type *ValueType = Val->getType();
228 Val = EmitToInt(CGF, Val, T, IntType);
229
230 llvm::Value *Result =
231 CGF.Builder.CreateAtomicRMW(Kind, DestAddr, Val, Ordering);
232 return EmitFromInt(CGF, Result, T, ValueType);
233}
234
236 Value *Val = CGF.EmitScalarExpr(E->getArg(0));
237 Address Addr = CGF.EmitPointerWithAlignment(E->getArg(1));
238
239 Val = CGF.EmitToMemory(Val, E->getArg(0)->getType());
240 LValue LV = CGF.MakeAddrLValue(Addr, E->getArg(0)->getType());
241 LV.setNontemporal(true);
242 CGF.EmitStoreOfScalar(Val, LV, false);
243 return nullptr;
244}
245
247 Address Addr = CGF.EmitPointerWithAlignment(E->getArg(0));
248
249 LValue LV = CGF.MakeAddrLValue(Addr, E->getType());
250 LV.setNontemporal(true);
251 return CGF.EmitLoadOfScalar(LV, E->getExprLoc());
252}
253
255 llvm::AtomicRMWInst::BinOp Kind,
256 const CallExpr *E) {
257 return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E));
258}
259
260/// Utility to insert an atomic instruction based Intrinsic::ID and
261/// the expression node, where the return value is the result of the
262/// operation.
264 llvm::AtomicRMWInst::BinOp Kind,
265 const CallExpr *E,
266 Instruction::BinaryOps Op,
267 bool Invert = false) {
268 QualType T = E->getType();
269 assert(E->getArg(0)->getType()->isPointerType());
271 E->getArg(0)->getType()->getPointeeType()));
272 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
273
274 Address DestAddr = CheckAtomicAlignment(CGF, E);
275
276 llvm::IntegerType *IntType = llvm::IntegerType::get(
277 CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
278
279 llvm::Value *Val = CGF.EmitScalarExpr(E->getArg(1));
280 llvm::Type *ValueType = Val->getType();
281 Val = EmitToInt(CGF, Val, T, IntType);
282
283 llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
284 Kind, DestAddr, Val, llvm::AtomicOrdering::SequentiallyConsistent);
285 Result = CGF.Builder.CreateBinOp(Op, Result, Val);
286 if (Invert)
287 Result =
288 CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,
289 llvm::ConstantInt::getAllOnesValue(IntType));
290 Result = EmitFromInt(CGF, Result, T, ValueType);
291 return RValue::get(Result);
292}
293
294/// Utility to insert an atomic cmpxchg instruction.
295///
296/// @param CGF The current codegen function.
297/// @param E Builtin call expression to convert to cmpxchg.
298/// arg0 - address to operate on
299/// arg1 - value to compare with
300/// arg2 - new value
301/// @param ReturnBool Specifies whether to return success flag of
302/// cmpxchg result or the old value.
303///
304/// @returns result of cmpxchg, according to ReturnBool
305///
306/// Note: In order to lower Microsoft's _InterlockedCompareExchange* intrinsics
307/// invoke the function EmitAtomicCmpXchgForMSIntrin.
309 bool ReturnBool) {
310 QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType();
311 Address DestAddr = CheckAtomicAlignment(CGF, E);
312
313 llvm::IntegerType *IntType = llvm::IntegerType::get(
314 CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
315
316 Value *Cmp = CGF.EmitScalarExpr(E->getArg(1));
317 llvm::Type *ValueType = Cmp->getType();
318 Cmp = EmitToInt(CGF, Cmp, T, IntType);
319 Value *New = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);
320
322 DestAddr, Cmp, New, llvm::AtomicOrdering::SequentiallyConsistent,
323 llvm::AtomicOrdering::SequentiallyConsistent);
324 if (ReturnBool)
325 // Extract boolean success flag and zext it to int.
326 return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1),
327 CGF.ConvertType(E->getType()));
328 else
329 // Extract old value and emit it using the same type as compare value.
330 return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T,
331 ValueType);
332}
333
334/// This function should be invoked to emit atomic cmpxchg for Microsoft's
335/// _InterlockedCompareExchange* intrinsics which have the following signature:
336/// T _InterlockedCompareExchange(T volatile *Destination,
337/// T Exchange,
338/// T Comparand);
339///
340/// Whereas the llvm 'cmpxchg' instruction has the following syntax:
341/// cmpxchg *Destination, Comparand, Exchange.
342/// So we need to swap Comparand and Exchange when invoking
343/// CreateAtomicCmpXchg. That is the reason we could not use the above utility
344/// function MakeAtomicCmpXchgValue since it expects the arguments to be
345/// already swapped.
346
347static
349 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
350 assert(E->getArg(0)->getType()->isPointerType());
352 E->getType(), E->getArg(0)->getType()->getPointeeType()));
353 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
354 E->getArg(1)->getType()));
355 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
356 E->getArg(2)->getType()));
357
358 Address DestAddr = CheckAtomicAlignment(CGF, E);
359
360 auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
361 auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));
362
363 // For Release ordering, the failure ordering should be Monotonic.
364 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
365 AtomicOrdering::Monotonic :
366 SuccessOrdering;
367
368 // The atomic instruction is marked volatile for consistency with MSVC. This
369 // blocks the few atomics optimizations that LLVM has. If we want to optimize
370 // _Interlocked* operations in the future, we will have to remove the volatile
371 // marker.
373 DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
374 Result->setVolatile(true);
375 return CGF.Builder.CreateExtractValue(Result, 0);
376}
377
378// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
379// prototyped like this:
380//
381// unsigned char _InterlockedCompareExchange128...(
382// __int64 volatile * _Destination,
383// __int64 _ExchangeHigh,
384// __int64 _ExchangeLow,
385// __int64 * _ComparandResult);
386//
387// Note that Destination is assumed to be at least 16-byte aligned, despite
388// being typed int64.
389
391 const CallExpr *E,
392 AtomicOrdering SuccessOrdering) {
393 assert(E->getNumArgs() == 4);
394 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
395 llvm::Value *ExchangeHigh = CGF.EmitScalarExpr(E->getArg(1));
396 llvm::Value *ExchangeLow = CGF.EmitScalarExpr(E->getArg(2));
397 Address ComparandAddr = CGF.EmitPointerWithAlignment(E->getArg(3));
398
399 assert(DestPtr->getType()->isPointerTy());
400 assert(!ExchangeHigh->getType()->isPointerTy());
401 assert(!ExchangeLow->getType()->isPointerTy());
402
403 // For Release ordering, the failure ordering should be Monotonic.
404 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
405 ? AtomicOrdering::Monotonic
406 : SuccessOrdering;
407
408 // Convert to i128 pointers and values. Alignment is also overridden for
409 // destination pointer.
410 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.getLLVMContext(), 128);
411 Address DestAddr(DestPtr, Int128Ty,
413 ComparandAddr = ComparandAddr.withElementType(Int128Ty);
414
415 // (((i128)hi) << 64) | ((i128)lo)
416 ExchangeHigh = CGF.Builder.CreateZExt(ExchangeHigh, Int128Ty);
417 ExchangeLow = CGF.Builder.CreateZExt(ExchangeLow, Int128Ty);
418 ExchangeHigh =
419 CGF.Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
420 llvm::Value *Exchange = CGF.Builder.CreateOr(ExchangeHigh, ExchangeLow);
421
422 // Load the comparand for the instruction.
423 llvm::Value *Comparand = CGF.Builder.CreateLoad(ComparandAddr);
424
425 auto *CXI = CGF.Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
426 SuccessOrdering, FailureOrdering);
427
428 // The atomic instruction is marked volatile for consistency with MSVC. This
429 // blocks the few atomics optimizations that LLVM has. If we want to optimize
430 // _Interlocked* operations in the future, we will have to remove the volatile
431 // marker.
432 CXI->setVolatile(true);
433
434 // Store the result as an outparameter.
435 CGF.Builder.CreateStore(CGF.Builder.CreateExtractValue(CXI, 0),
436 ComparandAddr);
437
438 // Get the success boolean and zero extend it to i8.
439 Value *Success = CGF.Builder.CreateExtractValue(CXI, 1);
440 return CGF.Builder.CreateZExt(Success, CGF.Int8Ty);
441}
442
444 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
445 assert(E->getArg(0)->getType()->isPointerType());
446
447 auto *IntTy = CGF.ConvertType(E->getType());
448 Address DestAddr = CheckAtomicAlignment(CGF, E);
449 auto *Result = CGF.Builder.CreateAtomicRMW(
450 AtomicRMWInst::Add, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
451 return CGF.Builder.CreateAdd(Result, ConstantInt::get(IntTy, 1));
452}
453
455 CodeGenFunction &CGF, const CallExpr *E,
456 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
457 assert(E->getArg(0)->getType()->isPointerType());
458
459 auto *IntTy = CGF.ConvertType(E->getType());
460 Address DestAddr = CheckAtomicAlignment(CGF, E);
461 auto *Result = CGF.Builder.CreateAtomicRMW(
462 AtomicRMWInst::Sub, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
463 return CGF.Builder.CreateSub(Result, ConstantInt::get(IntTy, 1));
464}
465
466// Build a plain volatile load.
468 Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
469 QualType ElTy = E->getArg(0)->getType()->getPointeeType();
470 CharUnits LoadSize = CGF.getContext().getTypeSizeInChars(ElTy);
471 llvm::Type *ITy =
472 llvm::IntegerType::get(CGF.getLLVMContext(), LoadSize.getQuantity() * 8);
473 llvm::LoadInst *Load = CGF.Builder.CreateAlignedLoad(ITy, Ptr, LoadSize);
474 Load->setVolatile(true);
475 return Load;
476}
477
478// Build a plain volatile store.
480 Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
481 Value *Value = CGF.EmitScalarExpr(E->getArg(1));
482 QualType ElTy = E->getArg(0)->getType()->getPointeeType();
483 CharUnits StoreSize = CGF.getContext().getTypeSizeInChars(ElTy);
484 llvm::StoreInst *Store =
485 CGF.Builder.CreateAlignedStore(Value, Ptr, StoreSize);
486 Store->setVolatile(true);
487 return Store;
488}
489
490// Emit a simple mangled intrinsic that has 1 argument and a return type
491// matching the argument type. Depending on mode, this may be a constrained
492// floating-point intrinsic.
494 const CallExpr *E, unsigned IntrinsicID,
495 unsigned ConstrainedIntrinsicID) {
496 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
497
498 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
499 if (CGF.Builder.getIsFPConstrained()) {
500 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
501 return CGF.Builder.CreateConstrainedFPCall(F, { Src0 });
502 } else {
503 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
504 return CGF.Builder.CreateCall(F, Src0);
505 }
506}
507
508// Emit an intrinsic that has 2 operands of the same type as its result.
509// Depending on mode, this may be a constrained floating-point intrinsic.
511 const CallExpr *E, unsigned IntrinsicID,
512 unsigned ConstrainedIntrinsicID) {
513 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
514 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
515
516 if (CGF.Builder.getIsFPConstrained()) {
517 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
518 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
519 return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
520 } else {
521 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
522 return CGF.Builder.CreateCall(F, { Src0, Src1 });
523 }
524}
525
526// Has second type mangled argument.
528 CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID,
529 llvm::Intrinsic::ID ConstrainedIntrinsicID) {
530 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
531 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
532
533 if (CGF.Builder.getIsFPConstrained()) {
534 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
535 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID,
536 {Src0->getType(), Src1->getType()});
537 return CGF.Builder.CreateConstrainedFPCall(F, {Src0, Src1});
538 }
539
540 Function *F =
541 CGF.CGM.getIntrinsic(IntrinsicID, {Src0->getType(), Src1->getType()});
542 return CGF.Builder.CreateCall(F, {Src0, Src1});
543}
544
545// Emit an intrinsic that has 3 operands of the same type as its result.
546// Depending on mode, this may be a constrained floating-point intrinsic.
548 const CallExpr *E, unsigned IntrinsicID,
549 unsigned ConstrainedIntrinsicID) {
550 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
551 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
552 llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
553
554 if (CGF.Builder.getIsFPConstrained()) {
555 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
556 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
557 return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
558 } else {
559 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
560 return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });
561 }
562}
563
564// Emit an intrinsic where all operands are of the same type as the result.
565// Depending on mode, this may be a constrained floating-point intrinsic.
567 unsigned IntrinsicID,
568 unsigned ConstrainedIntrinsicID,
569 llvm::Type *Ty,
570 ArrayRef<Value *> Args) {
571 Function *F;
572 if (CGF.Builder.getIsFPConstrained())
573 F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Ty);
574 else
575 F = CGF.CGM.getIntrinsic(IntrinsicID, Ty);
576
577 if (CGF.Builder.getIsFPConstrained())
578 return CGF.Builder.CreateConstrainedFPCall(F, Args);
579 else
580 return CGF.Builder.CreateCall(F, Args);
581}
582
583// Emit a simple mangled intrinsic that has 1 argument and a return type
584// matching the argument type.
586 unsigned IntrinsicID,
587 llvm::StringRef Name = "") {
588 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
589
590 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
591 return CGF.Builder.CreateCall(F, Src0, Name);
592}
593
594// Emit an intrinsic that has 2 operands of the same type as its result.
596 const CallExpr *E,
597 unsigned IntrinsicID) {
598 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
599 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
600
601 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
602 return CGF.Builder.CreateCall(F, { Src0, Src1 });
603}
604
605// Emit an intrinsic that has 3 operands of the same type as its result.
607 const CallExpr *E,
608 unsigned IntrinsicID) {
609 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
610 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
611 llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
612
613 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
614 return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });
615}
616
617// Emit an intrinsic that has 1 float or double operand, and 1 integer.
619 const CallExpr *E,
620 unsigned IntrinsicID) {
621 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
622 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
623
624 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
625 return CGF.Builder.CreateCall(F, {Src0, Src1});
626}
627
628// Emit an intrinsic that has overloaded integer result and fp operand.
629static Value *
631 unsigned IntrinsicID,
632 unsigned ConstrainedIntrinsicID) {
633 llvm::Type *ResultType = CGF.ConvertType(E->getType());
634 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
635
636 if (CGF.Builder.getIsFPConstrained()) {
637 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
638 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID,
639 {ResultType, Src0->getType()});
640 return CGF.Builder.CreateConstrainedFPCall(F, {Src0});
641 } else {
642 Function *F =
643 CGF.CGM.getIntrinsic(IntrinsicID, {ResultType, Src0->getType()});
644 return CGF.Builder.CreateCall(F, Src0);
645 }
646}
647
649 llvm::Intrinsic::ID IntrinsicID) {
650 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
651 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
652
653 QualType IntPtrTy = E->getArg(1)->getType()->getPointeeType();
654 llvm::Type *IntTy = CGF.ConvertType(IntPtrTy);
655 llvm::Function *F =
656 CGF.CGM.getIntrinsic(IntrinsicID, {Src0->getType(), IntTy});
657 llvm::Value *Call = CGF.Builder.CreateCall(F, Src0);
658
659 llvm::Value *Exp = CGF.Builder.CreateExtractValue(Call, 1);
660 LValue LV = CGF.MakeNaturalAlignAddrLValue(Src1, IntPtrTy);
661 CGF.EmitStoreOfScalar(Exp, LV);
662
663 return CGF.Builder.CreateExtractValue(Call, 0);
664}
665
666/// EmitFAbs - Emit a call to @llvm.fabs().
668 Function *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType());
669 llvm::CallInst *Call = CGF.Builder.CreateCall(F, V);
670 Call->setDoesNotAccessMemory();
671 return Call;
672}
673
674/// Emit the computation of the sign bit for a floating point value. Returns
675/// the i1 sign bit value.
677 LLVMContext &C = CGF.CGM.getLLVMContext();
678
679 llvm::Type *Ty = V->getType();
680 int Width = Ty->getPrimitiveSizeInBits();
681 llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
682 V = CGF.Builder.CreateBitCast(V, IntTy);
683 if (Ty->isPPC_FP128Ty()) {
684 // We want the sign bit of the higher-order double. The bitcast we just
685 // did works as if the double-double was stored to memory and then
686 // read as an i128. The "store" will put the higher-order double in the
687 // lower address in both little- and big-Endian modes, but the "load"
688 // will treat those bits as a different part of the i128: the low bits in
689 // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
690 // we need to shift the high bits down to the low before truncating.
691 Width >>= 1;
692 if (CGF.getTarget().isBigEndian()) {
693 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
694 V = CGF.Builder.CreateLShr(V, ShiftCst);
695 }
696 // We are truncating value in order to extract the higher-order
697 // double, which we will be using to extract the sign from.
698 IntTy = llvm::IntegerType::get(C, Width);
699 V = CGF.Builder.CreateTrunc(V, IntTy);
700 }
701 Value *Zero = llvm::Constant::getNullValue(IntTy);
702 return CGF.Builder.CreateICmpSLT(V, Zero);
703}
704
706 const CallExpr *E, llvm::Constant *calleeValue) {
707 CGCallee callee = CGCallee::forDirect(calleeValue, GlobalDecl(FD));
708 return CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot());
709}
710
711/// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
712/// depending on IntrinsicID.
713///
714/// \arg CGF The current codegen function.
715/// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
716/// \arg X The first argument to the llvm.*.with.overflow.*.
717/// \arg Y The second argument to the llvm.*.with.overflow.*.
718/// \arg Carry The carry returned by the llvm.*.with.overflow.*.
719/// \returns The result (i.e. sum/product) returned by the intrinsic.
720static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
721 const llvm::Intrinsic::ID IntrinsicID,
722 llvm::Value *X, llvm::Value *Y,
723 llvm::Value *&Carry) {
724 // Make sure we have integers of the same width.
725 assert(X->getType() == Y->getType() &&
726 "Arguments must be the same type. (Did you forget to make sure both "
727 "arguments have the same integer width?)");
728
729 Function *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType());
730 llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y});
731 Carry = CGF.Builder.CreateExtractValue(Tmp, 1);
732 return CGF.Builder.CreateExtractValue(Tmp, 0);
733}
734
736 unsigned IntrinsicID,
737 int low, int high) {
738 llvm::MDBuilder MDHelper(CGF.getLLVMContext());
739 llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
740 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, {});
741 llvm::Instruction *Call = CGF.Builder.CreateCall(F);
742 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
743 Call->setMetadata(llvm::LLVMContext::MD_noundef,
744 llvm::MDNode::get(CGF.getLLVMContext(), std::nullopt));
745 return Call;
746}
747
748namespace {
749 struct WidthAndSignedness {
750 unsigned Width;
751 bool Signed;
752 };
753}
754
755static WidthAndSignedness
757 const clang::QualType Type) {
758 assert(Type->isIntegerType() && "Given type is not an integer.");
759 unsigned Width = Type->isBooleanType() ? 1
760 : Type->isBitIntType() ? context.getIntWidth(Type)
761 : context.getTypeInfo(Type).Width;
763 return {Width, Signed};
764}
765
766// Given one or more integer types, this function produces an integer type that
767// encompasses them: any value in one of the given types could be expressed in
768// the encompassing type.
769static struct WidthAndSignedness
770EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
771 assert(Types.size() > 0 && "Empty list of types.");
772
773 // If any of the given types is signed, we must return a signed type.
774 bool Signed = false;
775 for (const auto &Type : Types) {
776 Signed |= Type.Signed;
777 }
778
779 // The encompassing type must have a width greater than or equal to the width
780 // of the specified types. Additionally, if the encompassing type is signed,
781 // its width must be strictly greater than the width of any unsigned types
782 // given.
783 unsigned Width = 0;
784 for (const auto &Type : Types) {
785 unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
786 if (Width < MinWidth) {
787 Width = MinWidth;
788 }
789 }
790
791 return {Width, Signed};
792}
793
794Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
795 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
796 return Builder.CreateCall(CGM.getIntrinsic(inst, {ArgValue->getType()}),
797 ArgValue);
798}
799
800/// Checks if using the result of __builtin_object_size(p, @p From) in place of
801/// __builtin_object_size(p, @p To) is correct
802static bool areBOSTypesCompatible(int From, int To) {
803 // Note: Our __builtin_object_size implementation currently treats Type=0 and
804 // Type=2 identically. Encoding this implementation detail here may make
805 // improving __builtin_object_size difficult in the future, so it's omitted.
806 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
807}
808
809static llvm::Value *
810getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
811 return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true);
812}
813
814llvm::Value *
815CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
816 llvm::IntegerType *ResType,
817 llvm::Value *EmittedE,
818 bool IsDynamic) {
819 uint64_t ObjectSize;
820 if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type))
821 return emitBuiltinObjectSize(E, Type, ResType, EmittedE, IsDynamic);
822 return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true);
823}
824
826 ASTContext &Ctx, const RecordDecl *RD, StringRef Name, uint64_t &Offset) {
827 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
828 getLangOpts().getStrictFlexArraysLevel();
829 unsigned FieldNo = 0;
830 bool IsUnion = RD->isUnion();
831
832 for (const Decl *D : RD->decls()) {
833 if (const auto *Field = dyn_cast<FieldDecl>(D);
834 Field && (Name.empty() || Field->getNameAsString() == Name) &&
836 Ctx, Field, Field->getType(), StrictFlexArraysLevel,
837 /*IgnoreTemplateOrMacroSubstitution=*/true)) {
838 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
839 Offset += Layout.getFieldOffset(FieldNo);
840 return Field;
841 }
842
843 if (const auto *Record = dyn_cast<RecordDecl>(D))
844 if (const FieldDecl *Field =
845 FindFlexibleArrayMemberField(Ctx, Record, Name, Offset)) {
846 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
847 Offset += Layout.getFieldOffset(FieldNo);
848 return Field;
849 }
850
851 if (!IsUnion && isa<FieldDecl>(D))
852 ++FieldNo;
853 }
854
855 return nullptr;
856}
857
858static unsigned CountCountedByAttrs(const RecordDecl *RD) {
859 unsigned Num = 0;
860
861 for (const Decl *D : RD->decls()) {
862 if (const auto *FD = dyn_cast<FieldDecl>(D);
863 FD && FD->getType()->isCountAttributedType()) {
864 return ++Num;
865 }
866
867 if (const auto *Rec = dyn_cast<RecordDecl>(D))
868 Num += CountCountedByAttrs(Rec);
869 }
870
871 return Num;
872}
873
874llvm::Value *
875CodeGenFunction::emitFlexibleArrayMemberSize(const Expr *E, unsigned Type,
876 llvm::IntegerType *ResType) {
877 // The code generated here calculates the size of a struct with a flexible
878 // array member that uses the counted_by attribute. There are two instances
879 // we handle:
880 //
881 // struct s {
882 // unsigned long flags;
883 // int count;
884 // int array[] __attribute__((counted_by(count)));
885 // }
886 //
887 // 1) bdos of the flexible array itself:
888 //
889 // __builtin_dynamic_object_size(p->array, 1) ==
890 // p->count * sizeof(*p->array)
891 //
892 // 2) bdos of a pointer into the flexible array:
893 //
894 // __builtin_dynamic_object_size(&p->array[42], 1) ==
895 // (p->count - 42) * sizeof(*p->array)
896 //
897 // 2) bdos of the whole struct, including the flexible array:
898 //
899 // __builtin_dynamic_object_size(p, 1) ==
900 // max(sizeof(struct s),
901 // offsetof(struct s, array) + p->count * sizeof(*p->array))
902 //
903 ASTContext &Ctx = getContext();
904 const Expr *Base = E->IgnoreParenImpCasts();
905 const Expr *Idx = nullptr;
906
907 if (const auto *UO = dyn_cast<UnaryOperator>(Base);
908 UO && UO->getOpcode() == UO_AddrOf) {
909 Expr *SubExpr = UO->getSubExpr()->IgnoreParenImpCasts();
910 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) {
911 Base = ASE->getBase()->IgnoreParenImpCasts();
912 Idx = ASE->getIdx()->IgnoreParenImpCasts();
913
914 if (const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
915 int64_t Val = IL->getValue().getSExtValue();
916 if (Val < 0)
918
919 if (Val == 0)
920 // The index is 0, so we don't need to take it into account.
921 Idx = nullptr;
922 }
923 } else {
924 // Potential pointer to another element in the struct.
925 Base = SubExpr;
926 }
927 }
928
929 // Get the flexible array member Decl.
930 const RecordDecl *OuterRD = nullptr;
931 std::string FAMName;
932 if (const auto *ME = dyn_cast<MemberExpr>(Base)) {
933 // Check if \p Base is referencing the FAM itself.
934 const ValueDecl *VD = ME->getMemberDecl();
936 FAMName = VD->getNameAsString();
937 } else if (const auto *DRE = dyn_cast<DeclRefExpr>(Base)) {
938 // Check if we're pointing to the whole struct.
939 QualType Ty = DRE->getDecl()->getType();
940 if (Ty->isPointerType())
941 Ty = Ty->getPointeeType();
942 OuterRD = Ty->getAsRecordDecl();
943
944 // If we have a situation like this:
945 //
946 // struct union_of_fams {
947 // int flags;
948 // union {
949 // signed char normal_field;
950 // struct {
951 // int count1;
952 // int arr1[] __counted_by(count1);
953 // };
954 // struct {
955 // signed char count2;
956 // int arr2[] __counted_by(count2);
957 // };
958 // };
959 // };
960 //
961 // We don't know which 'count' to use in this scenario:
962 //
963 // size_t get_size(struct union_of_fams *p) {
964 // return __builtin_dynamic_object_size(p, 1);
965 // }
966 //
967 // Instead of calculating a wrong number, we give up.
968 if (OuterRD && CountCountedByAttrs(OuterRD) > 1)
969 return nullptr;
970 }
971
972 if (!OuterRD)
973 return nullptr;
974
975 uint64_t Offset = 0;
976 const FieldDecl *FAMDecl =
977 FindFlexibleArrayMemberField(Ctx, OuterRD, FAMName, Offset);
978 Offset = Ctx.toCharUnitsFromBits(Offset).getQuantity();
979
980 if (!FAMDecl || !FAMDecl->getType()->isCountAttributedType())
981 // No flexible array member found or it doesn't have the "counted_by"
982 // attribute.
983 return nullptr;
984
985 const FieldDecl *CountedByFD = FindCountedByField(FAMDecl);
986 if (!CountedByFD)
987 // Can't find the field referenced by the "counted_by" attribute.
988 return nullptr;
989
990 // Build a load of the counted_by field.
991 bool IsSigned = CountedByFD->getType()->isSignedIntegerType();
992 Value *CountedByInst = EmitCountedByFieldExpr(Base, FAMDecl, CountedByFD);
993 if (!CountedByInst)
995
996 CountedByInst = Builder.CreateIntCast(CountedByInst, ResType, IsSigned);
997
998 // Build a load of the index and subtract it from the count.
999 Value *IdxInst = nullptr;
1000 if (Idx) {
1001 if (Idx->HasSideEffects(getContext()))
1002 // We can't have side-effects.
1003 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1004
1005 bool IdxSigned = Idx->getType()->isSignedIntegerType();
1006 IdxInst = EmitAnyExprToTemp(Idx).getScalarVal();
1007 IdxInst = Builder.CreateIntCast(IdxInst, ResType, IdxSigned);
1008
1009 // We go ahead with the calculation here. If the index turns out to be
1010 // negative, we'll catch it at the end.
1011 CountedByInst =
1012 Builder.CreateSub(CountedByInst, IdxInst, "", !IsSigned, IsSigned);
1013 }
1014
1015 // Calculate how large the flexible array member is in bytes.
1016 const ArrayType *ArrayTy = Ctx.getAsArrayType(FAMDecl->getType());
1018 llvm::Constant *ElemSize =
1019 llvm::ConstantInt::get(ResType, Size.getQuantity(), IsSigned);
1020 Value *FAMSize =
1021 Builder.CreateMul(CountedByInst, ElemSize, "", !IsSigned, IsSigned);
1022 FAMSize = Builder.CreateIntCast(FAMSize, ResType, IsSigned);
1023 Value *Res = FAMSize;
1024
1025 if (isa<DeclRefExpr>(Base)) {
1026 // The whole struct is specificed in the __bdos.
1027 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(OuterRD);
1028
1029 // Get the offset of the FAM.
1030 llvm::Constant *FAMOffset = ConstantInt::get(ResType, Offset, IsSigned);
1031 Value *OffsetAndFAMSize =
1032 Builder.CreateAdd(FAMOffset, Res, "", !IsSigned, IsSigned);
1033
1034 // Get the full size of the struct.
1035 llvm::Constant *SizeofStruct =
1036 ConstantInt::get(ResType, Layout.getSize().getQuantity(), IsSigned);
1037
1038 // max(sizeof(struct s),
1039 // offsetof(struct s, array) + p->count * sizeof(*p->array))
1040 Res = IsSigned
1041 ? Builder.CreateBinaryIntrinsic(llvm::Intrinsic::smax,
1042 OffsetAndFAMSize, SizeofStruct)
1043 : Builder.CreateBinaryIntrinsic(llvm::Intrinsic::umax,
1044 OffsetAndFAMSize, SizeofStruct);
1045 }
1046
1047 // A negative \p IdxInst or \p CountedByInst means that the index lands
1048 // outside of the flexible array member. If that's the case, we want to
1049 // return 0.
1050 Value *Cmp = Builder.CreateIsNotNeg(CountedByInst);
1051 if (IdxInst)
1052 Cmp = Builder.CreateAnd(Builder.CreateIsNotNeg(IdxInst), Cmp);
1053
1054 return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
1055}
1056
1057/// Returns a Value corresponding to the size of the given expression.
1058/// This Value may be either of the following:
1059/// - A llvm::Argument (if E is a param with the pass_object_size attribute on
1060/// it)
1061/// - A call to the @llvm.objectsize intrinsic
1062///
1063/// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
1064/// and we wouldn't otherwise try to reference a pass_object_size parameter,
1065/// we'll call @llvm.objectsize on EmittedE, rather than emitting E.
1066llvm::Value *
1067CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
1068 llvm::IntegerType *ResType,
1069 llvm::Value *EmittedE, bool IsDynamic) {
1070 // We need to reference an argument if the pointer is a parameter with the
1071 // pass_object_size attribute.
1072 if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) {
1073 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
1074 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
1075 if (Param != nullptr && PS != nullptr &&
1076 areBOSTypesCompatible(PS->getType(), Type)) {
1077 auto Iter = SizeArguments.find(Param);
1078 assert(Iter != SizeArguments.end());
1079
1080 const ImplicitParamDecl *D = Iter->second;
1081 auto DIter = LocalDeclMap.find(D);
1082 assert(DIter != LocalDeclMap.end());
1083
1084 return EmitLoadOfScalar(DIter->second, /*Volatile=*/false,
1085 getContext().getSizeType(), E->getBeginLoc());
1086 }
1087 }
1088
1089 if (IsDynamic) {
1090 // Emit special code for a flexible array member with the "counted_by"
1091 // attribute.
1092 if (Value *V = emitFlexibleArrayMemberSize(E, Type, ResType))
1093 return V;
1094 }
1095
1096 // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
1097 // evaluate E for side-effects. In either case, we shouldn't lower to
1098 // @llvm.objectsize.
1099 if (Type == 3 || (!EmittedE && E->HasSideEffects(getContext())))
1100 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1101
1102 Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
1103 assert(Ptr->getType()->isPointerTy() &&
1104 "Non-pointer passed to __builtin_object_size?");
1105
1106 Function *F =
1107 CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
1108
1109 // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.
1110 Value *Min = Builder.getInt1((Type & 2) != 0);
1111 // For GCC compatibility, __builtin_object_size treat NULL as unknown size.
1112 Value *NullIsUnknown = Builder.getTrue();
1113 Value *Dynamic = Builder.getInt1(IsDynamic);
1114 return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic});
1115}
1116
1117namespace {
1118/// A struct to generically describe a bit test intrinsic.
1119struct BitTest {
1120 enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
1121 enum InterlockingKind : uint8_t {
1122 Unlocked,
1123 Sequential,
1124 Acquire,
1125 Release,
1126 NoFence
1127 };
1128
1129 ActionKind Action;
1130 InterlockingKind Interlocking;
1131 bool Is64Bit;
1132
1133 static BitTest decodeBitTestBuiltin(unsigned BuiltinID);
1134};
1135
1136// Returns the first convergence entry/loop/anchor instruction found in |BB|.
1137// std::nullptr otherwise.
1138llvm::IntrinsicInst *getConvergenceToken(llvm::BasicBlock *BB) {
1139 for (auto &I : *BB) {
1140 auto *II = dyn_cast<llvm::IntrinsicInst>(&I);
1141 if (II && isConvergenceControlIntrinsic(II->getIntrinsicID()))
1142 return II;
1143 }
1144 return nullptr;
1145}
1146
1147} // namespace
1148
1149llvm::CallBase *
1150CodeGenFunction::addConvergenceControlToken(llvm::CallBase *Input,
1151 llvm::Value *ParentToken) {
1152 llvm::Value *bundleArgs[] = {ParentToken};
1153 llvm::OperandBundleDef OB("convergencectrl", bundleArgs);
1154 auto Output = llvm::CallBase::addOperandBundle(
1155 Input, llvm::LLVMContext::OB_convergencectrl, OB, Input);
1156 Input->replaceAllUsesWith(Output);
1157 Input->eraseFromParent();
1158 return Output;
1159}
1160
1161llvm::IntrinsicInst *
1162CodeGenFunction::emitConvergenceLoopToken(llvm::BasicBlock *BB,
1163 llvm::Value *ParentToken) {
1164 CGBuilderTy::InsertPoint IP = Builder.saveIP();
1165 Builder.SetInsertPoint(&BB->front());
1166 auto CB = Builder.CreateIntrinsic(
1167 llvm::Intrinsic::experimental_convergence_loop, {}, {});
1168 Builder.restoreIP(IP);
1169
1170 auto I = addConvergenceControlToken(CB, ParentToken);
1171 return cast<llvm::IntrinsicInst>(I);
1172}
1173
1174llvm::IntrinsicInst *
1175CodeGenFunction::getOrEmitConvergenceEntryToken(llvm::Function *F) {
1176 auto *BB = &F->getEntryBlock();
1177 auto *token = getConvergenceToken(BB);
1178 if (token)
1179 return token;
1180
1181 // Adding a convergence token requires the function to be marked as
1182 // convergent.
1183 F->setConvergent();
1184
1185 CGBuilderTy::InsertPoint IP = Builder.saveIP();
1186 Builder.SetInsertPoint(&BB->front());
1187 auto I = Builder.CreateIntrinsic(
1188 llvm::Intrinsic::experimental_convergence_entry, {}, {});
1189 assert(isa<llvm::IntrinsicInst>(I));
1190 Builder.restoreIP(IP);
1191
1192 return cast<llvm::IntrinsicInst>(I);
1193}
1194
1195llvm::IntrinsicInst *
1196CodeGenFunction::getOrEmitConvergenceLoopToken(const LoopInfo *LI) {
1197 assert(LI != nullptr);
1198
1199 auto *token = getConvergenceToken(LI->getHeader());
1200 if (token)
1201 return token;
1202
1203 llvm::IntrinsicInst *PII =
1204 LI->getParent()
1205 ? emitConvergenceLoopToken(
1206 LI->getHeader(), getOrEmitConvergenceLoopToken(LI->getParent()))
1207 : getOrEmitConvergenceEntryToken(LI->getHeader()->getParent());
1208
1209 return emitConvergenceLoopToken(LI->getHeader(), PII);
1210}
1211
1212llvm::CallBase *
1214 llvm::Value *ParentToken =
1216 ? getOrEmitConvergenceLoopToken(&LoopStack.getInfo())
1217 : getOrEmitConvergenceEntryToken(Input->getFunction());
1218 return addConvergenceControlToken(Input, ParentToken);
1219}
1220
1221BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
1222 switch (BuiltinID) {
1223 // Main portable variants.
1224 case Builtin::BI_bittest:
1225 return {TestOnly, Unlocked, false};
1226 case Builtin::BI_bittestandcomplement:
1227 return {Complement, Unlocked, false};
1228 case Builtin::BI_bittestandreset:
1229 return {Reset, Unlocked, false};
1230 case Builtin::BI_bittestandset:
1231 return {Set, Unlocked, false};
1232 case Builtin::BI_interlockedbittestandreset:
1233 return {Reset, Sequential, false};
1234 case Builtin::BI_interlockedbittestandset:
1235 return {Set, Sequential, false};
1236
1237 // X86-specific 64-bit variants.
1238 case Builtin::BI_bittest64:
1239 return {TestOnly, Unlocked, true};
1240 case Builtin::BI_bittestandcomplement64:
1241 return {Complement, Unlocked, true};
1242 case Builtin::BI_bittestandreset64:
1243 return {Reset, Unlocked, true};
1244 case Builtin::BI_bittestandset64:
1245 return {Set, Unlocked, true};
1246 case Builtin::BI_interlockedbittestandreset64:
1247 return {Reset, Sequential, true};
1248 case Builtin::BI_interlockedbittestandset64:
1249 return {Set, Sequential, true};
1250
1251 // ARM/AArch64-specific ordering variants.
1252 case Builtin::BI_interlockedbittestandset_acq:
1253 return {Set, Acquire, false};
1254 case Builtin::BI_interlockedbittestandset_rel:
1255 return {Set, Release, false};
1256 case Builtin::BI_interlockedbittestandset_nf:
1257 return {Set, NoFence, false};
1258 case Builtin::BI_interlockedbittestandreset_acq:
1259 return {Reset, Acquire, false};
1260 case Builtin::BI_interlockedbittestandreset_rel:
1261 return {Reset, Release, false};
1262 case Builtin::BI_interlockedbittestandreset_nf:
1263 return {Reset, NoFence, false};
1264 }
1265 llvm_unreachable("expected only bittest intrinsics");
1266}
1267
1268static char bitActionToX86BTCode(BitTest::ActionKind A) {
1269 switch (A) {
1270 case BitTest::TestOnly: return '\0';
1271 case BitTest::Complement: return 'c';
1272 case BitTest::Reset: return 'r';
1273 case BitTest::Set: return 's';
1274 }
1275 llvm_unreachable("invalid action");
1276}
1277
1279 BitTest BT,
1280 const CallExpr *E, Value *BitBase,
1281 Value *BitPos) {
1282 char Action = bitActionToX86BTCode(BT.Action);
1283 char SizeSuffix = BT.Is64Bit ? 'q' : 'l';
1284
1285 // Build the assembly.
1287 raw_svector_ostream AsmOS(Asm);
1288 if (BT.Interlocking != BitTest::Unlocked)
1289 AsmOS << "lock ";
1290 AsmOS << "bt";
1291 if (Action)
1292 AsmOS << Action;
1293 AsmOS << SizeSuffix << " $2, ($1)";
1294
1295 // Build the constraints. FIXME: We should support immediates when possible.
1296 std::string Constraints = "={@ccc},r,r,~{cc},~{memory}";
1297 std::string_view MachineClobbers = CGF.getTarget().getClobbers();
1298 if (!MachineClobbers.empty()) {
1299 Constraints += ',';
1300 Constraints += MachineClobbers;
1301 }
1302 llvm::IntegerType *IntType = llvm::IntegerType::get(
1303 CGF.getLLVMContext(),
1304 CGF.getContext().getTypeSize(E->getArg(1)->getType()));
1305 llvm::FunctionType *FTy =
1306 llvm::FunctionType::get(CGF.Int8Ty, {CGF.UnqualPtrTy, IntType}, false);
1307
1308 llvm::InlineAsm *IA =
1309 llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);
1310 return CGF.Builder.CreateCall(IA, {BitBase, BitPos});
1311}
1312
1313static llvm::AtomicOrdering
1314getBitTestAtomicOrdering(BitTest::InterlockingKind I) {
1315 switch (I) {
1316 case BitTest::Unlocked: return llvm::AtomicOrdering::NotAtomic;
1317 case BitTest::Sequential: return llvm::AtomicOrdering::SequentiallyConsistent;
1318 case BitTest::Acquire: return llvm::AtomicOrdering::Acquire;
1319 case BitTest::Release: return llvm::AtomicOrdering::Release;
1320 case BitTest::NoFence: return llvm::AtomicOrdering::Monotonic;
1321 }
1322 llvm_unreachable("invalid interlocking");
1323}
1324
1325/// Emit a _bittest* intrinsic. These intrinsics take a pointer to an array of
1326/// bits and a bit position and read and optionally modify the bit at that
1327/// position. The position index can be arbitrarily large, i.e. it can be larger
1328/// than 31 or 63, so we need an indexed load in the general case.
1329static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
1330 unsigned BuiltinID,
1331 const CallExpr *E) {
1332 Value *BitBase = CGF.EmitScalarExpr(E->getArg(0));
1333 Value *BitPos = CGF.EmitScalarExpr(E->getArg(1));
1334
1335 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1336
1337 // X86 has special BT, BTC, BTR, and BTS instructions that handle the array
1338 // indexing operation internally. Use them if possible.
1339 if (CGF.getTarget().getTriple().isX86())
1340 return EmitX86BitTestIntrinsic(CGF, BT, E, BitBase, BitPos);
1341
1342 // Otherwise, use generic code to load one byte and test the bit. Use all but
1343 // the bottom three bits as the array index, and the bottom three bits to form
1344 // a mask.
1345 // Bit = BitBaseI8[BitPos >> 3] & (1 << (BitPos & 0x7)) != 0;
1346 Value *ByteIndex = CGF.Builder.CreateAShr(
1347 BitPos, llvm::ConstantInt::get(BitPos->getType(), 3), "bittest.byteidx");
1348 Value *BitBaseI8 = CGF.Builder.CreatePointerCast(BitBase, CGF.Int8PtrTy);
1349 Address ByteAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, BitBaseI8,
1350 ByteIndex, "bittest.byteaddr"),
1351 CGF.Int8Ty, CharUnits::One());
1352 Value *PosLow =
1353 CGF.Builder.CreateAnd(CGF.Builder.CreateTrunc(BitPos, CGF.Int8Ty),
1354 llvm::ConstantInt::get(CGF.Int8Ty, 0x7));
1355
1356 // The updating instructions will need a mask.
1357 Value *Mask = nullptr;
1358 if (BT.Action != BitTest::TestOnly) {
1359 Mask = CGF.Builder.CreateShl(llvm::ConstantInt::get(CGF.Int8Ty, 1), PosLow,
1360 "bittest.mask");
1361 }
1362
1363 // Check the action and ordering of the interlocked intrinsics.
1364 llvm::AtomicOrdering Ordering = getBitTestAtomicOrdering(BT.Interlocking);
1365
1366 Value *OldByte = nullptr;
1367 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1368 // Emit a combined atomicrmw load/store operation for the interlocked
1369 // intrinsics.
1370 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1371 if (BT.Action == BitTest::Reset) {
1372 Mask = CGF.Builder.CreateNot(Mask);
1373 RMWOp = llvm::AtomicRMWInst::And;
1374 }
1375 OldByte = CGF.Builder.CreateAtomicRMW(RMWOp, ByteAddr, Mask, Ordering);
1376 } else {
1377 // Emit a plain load for the non-interlocked intrinsics.
1378 OldByte = CGF.Builder.CreateLoad(ByteAddr, "bittest.byte");
1379 Value *NewByte = nullptr;
1380 switch (BT.Action) {
1381 case BitTest::TestOnly:
1382 // Don't store anything.
1383 break;
1384 case BitTest::Complement:
1385 NewByte = CGF.Builder.CreateXor(OldByte, Mask);
1386 break;
1387 case BitTest::Reset:
1388 NewByte = CGF.Builder.CreateAnd(OldByte, CGF.Builder.CreateNot(Mask));
1389 break;
1390 case BitTest::Set:
1391 NewByte = CGF.Builder.CreateOr(OldByte, Mask);
1392 break;
1393 }
1394 if (NewByte)
1395 CGF.Builder.CreateStore(NewByte, ByteAddr);
1396 }
1397
1398 // However we loaded the old byte, either by plain load or atomicrmw, shift
1399 // the bit into the low position and mask it to 0 or 1.
1400 Value *ShiftedByte = CGF.Builder.CreateLShr(OldByte, PosLow, "bittest.shr");
1401 return CGF.Builder.CreateAnd(
1402 ShiftedByte, llvm::ConstantInt::get(CGF.Int8Ty, 1), "bittest.res");
1403}
1404
1406 unsigned BuiltinID,
1407 const CallExpr *E) {
1408 Value *Addr = CGF.EmitScalarExpr(E->getArg(0));
1409
1411 raw_svector_ostream AsmOS(Asm);
1412 llvm::IntegerType *RetType = CGF.Int32Ty;
1413
1414 switch (BuiltinID) {
1415 case clang::PPC::BI__builtin_ppc_ldarx:
1416 AsmOS << "ldarx ";
1417 RetType = CGF.Int64Ty;
1418 break;
1419 case clang::PPC::BI__builtin_ppc_lwarx:
1420 AsmOS << "lwarx ";
1421 RetType = CGF.Int32Ty;
1422 break;
1423 case clang::PPC::BI__builtin_ppc_lharx:
1424 AsmOS << "lharx ";
1425 RetType = CGF.Int16Ty;
1426 break;
1427 case clang::PPC::BI__builtin_ppc_lbarx:
1428 AsmOS << "lbarx ";
1429 RetType = CGF.Int8Ty;
1430 break;
1431 default:
1432 llvm_unreachable("Expected only PowerPC load reserve intrinsics");
1433 }
1434
1435 AsmOS << "$0, ${1:y}";
1436
1437 std::string Constraints = "=r,*Z,~{memory}";
1438 std::string_view MachineClobbers = CGF.getTarget().getClobbers();
1439 if (!MachineClobbers.empty()) {
1440 Constraints += ',';
1441 Constraints += MachineClobbers;
1442 }
1443
1444 llvm::Type *PtrType = CGF.UnqualPtrTy;
1445 llvm::FunctionType *FTy = llvm::FunctionType::get(RetType, {PtrType}, false);
1446
1447 llvm::InlineAsm *IA =
1448 llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);
1449 llvm::CallInst *CI = CGF.Builder.CreateCall(IA, {Addr});
1450 CI->addParamAttr(
1451 0, Attribute::get(CGF.getLLVMContext(), Attribute::ElementType, RetType));
1452 return CI;
1453}
1454
1455namespace {
1456enum class MSVCSetJmpKind {
1457 _setjmpex,
1458 _setjmp3,
1459 _setjmp
1460};
1461}
1462
1463/// MSVC handles setjmp a bit differently on different platforms. On every
1464/// architecture except 32-bit x86, the frame address is passed. On x86, extra
1465/// parameters can be passed as variadic arguments, but we always pass none.
1466static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind,
1467 const CallExpr *E) {
1468 llvm::Value *Arg1 = nullptr;
1469 llvm::Type *Arg1Ty = nullptr;
1470 StringRef Name;
1471 bool IsVarArg = false;
1472 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1473 Name = "_setjmp3";
1474 Arg1Ty = CGF.Int32Ty;
1475 Arg1 = llvm::ConstantInt::get(CGF.IntTy, 0);
1476 IsVarArg = true;
1477 } else {
1478 Name = SJKind == MSVCSetJmpKind::_setjmp ? "_setjmp" : "_setjmpex";
1479 Arg1Ty = CGF.Int8PtrTy;
1480 if (CGF.getTarget().getTriple().getArch() == llvm::Triple::aarch64) {
1481 Arg1 = CGF.Builder.CreateCall(
1482 CGF.CGM.getIntrinsic(Intrinsic::sponentry, CGF.AllocaInt8PtrTy));
1483 } else
1484 Arg1 = CGF.Builder.CreateCall(
1485 CGF.CGM.getIntrinsic(Intrinsic::frameaddress, CGF.AllocaInt8PtrTy),
1486 llvm::ConstantInt::get(CGF.Int32Ty, 0));
1487 }
1488
1489 // Mark the call site and declaration with ReturnsTwice.
1490 llvm::Type *ArgTypes[2] = {CGF.Int8PtrTy, Arg1Ty};
1491 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1492 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex,
1493 llvm::Attribute::ReturnsTwice);
1494 llvm::FunctionCallee SetJmpFn = CGF.CGM.CreateRuntimeFunction(
1495 llvm::FunctionType::get(CGF.IntTy, ArgTypes, IsVarArg), Name,
1496 ReturnsTwiceAttr, /*Local=*/true);
1497
1498 llvm::Value *Buf = CGF.Builder.CreateBitOrPointerCast(
1499 CGF.EmitScalarExpr(E->getArg(0)), CGF.Int8PtrTy);
1500 llvm::Value *Args[] = {Buf, Arg1};
1501 llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(SetJmpFn, Args);
1502 CB->setAttributes(ReturnsTwiceAttr);
1503 return RValue::get(CB);
1504}
1505
1506// Many of MSVC builtins are on x64, ARM and AArch64; to avoid repeating code,
1507// we handle them here.
1547 __fastfail,
1548};
1549
1550static std::optional<CodeGenFunction::MSVCIntrin>
1551translateArmToMsvcIntrin(unsigned BuiltinID) {
1552 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
1553 switch (BuiltinID) {
1554 default:
1555 return std::nullopt;
1556 case clang::ARM::BI_BitScanForward:
1557 case clang::ARM::BI_BitScanForward64:
1558 return MSVCIntrin::_BitScanForward;
1559 case clang::ARM::BI_BitScanReverse:
1560 case clang::ARM::BI_BitScanReverse64:
1561 return MSVCIntrin::_BitScanReverse;
1562 case clang::ARM::BI_InterlockedAnd64:
1563 return MSVCIntrin::_InterlockedAnd;
1564 case clang::ARM::BI_InterlockedExchange64:
1565 return MSVCIntrin::_InterlockedExchange;
1566 case clang::ARM::BI_InterlockedExchangeAdd64:
1567 return MSVCIntrin::_InterlockedExchangeAdd;
1568 case clang::ARM::BI_InterlockedExchangeSub64:
1569 return MSVCIntrin::_InterlockedExchangeSub;
1570 case clang::ARM::BI_InterlockedOr64:
1571 return MSVCIntrin::_InterlockedOr;
1572 case clang::ARM::BI_InterlockedXor64:
1573 return MSVCIntrin::_InterlockedXor;
1574 case clang::ARM::BI_InterlockedDecrement64:
1575 return MSVCIntrin::_InterlockedDecrement;
1576 case clang::ARM::BI_InterlockedIncrement64:
1577 return MSVCIntrin::_InterlockedIncrement;
1578 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1579 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1580 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1581 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1582 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1583 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1584 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1585 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1586 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1587 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1588 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1589 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1590 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1591 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1592 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1593 case clang::ARM::BI_InterlockedExchange8_acq:
1594 case clang::ARM::BI_InterlockedExchange16_acq:
1595 case clang::ARM::BI_InterlockedExchange_acq:
1596 case clang::ARM::BI_InterlockedExchange64_acq:
1597 return MSVCIntrin::_InterlockedExchange_acq;
1598 case clang::ARM::BI_InterlockedExchange8_rel:
1599 case clang::ARM::BI_InterlockedExchange16_rel:
1600 case clang::ARM::BI_InterlockedExchange_rel:
1601 case clang::ARM::BI_InterlockedExchange64_rel:
1602 return MSVCIntrin::_InterlockedExchange_rel;
1603 case clang::ARM::BI_InterlockedExchange8_nf:
1604 case clang::ARM::BI_InterlockedExchange16_nf:
1605 case clang::ARM::BI_InterlockedExchange_nf:
1606 case clang::ARM::BI_InterlockedExchange64_nf:
1607 return MSVCIntrin::_InterlockedExchange_nf;
1608 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1609 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1610 case clang::ARM::BI_InterlockedCompareExchange_acq:
1611 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1612 return MSVCIntrin::_InterlockedCompareExchange_acq;
1613 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1614 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1615 case clang::ARM::BI_InterlockedCompareExchange_rel:
1616 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1617 return MSVCIntrin::_InterlockedCompareExchange_rel;
1618 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1619 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1620 case clang::ARM::BI_InterlockedCompareExchange_nf:
1621 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1622 return MSVCIntrin::_InterlockedCompareExchange_nf;
1623 case clang::ARM::BI_InterlockedOr8_acq:
1624 case clang::ARM::BI_InterlockedOr16_acq:
1625 case clang::ARM::BI_InterlockedOr_acq:
1626 case clang::ARM::BI_InterlockedOr64_acq:
1627 return MSVCIntrin::_InterlockedOr_acq;
1628 case clang::ARM::BI_InterlockedOr8_rel:
1629 case clang::ARM::BI_InterlockedOr16_rel:
1630 case clang::ARM::BI_InterlockedOr_rel:
1631 case clang::ARM::BI_InterlockedOr64_rel:
1632 return MSVCIntrin::_InterlockedOr_rel;
1633 case clang::ARM::BI_InterlockedOr8_nf:
1634 case clang::ARM::BI_InterlockedOr16_nf:
1635 case clang::ARM::BI_InterlockedOr_nf:
1636 case clang::ARM::BI_InterlockedOr64_nf:
1637 return MSVCIntrin::_InterlockedOr_nf;
1638 case clang::ARM::BI_InterlockedXor8_acq:
1639 case clang::ARM::BI_InterlockedXor16_acq:
1640 case clang::ARM::BI_InterlockedXor_acq:
1641 case clang::ARM::BI_InterlockedXor64_acq:
1642 return MSVCIntrin::_InterlockedXor_acq;
1643 case clang::ARM::BI_InterlockedXor8_rel:
1644 case clang::ARM::BI_InterlockedXor16_rel:
1645 case clang::ARM::BI_InterlockedXor_rel:
1646 case clang::ARM::BI_InterlockedXor64_rel:
1647 return MSVCIntrin::_InterlockedXor_rel;
1648 case clang::ARM::BI_InterlockedXor8_nf:
1649 case clang::ARM::BI_InterlockedXor16_nf:
1650 case clang::ARM::BI_InterlockedXor_nf:
1651 case clang::ARM::BI_InterlockedXor64_nf:
1652 return MSVCIntrin::_InterlockedXor_nf;
1653 case clang::ARM::BI_InterlockedAnd8_acq:
1654 case clang::ARM::BI_InterlockedAnd16_acq:
1655 case clang::ARM::BI_InterlockedAnd_acq:
1656 case clang::ARM::BI_InterlockedAnd64_acq:
1657 return MSVCIntrin::_InterlockedAnd_acq;
1658 case clang::ARM::BI_InterlockedAnd8_rel:
1659 case clang::ARM::BI_InterlockedAnd16_rel:
1660 case clang::ARM::BI_InterlockedAnd_rel:
1661 case clang::ARM::BI_InterlockedAnd64_rel:
1662 return MSVCIntrin::_InterlockedAnd_rel;
1663 case clang::ARM::BI_InterlockedAnd8_nf:
1664 case clang::ARM::BI_InterlockedAnd16_nf:
1665 case clang::ARM::BI_InterlockedAnd_nf:
1666 case clang::ARM::BI_InterlockedAnd64_nf:
1667 return MSVCIntrin::_InterlockedAnd_nf;
1668 case clang::ARM::BI_InterlockedIncrement16_acq:
1669 case clang::ARM::BI_InterlockedIncrement_acq:
1670 case clang::ARM::BI_InterlockedIncrement64_acq:
1671 return MSVCIntrin::_InterlockedIncrement_acq;
1672 case clang::ARM::BI_InterlockedIncrement16_rel:
1673 case clang::ARM::BI_InterlockedIncrement_rel:
1674 case clang::ARM::BI_InterlockedIncrement64_rel:
1675 return MSVCIntrin::_InterlockedIncrement_rel;
1676 case clang::ARM::BI_InterlockedIncrement16_nf:
1677 case clang::ARM::BI_InterlockedIncrement_nf:
1678 case clang::ARM::BI_InterlockedIncrement64_nf:
1679 return MSVCIntrin::_InterlockedIncrement_nf;
1680 case clang::ARM::BI_InterlockedDecrement16_acq:
1681 case clang::ARM::BI_InterlockedDecrement_acq:
1682 case clang::ARM::BI_InterlockedDecrement64_acq:
1683 return MSVCIntrin::_InterlockedDecrement_acq;
1684 case clang::ARM::BI_InterlockedDecrement16_rel:
1685 case clang::ARM::BI_InterlockedDecrement_rel:
1686 case clang::ARM::BI_InterlockedDecrement64_rel:
1687 return MSVCIntrin::_InterlockedDecrement_rel;
1688 case clang::ARM::BI_InterlockedDecrement16_nf:
1689 case clang::ARM::BI_InterlockedDecrement_nf:
1690 case clang::ARM::BI_InterlockedDecrement64_nf:
1691 return MSVCIntrin::_InterlockedDecrement_nf;
1692 }
1693 llvm_unreachable("must return from switch");
1694}
1695
1696static std::optional<CodeGenFunction::MSVCIntrin>
1697translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
1698 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
1699 switch (BuiltinID) {
1700 default:
1701 return std::nullopt;
1702 case clang::AArch64::BI_BitScanForward:
1703 case clang::AArch64::BI_BitScanForward64:
1704 return MSVCIntrin::_BitScanForward;
1705 case clang::AArch64::BI_BitScanReverse:
1706 case clang::AArch64::BI_BitScanReverse64:
1707 return MSVCIntrin::_BitScanReverse;
1708 case clang::AArch64::BI_InterlockedAnd64:
1709 return MSVCIntrin::_InterlockedAnd;
1710 case clang::AArch64::BI_InterlockedExchange64:
1711 return MSVCIntrin::_InterlockedExchange;
1712 case clang::AArch64::BI_InterlockedExchangeAdd64:
1713 return MSVCIntrin::_InterlockedExchangeAdd;
1714 case clang::AArch64::BI_InterlockedExchangeSub64:
1715 return MSVCIntrin::_InterlockedExchangeSub;
1716 case clang::AArch64::BI_InterlockedOr64:
1717 return MSVCIntrin::_InterlockedOr;
1718 case clang::AArch64::BI_InterlockedXor64:
1719 return MSVCIntrin::_InterlockedXor;
1720 case clang::AArch64::BI_InterlockedDecrement64:
1721 return MSVCIntrin::_InterlockedDecrement;
1722 case clang::AArch64::BI_InterlockedIncrement64:
1723 return MSVCIntrin::_InterlockedIncrement;
1724 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1725 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1726 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1727 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1728 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1729 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1730 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1731 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1732 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1733 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1734 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1735 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1736 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1737 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1738 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1739 case clang::AArch64::BI_InterlockedExchange8_acq:
1740 case clang::AArch64::BI_InterlockedExchange16_acq:
1741 case clang::AArch64::BI_InterlockedExchange_acq:
1742 case clang::AArch64::BI_InterlockedExchange64_acq:
1743 return MSVCIntrin::_InterlockedExchange_acq;
1744 case clang::AArch64::BI_InterlockedExchange8_rel:
1745 case clang::AArch64::BI_InterlockedExchange16_rel:
1746 case clang::AArch64::BI_InterlockedExchange_rel:
1747 case clang::AArch64::BI_InterlockedExchange64_rel:
1748 return MSVCIntrin::_InterlockedExchange_rel;
1749 case clang::AArch64::BI_InterlockedExchange8_nf:
1750 case clang::AArch64::BI_InterlockedExchange16_nf:
1751 case clang::AArch64::BI_InterlockedExchange_nf:
1752 case clang::AArch64::BI_InterlockedExchange64_nf:
1753 return MSVCIntrin::_InterlockedExchange_nf;
1754 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1755 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1756 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1757 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1758 return MSVCIntrin::_InterlockedCompareExchange_acq;
1759 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1760 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1761 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1762 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1763 return MSVCIntrin::_InterlockedCompareExchange_rel;
1764 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1765 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1766 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1767 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1768 return MSVCIntrin::_InterlockedCompareExchange_nf;
1769 case clang::AArch64::BI_InterlockedCompareExchange128:
1770 return MSVCIntrin::_InterlockedCompareExchange128;
1771 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1772 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1773 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1774 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1775 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1776 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1777 case clang::AArch64::BI_InterlockedOr8_acq:
1778 case clang::AArch64::BI_InterlockedOr16_acq:
1779 case clang::AArch64::BI_InterlockedOr_acq:
1780 case clang::AArch64::BI_InterlockedOr64_acq:
1781 return MSVCIntrin::_InterlockedOr_acq;
1782 case clang::AArch64::BI_InterlockedOr8_rel:
1783 case clang::AArch64::BI_InterlockedOr16_rel:
1784 case clang::AArch64::BI_InterlockedOr_rel:
1785 case clang::AArch64::BI_InterlockedOr64_rel:
1786 return MSVCIntrin::_InterlockedOr_rel;
1787 case clang::AArch64::BI_InterlockedOr8_nf:
1788 case clang::AArch64::BI_InterlockedOr16_nf:
1789 case clang::AArch64::BI_InterlockedOr_nf:
1790 case clang::AArch64::BI_InterlockedOr64_nf:
1791 return MSVCIntrin::_InterlockedOr_nf;
1792 case clang::AArch64::BI_InterlockedXor8_acq:
1793 case clang::AArch64::BI_InterlockedXor16_acq:
1794 case clang::AArch64::BI_InterlockedXor_acq:
1795 case clang::AArch64::BI_InterlockedXor64_acq:
1796 return MSVCIntrin::_InterlockedXor_acq;
1797 case clang::AArch64::BI_InterlockedXor8_rel:
1798 case clang::AArch64::BI_InterlockedXor16_rel:
1799 case clang::AArch64::BI_InterlockedXor_rel:
1800 case clang::AArch64::BI_InterlockedXor64_rel:
1801 return MSVCIntrin::_InterlockedXor_rel;
1802 case clang::AArch64::BI_InterlockedXor8_nf:
1803 case clang::AArch64::BI_InterlockedXor16_nf:
1804 case clang::AArch64::BI_InterlockedXor_nf:
1805 case clang::AArch64::BI_InterlockedXor64_nf:
1806 return MSVCIntrin::_InterlockedXor_nf;
1807 case clang::AArch64::BI_InterlockedAnd8_acq:
1808 case clang::AArch64::BI_InterlockedAnd16_acq:
1809 case clang::AArch64::BI_InterlockedAnd_acq:
1810 case clang::AArch64::BI_InterlockedAnd64_acq:
1811 return MSVCIntrin::_InterlockedAnd_acq;
1812 case clang::AArch64::BI_InterlockedAnd8_rel:
1813 case clang::AArch64::BI_InterlockedAnd16_rel:
1814 case clang::AArch64::BI_InterlockedAnd_rel:
1815 case clang::AArch64::BI_InterlockedAnd64_rel:
1816 return MSVCIntrin::_InterlockedAnd_rel;
1817 case clang::AArch64::BI_InterlockedAnd8_nf:
1818 case clang::AArch64::BI_InterlockedAnd16_nf:
1819 case clang::AArch64::BI_InterlockedAnd_nf:
1820 case clang::AArch64::BI_InterlockedAnd64_nf:
1821 return MSVCIntrin::_InterlockedAnd_nf;
1822 case clang::AArch64::BI_InterlockedIncrement16_acq:
1823 case clang::AArch64::BI_InterlockedIncrement_acq:
1824 case clang::AArch64::BI_InterlockedIncrement64_acq:
1825 return MSVCIntrin::_InterlockedIncrement_acq;
1826 case clang::AArch64::BI_InterlockedIncrement16_rel:
1827 case clang::AArch64::BI_InterlockedIncrement_rel:
1828 case clang::AArch64::BI_InterlockedIncrement64_rel:
1829 return MSVCIntrin::_InterlockedIncrement_rel;
1830 case clang::AArch64::BI_InterlockedIncrement16_nf:
1831 case clang::AArch64::BI_InterlockedIncrement_nf:
1832 case clang::AArch64::BI_InterlockedIncrement64_nf:
1833 return MSVCIntrin::_InterlockedIncrement_nf;
1834 case clang::AArch64::BI_InterlockedDecrement16_acq:
1835 case clang::AArch64::BI_InterlockedDecrement_acq:
1836 case clang::AArch64::BI_InterlockedDecrement64_acq:
1837 return MSVCIntrin::_InterlockedDecrement_acq;
1838 case clang::AArch64::BI_InterlockedDecrement16_rel:
1839 case clang::AArch64::BI_InterlockedDecrement_rel:
1840 case clang::AArch64::BI_InterlockedDecrement64_rel:
1841 return MSVCIntrin::_InterlockedDecrement_rel;
1842 case clang::AArch64::BI_InterlockedDecrement16_nf:
1843 case clang::AArch64::BI_InterlockedDecrement_nf:
1844 case clang::AArch64::BI_InterlockedDecrement64_nf:
1845 return MSVCIntrin::_InterlockedDecrement_nf;
1846 }
1847 llvm_unreachable("must return from switch");
1848}
1849
1850static std::optional<CodeGenFunction::MSVCIntrin>
1851translateX86ToMsvcIntrin(unsigned BuiltinID) {
1852 using MSVCIntrin = CodeGenFunction::MSVCIntrin;
1853 switch (BuiltinID) {
1854 default:
1855 return std::nullopt;
1856 case clang::X86::BI_BitScanForward:
1857 case clang::X86::BI_BitScanForward64:
1858 return MSVCIntrin::_BitScanForward;
1859 case clang::X86::BI_BitScanReverse:
1860 case clang::X86::BI_BitScanReverse64:
1861 return MSVCIntrin::_BitScanReverse;
1862 case clang::X86::BI_InterlockedAnd64:
1863 return MSVCIntrin::_InterlockedAnd;
1864 case clang::X86::BI_InterlockedCompareExchange128:
1865 return MSVCIntrin::_InterlockedCompareExchange128;
1866 case clang::X86::BI_InterlockedExchange64:
1867 return MSVCIntrin::_InterlockedExchange;
1868 case clang::X86::BI_InterlockedExchangeAdd64:
1869 return MSVCIntrin::_InterlockedExchangeAdd;
1870 case clang::X86::BI_InterlockedExchangeSub64:
1871 return MSVCIntrin::_InterlockedExchangeSub;
1872 case clang::X86::BI_InterlockedOr64:
1873 return MSVCIntrin::_InterlockedOr;
1874 case clang::X86::BI_InterlockedXor64:
1875 return MSVCIntrin::_InterlockedXor;
1876 case clang::X86::BI_InterlockedDecrement64:
1877 return MSVCIntrin::_InterlockedDecrement;
1878 case clang::X86::BI_InterlockedIncrement64:
1879 return MSVCIntrin::_InterlockedIncrement;
1880 }
1881 llvm_unreachable("must return from switch");
1882}
1883
1884// Emit an MSVC intrinsic. Assumes that arguments have *not* been evaluated.
1885Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
1886 const CallExpr *E) {
1887 switch (BuiltinID) {
1888 case MSVCIntrin::_BitScanForward:
1889 case MSVCIntrin::_BitScanReverse: {
1890 Address IndexAddress(EmitPointerWithAlignment(E->getArg(0)));
1891 Value *ArgValue = EmitScalarExpr(E->getArg(1));
1892
1893 llvm::Type *ArgType = ArgValue->getType();
1894 llvm::Type *IndexType = IndexAddress.getElementType();
1895 llvm::Type *ResultType = ConvertType(E->getType());
1896
1897 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1898 Value *ResZero = llvm::Constant::getNullValue(ResultType);
1899 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
1900
1901 BasicBlock *Begin = Builder.GetInsertBlock();
1902 BasicBlock *End = createBasicBlock("bitscan_end", this->CurFn);
1903 Builder.SetInsertPoint(End);
1904 PHINode *Result = Builder.CreatePHI(ResultType, 2, "bitscan_result");
1905
1906 Builder.SetInsertPoint(Begin);
1907 Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
1908 BasicBlock *NotZero = createBasicBlock("bitscan_not_zero", this->CurFn);
1909 Builder.CreateCondBr(IsZero, End, NotZero);
1910 Result->addIncoming(ResZero, Begin);
1911
1912 Builder.SetInsertPoint(NotZero);
1913
1914 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1915 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1916 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1917 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
1918 Builder.CreateStore(ZeroCount, IndexAddress, false);
1919 } else {
1920 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
1921 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
1922
1923 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
1924 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1925 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
1926 Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
1927 Builder.CreateStore(Index, IndexAddress, false);
1928 }
1929 Builder.CreateBr(End);
1930 Result->addIncoming(ResOne, NotZero);
1931
1932 Builder.SetInsertPoint(End);
1933 return Result;
1934 }
1935 case MSVCIntrin::_InterlockedAnd:
1936 return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E);
1937 case MSVCIntrin::_InterlockedExchange:
1938 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E);
1939 case MSVCIntrin::_InterlockedExchangeAdd:
1940 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E);
1941 case MSVCIntrin::_InterlockedExchangeSub:
1942 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Sub, E);
1943 case MSVCIntrin::_InterlockedOr:
1944 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E);
1945 case MSVCIntrin::_InterlockedXor:
1946 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E);
1947 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1948 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
1949 AtomicOrdering::Acquire);
1950 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1951 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
1952 AtomicOrdering::Release);
1953 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1954 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
1955 AtomicOrdering::Monotonic);
1956 case MSVCIntrin::_InterlockedExchange_acq:
1957 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
1958 AtomicOrdering::Acquire);
1959 case MSVCIntrin::_InterlockedExchange_rel:
1960 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
1961 AtomicOrdering::Release);
1962 case MSVCIntrin::_InterlockedExchange_nf:
1963 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
1964 AtomicOrdering::Monotonic);
1965 case MSVCIntrin::_InterlockedCompareExchange_acq:
1966 return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Acquire);
1967 case MSVCIntrin::_InterlockedCompareExchange_rel:
1968 return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Release);
1969 case MSVCIntrin::_InterlockedCompareExchange_nf:
1970 return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Monotonic);
1971 case MSVCIntrin::_InterlockedCompareExchange128:
1973 *this, E, AtomicOrdering::SequentiallyConsistent);
1974 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1975 return EmitAtomicCmpXchg128ForMSIntrin(*this, E, AtomicOrdering::Acquire);
1976 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1977 return EmitAtomicCmpXchg128ForMSIntrin(*this, E, AtomicOrdering::Release);
1978 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1979 return EmitAtomicCmpXchg128ForMSIntrin(*this, E, AtomicOrdering::Monotonic);
1980 case MSVCIntrin::_InterlockedOr_acq:
1981 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
1982 AtomicOrdering::Acquire);
1983 case MSVCIntrin::_InterlockedOr_rel:
1984 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
1985 AtomicOrdering::Release);
1986 case MSVCIntrin::_InterlockedOr_nf:
1987 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
1988 AtomicOrdering::Monotonic);
1989 case MSVCIntrin::_InterlockedXor_acq:
1990 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
1991 AtomicOrdering::Acquire);
1992 case MSVCIntrin::_InterlockedXor_rel:
1993 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
1994 AtomicOrdering::Release);
1995 case MSVCIntrin::_InterlockedXor_nf:
1996 return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
1997 AtomicOrdering::Monotonic);
1998 case MSVCIntrin::_InterlockedAnd_acq:
1999 return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
2000 AtomicOrdering::Acquire);
2001 case MSVCIntrin::_InterlockedAnd_rel:
2002 return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
2003 AtomicOrdering::Release);
2004 case MSVCIntrin::_InterlockedAnd_nf:
2005 return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
2006 AtomicOrdering::Monotonic);
2007 case MSVCIntrin::_InterlockedIncrement_acq:
2008 return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Acquire);
2009 case MSVCIntrin::_InterlockedIncrement_rel:
2010 return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Release);
2011 case MSVCIntrin::_InterlockedIncrement_nf:
2012 return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Monotonic);
2013 case MSVCIntrin::_InterlockedDecrement_acq:
2014 return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Acquire);
2015 case MSVCIntrin::_InterlockedDecrement_rel:
2016 return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Release);
2017 case MSVCIntrin::_InterlockedDecrement_nf:
2018 return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Monotonic);
2019
2020 case MSVCIntrin::_InterlockedDecrement:
2021 return EmitAtomicDecrementValue(*this, E);
2022 case MSVCIntrin::_InterlockedIncrement:
2023 return EmitAtomicIncrementValue(*this, E);
2024
2025 case MSVCIntrin::__fastfail: {
2026 // Request immediate process termination from the kernel. The instruction
2027 // sequences to do this are documented on MSDN:
2028 // https://msdn.microsoft.com/en-us/library/dn774154.aspx
2029 llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
2030 StringRef Asm, Constraints;
2031 switch (ISA) {
2032 default:
2033 ErrorUnsupported(E, "__fastfail call for this architecture");
2034 break;
2035 case llvm::Triple::x86:
2036 case llvm::Triple::x86_64:
2037 Asm = "int $$0x29";
2038 Constraints = "{cx}";
2039 break;
2040 case llvm::Triple::thumb:
2041 Asm = "udf #251";
2042 Constraints = "{r0}";
2043 break;
2044 case llvm::Triple::aarch64:
2045 Asm = "brk #0xF003";
2046 Constraints = "{w0}";
2047 }
2048 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, {Int32Ty}, false);
2049 llvm::InlineAsm *IA =
2050 llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);
2051 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
2052 getLLVMContext(), llvm::AttributeList::FunctionIndex,
2053 llvm::Attribute::NoReturn);
2054 llvm::CallInst *CI = Builder.CreateCall(IA, EmitScalarExpr(E->getArg(0)));
2055 CI->setAttributes(NoReturnAttr);
2056 return CI;
2057 }
2058 }
2059 llvm_unreachable("Incorrect MSVC intrinsic!");
2060}
2061
2062namespace {
2063// ARC cleanup for __builtin_os_log_format
2064struct CallObjCArcUse final : EHScopeStack::Cleanup {
2065 CallObjCArcUse(llvm::Value *object) : object(object) {}
2066 llvm::Value *object;
2067
2068 void Emit(CodeGenFunction &CGF, Flags flags) override {
2069 CGF.EmitARCIntrinsicUse(object);
2070 }
2071};
2072}
2073
2075 BuiltinCheckKind Kind) {
2076 assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero)
2077 && "Unsupported builtin check kind");
2078
2079 Value *ArgValue = EmitScalarExpr(E);
2080 if (!SanOpts.has(SanitizerKind::Builtin))
2081 return ArgValue;
2082
2083 SanitizerScope SanScope(this);
2084 Value *Cond = Builder.CreateICmpNE(
2085 ArgValue, llvm::Constant::getNullValue(ArgValue->getType()));
2086 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
2087 SanitizerHandler::InvalidBuiltin,
2089 llvm::ConstantInt::get(Builder.getInt8Ty(), Kind)},
2090 std::nullopt);
2091 return ArgValue;
2092}
2093
2094static Value *EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW) {
2095 return CGF.Builder.CreateBinaryIntrinsic(
2096 Intrinsic::abs, ArgValue,
2097 ConstantInt::get(CGF.Builder.getInt1Ty(), HasNSW));
2098}
2099
2101 bool SanitizeOverflow) {
2102 Value *ArgValue = CGF.EmitScalarExpr(E->getArg(0));
2103
2104 // Try to eliminate overflow check.
2105 if (const auto *VCI = dyn_cast<llvm::ConstantInt>(ArgValue)) {
2106 if (!VCI->isMinSignedValue())
2107 return EmitAbs(CGF, ArgValue, true);
2108 }
2109
2110 CodeGenFunction::SanitizerScope SanScope(&CGF);
2111
2112 Constant *Zero = Constant::getNullValue(ArgValue->getType());
2113 Value *ResultAndOverflow = CGF.Builder.CreateBinaryIntrinsic(
2114 Intrinsic::ssub_with_overflow, Zero, ArgValue);
2115 Value *Result = CGF.Builder.CreateExtractValue(ResultAndOverflow, 0);
2116 Value *NotOverflow = CGF.Builder.CreateNot(
2117 CGF.Builder.CreateExtractValue(ResultAndOverflow, 1));
2118
2119 // TODO: support -ftrapv-handler.
2120 if (SanitizeOverflow) {
2121 CGF.EmitCheck({{NotOverflow, SanitizerKind::SignedIntegerOverflow}},
2122 SanitizerHandler::NegateOverflow,
2125 {ArgValue});
2126 } else
2127 CGF.EmitTrapCheck(NotOverflow, SanitizerHandler::SubOverflow);
2128
2129 Value *CmpResult = CGF.Builder.CreateICmpSLT(ArgValue, Zero, "abscond");
2130 return CGF.Builder.CreateSelect(CmpResult, Result, ArgValue, "abs");
2131}
2132
2133/// Get the argument type for arguments to os_log_helper.
2135 QualType UnsignedTy = C.getIntTypeForBitwidth(Size * 8, /*Signed=*/false);
2136 return C.getCanonicalType(UnsignedTy);
2137}
2138
2141 CharUnits BufferAlignment) {
2142 ASTContext &Ctx = getContext();
2143
2145 {
2146 raw_svector_ostream OS(Name);
2147 OS << "__os_log_helper";
2148 OS << "_" << BufferAlignment.getQuantity();
2149 OS << "_" << int(Layout.getSummaryByte());
2150 OS << "_" << int(Layout.getNumArgsByte());
2151 for (const auto &Item : Layout.Items)
2152 OS << "_" << int(Item.getSizeByte()) << "_"
2153 << int(Item.getDescriptorByte());
2154 }
2155
2156 if (llvm::Function *F = CGM.getModule().getFunction(Name))
2157 return F;
2158
2160 FunctionArgList Args;
2161 Args.push_back(ImplicitParamDecl::Create(
2162 Ctx, nullptr, SourceLocation(), &Ctx.Idents.get("buffer"), Ctx.VoidPtrTy,
2164 ArgTys.emplace_back(Ctx.VoidPtrTy);
2165
2166 for (unsigned int I = 0, E = Layout.Items.size(); I < E; ++I) {
2167 char Size = Layout.Items[I].getSizeByte();
2168 if (!Size)
2169 continue;
2170
2171 QualType ArgTy = getOSLogArgType(Ctx, Size);
2172 Args.push_back(ImplicitParamDecl::Create(
2173 Ctx, nullptr, SourceLocation(),
2174 &Ctx.Idents.get(std::string("arg") + llvm::to_string(I)), ArgTy,
2176 ArgTys.emplace_back(ArgTy);
2177 }
2178
2179 QualType ReturnTy = Ctx.VoidTy;
2180
2181 // The helper function has linkonce_odr linkage to enable the linker to merge
2182 // identical functions. To ensure the merging always happens, 'noinline' is
2183 // attached to the function when compiling with -Oz.
2184 const CGFunctionInfo &FI =
2186 llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
2187 llvm::Function *Fn = llvm::Function::Create(
2188 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &CGM.getModule());
2189 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2190 CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, Fn, /*IsThunk=*/false);
2192 Fn->setDoesNotThrow();
2193
2194 // Attach 'noinline' at -Oz.
2195 if (CGM.getCodeGenOpts().OptimizeSize == 2)
2196 Fn->addFnAttr(llvm::Attribute::NoInline);
2197
2198 auto NL = ApplyDebugLocation::CreateEmpty(*this);
2199 StartFunction(GlobalDecl(), ReturnTy, Fn, FI, Args);
2200
2201 // Create a scope with an artificial location for the body of this function.
2202 auto AL = ApplyDebugLocation::CreateArtificial(*this);
2203
2204 CharUnits Offset;
2206 Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Ctx.VoidTy,
2207 BufferAlignment);
2208 Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()),
2209 Builder.CreateConstByteGEP(BufAddr, Offset++, "summary"));
2210 Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()),
2211 Builder.CreateConstByteGEP(BufAddr, Offset++, "numArgs"));
2212
2213 unsigned I = 1;
2214 for (const auto &Item : Layout.Items) {
2216 Builder.getInt8(Item.getDescriptorByte()),
2217 Builder.CreateConstByteGEP(BufAddr, Offset++, "argDescriptor"));
2219 Builder.getInt8(Item.getSizeByte()),
2220 Builder.CreateConstByteGEP(BufAddr, Offset++, "argSize"));
2221
2222 CharUnits Size = Item.size();
2223 if (!Size.getQuantity())
2224 continue;
2225
2226 Address Arg = GetAddrOfLocalVar(Args[I]);
2227 Address Addr = Builder.CreateConstByteGEP(BufAddr, Offset, "argData");
2228 Addr = Addr.withElementType(Arg.getElementType());
2230 Offset += Size;
2231 ++I;
2232 }
2233
2235
2236 return Fn;
2237}
2238
2240 assert(E.getNumArgs() >= 2 &&
2241 "__builtin_os_log_format takes at least 2 arguments");
2242 ASTContext &Ctx = getContext();
2245 Address BufAddr = EmitPointerWithAlignment(E.getArg(0));
2246 llvm::SmallVector<llvm::Value *, 4> RetainableOperands;
2247
2248 // Ignore argument 1, the format string. It is not currently used.
2249 CallArgList Args;
2250 Args.add(RValue::get(BufAddr.emitRawPointer(*this)), Ctx.VoidPtrTy);
2251
2252 for (const auto &Item : Layout.Items) {
2253 int Size = Item.getSizeByte();
2254 if (!Size)
2255 continue;
2256
2257 llvm::Value *ArgVal;
2258
2259 if (Item.getKind() == analyze_os_log::OSLogBufferItem::MaskKind) {
2260 uint64_t Val = 0;
2261 for (unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
2262 Val |= ((uint64_t)Item.getMaskType()[I]) << I * 8;
2263 ArgVal = llvm::Constant::getIntegerValue(Int64Ty, llvm::APInt(64, Val));
2264 } else if (const Expr *TheExpr = Item.getExpr()) {
2265 ArgVal = EmitScalarExpr(TheExpr, /*Ignore*/ false);
2266
2267 // If a temporary object that requires destruction after the full
2268 // expression is passed, push a lifetime-extended cleanup to extend its
2269 // lifetime to the end of the enclosing block scope.
2270 auto LifetimeExtendObject = [&](const Expr *E) {
2271 E = E->IgnoreParenCasts();
2272 // Extend lifetimes of objects returned by function calls and message
2273 // sends.
2274
2275 // FIXME: We should do this in other cases in which temporaries are
2276 // created including arguments of non-ARC types (e.g., C++
2277 // temporaries).
2278 if (isa<CallExpr>(E) || isa<ObjCMessageExpr>(E))
2279 return true;
2280 return false;
2281 };
2282
2283 if (TheExpr->getType()->isObjCRetainableType() &&
2284 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2285 assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
2286 "Only scalar can be a ObjC retainable type");
2287 if (!isa<Constant>(ArgVal)) {
2288 CleanupKind Cleanup = getARCCleanupKind();
2289 QualType Ty = TheExpr->getType();
2291 RawAddress Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca);
2292 ArgVal = EmitARCRetain(Ty, ArgVal);
2293 Builder.CreateStore(ArgVal, Addr);
2294 pushLifetimeExtendedDestroy(Cleanup, Alloca, Ty,
2296 Cleanup & EHCleanup);
2297
2298 // Push a clang.arc.use call to ensure ARC optimizer knows that the
2299 // argument has to be alive.
2300 if (CGM.getCodeGenOpts().OptimizationLevel != 0)
2301 pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
2302 }
2303 }
2304 } else {
2305 ArgVal = Builder.getInt32(Item.getConstValue().getQuantity());
2306 }
2307
2308 unsigned ArgValSize =
2309 CGM.getDataLayout().getTypeSizeInBits(ArgVal->getType());
2310 llvm::IntegerType *IntTy = llvm::Type::getIntNTy(getLLVMContext(),
2311 ArgValSize);
2312 ArgVal = Builder.CreateBitOrPointerCast(ArgVal, IntTy);
2313 CanQualType ArgTy = getOSLogArgType(Ctx, Size);
2314 // If ArgVal has type x86_fp80, zero-extend ArgVal.
2315 ArgVal = Builder.CreateZExtOrBitCast(ArgVal, ConvertType(ArgTy));
2316 Args.add(RValue::get(ArgVal), ArgTy);
2317 }
2318
2319 const CGFunctionInfo &FI =
2322 Layout, BufAddr.getAlignment());
2324 return RValue::get(BufAddr, *this);
2325}
2326
2328 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2329 WidthAndSignedness ResultInfo) {
2330 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2331 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2332 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2333}
2334
2336 CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info,
2337 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2338 const clang::Expr *ResultArg, QualType ResultQTy,
2339 WidthAndSignedness ResultInfo) {
2341 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2342 "Cannot specialize this multiply");
2343
2344 llvm::Value *V1 = CGF.EmitScalarExpr(Op1);
2345 llvm::Value *V2 = CGF.EmitScalarExpr(Op2);
2346
2347 llvm::Value *HasOverflow;
2348 llvm::Value *Result = EmitOverflowIntrinsic(
2349 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2350
2351 // The intrinsic call will detect overflow when the value is > UINT_MAX,
2352 // however, since the original builtin had a signed result, we need to report
2353 // an overflow when the result is greater than INT_MAX.
2354 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
2355 llvm::Value *IntMaxValue = llvm::ConstantInt::get(Result->getType(), IntMax);
2356
2357 llvm::Value *IntMaxOverflow = CGF.Builder.CreateICmpUGT(Result, IntMaxValue);
2358 HasOverflow = CGF.Builder.CreateOr(HasOverflow, IntMaxOverflow);
2359
2360 bool isVolatile =
2361 ResultArg->getType()->getPointeeType().isVolatileQualified();
2362 Address ResultPtr = CGF.EmitPointerWithAlignment(ResultArg);
2363 CGF.Builder.CreateStore(CGF.EmitToMemory(Result, ResultQTy), ResultPtr,
2364 isVolatile);
2365 return RValue::get(HasOverflow);
2366}
2367
2368/// Determine if a binop is a checked mixed-sign multiply we can specialize.
2369static bool isSpecialMixedSignMultiply(unsigned BuiltinID,
2370 WidthAndSignedness Op1Info,
2371 WidthAndSignedness Op2Info,
2372 WidthAndSignedness ResultInfo) {
2373 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2374 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2375 Op1Info.Signed != Op2Info.Signed;
2376}
2377
2378/// Emit a checked mixed-sign multiply. This is a cheaper specialization of
2379/// the generic checked-binop irgen.
2380static RValue
2382 WidthAndSignedness Op1Info, const clang::Expr *Op2,
2383 WidthAndSignedness Op2Info,
2384 const clang::Expr *ResultArg, QualType ResultQTy,
2385 WidthAndSignedness ResultInfo) {
2386 assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info,
2387 Op2Info, ResultInfo) &&
2388 "Not a mixed-sign multipliction we can specialize");
2389
2390 // Emit the signed and unsigned operands.
2391 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2392 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2393 llvm::Value *Signed = CGF.EmitScalarExpr(SignedOp);
2394 llvm::Value *Unsigned = CGF.EmitScalarExpr(UnsignedOp);
2395 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2396 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2397
2398 // One of the operands may be smaller than the other. If so, [s|z]ext it.
2399 if (SignedOpWidth < UnsignedOpWidth)
2400 Signed = CGF.Builder.CreateSExt(Signed, Unsigned->getType(), "op.sext");
2401 if (UnsignedOpWidth < SignedOpWidth)
2402 Unsigned = CGF.Builder.CreateZExt(Unsigned, Signed->getType(), "op.zext");
2403
2404 llvm::Type *OpTy = Signed->getType();
2405 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2406 Address ResultPtr = CGF.EmitPointerWithAlignment(ResultArg);
2407 llvm::Type *ResTy = ResultPtr.getElementType();
2408 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
2409
2410 // Take the absolute value of the signed operand.
2411 llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(Signed, Zero);
2412 llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(Zero, Signed);
2413 llvm::Value *AbsSigned =
2414 CGF.Builder.CreateSelect(IsNegative, AbsOfNegative, Signed);
2415
2416 // Perform a checked unsigned multiplication.
2417 llvm::Value *UnsignedOverflow;
2418 llvm::Value *UnsignedResult =
2419 EmitOverflowIntrinsic(CGF, llvm::Intrinsic::umul_with_overflow, AbsSigned,
2420 Unsigned, UnsignedOverflow);
2421
2422 llvm::Value *Overflow, *Result;
2423 if (ResultInfo.Signed) {
2424 // Signed overflow occurs if the result is greater than INT_MAX or lesser
2425 // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
2426 auto IntMax =
2427 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2428 llvm::Value *MaxResult =
2429 CGF.Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2430 CGF.Builder.CreateZExt(IsNegative, OpTy));
2431 llvm::Value *SignedOverflow =
2432 CGF.Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2433 Overflow = CGF.Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2434
2435 // Prepare the signed result (possibly by negating it).
2436 llvm::Value *NegativeResult = CGF.Builder.CreateNeg(UnsignedResult);
2437 llvm::Value *SignedResult =
2438 CGF.Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2439 Result = CGF.Builder.CreateTrunc(SignedResult, ResTy);
2440 } else {
2441 // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX.
2442 llvm::Value *Underflow = CGF.Builder.CreateAnd(
2443 IsNegative, CGF.Builder.CreateIsNotNull(UnsignedResult));
2444 Overflow = CGF.Builder.CreateOr(UnsignedOverflow, Underflow);
2445 if (ResultInfo.Width < OpWidth) {
2446 auto IntMax =
2447 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2448 llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
2449 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2450 Overflow = CGF.Builder.CreateOr(Overflow, TruncOverflow);
2451 }
2452
2453 // Negate the product if it would be negative in infinite precision.
2454 Result = CGF.Builder.CreateSelect(
2455 IsNegative, CGF.Builder.CreateNeg(UnsignedResult), UnsignedResult);
2456
2457 Result = CGF.Builder.CreateTrunc(Result, ResTy);
2458 }
2459 assert(Overflow && Result && "Missing overflow or result");
2460
2461 bool isVolatile =
2462 ResultArg->getType()->getPointeeType().isVolatileQualified();
2463 CGF.Builder.CreateStore(CGF.EmitToMemory(Result, ResultQTy), ResultPtr,
2464 isVolatile);
2465 return RValue::get(Overflow);
2466}
2467
2468static bool
2470 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2471 if (const auto *Arr = Ctx.getAsArrayType(Ty))
2472 Ty = Ctx.getBaseElementType(Arr);
2473
2474 const auto *Record = Ty->getAsCXXRecordDecl();
2475 if (!Record)
2476 return false;
2477
2478 // We've already checked this type, or are in the process of checking it.
2479 if (!Seen.insert(Record).second)
2480 return false;
2481
2482 assert(Record->hasDefinition() &&
2483 "Incomplete types should already be diagnosed");
2484
2485 if (Record->isDynamicClass())
2486 return true;
2487
2488 for (FieldDecl *F : Record->fields()) {
2489 if (TypeRequiresBuiltinLaunderImp(Ctx, F->getType(), Seen))
2490 return true;
2491 }
2492 return false;
2493}
2494
2495/// Determine if the specified type requires laundering by checking if it is a
2496/// dynamic class type or contains a subobject which is a dynamic class type.
2498 if (!CGM.getCodeGenOpts().StrictVTablePointers)
2499 return false;
2501 return TypeRequiresBuiltinLaunderImp(CGM.getContext(), Ty, Seen);
2502}
2503
2504RValue CodeGenFunction::emitRotate(const CallExpr *E, bool IsRotateRight) {
2505 llvm::Value *Src = EmitScalarExpr(E->getArg(0));
2506 llvm::Value *ShiftAmt = EmitScalarExpr(E->getArg(1));
2507
2508 // The builtin's shift arg may have a different type than the source arg and
2509 // result, but the LLVM intrinsic uses the same type for all values.
2510 llvm::Type *Ty = Src->getType();
2511 ShiftAmt = Builder.CreateIntCast(ShiftAmt, Ty, false);
2512
2513 // Rotate is a special case of LLVM funnel shift - 1st 2 args are the same.
2514 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2515 Function *F = CGM.getIntrinsic(IID, Ty);
2516 return RValue::get(Builder.CreateCall(F, { Src, Src, ShiftAmt }));
2517}
2518
2519// Map math builtins for long-double to f128 version.
2520static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID) {
2521 switch (BuiltinID) {
2522#define MUTATE_LDBL(func) \
2523 case Builtin::BI__builtin_##func##l: \
2524 return Builtin::BI__builtin_##func##f128;
2555 MUTATE_LDBL(nans)
2556 MUTATE_LDBL(inf)
2575 MUTATE_LDBL(huge_val)
2585#undef MUTATE_LDBL
2586 default:
2587 return BuiltinID;
2588 }
2589}
2590
2591static Value *tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID,
2592 Value *V) {
2593 if (CGF.Builder.getIsFPConstrained() &&
2594 CGF.Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2595 if (Value *Result =
2596 CGF.getTargetHooks().testFPKind(V, BuiltinID, CGF.Builder, CGF.CGM))
2597 return Result;
2598 }
2599 return nullptr;
2600}
2601
2603 const FunctionDecl *FD) {
2604 auto Name = FD->getNameAsString() + "__hipstdpar_unsupported";
2605 auto FnTy = CGF->CGM.getTypes().GetFunctionType(FD);
2606 auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, FnTy);
2607
2609 for (auto &&FormalTy : FnTy->params())
2610 Args.push_back(llvm::PoisonValue::get(FormalTy));
2611
2612 return RValue::get(CGF->Builder.CreateCall(UBF, Args));
2613}
2614
2615RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
2616 const CallExpr *E,
2617 ReturnValueSlot ReturnValue) {
2618 const FunctionDecl *FD = GD.getDecl()->getAsFunction();
2619 // See if we can constant fold this builtin. If so, don't emit it at all.
2620 // TODO: Extend this handling to all builtin calls that we can constant-fold.
2622 if (E->isPRValue() && E->EvaluateAsRValue(Result, CGM.getContext()) &&
2623 !Result.hasSideEffects()) {
2624 if (Result.Val.isInt())
2625 return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
2626 Result.Val.getInt()));
2627 if (Result.Val.isFloat())
2628 return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
2629 Result.Val.getFloat()));
2630 }
2631
2632 // If current long-double semantics is IEEE 128-bit, replace math builtins
2633 // of long-double with f128 equivalent.
2634 // TODO: This mutation should also be applied to other targets other than PPC,
2635 // after backend supports IEEE 128-bit style libcalls.
2636 if (getTarget().getTriple().isPPC64() &&
2637 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2638 BuiltinID = mutateLongDoubleBuiltin(BuiltinID);
2639
2640 // If the builtin has been declared explicitly with an assembler label,
2641 // disable the specialized emitting below. Ideally we should communicate the
2642 // rename in IR, or at least avoid generating the intrinsic calls that are
2643 // likely to get lowered to the renamed library functions.
2644 const unsigned BuiltinIDIfNoAsmLabel =
2645 FD->hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2646
2647 std::optional<bool> ErrnoOverriden;
2648 // ErrnoOverriden is true if math-errno is overriden via the
2649 // '#pragma float_control(precise, on)'. This pragma disables fast-math,
2650 // which implies math-errno.
2651 if (E->hasStoredFPFeatures()) {
2653 if (OP.hasMathErrnoOverride())
2654 ErrnoOverriden = OP.getMathErrnoOverride();
2655 }
2656 // True if 'atttibute__((optnone)) is used. This attibute overrides
2657 // fast-math which implies math-errno.
2658 bool OptNone = CurFuncDecl && CurFuncDecl->hasAttr<OptimizeNoneAttr>();
2659
2660 // True if we are compiling at -O2 and errno has been disabled
2661 // using the '#pragma float_control(precise, off)', and
2662 // attribute opt-none hasn't been seen.
2663 bool ErrnoOverridenToFalseWithOpt =
2664 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2665 CGM.getCodeGenOpts().OptimizationLevel != 0;
2666
2667 // There are LLVM math intrinsics/instructions corresponding to math library
2668 // functions except the LLVM op will never set errno while the math library
2669 // might. Also, math builtins have the same semantics as their math library
2670 // twins. Thus, we can transform math library and builtin calls to their
2671 // LLVM counterparts if the call is marked 'const' (known to never set errno).
2672 // In case FP exceptions are enabled, the experimental versions of the
2673 // intrinsics model those.
2674 bool ConstAlways =
2675 getContext().BuiltinInfo.isConst(BuiltinID);
2676
2677 // There's a special case with the fma builtins where they are always const
2678 // if the target environment is GNU or the target is OS is Windows and we're
2679 // targeting the MSVCRT.dll environment.
2680 // FIXME: This list can be become outdated. Need to find a way to get it some
2681 // other way.
2682 switch (BuiltinID) {
2683 case Builtin::BI__builtin_fma:
2684 case Builtin::BI__builtin_fmaf:
2685 case Builtin::BI__builtin_fmal:
2686 case Builtin::BIfma:
2687 case Builtin::BIfmaf:
2688 case Builtin::BIfmal: {
2689 auto &Trip = CGM.getTriple();
2690 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2691 ConstAlways = true;
2692 break;
2693 }
2694 default:
2695 break;
2696 }
2697
2698 bool ConstWithoutErrnoAndExceptions =
2700 bool ConstWithoutExceptions =
2702
2703 // ConstAttr is enabled in fast-math mode. In fast-math mode, math-errno is
2704 // disabled.
2705 // Math intrinsics are generated only when math-errno is disabled. Any pragmas
2706 // or attributes that affect math-errno should prevent or allow math
2707 // intrincs to be generated. Intrinsics are generated:
2708 // 1- In fast math mode, unless math-errno is overriden
2709 // via '#pragma float_control(precise, on)', or via an
2710 // 'attribute__((optnone))'.
2711 // 2- If math-errno was enabled on command line but overriden
2712 // to false via '#pragma float_control(precise, off))' and
2713 // 'attribute__((optnone))' hasn't been used.
2714 // 3- If we are compiling with optimization and errno has been disabled
2715 // via '#pragma float_control(precise, off)', and
2716 // 'attribute__((optnone))' hasn't been used.
2717
2718 bool ConstWithoutErrnoOrExceptions =
2719 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2720 bool GenerateIntrinsics =
2721 (ConstAlways && !OptNone) ||
2722 (!getLangOpts().MathErrno &&
2723 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2724 if (!GenerateIntrinsics) {
2725 GenerateIntrinsics =
2726 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2727 if (!GenerateIntrinsics)
2728 GenerateIntrinsics =
2729 ConstWithoutErrnoOrExceptions &&
2730 (!getLangOpts().MathErrno &&
2731 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2732 if (!GenerateIntrinsics)
2733 GenerateIntrinsics =
2734 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2735 }
2736 if (GenerateIntrinsics) {
2737 switch (BuiltinIDIfNoAsmLabel) {
2738 case Builtin::BIceil:
2739 case Builtin::BIceilf:
2740 case Builtin::BIceill:
2741 case Builtin::BI__builtin_ceil:
2742 case Builtin::BI__builtin_ceilf:
2743 case Builtin::BI__builtin_ceilf16:
2744 case Builtin::BI__builtin_ceill:
2745 case Builtin::BI__builtin_ceilf128:
2747 Intrinsic::ceil,
2748 Intrinsic::experimental_constrained_ceil));
2749
2750 case Builtin::BIcopysign:
2751 case Builtin::BIcopysignf:
2752 case Builtin::BIcopysignl:
2753 case Builtin::BI__builtin_copysign:
2754 case Builtin::BI__builtin_copysignf:
2755 case Builtin::BI__builtin_copysignf16:
2756 case Builtin::BI__builtin_copysignl:
2757 case Builtin::BI__builtin_copysignf128:
2758 return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::copysign));
2759
2760 case Builtin::BIcos:
2761 case Builtin::BIcosf:
2762 case Builtin::BIcosl:
2763 case Builtin::BI__builtin_cos:
2764 case Builtin::BI__builtin_cosf:
2765 case Builtin::BI__builtin_cosf16:
2766 case Builtin::BI__builtin_cosl:
2767 case Builtin::BI__builtin_cosf128:
2769 Intrinsic::cos,
2770 Intrinsic::experimental_constrained_cos));
2771
2772 case Builtin::BIexp:
2773 case Builtin::BIexpf:
2774 case Builtin::BIexpl:
2775 case Builtin::BI__builtin_exp:
2776 case Builtin::BI__builtin_expf:
2777 case Builtin::BI__builtin_expf16:
2778 case Builtin::BI__builtin_expl:
2779 case Builtin::BI__builtin_expf128:
2781 Intrinsic::exp,
2782 Intrinsic::experimental_constrained_exp));
2783
2784 case Builtin::BIexp2:
2785 case Builtin::BIexp2f:
2786 case Builtin::BIexp2l:
2787 case Builtin::BI__builtin_exp2:
2788 case Builtin::BI__builtin_exp2f:
2789 case Builtin::BI__builtin_exp2f16:
2790 case Builtin::BI__builtin_exp2l:
2791 case Builtin::BI__builtin_exp2f128:
2793 Intrinsic::exp2,
2794 Intrinsic::experimental_constrained_exp2));
2795 case Builtin::BI__builtin_exp10:
2796 case Builtin::BI__builtin_exp10f:
2797 case Builtin::BI__builtin_exp10f16:
2798 case Builtin::BI__builtin_exp10l:
2799 case Builtin::BI__builtin_exp10f128: {
2800 // TODO: strictfp support
2801 if (Builder.getIsFPConstrained())
2802 break;
2803 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::exp10));
2804 }
2805 case Builtin::BIfabs:
2806 case Builtin::BIfabsf:
2807 case Builtin::BIfabsl:
2808 case Builtin::BI__builtin_fabs:
2809 case Builtin::BI__builtin_fabsf:
2810 case Builtin::BI__builtin_fabsf16:
2811 case Builtin::BI__builtin_fabsl:
2812 case Builtin::BI__builtin_fabsf128:
2813 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::fabs));
2814
2815 case Builtin::BIfloor:
2816 case Builtin::BIfloorf:
2817 case Builtin::BIfloorl:
2818 case Builtin::BI__builtin_floor:
2819 case Builtin::BI__builtin_floorf:
2820 case Builtin::BI__builtin_floorf16:
2821 case Builtin::BI__builtin_floorl:
2822 case Builtin::BI__builtin_floorf128:
2824 Intrinsic::floor,
2825 Intrinsic::experimental_constrained_floor));
2826
2827 case Builtin::BIfma:
2828 case Builtin::BIfmaf:
2829 case Builtin::BIfmal:
2830 case Builtin::BI__builtin_fma:
2831 case Builtin::BI__builtin_fmaf:
2832 case Builtin::BI__builtin_fmaf16:
2833 case Builtin::BI__builtin_fmal:
2834 case Builtin::BI__builtin_fmaf128:
2836 Intrinsic::fma,
2837 Intrinsic::experimental_constrained_fma));
2838
2839 case Builtin::BIfmax:
2840 case Builtin::BIfmaxf:
2841 case Builtin::BIfmaxl:
2842 case Builtin::BI__builtin_fmax:
2843 case Builtin::BI__builtin_fmaxf:
2844 case Builtin::BI__builtin_fmaxf16:
2845 case Builtin::BI__builtin_fmaxl:
2846 case Builtin::BI__builtin_fmaxf128:
2848 Intrinsic::maxnum,
2849 Intrinsic::experimental_constrained_maxnum));
2850
2851 case Builtin::BIfmin:
2852 case Builtin::BIfminf:
2853 case Builtin::BIfminl:
2854 case Builtin::BI__builtin_fmin:
2855 case Builtin::BI__builtin_fminf:
2856 case Builtin::BI__builtin_fminf16:
2857 case Builtin::BI__builtin_fminl:
2858 case Builtin::BI__builtin_fminf128:
2860 Intrinsic::minnum,
2861 Intrinsic::experimental_constrained_minnum));
2862
2863 // fmod() is a special-case. It maps to the frem instruction rather than an
2864 // LLVM intrinsic.
2865 case Builtin::BIfmod:
2866 case Builtin::BIfmodf:
2867 case Builtin::BIfmodl:
2868 case Builtin::BI__builtin_fmod:
2869 case Builtin::BI__builtin_fmodf:
2870 case Builtin::BI__builtin_fmodf16:
2871 case Builtin::BI__builtin_fmodl:
2872 case Builtin::BI__builtin_fmodf128: {
2873 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
2874 Value *Arg1 = EmitScalarExpr(E->getArg(0));
2875 Value *Arg2 = EmitScalarExpr(E->getArg(1));
2876 return RValue::get(Builder.CreateFRem(Arg1, Arg2, "fmod"));
2877 }
2878
2879 case Builtin::BIlog:
2880 case Builtin::BIlogf:
2881 case Builtin::BIlogl:
2882 case Builtin::BI__builtin_log:
2883 case Builtin::BI__builtin_logf:
2884 case Builtin::BI__builtin_logf16:
2885 case Builtin::BI__builtin_logl:
2886 case Builtin::BI__builtin_logf128:
2888 Intrinsic::log,
2889 Intrinsic::experimental_constrained_log));
2890
2891 case Builtin::BIlog10:
2892 case Builtin::BIlog10f:
2893 case Builtin::BIlog10l:
2894 case Builtin::BI__builtin_log10:
2895 case Builtin::BI__builtin_log10f:
2896 case Builtin::BI__builtin_log10f16:
2897 case Builtin::BI__builtin_log10l:
2898 case Builtin::BI__builtin_log10f128:
2900 Intrinsic::log10,
2901 Intrinsic::experimental_constrained_log10));
2902
2903 case Builtin::BIlog2:
2904 case Builtin::BIlog2f:
2905 case Builtin::BIlog2l:
2906 case Builtin::BI__builtin_log2:
2907 case Builtin::BI__builtin_log2f:
2908 case Builtin::BI__builtin_log2f16:
2909 case Builtin::BI__builtin_log2l:
2910 case Builtin::BI__builtin_log2f128:
2912 Intrinsic::log2,
2913 Intrinsic::experimental_constrained_log2));
2914
2915 case Builtin::BInearbyint:
2916 case Builtin::BInearbyintf:
2917 case Builtin::BInearbyintl:
2918 case Builtin::BI__builtin_nearbyint:
2919 case Builtin::BI__builtin_nearbyintf:
2920 case Builtin::BI__builtin_nearbyintl:
2921 case Builtin::BI__builtin_nearbyintf128:
2923 Intrinsic::nearbyint,
2924 Intrinsic::experimental_constrained_nearbyint));
2925
2926 case Builtin::BIpow:
2927 case Builtin::BIpowf:
2928 case Builtin::BIpowl:
2929 case Builtin::BI__builtin_pow:
2930 case Builtin::BI__builtin_powf:
2931 case Builtin::BI__builtin_powf16:
2932 case Builtin::BI__builtin_powl:
2933 case Builtin::BI__builtin_powf128:
2935 Intrinsic::pow,
2936 Intrinsic::experimental_constrained_pow));
2937
2938 case Builtin::BIrint:
2939 case Builtin::BIrintf:
2940 case Builtin::BIrintl:
2941 case Builtin::BI__builtin_rint:
2942 case Builtin::BI__builtin_rintf:
2943 case Builtin::BI__builtin_rintf16:
2944 case Builtin::BI__builtin_rintl:
2945 case Builtin::BI__builtin_rintf128:
2947 Intrinsic::rint,
2948 Intrinsic::experimental_constrained_rint));
2949
2950 case Builtin::BIround:
2951 case Builtin::BIroundf:
2952 case Builtin::BIroundl:
2953 case Builtin::BI__builtin_round:
2954 case Builtin::BI__builtin_roundf:
2955 case Builtin::BI__builtin_roundf16:
2956 case Builtin::BI__builtin_roundl:
2957 case Builtin::BI__builtin_roundf128:
2959 Intrinsic::round,
2960 Intrinsic::experimental_constrained_round));
2961
2962 case Builtin::BIroundeven:
2963 case Builtin::BIroundevenf:
2964 case Builtin::BIroundevenl:
2965 case Builtin::BI__builtin_roundeven:
2966 case Builtin::BI__builtin_roundevenf:
2967 case Builtin::BI__builtin_roundevenf16:
2968 case Builtin::BI__builtin_roundevenl:
2969 case Builtin::BI__builtin_roundevenf128:
2971 Intrinsic::roundeven,
2972 Intrinsic::experimental_constrained_roundeven));
2973
2974 case Builtin::BIsin:
2975 case Builtin::BIsinf:
2976 case Builtin::BIsinl:
2977 case Builtin::BI__builtin_sin:
2978 case Builtin::BI__builtin_sinf:
2979 case Builtin::BI__builtin_sinf16:
2980 case Builtin::BI__builtin_sinl:
2981 case Builtin::BI__builtin_sinf128:
2983 Intrinsic::sin,
2984 Intrinsic::experimental_constrained_sin));
2985
2986 case Builtin::BIsqrt:
2987 case Builtin::BIsqrtf:
2988 case Builtin::BIsqrtl:
2989 case Builtin::BI__builtin_sqrt:
2990 case Builtin::BI__builtin_sqrtf:
2991 case Builtin::BI__builtin_sqrtf16:
2992 case Builtin::BI__builtin_sqrtl:
2993 case Builtin::BI__builtin_sqrtf128:
2994 case Builtin::BI__builtin_elementwise_sqrt: {
2996 *this, E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
2998 return RValue::get(Call);
2999 }
3000 case Builtin::BItrunc:
3001 case Builtin::BItruncf:
3002 case Builtin::BItruncl:
3003 case Builtin::BI__builtin_trunc:
3004 case Builtin::BI__builtin_truncf:
3005 case Builtin::BI__builtin_truncf16:
3006 case Builtin::BI__builtin_truncl:
3007 case Builtin::BI__builtin_truncf128:
3009 Intrinsic::trunc,
3010 Intrinsic::experimental_constrained_trunc));
3011
3012 case Builtin::BIlround:
3013 case Builtin::BIlroundf:
3014 case Builtin::BIlroundl:
3015 case Builtin::BI__builtin_lround:
3016 case Builtin::BI__builtin_lroundf:
3017 case Builtin::BI__builtin_lroundl:
3018 case Builtin::BI__builtin_lroundf128:
3020 *this, E, Intrinsic::lround,
3021 Intrinsic::experimental_constrained_lround));
3022
3023 case Builtin::BIllround:
3024 case Builtin::BIllroundf:
3025 case Builtin::BIllroundl:
3026 case Builtin::BI__builtin_llround:
3027 case Builtin::BI__builtin_llroundf:
3028 case Builtin::BI__builtin_llroundl:
3029 case Builtin::BI__builtin_llroundf128:
3031 *this, E, Intrinsic::llround,
3032 Intrinsic::experimental_constrained_llround));
3033
3034 case Builtin::BIlrint:
3035 case Builtin::BIlrintf:
3036 case Builtin::BIlrintl:
3037 case Builtin::BI__builtin_lrint:
3038 case Builtin::BI__builtin_lrintf:
3039 case Builtin::BI__builtin_lrintl:
3040 case Builtin::BI__builtin_lrintf128:
3042 *this, E, Intrinsic::lrint,
3043 Intrinsic::experimental_constrained_lrint));
3044
3045 case Builtin::BIllrint:
3046 case Builtin::BIllrintf:
3047 case Builtin::BIllrintl:
3048 case Builtin::BI__builtin_llrint:
3049 case Builtin::BI__builtin_llrintf:
3050 case Builtin::BI__builtin_llrintl:
3051 case Builtin::BI__builtin_llrintf128:
3053 *this, E, Intrinsic::llrint,
3054 Intrinsic::experimental_constrained_llrint));
3055 case Builtin::BI__builtin_ldexp:
3056 case Builtin::BI__builtin_ldexpf:
3057 case Builtin::BI__builtin_ldexpl:
3058 case Builtin::BI__builtin_ldexpf16:
3059 case Builtin::BI__builtin_ldexpf128: {
3061 *this, E, Intrinsic::ldexp,
3062 Intrinsic::experimental_constrained_ldexp));
3063 }
3064 default:
3065 break;
3066 }
3067 }
3068
3069 // Check NonnullAttribute/NullabilityArg and Alignment.
3070 auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg,
3071 unsigned ParmNum) {
3072 Value *Val = A.emitRawPointer(*this);
3073 EmitNonNullArgCheck(RValue::get(Val), Arg->getType(), Arg->getExprLoc(), FD,
3074 ParmNum);
3075
3076 if (SanOpts.has(SanitizerKind::Alignment)) {
3077 SanitizerSet SkippedChecks;
3078 SkippedChecks.set(SanitizerKind::All);
3079 SkippedChecks.clear(SanitizerKind::Alignment);
3080 SourceLocation Loc = Arg->getExprLoc();
3081 // Strip an implicit cast.
3082 if (auto *CE = dyn_cast<ImplicitCastExpr>(Arg))
3083 if (CE->getCastKind() == CK_BitCast)
3084 Arg = CE->getSubExpr();
3085 EmitTypeCheck(Kind, Loc, Val, Arg->getType(), A.getAlignment(),
3086 SkippedChecks);
3087 }
3088 };
3089
3090 switch (BuiltinIDIfNoAsmLabel) {
3091 default: break;
3092 case Builtin::BI__builtin___CFStringMakeConstantString:
3093 case Builtin::BI__builtin___NSStringMakeConstantString:
3094 return RValue::get(ConstantEmitter(*this).emitAbstract(E, E->getType()));
3095 case Builtin::BI__builtin_stdarg_start:
3096 case Builtin::BI__builtin_va_start:
3097 case Builtin::BI__va_start:
3098 case Builtin::BI__builtin_va_end:
3099 EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
3100 ? EmitScalarExpr(E->getArg(0))
3101 : EmitVAListRef(E->getArg(0)).emitRawPointer(*this),
3102 BuiltinID != Builtin::BI__builtin_va_end);
3103 return RValue::get(nullptr);
3104 case Builtin::BI__builtin_va_copy: {
3105 Value *DstPtr = EmitVAListRef(E->getArg(0)).emitRawPointer(*this);
3106 Value *SrcPtr = EmitVAListRef(E->getArg(1)).emitRawPointer(*this);
3107 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy, {DstPtr->getType()}),
3108 {DstPtr, SrcPtr});
3109 return RValue::get(nullptr);
3110 }
3111 case Builtin::BIabs:
3112 case Builtin::BIlabs:
3113 case Builtin::BIllabs:
3114 case Builtin::BI__builtin_abs:
3115 case Builtin::BI__builtin_labs:
3116 case Builtin::BI__builtin_llabs: {
3117 bool SanitizeOverflow = SanOpts.has(SanitizerKind::SignedIntegerOverflow);
3118
3119 Value *Result;
3120 switch (getLangOpts().getSignedOverflowBehavior()) {
3122 Result = EmitAbs(*this, EmitScalarExpr(E->getArg(0)), false);
3123 break;
3125 if (!SanitizeOverflow) {
3126 Result = EmitAbs(*this, EmitScalarExpr(E->getArg(0)), true);
3127 break;
3128 }
3129 [[fallthrough]];
3131 // TODO: Somehow handle the corner case when the address of abs is taken.
3132 Result = EmitOverflowCheckedAbs(*this, E, SanitizeOverflow);
3133 break;
3134 }
3135 return RValue::get(Result);
3136 }
3137 case Builtin::BI__builtin_complex: {
3138 Value *Real = EmitScalarExpr(E->getArg(0));
3139 Value *Imag = EmitScalarExpr(E->getArg(1));
3140 return RValue::getComplex({Real, Imag});
3141 }
3142 case Builtin::BI__builtin_conj:
3143 case Builtin::BI__builtin_conjf:
3144 case Builtin::BI__builtin_conjl:
3145 case Builtin::BIconj:
3146 case Builtin::BIconjf:
3147 case Builtin::BIconjl: {
3148 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3149 Value *Real = ComplexVal.first;
3150 Value *Imag = ComplexVal.second;
3151 Imag = Builder.CreateFNeg(Imag, "neg");
3152 return RValue::getComplex(std::make_pair(Real, Imag));
3153 }
3154 case Builtin::BI__builtin_creal:
3155 case Builtin::BI__builtin_crealf:
3156 case Builtin::BI__builtin_creall:
3157 case Builtin::BIcreal:
3158 case Builtin::BIcrealf:
3159 case Builtin::BIcreall: {
3160 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3161 return RValue::get(ComplexVal.first);
3162 }
3163
3164 case Builtin::BI__builtin_preserve_access_index: {
3165 // Only enabled preserved access index region when debuginfo
3166 // is available as debuginfo is needed to preserve user-level
3167 // access pattern.
3168 if (!getDebugInfo()) {
3169 CGM.Error(E->getExprLoc(), "using builtin_preserve_access_index() without -g");
3170 return RValue::get(EmitScalarExpr(E->getArg(0)));
3171 }
3172
3173 // Nested builtin_preserve_access_index() not supported
3175 CGM.Error(E->getExprLoc(), "nested builtin_preserve_access_index() not supported");
3176 return RValue::get(EmitScalarExpr(E->getArg(0)));
3177 }
3178
3179 IsInPreservedAIRegion = true;
3180 Value *Res = EmitScalarExpr(E->getArg(0));
3181 IsInPreservedAIRegion = false;
3182 return RValue::get(Res);
3183 }
3184
3185 case Builtin::BI__builtin_cimag:
3186 case Builtin::BI__builtin_cimagf:
3187 case Builtin::BI__builtin_cimagl:
3188 case Builtin::BIcimag:
3189 case Builtin::BIcimagf:
3190 case Builtin::BIcimagl: {
3191 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3192 return RValue::get(ComplexVal.second);
3193 }
3194
3195 case Builtin::BI__builtin_clrsb:
3196 case Builtin::BI__builtin_clrsbl:
3197 case Builtin::BI__builtin_clrsbll: {
3198 // clrsb(x) -> clz(x < 0 ? ~x : x) - 1 or
3199 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3200
3201 llvm::Type *ArgType = ArgValue->getType();
3202 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3203
3204 llvm::Type *ResultType = ConvertType(E->getType());
3205 Value *Zero = llvm::Constant::getNullValue(ArgType);
3206 Value *IsNeg = Builder.CreateICmpSLT(ArgValue, Zero, "isneg");
3207 Value *Inverse = Builder.CreateNot(ArgValue, "not");
3208 Value *Tmp = Builder.CreateSelect(IsNeg, Inverse, ArgValue);
3209 Value *Ctlz = Builder.CreateCall(F, {Tmp, Builder.getFalse()});
3210 Value *Result = Builder.CreateSub(Ctlz, llvm::ConstantInt::get(ArgType, 1));
3211 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3212 "cast");
3213 return RValue::get(Result);
3214 }
3215 case Builtin::BI__builtin_ctzs:
3216 case Builtin::BI__builtin_ctz:
3217 case Builtin::BI__builtin_ctzl:
3218 case Builtin::BI__builtin_ctzll:
3219 case Builtin::BI__builtin_ctzg: {
3220 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
3221 E->getNumArgs() > 1;
3222
3223 Value *ArgValue =
3224 HasFallback ? EmitScalarExpr(E->getArg(0))
3226
3227 llvm::Type *ArgType = ArgValue->getType();
3228 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
3229
3230 llvm::Type *ResultType = ConvertType(E->getType());
3231 Value *ZeroUndef =
3232 Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef());
3233 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
3234 if (Result->getType() != ResultType)
3235 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3236 "cast");
3237 if (!HasFallback)
3238 return RValue::get(Result);
3239
3240 Value *Zero = Constant::getNullValue(ArgType);
3241 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
3242 Value *FallbackValue = EmitScalarExpr(E->getArg(1));
3243 Value *ResultOrFallback =
3244 Builder.CreateSelect(IsZero, FallbackValue, Result, "ctzg");
3245 return RValue::get(ResultOrFallback);
3246 }
3247 case Builtin::BI__builtin_clzs:
3248 case Builtin::BI__builtin_clz:
3249 case Builtin::BI__builtin_clzl:
3250 case Builtin::BI__builtin_clzll:
3251 case Builtin::BI__builtin_clzg: {
3252 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
3253 E->getNumArgs() > 1;
3254
3255 Value *ArgValue =
3256 HasFallback ? EmitScalarExpr(E->getArg(0))
3258
3259 llvm::Type *ArgType = ArgValue->getType();
3260 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3261
3262 llvm::Type *ResultType = ConvertType(E->getType());
3263 Value *ZeroUndef =
3264 Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef());
3265 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
3266 if (Result->getType() != ResultType)
3267 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3268 "cast");
3269 if (!HasFallback)
3270 return RValue::get(Result);
3271
3272 Value *Zero = Constant::getNullValue(ArgType);
3273 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
3274 Value *FallbackValue = EmitScalarExpr(E->getArg(1));
3275 Value *ResultOrFallback =
3276 Builder.CreateSelect(IsZero, FallbackValue, Result, "clzg");
3277 return RValue::get(ResultOrFallback);
3278 }
3279 case Builtin::BI__builtin_ffs:
3280 case Builtin::BI__builtin_ffsl:
3281 case Builtin::BI__builtin_ffsll: {
3282 // ffs(x) -> x ? cttz(x) + 1 : 0
3283 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3284
3285 llvm::Type *ArgType = ArgValue->getType();
3286 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
3287
3288 llvm::Type *ResultType = ConvertType(E->getType());
3289 Value *Tmp =
3290 Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
3291 llvm::ConstantInt::get(ArgType, 1));
3292 Value *Zero = llvm::Constant::getNullValue(ArgType);
3293 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
3294 Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
3295 if (Result->getType() != ResultType)
3296 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3297 "cast");
3298 return RValue::get(Result);
3299 }
3300 case Builtin::BI__builtin_parity:
3301 case Builtin::BI__builtin_parityl:
3302 case Builtin::BI__builtin_parityll: {
3303 // parity(x) -> ctpop(x) & 1
3304 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3305
3306 llvm::Type *ArgType = ArgValue->getType();
3307 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
3308
3309 llvm::Type *ResultType = ConvertType(E->getType());
3310 Value *Tmp = Builder.CreateCall(F, ArgValue);
3311 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
3312 if (Result->getType() != ResultType)
3313 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3314 "cast");
3315 return RValue::get(Result);
3316 }
3317 case Builtin::BI__lzcnt16:
3318 case Builtin::BI__lzcnt:
3319 case Builtin::BI__lzcnt64: {
3320 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3321
3322 llvm::Type *ArgType = ArgValue->getType();
3323 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3324
3325 llvm::Type *ResultType = ConvertType(E->getType());
3326 Value *Result = Builder.CreateCall(F, {ArgValue, Builder.getFalse()});
3327 if (Result->getType() != ResultType)
3328 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3329 "cast");
3330 return RValue::get(Result);
3331 }
3332 case Builtin::BI__popcnt16:
3333 case Builtin::BI__popcnt:
3334 case Builtin::BI__popcnt64:
3335 case Builtin::BI__builtin_popcount:
3336 case Builtin::BI__builtin_popcountl:
3337 case Builtin::BI__builtin_popcountll:
3338 case Builtin::BI__builtin_popcountg: {
3339 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3340
3341 llvm::Type *ArgType = ArgValue->getType();
3342 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
3343
3344 llvm::Type *ResultType = ConvertType(E->getType());
3345 Value *Result = Builder.CreateCall(F, ArgValue);
3346 if (Result->getType() != ResultType)
3347 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3348 "cast");
3349 return RValue::get(Result);
3350 }
3351 case Builtin::BI__builtin_unpredictable: {
3352 // Always return the argument of __builtin_unpredictable. LLVM does not
3353 // handle this builtin. Metadata for this builtin should be added directly
3354 // to instructions such as branches or switches that use it.
3355 return RValue::get(EmitScalarExpr(E->getArg(0)));
3356 }
3357 case Builtin::BI__builtin_expect: {
3358 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3359 llvm::Type *ArgType = ArgValue->getType();
3360
3361 Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
3362 // Don't generate llvm.expect on -O0 as the backend won't use it for
3363 // anything.
3364 // Note, we still IRGen ExpectedValue because it could have side-effects.
3365 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3366 return RValue::get(ArgValue);
3367
3368 Function *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
3369 Value *Result =
3370 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval");
3371 return RValue::get(Result);
3372 }
3373 case Builtin::BI__builtin_expect_with_probability: {
3374 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3375 llvm::Type *ArgType = ArgValue->getType();
3376
3377 Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
3378 llvm::APFloat Probability(0.0);
3379 const Expr *ProbArg = E->getArg(2);
3380 bool EvalSucceed = ProbArg->EvaluateAsFloat(Probability, CGM.getContext());
3381 assert(EvalSucceed && "probability should be able to evaluate as float");
3382 (void)EvalSucceed;
3383 bool LoseInfo = false;
3384 Probability.convert(llvm::APFloat::IEEEdouble(),
3385 llvm::RoundingMode::Dynamic, &LoseInfo);
3386 llvm::Type *Ty = ConvertType(ProbArg->getType());
3387 Constant *Confidence = ConstantFP::get(Ty, Probability);
3388 // Don't generate llvm.expect.with.probability on -O0 as the backend
3389 // won't use it for anything.
3390 // Note, we still IRGen ExpectedValue because it could have side-effects.
3391 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3392 return RValue::get(ArgValue);
3393
3394 Function *FnExpect =
3395 CGM.getIntrinsic(Intrinsic::expect_with_probability, ArgType);
3396 Value *Result = Builder.CreateCall(
3397 FnExpect, {ArgValue, ExpectedValue, Confidence}, "expval");
3398 return RValue::get(Result);
3399 }
3400 case Builtin::BI__builtin_assume_aligned: {
3401 const Expr *Ptr = E->getArg(0);
3402 Value *PtrValue = EmitScalarExpr(Ptr);
3403 Value *OffsetValue =
3404 (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr;
3405
3406 Value *AlignmentValue = EmitScalarExpr(E->getArg(1));
3407 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
3408 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
3409 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
3410 llvm::Value::MaximumAlignment);
3411
3412 emitAlignmentAssumption(PtrValue, Ptr,
3413 /*The expr loc is sufficient.*/ SourceLocation(),
3414 AlignmentCI, OffsetValue);
3415 return RValue::get(PtrValue);
3416 }
3417 case Builtin::BI__assume:
3418 case Builtin::BI__builtin_assume: {
3419 if (E->getArg(0)->HasSideEffects(getContext()))
3420 return RValue::get(nullptr);
3421
3422 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3423 Function *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
3424 Builder.CreateCall(FnAssume, ArgValue);
3425 return RValue::get(nullptr);
3426 }
3427 case Builtin::BI__builtin_assume_separate_storage: {
3428 const Expr *Arg0 = E->getArg(0);
3429 const Expr *Arg1 = E->getArg(1);
3430
3431 Value *Value0 = EmitScalarExpr(Arg0);
3432 Value *Value1 = EmitScalarExpr(Arg1);
3433
3434 Value *Values[] = {Value0, Value1};
3435 OperandBundleDefT<Value *> OBD("separate_storage", Values);
3436 Builder.CreateAssumption(ConstantInt::getTrue(getLLVMContext()), {OBD});
3437 return RValue::get(nullptr);
3438 }
3439 case Builtin::BI__arithmetic_fence: {
3440 // Create the builtin call if FastMath is selected, and the target
3441 // supports the builtin, otherwise just return the argument.
3442 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3443 llvm::FastMathFlags FMF = Builder.getFastMathFlags();
3444 bool isArithmeticFenceEnabled =
3445 FMF.allowReassoc() &&
3447 QualType ArgType = E->getArg(0)->getType();
3448 if (ArgType->isComplexType()) {
3449 if (isArithmeticFenceEnabled) {
3450 QualType ElementType = ArgType->castAs<ComplexType>()->getElementType();
3451 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3452 Value *Real = Builder.CreateArithmeticFence(ComplexVal.first,
3453 ConvertType(ElementType));
3454 Value *Imag = Builder.CreateArithmeticFence(ComplexVal.second,
3455 ConvertType(ElementType));
3456 return RValue::getComplex(std::make_pair(Real, Imag));
3457 }
3458 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3459 Value *Real = ComplexVal.first;
3460 Value *Imag = ComplexVal.second;
3461 return RValue::getComplex(std::make_pair(Real, Imag));
3462 }
3463 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3464 if (isArithmeticFenceEnabled)
3465 return RValue::get(
3466 Builder.CreateArithmeticFence(ArgValue, ConvertType(ArgType)));
3467 return RValue::get(ArgValue);
3468 }
3469 case Builtin::BI__builtin_bswap16:
3470 case Builtin::BI__builtin_bswap32:
3471 case Builtin::BI__builtin_bswap64:
3472 case Builtin::BI_byteswap_ushort:
3473 case Builtin::BI_byteswap_ulong:
3474 case Builtin::BI_byteswap_uint64: {
3475 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bswap));
3476 }
3477 case Builtin::BI__builtin_bitreverse8:
3478 case Builtin::BI__builtin_bitreverse16:
3479 case Builtin::BI__builtin_bitreverse32:
3480 case Builtin::BI__builtin_bitreverse64: {
3481 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bitreverse));
3482 }
3483 case Builtin::BI__builtin_rotateleft8:
3484 case Builtin::BI__builtin_rotateleft16:
3485 case Builtin::BI__builtin_rotateleft32:
3486 case Builtin::BI__builtin_rotateleft64:
3487 case Builtin::BI_rotl8: // Microsoft variants of rotate left
3488 case Builtin::BI_rotl16:
3489 case Builtin::BI_rotl:
3490 case Builtin::BI_lrotl:
3491 case Builtin::BI_rotl64:
3492 return emitRotate(E, false);
3493
3494 case Builtin::BI__builtin_rotateright8:
3495 case Builtin::BI__builtin_rotateright16:
3496 case Builtin::BI__builtin_rotateright32:
3497 case Builtin::BI__builtin_rotateright64:
3498 case Builtin::BI_rotr8: // Microsoft variants of rotate right
3499 case Builtin::BI_rotr16:
3500 case Builtin::BI_rotr:
3501 case Builtin::BI_lrotr:
3502 case Builtin::BI_rotr64:
3503 return emitRotate(E, true);
3504
3505 case Builtin::BI__builtin_constant_p: {
3506 llvm::Type *ResultType = ConvertType(E->getType());
3507
3508 const Expr *Arg = E->getArg(0);
3509 QualType ArgType = Arg->getType();
3510 // FIXME: The allowance for Obj-C pointers and block pointers is historical
3511 // and likely a mistake.
3512 if (!ArgType->isIntegralOrEnumerationType() && !ArgType->isFloatingType() &&
3513 !ArgType->isObjCObjectPointerType() && !ArgType->isBlockPointerType())
3514 // Per the GCC documentation, only numeric constants are recognized after
3515 // inlining.
3516 return RValue::get(ConstantInt::get(ResultType, 0));
3517
3518 if (Arg->HasSideEffects(getContext()))
3519 // The argument is unevaluated, so be conservative if it might have
3520 // side-effects.
3521 return RValue::get(ConstantInt::get(ResultType, 0));
3522
3523 Value *ArgValue = EmitScalarExpr(Arg);
3524 if (ArgType->isObjCObjectPointerType()) {
3525 // Convert Objective-C objects to id because we cannot distinguish between
3526 // LLVM types for Obj-C classes as they are opaque.
3527 ArgType = CGM.getContext().getObjCIdType();
3528 ArgValue = Builder.CreateBitCast(ArgValue, ConvertType(ArgType));
3529 }
3530 Function *F =
3531 CGM.getIntrinsic(Intrinsic::is_constant, ConvertType(ArgType));
3532 Value *Result = Builder.CreateCall(F, ArgValue);
3533 if (Result->getType() != ResultType)
3534 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/false);
3535 return RValue::get(Result);
3536 }
3537 case Builtin::BI__builtin_dynamic_object_size:
3538 case Builtin::BI__builtin_object_size: {
3539 unsigned Type =
3540 E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue();
3541 auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType()));
3542
3543 // We pass this builtin onto the optimizer so that it can figure out the
3544 // object size in more complex cases.
3545 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3546 return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType,
3547 /*EmittedE=*/nullptr, IsDynamic));
3548 }
3549 case Builtin::BI__builtin_prefetch: {
3550 Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
3551 // FIXME: Technically these constants should of type 'int', yes?
3552 RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) :
3553 llvm::ConstantInt::get(Int32Ty, 0);
3554 Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) :
3555 llvm::ConstantInt::get(Int32Ty, 3);
3556 Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
3557 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
3558 Builder.CreateCall(F, {Address, RW, Locality, Data});
3559 return RValue::get(nullptr);
3560 }
3561 case Builtin::BI__builtin_readcyclecounter: {
3562 Function *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
3563 return RValue::get(Builder.CreateCall(F));
3564 }
3565 case Builtin::BI__builtin_readsteadycounter: {
3566 Function *F = CGM.getIntrinsic(Intrinsic::readsteadycounter);
3567 return RValue::get(Builder.CreateCall(F));
3568 }
3569 case Builtin::BI__builtin___clear_cache: {
3570 Value *Begin = EmitScalarExpr(E->getArg(0));
3571 Value *End = EmitScalarExpr(E->getArg(1));
3572 Function *F = CGM.getIntrinsic(Intrinsic::clear_cache);
3573 return RValue::get(Builder.CreateCall(F, {Begin, End}));
3574 }
3575 case Builtin::BI__builtin_trap:
3576 EmitTrapCall(Intrinsic::trap);
3577 return RValue::get(nullptr);
3578 case Builtin::BI__debugbreak:
3579 EmitTrapCall(Intrinsic::debugtrap);
3580 return RValue::get(nullptr);
3581 case Builtin::BI__builtin_unreachable: {
3583
3584 // We do need to preserve an insertion point.
3585 EmitBlock(createBasicBlock("unreachable.cont"));
3586
3587 return RValue::get(nullptr);
3588 }
3589
3590 case Builtin::BI__builtin_powi:
3591 case Builtin::BI__builtin_powif:
3592 case Builtin::BI__builtin_powil: {
3593 llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
3594 llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
3595
3596 if (Builder.getIsFPConstrained()) {
3597 // FIXME: llvm.powi has 2 mangling types,
3598 // llvm.experimental.constrained.powi has one.
3599 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3600 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_powi,
3601 Src0->getType());
3602 return RValue::get(Builder.CreateConstrainedFPCall(F, { Src0, Src1 }));
3603 }
3604
3605 Function *F = CGM.getIntrinsic(Intrinsic::powi,
3606 { Src0->getType(), Src1->getType() });
3607 return RValue::get(Builder.CreateCall(F, { Src0, Src1 }));
3608 }
3609 case Builtin::BI__builtin_frexpl: {
3610 // Linux PPC will not be adding additional PPCDoubleDouble support.
3611 // WIP to switch default to IEEE long double. Will emit libcall for
3612 // frexpl instead of legalizing this type in the BE.
3613 if (&getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
3614 break;
3615 LLVM_FALLTHROUGH;
3616 }
3617 case Builtin::BI__builtin_frexp:
3618 case Builtin::BI__builtin_frexpf:
3619 case Builtin::BI__builtin_frexpf128:
3620 case Builtin::BI__builtin_frexpf16:
3621 return RValue::get(emitFrexpBuiltin(*this, E, Intrinsic::frexp));
3622 case Builtin::BI__builtin_isgreater:
3623 case Builtin::BI__builtin_isgreaterequal:
3624 case Builtin::BI__builtin_isless:
3625 case Builtin::BI__builtin_islessequal:
3626 case Builtin::BI__builtin_islessgreater:
3627 case Builtin::BI__builtin_isunordered: {
3628 // Ordered comparisons: we know the arguments to these are matching scalar
3629 // floating point values.
3630 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3631 Value *LHS = EmitScalarExpr(E->getArg(0));
3632 Value *RHS = EmitScalarExpr(E->getArg(1));
3633
3634 switch (BuiltinID) {
3635 default: llvm_unreachable("Unknown ordered comparison");
3636 case Builtin::BI__builtin_isgreater:
3637 LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp");
3638 break;
3639 case Builtin::BI__builtin_isgreaterequal:
3640 LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp");
3641 break;
3642 case Builtin::BI__builtin_isless:
3643 LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp");
3644 break;
3645 case Builtin::BI__builtin_islessequal:
3646 LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp");
3647 break;
3648 case Builtin::BI__builtin_islessgreater:
3649 LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp");
3650 break;
3651 case Builtin::BI__builtin_isunordered:
3652 LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp");
3653 break;
3654 }
3655 // ZExt bool to int type.
3656 return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
3657 }
3658
3659 case Builtin::BI__builtin_isnan: {
3660 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3661 Value *V = EmitScalarExpr(E->getArg(0));
3662 if (Value *Result = tryUseTestFPKind(*this, BuiltinID, V))
3663 return RValue::get(Result);
3664 return RValue::get(
3665 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcNan),
3666 ConvertType(E->getType())));
3667 }
3668
3669 case Builtin::BI__builtin_issignaling: {
3670 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3671 Value *V = EmitScalarExpr(E->getArg(0));
3672 return RValue::get(
3673 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcSNan),
3674 ConvertType(E->getType())));
3675 }
3676
3677 case Builtin::BI__builtin_isinf: {
3678 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3679 Value *V = EmitScalarExpr(E->getArg(0));
3680 if (Value *Result = tryUseTestFPKind(*this, BuiltinID, V))
3681 return RValue::get(Result);
3682 return RValue::get(
3683 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcInf),
3684 ConvertType(E->getType())));
3685 }
3686
3687 case Builtin::BIfinite:
3688 case Builtin::BI__finite:
3689 case Builtin::BIfinitef:
3690 case Builtin::BI__finitef:
3691 case Builtin::BIfinitel:
3692 case Builtin::BI__finitel:
3693 case Builtin::BI__builtin_isfinite: {
3694 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3695 Value *V = EmitScalarExpr(E->getArg(0));
3696 if (Value *Result = tryUseTestFPKind(*this, BuiltinID, V))
3697 return RValue::get(Result);
3698 return RValue::get(
3699 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcFinite),
3700 ConvertType(E->getType())));
3701 }
3702
3703 case Builtin::BI__builtin_isnormal: {
3704 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3705 Value *V = EmitScalarExpr(E->getArg(0));
3706 return RValue::get(
3707 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcNormal),
3708 ConvertType(E->getType())));
3709 }
3710
3711 case Builtin::BI__builtin_issubnormal: {
3712 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3713 Value *V = EmitScalarExpr(E->getArg(0));
3714 return RValue::get(
3715 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcSubnormal),
3716 ConvertType(E->getType())));
3717 }
3718
3719 case Builtin::BI__builtin_iszero: {
3720 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3721 Value *V = EmitScalarExpr(E->getArg(0));
3722 return RValue::get(
3723 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcZero),
3724 ConvertType(E->getType())));
3725 }
3726
3727 case Builtin::BI__builtin_isfpclass: {
3729 if (!E->getArg(1)->EvaluateAsInt(Result, CGM.getContext()))
3730 break;
3731 uint64_t Test = Result.Val.getInt().getLimitedValue();
3732 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3733 Value *V = EmitScalarExpr(E->getArg(0));
3734 return RValue::get(Builder.CreateZExt(Builder.createIsFPClass(V, Test),
3735 ConvertType(E->getType())));
3736 }
3737
3738 case Builtin::BI__builtin_nondeterministic_value: {
3739 llvm::Type *Ty = ConvertType(E->getArg(0)->getType());
3740
3741 Value *Result = PoisonValue::get(Ty);
3742 Result = Builder.CreateFreeze(Result);
3743
3744 return RValue::get(Result);
3745 }
3746
3747 case Builtin::BI__builtin_elementwise_abs: {
3748 Value *Result;
3749 QualType QT = E->getArg(0)->getType();
3750
3751 if (auto *VecTy = QT->getAs<VectorType>())
3752 QT = VecTy->getElementType();
3753 if (QT->isIntegerType())
3754 Result = Builder.CreateBinaryIntrinsic(
3755 llvm::Intrinsic::abs, EmitScalarExpr(E->getArg(0)),
3756 Builder.getFalse(), nullptr, "elt.abs");
3757 else
3758 Result = emitUnaryBuiltin(*this, E, llvm::Intrinsic::fabs, "elt.abs");
3759
3760 return RValue::get(Result);
3761 }
3762
3763 case Builtin::BI__builtin_elementwise_ceil:
3764 return RValue::get(
3765 emitUnaryBuiltin(*this, E, llvm::Intrinsic::ceil, "elt.ceil"));
3766 case Builtin::BI__builtin_elementwise_exp:
3767 return RValue::get(
3768 emitUnaryBuiltin(*this, E, llvm::Intrinsic::exp, "elt.exp"));
3769 case Builtin::BI__builtin_elementwise_exp2:
3770 return RValue::get(
3771 emitUnaryBuiltin(*this, E, llvm::Intrinsic::exp2, "elt.exp2"));
3772 case Builtin::BI__builtin_elementwise_log:
3773 return RValue::get(
3774 emitUnaryBuiltin(*this, E, llvm::Intrinsic::log, "elt.log"));
3775 case Builtin::BI__builtin_elementwise_log2:
3776 return RValue::get(
3777 emitUnaryBuiltin(*this, E, llvm::Intrinsic::log2, "elt.log2"));
3778 case Builtin::BI__builtin_elementwise_log10:
3779 return RValue::get(
3780 emitUnaryBuiltin(*this, E, llvm::Intrinsic::log10, "elt.log10"));
3781 case Builtin::BI__builtin_elementwise_pow: {
3782 return RValue::get(emitBinaryBuiltin(*this, E, llvm::Intrinsic::pow));
3783 }
3784 case Builtin::BI__builtin_elementwise_bitreverse:
3785 return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::bitreverse,
3786 "elt.bitreverse"));
3787 case Builtin::BI__builtin_elementwise_cos:
3788 return RValue::get(
3789 emitUnaryBuiltin(*this, E, llvm::Intrinsic::cos, "elt.cos"));
3790 case Builtin::BI__builtin_elementwise_floor:
3791 return RValue::get(
3792 emitUnaryBuiltin(*this, E, llvm::Intrinsic::floor, "elt.floor"));
3793 case Builtin::BI__builtin_elementwise_roundeven:
3794 return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::roundeven,
3795 "elt.roundeven"));
3796 case Builtin::BI__builtin_elementwise_round:
3797 return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::round,
3798 "elt.round"));
3799 case Builtin::BI__builtin_elementwise_rint:
3800 return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::rint,
3801 "elt.rint"));
3802 case Builtin::BI__builtin_elementwise_nearbyint:
3803 return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::nearbyint,
3804 "elt.nearbyint"));
3805 case Builtin::BI__builtin_elementwise_sin:
3806 return RValue::get(
3807 emitUnaryBuiltin(*this, E, llvm::Intrinsic::sin, "elt.sin"));
3808
3809 case Builtin::BI__builtin_elementwise_trunc:
3810 return RValue::get(
3811 emitUnaryBuiltin(*this, E, llvm::Intrinsic::trunc, "elt.trunc"));
3812 case Builtin::BI__builtin_elementwise_canonicalize:
3813 return RValue::get(
3814 emitUnaryBuiltin(*this, E, llvm::Intrinsic::canonicalize, "elt.canonicalize"));
3815 case Builtin::BI__builtin_elementwise_copysign:
3816 return RValue::get(emitBinaryBuiltin(*this, E, llvm::Intrinsic::copysign));
3817 case Builtin::BI__builtin_elementwise_fma:
3818 return RValue::get(emitTernaryBuiltin(*this, E, llvm::Intrinsic::fma));
3819 case Builtin::BI__builtin_elementwise_add_sat:
3820 case Builtin::BI__builtin_elementwise_sub_sat: {
3821 Value *Op0 = EmitScalarExpr(E->getArg(0));
3822 Value *Op1 = EmitScalarExpr(E->getArg(1));
3823 Value *Result;
3824 assert(Op0->getType()->isIntOrIntVectorTy() && "integer type expected");
3825 QualType Ty = E->getArg(0)->getType();
3826 if (auto *VecTy = Ty->getAs<VectorType>())
3827 Ty = VecTy->getElementType();
3828 bool IsSigned = Ty->isSignedIntegerType();
3829 unsigned Opc;
3830 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
3831 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
3832 else
3833 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
3834 Result = Builder.CreateBinaryIntrinsic(Opc, Op0, Op1, nullptr, "elt.sat");
3835 return RValue::get(Result);
3836 }
3837
3838 case Builtin::BI__builtin_elementwise_max: {
3839 Value *Op0 = EmitScalarExpr(E->getArg(0));
3840 Value *Op1 = EmitScalarExpr(E->getArg(1));
3841 Value *Result;
3842 if (Op0->getType()->isIntOrIntVectorTy()) {
3843 QualType Ty = E->getArg(0)->getType();
3844 if (auto *VecTy = Ty->getAs<VectorType>())
3845 Ty = VecTy->getElementType();
3846 Result = Builder.CreateBinaryIntrinsic(Ty->isSignedIntegerType()
3847 ? llvm::Intrinsic::smax
3848 : llvm::Intrinsic::umax,
3849 Op0, Op1, nullptr, "elt.max");
3850 } else
3851 Result = Builder.CreateMaxNum(Op0, Op1, "elt.max");
3852 return RValue::get(Result);
3853 }
3854 case Builtin::BI__builtin_elementwise_min: {
3855 Value *Op0 = EmitScalarExpr(E->getArg(0));
3856 Value *Op1 = EmitScalarExpr(E->getArg(1));
3857 Value *Result;
3858 if (Op0->getType()->isIntOrIntVectorTy()) {
3859 QualType Ty = E->getArg(0)->getType();
3860 if (auto *VecTy = Ty->getAs<VectorType>())
3861 Ty = VecTy->getElementType();
3862 Result = Builder.CreateBinaryIntrinsic(Ty->isSignedIntegerType()
3863 ? llvm::Intrinsic::smin
3864 : llvm::Intrinsic::umin,
3865 Op0, Op1, nullptr, "elt.min");
3866 } else
3867 Result = Builder.CreateMinNum(Op0, Op1, "elt.min");
3868 return RValue::get(Result);
3869 }
3870
3871 case Builtin::BI__builtin_reduce_max: {
3872 auto GetIntrinsicID = [](QualType QT) {
3873 if (auto *VecTy = QT->getAs<VectorType>())
3874 QT = VecTy->getElementType();
3875 if (QT->isSignedIntegerType())
3876 return llvm::Intrinsic::vector_reduce_smax;
3877 if (QT->isUnsignedIntegerType())
3878 return llvm::Intrinsic::vector_reduce_umax;
3879 assert(QT->isFloatingType() && "must have a float here");
3880 return llvm::Intrinsic::vector_reduce_fmax;
3881 };
3883 *this, E, GetIntrinsicID(E->getArg(0)->getType()), "rdx.min"));
3884 }
3885
3886 case Builtin::BI__builtin_reduce_min: {
3887 auto GetIntrinsicID = [](QualType QT) {
3888 if (auto *VecTy = QT->getAs<VectorType>())
3889 QT = VecTy->getElementType();
3890 if (QT->isSignedIntegerType())
3891 return llvm::Intrinsic::vector_reduce_smin;
3892 if (QT->isUnsignedIntegerType())
3893 return llvm::Intrinsic::vector_reduce_umin;
3894 assert(QT->isFloatingType() && "must have a float here");
3895 return llvm::Intrinsic::vector_reduce_fmin;
3896 };
3897
3899 *this, E, GetIntrinsicID(E->getArg(0)->getType()), "rdx.min"));
3900 }
3901
3902 case Builtin::BI__builtin_reduce_add:
3904 *this, E, llvm::Intrinsic::vector_reduce_add, "rdx.add"));
3905 case Builtin::BI__builtin_reduce_mul:
3907 *this, E, llvm::Intrinsic::vector_reduce_mul, "rdx.mul"));
3908 case Builtin::BI__builtin_reduce_xor:
3910 *this, E, llvm::Intrinsic::vector_reduce_xor, "rdx.xor"));
3911 case Builtin::BI__builtin_reduce_or:
3913 *this, E, llvm::Intrinsic::vector_reduce_or, "rdx.or"));
3914 case Builtin::BI__builtin_reduce_and:
3916 *this, E, llvm::Intrinsic::vector_reduce_and, "rdx.and"));
3917
3918 case Builtin::BI__builtin_matrix_transpose: {
3919 auto *MatrixTy = E->getArg(0)->getType()->castAs<ConstantMatrixType>();
3920 Value *MatValue = EmitScalarExpr(E->getArg(0));
3921 MatrixBuilder MB(Builder);
3922 Value *Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
3923 MatrixTy->getNumColumns());
3924 return RValue::get(Result);
3925 }
3926
3927 case Builtin::BI__builtin_matrix_column_major_load: {
3928 MatrixBuilder MB(Builder);
3929 // Emit everything that isn't dependent on the first parameter type
3930 Value *Stride = EmitScalarExpr(E->getArg(3));
3931 const auto *ResultTy = E->getType()->getAs<ConstantMatrixType>();
3932 auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
3933 assert(PtrTy && "arg0 must be of pointer type");
3934 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
3935
3938 E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD,
3939 0);
3940 Value *Result = MB.CreateColumnMajorLoad(
3941 Src.getElementType(), Src.emitRawPointer(*this),
3942 Align(Src.getAlignment().getQuantity()), Stride, IsVolatile,
3943 ResultTy->getNumRows(), ResultTy->getNumColumns(), "matrix");
3944 return RValue::get(Result);
3945 }
3946
3947 case Builtin::BI__builtin_matrix_column_major_store: {
3948 MatrixBuilder MB(Builder);
3949 Value *Matrix = EmitScalarExpr(E->getArg(0));
3951 Value *Stride = EmitScalarExpr(E->getArg(2));
3952
3953 const auto *MatrixTy = E->getArg(0)->getType()->getAs<ConstantMatrixType>();
3954 auto *PtrTy = E->getArg(1)->getType()->getAs<PointerType>();
3955 assert(PtrTy && "arg1 must be of pointer type");
3956 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
3957
3959 E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD,
3960 0);
3961 Value *Result = MB.CreateColumnMajorStore(
3962 Matrix, Dst.emitRawPointer(*this),
3963 Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile,
3964 MatrixTy->getNumRows(), MatrixTy->getNumColumns());
3965 return RValue::get(Result);
3966 }
3967
3968 case Builtin::BI__builtin_isinf_sign: {
3969 // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
3970 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3971 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
3972 Value *Arg = EmitScalarExpr(E->getArg(0));
3973 Value *AbsArg = EmitFAbs(*this, Arg);
3974 Value *IsInf = Builder.CreateFCmpOEQ(
3975 AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf");
3976 Value *IsNeg = EmitSignBit(*this, Arg);
3977
3978 llvm::Type *IntTy = ConvertType(E->getType());
3979 Value *Zero = Constant::getNullValue(IntTy);
3980 Value *One = ConstantInt::get(IntTy, 1);
3981 Value *NegativeOne = ConstantInt::get(IntTy, -1);
3982 Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
3983 Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero);
3984 return RValue::get(Result);
3985 }
3986
3987 case Builtin::BI__builtin_flt_rounds: {
3988 Function *F = CGM.getIntrinsic(Intrinsic::get_rounding);
3989
3990 llvm::Type *ResultType = ConvertType(E->getType());
3991 Value *Result = Builder.CreateCall(F);
3992 if (Result->getType() != ResultType)
3993 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3994 "cast");
3995 return RValue::get(Result);
3996 }
3997
3998 case Builtin::BI__builtin_set_flt_rounds: {
3999 Function *F = CGM.getIntrinsic(Intrinsic::set_rounding);
4000
4001 Value *V = EmitScalarExpr(E->getArg(0));
4002 Builder.CreateCall(F, V);
4003 return RValue::get(nullptr);
4004 }
4005
4006 case Builtin::BI__builtin_fpclassify: {
4007 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4008 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4009 Value *V = EmitScalarExpr(E->getArg(5));
4010 llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
4011
4012 // Create Result
4013 BasicBlock *Begin = Builder.GetInsertBlock();
4014 BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn);
4015 Builder.SetInsertPoint(End);
4016 PHINode *Result =
4017 Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4,
4018 "fpclassify_result");
4019
4020 // if (V==0) return FP_ZERO
4021 Builder.SetInsertPoint(Begin);
4022 Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
4023 "iszero");
4024 Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
4025 BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn);
4026 Builder.CreateCondBr(IsZero, End, NotZero);
4027 Result->addIncoming(ZeroLiteral, Begin);
4028
4029 // if (V != V) return FP_NAN
4030 Builder.SetInsertPoint(NotZero);
4031 Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp");
4032 Value *NanLiteral = EmitScalarExpr(E->getArg(0));
4033 BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn);
4034 Builder.CreateCondBr(IsNan, End, NotNan);
4035 Result->addIncoming(NanLiteral, NotZero);
4036
4037 // if (fabs(V) == infinity) return FP_INFINITY
4038 Builder.SetInsertPoint(NotNan);
4039 Value *VAbs = EmitFAbs(*this, V);
4040 Value *IsInf =
4041 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
4042 "isinf");
4043 Value *InfLiteral = EmitScalarExpr(E->getArg(1));
4044 BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn);
4045 Builder.CreateCondBr(IsInf, End, NotInf);
4046 Result->addIncoming(InfLiteral, NotNan);
4047
4048 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
4049 Builder.SetInsertPoint(NotInf);
4050 APFloat Smallest = APFloat::getSmallestNormalized(
4051 getContext().getFloatTypeSemantics(E->getArg(5)->getType()));
4052 Value *IsNormal =
4053 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
4054 "isnormal");
4055 Value *NormalResult =
4056 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
4057 EmitScalarExpr(E->getArg(3)));
4058 Builder.CreateBr(End);
4059 Result->addIncoming(NormalResult, NotInf);
4060
4061 // return Result
4062 Builder.SetInsertPoint(End);
4063 return RValue::get(Result);
4064 }
4065
4066 // An alloca will always return a pointer to the alloca (stack) address
4067 // space. This address space need not be the same as the AST / Language
4068 // default (e.g. in C / C++ auto vars are in the generic address space). At
4069 // the AST level this is handled within CreateTempAlloca et al., but for the
4070 // builtin / dynamic alloca we have to handle it here. We use an explicit cast
4071 // instead of passing an AS to CreateAlloca so as to not inhibit optimisation.
4072 case Builtin::BIalloca:
4073 case Builtin::BI_alloca:
4074 case Builtin::BI__builtin_alloca_uninitialized:
4075 case Builtin::BI__builtin_alloca: {
4076 Value *Size = EmitScalarExpr(E->getArg(0));
4077 const TargetInfo &TI = getContext().getTargetInfo();
4078 // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
4079 const Align SuitableAlignmentInBytes =
4080 CGM.getContext()
4082 .getAsAlign();
4083 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
4084 AI->setAlignment(SuitableAlignmentInBytes);
4085 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4086 initializeAlloca(*this, AI, Size, SuitableAlignmentInBytes);
4089 if (AAS != EAS) {
4090 llvm::Type *Ty = CGM.getTypes().ConvertType(E->getType());
4091 return RValue::get(getTargetHooks().performAddrSpaceCast(*this, AI, AAS,
4092 EAS, Ty));
4093 }
4094 return RValue::get(AI);
4095 }
4096
4097 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4098 case Builtin::BI__builtin_alloca_with_align: {
4099 Value *Size = EmitScalarExpr(E->getArg(0));
4100 Value *AlignmentInBitsValue = EmitScalarExpr(E->getArg(1));
4101 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
4102 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4103 const Align AlignmentInBytes =
4104 CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getAsAlign();
4105 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
4106 AI->setAlignment(AlignmentInBytes);
4107 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4108 initializeAlloca(*this, AI, Size, AlignmentInBytes);
4111 if (AAS != EAS) {
4112 llvm::Type *Ty = CGM.getTypes().ConvertType(E->getType());
4113 return RValue::get(getTargetHooks().performAddrSpaceCast(*this, AI, AAS,
4114 EAS, Ty));
4115 }
4116 return RValue::get(AI);
4117 }
4118
4119 case Builtin::BIbzero:
4120 case Builtin::BI__builtin_bzero: {
4122 Value *SizeVal = EmitScalarExpr(E->getArg(1));
4123 EmitNonNullArgCheck(Dest, E->getArg(0)->getType(),
4124 E->getArg(0)->getExprLoc(), FD, 0);
4125 Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false);
4126 return RValue::get(nullptr);
4127 }
4128
4129 case Builtin::BIbcopy:
4130 case Builtin::BI__builtin_bcopy: {
4133 Value *SizeVal = EmitScalarExpr(E->getArg(2));
4135 E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD,
4136 0);
4138 E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD,
4139 0);
4140 Builder.CreateMemMove(Dest, Src, SizeVal, false);
4141 return RValue::get(nullptr);
4142 }
4143
4144 case Builtin::BImemcpy:
4145 case Builtin::BI__builtin_memcpy:
4146 case Builtin::BImempcpy:
4147 case Builtin::BI__builtin_mempcpy: {
4150 Value *SizeVal = EmitScalarExpr(E->getArg(2));
4151 EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0);
4152 EmitArgCheck(TCK_Load, Src, E->getArg(1), 1);
4153 Builder.CreateMemCpy(Dest, Src, SizeVal, false);
4154 if (BuiltinID == Builtin::BImempcpy ||
4155 BuiltinID == Builtin::BI__builtin_mempcpy)
4157 Dest.getElementType(), Dest.emitRawPointer(*this), SizeVal));
4158 else
4159 return RValue::get(Dest, *this);
4160 }
4161
4162 case Builtin::BI__builtin_memcpy_inline: {
4165 uint64_t Size =
4166 E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue();
4167 EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0);
4168 EmitArgCheck(TCK_Load, Src, E->getArg(1), 1);
4169 Builder.CreateMemCpyInline(Dest, Src, Size);
4170 return RValue::get(nullptr);
4171 }
4172
4173 case Builtin::BI__builtin_char_memchr:
4174 BuiltinID = Builtin::BI__builtin_memchr;
4175 break;
4176
4177 case Builtin::BI__builtin___memcpy_chk: {
4178 // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
4179 Expr::EvalResult SizeResult, DstSizeResult;
4180 if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
4181 !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
4182 break;
4183 llvm::APSInt Size = SizeResult.Val.getInt();
4184 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4185 if (Size.ugt(DstSize))
4186 break;
4189 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
4190 Builder.CreateMemCpy(Dest, Src, SizeVal, false);
4191 return RValue::get(Dest, *this);
4192 }
4193
4194 case Builtin::BI__builtin_objc_memmove_collectable: {
4195 Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
4196 Address SrcAddr = EmitPointerWithAlignment(E->getArg(1));
4197 Value *SizeVal = EmitScalarExpr(E->getArg(2));
4199 DestAddr, SrcAddr, SizeVal);
4200 return RValue::get(DestAddr, *this);
4201 }
4202
4203 case Builtin::BI__builtin___memmove_chk: {
4204 // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
4205 Expr::EvalResult SizeResult, DstSizeResult;
4206 if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
4207 !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
4208 break;
4209 llvm::APSInt Size = SizeResult.Val.getInt();
4210 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4211 if (Size.ugt(DstSize))
4212 break;
4215 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
4216 Builder.CreateMemMove(Dest, Src, SizeVal, false);
4217 return RValue::get(Dest, *this);
4218 }
4219
4220 case Builtin::BImemmove:
4221 case Builtin::BI__builtin_memmove: {
4224 Value *SizeVal = EmitScalarExpr(E->getArg(2));
4225 EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0);
4226 EmitArgCheck(TCK_Load, Src, E->getArg(1), 1);
4227 Builder.CreateMemMove(Dest, Src, SizeVal, false);
4228 return RValue::get(Dest, *this);
4229 }
4230 case Builtin::BImemset:
4231 case Builtin::BI__builtin_memset: {
4233 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
4234 Builder.getInt8Ty());
4235 Value *SizeVal = EmitScalarExpr(E->getArg(2));
4236 EmitNonNullArgCheck(Dest, E->getArg(0)->getType(),
4237 E->getArg(0)->getExprLoc(), FD, 0);
4238 Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
4239 return RValue::get(Dest, *this);
4240 }
4241 case Builtin::BI__builtin_memset_inline: {
4243 Value *ByteVal =
4244 Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty());
4245 uint64_t Size =
4246 E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue();
4248 E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD,
4249 0);
4250 Builder.CreateMemSetInline(Dest, ByteVal, Size);
4251 return RValue::get(nullptr);
4252 }
4253 case Builtin::BI__builtin___memset_chk: {
4254 // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
4255 Expr::EvalResult SizeResult, DstSizeResult;
4256 if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
4257 !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
4258 break;
4259 llvm::APSInt Size = SizeResult.Val.getInt();
4260 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4261 if (Size.ugt(DstSize))
4262 break;
4264 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
4265 Builder.getInt8Ty());
4266 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
4267 Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
4268 return RValue::get(Dest, *this);
4269 }
4270 case Builtin::BI__builtin_wmemchr: {
4271 // The MSVC runtime library does not provide a definition of wmemchr, so we
4272 // need an inline implementation.
4273 if (!getTarget().getTriple().isOSMSVCRT())
4274 break;
4275
4276 llvm::Type *WCharTy = ConvertType(getContext().WCharTy);
4277 Value *Str = EmitScalarExpr(E->getArg(0));
4278 Value *Chr = EmitScalarExpr(E->getArg(1));
4279 Value *Size = EmitScalarExpr(E->getArg(2));
4280
4281 BasicBlock *Entry = Builder.GetInsertBlock();
4282 BasicBlock *CmpEq = createBasicBlock("wmemchr.eq");
4283 BasicBlock *Next = createBasicBlock("wmemchr.next");
4284 BasicBlock *Exit = createBasicBlock("wmemchr.exit");
4285 Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(SizeTy, 0));
4286 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
4287
4288 EmitBlock(CmpEq);
4289 PHINode *StrPhi = Builder.CreatePHI(Str->getType(), 2);
4290 StrPhi->addIncoming(Str, Entry);
4291 PHINode *SizePhi = Builder.CreatePHI(SizeTy, 2);
4292 SizePhi->addIncoming(Size, Entry);
4293 CharUnits WCharAlign =
4295 Value *StrCh = Builder.CreateAlignedLoad(WCharTy, StrPhi, WCharAlign);
4296 Value *FoundChr = Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
4297 Value *StrEqChr = Builder.CreateICmpEQ(StrCh, Chr);
4298 Builder.CreateCondBr(StrEqChr, Exit, Next);
4299
4300 EmitBlock(Next);
4301 Value *NextStr = Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
4302 Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(SizeTy, 1));
4303 Value *NextSizeEq0 =
4304 Builder.CreateICmpEQ(NextSize, ConstantInt::get(SizeTy, 0));
4305 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
4306 StrPhi->addIncoming(NextStr, Next);
4307 SizePhi->addIncoming(NextSize, Next);
4308
4309 EmitBlock(Exit);
4310 PHINode *Ret = Builder.CreatePHI(Str->getType(), 3);
4311 Ret->addIncoming(llvm::Constant::getNullValue(Str->getType()), Entry);
4312 Ret->addIncoming(llvm::Constant::getNullValue(Str->getType()), Next);
4313 Ret->addIncoming(FoundChr, CmpEq);
4314 return RValue::get(Ret);
4315 }
4316 case Builtin::BI__builtin_wmemcmp: {
4317 // The MSVC runtime library does not provide a definition of wmemcmp, so we
4318 // need an inline implementation.
4319 if (!getTarget().getTriple().isOSMSVCRT())
4320 break;
4321
4322 llvm::Type *WCharTy = ConvertType(getContext().WCharTy);
4323
4324 Value *Dst = EmitScalarExpr(E->getArg(0));
4325 Value *Src = EmitScalarExpr(E->getArg(1));
4326 Value *Size = EmitScalarExpr(E->getArg(2));
4327
4328 BasicBlock *Entry = Builder.GetInsertBlock();
4329 BasicBlock *CmpGT = createBasicBlock("wmemcmp.gt");
4330 BasicBlock *CmpLT = createBasicBlock("wmemcmp.lt");
4331 BasicBlock *Next = createBasicBlock("wmemcmp.next");
4332 BasicBlock *Exit = createBasicBlock("wmemcmp.exit");
4333 Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(SizeTy, 0));
4334 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
4335
4336 EmitBlock(CmpGT);
4337 PHINode *DstPhi = Builder.CreatePHI(Dst->getType(), 2);
4338 DstPhi->addIncoming(Dst, Entry);
4339 PHINode *SrcPhi = Builder.CreatePHI(Src->getType(), 2);
4340 SrcPhi->addIncoming(Src, Entry);
4341 PHINode *SizePhi = Builder.CreatePHI(SizeTy, 2);
4342 SizePhi->addIncoming(Size, Entry);
4343 CharUnits WCharAlign =
4345 Value *DstCh = Builder.CreateAlignedLoad(WCharTy, DstPhi, WCharAlign);
4346 Value *SrcCh = Builder.CreateAlignedLoad(WCharTy, SrcPhi, WCharAlign);
4347 Value *DstGtSrc = Builder.CreateICmpUGT(DstCh, SrcCh);
4348 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
4349
4350 EmitBlock(CmpLT);
4351 Value *DstLtSrc = Builder.CreateICmpULT(DstCh, SrcCh);
4352 Builder.CreateCondBr(DstLtSrc, Exit, Next);
4353
4354 EmitBlock(Next);
4355 Value *NextDst = Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
4356 Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
4357 Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(SizeTy, 1));
4358 Value *NextSizeEq0 =
4359 Builder.CreateICmpEQ(NextSize, ConstantInt::get(SizeTy, 0));
4360 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
4361 DstPhi->addIncoming(NextDst, Next);
4362 SrcPhi->addIncoming(NextSrc, Next);
4363 SizePhi->addIncoming(NextSize, Next);
4364
4365 EmitBlock(Exit);
4366 PHINode *Ret = Builder.CreatePHI(IntTy, 4);
4367 Ret->addIncoming(ConstantInt::get(IntTy, 0), Entry);
4368 Ret->addIncoming(ConstantInt::get(IntTy, 1), CmpGT);
4369 Ret->addIncoming(ConstantInt::get(IntTy, -1), CmpLT);
4370 Ret->addIncoming(ConstantInt::get(IntTy, 0), Next);
4371 return RValue::get(Ret);
4372 }
4373 case Builtin::BI__builtin_dwarf_cfa: {
4374 // The offset in bytes from the first argument to the CFA.
4375 //
4376 // Why on earth is this in the frontend? Is there any reason at
4377 // all that the backend can't reasonably determine this while
4378 // lowering llvm.eh.dwarf.cfa()?
4379 //
4380 // TODO: If there's a satisfactory reason, add a target hook for
4381 // this instead of hard-coding 0, which is correct for most targets.
4382 int32_t Offset = 0;
4383
4384 Function *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
4385 return RValue::get(Builder.CreateCall(F,
4386 llvm::ConstantInt::get(Int32Ty, Offset)));
4387 }
4388 case Builtin::BI__builtin_return_address: {
4389 Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
4390 getContext().UnsignedIntTy);
4391 Function *F = CGM.getIntrinsic(Intrinsic::returnaddress);
4392 return RValue::get(Builder.CreateCall(F, Depth));
4393 }
4394 case Builtin::BI_ReturnAddress: {
4395 Function *F = CGM.getIntrinsic(Intrinsic::returnaddress);
4396 return RValue::get(Builder.CreateCall(F, Builder.getInt32(0)));
4397 }
4398 case Builtin::BI__builtin_frame_address: {
4399 Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
4400 getContext().UnsignedIntTy);
4401 Function *F = CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy);
4402 return RValue::get(Builder.CreateCall(F, Depth));
4403 }
4404 case Builtin::BI__builtin_extract_return_addr: {
4407 return RValue::get(Result);
4408 }
4409 case Builtin::BI__builtin_frob_return_addr: {
4412 return RValue::get(Result);
4413 }
4414 case Builtin::BI__builtin_dwarf_sp_column: {
4415 llvm::IntegerType *Ty
4416 = cast<llvm::IntegerType>(ConvertType(E->getType()));
4418 if (Column == -1) {
4419 CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column");
4420 return RValue::get(llvm::UndefValue::get(Ty));
4421 }
4422 return RValue::get(llvm::ConstantInt::get(Ty, Column, true));
4423 }
4424 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
4426 if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address))
4427 CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table");
4428 return RValue::get(llvm::UndefValue::get(ConvertType(E->getType())));
4429 }
4430 case Builtin::BI__builtin_eh_return: {
4431 Value *Int = EmitScalarExpr(E->getArg(0));
4432 Value *Ptr = EmitScalarExpr(E->getArg(1));
4433
4434 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
4435 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
4436 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
4437 Function *F =
4438 CGM.getIntrinsic(IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32
4439 : Intrinsic::eh_return_i64);
4440 Builder.CreateCall(F, {Int, Ptr});
4441 Builder.CreateUnreachable();
4442
4443 // We do need to preserve an insertion point.
4444 EmitBlock(createBasicBlock("builtin_eh_return.cont"));
4445
4446 return RValue::get(nullptr);
4447 }
4448 case Builtin::BI__builtin_unwind_init: {
4449 Function *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
4450 Builder.CreateCall(F);
4451 return RValue::get(nullptr);
4452 }
4453 case Builtin::BI__builtin_extend_pointer: {
4454 // Extends a pointer to the size of an _Unwind_Word, which is
4455 // uint64_t on all platforms. Generally this gets poked into a
4456 // register and eventually used as an address, so if the
4457 // addressing registers are wider than pointers and the platform
4458 // doesn't implicitly ignore high-order bits when doing
4459 // addressing, we need to make sure we zext / sext based on
4460 // the platform's expectations.
4461 //
4462 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
4463
4464 // Cast the pointer to intptr_t.
4465 Value *Ptr = EmitScalarExpr(E->getArg(0));
4466 Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast");
4467
4468 // If that's 64 bits, we're done.
4469 if (IntPtrTy->getBitWidth() == 64)
4470 return RValue::get(Result);
4471
4472 // Otherwise, ask the codegen data what to do.
4473 if (getTargetHooks().extendPointerWithSExt())
4474 return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext"));
4475 else
4476 return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext"));
4477 }
4478 case Builtin::BI__builtin_setjmp: {
4479 // Buffer is a void**.
4481
4482 // Store the frame pointer to the setjmp buffer.
4483 Value *FrameAddr = Builder.CreateCall(
4484 CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy),
4485 ConstantInt::get(Int32Ty, 0));
4486 Builder.CreateStore(FrameAddr, Buf);
4487
4488 // Store the stack pointer to the setjmp buffer.
4489 Value *StackAddr = Builder.CreateStackSave();
4490 assert(Buf.emitRawPointer(*this)->getType() == StackAddr->getType());
4491
4492 Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Buf, 2);
4493 Builder.CreateStore(StackAddr, StackSaveSlot);
4494
4495 // Call LLVM's EH setjmp, which is lightweight.
4496 Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
4497 return RValue::get(Builder.CreateCall(F, Buf.emitRawPointer(*this)));
4498 }
4499 case Builtin::BI__builtin_longjmp: {
4500 Value *Buf = EmitScalarExpr(E->getArg(0));
4501
4502 // Call LLVM's EH longjmp, which is lightweight.
4503 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
4504
4505 // longjmp doesn't return; mark this as unreachable.
4506 Builder.CreateUnreachable();
4507
4508 // We do need to preserve an insertion point.
4509 EmitBlock(createBasicBlock("longjmp.cont"));
4510
4511 return RValue::get(nullptr);
4512 }
4513 case Builtin::BI__builtin_launder: {
4514 const Expr *Arg = E->getArg(0);
4515 QualType ArgTy = Arg->getType()->getPointeeType();
4516 Value *Ptr = EmitScalarExpr(Arg);
4517 if (TypeRequiresBuiltinLaunder(CGM, ArgTy))
4519
4520 return RValue::get(Ptr);
4521 }
4522 case Builtin::BI__sync_fetch_and_add:
4523 case Builtin::BI__sync_fetch_and_sub:
4524 case Builtin::BI__sync_fetch_and_or:
4525 case Builtin::BI__sync_fetch_and_and:
4526 case Builtin::BI__sync_fetch_and_xor:
4527 case Builtin::BI__sync_fetch_and_nand:
4528 case Builtin::BI__sync_add_and_fetch:
4529 case Builtin::BI__sync_sub_and_fetch:
4530 case Builtin::BI__sync_and_and_fetch:
4531 case Builtin::BI__sync_or_and_fetch:
4532 case Builtin::BI__sync_xor_and_fetch:
4533 case Builtin::BI__sync_nand_and_fetch:
4534 case Builtin::BI__sync_val_compare_and_swap:
4535 case Builtin::BI__sync_bool_compare_and_swap:
4536 case Builtin::BI__sync_lock_test_and_set:
4537 case Builtin::BI__sync_lock_release:
4538 case Builtin::BI__sync_swap:
4539 llvm_unreachable("Shouldn't make it through sema");
4540 case Builtin::BI__sync_fetch_and_add_1:
4541 case Builtin::BI__sync_fetch_and_add_2:
4542 case Builtin::BI__sync_fetch_and_add_4:
4543 case Builtin::BI__sync_fetch_and_add_8:
4544 case Builtin::BI__sync_fetch_and_add_16:
4545 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E);
4546 case Builtin::BI__sync_fetch_and_sub_1:
4547 case Builtin::BI__sync_fetch_and_sub_2:
4548 case Builtin::BI__sync_fetch_and_sub_4:
4549 case Builtin::BI__sync_fetch_and_sub_8:
4550 case Builtin::BI__sync_fetch_and_sub_16:
4551 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E);
4552 case Builtin::BI__sync_fetch_and_or_1:
4553 case Builtin::BI__sync_fetch_and_or_2: