clang 23.0.0git
LowerToLLVM.cpp
Go to the documentation of this file.
1//====- LowerToLLVM.cpp - Lowering from CIR to LLVMIR ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements lowering of CIR operations to LLVMIR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "LowerToLLVM.h"
14
15#include <array>
16#include <deque>
17#include <optional>
18
19#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
20#include "mlir/Dialect/DLTI/DLTI.h"
21#include "mlir/Dialect/Func/IR/FuncOps.h"
22#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
23#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
24#include "mlir/Dialect/Ptr/IR/MemorySpaceInterfaces.h"
25#include "mlir/IR/BuiltinAttributes.h"
26#include "mlir/IR/BuiltinDialect.h"
27#include "mlir/IR/BuiltinOps.h"
28#include "mlir/IR/Types.h"
29#include "mlir/Pass/Pass.h"
30#include "mlir/Pass/PassManager.h"
31#include "mlir/Support/LLVM.h"
32#include "mlir/Target/LLVMIR/Dialect/Builtin/BuiltinToLLVMIRTranslation.h"
33#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
34#include "mlir/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.h"
35#include "mlir/Target/LLVMIR/Export.h"
36#include "mlir/Transforms/DialectConversion.h"
37#include "clang/CIR/Dialect/IR/CIRAttrs.h"
38#include "clang/CIR/Dialect/IR/CIRDialect.h"
39#include "clang/CIR/Dialect/IR/CIRTypes.h"
40#include "clang/CIR/Dialect/Passes.h"
41#include "clang/CIR/LoweringHelpers.h"
42#include "clang/CIR/MissingFeatures.h"
43#include "clang/CIR/Passes.h"
44#include "llvm/ADT/TypeSwitch.h"
45#include "llvm/IR/Module.h"
46#include "llvm/Support/Casting.h"
47#include "llvm/Support/ErrorHandling.h"
48#include "llvm/Support/TimeProfiler.h"
49#include "llvm/Support/raw_ostream.h"
50
51using namespace cir;
52using namespace llvm;
53
54namespace cir {
55namespace direct {
56
57//===----------------------------------------------------------------------===//
58// Helper Methods
59//===----------------------------------------------------------------------===//
60
61namespace {
62/// If the given type is a vector type, return the vector's element type.
63/// Otherwise return the given type unchanged.
64mlir::Type elementTypeIfVector(mlir::Type type) {
65 return llvm::TypeSwitch<mlir::Type, mlir::Type>(type)
66 .Case<cir::VectorType, mlir::VectorType>(
67 [](auto p) { return p.getElementType(); })
68 .Default([](mlir::Type p) { return p; });
69}
70} // namespace
71
72/// Given a type convertor and a data layout, convert the given type to a type
73/// that is suitable for memory operations. For example, this can be used to
74/// lower cir.bool accesses to i8.
75static mlir::Type convertTypeForMemory(const mlir::TypeConverter &converter,
76 mlir::DataLayout const &dataLayout,
77 mlir::Type type) {
78 // TODO(cir): Handle other types similarly to clang's codegen
79 // convertTypeForMemory
80 if (isa<cir::BoolType>(type)) {
81 return mlir::IntegerType::get(type.getContext(),
82 dataLayout.getTypeSizeInBits(type));
83 }
84
85 return converter.convertType(type);
86}
87
88static mlir::Value createIntCast(mlir::OpBuilder &bld, mlir::Value src,
89 mlir::IntegerType dstTy,
90 bool isSigned = false) {
91 mlir::Type srcTy = src.getType();
92 assert(mlir::isa<mlir::IntegerType>(srcTy));
93
94 unsigned srcWidth = mlir::cast<mlir::IntegerType>(srcTy).getWidth();
95 unsigned dstWidth = mlir::cast<mlir::IntegerType>(dstTy).getWidth();
96 mlir::Location loc = src.getLoc();
97
98 if (dstWidth > srcWidth && isSigned)
99 return mlir::LLVM::SExtOp::create(bld, loc, dstTy, src);
100 if (dstWidth > srcWidth)
101 return mlir::LLVM::ZExtOp::create(bld, loc, dstTy, src);
102 if (dstWidth < srcWidth)
103 return mlir::LLVM::TruncOp::create(bld, loc, dstTy, src);
104 return mlir::LLVM::BitcastOp::create(bld, loc, dstTy, src);
105}
106
107static mlir::LLVM::Visibility
108lowerCIRVisibilityToLLVMVisibility(cir::VisibilityKind visibilityKind) {
109 switch (visibilityKind) {
110 case cir::VisibilityKind::Default:
111 return ::mlir::LLVM::Visibility::Default;
112 case cir::VisibilityKind::Hidden:
113 return ::mlir::LLVM::Visibility::Hidden;
114 case cir::VisibilityKind::Protected:
115 return ::mlir::LLVM::Visibility::Protected;
116 }
117}
118
119/// Emits the value from memory as expected by its users. Should be called when
120/// the memory represetnation of a CIR type is not equal to its scalar
121/// representation.
122static mlir::Value emitFromMemory(mlir::ConversionPatternRewriter &rewriter,
123 mlir::DataLayout const &dataLayout,
124 cir::LoadOp op, mlir::Value value) {
125
126 // TODO(cir): Handle other types similarly to clang's codegen EmitFromMemory
127 if (auto boolTy = mlir::dyn_cast<cir::BoolType>(op.getType())) {
128 // Create a cast value from specified size in datalayout to i1
129 assert(value.getType().isInteger(dataLayout.getTypeSizeInBits(boolTy)));
130 return createIntCast(rewriter, value, rewriter.getI1Type());
131 }
132
133 return value;
134}
135
136/// Emits a value to memory with the expected scalar type. Should be called when
137/// the memory represetnation of a CIR type is not equal to its scalar
138/// representation.
139static mlir::Value emitToMemory(mlir::ConversionPatternRewriter &rewriter,
140 mlir::DataLayout const &dataLayout,
141 mlir::Type origType, mlir::Value value) {
142
143 // TODO(cir): Handle other types similarly to clang's codegen EmitToMemory
144 if (auto boolTy = mlir::dyn_cast<cir::BoolType>(origType)) {
145 // Create zext of value from i1 to i8
146 mlir::IntegerType memType =
147 rewriter.getIntegerType(dataLayout.getTypeSizeInBits(boolTy));
148 return createIntCast(rewriter, value, memType);
149 }
150
151 return value;
152}
153
154mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage) {
155 using CIR = cir::GlobalLinkageKind;
156 using LLVM = mlir::LLVM::Linkage;
157
158 switch (linkage) {
159 case CIR::AppendingLinkage:
160 return LLVM::Appending;
161 case CIR::AvailableExternallyLinkage:
162 return LLVM::AvailableExternally;
163 case CIR::CommonLinkage:
164 return LLVM::Common;
165 case CIR::ExternalLinkage:
166 return LLVM::External;
167 case CIR::ExternalWeakLinkage:
168 return LLVM::ExternWeak;
169 case CIR::InternalLinkage:
170 return LLVM::Internal;
171 case CIR::LinkOnceAnyLinkage:
172 return LLVM::Linkonce;
173 case CIR::LinkOnceODRLinkage:
174 return LLVM::LinkonceODR;
175 case CIR::PrivateLinkage:
176 return LLVM::Private;
177 case CIR::WeakAnyLinkage:
178 return LLVM::Weak;
179 case CIR::WeakODRLinkage:
180 return LLVM::WeakODR;
181 };
182 llvm_unreachable("Unknown CIR linkage type");
183}
184
185mlir::LogicalResult CIRToLLVMCopyOpLowering::matchAndRewrite(
186 cir::CopyOp op, OpAdaptor adaptor,
187 mlir::ConversionPatternRewriter &rewriter) const {
188 mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>());
189 const mlir::Value length = mlir::LLVM::ConstantOp::create(
190 rewriter, op.getLoc(), rewriter.getI64Type(), op.getLength(layout));
191 assert(!cir::MissingFeatures::aggValueSlotVolatile());
192 rewriter.replaceOpWithNewOp<mlir::LLVM::MemcpyOp>(
193 op, adaptor.getDst(), adaptor.getSrc(), length, op.getIsVolatile());
194 return mlir::success();
195}
196
197mlir::LogicalResult CIRToLLVMMemCpyOpLowering::matchAndRewrite(
198 cir::MemCpyOp op, OpAdaptor adaptor,
199 mlir::ConversionPatternRewriter &rewriter) const {
200 rewriter.replaceOpWithNewOp<mlir::LLVM::MemcpyOp>(
201 op, adaptor.getDst(), adaptor.getSrc(), adaptor.getLen(),
202 /*isVolatile=*/false);
203 return mlir::success();
204}
205
206mlir::LogicalResult CIRToLLVMMemMoveOpLowering::matchAndRewrite(
207 cir::MemMoveOp op, OpAdaptor adaptor,
208 mlir::ConversionPatternRewriter &rewriter) const {
209 rewriter.replaceOpWithNewOp<mlir::LLVM::MemmoveOp>(
210 op, adaptor.getDst(), adaptor.getSrc(), adaptor.getLen(),
211 /*isVolatile=*/false);
212 return mlir::success();
213}
214
215mlir::LogicalResult CIRToLLVMMemSetOpLowering::matchAndRewrite(
216 cir::MemSetOp op, OpAdaptor adaptor,
217 mlir::ConversionPatternRewriter &rewriter) const {
218
219 auto memset = rewriter.replaceOpWithNewOp<mlir::LLVM::MemsetOp>(
220 op, adaptor.getDst(), adaptor.getVal(), adaptor.getLen(),
221 /*isVolatile=*/false);
222
223 if (op.getAlignmentAttr()) {
224 // Construct a list full of empty attributes.
225 llvm::SmallVector<mlir::Attribute> attrs{memset.getNumOperands(),
226 rewriter.getDictionaryAttr({})};
227 llvm::SmallVector<mlir::NamedAttribute> destAttrs;
228 destAttrs.push_back(
229 {mlir::LLVM::LLVMDialect::getAlignAttrName(), op.getAlignmentAttr()});
230 attrs[memset.odsIndex_dst] = rewriter.getDictionaryAttr(destAttrs);
231
232 auto arrayAttr = rewriter.getArrayAttr(attrs);
233 memset.setArgAttrsAttr(arrayAttr);
234 }
235
236 return mlir::success();
237}
238
239mlir::LogicalResult CIRToLLVMSqrtOpLowering::matchAndRewrite(
240 cir::SqrtOp op, OpAdaptor adaptor,
241 mlir::ConversionPatternRewriter &rewriter) const {
242 mlir::Type resTy = typeConverter->convertType(op.getType());
243 rewriter.replaceOpWithNewOp<mlir::LLVM::SqrtOp>(op, resTy, adaptor.getSrc());
244 return mlir::success();
245}
246
247mlir::LogicalResult CIRToLLVMCosOpLowering::matchAndRewrite(
248 cir::CosOp op, OpAdaptor adaptor,
249 mlir::ConversionPatternRewriter &rewriter) const {
250 mlir::Type resTy = typeConverter->convertType(op.getType());
251 rewriter.replaceOpWithNewOp<mlir::LLVM::CosOp>(op, resTy, adaptor.getSrc());
252 return mlir::success();
253}
254
255mlir::LogicalResult CIRToLLVMExpOpLowering::matchAndRewrite(
256 cir::ExpOp op, OpAdaptor adaptor,
257 mlir::ConversionPatternRewriter &rewriter) const {
258 mlir::Type resTy = typeConverter->convertType(op.getType());
259 rewriter.replaceOpWithNewOp<mlir::LLVM::ExpOp>(op, resTy, adaptor.getSrc());
260 return mlir::success();
261}
262
263mlir::LogicalResult CIRToLLVMExp2OpLowering::matchAndRewrite(
264 cir::Exp2Op op, OpAdaptor adaptor,
265 mlir::ConversionPatternRewriter &rewriter) const {
266 mlir::Type resTy = typeConverter->convertType(op.getType());
267 rewriter.replaceOpWithNewOp<mlir::LLVM::Exp2Op>(op, resTy, adaptor.getSrc());
268 return mlir::success();
269}
270
271mlir::LogicalResult CIRToLLVMLogOpLowering::matchAndRewrite(
272 cir::LogOp op, OpAdaptor adaptor,
273 mlir::ConversionPatternRewriter &rewriter) const {
274 mlir::Type resTy = typeConverter->convertType(op.getType());
275 rewriter.replaceOpWithNewOp<mlir::LLVM::LogOp>(op, resTy, adaptor.getSrc());
276 return mlir::success();
277}
278
279mlir::LogicalResult CIRToLLVMLog10OpLowering::matchAndRewrite(
280 cir::Log10Op op, OpAdaptor adaptor,
281 mlir::ConversionPatternRewriter &rewriter) const {
282 mlir::Type resTy = typeConverter->convertType(op.getType());
283 rewriter.replaceOpWithNewOp<mlir::LLVM::Log10Op>(op, resTy, adaptor.getSrc());
284 return mlir::success();
285}
286
287mlir::LogicalResult CIRToLLVMLog2OpLowering::matchAndRewrite(
288 cir::Log2Op op, OpAdaptor adaptor,
289 mlir::ConversionPatternRewriter &rewriter) const {
290 mlir::Type resTy = typeConverter->convertType(op.getType());
291 rewriter.replaceOpWithNewOp<mlir::LLVM::Log2Op>(op, resTy, adaptor.getSrc());
292 return mlir::success();
293}
294
295mlir::LogicalResult CIRToLLVMNearbyintOpLowering::matchAndRewrite(
296 cir::NearbyintOp op, OpAdaptor adaptor,
297 mlir::ConversionPatternRewriter &rewriter) const {
298 mlir::Type resTy = typeConverter->convertType(op.getType());
299 rewriter.replaceOpWithNewOp<mlir::LLVM::NearbyintOp>(op, resTy,
300 adaptor.getSrc());
301 return mlir::success();
302}
303
304mlir::LogicalResult CIRToLLVMRintOpLowering::matchAndRewrite(
305 cir::RintOp op, OpAdaptor adaptor,
306 mlir::ConversionPatternRewriter &rewriter) const {
307 mlir::Type resTy = typeConverter->convertType(op.getType());
308 rewriter.replaceOpWithNewOp<mlir::LLVM::RintOp>(op, resTy, adaptor.getSrc());
309 return mlir::success();
310}
311
312mlir::LogicalResult CIRToLLVMRoundOpLowering::matchAndRewrite(
313 cir::RoundOp op, OpAdaptor adaptor,
314 mlir::ConversionPatternRewriter &rewriter) const {
315 mlir::Type resTy = typeConverter->convertType(op.getType());
316 rewriter.replaceOpWithNewOp<mlir::LLVM::RoundOp>(op, resTy, adaptor.getSrc());
317 return mlir::success();
318}
319
320mlir::LogicalResult CIRToLLVMRoundEvenOpLowering::matchAndRewrite(
321 cir::RoundEvenOp op, OpAdaptor adaptor,
322 mlir::ConversionPatternRewriter &rewriter) const {
323 mlir::Type resTy = typeConverter->convertType(op.getType());
324 rewriter.replaceOpWithNewOp<mlir::LLVM::RoundEvenOp>(op, resTy,
325 adaptor.getSrc());
326 return mlir::success();
327}
328
329mlir::LogicalResult CIRToLLVMSinOpLowering::matchAndRewrite(
330 cir::SinOp op, OpAdaptor adaptor,
331 mlir::ConversionPatternRewriter &rewriter) const {
332 mlir::Type resTy = typeConverter->convertType(op.getType());
333 rewriter.replaceOpWithNewOp<mlir::LLVM::SinOp>(op, resTy, adaptor.getSrc());
334 return mlir::success();
335}
336
337mlir::LogicalResult CIRToLLVMTanOpLowering::matchAndRewrite(
338 cir::TanOp op, OpAdaptor adaptor,
339 mlir::ConversionPatternRewriter &rewriter) const {
340 mlir::Type resTy = typeConverter->convertType(op.getType());
341 rewriter.replaceOpWithNewOp<mlir::LLVM::TanOp>(op, resTy, adaptor.getSrc());
342 return mlir::success();
343}
344
345mlir::LogicalResult CIRToLLVMTruncOpLowering::matchAndRewrite(
346 cir::TruncOp op, OpAdaptor adaptor,
347 mlir::ConversionPatternRewriter &rewriter) const {
348 mlir::Type resTy = typeConverter->convertType(op.getType());
349 rewriter.replaceOpWithNewOp<mlir::LLVM::FTruncOp>(op, resTy,
350 adaptor.getSrc());
351 return mlir::success();
352}
353
354mlir::LogicalResult CIRToLLVMFloorOpLowering::matchAndRewrite(
355 cir::FloorOp op, OpAdaptor adaptor,
356 mlir::ConversionPatternRewriter &rewriter) const {
357 mlir::Type resTy = typeConverter->convertType(op.getType());
358 rewriter.replaceOpWithNewOp<mlir::LLVM::FFloorOp>(op, resTy,
359 adaptor.getSrc());
360 return mlir::success();
361}
362
363static mlir::Value getLLVMIntCast(mlir::ConversionPatternRewriter &rewriter,
364 mlir::Value llvmSrc, mlir::Type llvmDstIntTy,
365 bool isUnsigned, uint64_t cirSrcWidth,
366 uint64_t cirDstIntWidth) {
367 if (cirSrcWidth == cirDstIntWidth)
368 return llvmSrc;
369
370 auto loc = llvmSrc.getLoc();
371 if (cirSrcWidth < cirDstIntWidth) {
372 if (isUnsigned)
373 return mlir::LLVM::ZExtOp::create(rewriter, loc, llvmDstIntTy, llvmSrc);
374 return mlir::LLVM::SExtOp::create(rewriter, loc, llvmDstIntTy, llvmSrc);
375 }
376
377 // Otherwise truncate
378 return mlir::LLVM::TruncOp::create(rewriter, loc, llvmDstIntTy, llvmSrc);
379}
380
381class CIRAttrToValue {
382public:
383 CIRAttrToValue(mlir::Operation *parentOp,
384 mlir::ConversionPatternRewriter &rewriter,
385 const mlir::TypeConverter *converter)
386 : parentOp(parentOp), rewriter(rewriter), converter(converter) {}
387
388 mlir::Value visit(mlir::Attribute attr) {
389 return llvm::TypeSwitch<mlir::Attribute, mlir::Value>(attr)
390 .Case<cir::BoolAttr, cir::IntAttr, cir::FPAttr, cir::ConstComplexAttr,
391 cir::ConstArrayAttr, cir::ConstRecordAttr, cir::ConstVectorAttr,
392 cir::ConstPtrAttr, cir::GlobalViewAttr, cir::TypeInfoAttr,
393 cir::UndefAttr, cir::VTableAttr, cir::ZeroAttr>(
394 [&](auto attrT) { return visitCirAttr(attrT); })
395 .Default([&](auto attrT) { return mlir::Value(); });
396 }
397
398 mlir::Value visitCirAttr(cir::BoolAttr boolAttr);
399 mlir::Value visitCirAttr(cir::IntAttr intAttr);
400 mlir::Value visitCirAttr(cir::FPAttr fltAttr);
401 mlir::Value visitCirAttr(cir::ConstComplexAttr complexAttr);
402 mlir::Value visitCirAttr(cir::ConstPtrAttr ptrAttr);
403 mlir::Value visitCirAttr(cir::ConstArrayAttr attr);
404 mlir::Value visitCirAttr(cir::ConstRecordAttr attr);
405 mlir::Value visitCirAttr(cir::ConstVectorAttr attr);
406 mlir::Value visitCirAttr(cir::GlobalViewAttr attr);
407 mlir::Value visitCirAttr(cir::TypeInfoAttr attr);
408 mlir::Value visitCirAttr(cir::UndefAttr attr);
409 mlir::Value visitCirAttr(cir::VTableAttr attr);
410 mlir::Value visitCirAttr(cir::ZeroAttr attr);
411
412private:
413 mlir::Operation *parentOp;
414 mlir::ConversionPatternRewriter &rewriter;
415 const mlir::TypeConverter *converter;
416};
417
418/// Switches on the type of attribute and calls the appropriate conversion.
419mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp,
420 const mlir::Attribute attr,
421 mlir::ConversionPatternRewriter &rewriter,
422 const mlir::TypeConverter *converter) {
423 CIRAttrToValue valueConverter(parentOp, rewriter, converter);
424 mlir::Value value = valueConverter.visit(attr);
425 if (!value)
426 llvm_unreachable("unhandled attribute type");
427 return value;
428}
429
430void convertSideEffectForCall(mlir::Operation *callOp, bool isNothrow,
431 cir::SideEffect sideEffect,
432 mlir::LLVM::MemoryEffectsAttr &memoryEffect,
433 bool &noUnwind, bool &willReturn,
434 bool &noReturn) {
435 using mlir::LLVM::ModRefInfo;
436
437 switch (sideEffect) {
438 case cir::SideEffect::All:
439 memoryEffect = {};
440 noUnwind = isNothrow;
441 willReturn = false;
442 break;
443
444 case cir::SideEffect::Pure:
445 memoryEffect = mlir::LLVM::MemoryEffectsAttr::get(
446 callOp->getContext(), /*other=*/ModRefInfo::Ref,
447 /*argMem=*/ModRefInfo::Ref,
448 /*inaccessibleMem=*/ModRefInfo::Ref,
449 /*errnoMem=*/ModRefInfo::Ref,
450 /*targetMem0=*/ModRefInfo::Ref,
451 /*targetMem1=*/ModRefInfo::Ref);
452 noUnwind = true;
453 willReturn = true;
454 break;
455
456 case cir::SideEffect::Const:
457 memoryEffect = mlir::LLVM::MemoryEffectsAttr::get(
458 callOp->getContext(), /*other=*/ModRefInfo::NoModRef,
459 /*argMem=*/ModRefInfo::NoModRef,
460 /*inaccessibleMem=*/ModRefInfo::NoModRef,
461 /*errnoMem=*/ModRefInfo::NoModRef,
462 /*targetMem0=*/ModRefInfo::NoModRef,
463 /*targetMem1=*/ModRefInfo::NoModRef);
464 noUnwind = true;
465 willReturn = true;
466 break;
467 }
468
469 noReturn = callOp->hasAttr(CIRDialect::getNoReturnAttrName());
470}
471
472static mlir::LLVM::CallIntrinsicOp
473createCallLLVMIntrinsicOp(mlir::ConversionPatternRewriter &rewriter,
474 mlir::Location loc, const llvm::Twine &intrinsicName,
475 mlir::Type resultTy, mlir::ValueRange operands) {
476 auto intrinsicNameAttr =
477 mlir::StringAttr::get(rewriter.getContext(), intrinsicName);
478 return mlir::LLVM::CallIntrinsicOp::create(rewriter, loc, resultTy,
479 intrinsicNameAttr, operands);
480}
481
482static mlir::LLVM::CallIntrinsicOp replaceOpWithCallLLVMIntrinsicOp(
483 mlir::ConversionPatternRewriter &rewriter, mlir::Operation *op,
484 const llvm::Twine &intrinsicName, mlir::Type resultTy,
485 mlir::ValueRange operands) {
486 mlir::LLVM::CallIntrinsicOp callIntrinOp = createCallLLVMIntrinsicOp(
487 rewriter, op->getLoc(), intrinsicName, resultTy, operands);
488 rewriter.replaceOp(op, callIntrinOp.getOperation());
489 return callIntrinOp;
490}
491
492mlir::LogicalResult CIRToLLVMLLVMIntrinsicCallOpLowering::matchAndRewrite(
493 cir::LLVMIntrinsicCallOp op, OpAdaptor adaptor,
494 mlir::ConversionPatternRewriter &rewriter) const {
495 mlir::Type llvmResTy =
496 getTypeConverter()->convertType(op->getResultTypes()[0]);
497 if (!llvmResTy)
498 return op.emitError("expected LLVM result type");
499 StringRef name = op.getIntrinsicName();
500
501 // Some LLVM intrinsics require ElementType attribute to be attached to
502 // the argument of pointer type. That prevents us from generating LLVM IR
503 // because from LLVM dialect, we have LLVM IR like the below which fails
504 // LLVM IR verification.
505 // %3 = call i64 @llvm.aarch64.ldxr.p0(ptr %2)
506 // The expected LLVM IR should be like
507 // %3 = call i64 @llvm.aarch64.ldxr.p0(ptr elementtype(i32) %2)
508 // TODO(cir): MLIR LLVM dialect should handle this part as CIR has no way
509 // to set LLVM IR attribute.
510 assert(!cir::MissingFeatures::intrinsicElementTypeSupport());
511 replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm." + name, llvmResTy,
512 adaptor.getOperands());
513 return mlir::success();
514}
515
516/// BoolAttr visitor.
517mlir::Value CIRAttrToValue::visitCirAttr(cir::BoolAttr boolAttr) {
518 mlir::Location loc = parentOp->getLoc();
519 mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>());
520 mlir::Value boolVal = mlir::LLVM::ConstantOp::create(
521 rewriter, loc, converter->convertType(boolAttr.getType()),
522 boolAttr.getValue());
523 return emitToMemory(rewriter, layout, boolAttr.getType(), boolVal);
524}
525
526/// IntAttr visitor.
527mlir::Value CIRAttrToValue::visitCirAttr(cir::IntAttr intAttr) {
528 mlir::Location loc = parentOp->getLoc();
529 return mlir::LLVM::ConstantOp::create(
530 rewriter, loc, converter->convertType(intAttr.getType()),
531 intAttr.getValue());
532}
533
534/// FPAttr visitor.
535mlir::Value CIRAttrToValue::visitCirAttr(cir::FPAttr fltAttr) {
536 mlir::Location loc = parentOp->getLoc();
537 return mlir::LLVM::ConstantOp::create(
538 rewriter, loc, converter->convertType(fltAttr.getType()),
539 fltAttr.getValue());
540}
541
542/// ConstComplexAttr visitor.
543mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstComplexAttr complexAttr) {
544 auto complexType = mlir::cast<cir::ComplexType>(complexAttr.getType());
545 mlir::Type complexElemTy = complexType.getElementType();
546 mlir::Type complexElemLLVMTy = converter->convertType(complexElemTy);
547
548 mlir::Attribute components[2];
549 if (const auto intType = mlir::dyn_cast<cir::IntType>(complexElemTy)) {
550 components[0] = rewriter.getIntegerAttr(
551 complexElemLLVMTy,
552 mlir::cast<cir::IntAttr>(complexAttr.getReal()).getValue());
553 components[1] = rewriter.getIntegerAttr(
554 complexElemLLVMTy,
555 mlir::cast<cir::IntAttr>(complexAttr.getImag()).getValue());
556 } else {
557 components[0] = rewriter.getFloatAttr(
558 complexElemLLVMTy,
559 mlir::cast<cir::FPAttr>(complexAttr.getReal()).getValue());
560 components[1] = rewriter.getFloatAttr(
561 complexElemLLVMTy,
562 mlir::cast<cir::FPAttr>(complexAttr.getImag()).getValue());
563 }
564
565 mlir::Location loc = parentOp->getLoc();
566 return mlir::LLVM::ConstantOp::create(
567 rewriter, loc, converter->convertType(complexAttr.getType()),
568 rewriter.getArrayAttr(components));
569}
570
571/// ConstPtrAttr visitor.
572mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstPtrAttr ptrAttr) {
573 mlir::Location loc = parentOp->getLoc();
574 if (ptrAttr.isNullValue()) {
575 return mlir::LLVM::ZeroOp::create(
576 rewriter, loc, converter->convertType(ptrAttr.getType()));
577 }
578 mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>());
579 mlir::Value ptrVal = mlir::LLVM::ConstantOp::create(
580 rewriter, loc,
581 rewriter.getIntegerType(layout.getTypeSizeInBits(ptrAttr.getType())),
582 ptrAttr.getValue().getInt());
583 return mlir::LLVM::IntToPtrOp::create(
584 rewriter, loc, converter->convertType(ptrAttr.getType()), ptrVal);
585}
586
587// ConstArrayAttr visitor
588mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstArrayAttr attr) {
589 mlir::Type llvmTy = converter->convertType(attr.getType());
590 mlir::Location loc = parentOp->getLoc();
591 mlir::Value result;
592
593 if (attr.hasTrailingZeros()) {
594 mlir::Type arrayTy = attr.getType();
595 result = mlir::LLVM::ZeroOp::create(rewriter, loc,
596 converter->convertType(arrayTy));
597 } else {
598 result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);
599 }
600
601 // Iteratively lower each constant element of the array.
602 if (auto arrayAttr = mlir::dyn_cast<mlir::ArrayAttr>(attr.getElts())) {
603 for (auto [idx, elt] : llvm::enumerate(arrayAttr)) {
604 mlir::DataLayout dataLayout(parentOp->getParentOfType<mlir::ModuleOp>());
605 mlir::Value init = visit(elt);
606 result =
607 mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);
608 }
609 } else if (auto strAttr = mlir::dyn_cast<mlir::StringAttr>(attr.getElts())) {
610 // TODO(cir): this diverges from traditional lowering. Normally the string
611 // would be a global constant that is memcopied.
612 auto arrayTy = mlir::dyn_cast<cir::ArrayType>(strAttr.getType());
613 assert(arrayTy && "String attribute must have an array type");
614 mlir::Type eltTy = arrayTy.getElementType();
615 for (auto [idx, elt] : llvm::enumerate(strAttr)) {
616 auto init = mlir::LLVM::ConstantOp::create(
617 rewriter, loc, converter->convertType(eltTy), elt);
618 result =
619 mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);
620 }
621 } else {
622 llvm_unreachable("unexpected ConstArrayAttr elements");
623 }
624
625 return result;
626}
627
628/// ConstRecord visitor.
629mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstRecordAttr constRecord) {
630 const mlir::Type llvmTy = converter->convertType(constRecord.getType());
631 const mlir::Location loc = parentOp->getLoc();
632 mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);
633
634 // Iteratively lower each constant element of the record.
635 for (auto [idx, elt] : llvm::enumerate(constRecord.getMembers())) {
636 mlir::Value init = visit(elt);
637 result =
638 mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);
639 }
640
641 return result;
642}
643
644/// ConstVectorAttr visitor.
645mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstVectorAttr attr) {
646 const mlir::Type llvmTy = converter->convertType(attr.getType());
647 const mlir::Location loc = parentOp->getLoc();
648
649 SmallVector<mlir::Attribute> mlirValues;
650 for (const mlir::Attribute elementAttr : attr.getElts()) {
651 mlir::Attribute mlirAttr;
652 if (auto intAttr = mlir::dyn_cast<cir::IntAttr>(elementAttr)) {
653 mlirAttr = rewriter.getIntegerAttr(
654 converter->convertType(intAttr.getType()), intAttr.getValue());
655 } else if (auto floatAttr = mlir::dyn_cast<cir::FPAttr>(elementAttr)) {
656 mlirAttr = rewriter.getFloatAttr(
657 converter->convertType(floatAttr.getType()), floatAttr.getValue());
658 } else {
659 llvm_unreachable(
660 "vector constant with an element that is neither an int nor a float");
661 }
662 mlirValues.push_back(mlirAttr);
663 }
664
665 return mlir::LLVM::ConstantOp::create(
666 rewriter, loc, llvmTy,
667 mlir::DenseElementsAttr::get(mlir::cast<mlir::ShapedType>(llvmTy),
668 mlirValues));
669}
670
671// GlobalViewAttr visitor.
672mlir::Value CIRAttrToValue::visitCirAttr(cir::GlobalViewAttr globalAttr) {
673 auto moduleOp = parentOp->getParentOfType<mlir::ModuleOp>();
674 mlir::DataLayout dataLayout(moduleOp);
675 mlir::Type sourceType;
676 assert(!cir::MissingFeatures::addressSpace());
677 llvm::StringRef symName;
678 mlir::Operation *sourceSymbol =
679 mlir::SymbolTable::lookupSymbolIn(moduleOp, globalAttr.getSymbol());
680 if (auto llvmSymbol = dyn_cast<mlir::LLVM::GlobalOp>(sourceSymbol)) {
681 sourceType = llvmSymbol.getType();
682 symName = llvmSymbol.getSymName();
683 } else if (auto cirSymbol = dyn_cast<cir::GlobalOp>(sourceSymbol)) {
684 sourceType =
685 convertTypeForMemory(*converter, dataLayout, cirSymbol.getSymType());
686 symName = cirSymbol.getSymName();
687 } else if (auto llvmFun = dyn_cast<mlir::LLVM::LLVMFuncOp>(sourceSymbol)) {
688 sourceType = llvmFun.getFunctionType();
689 symName = llvmFun.getSymName();
690 } else if (auto fun = dyn_cast<cir::FuncOp>(sourceSymbol)) {
691 sourceType = converter->convertType(fun.getFunctionType());
692 symName = fun.getSymName();
693 } else if (auto alias = dyn_cast<mlir::LLVM::AliasOp>(sourceSymbol)) {
694 sourceType = alias.getType();
695 symName = alias.getSymName();
696 } else {
697 llvm_unreachable("Unexpected GlobalOp type");
698 }
699
700 mlir::Location loc = parentOp->getLoc();
701 mlir::Value addrOp = mlir::LLVM::AddressOfOp::create(
702 rewriter, loc, mlir::LLVM::LLVMPointerType::get(rewriter.getContext()),
703 symName);
704
705 if (globalAttr.getIndices()) {
706 llvm::SmallVector<mlir::LLVM::GEPArg> indices;
707
708 if (mlir::isa<mlir::LLVM::LLVMArrayType, mlir::LLVM::LLVMStructType>(
709 sourceType))
710 indices.push_back(0);
711
712 for (mlir::Attribute idx : globalAttr.getIndices()) {
713 auto intAttr = mlir::cast<mlir::IntegerAttr>(idx);
714 indices.push_back(intAttr.getValue().getSExtValue());
715 }
716 mlir::Type resTy = addrOp.getType();
717 mlir::Type eltTy = converter->convertType(sourceType);
718 addrOp =
719 mlir::LLVM::GEPOp::create(rewriter, loc, resTy, eltTy, addrOp, indices,
720 mlir::LLVM::GEPNoWrapFlags::none);
721 }
722
723 // We can have a global view with an integer type in the case of method
724 // pointers. With the Itanium ABI, the #cir.method attribute is lowered to a
725 // #cir.global_view with a pointer-sized integer representing the address of
726 // the method.
727 if (auto intTy = mlir::dyn_cast<cir::IntType>(globalAttr.getType())) {
728 mlir::Type llvmDstTy = converter->convertType(globalAttr.getType());
729 return mlir::LLVM::PtrToIntOp::create(rewriter, parentOp->getLoc(),
730 llvmDstTy, addrOp);
731 }
732
733 if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(globalAttr.getType())) {
734 mlir::Type llvmEltTy =
735 convertTypeForMemory(*converter, dataLayout, ptrTy.getPointee());
736
737 if (llvmEltTy == sourceType)
738 return addrOp;
739
740 mlir::Type llvmDstTy = converter->convertType(globalAttr.getType());
741 return mlir::LLVM::BitcastOp::create(rewriter, parentOp->getLoc(),
742 llvmDstTy, addrOp);
743 }
744
745 llvm_unreachable("Expecting pointer or integer type for GlobalViewAttr");
746}
747
748// TypeInfoAttr visitor.
749mlir::Value CIRAttrToValue::visitCirAttr(cir::TypeInfoAttr typeInfoAttr) {
750 mlir::Type llvmTy = converter->convertType(typeInfoAttr.getType());
751 mlir::Location loc = parentOp->getLoc();
752 mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);
753
754 for (auto [idx, elt] : llvm::enumerate(typeInfoAttr.getData())) {
755 mlir::Value init = visit(elt);
756 result =
757 mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);
758 }
759
760 return result;
761}
762
763/// UndefAttr visitor.
764mlir::Value CIRAttrToValue::visitCirAttr(cir::UndefAttr undefAttr) {
765 mlir::Location loc = parentOp->getLoc();
766 return mlir::LLVM::UndefOp::create(
767 rewriter, loc, converter->convertType(undefAttr.getType()));
768}
769
770// VTableAttr visitor.
771mlir::Value CIRAttrToValue::visitCirAttr(cir::VTableAttr vtableArr) {
772 mlir::Type llvmTy = converter->convertType(vtableArr.getType());
773 mlir::Location loc = parentOp->getLoc();
774 mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);
775
776 for (auto [idx, elt] : llvm::enumerate(vtableArr.getData())) {
777 mlir::Value init = visit(elt);
778 result =
779 mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);
780 }
781
782 return result;
783}
784
785/// ZeroAttr visitor.
786mlir::Value CIRAttrToValue::visitCirAttr(cir::ZeroAttr attr) {
787 mlir::Location loc = parentOp->getLoc();
788 return mlir::LLVM::ZeroOp::create(rewriter, loc,
789 converter->convertType(attr.getType()));
790}
791
792// This class handles rewriting initializer attributes for types that do not
793// require region initialization.
794class GlobalInitAttrRewriter {
795public:
796 GlobalInitAttrRewriter(mlir::Type type,
797 mlir::ConversionPatternRewriter &rewriter)
798 : llvmType(type), rewriter(rewriter) {}
799
800 mlir::Attribute visit(mlir::Attribute attr) {
801 return llvm::TypeSwitch<mlir::Attribute, mlir::Attribute>(attr)
802 .Case<cir::IntAttr, cir::FPAttr, cir::BoolAttr>(
803 [&](auto attrT) { return visitCirAttr(attrT); })
804 .Default([&](auto attrT) { return mlir::Attribute(); });
805 }
806
807 mlir::Attribute visitCirAttr(cir::IntAttr attr) {
808 return rewriter.getIntegerAttr(llvmType, attr.getValue());
809 }
810
811 mlir::Attribute visitCirAttr(cir::FPAttr attr) {
812 return rewriter.getFloatAttr(llvmType, attr.getValue());
813 }
814
815 mlir::Attribute visitCirAttr(cir::BoolAttr attr) {
816 return rewriter.getBoolAttr(attr.getValue());
817 }
818
819private:
820 mlir::Type llvmType;
821 mlir::ConversionPatternRewriter &rewriter;
822};
823
824// This pass requires the CIR to be in a "flat" state. All blocks in each
825// function must belong to the parent region. Once scopes and control flow
826// are implemented in CIR, a pass will be run before this one to flatten
827// the CIR and get it into the state that this pass requires.
828struct ConvertCIRToLLVMPass
829 : public mlir::PassWrapper<ConvertCIRToLLVMPass,
830 mlir::OperationPass<mlir::ModuleOp>> {
831 void getDependentDialects(mlir::DialectRegistry &registry) const override {
832 registry.insert<mlir::BuiltinDialect, mlir::DLTIDialect,
833 mlir::LLVM::LLVMDialect, mlir::func::FuncDialect>();
834 }
835 void runOnOperation() final;
836
837 void processCIRAttrs(mlir::ModuleOp module);
838
839 void resolveBlockAddressOp(LLVMBlockAddressInfo &blockInfoAddr);
840
841 StringRef getDescription() const override {
842 return "Convert the prepared CIR dialect module to LLVM dialect";
843 }
844
845 StringRef getArgument() const override { return "cir-flat-to-llvm"; }
846};
847
848mlir::LogicalResult CIRToLLVMACosOpLowering::matchAndRewrite(
849 cir::ACosOp op, OpAdaptor adaptor,
850 mlir::ConversionPatternRewriter &rewriter) const {
851 mlir::Type resTy = typeConverter->convertType(op.getType());
852 rewriter.replaceOpWithNewOp<mlir::LLVM::ACosOp>(op, resTy,
853 adaptor.getOperands()[0]);
854 return mlir::success();
855}
856
857mlir::LogicalResult CIRToLLVMASinOpLowering::matchAndRewrite(
858 cir::ASinOp op, OpAdaptor adaptor,
859 mlir::ConversionPatternRewriter &rewriter) const {
860 mlir::Type resTy = typeConverter->convertType(op.getType());
861 rewriter.replaceOpWithNewOp<mlir::LLVM::ASinOp>(op, resTy, adaptor.getSrc());
862 return mlir::success();
863}
864
865mlir::LogicalResult CIRToLLVMIsFPClassOpLowering::matchAndRewrite(
866 cir::IsFPClassOp op, OpAdaptor adaptor,
867 mlir::ConversionPatternRewriter &rewriter) const {
868 mlir::Value src = adaptor.getSrc();
869 cir::FPClassTest flags = adaptor.getFlags();
870 mlir::IntegerType retTy = rewriter.getI1Type();
871
872 rewriter.replaceOpWithNewOp<mlir::LLVM::IsFPClass>(
873 op, retTy, src, static_cast<uint32_t>(flags));
874 return mlir::success();
875}
876
877mlir::LogicalResult CIRToLLVMSignBitOpLowering::matchAndRewrite(
878 cir::SignBitOp op, OpAdaptor adaptor,
879 mlir::ConversionPatternRewriter &rewriter) const {
880 assert(!cir::MissingFeatures::isPPC_FP128Ty());
881
882 mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>());
883 int width = layout.getTypeSizeInBits(op.getInput().getType());
884 if (auto longDoubleType =
885 mlir::dyn_cast<cir::LongDoubleType>(op.getInput().getType())) {
886 if (mlir::isa<cir::FP80Type>(longDoubleType.getUnderlying())) {
887 // If the underlying type of LongDouble is FP80Type,
888 // DataLayout::getTypeSizeInBits returns 128.
889 // See https://github.com/llvm/clangir/issues/1057.
890 // Set the width to 80 manually.
891 width = 80;
892 }
893 }
894 mlir::Type intTy = mlir::IntegerType::get(rewriter.getContext(), width);
895 auto bitcast = mlir::LLVM::BitcastOp::create(rewriter, op->getLoc(), intTy,
896 adaptor.getInput());
897
898 auto zero = mlir::LLVM::ConstantOp::create(rewriter, op->getLoc(), intTy, 0);
899 auto cmpResult = mlir::LLVM::ICmpOp::create(rewriter, op.getLoc(),
900 mlir::LLVM::ICmpPredicate::slt,
901 bitcast.getResult(), zero);
902 rewriter.replaceOp(op, cmpResult);
903 return mlir::success();
904}
905
906mlir::LogicalResult CIRToLLVMAssumeOpLowering::matchAndRewrite(
907 cir::AssumeOp op, OpAdaptor adaptor,
908 mlir::ConversionPatternRewriter &rewriter) const {
909 auto cond = adaptor.getPredicate();
910 rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(op, cond);
911 return mlir::success();
912}
913
914mlir::LogicalResult CIRToLLVMAssumeAlignedOpLowering::matchAndRewrite(
915 cir::AssumeAlignedOp op, OpAdaptor adaptor,
916 mlir::ConversionPatternRewriter &rewriter) const {
917 SmallVector<mlir::Value, 3> opBundleArgs{adaptor.getPointer()};
918
919 auto alignment = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
920 adaptor.getAlignmentAttr());
921 opBundleArgs.push_back(alignment);
922
923 if (mlir::Value offset = adaptor.getOffset())
924 opBundleArgs.push_back(offset);
925
926 auto cond = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
927 rewriter.getI1Type(), 1);
928 mlir::LLVM::AssumeOp::create(rewriter, op.getLoc(), cond, "align",
929 opBundleArgs);
930
931 // The llvm.assume operation does not have a result, so we need to replace
932 // all uses of this cir.assume_aligned operation with the input ptr itself.
933 rewriter.replaceOp(op, adaptor.getPointer());
934 return mlir::success();
935}
936
937mlir::LogicalResult CIRToLLVMAssumeSepStorageOpLowering::matchAndRewrite(
938 cir::AssumeSepStorageOp op, OpAdaptor adaptor,
939 mlir::ConversionPatternRewriter &rewriter) const {
940 auto cond = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
941 rewriter.getI1Type(), 1);
942 rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(
943 op, cond, mlir::LLVM::AssumeSeparateStorageTag{}, adaptor.getPtr1(),
944 adaptor.getPtr2());
945 return mlir::success();
946}
947
948static mlir::LLVM::AtomicOrdering
949getLLVMMemOrder(std::optional<cir::MemOrder> memorder) {
950 if (!memorder)
951 return mlir::LLVM::AtomicOrdering::not_atomic;
952 switch (*memorder) {
953 case cir::MemOrder::Relaxed:
954 return mlir::LLVM::AtomicOrdering::monotonic;
955 case cir::MemOrder::Consume:
956 case cir::MemOrder::Acquire:
957 return mlir::LLVM::AtomicOrdering::acquire;
958 case cir::MemOrder::Release:
959 return mlir::LLVM::AtomicOrdering::release;
960 case cir::MemOrder::AcquireRelease:
961 return mlir::LLVM::AtomicOrdering::acq_rel;
962 case cir::MemOrder::SequentiallyConsistent:
963 return mlir::LLVM::AtomicOrdering::seq_cst;
964 }
965 llvm_unreachable("unknown memory order");
966}
967
968static llvm::StringRef getLLVMSyncScope(cir::SyncScopeKind syncScope) {
969 return syncScope == cir::SyncScopeKind::SingleThread ? "singlethread" : "";
970}
971
972static std::optional<llvm::StringRef>
973getLLVMSyncScope(std::optional<cir::SyncScopeKind> syncScope) {
974 if (syncScope.has_value())
975 return getLLVMSyncScope(*syncScope);
976 return std::nullopt;
977}
978
979mlir::LogicalResult CIRToLLVMAtomicCmpXchgOpLowering::matchAndRewrite(
980 cir::AtomicCmpXchgOp op, OpAdaptor adaptor,
981 mlir::ConversionPatternRewriter &rewriter) const {
982 mlir::Value expected = adaptor.getExpected();
983 mlir::Value desired = adaptor.getDesired();
984
985 auto cmpxchg = mlir::LLVM::AtomicCmpXchgOp::create(
986 rewriter, op.getLoc(), adaptor.getPtr(), expected, desired,
987 getLLVMMemOrder(adaptor.getSuccOrder()),
988 getLLVMMemOrder(adaptor.getFailOrder()),
989 getLLVMSyncScope(op.getSyncScope()));
990
991 cmpxchg.setAlignment(adaptor.getAlignment());
992 cmpxchg.setWeak(adaptor.getWeak());
993 cmpxchg.setVolatile_(adaptor.getIsVolatile());
994
995 // Check result and apply stores accordingly.
996 auto old = mlir::LLVM::ExtractValueOp::create(rewriter, op.getLoc(),
997 cmpxchg.getResult(), 0);
998 auto cmp = mlir::LLVM::ExtractValueOp::create(rewriter, op.getLoc(),
999 cmpxchg.getResult(), 1);
1000
1001 rewriter.replaceOp(op, {old, cmp});
1002 return mlir::success();
1003}
1004
1005mlir::LogicalResult CIRToLLVMAtomicXchgOpLowering::matchAndRewrite(
1006 cir::AtomicXchgOp op, OpAdaptor adaptor,
1007 mlir::ConversionPatternRewriter &rewriter) const {
1008 assert(!cir::MissingFeatures::atomicSyncScopeID());
1009 mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(adaptor.getMemOrder());
1010 llvm::StringRef llvmSyncScope = getLLVMSyncScope(adaptor.getSyncScope());
1011 rewriter.replaceOpWithNewOp<mlir::LLVM::AtomicRMWOp>(
1012 op, mlir::LLVM::AtomicBinOp::xchg, adaptor.getPtr(), adaptor.getVal(),
1013 llvmOrder, llvmSyncScope);
1014 return mlir::success();
1015}
1016
1017mlir::LogicalResult CIRToLLVMAtomicTestAndSetOpLowering::matchAndRewrite(
1018 cir::AtomicTestAndSetOp op, OpAdaptor adaptor,
1019 mlir::ConversionPatternRewriter &rewriter) const {
1020 assert(!cir::MissingFeatures::atomicSyncScopeID());
1021
1022 mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(op.getMemOrder());
1023
1024 auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
1025 rewriter.getI8Type(), 1);
1026 auto rmw = mlir::LLVM::AtomicRMWOp::create(
1027 rewriter, op.getLoc(), mlir::LLVM::AtomicBinOp::xchg, adaptor.getPtr(),
1028 one, llvmOrder, /*syncscope=*/llvm::StringRef(),
1029 adaptor.getAlignment().value_or(0), op.getIsVolatile());
1030
1031 auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
1032 rewriter.getI8Type(), 0);
1033 auto cmp = mlir::LLVM::ICmpOp::create(
1034 rewriter, op.getLoc(), mlir::LLVM::ICmpPredicate::ne, rmw, zero);
1035
1036 rewriter.replaceOp(op, cmp);
1037 return mlir::success();
1038}
1039
1040mlir::LogicalResult CIRToLLVMAtomicClearOpLowering::matchAndRewrite(
1041 cir::AtomicClearOp op, OpAdaptor adaptor,
1042 mlir::ConversionPatternRewriter &rewriter) const {
1043 assert(!cir::MissingFeatures::atomicSyncScopeID());
1044
1045 mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(op.getMemOrder());
1046 auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
1047 rewriter.getI8Type(), 0);
1048 auto store = mlir::LLVM::StoreOp::create(
1049 rewriter, op.getLoc(), zero, adaptor.getPtr(),
1050 adaptor.getAlignment().value_or(0), op.getIsVolatile(),
1051 /*isNonTemporal=*/false, /*isInvariantGroup=*/false, llvmOrder);
1052
1053 rewriter.replaceOp(op, store);
1054 return mlir::success();
1055}
1056
1057mlir::LogicalResult CIRToLLVMAtomicFenceOpLowering::matchAndRewrite(
1058 cir::AtomicFenceOp op, OpAdaptor adaptor,
1059 mlir::ConversionPatternRewriter &rewriter) const {
1060 mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(adaptor.getOrdering());
1061
1062 auto fence = mlir::LLVM::FenceOp::create(rewriter, op.getLoc(), llvmOrder);
1063 fence.setSyncscope(getLLVMSyncScope(adaptor.getSyncscope()));
1064
1065 rewriter.replaceOp(op, fence);
1066
1067 return mlir::success();
1068}
1069
1070static mlir::LLVM::AtomicBinOp
1071getLLVMAtomicBinOp(cir::AtomicFetchKind k, bool isInt, bool isSignedInt) {
1072 switch (k) {
1073 case cir::AtomicFetchKind::Add:
1074 return isInt ? mlir::LLVM::AtomicBinOp::add : mlir::LLVM::AtomicBinOp::fadd;
1075 case cir::AtomicFetchKind::Sub:
1076 return isInt ? mlir::LLVM::AtomicBinOp::sub : mlir::LLVM::AtomicBinOp::fsub;
1077 case cir::AtomicFetchKind::And:
1078 return mlir::LLVM::AtomicBinOp::_and;
1079 case cir::AtomicFetchKind::Xor:
1080 return mlir::LLVM::AtomicBinOp::_xor;
1081 case cir::AtomicFetchKind::Or:
1082 return mlir::LLVM::AtomicBinOp::_or;
1083 case cir::AtomicFetchKind::Nand:
1084 return mlir::LLVM::AtomicBinOp::nand;
1085 case cir::AtomicFetchKind::Max: {
1086 if (!isInt)
1087 return mlir::LLVM::AtomicBinOp::fmax;
1088 return isSignedInt ? mlir::LLVM::AtomicBinOp::max
1089 : mlir::LLVM::AtomicBinOp::umax;
1090 }
1091 case cir::AtomicFetchKind::Min: {
1092 if (!isInt)
1093 return mlir::LLVM::AtomicBinOp::fmin;
1094 return isSignedInt ? mlir::LLVM::AtomicBinOp::min
1095 : mlir::LLVM::AtomicBinOp::umin;
1096 }
1097 case cir::AtomicFetchKind::UIncWrap:
1098 return mlir::LLVM::AtomicBinOp::uinc_wrap;
1099 case cir::AtomicFetchKind::UDecWrap:
1100 return mlir::LLVM::AtomicBinOp::udec_wrap;
1101 }
1102 llvm_unreachable("Unknown atomic fetch opcode");
1103}
1104
1105static llvm::StringLiteral getLLVMBinopForPostAtomic(cir::AtomicFetchKind k,
1106 bool isInt) {
1107 switch (k) {
1108 case cir::AtomicFetchKind::Add:
1109 return isInt ? mlir::LLVM::AddOp::getOperationName()
1110 : mlir::LLVM::FAddOp::getOperationName();
1111 case cir::AtomicFetchKind::Sub:
1112 return isInt ? mlir::LLVM::SubOp::getOperationName()
1113 : mlir::LLVM::FSubOp::getOperationName();
1114 case cir::AtomicFetchKind::And:
1115 return mlir::LLVM::AndOp::getOperationName();
1116 case cir::AtomicFetchKind::Xor:
1117 return mlir::LLVM::XOrOp::getOperationName();
1118 case cir::AtomicFetchKind::Or:
1119 return mlir::LLVM::OrOp::getOperationName();
1120 case cir::AtomicFetchKind::Nand:
1121 // There's no nand binop in LLVM, this is later fixed with a not.
1122 return mlir::LLVM::AndOp::getOperationName();
1123 case cir::AtomicFetchKind::Max:
1124 case cir::AtomicFetchKind::Min:
1125 llvm_unreachable("handled in buildMinMaxPostOp");
1126 case cir::AtomicFetchKind::UIncWrap:
1127 case cir::AtomicFetchKind::UDecWrap:
1128 llvm_unreachable("uinc_wrap and udec_wrap are always fetch_first");
1129 }
1130 llvm_unreachable("Unknown atomic fetch opcode");
1131}
1132
1133mlir::Value CIRToLLVMAtomicFetchOpLowering::buildPostOp(
1134 cir::AtomicFetchOp op, OpAdaptor adaptor,
1135 mlir::ConversionPatternRewriter &rewriter, mlir::Value rmwVal,
1136 bool isInt) const {
1137 SmallVector<mlir::Value> atomicOperands = {rmwVal, adaptor.getVal()};
1138 SmallVector<mlir::Type> atomicResTys = {rmwVal.getType()};
1139 return rewriter
1140 .create(op.getLoc(),
1141 rewriter.getStringAttr(
1142 getLLVMBinopForPostAtomic(op.getBinop(), isInt)),
1143 atomicOperands, atomicResTys, {})
1144 ->getResult(0);
1145}
1146
1147mlir::Value CIRToLLVMAtomicFetchOpLowering::buildMinMaxPostOp(
1148 cir::AtomicFetchOp op, OpAdaptor adaptor,
1149 mlir::ConversionPatternRewriter &rewriter, mlir::Value rmwVal, bool isInt,
1150 bool isSigned) const {
1151 mlir::Location loc = op.getLoc();
1152
1153 if (!isInt) {
1154 if (op.getBinop() == cir::AtomicFetchKind::Max)
1155 return mlir::LLVM::MaxNumOp::create(rewriter, loc, rmwVal,
1156 adaptor.getVal());
1157 return mlir::LLVM::MinNumOp::create(rewriter, loc, rmwVal,
1158 adaptor.getVal());
1159 }
1160
1161 mlir::LLVM::ICmpPredicate pred;
1162 if (op.getBinop() == cir::AtomicFetchKind::Max) {
1163 pred = isSigned ? mlir::LLVM::ICmpPredicate::sgt
1164 : mlir::LLVM::ICmpPredicate::ugt;
1165 } else { // Min
1166 pred = isSigned ? mlir::LLVM::ICmpPredicate::slt
1167 : mlir::LLVM::ICmpPredicate::ult;
1168 }
1169 mlir::Value cmp = mlir::LLVM::ICmpOp::create(
1170 rewriter, loc,
1171 mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(), pred), rmwVal,
1172 adaptor.getVal());
1173 return mlir::LLVM::SelectOp::create(rewriter, loc, cmp, rmwVal,
1174 adaptor.getVal());
1175}
1176
1177mlir::LogicalResult CIRToLLVMAtomicFetchOpLowering::matchAndRewrite(
1178 cir::AtomicFetchOp op, OpAdaptor adaptor,
1179 mlir::ConversionPatternRewriter &rewriter) const {
1180 bool isInt = false;
1181 bool isSignedInt = false;
1182 if (auto intTy = mlir::dyn_cast<cir::IntType>(op.getVal().getType())) {
1183 isInt = true;
1184 isSignedInt = intTy.isSigned();
1185 } else if (mlir::isa<cir::SingleType, cir::DoubleType>(
1186 op.getVal().getType())) {
1187 isInt = false;
1188 } else {
1189 return op.emitError() << "Unsupported type: " << op.getVal().getType();
1190 }
1191
1192 mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(op.getMemOrder());
1193 llvm::StringRef llvmSyncScope = getLLVMSyncScope(op.getSyncScope());
1194 mlir::LLVM::AtomicBinOp llvmBinOp =
1195 getLLVMAtomicBinOp(op.getBinop(), isInt, isSignedInt);
1196 auto rmwVal = mlir::LLVM::AtomicRMWOp::create(
1197 rewriter, op.getLoc(), llvmBinOp, adaptor.getPtr(), adaptor.getVal(),
1198 llvmOrder, llvmSyncScope);
1199
1200 mlir::Value result = rmwVal.getResult();
1201 if (!op.getFetchFirst()) {
1202 if (op.getBinop() == cir::AtomicFetchKind::Max ||
1203 op.getBinop() == cir::AtomicFetchKind::Min)
1204 result = buildMinMaxPostOp(op, adaptor, rewriter, rmwVal.getRes(), isInt,
1205 isSignedInt);
1206 else
1207 result = buildPostOp(op, adaptor, rewriter, rmwVal.getRes(), isInt);
1208
1209 // Compensate lack of nand binop in LLVM IR.
1210 if (op.getBinop() == cir::AtomicFetchKind::Nand) {
1211 auto negOne = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
1212 result.getType(), -1);
1213 result = mlir::LLVM::XOrOp::create(rewriter, op.getLoc(), result, negOne);
1214 }
1215 }
1216
1217 rewriter.replaceOp(op, result);
1218 return mlir::success();
1219}
1220
1221mlir::LogicalResult CIRToLLVMBitClrsbOpLowering::matchAndRewrite(
1222 cir::BitClrsbOp op, OpAdaptor adaptor,
1223 mlir::ConversionPatternRewriter &rewriter) const {
1224 auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
1225 adaptor.getInput().getType(), 0);
1226 auto isNeg = mlir::LLVM::ICmpOp::create(
1227 rewriter, op.getLoc(),
1228 mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(),
1229 mlir::LLVM::ICmpPredicate::slt),
1230 adaptor.getInput(), zero);
1231
1232 auto negOne = mlir::LLVM::ConstantOp::create(
1233 rewriter, op.getLoc(), adaptor.getInput().getType(), -1);
1234 auto flipped = mlir::LLVM::XOrOp::create(rewriter, op.getLoc(),
1235 adaptor.getInput(), negOne);
1236
1237 auto select = mlir::LLVM::SelectOp::create(rewriter, op.getLoc(), isNeg,
1238 flipped, adaptor.getInput());
1239
1240 auto resTy = getTypeConverter()->convertType(op.getType());
1241 auto clz = mlir::LLVM::CountLeadingZerosOp::create(
1242 rewriter, op.getLoc(), resTy, select, /*is_zero_poison=*/false);
1243
1244 auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1);
1245 auto res = mlir::LLVM::SubOp::create(rewriter, op.getLoc(), clz, one,
1246 mlir::LLVM::IntegerOverflowFlags::nuw);
1247 rewriter.replaceOp(op, res);
1248
1249 return mlir::LogicalResult::success();
1250}
1251
1252mlir::LogicalResult CIRToLLVMBitClzOpLowering::matchAndRewrite(
1253 cir::BitClzOp op, OpAdaptor adaptor,
1254 mlir::ConversionPatternRewriter &rewriter) const {
1255 auto resTy = getTypeConverter()->convertType(op.getType());
1256 auto llvmOp = mlir::LLVM::CountLeadingZerosOp::create(
1257 rewriter, op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero());
1258 rewriter.replaceOp(op, llvmOp);
1259 return mlir::LogicalResult::success();
1260}
1261
1262mlir::LogicalResult CIRToLLVMBitCtzOpLowering::matchAndRewrite(
1263 cir::BitCtzOp op, OpAdaptor adaptor,
1264 mlir::ConversionPatternRewriter &rewriter) const {
1265 auto resTy = getTypeConverter()->convertType(op.getType());
1266 auto llvmOp = mlir::LLVM::CountTrailingZerosOp::create(
1267 rewriter, op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero());
1268 rewriter.replaceOp(op, llvmOp);
1269 return mlir::LogicalResult::success();
1270}
1271
1272mlir::LogicalResult CIRToLLVMBitFfsOpLowering::matchAndRewrite(
1273 cir::BitFfsOp op, OpAdaptor adaptor,
1274 mlir::ConversionPatternRewriter &rewriter) const {
1275 auto resTy = getTypeConverter()->convertType(op.getType());
1276 auto ctz = mlir::LLVM::CountTrailingZerosOp::create(rewriter, op.getLoc(),
1277 resTy, adaptor.getInput(),
1278 /*is_zero_poison=*/true);
1279
1280 auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1);
1281 auto ctzAddOne = mlir::LLVM::AddOp::create(rewriter, op.getLoc(), ctz, one);
1282
1283 auto zeroInputTy = mlir::LLVM::ConstantOp::create(
1284 rewriter, op.getLoc(), adaptor.getInput().getType(), 0);
1285 auto isZero = mlir::LLVM::ICmpOp::create(
1286 rewriter, op.getLoc(),
1287 mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(),
1288 mlir::LLVM::ICmpPredicate::eq),
1289 adaptor.getInput(), zeroInputTy);
1290
1291 auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 0);
1292 auto res = mlir::LLVM::SelectOp::create(rewriter, op.getLoc(), isZero, zero,
1293 ctzAddOne);
1294 rewriter.replaceOp(op, res);
1295
1296 return mlir::LogicalResult::success();
1297}
1298
1299mlir::LogicalResult CIRToLLVMBitParityOpLowering::matchAndRewrite(
1300 cir::BitParityOp op, OpAdaptor adaptor,
1301 mlir::ConversionPatternRewriter &rewriter) const {
1302 auto resTy = getTypeConverter()->convertType(op.getType());
1303 auto popcnt = mlir::LLVM::CtPopOp::create(rewriter, op.getLoc(), resTy,
1304 adaptor.getInput());
1305
1306 auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1);
1307 auto popcntMod2 =
1308 mlir::LLVM::AndOp::create(rewriter, op.getLoc(), popcnt, one);
1309 rewriter.replaceOp(op, popcntMod2);
1310
1311 return mlir::LogicalResult::success();
1312}
1313
1314mlir::LogicalResult CIRToLLVMBitPopcountOpLowering::matchAndRewrite(
1315 cir::BitPopcountOp op, OpAdaptor adaptor,
1316 mlir::ConversionPatternRewriter &rewriter) const {
1317 auto resTy = getTypeConverter()->convertType(op.getType());
1318 auto llvmOp = mlir::LLVM::CtPopOp::create(rewriter, op.getLoc(), resTy,
1319 adaptor.getInput());
1320 rewriter.replaceOp(op, llvmOp);
1321 return mlir::LogicalResult::success();
1322}
1323
1324mlir::LogicalResult CIRToLLVMBitReverseOpLowering::matchAndRewrite(
1325 cir::BitReverseOp op, OpAdaptor adaptor,
1326 mlir::ConversionPatternRewriter &rewriter) const {
1327 rewriter.replaceOpWithNewOp<mlir::LLVM::BitReverseOp>(op, adaptor.getInput());
1328 return mlir::success();
1329}
1330
1331mlir::LogicalResult CIRToLLVMBrCondOpLowering::matchAndRewrite(
1332 cir::BrCondOp brOp, OpAdaptor adaptor,
1333 mlir::ConversionPatternRewriter &rewriter) const {
1334 // When ZExtOp is implemented, we'll need to check if the condition is a
1335 // ZExtOp and if so, delete it if it has a single use.
1336 assert(!cir::MissingFeatures::zextOp());
1337
1338 mlir::Value i1Condition = adaptor.getCond();
1339
1340 rewriter.replaceOpWithNewOp<mlir::LLVM::CondBrOp>(
1341 brOp, i1Condition, brOp.getDestTrue(), adaptor.getDestOperandsTrue(),
1342 brOp.getDestFalse(), adaptor.getDestOperandsFalse());
1343
1344 return mlir::success();
1345}
1346
1347mlir::LogicalResult CIRToLLVMByteSwapOpLowering::matchAndRewrite(
1348 cir::ByteSwapOp op, OpAdaptor adaptor,
1349 mlir::ConversionPatternRewriter &rewriter) const {
1350 rewriter.replaceOpWithNewOp<mlir::LLVM::ByteSwapOp>(op, adaptor.getInput());
1351 return mlir::LogicalResult::success();
1352}
1353
1354mlir::Type CIRToLLVMCastOpLowering::convertTy(mlir::Type ty) const {
1355 return getTypeConverter()->convertType(ty);
1356}
1357
1358mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
1359 cir::CastOp castOp, OpAdaptor adaptor,
1360 mlir::ConversionPatternRewriter &rewriter) const {
1361 // For arithmetic conversions, LLVM IR uses the same instruction to convert
1362 // both individual scalars and entire vectors. This lowering pass handles
1363 // both situations.
1364
1365 switch (castOp.getKind()) {
1366 case cir::CastKind::array_to_ptrdecay: {
1367 const auto ptrTy = mlir::cast<cir::PointerType>(castOp.getType());
1368 mlir::Value sourceValue = adaptor.getSrc();
1369 mlir::Type targetType = convertTy(ptrTy);
1370 mlir::Type elementTy = convertTypeForMemory(*getTypeConverter(), dataLayout,
1371 ptrTy.getPointee());
1372 llvm::SmallVector<mlir::LLVM::GEPArg> offset{0};
1373 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
1374 castOp, targetType, elementTy, sourceValue, offset);
1375 break;
1376 }
1377 case cir::CastKind::int_to_bool: {
1378 mlir::Value llvmSrcVal = adaptor.getSrc();
1379 mlir::Value zeroInt = mlir::LLVM::ConstantOp::create(
1380 rewriter, castOp.getLoc(), llvmSrcVal.getType(), 0);
1381 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
1382 castOp, mlir::LLVM::ICmpPredicate::ne, llvmSrcVal, zeroInt);
1383 break;
1384 }
1385 case cir::CastKind::integral: {
1386 mlir::Type srcType = castOp.getSrc().getType();
1387 mlir::Type dstType = castOp.getType();
1388 mlir::Value llvmSrcVal = adaptor.getSrc();
1389 mlir::Type llvmDstType = getTypeConverter()->convertType(dstType);
1390 cir::IntType srcIntType =
1391 mlir::cast<cir::IntType>(elementTypeIfVector(srcType));
1392 cir::IntType dstIntType =
1393 mlir::cast<cir::IntType>(elementTypeIfVector(dstType));
1394 rewriter.replaceOp(castOp, getLLVMIntCast(rewriter, llvmSrcVal, llvmDstType,
1395 srcIntType.isUnsigned(),
1396 srcIntType.getWidth(),
1397 dstIntType.getWidth()));
1398 break;
1399 }
1400 case cir::CastKind::floating: {
1401 mlir::Value llvmSrcVal = adaptor.getSrc();
1402 mlir::Type llvmDstTy = getTypeConverter()->convertType(castOp.getType());
1403
1404 mlir::Type srcTy = elementTypeIfVector(castOp.getSrc().getType());
1405 mlir::Type dstTy = elementTypeIfVector(castOp.getType());
1406
1407 if (!mlir::isa<cir::FPTypeInterface>(dstTy) ||
1408 !mlir::isa<cir::FPTypeInterface>(srcTy))
1409 return castOp.emitError() << "NYI cast from " << srcTy << " to " << dstTy;
1410
1411 auto getFloatWidth = [](mlir::Type ty) -> unsigned {
1412 return mlir::cast<cir::FPTypeInterface>(ty).getWidth();
1413 };
1414
1415 if (getFloatWidth(srcTy) > getFloatWidth(dstTy))
1416 rewriter.replaceOpWithNewOp<mlir::LLVM::FPTruncOp>(castOp, llvmDstTy,
1417 llvmSrcVal);
1418 else
1419 rewriter.replaceOpWithNewOp<mlir::LLVM::FPExtOp>(castOp, llvmDstTy,
1420 llvmSrcVal);
1421 return mlir::success();
1422 }
1423 case cir::CastKind::int_to_ptr: {
1424 auto dstTy = mlir::cast<cir::PointerType>(castOp.getType());
1425 mlir::Value llvmSrcVal = adaptor.getSrc();
1426 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
1427 rewriter.replaceOpWithNewOp<mlir::LLVM::IntToPtrOp>(castOp, llvmDstTy,
1428 llvmSrcVal);
1429 return mlir::success();
1430 }
1431 case cir::CastKind::ptr_to_int: {
1432 auto dstTy = mlir::cast<cir::IntType>(castOp.getType());
1433 mlir::Value llvmSrcVal = adaptor.getSrc();
1434 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
1435 rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(castOp, llvmDstTy,
1436 llvmSrcVal);
1437 return mlir::success();
1438 }
1439 case cir::CastKind::float_to_bool: {
1440 mlir::Value llvmSrcVal = adaptor.getSrc();
1441 auto kind = mlir::LLVM::FCmpPredicate::une;
1442
1443 // Check if float is not equal to zero.
1444 auto zeroFloat = mlir::LLVM::ConstantOp::create(
1445 rewriter, castOp.getLoc(), llvmSrcVal.getType(),
1446 mlir::FloatAttr::get(llvmSrcVal.getType(), 0.0));
1447
1448 // Extend comparison result to either bool (C++) or int (C).
1449 rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(castOp, kind, llvmSrcVal,
1450 zeroFloat);
1451
1452 return mlir::success();
1453 }
1454 case cir::CastKind::bool_to_int: {
1455 auto dstTy = mlir::cast<cir::IntType>(castOp.getType());
1456 mlir::Value llvmSrcVal = adaptor.getSrc();
1457 auto llvmSrcTy = mlir::cast<mlir::IntegerType>(llvmSrcVal.getType());
1458 auto llvmDstTy =
1459 mlir::cast<mlir::IntegerType>(getTypeConverter()->convertType(dstTy));
1460
1461 if (llvmSrcTy.getWidth() == llvmDstTy.getWidth())
1462 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(castOp, llvmDstTy,
1463 llvmSrcVal);
1464 else
1465 rewriter.replaceOpWithNewOp<mlir::LLVM::ZExtOp>(castOp, llvmDstTy,
1466 llvmSrcVal);
1467 return mlir::success();
1468 }
1469 case cir::CastKind::bool_to_float: {
1470 mlir::Type dstTy = castOp.getType();
1471 mlir::Value llvmSrcVal = adaptor.getSrc();
1472 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
1473 rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(castOp, llvmDstTy,
1474 llvmSrcVal);
1475 return mlir::success();
1476 }
1477 case cir::CastKind::int_to_float: {
1478 mlir::Type dstTy = castOp.getType();
1479 mlir::Value llvmSrcVal = adaptor.getSrc();
1480 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
1481 if (mlir::cast<cir::IntType>(elementTypeIfVector(castOp.getSrc().getType()))
1482 .isSigned())
1483 rewriter.replaceOpWithNewOp<mlir::LLVM::SIToFPOp>(castOp, llvmDstTy,
1484 llvmSrcVal);
1485 else
1486 rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(castOp, llvmDstTy,
1487 llvmSrcVal);
1488 return mlir::success();
1489 }
1490 case cir::CastKind::float_to_int: {
1491 mlir::Type dstTy = castOp.getType();
1492 mlir::Value llvmSrcVal = adaptor.getSrc();
1493 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
1494 if (mlir::cast<cir::IntType>(elementTypeIfVector(castOp.getType()))
1495 .isSigned())
1496 rewriter.replaceOpWithNewOp<mlir::LLVM::FPToSIOp>(castOp, llvmDstTy,
1497 llvmSrcVal);
1498 else
1499 rewriter.replaceOpWithNewOp<mlir::LLVM::FPToUIOp>(castOp, llvmDstTy,
1500 llvmSrcVal);
1501 return mlir::success();
1502 }
1503 case cir::CastKind::bitcast: {
1504 mlir::Type dstTy = castOp.getType();
1505 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
1506
1507 assert(!MissingFeatures::cxxABI());
1508 assert(!MissingFeatures::dataMemberType());
1509
1510 mlir::Value llvmSrcVal = adaptor.getSrc();
1511 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(castOp, llvmDstTy,
1512 llvmSrcVal);
1513 return mlir::success();
1514 }
1515 case cir::CastKind::ptr_to_bool: {
1516 mlir::Value llvmSrcVal = adaptor.getSrc();
1517 mlir::Value zeroPtr = mlir::LLVM::ZeroOp::create(rewriter, castOp.getLoc(),
1518 llvmSrcVal.getType());
1519 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
1520 castOp, mlir::LLVM::ICmpPredicate::ne, llvmSrcVal, zeroPtr);
1521 break;
1522 }
1523 case cir::CastKind::address_space: {
1524 mlir::Type dstTy = castOp.getType();
1525 mlir::Value llvmSrcVal = adaptor.getSrc();
1526 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
1527 rewriter.replaceOpWithNewOp<mlir::LLVM::AddrSpaceCastOp>(castOp, llvmDstTy,
1528 llvmSrcVal);
1529 break;
1530 }
1531 case cir::CastKind::member_ptr_to_bool:
1532 assert(!MissingFeatures::cxxABI());
1533 assert(!MissingFeatures::methodType());
1534 break;
1535 default: {
1536 return castOp.emitError("Unhandled cast kind: ")
1537 << castOp.getKindAttrName();
1538 }
1539 }
1540
1541 return mlir::success();
1542}
1543
1544static mlir::Value convertToIndexTy(mlir::ConversionPatternRewriter &rewriter,
1545 mlir::ModuleOp mod, mlir::Value index,
1546 mlir::Type baseTy, cir::IntType strideTy) {
1547 mlir::Operation *indexOp = index.getDefiningOp();
1548 if (!indexOp)
1549 return index;
1550
1551 auto indexType = mlir::cast<mlir::IntegerType>(index.getType());
1552 mlir::DataLayout llvmLayout(mod);
1553 std::optional<uint64_t> layoutWidth = llvmLayout.getTypeIndexBitwidth(baseTy);
1554
1555 // If there is no change in width, don't do anything.
1556 if (!layoutWidth || *layoutWidth == indexType.getWidth())
1557 return index;
1558
1559 // If the index comes from a subtraction, make sure the extension happens
1560 // before it. To achieve that, look at unary minus, which already got
1561 // lowered to "sub 0, x".
1562 auto sub = dyn_cast<mlir::LLVM::SubOp>(indexOp);
1563 bool rewriteSub = false;
1564 if (sub) {
1565 if (auto lhsConst =
1566 dyn_cast<mlir::LLVM::ConstantOp>(sub.getLhs().getDefiningOp())) {
1567 auto lhsConstInt = mlir::dyn_cast<mlir::IntegerAttr>(lhsConst.getValue());
1568 if (lhsConstInt && lhsConstInt.getValue() == 0) {
1569 index = sub.getRhs();
1570 rewriteSub = true;
1571 }
1572 }
1573 }
1574
1575 auto llvmDstType = rewriter.getIntegerType(*layoutWidth);
1576 bool isUnsigned = strideTy && strideTy.isUnsigned();
1577 index = getLLVMIntCast(rewriter, index, llvmDstType, isUnsigned,
1578 indexType.getWidth(), *layoutWidth);
1579
1580 if (rewriteSub) {
1581 index = mlir::LLVM::SubOp::create(
1582 rewriter, index.getLoc(),
1583 mlir::LLVM::ConstantOp::create(rewriter, index.getLoc(),
1584 index.getType(), 0),
1585 index);
1586 // TODO: ensure sub is trivially dead now.
1587 rewriter.eraseOp(sub);
1588 }
1589
1590 return index;
1591}
1592
1593mlir::LogicalResult CIRToLLVMPtrStrideOpLowering::matchAndRewrite(
1594 cir::PtrStrideOp ptrStrideOp, OpAdaptor adaptor,
1595 mlir::ConversionPatternRewriter &rewriter) const {
1596
1597 const mlir::TypeConverter *tc = getTypeConverter();
1598 const mlir::Type resultTy = tc->convertType(ptrStrideOp.getType());
1599
1600 mlir::Type elementTy =
1601 convertTypeForMemory(*tc, dataLayout, ptrStrideOp.getElementType());
1602
1603 // void and function types doesn't really have a layout to use in GEPs,
1604 // make it i8 instead.
1605 if (mlir::isa<mlir::LLVM::LLVMVoidType>(elementTy) ||
1606 mlir::isa<mlir::LLVM::LLVMFunctionType>(elementTy))
1607 elementTy = mlir::IntegerType::get(elementTy.getContext(), 8,
1608 mlir::IntegerType::Signless);
1609 // Zero-extend, sign-extend or trunc the pointer value.
1610 mlir::Value index = adaptor.getStride();
1611 index = convertToIndexTy(
1612 rewriter, ptrStrideOp->getParentOfType<mlir::ModuleOp>(), index,
1613 adaptor.getBase().getType(),
1614 dyn_cast<cir::IntType>(ptrStrideOp.getOperand(1).getType()));
1615
1616 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
1617 ptrStrideOp, resultTy, elementTy, adaptor.getBase(), index);
1618 return mlir::success();
1619}
1620
1621mlir::LogicalResult CIRToLLVMGetElementOpLowering::matchAndRewrite(
1622 cir::GetElementOp op, OpAdaptor adaptor,
1623 mlir::ConversionPatternRewriter &rewriter) const {
1624 if (auto arrayTy =
1625 mlir::dyn_cast<cir::ArrayType>(op.getBaseType().getPointee())) {
1626 const mlir::TypeConverter *converter = getTypeConverter();
1627 const mlir::Type llArrayTy = converter->convertType(arrayTy);
1628 const mlir::Type llResultTy = converter->convertType(op.getType());
1629 mlir::Type elementTy =
1630 convertTypeForMemory(*converter, dataLayout, op.getElementType());
1631
1632 // void and function types don't really have a layout to use in GEPs,
1633 // make it i8 instead.
1634 if (mlir::isa<mlir::LLVM::LLVMVoidType>(elementTy) ||
1635 mlir::isa<mlir::LLVM::LLVMFunctionType>(elementTy))
1636 elementTy = rewriter.getIntegerType(8);
1637
1638 mlir::Value index = adaptor.getIndex();
1639 index =
1640 convertToIndexTy(rewriter, op->getParentOfType<mlir::ModuleOp>(), index,
1641 adaptor.getBase().getType(),
1642 dyn_cast<cir::IntType>(op.getOperand(1).getType()));
1643
1644 // Since the base address is a pointer to an aggregate, the first
1645 // offset is always zero. The second offset tell us which member it
1646 // will access.
1647 std::array<mlir::LLVM::GEPArg, 2> offset{0, index};
1648 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, llResultTy, llArrayTy,
1649 adaptor.getBase(), offset);
1650 return mlir::success();
1651 }
1652
1653 op.emitError() << "NYI: GetElementOp lowering to LLVM for non-array";
1654 return mlir::failure();
1655}
1656
1657mlir::LogicalResult CIRToLLVMBaseClassAddrOpLowering::matchAndRewrite(
1658 cir::BaseClassAddrOp baseClassOp, OpAdaptor adaptor,
1659 mlir::ConversionPatternRewriter &rewriter) const {
1660 const mlir::Type resultType =
1661 getTypeConverter()->convertType(baseClassOp.getType());
1662 mlir::Value derivedAddr = adaptor.getDerivedAddr();
1663 llvm::SmallVector<mlir::LLVM::GEPArg, 1> offset = {
1664 adaptor.getOffset().getZExtValue()};
1665 mlir::Type byteType = mlir::IntegerType::get(resultType.getContext(), 8,
1666 mlir::IntegerType::Signless);
1667 if (adaptor.getOffset().getZExtValue() == 0) {
1668 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(
1669 baseClassOp, resultType, adaptor.getDerivedAddr());
1670 return mlir::success();
1671 }
1672
1673 if (baseClassOp.getAssumeNotNull()) {
1674 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
1675 baseClassOp, resultType, byteType, derivedAddr, offset);
1676 } else {
1677 auto loc = baseClassOp.getLoc();
1678 mlir::Value isNull = mlir::LLVM::ICmpOp::create(
1679 rewriter, loc, mlir::LLVM::ICmpPredicate::eq, derivedAddr,
1680 mlir::LLVM::ZeroOp::create(rewriter, loc, derivedAddr.getType()));
1681 mlir::Value adjusted = mlir::LLVM::GEPOp::create(
1682 rewriter, loc, resultType, byteType, derivedAddr, offset);
1683 rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(baseClassOp, isNull,
1684 derivedAddr, adjusted);
1685 }
1686 return mlir::success();
1687}
1688
1689mlir::LogicalResult CIRToLLVMDerivedClassAddrOpLowering::matchAndRewrite(
1690 cir::DerivedClassAddrOp derivedClassOp, OpAdaptor adaptor,
1691 mlir::ConversionPatternRewriter &rewriter) const {
1692 const mlir::Type resultType =
1693 getTypeConverter()->convertType(derivedClassOp.getType());
1694 mlir::Value baseAddr = adaptor.getBaseAddr();
1695 // The offset is set in the operation as an unsigned value, but it must be
1696 // applied as a negative offset.
1697 int64_t offsetVal = -(adaptor.getOffset().getZExtValue());
1698 if (offsetVal == 0) {
1699 // If the offset is zero, we can just return the base address,
1700 rewriter.replaceOp(derivedClassOp, baseAddr);
1701 return mlir::success();
1702 }
1703 llvm::SmallVector<mlir::LLVM::GEPArg, 1> offset = {offsetVal};
1704 mlir::Type byteType = mlir::IntegerType::get(resultType.getContext(), 8,
1705 mlir::IntegerType::Signless);
1706 if (derivedClassOp.getAssumeNotNull()) {
1707 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
1708 derivedClassOp, resultType, byteType, baseAddr, offset,
1709 mlir::LLVM::GEPNoWrapFlags::inbounds);
1710 } else {
1711 mlir::Location loc = derivedClassOp.getLoc();
1712 mlir::Value isNull = mlir::LLVM::ICmpOp::create(
1713 rewriter, loc, mlir::LLVM::ICmpPredicate::eq, baseAddr,
1714 mlir::LLVM::ZeroOp::create(rewriter, loc, baseAddr.getType()));
1715 mlir::Value adjusted =
1716 mlir::LLVM::GEPOp::create(rewriter, loc, resultType, byteType, baseAddr,
1717 offset, mlir::LLVM::GEPNoWrapFlags::inbounds);
1718 rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(derivedClassOp, isNull,
1719 baseAddr, adjusted);
1720 }
1721 return mlir::success();
1722}
1723
1724mlir::LogicalResult CIRToLLVMATanOpLowering::matchAndRewrite(
1725 cir::ATanOp op, OpAdaptor adaptor,
1726 mlir::ConversionPatternRewriter &rewriter) const {
1727 mlir::Type resTy = typeConverter->convertType(op.getType());
1728 rewriter.replaceOpWithNewOp<mlir::LLVM::ATanOp>(op, resTy, adaptor.getSrc());
1729 return mlir::success();
1730}
1731
1732mlir::LogicalResult CIRToLLVMCeilOpLowering::matchAndRewrite(
1733 cir::CeilOp op, OpAdaptor adaptor,
1734 mlir::ConversionPatternRewriter &rewriter) const {
1735 mlir::Type resTy = typeConverter->convertType(op.getType());
1736 rewriter.replaceOpWithNewOp<mlir::LLVM::FCeilOp>(op, resTy, adaptor.getSrc());
1737 return mlir::success();
1738}
1739
1740mlir::LogicalResult CIRToLLVMCopysignOpLowering::matchAndRewrite(
1741 cir::CopysignOp op, OpAdaptor adaptor,
1742 mlir::ConversionPatternRewriter &rewriter) const {
1743 mlir::Type resTy = typeConverter->convertType(op.getType());
1744 rewriter.replaceOpWithNewOp<mlir::LLVM::CopySignOp>(
1745 op, resTy, adaptor.getLhs(), adaptor.getRhs());
1746 return mlir::success();
1747}
1748
1749mlir::LogicalResult CIRToLLVMFMaxNumOpLowering::matchAndRewrite(
1750 cir::FMaxNumOp op, OpAdaptor adaptor,
1751 mlir::ConversionPatternRewriter &rewriter) const {
1752 mlir::Type resTy = typeConverter->convertType(op.getType());
1753 rewriter.replaceOpWithNewOp<mlir::LLVM::MaxNumOp>(
1754 op, resTy, adaptor.getLhs(), adaptor.getRhs(),
1755 mlir::LLVM::FastmathFlags::nsz);
1756 return mlir::success();
1757}
1758
1759mlir::LogicalResult CIRToLLVMFMinNumOpLowering::matchAndRewrite(
1760 cir::FMinNumOp op, OpAdaptor adaptor,
1761 mlir::ConversionPatternRewriter &rewriter) const {
1762 mlir::Type resTy = typeConverter->convertType(op.getType());
1763 rewriter.replaceOpWithNewOp<mlir::LLVM::MinNumOp>(
1764 op, resTy, adaptor.getLhs(), adaptor.getRhs(),
1765 mlir::LLVM::FastmathFlags::nsz);
1766 return mlir::success();
1767}
1768
1769mlir::LogicalResult CIRToLLVMFMaximumOpLowering::matchAndRewrite(
1770 cir::FMaximumOp op, OpAdaptor adaptor,
1771 mlir::ConversionPatternRewriter &rewriter) const {
1772 mlir::Type resTy = typeConverter->convertType(op.getType());
1773 rewriter.replaceOpWithNewOp<mlir::LLVM::MaximumOp>(
1774 op, resTy, adaptor.getLhs(), adaptor.getRhs());
1775 return mlir::success();
1776}
1777
1778mlir::LogicalResult CIRToLLVMFMinimumOpLowering::matchAndRewrite(
1779 cir::FMinimumOp op, OpAdaptor adaptor,
1780 mlir::ConversionPatternRewriter &rewriter) const {
1781 mlir::Type resTy = typeConverter->convertType(op.getType());
1782 rewriter.replaceOpWithNewOp<mlir::LLVM::MinimumOp>(
1783 op, resTy, adaptor.getLhs(), adaptor.getRhs());
1784 return mlir::success();
1785}
1786
1787mlir::LogicalResult CIRToLLVMFModOpLowering::matchAndRewrite(
1788 cir::FModOp op, OpAdaptor adaptor,
1789 mlir::ConversionPatternRewriter &rewriter) const {
1790 mlir::Type resTy = typeConverter->convertType(op.getType());
1791 rewriter.replaceOpWithNewOp<mlir::LLVM::FRemOp>(op, resTy, adaptor.getLhs(),
1792 adaptor.getRhs());
1793 return mlir::success();
1794}
1795
1796mlir::LogicalResult CIRToLLVMPowOpLowering::matchAndRewrite(
1797 cir::PowOp op, OpAdaptor adaptor,
1798 mlir::ConversionPatternRewriter &rewriter) const {
1799 mlir::Type resTy = typeConverter->convertType(op.getType());
1800 rewriter.replaceOpWithNewOp<mlir::LLVM::PowOp>(op, resTy, adaptor.getLhs(),
1801 adaptor.getRhs());
1802 return mlir::success();
1803}
1804
1805mlir::LogicalResult CIRToLLVMATan2OpLowering::matchAndRewrite(
1806 cir::ATan2Op op, OpAdaptor adaptor,
1807 mlir::ConversionPatternRewriter &rewriter) const {
1808 mlir::Type resTy = typeConverter->convertType(op.getType());
1809 rewriter.replaceOpWithNewOp<mlir::LLVM::ATan2Op>(op, resTy, adaptor.getLhs(),
1810 adaptor.getRhs());
1811 return mlir::success();
1812}
1813
1814mlir::LogicalResult CIRToLLVMLroundOpLowering::matchAndRewrite(
1815 cir::LroundOp op, OpAdaptor adaptor,
1816 mlir::ConversionPatternRewriter &rewriter) const {
1817 mlir::Type resTy = typeConverter->convertType(op.getType());
1818 rewriter.replaceOpWithNewOp<mlir::LLVM::LroundOp>(op, resTy,
1819 adaptor.getSrc());
1820 return mlir::success();
1821}
1822
1823mlir::LogicalResult CIRToLLVMLlroundOpLowering::matchAndRewrite(
1824 cir::LlroundOp op, OpAdaptor adaptor,
1825 mlir::ConversionPatternRewriter &rewriter) const {
1826 mlir::Type resTy = typeConverter->convertType(op.getType());
1827 rewriter.replaceOpWithNewOp<mlir::LLVM::LlroundOp>(op, resTy,
1828 adaptor.getSrc());
1829 return mlir::success();
1830}
1831
1832mlir::LogicalResult CIRToLLVMLrintOpLowering::matchAndRewrite(
1833 cir::LrintOp op, OpAdaptor adaptor,
1834 mlir::ConversionPatternRewriter &rewriter) const {
1835 mlir::Type resTy = typeConverter->convertType(op.getType());
1836 rewriter.replaceOpWithNewOp<mlir::LLVM::LrintOp>(op, resTy, adaptor.getSrc());
1837 return mlir::success();
1838}
1839
1840mlir::LogicalResult CIRToLLVMLlrintOpLowering::matchAndRewrite(
1841 cir::LlrintOp op, OpAdaptor adaptor,
1842 mlir::ConversionPatternRewriter &rewriter) const {
1843 mlir::Type resTy = typeConverter->convertType(op.getType());
1844 rewriter.replaceOpWithNewOp<mlir::LLVM::LlrintOp>(op, resTy,
1845 adaptor.getSrc());
1846 return mlir::success();
1847}
1848
1849mlir::LogicalResult CIRToLLVMAllocaOpLowering::matchAndRewrite(
1850 cir::AllocaOp op, OpAdaptor adaptor,
1851 mlir::ConversionPatternRewriter &rewriter) const {
1852 mlir::Value size =
1853 op.isDynamic()
1854 ? adaptor.getDynAllocSize()
1855 : mlir::LLVM::ConstantOp::create(
1856 rewriter, op.getLoc(),
1857 typeConverter->convertType(rewriter.getIndexType()), 1);
1858 mlir::Type elementTy =
1859 convertTypeForMemory(*getTypeConverter(), dataLayout, op.getAllocaType());
1860 mlir::Type resultTy =
1861 convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType());
1862
1863 assert(!cir::MissingFeatures::addressSpace());
1864 assert(!cir::MissingFeatures::opAllocaAnnotations());
1865
1866 rewriter.replaceOpWithNewOp<mlir::LLVM::AllocaOp>(op, resultTy, elementTy,
1867 size, op.getAlignment());
1868
1869 return mlir::success();
1870}
1871
1872mlir::LogicalResult CIRToLLVMReturnOpLowering::matchAndRewrite(
1873 cir::ReturnOp op, OpAdaptor adaptor,
1874 mlir::ConversionPatternRewriter &rewriter) const {
1875 rewriter.replaceOpWithNewOp<mlir::LLVM::ReturnOp>(op, adaptor.getOperands());
1876 return mlir::LogicalResult::success();
1877}
1878
1879mlir::LogicalResult CIRToLLVMRotateOpLowering::matchAndRewrite(
1880 cir::RotateOp op, OpAdaptor adaptor,
1881 mlir::ConversionPatternRewriter &rewriter) const {
1882 // Note that LLVM intrinsic calls to @llvm.fsh{r,l}.i* have the same type as
1883 // the operand.
1884 mlir::Value input = adaptor.getInput();
1885 if (op.isRotateLeft())
1886 rewriter.replaceOpWithNewOp<mlir::LLVM::FshlOp>(op, input, input,
1887 adaptor.getAmount());
1888 else
1889 rewriter.replaceOpWithNewOp<mlir::LLVM::FshrOp>(op, input, input,
1890 adaptor.getAmount());
1891 return mlir::LogicalResult::success();
1892}
1893
1894static void lowerCallAttributes(cir::CIRCallOpInterface op,
1895 SmallVectorImpl<mlir::NamedAttribute> &result) {
1896 for (mlir::NamedAttribute attr : op->getAttrs()) {
1897 assert(!cir::MissingFeatures::opFuncCallingConv());
1898 if (attr.getName() == CIRDialect::getCalleeAttrName() ||
1899 attr.getName() == CIRDialect::getSideEffectAttrName() ||
1900 attr.getName() == CIRDialect::getNoThrowAttrName() ||
1901 attr.getName() == CIRDialect::getNoUnwindAttrName() ||
1902 attr.getName() == CIRDialect::getNoReturnAttrName())
1903 continue;
1904
1905 assert(!cir::MissingFeatures::opFuncExtraAttrs());
1906 result.push_back(attr);
1907 }
1908}
1909
1910static mlir::LogicalResult
1911rewriteCallOrInvoke(mlir::Operation *op, mlir::ValueRange callOperands,
1912 mlir::ConversionPatternRewriter &rewriter,
1913 const mlir::TypeConverter *converter,
1914 mlir::FlatSymbolRefAttr calleeAttr,
1915 mlir::Block *continueBlock = nullptr,
1916 mlir::Block *landingPadBlock = nullptr) {
1917 llvm::SmallVector<mlir::Type, 8> llvmResults;
1918 mlir::ValueTypeRange<mlir::ResultRange> cirResults = op->getResultTypes();
1919 auto call = cast<cir::CIRCallOpInterface>(op);
1920
1921 if (converter->convertTypes(cirResults, llvmResults).failed())
1922 return mlir::failure();
1923
1924 assert(!cir::MissingFeatures::opCallCallConv());
1925
1926 mlir::LLVM::MemoryEffectsAttr memoryEffects;
1927 bool noUnwind = false;
1928 bool willReturn = false;
1929 bool noReturn = false;
1930 convertSideEffectForCall(op, call.getNothrow(), call.getSideEffect(),
1931 memoryEffects, noUnwind, willReturn, noReturn);
1932
1933 SmallVector<mlir::NamedAttribute, 4> attributes;
1934 lowerCallAttributes(call, attributes);
1935
1936 mlir::LLVM::LLVMFunctionType llvmFnTy;
1937
1938 // Temporary to handle the case where we need to prepend an operand if the
1939 // callee is an alias.
1940 SmallVector<mlir::Value> adjustedCallOperands;
1941
1942 if (calleeAttr) { // direct call
1943 mlir::Operation *callee =
1944 mlir::SymbolTable::lookupNearestSymbolFrom(op, calleeAttr);
1945 if (auto fn = mlir::dyn_cast<mlir::FunctionOpInterface>(callee)) {
1946 llvmFnTy = converter->convertType<mlir::LLVM::LLVMFunctionType>(
1947 fn.getFunctionType());
1948 assert(llvmFnTy && "Failed to convert function type");
1949 } else if (auto alias = mlir::cast<mlir::LLVM::AliasOp>(callee)) {
1950 // If the callee was an alias. In that case,
1951 // we need to prepend the address of the alias to the operands. The
1952 // way aliases work in the LLVM dialect is a little counter-intuitive.
1953 // The AliasOp itself is a pseudo-function that returns the address of
1954 // the global value being aliased, but when we generate the call we
1955 // need to insert an operation that gets the address of the AliasOp.
1956 // This all gets sorted out when the LLVM dialect is lowered to LLVM IR.
1957 auto symAttr = mlir::cast<mlir::FlatSymbolRefAttr>(calleeAttr);
1958 auto addrOfAlias =
1959 mlir::LLVM::AddressOfOp::create(
1960 rewriter, op->getLoc(),
1961 mlir::LLVM::LLVMPointerType::get(rewriter.getContext()), symAttr)
1962 .getResult();
1963 adjustedCallOperands.push_back(addrOfAlias);
1964
1965 // Now add the regular operands and assign this to the range value.
1966 llvm::append_range(adjustedCallOperands, callOperands);
1967 callOperands = adjustedCallOperands;
1968
1969 // Clear the callee attribute because we're calling an alias.
1970 calleeAttr = {};
1971 llvmFnTy = mlir::cast<mlir::LLVM::LLVMFunctionType>(alias.getType());
1972 } else {
1973 // Was this an ifunc?
1974 return op->emitError("Unexpected callee type!");
1975 }
1976 } else { // indirect call
1977 assert(!op->getOperands().empty() &&
1978 "operands list must no be empty for the indirect call");
1979 auto calleeTy = op->getOperands().front().getType();
1980 auto calleePtrTy = cast<cir::PointerType>(calleeTy);
1981 auto calleeFuncTy = cast<cir::FuncType>(calleePtrTy.getPointee());
1982 llvm::append_range(adjustedCallOperands, callOperands);
1983 llvmFnTy = cast<mlir::LLVM::LLVMFunctionType>(
1984 converter->convertType(calleeFuncTy));
1985 }
1986
1987 assert(!cir::MissingFeatures::opCallCallConv());
1988
1989 if (landingPadBlock) {
1990 auto newOp = rewriter.replaceOpWithNewOp<mlir::LLVM::InvokeOp>(
1991 op, llvmFnTy, calleeAttr, callOperands, continueBlock,
1992 mlir::ValueRange{}, landingPadBlock, mlir::ValueRange{});
1993 newOp->setAttrs(attributes);
1994 } else {
1995 auto newOp = rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
1996 op, llvmFnTy, calleeAttr, callOperands);
1997 newOp->setAttrs(attributes);
1998 if (memoryEffects)
1999 newOp.setMemoryEffectsAttr(memoryEffects);
2000 newOp.setNoUnwind(noUnwind);
2001 newOp.setWillReturn(willReturn);
2002 newOp.setNoreturn(noReturn);
2003 }
2004
2005 return mlir::success();
2006}
2007
2008mlir::LogicalResult CIRToLLVMCallOpLowering::matchAndRewrite(
2009 cir::CallOp op, OpAdaptor adaptor,
2010 mlir::ConversionPatternRewriter &rewriter) const {
2011 return rewriteCallOrInvoke(op.getOperation(), adaptor.getOperands(), rewriter,
2012 getTypeConverter(), op.getCalleeAttr());
2013}
2014
2015mlir::LogicalResult CIRToLLVMTryCallOpLowering::matchAndRewrite(
2016 cir::TryCallOp op, OpAdaptor adaptor,
2017 mlir::ConversionPatternRewriter &rewriter) const {
2018 assert(!cir::MissingFeatures::opCallCallConv());
2019 return rewriteCallOrInvoke(op.getOperation(), adaptor.getOperands(), rewriter,
2020 getTypeConverter(), op.getCalleeAttr(),
2021 op.getNormalDest(), op.getUnwindDest());
2022}
2023
2024mlir::LogicalResult CIRToLLVMReturnAddrOpLowering::matchAndRewrite(
2025 cir::ReturnAddrOp op, OpAdaptor adaptor,
2026 mlir::ConversionPatternRewriter &rewriter) const {
2027 const mlir::Type llvmPtrTy = getTypeConverter()->convertType(op.getType());
2028 replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.returnaddress",
2029 llvmPtrTy, adaptor.getOperands());
2030 return mlir::success();
2031}
2032
2033mlir::LogicalResult CIRToLLVMFrameAddrOpLowering::matchAndRewrite(
2034 cir::FrameAddrOp op, OpAdaptor adaptor,
2035 mlir::ConversionPatternRewriter &rewriter) const {
2036 const mlir::Type llvmPtrTy = getTypeConverter()->convertType(op.getType());
2037 replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.frameaddress", llvmPtrTy,
2038 adaptor.getOperands());
2039 return mlir::success();
2040}
2041
2042mlir::LogicalResult CIRToLLVMClearCacheOpLowering::matchAndRewrite(
2043 cir::ClearCacheOp op, OpAdaptor adaptor,
2044 mlir::ConversionPatternRewriter &rewriter) const {
2045 mlir::Value begin = adaptor.getBegin();
2046 mlir::Value end = adaptor.getEnd();
2047 auto intrinNameAttr =
2048 mlir::StringAttr::get(op.getContext(), "llvm.clear_cache");
2049 rewriter.replaceOpWithNewOp<mlir::LLVM::CallIntrinsicOp>(
2050 op, mlir::Type{}, intrinNameAttr, mlir::ValueRange{begin, end});
2051
2052 return mlir::success();
2053}
2054
2055mlir::LogicalResult CIRToLLVMAddrOfReturnAddrOpLowering::matchAndRewrite(
2056 cir::AddrOfReturnAddrOp op, OpAdaptor adaptor,
2057 mlir::ConversionPatternRewriter &rewriter) const {
2058 const mlir::Type llvmPtrTy = getTypeConverter()->convertType(op.getType());
2059 replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.addressofreturnaddress",
2060 llvmPtrTy, adaptor.getOperands());
2061 return mlir::success();
2062}
2063
2064mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
2065 cir::LoadOp op, OpAdaptor adaptor,
2066 mlir::ConversionPatternRewriter &rewriter) const {
2067 const mlir::Type llvmTy =
2068 convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType());
2069 mlir::LLVM::AtomicOrdering ordering = getLLVMMemOrder(op.getMemOrder());
2070 std::optional<size_t> opAlign = op.getAlignment();
2071 unsigned alignment =
2072 (unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmTy));
2073
2074 assert(!cir::MissingFeatures::lowerModeOptLevel());
2075
2076 // TODO: nontemporal.
2077 assert(!cir::MissingFeatures::opLoadStoreNontemporal());
2078
2079 std::optional<llvm::StringRef> llvmSyncScope =
2080 getLLVMSyncScope(op.getSyncScope());
2081
2082 mlir::LLVM::LoadOp newLoad = mlir::LLVM::LoadOp::create(
2083 rewriter, op->getLoc(), llvmTy, adaptor.getAddr(), alignment,
2084 op.getIsVolatile(), /*isNonTemporal=*/false,
2085 /*isInvariant=*/false, /*isInvariantGroup=*/false, ordering,
2086 llvmSyncScope.value_or(std::string()));
2087
2088 // Convert adapted result to its original type if needed.
2089 mlir::Value result =
2090 emitFromMemory(rewriter, dataLayout, op, newLoad.getResult());
2091 rewriter.replaceOp(op, result);
2092 assert(!cir::MissingFeatures::opLoadStoreTbaa());
2093 return mlir::LogicalResult::success();
2094}
2095
2096mlir::LogicalResult
2097cir::direct::CIRToLLVMVecMaskedLoadOpLowering::matchAndRewrite(
2098 cir::VecMaskedLoadOp op, OpAdaptor adaptor,
2099 mlir::ConversionPatternRewriter &rewriter) const {
2100 const mlir::Type llvmResTy =
2101 convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType());
2102
2103 std::optional<size_t> opAlign = op.getAlignment();
2104 unsigned alignment =
2105 (unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmResTy));
2106
2107 mlir::IntegerAttr alignAttr = rewriter.getI32IntegerAttr(alignment);
2108
2109 auto newLoad = mlir::LLVM::MaskedLoadOp::create(
2110 rewriter, op.getLoc(), llvmResTy, adaptor.getAddr(), adaptor.getMask(),
2111 adaptor.getPassThru(), alignAttr);
2112
2113 rewriter.replaceOp(op, newLoad.getResult());
2114 return mlir::success();
2115}
2116
2117mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite(
2118 cir::StoreOp op, OpAdaptor adaptor,
2119 mlir::ConversionPatternRewriter &rewriter) const {
2120 mlir::LLVM::AtomicOrdering memorder = getLLVMMemOrder(op.getMemOrder());
2121 const mlir::Type llvmTy =
2122 getTypeConverter()->convertType(op.getValue().getType());
2123 std::optional<size_t> opAlign = op.getAlignment();
2124 unsigned alignment =
2125 (unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmTy));
2126
2127 assert(!cir::MissingFeatures::lowerModeOptLevel());
2128
2129 // Convert adapted value to its memory type if needed.
2130 mlir::Value value = emitToMemory(rewriter, dataLayout,
2131 op.getValue().getType(), adaptor.getValue());
2132 // TODO: nontemporal.
2133 assert(!cir::MissingFeatures::opLoadStoreNontemporal());
2134 assert(!cir::MissingFeatures::opLoadStoreTbaa());
2135
2136 std::optional<llvm::StringRef> llvmSyncScope =
2137 getLLVMSyncScope(op.getSyncScope());
2138
2139 mlir::LLVM::StoreOp storeOp = mlir::LLVM::StoreOp::create(
2140 rewriter, op->getLoc(), value, adaptor.getAddr(), alignment,
2141 op.getIsVolatile(),
2142 /*isNonTemporal=*/false, /*isInvariantGroup=*/false, memorder,
2143 llvmSyncScope.value_or(std::string()));
2144 rewriter.replaceOp(op, storeOp);
2145 assert(!cir::MissingFeatures::opLoadStoreTbaa());
2146 return mlir::LogicalResult::success();
2147}
2148
2149bool hasTrailingZeros(cir::ConstArrayAttr attr) {
2150 auto array = mlir::dyn_cast<mlir::ArrayAttr>(attr.getElts());
2151 return attr.hasTrailingZeros() ||
2152 (array && std::count_if(array.begin(), array.end(), [](auto elt) {
2153 auto ar = dyn_cast<cir::ConstArrayAttr>(elt);
2154 return ar && hasTrailingZeros(ar);
2155 }));
2156}
2157
2158mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
2159 cir::ConstantOp op, OpAdaptor adaptor,
2160 mlir::ConversionPatternRewriter &rewriter) const {
2161 mlir::Attribute attr = op.getValue();
2162
2163 if (mlir::isa<cir::PoisonAttr>(attr)) {
2164 rewriter.replaceOpWithNewOp<mlir::LLVM::PoisonOp>(
2165 op, getTypeConverter()->convertType(op.getType()));
2166 return mlir::success();
2167 }
2168
2169 if (mlir::isa<mlir::IntegerType>(op.getType())) {
2170 // Verified cir.const operations cannot actually be of these types, but the
2171 // lowering pass may generate temporary cir.const operations with these
2172 // types. This is OK since MLIR allows unverified operations to be alive
2173 // during a pass as long as they don't live past the end of the pass.
2174 attr = op.getValue();
2175 } else if (mlir::isa<cir::BoolType>(op.getType())) {
2176 int value = mlir::cast<cir::BoolAttr>(op.getValue()).getValue();
2177 attr = rewriter.getIntegerAttr(typeConverter->convertType(op.getType()),
2178 value);
2179 } else if (mlir::isa<cir::IntType>(op.getType())) {
2180 // Lower GlobalViewAttr to llvm.mlir.addressof + llvm.mlir.ptrtoint
2181 if (auto ga = mlir::dyn_cast<cir::GlobalViewAttr>(op.getValue())) {
2182 // We can have a global view with an integer type in the case of method
2183 // pointers, but the lowering of those doesn't go through this path.
2184 // They are handled in the visitCirAttr. This is left as an error until
2185 // we have a test case that reaches it.
2186 assert(!cir::MissingFeatures::globalViewIntLowering());
2187 op.emitError() << "global view with integer type";
2188 return mlir::failure();
2189 }
2190
2191 attr = rewriter.getIntegerAttr(
2192 typeConverter->convertType(op.getType()),
2193 mlir::cast<cir::IntAttr>(op.getValue()).getValue());
2194 } else if (mlir::isa<cir::FPTypeInterface>(op.getType())) {
2195 attr = rewriter.getFloatAttr(
2196 typeConverter->convertType(op.getType()),
2197 mlir::cast<cir::FPAttr>(op.getValue()).getValue());
2198 } else if (mlir::isa<cir::PointerType>(op.getType())) {
2199 // Optimize with dedicated LLVM op for null pointers.
2200 if (mlir::isa<cir::ConstPtrAttr>(op.getValue())) {
2201 if (mlir::cast<cir::ConstPtrAttr>(op.getValue()).isNullValue()) {
2202 rewriter.replaceOpWithNewOp<mlir::LLVM::ZeroOp>(
2203 op, typeConverter->convertType(op.getType()));
2204 return mlir::success();
2205 }
2206 }
2207 // Lower GlobalViewAttr to llvm.mlir.addressof
2208 if (auto gv = mlir::dyn_cast<cir::GlobalViewAttr>(op.getValue())) {
2209 auto newOp = lowerCirAttrAsValue(op, gv, rewriter, getTypeConverter());
2210 rewriter.replaceOp(op, newOp);
2211 return mlir::success();
2212 }
2213 attr = op.getValue();
2214 } else if (const auto arrTy = mlir::dyn_cast<cir::ArrayType>(op.getType())) {
2215 const auto constArr = mlir::dyn_cast<cir::ConstArrayAttr>(op.getValue());
2216 if (!constArr && !isa<cir::ZeroAttr, cir::UndefAttr>(op.getValue()))
2217 return op.emitError() << "array does not have a constant initializer";
2218
2219 std::optional<mlir::Attribute> denseAttr;
2220 if (constArr && hasTrailingZeros(constArr)) {
2221 const mlir::Value newOp =
2222 lowerCirAttrAsValue(op, constArr, rewriter, getTypeConverter());
2223 rewriter.replaceOp(op, newOp);
2224 return mlir::success();
2225 } else if (constArr &&
2226 (denseAttr = lowerConstArrayAttr(constArr, typeConverter))) {
2227 attr = denseAttr.value();
2228 } else {
2229 const mlir::Value initVal =
2230 lowerCirAttrAsValue(op, op.getValue(), rewriter, typeConverter);
2231 rewriter.replaceOp(op, initVal);
2232 return mlir::success();
2233 }
2234 } else if (const auto recordAttr =
2235 mlir::dyn_cast<cir::ConstRecordAttr>(op.getValue())) {
2236 auto initVal = lowerCirAttrAsValue(op, recordAttr, rewriter, typeConverter);
2237 rewriter.replaceOp(op, initVal);
2238 return mlir::success();
2239 } else if (const auto vecTy = mlir::dyn_cast<cir::VectorType>(op.getType())) {
2240 rewriter.replaceOp(op, lowerCirAttrAsValue(op, op.getValue(), rewriter,
2241 getTypeConverter()));
2242 return mlir::success();
2243 } else if (auto recTy = mlir::dyn_cast<cir::RecordType>(op.getType())) {
2244 if (mlir::isa<cir::ZeroAttr, cir::UndefAttr>(attr)) {
2245 mlir::Value initVal =
2246 lowerCirAttrAsValue(op, attr, rewriter, typeConverter);
2247 rewriter.replaceOp(op, initVal);
2248 return mlir::success();
2249 }
2250 return op.emitError() << "unsupported lowering for record constant type "
2251 << op.getType();
2252 } else if (auto complexTy = mlir::dyn_cast<cir::ComplexType>(op.getType())) {
2253 mlir::Type complexElemTy = complexTy.getElementType();
2254 mlir::Type complexElemLLVMTy = typeConverter->convertType(complexElemTy);
2255
2256 if (auto zeroInitAttr = mlir::dyn_cast<cir::ZeroAttr>(op.getValue())) {
2257 mlir::TypedAttr zeroAttr = rewriter.getZeroAttr(complexElemLLVMTy);
2258 mlir::ArrayAttr array = rewriter.getArrayAttr({zeroAttr, zeroAttr});
2259 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
2260 op, getTypeConverter()->convertType(op.getType()), array);
2261 return mlir::success();
2262 }
2263
2264 auto complexAttr = mlir::cast<cir::ConstComplexAttr>(op.getValue());
2265
2266 mlir::Attribute components[2];
2267 if (mlir::isa<cir::IntType>(complexElemTy)) {
2268 components[0] = rewriter.getIntegerAttr(
2269 complexElemLLVMTy,
2270 mlir::cast<cir::IntAttr>(complexAttr.getReal()).getValue());
2271 components[1] = rewriter.getIntegerAttr(
2272 complexElemLLVMTy,
2273 mlir::cast<cir::IntAttr>(complexAttr.getImag()).getValue());
2274 } else {
2275 components[0] = rewriter.getFloatAttr(
2276 complexElemLLVMTy,
2277 mlir::cast<cir::FPAttr>(complexAttr.getReal()).getValue());
2278 components[1] = rewriter.getFloatAttr(
2279 complexElemLLVMTy,
2280 mlir::cast<cir::FPAttr>(complexAttr.getImag()).getValue());
2281 }
2282
2283 attr = rewriter.getArrayAttr(components);
2284 } else {
2285 return op.emitError() << "unsupported constant type " << op.getType();
2286 }
2287
2288 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
2289 op, getTypeConverter()->convertType(op.getType()), attr);
2290
2291 return mlir::success();
2292}
2293
2294static uint64_t getTypeSize(mlir::Type type, mlir::Operation &op) {
2295 mlir::DataLayout layout(op.getParentOfType<mlir::ModuleOp>());
2296 // For LLVM purposes we treat void as u8.
2297 if (isa<cir::VoidType>(type))
2298 type = cir::IntType::get(type.getContext(), 8, /*isSigned=*/false);
2299 return llvm::divideCeil(layout.getTypeSizeInBits(type), 8);
2300}
2301
2302mlir::LogicalResult CIRToLLVMPrefetchOpLowering::matchAndRewrite(
2303 cir::PrefetchOp op, OpAdaptor adaptor,
2304 mlir::ConversionPatternRewriter &rewriter) const {
2305 rewriter.replaceOpWithNewOp<mlir::LLVM::Prefetch>(
2306 op, adaptor.getAddr(), adaptor.getIsWrite(), adaptor.getLocality(),
2307 /*DataCache=*/1);
2308 return mlir::success();
2309}
2310
2311mlir::LogicalResult CIRToLLVMPtrDiffOpLowering::matchAndRewrite(
2312 cir::PtrDiffOp op, OpAdaptor adaptor,
2313 mlir::ConversionPatternRewriter &rewriter) const {
2314 auto dstTy = mlir::cast<cir::IntType>(op.getType());
2315 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
2316
2317 auto lhs = mlir::LLVM::PtrToIntOp::create(rewriter, op.getLoc(), llvmDstTy,
2318 adaptor.getLhs());
2319 auto rhs = mlir::LLVM::PtrToIntOp::create(rewriter, op.getLoc(), llvmDstTy,
2320 adaptor.getRhs());
2321
2322 auto diff =
2323 mlir::LLVM::SubOp::create(rewriter, op.getLoc(), llvmDstTy, lhs, rhs);
2324
2325 cir::PointerType ptrTy = op.getLhs().getType();
2326 assert(!cir::MissingFeatures::llvmLoweringPtrDiffConsidersPointee());
2327 uint64_t typeSize = getTypeSize(ptrTy.getPointee(), *op);
2328
2329 // Avoid silly division by 1.
2330 mlir::Value resultVal = diff.getResult();
2331 if (typeSize != 1) {
2332 auto typeSizeVal = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
2333 llvmDstTy, typeSize);
2334
2335 if (dstTy.isUnsigned()) {
2336 auto uDiv =
2337 mlir::LLVM::UDivOp::create(rewriter, op.getLoc(), diff, typeSizeVal);
2338 uDiv.setIsExact(true);
2339 resultVal = uDiv.getResult();
2340 } else {
2341 auto sDiv =
2342 mlir::LLVM::SDivOp::create(rewriter, op.getLoc(), diff, typeSizeVal);
2343 sDiv.setIsExact(true);
2344 resultVal = sDiv.getResult();
2345 }
2346 }
2347 rewriter.replaceOp(op, resultVal);
2348 return mlir::success();
2349}
2350
2351mlir::LogicalResult CIRToLLVMExpectOpLowering::matchAndRewrite(
2352 cir::ExpectOp op, OpAdaptor adaptor,
2353 mlir::ConversionPatternRewriter &rewriter) const {
2354 // TODO(cir): do not generate LLVM intrinsics under -O0
2355 assert(!cir::MissingFeatures::optInfoAttr());
2356
2357 std::optional<llvm::APFloat> prob = op.getProb();
2358 if (prob)
2359 rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectWithProbabilityOp>(
2360 op, adaptor.getVal(), adaptor.getExpected(), prob.value());
2361 else
2362 rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectOp>(op, adaptor.getVal(),
2363 adaptor.getExpected());
2364 return mlir::success();
2365}
2366
2367mlir::LogicalResult CIRToLLVMFAbsOpLowering::matchAndRewrite(
2368 cir::FAbsOp op, OpAdaptor adaptor,
2369 mlir::ConversionPatternRewriter &rewriter) const {
2370 mlir::Type resTy = typeConverter->convertType(op.getType());
2371 rewriter.replaceOpWithNewOp<mlir::LLVM::FAbsOp>(op, resTy,
2372 adaptor.getOperands()[0]);
2373 return mlir::success();
2374}
2375
2376mlir::LogicalResult CIRToLLVMAbsOpLowering::matchAndRewrite(
2377 cir::AbsOp op, OpAdaptor adaptor,
2378 mlir::ConversionPatternRewriter &rewriter) const {
2379 mlir::Type resTy = typeConverter->convertType(op.getType());
2380 auto absOp = mlir::LLVM::AbsOp::create(rewriter, op.getLoc(), resTy,
2381 adaptor.getOperands()[0],
2382 adaptor.getMinIsPoison());
2383 rewriter.replaceOp(op, absOp);
2384 return mlir::success();
2385}
2386
2387/// Convert the `cir.func` attributes to `llvm.func` attributes.
2388/// Only retain those attributes that are not constructed by
2389/// `LLVMFuncOp::build`. If `filterArgAttrs` is set, also filter out
2390/// argument attributes.
2391void CIRToLLVMFuncOpLowering::lowerFuncAttributes(
2392 cir::FuncOp func, bool filterArgAndResAttrs,
2393 SmallVectorImpl<mlir::NamedAttribute> &result) const {
2394 assert(!cir::MissingFeatures::opFuncCallingConv());
2395 for (mlir::NamedAttribute attr : func->getAttrs()) {
2396 assert(!cir::MissingFeatures::opFuncCallingConv());
2397 if (attr.getName() == mlir::SymbolTable::getSymbolAttrName() ||
2398 attr.getName() == func.getFunctionTypeAttrName() ||
2399 attr.getName() == getLinkageAttrNameString() ||
2400 attr.getName() == func.getGlobalVisibilityAttrName() ||
2401 attr.getName() == func.getDsoLocalAttrName() ||
2402 attr.getName() == func.getInlineKindAttrName() ||
2403 attr.getName() == func.getSideEffectAttrName() ||
2404 attr.getName() == CIRDialect::getNoReturnAttrName() ||
2405 (filterArgAndResAttrs &&
2406 (attr.getName() == func.getArgAttrsAttrName() ||
2407 attr.getName() == func.getResAttrsAttrName())))
2408 continue;
2409
2410 assert(!cir::MissingFeatures::opFuncExtraAttrs());
2411 result.push_back(attr);
2412 }
2413}
2414
2415mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewriteAlias(
2416 cir::FuncOp op, llvm::StringRef aliasee, mlir::Type ty, OpAdaptor adaptor,
2417 mlir::ConversionPatternRewriter &rewriter) const {
2418 SmallVector<mlir::NamedAttribute, 4> attributes;
2419 lowerFuncAttributes(op, /*filterArgAndResAttrs=*/false, attributes);
2420
2421 mlir::Location loc = op.getLoc();
2422 auto aliasOp = rewriter.replaceOpWithNewOp<mlir::LLVM::AliasOp>(
2423 op, ty, convertLinkage(op.getLinkage()), op.getName(), op.getDsoLocal(),
2424 /*threadLocal=*/false, attributes);
2425
2426 // Create the alias body
2427 mlir::OpBuilder builder(op.getContext());
2428 mlir::Block *block = builder.createBlock(&aliasOp.getInitializerRegion());
2429 builder.setInsertionPointToStart(block);
2430 // The type of AddressOfOp is always a pointer.
2431 assert(!cir::MissingFeatures::addressSpace());
2432 mlir::Type ptrTy = mlir::LLVM::LLVMPointerType::get(ty.getContext());
2433 auto addrOp = mlir::LLVM::AddressOfOp::create(builder, loc, ptrTy, aliasee);
2434 mlir::LLVM::ReturnOp::create(builder, loc, addrOp);
2435
2436 return mlir::success();
2437}
2438
2439mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewrite(
2440 cir::FuncOp op, OpAdaptor adaptor,
2441 mlir::ConversionPatternRewriter &rewriter) const {
2442
2443 cir::FuncType fnType = op.getFunctionType();
2444 bool isDsoLocal = op.getDsoLocal();
2445 mlir::TypeConverter::SignatureConversion signatureConversion(
2446 fnType.getNumInputs());
2447
2448 for (const auto &argType : llvm::enumerate(fnType.getInputs())) {
2449 mlir::Type convertedType = typeConverter->convertType(argType.value());
2450 if (!convertedType)
2451 return mlir::failure();
2452 signatureConversion.addInputs(argType.index(), convertedType);
2453 }
2454
2455 mlir::Type resultType =
2456 getTypeConverter()->convertType(fnType.getReturnType());
2457
2458 // Create the LLVM function operation.
2459 mlir::Type llvmFnTy = mlir::LLVM::LLVMFunctionType::get(
2460 resultType ? resultType : mlir::LLVM::LLVMVoidType::get(getContext()),
2461 signatureConversion.getConvertedTypes(),
2462 /*isVarArg=*/fnType.isVarArg());
2463
2464 // If this is an alias, it needs to be lowered to llvm::AliasOp.
2465 if (std::optional<llvm::StringRef> aliasee = op.getAliasee())
2466 return matchAndRewriteAlias(op, *aliasee, llvmFnTy, adaptor, rewriter);
2467
2468 // LLVMFuncOp expects a single FileLine Location instead of a fused
2469 // location.
2470 mlir::Location loc = op.getLoc();
2471 if (mlir::FusedLoc fusedLoc = mlir::dyn_cast<mlir::FusedLoc>(loc))
2472 loc = fusedLoc.getLocations()[0];
2473 assert((mlir::isa<mlir::FileLineColLoc>(loc) ||
2474 mlir::isa<mlir::UnknownLoc>(loc)) &&
2475 "expected single location or unknown location here");
2476
2477 mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());
2478 assert(!cir::MissingFeatures::opFuncCallingConv());
2479 mlir::LLVM::CConv cconv = mlir::LLVM::CConv::C;
2480 SmallVector<mlir::NamedAttribute, 4> attributes;
2481 lowerFuncAttributes(op, /*filterArgAndResAttrs=*/false, attributes);
2482
2483 mlir::LLVM::LLVMFuncOp fn = mlir::LLVM::LLVMFuncOp::create(
2484 rewriter, loc, op.getName(), llvmFnTy, linkage, isDsoLocal, cconv,
2485 mlir::SymbolRefAttr(), attributes);
2486
2487 assert(!cir::MissingFeatures::opFuncMultipleReturnVals());
2488
2489 if (std::optional<cir::SideEffect> sideEffectKind = op.getSideEffect()) {
2490 switch (*sideEffectKind) {
2491 case cir::SideEffect::All:
2492 break;
2493 case cir::SideEffect::Pure:
2494 fn.setMemoryEffectsAttr(mlir::LLVM::MemoryEffectsAttr::get(
2495 fn.getContext(),
2496 /*other=*/mlir::LLVM::ModRefInfo::Ref,
2497 /*argMem=*/mlir::LLVM::ModRefInfo::Ref,
2498 /*inaccessibleMem=*/mlir::LLVM::ModRefInfo::Ref,
2499 /*errnoMem=*/mlir::LLVM::ModRefInfo::Ref,
2500 /*targetMem0=*/mlir::LLVM::ModRefInfo::Ref,
2501 /*targetMem1=*/mlir::LLVM::ModRefInfo::Ref));
2502 fn.setNoUnwind(true);
2503 fn.setWillReturn(true);
2504 break;
2505 case cir::SideEffect::Const:
2506 fn.setMemoryEffectsAttr(mlir::LLVM::MemoryEffectsAttr::get(
2507 fn.getContext(),
2508 /*other=*/mlir::LLVM::ModRefInfo::NoModRef,
2509 /*argMem=*/mlir::LLVM::ModRefInfo::NoModRef,
2510 /*inaccessibleMem=*/mlir::LLVM::ModRefInfo::NoModRef,
2511 /*errnoMem=*/mlir::LLVM::ModRefInfo::NoModRef,
2512 /*targetMem0=*/mlir::LLVM::ModRefInfo::NoModRef,
2513 /*targetMem1=*/mlir::LLVM::ModRefInfo::NoModRef));
2514 fn.setNoUnwind(true);
2515 fn.setWillReturn(true);
2516 break;
2517 }
2518 }
2519
2520 if (op->hasAttr(CIRDialect::getNoReturnAttrName()))
2521 fn.setNoreturn(true);
2522
2523 if (std::optional<cir::InlineKind> inlineKind = op.getInlineKind()) {
2524 fn.setNoInline(*inlineKind == cir::InlineKind::NoInline);
2525 fn.setInlineHint(*inlineKind == cir::InlineKind::InlineHint);
2526 fn.setAlwaysInline(*inlineKind == cir::InlineKind::AlwaysInline);
2527 }
2528
2529 if (std::optional<llvm::StringRef> personality = op.getPersonality())
2530 fn.setPersonality(*personality);
2531
2532 fn.setVisibility_(
2533 lowerCIRVisibilityToLLVMVisibility(op.getGlobalVisibility()));
2534
2535 rewriter.inlineRegionBefore(op.getBody(), fn.getBody(), fn.end());
2536 if (failed(rewriter.convertRegionTypes(&fn.getBody(), *typeConverter,
2537 &signatureConversion)))
2538 return mlir::failure();
2539
2540 rewriter.eraseOp(op);
2541
2542 return mlir::LogicalResult::success();
2543}
2544
2545mlir::LogicalResult CIRToLLVMGetGlobalOpLowering::matchAndRewrite(
2546 cir::GetGlobalOp op, OpAdaptor adaptor,
2547 mlir::ConversionPatternRewriter &rewriter) const {
2548 // FIXME(cir): Premature DCE to avoid lowering stuff we're not using.
2549 // CIRGen should mitigate this and not emit the get_global.
2550 if (op->getUses().empty()) {
2551 rewriter.eraseOp(op);
2552 return mlir::success();
2553 }
2554
2555 mlir::Type type = getTypeConverter()->convertType(op.getType());
2556 mlir::Operation *newop = mlir::LLVM::AddressOfOp::create(
2557 rewriter, op.getLoc(), type, op.getName());
2558
2559 if (op.getTls()) {
2560 // Handle access to TLS via intrinsic.
2561 newop = mlir::LLVM::ThreadlocalAddressOp::create(rewriter, op.getLoc(),
2562 type, newop->getResult(0));
2563 }
2564
2565 rewriter.replaceOp(op, newop);
2566 return mlir::success();
2567}
2568
2569/// Replace CIR global with a region initialized LLVM global and update
2570/// insertion point to the end of the initializer block.
2571void CIRToLLVMGlobalOpLowering::setupRegionInitializedLLVMGlobalOp(
2572 cir::GlobalOp op, mlir::ConversionPatternRewriter &rewriter) const {
2573 const mlir::Type llvmType =
2574 convertTypeForMemory(*getTypeConverter(), dataLayout, op.getSymType());
2575
2576 // FIXME: These default values are placeholders until the the equivalent
2577 // attributes are available on cir.global ops. This duplicates code
2578 // in CIRToLLVMGlobalOpLowering::matchAndRewrite() but that will go
2579 // away when the placeholders are no longer needed.
2580 const bool isConst = op.getConstant();
2581 unsigned addrSpace = 0;
2582 if (auto targetAS = mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>(
2583 op.getAddrSpaceAttr()))
2584 addrSpace = targetAS.getValue();
2585 const bool isDsoLocal = op.getDsoLocal();
2586 const bool isThreadLocal = (bool)op.getTlsModelAttr();
2587 const uint64_t alignment = op.getAlignment().value_or(0);
2588 const mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());
2589 const StringRef symbol = op.getSymName();
2590 mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter);
2591
2592 SmallVector<mlir::NamedAttribute> attributes;
2593 mlir::LLVM::GlobalOp newGlobalOp =
2594 rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(
2595 op, llvmType, isConst, linkage, symbol, nullptr, alignment, addrSpace,
2596 isDsoLocal, isThreadLocal, comdatAttr, attributes);
2597 newGlobalOp.getRegion().emplaceBlock();
2598 rewriter.setInsertionPointToEnd(newGlobalOp.getInitializerBlock());
2599}
2600
2601mlir::LogicalResult
2602CIRToLLVMGlobalOpLowering::matchAndRewriteRegionInitializedGlobal(
2603 cir::GlobalOp op, mlir::Attribute init,
2604 mlir::ConversionPatternRewriter &rewriter) const {
2605 // TODO: Generalize this handling when more types are needed here.
2606 assert(
2607 (isa<cir::ConstArrayAttr, cir::ConstRecordAttr, cir::ConstVectorAttr,
2608 cir::ConstPtrAttr, cir::ConstComplexAttr, cir::GlobalViewAttr,
2609 cir::TypeInfoAttr, cir::UndefAttr, cir::VTableAttr, cir::ZeroAttr>(
2610 init)));
2611
2612 // TODO(cir): once LLVM's dialect has proper equivalent attributes this
2613 // should be updated. For now, we use a custom op to initialize globals
2614 // to the appropriate value.
2615 const mlir::Location loc = op.getLoc();
2616 setupRegionInitializedLLVMGlobalOp(op, rewriter);
2617 CIRAttrToValue valueConverter(op, rewriter, typeConverter);
2618 mlir::Value value = valueConverter.visit(init);
2619 mlir::LLVM::ReturnOp::create(rewriter, loc, value);
2620 return mlir::success();
2621}
2622
2623mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite(
2624 cir::GlobalOp op, OpAdaptor adaptor,
2625 mlir::ConversionPatternRewriter &rewriter) const {
2626 // If this global requires non-trivial initialization or destruction,
2627 // that needs to be moved to runtime handlers during LoweringPrepare.
2628 if (!op.getCtorRegion().empty() || !op.getDtorRegion().empty())
2629 return op.emitError() << "GlobalOp ctor and dtor regions should be removed "
2630 "in LoweringPrepare";
2631
2632 std::optional<mlir::Attribute> init = op.getInitialValue();
2633
2634 // Fetch required values to create LLVM op.
2635 const mlir::Type cirSymType = op.getSymType();
2636
2637 // This is the LLVM dialect type.
2638 const mlir::Type llvmType =
2639 convertTypeForMemory(*getTypeConverter(), dataLayout, cirSymType);
2640
2641 // FIXME: These default values are placeholders until the the equivalent
2642 // attributes are available on cir.global ops.
2643 const bool isConst = op.getConstant();
2644 unsigned addrSpace = 0;
2645 if (auto targetAS = mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>(
2646 op.getAddrSpaceAttr()))
2647 addrSpace = targetAS.getValue();
2648 const bool isDsoLocal = op.getDsoLocal();
2649 const bool isThreadLocal = (bool)op.getTlsModelAttr();
2650 const uint64_t alignment = op.getAlignment().value_or(0);
2651 const mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());
2652 const StringRef symbol = op.getSymName();
2653 SmallVector<mlir::NamedAttribute> attributes;
2654
2655 // Mark externally_initialized for __device__ and __constant__
2656 if (auto extInit =
2657 op->getAttr(CUDAExternallyInitializedAttr::getMnemonic())) {
2658 attributes.push_back(rewriter.getNamedAttr("externally_initialized",
2659 rewriter.getUnitAttr()));
2660 }
2661
2662 if (init.has_value()) {
2663 if (mlir::isa<cir::FPAttr, cir::IntAttr, cir::BoolAttr>(init.value())) {
2664 GlobalInitAttrRewriter initRewriter(llvmType, rewriter);
2665 init = initRewriter.visit(init.value());
2666 // If initRewriter returned a null attribute, init will have a value but
2667 // the value will be null. If that happens, initRewriter didn't handle the
2668 // attribute type. It probably needs to be added to
2669 // GlobalInitAttrRewriter.
2670 if (!init.value()) {
2671 op.emitError() << "unsupported initializer '" << init.value() << "'";
2672 return mlir::failure();
2673 }
2674 } else if (mlir::isa<cir::ConstArrayAttr, cir::ConstVectorAttr,
2675 cir::ConstRecordAttr, cir::ConstPtrAttr,
2676 cir::ConstComplexAttr, cir::GlobalViewAttr,
2677 cir::TypeInfoAttr, cir::UndefAttr, cir::VTableAttr,
2678 cir::ZeroAttr>(init.value())) {
2679 // TODO(cir): once LLVM's dialect has proper equivalent attributes this
2680 // should be updated. For now, we use a custom op to initialize globals
2681 // to the appropriate value.
2682 return matchAndRewriteRegionInitializedGlobal(op, init.value(), rewriter);
2683 } else {
2684 // We will only get here if new initializer types are added and this
2685 // code is not updated to handle them.
2686 op.emitError() << "unsupported initializer '" << init.value() << "'";
2687 return mlir::failure();
2688 }
2689 }
2690
2691 mlir::LLVM::Visibility visibility =
2692 lowerCIRVisibilityToLLVMVisibility(op.getGlobalVisibility());
2693 mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter);
2694 auto newOp = rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(
2695 op, llvmType, isConst, linkage, symbol, init.value_or(mlir::Attribute()),
2696 alignment, addrSpace, isDsoLocal, isThreadLocal, comdatAttr, attributes);
2697 newOp.setVisibility_(visibility);
2698
2699 return mlir::success();
2700}
2701
2702mlir::SymbolRefAttr
2703CIRToLLVMGlobalOpLowering::getComdatAttr(cir::GlobalOp &op,
2704 mlir::OpBuilder &builder) const {
2705 if (!op.getComdat())
2706 return mlir::SymbolRefAttr{};
2707
2708 mlir::ModuleOp module = op->getParentOfType<mlir::ModuleOp>();
2709 mlir::OpBuilder::InsertionGuard guard(builder);
2710 StringRef comdatName("__llvm_comdat_globals");
2711 if (!comdatOp) {
2712 builder.setInsertionPointToStart(module.getBody());
2713 comdatOp =
2714 mlir::LLVM::ComdatOp::create(builder, module.getLoc(), comdatName);
2715 }
2716
2717 if (auto comdatSelector = comdatOp.lookupSymbol<mlir::LLVM::ComdatSelectorOp>(
2718 op.getSymName())) {
2719 return mlir::SymbolRefAttr::get(
2720 builder.getContext(), comdatName,
2721 mlir::FlatSymbolRefAttr::get(comdatSelector.getSymNameAttr()));
2722 }
2723
2724 builder.setInsertionPointToStart(&comdatOp.getBody().back());
2725 auto selectorOp = mlir::LLVM::ComdatSelectorOp::create(
2726 builder, comdatOp.getLoc(), op.getSymName(),
2727 mlir::LLVM::comdat::Comdat::Any);
2728 return mlir::SymbolRefAttr::get(
2729 builder.getContext(), comdatName,
2730 mlir::FlatSymbolRefAttr::get(selectorOp.getSymNameAttr()));
2731}
2732
2733mlir::LogicalResult CIRToLLVMSwitchFlatOpLowering::matchAndRewrite(
2734 cir::SwitchFlatOp op, OpAdaptor adaptor,
2735 mlir::ConversionPatternRewriter &rewriter) const {
2736
2737 llvm::SmallVector<mlir::APInt, 8> caseValues;
2738 for (mlir::Attribute val : op.getCaseValues()) {
2739 auto intAttr = cast<cir::IntAttr>(val);
2740 caseValues.push_back(intAttr.getValue());
2741 }
2742
2743 llvm::SmallVector<mlir::Block *, 8> caseDestinations;
2744 llvm::SmallVector<mlir::ValueRange, 8> caseOperands;
2745
2746 for (mlir::Block *x : op.getCaseDestinations())
2747 caseDestinations.push_back(x);
2748
2749 for (mlir::OperandRange x : op.getCaseOperands())
2750 caseOperands.push_back(x);
2751
2752 // Set switch op to branch to the newly created blocks.
2753 rewriter.setInsertionPoint(op);
2754 rewriter.replaceOpWithNewOp<mlir::LLVM::SwitchOp>(
2755 op, adaptor.getCondition(), op.getDefaultDestination(),
2756 op.getDefaultOperands(), caseValues, caseDestinations, caseOperands);
2757 return mlir::success();
2758}
2759
2760static mlir::LLVM::IntegerOverflowFlags nswFlag(bool nsw) {
2761 return nsw ? mlir::LLVM::IntegerOverflowFlags::nsw
2762 : mlir::LLVM::IntegerOverflowFlags::none;
2763}
2764
2765template <typename CIROp, typename LLVMIntOp>
2766static mlir::LogicalResult
2767lowerIncDecOp(CIROp op, typename CIROp::Adaptor adaptor,
2768 mlir::ConversionPatternRewriter &rewriter, double fpConstant) {
2769 mlir::Type elementType = elementTypeIfVector(op.getType());
2770 mlir::Type llvmType = adaptor.getInput().getType();
2771 mlir::Location loc = op.getLoc();
2772
2773 if (mlir::isa<cir::IntType>(elementType)) {
2774 auto maybeNSW = nswFlag(op.getNoSignedWrap());
2775 auto one = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 1);
2776 rewriter.replaceOpWithNewOp<LLVMIntOp>(op, adaptor.getInput(), one,
2777 maybeNSW);
2778 return mlir::success();
2779 }
2780 if (mlir::isa<cir::FPTypeInterface>(elementType)) {
2781 auto fpConst = mlir::LLVM::ConstantOp::create(
2782 rewriter, loc, rewriter.getFloatAttr(llvmType, fpConstant));
2783 rewriter.replaceOpWithNewOp<mlir::LLVM::FAddOp>(op, fpConst,
2784 adaptor.getInput());
2785 return mlir::success();
2786 }
2787 return op.emitError() << "Unsupported type for IncOp/DecOp";
2788}
2789
2790mlir::LogicalResult CIRToLLVMIncOpLowering::matchAndRewrite(
2791 cir::IncOp op, OpAdaptor adaptor,
2792 mlir::ConversionPatternRewriter &rewriter) const {
2793 return lowerIncDecOp<cir::IncOp, mlir::LLVM::AddOp>(op, adaptor, rewriter,
2794 1.0);
2795}
2796
2797mlir::LogicalResult CIRToLLVMDecOpLowering::matchAndRewrite(
2798 cir::DecOp op, OpAdaptor adaptor,
2799 mlir::ConversionPatternRewriter &rewriter) const {
2800 return lowerIncDecOp<cir::DecOp, mlir::LLVM::SubOp>(op, adaptor, rewriter,
2801 -1.0);
2802}
2803
2804mlir::LogicalResult CIRToLLVMMinusOpLowering::matchAndRewrite(
2805 cir::MinusOp op, OpAdaptor adaptor,
2806 mlir::ConversionPatternRewriter &rewriter) const {
2807 mlir::Type elementType = elementTypeIfVector(op.getType());
2808 bool isVector = mlir::isa<cir::VectorType>(op.getType());
2809 mlir::Type llvmType = adaptor.getInput().getType();
2810 mlir::Location loc = op.getLoc();
2811
2812 if (mlir::isa<cir::IntType>(elementType)) {
2813 auto maybeNSW = nswFlag(op.getNoSignedWrap());
2814 mlir::Value zero;
2815 if (isVector)
2816 zero = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmType);
2817 else
2818 zero = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 0);
2819 rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>(op, zero, adaptor.getInput(),
2820 maybeNSW);
2821 return mlir::success();
2822 }
2823 if (mlir::isa<cir::FPTypeInterface>(elementType)) {
2824 rewriter.replaceOpWithNewOp<mlir::LLVM::FNegOp>(op, adaptor.getInput());
2825 return mlir::success();
2826 }
2827 return op.emitError() << "Unsupported type for unary minus";
2828}
2829
2830mlir::LogicalResult CIRToLLVMNotOpLowering::matchAndRewrite(
2831 cir::NotOp op, OpAdaptor adaptor,
2832 mlir::ConversionPatternRewriter &rewriter) const {
2833 mlir::Type elementType = elementTypeIfVector(op.getType());
2834 bool isVector = mlir::isa<cir::VectorType>(op.getType());
2835 mlir::Type llvmType = adaptor.getInput().getType();
2836 mlir::Location loc = op.getLoc();
2837
2838 if (mlir::isa<cir::IntType>(elementType)) {
2839 mlir::Value minusOne;
2840 if (isVector) {
2841 const uint64_t numElements =
2842 mlir::dyn_cast<cir::VectorType>(op.getType()).getSize();
2843 SmallVector<int32_t> values(numElements, -1);
2844 mlir::DenseIntElementsAttr denseVec = rewriter.getI32VectorAttr(values);
2845 minusOne =
2846 mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, denseVec);
2847 } else {
2848 minusOne = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, -1);
2849 }
2850 rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getInput(),
2851 minusOne);
2852 return mlir::success();
2853 }
2854 if (mlir::isa<cir::BoolType>(elementType)) {
2855 auto one = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 1);
2856 rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getInput(), one);
2857 return mlir::success();
2858 }
2859 return op.emitError() << "Unsupported type for bitwise NOT";
2860}
2861
2862static bool isIntTypeUnsigned(mlir::Type type) {
2863 // TODO: Ideally, we should only need to check cir::IntType here.
2864 return mlir::isa<cir::IntType>(type)
2865 ? mlir::cast<cir::IntType>(type).isUnsigned()
2866 : mlir::cast<mlir::IntegerType>(type).isUnsigned();
2867}
2868
2869//===----------------------------------------------------------------------===//
2870// Binary Op Lowering
2871//===----------------------------------------------------------------------===//
2872
2873template <typename BinOp>
2874static mlir::LLVM::IntegerOverflowFlags intOverflowFlag(BinOp op) {
2875 if (op.getNoUnsignedWrap())
2876 return mlir::LLVM::IntegerOverflowFlags::nuw;
2877 if (op.getNoSignedWrap())
2878 return mlir::LLVM::IntegerOverflowFlags::nsw;
2879 return mlir::LLVM::IntegerOverflowFlags::none;
2880}
2881
2882/// Lower an arithmetic op that supports saturation, overflow flags, and an FP
2883/// variant. Used for Add and Sub which share identical dispatch logic.
2884template <typename UIntSatOp, typename SIntSatOp, typename IntOp, typename FPOp,
2885 typename CIROp>
2886static mlir::LogicalResult
2887lowerSaturatableArithOp(CIROp op, mlir::Value lhs, mlir::Value rhs,
2888 mlir::ConversionPatternRewriter &rewriter) {
2889 const mlir::Type eltType = elementTypeIfVector(op.getRhs().getType());
2890 if (cir::isIntOrBoolType(eltType)) {
2891 if (op.getSaturated()) {
2892 if (isIntTypeUnsigned(eltType))
2893 rewriter.replaceOpWithNewOp<UIntSatOp>(op, lhs, rhs);
2894 else
2895 rewriter.replaceOpWithNewOp<SIntSatOp>(op, lhs, rhs);
2896 return mlir::success();
2897 }
2898 rewriter.replaceOpWithNewOp<IntOp>(op, lhs, rhs, intOverflowFlag(op));
2899 } else {
2900 rewriter.replaceOpWithNewOp<FPOp>(op, lhs, rhs);
2901 }
2902 return mlir::success();
2903}
2904
2905mlir::LogicalResult CIRToLLVMAddOpLowering::matchAndRewrite(
2906 cir::AddOp op, OpAdaptor adaptor,
2907 mlir::ConversionPatternRewriter &rewriter) const {
2908 return lowerSaturatableArithOp<mlir::LLVM::UAddSat, mlir::LLVM::SAddSat,
2909 mlir::LLVM::AddOp, mlir::LLVM::FAddOp>(
2910 op, adaptor.getLhs(), adaptor.getRhs(), rewriter);
2911}
2912
2913mlir::LogicalResult CIRToLLVMSubOpLowering::matchAndRewrite(
2914 cir::SubOp op, OpAdaptor adaptor,
2915 mlir::ConversionPatternRewriter &rewriter) const {
2916 return lowerSaturatableArithOp<mlir::LLVM::USubSat, mlir::LLVM::SSubSat,
2917 mlir::LLVM::SubOp, mlir::LLVM::FSubOp>(
2918 op, adaptor.getLhs(), adaptor.getRhs(), rewriter);
2919}
2920
2921mlir::LogicalResult CIRToLLVMMulOpLowering::matchAndRewrite(
2922 cir::MulOp op, OpAdaptor adaptor,
2923 mlir::ConversionPatternRewriter &rewriter) const {
2924 const mlir::Value lhs = adaptor.getLhs();
2925 const mlir::Value rhs = adaptor.getRhs();
2926 if (cir::isIntOrBoolType(elementTypeIfVector(op.getRhs().getType()))) {
2927 rewriter.replaceOpWithNewOp<mlir::LLVM::MulOp>(op, lhs, rhs,
2928 intOverflowFlag(op));
2929 } else {
2930 rewriter.replaceOpWithNewOp<mlir::LLVM::FMulOp>(op, lhs, rhs);
2931 }
2932 return mlir::success();
2933}
2934
2935/// Lower a binary op that maps to unsigned/signed/FP LLVM ops depending on
2936/// operand type. Used for Div and Rem which share identical dispatch logic.
2937template <typename UIntOp, typename SIntOp, typename FPOp, typename CIROp>
2938static mlir::LogicalResult
2939lowerIntFPBinaryOp(CIROp op, mlir::Value lhs, mlir::Value rhs,
2940 mlir::ConversionPatternRewriter &rewriter) {
2941 const mlir::Type eltType = elementTypeIfVector(op.getRhs().getType());
2942 if (cir::isIntOrBoolType(eltType)) {
2943 if (isIntTypeUnsigned(eltType))
2944 rewriter.replaceOpWithNewOp<UIntOp>(op, lhs, rhs);
2945 else
2946 rewriter.replaceOpWithNewOp<SIntOp>(op, lhs, rhs);
2947 } else {
2948 rewriter.replaceOpWithNewOp<FPOp>(op, lhs, rhs);
2949 }
2950 return mlir::success();
2951}
2952
2953mlir::LogicalResult CIRToLLVMDivOpLowering::matchAndRewrite(
2954 cir::DivOp op, OpAdaptor adaptor,
2955 mlir::ConversionPatternRewriter &rewriter) const {
2956 return lowerIntFPBinaryOp<mlir::LLVM::UDivOp, mlir::LLVM::SDivOp,
2957 mlir::LLVM::FDivOp>(op, adaptor.getLhs(),
2958 adaptor.getRhs(), rewriter);
2959}
2960
2961mlir::LogicalResult CIRToLLVMRemOpLowering::matchAndRewrite(
2962 cir::RemOp op, OpAdaptor adaptor,
2963 mlir::ConversionPatternRewriter &rewriter) const {
2964 return lowerIntFPBinaryOp<mlir::LLVM::URemOp, mlir::LLVM::SRemOp,
2965 mlir::LLVM::FRemOp>(op, adaptor.getLhs(),
2966 adaptor.getRhs(), rewriter);
2967}
2968
2969mlir::LogicalResult CIRToLLVMAndOpLowering::matchAndRewrite(
2970 cir::AndOp op, OpAdaptor adaptor,
2971 mlir::ConversionPatternRewriter &rewriter) const {
2972 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op, adaptor.getLhs(),
2973 adaptor.getRhs());
2974 return mlir::success();
2975}
2976
2977mlir::LogicalResult CIRToLLVMOrOpLowering::matchAndRewrite(
2978 cir::OrOp op, OpAdaptor adaptor,
2979 mlir::ConversionPatternRewriter &rewriter) const {
2980 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(op, adaptor.getLhs(),
2981 adaptor.getRhs());
2982 return mlir::success();
2983}
2984
2985mlir::LogicalResult CIRToLLVMXorOpLowering::matchAndRewrite(
2986 cir::XorOp op, OpAdaptor adaptor,
2987 mlir::ConversionPatternRewriter &rewriter) const {
2988 rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getLhs(),
2989 adaptor.getRhs());
2990 return mlir::success();
2991}
2992
2993template <typename CIROp, typename UIntOp, typename SIntOp>
2994static mlir::LogicalResult
2995lowerMinMaxOp(CIROp op, typename CIROp::Adaptor adaptor,
2996 mlir::ConversionPatternRewriter &rewriter) {
2997 const mlir::Value lhs = adaptor.getLhs();
2998 const mlir::Value rhs = adaptor.getRhs();
2999 if (isIntTypeUnsigned(elementTypeIfVector(op.getRhs().getType())))
3000 rewriter.replaceOpWithNewOp<UIntOp>(op, lhs, rhs);
3001 else
3002 rewriter.replaceOpWithNewOp<SIntOp>(op, lhs, rhs);
3003 return mlir::success();
3004}
3005
3006mlir::LogicalResult CIRToLLVMMaxOpLowering::matchAndRewrite(
3007 cir::MaxOp op, OpAdaptor adaptor,
3008 mlir::ConversionPatternRewriter &rewriter) const {
3009 return lowerMinMaxOp<cir::MaxOp, mlir::LLVM::UMaxOp, mlir::LLVM::SMaxOp>(
3010 op, adaptor, rewriter);
3011}
3012
3013mlir::LogicalResult CIRToLLVMMinOpLowering::matchAndRewrite(
3014 cir::MinOp op, OpAdaptor adaptor,
3015 mlir::ConversionPatternRewriter &rewriter) const {
3016 return lowerMinMaxOp<cir::MinOp, mlir::LLVM::UMinOp, mlir::LLVM::SMinOp>(
3017 op, adaptor, rewriter);
3018}
3019
3020/// Convert from a CIR comparison kind to an LLVM IR integral comparison kind.
3021static mlir::LLVM::ICmpPredicate
3022convertCmpKindToICmpPredicate(cir::CmpOpKind kind, bool isSigned) {
3023 using CIR = cir::CmpOpKind;
3024 using LLVMICmp = mlir::LLVM::ICmpPredicate;
3025 switch (kind) {
3026 case CIR::eq:
3027 return LLVMICmp::eq;
3028 case CIR::ne:
3029 return LLVMICmp::ne;
3030 case CIR::lt:
3031 return (isSigned ? LLVMICmp::slt : LLVMICmp::ult);
3032 case CIR::le:
3033 return (isSigned ? LLVMICmp::sle : LLVMICmp::ule);
3034 case CIR::gt:
3035 return (isSigned ? LLVMICmp::sgt : LLVMICmp::ugt);
3036 case CIR::ge:
3037 return (isSigned ? LLVMICmp::sge : LLVMICmp::uge);
3038 }
3039 llvm_unreachable("Unknown CmpOpKind");
3040}
3041
3042/// Convert from a CIR comparison kind to an LLVM IR floating-point comparison
3043/// kind.
3044static mlir::LLVM::FCmpPredicate
3045convertCmpKindToFCmpPredicate(cir::CmpOpKind kind) {
3046 using CIR = cir::CmpOpKind;
3047 using LLVMFCmp = mlir::LLVM::FCmpPredicate;
3048 switch (kind) {
3049 case CIR::eq:
3050 return LLVMFCmp::oeq;
3051 case CIR::ne:
3052 return LLVMFCmp::une;
3053 case CIR::lt:
3054 return LLVMFCmp::olt;
3055 case CIR::le:
3056 return LLVMFCmp::ole;
3057 case CIR::gt:
3058 return LLVMFCmp::ogt;
3059 case CIR::ge:
3060 return LLVMFCmp::oge;
3061 }
3062 llvm_unreachable("Unknown CmpOpKind");
3063}
3064
3065mlir::LogicalResult CIRToLLVMCmpOpLowering::matchAndRewrite(
3066 cir::CmpOp cmpOp, OpAdaptor adaptor,
3067 mlir::ConversionPatternRewriter &rewriter) const {
3068 mlir::Type type = cmpOp.getLhs().getType();
3069
3070 if (mlir::isa<cir::IntType, mlir::IntegerType>(type)) {
3071 bool isSigned = mlir::isa<cir::IntType>(type)
3072 ? mlir::cast<cir::IntType>(type).isSigned()
3073 : mlir::cast<mlir::IntegerType>(type).isSigned();
3074 mlir::LLVM::ICmpPredicate kind =
3075 convertCmpKindToICmpPredicate(cmpOp.getKind(), isSigned);
3076 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
3077 cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
3078 return mlir::success();
3079 }
3080
3081 if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(type)) {
3082 mlir::LLVM::ICmpPredicate kind =
3083 convertCmpKindToICmpPredicate(cmpOp.getKind(),
3084 /* isSigned=*/false);
3085 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
3086 cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
3087 return mlir::success();
3088 }
3089
3090 if (auto vptrTy = mlir::dyn_cast<cir::VPtrType>(type)) {
3091 // !cir.vptr is a special case, but it's just a pointer to LLVM.
3092 auto kind = convertCmpKindToICmpPredicate(cmpOp.getKind(),
3093 /* isSigned=*/false);
3094 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
3095 cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
3096 return mlir::success();
3097 }
3098
3099 if (mlir::isa<cir::FPTypeInterface>(type)) {
3100 mlir::LLVM::FCmpPredicate kind =
3101 convertCmpKindToFCmpPredicate(cmpOp.getKind());
3102 rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(
3103 cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
3104 return mlir::success();
3105 }
3106
3107 if (mlir::isa<cir::ComplexType>(type)) {
3108 mlir::Value lhs = adaptor.getLhs();
3109 mlir::Value rhs = adaptor.getRhs();
3110 mlir::Location loc = cmpOp.getLoc();
3111
3112 auto complexType = mlir::cast<cir::ComplexType>(cmpOp.getLhs().getType());
3113 mlir::Type complexElemTy =
3114 getTypeConverter()->convertType(complexType.getElementType());
3115
3116 auto lhsReal = mlir::LLVM::ExtractValueOp::create(
3117 rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0}));
3118 auto lhsImag = mlir::LLVM::ExtractValueOp::create(
3119 rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1}));
3120 auto rhsReal = mlir::LLVM::ExtractValueOp::create(
3121 rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0}));
3122 auto rhsImag = mlir::LLVM::ExtractValueOp::create(
3123 rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1}));
3124
3125 if (cmpOp.getKind() == cir::CmpOpKind::eq) {
3126 if (complexElemTy.isInteger()) {
3127 auto realCmp = mlir::LLVM::ICmpOp::create(
3128 rewriter, loc, mlir::LLVM::ICmpPredicate::eq, lhsReal, rhsReal);
3129 auto imagCmp = mlir::LLVM::ICmpOp::create(
3130 rewriter, loc, mlir::LLVM::ICmpPredicate::eq, lhsImag, rhsImag);
3131 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp);
3132 return mlir::success();
3133 }
3134
3135 auto realCmp = mlir::LLVM::FCmpOp::create(
3136 rewriter, loc, mlir::LLVM::FCmpPredicate::oeq, lhsReal, rhsReal);
3137 auto imagCmp = mlir::LLVM::FCmpOp::create(
3138 rewriter, loc, mlir::LLVM::FCmpPredicate::oeq, lhsImag, rhsImag);
3139 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp);
3140 return mlir::success();
3141 }
3142
3143 if (cmpOp.getKind() == cir::CmpOpKind::ne) {
3144 if (complexElemTy.isInteger()) {
3145 auto realCmp = mlir::LLVM::ICmpOp::create(
3146 rewriter, loc, mlir::LLVM::ICmpPredicate::ne, lhsReal, rhsReal);
3147 auto imagCmp = mlir::LLVM::ICmpOp::create(
3148 rewriter, loc, mlir::LLVM::ICmpPredicate::ne, lhsImag, rhsImag);
3149 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp);
3150 return mlir::success();
3151 }
3152
3153 auto realCmp = mlir::LLVM::FCmpOp::create(
3154 rewriter, loc, mlir::LLVM::FCmpPredicate::une, lhsReal, rhsReal);
3155 auto imagCmp = mlir::LLVM::FCmpOp::create(
3156 rewriter, loc, mlir::LLVM::FCmpPredicate::une, lhsImag, rhsImag);
3157 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp);
3158 return mlir::success();
3159 }
3160 }
3161
3162 return cmpOp.emitError() << "unsupported type for CmpOp: " << type;
3163}
3164
3165/// Shared lowering logic for checked binary arithmetic overflow operations.
3166/// The \p opStr parameter specifies the arithmetic operation name used in the
3167/// LLVM intrinsic (e.g., "add", "sub", "mul").
3168template <typename OpTy>
3169static mlir::LogicalResult
3170lowerBinOpOverflow(OpTy op, typename OpTy::Adaptor adaptor,
3171 mlir::ConversionPatternRewriter &rewriter,
3172 const mlir::TypeConverter *typeConverter,
3173 llvm::StringRef opStr) {
3174 mlir::Location loc = op.getLoc();
3175 cir::IntType operandTy = op.getLhs().getType();
3176 cir::IntType resultTy = op.getResult().getType();
3177
3178 bool sign = operandTy.getIsSigned() || resultTy.getIsSigned();
3179 unsigned width =
3180 std::max(operandTy.getWidth() + (sign && operandTy.isUnsigned()),
3181 resultTy.getWidth() + (sign && resultTy.isUnsigned()));
3182
3183 mlir::IntegerType encompassedLLVMTy = rewriter.getIntegerType(width);
3184
3185 mlir::Value lhs = adaptor.getLhs();
3186 mlir::Value rhs = adaptor.getRhs();
3187 if (operandTy.getWidth() < width) {
3188 if (operandTy.isSigned()) {
3189 lhs = mlir::LLVM::SExtOp::create(rewriter, loc, encompassedLLVMTy, lhs);
3190 rhs = mlir::LLVM::SExtOp::create(rewriter, loc, encompassedLLVMTy, rhs);
3191 } else {
3192 lhs = mlir::LLVM::ZExtOp::create(rewriter, loc, encompassedLLVMTy, lhs);
3193 rhs = mlir::LLVM::ZExtOp::create(rewriter, loc, encompassedLLVMTy, rhs);
3194 }
3195 }
3196
3197 // The intrinsic name is `@llvm.{s|u}{op}.with.overflow.i{width}`
3198 std::string intrinName = ("llvm." + llvm::Twine(sign ? 's' : 'u') + opStr +
3199 ".with.overflow.i" + llvm::Twine(width))
3200 .str();
3201 auto intrinNameAttr = mlir::StringAttr::get(op.getContext(), intrinName);
3202
3203 mlir::IntegerType overflowLLVMTy = rewriter.getI1Type();
3204 auto intrinRetTy = mlir::LLVM::LLVMStructType::getLiteral(
3205 rewriter.getContext(), {encompassedLLVMTy, overflowLLVMTy});
3206
3207 auto callLLVMIntrinOp = mlir::LLVM::CallIntrinsicOp::create(
3208 rewriter, loc, intrinRetTy, intrinNameAttr, mlir::ValueRange{lhs, rhs});
3209 mlir::Value intrinRet = callLLVMIntrinOp.getResult(0);
3210
3211 mlir::Value result = mlir::LLVM::ExtractValueOp::create(
3212 rewriter, loc, intrinRet, ArrayRef<int64_t>{0})
3213 .getResult();
3214 mlir::Value overflow = mlir::LLVM::ExtractValueOp::create(
3215 rewriter, loc, intrinRet, ArrayRef<int64_t>{1})
3216 .getResult();
3217
3218 if (resultTy.getWidth() < width) {
3219 mlir::Type resultLLVMTy = typeConverter->convertType(resultTy);
3220 auto truncResult =
3221 mlir::LLVM::TruncOp::create(rewriter, loc, resultLLVMTy, result);
3222
3223 // Extend the truncated result back to the encompassing type to check for
3224 // any overflows during the truncation.
3225 mlir::Value truncResultExt;
3226 if (resultTy.isSigned())
3227 truncResultExt = mlir::LLVM::SExtOp::create(
3228 rewriter, loc, encompassedLLVMTy, truncResult);
3229 else
3230 truncResultExt = mlir::LLVM::ZExtOp::create(
3231 rewriter, loc, encompassedLLVMTy, truncResult);
3232 auto truncOverflow = mlir::LLVM::ICmpOp::create(
3233 rewriter, loc, mlir::LLVM::ICmpPredicate::ne, truncResultExt, result);
3234
3235 result = truncResult;
3236 overflow = mlir::LLVM::OrOp::create(rewriter, loc, overflow, truncOverflow);
3237 }
3238
3239 mlir::Type boolLLVMTy =
3240 typeConverter->convertType(op.getOverflow().getType());
3241 if (boolLLVMTy != rewriter.getI1Type())
3242 overflow = mlir::LLVM::ZExtOp::create(rewriter, loc, boolLLVMTy, overflow);
3243
3244 rewriter.replaceOp(op, mlir::ValueRange{result, overflow});
3245
3246 return mlir::success();
3247}
3248
3249mlir::LogicalResult CIRToLLVMAddOverflowOpLowering::matchAndRewrite(
3250 cir::AddOverflowOp op, OpAdaptor adaptor,
3251 mlir::ConversionPatternRewriter &rewriter) const {
3252 return lowerBinOpOverflow(op, adaptor, rewriter, getTypeConverter(), "add");
3253}
3254
3255mlir::LogicalResult CIRToLLVMSubOverflowOpLowering::matchAndRewrite(
3256 cir::SubOverflowOp op, OpAdaptor adaptor,
3257 mlir::ConversionPatternRewriter &rewriter) const {
3258 return lowerBinOpOverflow(op, adaptor, rewriter, getTypeConverter(), "sub");
3259}
3260
3261mlir::LogicalResult CIRToLLVMMulOverflowOpLowering::matchAndRewrite(
3262 cir::MulOverflowOp op, OpAdaptor adaptor,
3263 mlir::ConversionPatternRewriter &rewriter) const {
3264 return lowerBinOpOverflow(op, adaptor, rewriter, getTypeConverter(), "mul");
3265}
3266
3267mlir::LogicalResult CIRToLLVMShiftOpLowering::matchAndRewrite(
3268 cir::ShiftOp op, OpAdaptor adaptor,
3269 mlir::ConversionPatternRewriter &rewriter) const {
3270 assert((op.getValue().getType() == op.getType()) &&
3271 "inconsistent operands' types NYI");
3272
3273 const mlir::Type llvmTy = getTypeConverter()->convertType(op.getType());
3274 mlir::Value amt = adaptor.getAmount();
3275 mlir::Value val = adaptor.getValue();
3276
3277 auto cirAmtTy = mlir::dyn_cast<cir::IntType>(op.getAmount().getType());
3278 bool isUnsigned;
3279 if (cirAmtTy) {
3280 auto cirValTy = mlir::cast<cir::IntType>(op.getValue().getType());
3281 isUnsigned = cirValTy.isUnsigned();
3282
3283 // Ensure shift amount is the same type as the value. Some undefined
3284 // behavior might occur in the casts below as per [C99 6.5.7.3].
3285 // Vector type shift amount needs no cast as type consistency is expected to
3286 // be already be enforced at CIRGen.
3287 if (cirAmtTy)
3288 amt = getLLVMIntCast(rewriter, amt, llvmTy, true, cirAmtTy.getWidth(),
3289 cirValTy.getWidth());
3290 } else {
3291 auto cirValVTy = mlir::cast<cir::VectorType>(op.getValue().getType());
3292 isUnsigned =
3293 mlir::cast<cir::IntType>(cirValVTy.getElementType()).isUnsigned();
3294 }
3295
3296 // Lower to the proper LLVM shift operation.
3297 if (op.getIsShiftleft()) {
3298 rewriter.replaceOpWithNewOp<mlir::LLVM::ShlOp>(op, llvmTy, val, amt);
3299 return mlir::success();
3300 }
3301
3302 if (isUnsigned)
3303 rewriter.replaceOpWithNewOp<mlir::LLVM::LShrOp>(op, llvmTy, val, amt);
3304 else
3305 rewriter.replaceOpWithNewOp<mlir::LLVM::AShrOp>(op, llvmTy, val, amt);
3306 return mlir::success();
3307}
3308
3309mlir::LogicalResult CIRToLLVMSelectOpLowering::matchAndRewrite(
3310 cir::SelectOp op, OpAdaptor adaptor,
3311 mlir::ConversionPatternRewriter &rewriter) const {
3312 auto getConstantBool = [](mlir::Value value) -> cir::BoolAttr {
3313 auto definingOp = value.getDefiningOp<cir::ConstantOp>();
3314 if (!definingOp)
3315 return {};
3316
3317 auto constValue = definingOp.getValueAttr<cir::BoolAttr>();
3318 if (!constValue)
3319 return {};
3320
3321 return constValue;
3322 };
3323
3324 // Two special cases in the LLVMIR codegen of select op:
3325 // - select %0, %1, false => and %0, %1
3326 // - select %0, true, %1 => or %0, %1
3327 if (mlir::isa<cir::BoolType>(op.getTrueValue().getType())) {
3328 cir::BoolAttr trueValue = getConstantBool(op.getTrueValue());
3329 cir::BoolAttr falseValue = getConstantBool(op.getFalseValue());
3330 if (falseValue && !falseValue.getValue()) {
3331 // select %0, %1, false => and %0, %1
3332 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op, adaptor.getCondition(),
3333 adaptor.getTrueValue());
3334 return mlir::success();
3335 }
3336 if (trueValue && trueValue.getValue()) {
3337 // select %0, true, %1 => or %0, %1
3338 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(op, adaptor.getCondition(),
3339 adaptor.getFalseValue());
3340 return mlir::success();
3341 }
3342 }
3343
3344 mlir::Value llvmCondition = adaptor.getCondition();
3345 rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(
3346 op, llvmCondition, adaptor.getTrueValue(), adaptor.getFalseValue());
3347
3348 return mlir::success();
3349}
3350
3351static void prepareTypeConverter(mlir::LLVMTypeConverter &converter,
3352 mlir::DataLayout &dataLayout) {
3353 converter.addConversion([&](cir::PointerType type) -> mlir::Type {
3354 mlir::ptr::MemorySpaceAttrInterface addrSpaceAttr = type.getAddrSpace();
3355 unsigned numericAS = 0;
3356
3357 if (auto langAsAttr =
3358 mlir::dyn_cast_if_present<cir::LangAddressSpaceAttr>(addrSpaceAttr))
3359 llvm_unreachable("lowering LangAddressSpaceAttr NYI");
3360 else if (auto targetAsAttr =
3361 mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>(
3362 addrSpaceAttr))
3363 numericAS = targetAsAttr.getValue();
3364 return mlir::LLVM::LLVMPointerType::get(type.getContext(), numericAS);
3365 });
3366 converter.addConversion([&](cir::VPtrType type) -> mlir::Type {
3367 assert(!cir::MissingFeatures::addressSpace());
3368 return mlir::LLVM::LLVMPointerType::get(type.getContext());
3369 });
3370 converter.addConversion([&](cir::ArrayType type) -> mlir::Type {
3371 mlir::Type ty =
3372 convertTypeForMemory(converter, dataLayout, type.getElementType());
3373 return mlir::LLVM::LLVMArrayType::get(ty, type.getSize());
3374 });
3375 converter.addConversion([&](cir::VectorType type) -> mlir::Type {
3376 const mlir::Type ty = converter.convertType(type.getElementType());
3377 return mlir::VectorType::get(type.getSize(), ty, {type.getIsScalable()});
3378 });
3379 converter.addConversion([&](cir::BoolType type) -> mlir::Type {
3380 return mlir::IntegerType::get(type.getContext(), 1,
3381 mlir::IntegerType::Signless);
3382 });
3383 converter.addConversion([&](cir::IntType type) -> mlir::Type {
3384 // LLVM doesn't work with signed types, so we drop the CIR signs here.
3385 return mlir::IntegerType::get(type.getContext(), type.getWidth());
3386 });
3387 converter.addConversion([&](cir::SingleType type) -> mlir::Type {
3388 return mlir::Float32Type::get(type.getContext());
3389 });
3390 converter.addConversion([&](cir::DoubleType type) -> mlir::Type {
3391 return mlir::Float64Type::get(type.getContext());
3392 });
3393 converter.addConversion([&](cir::FP80Type type) -> mlir::Type {
3394 return mlir::Float80Type::get(type.getContext());
3395 });
3396 converter.addConversion([&](cir::FP128Type type) -> mlir::Type {
3397 return mlir::Float128Type::get(type.getContext());
3398 });
3399 converter.addConversion([&](cir::LongDoubleType type) -> mlir::Type {
3400 return converter.convertType(type.getUnderlying());
3401 });
3402 converter.addConversion([&](cir::FP16Type type) -> mlir::Type {
3403 return mlir::Float16Type::get(type.getContext());
3404 });
3405 converter.addConversion([&](cir::BF16Type type) -> mlir::Type {
3406 return mlir::BFloat16Type::get(type.getContext());
3407 });
3408 converter.addConversion([&](cir::ComplexType type) -> mlir::Type {
3409 // A complex type is lowered to an LLVM struct that contains the real and
3410 // imaginary part as data fields.
3411 mlir::Type elementTy = converter.convertType(type.getElementType());
3412 mlir::Type structFields[2] = {elementTy, elementTy};
3413 return mlir::LLVM::LLVMStructType::getLiteral(type.getContext(),
3414 structFields);
3415 });
3416 converter.addConversion([&](cir::FuncType type) -> std::optional<mlir::Type> {
3417 auto result = converter.convertType(type.getReturnType());
3418 llvm::SmallVector<mlir::Type> arguments;
3419 arguments.reserve(type.getNumInputs());
3420 if (converter.convertTypes(type.getInputs(), arguments).failed())
3421 return std::nullopt;
3422 auto varArg = type.isVarArg();
3423 return mlir::LLVM::LLVMFunctionType::get(result, arguments, varArg);
3424 });
3425 converter.addConversion([&](cir::RecordType type) -> mlir::Type {
3426 // Convert struct members.
3427 llvm::SmallVector<mlir::Type> llvmMembers;
3428 switch (type.getKind()) {
3429 case cir::RecordType::Class:
3430 case cir::RecordType::Struct:
3431 for (mlir::Type ty : type.getMembers())
3432 llvmMembers.push_back(convertTypeForMemory(converter, dataLayout, ty));
3433 break;
3434 // Unions are lowered as only the largest member.
3435 case cir::RecordType::Union:
3436 if (type.getMembers().empty())
3437 break;
3438 if (auto largestMember = type.getLargestMember(dataLayout))
3439 llvmMembers.push_back(
3440 convertTypeForMemory(converter, dataLayout, largestMember));
3441 if (type.getPadded()) {
3442 auto last = *type.getMembers().rbegin();
3443 llvmMembers.push_back(
3444 convertTypeForMemory(converter, dataLayout, last));
3445 }
3446 break;
3447 }
3448
3449 // Record has a name: lower as an identified record.
3450 mlir::LLVM::LLVMStructType llvmStruct;
3451 if (type.getName()) {
3452 llvmStruct = mlir::LLVM::LLVMStructType::getIdentified(
3453 type.getContext(), type.getPrefixedName());
3454 if (llvmStruct.setBody(llvmMembers, type.getPacked()).failed())
3455 llvm_unreachable("Failed to set body of record");
3456 } else { // Record has no name: lower as literal record.
3457 llvmStruct = mlir::LLVM::LLVMStructType::getLiteral(
3458 type.getContext(), llvmMembers, type.getPacked());
3459 }
3460
3461 return llvmStruct;
3462 });
3463 converter.addConversion([&](cir::VoidType type) -> mlir::Type {
3464 return mlir::LLVM::LLVMVoidType::get(type.getContext());
3465 });
3466}
3467
3468static void buildCtorDtorList(
3469 mlir::ModuleOp module, StringRef globalXtorName, StringRef llvmXtorName,
3470 llvm::function_ref<std::pair<StringRef, int>(mlir::Attribute)> createXtor) {
3471 llvm::SmallVector<std::pair<StringRef, int>> globalXtors;
3472 for (const mlir::NamedAttribute namedAttr : module->getAttrs()) {
3473 if (namedAttr.getName() == globalXtorName) {
3474 for (auto attr : mlir::cast<mlir::ArrayAttr>(namedAttr.getValue()))
3475 globalXtors.emplace_back(createXtor(attr));
3476 break;
3477 }
3478 }
3479
3480 if (globalXtors.empty())
3481 return;
3482
3483 mlir::OpBuilder builder(module.getContext());
3484 builder.setInsertionPointToEnd(&module.getBodyRegion().back());
3485
3486 // Create a global array llvm.global_ctors with element type of
3487 // struct { i32, ptr, ptr }
3488 auto ctorPFTy = mlir::LLVM::LLVMPointerType::get(builder.getContext());
3489 llvm::SmallVector<mlir::Type> ctorStructFields;
3490 ctorStructFields.push_back(builder.getI32Type());
3491 ctorStructFields.push_back(ctorPFTy);
3492 ctorStructFields.push_back(ctorPFTy);
3493
3494 auto ctorStructTy = mlir::LLVM::LLVMStructType::getLiteral(
3495 builder.getContext(), ctorStructFields);
3496 auto ctorStructArrayTy =
3497 mlir::LLVM::LLVMArrayType::get(ctorStructTy, globalXtors.size());
3498
3499 mlir::Location loc = module.getLoc();
3500 auto newGlobalOp = mlir::LLVM::GlobalOp::create(
3501 builder, loc, ctorStructArrayTy, /*constant=*/false,
3502 mlir::LLVM::Linkage::Appending, llvmXtorName, mlir::Attribute());
3503
3504 builder.createBlock(&newGlobalOp.getRegion());
3505 builder.setInsertionPointToEnd(newGlobalOp.getInitializerBlock());
3506
3507 mlir::Value result =
3508 mlir::LLVM::UndefOp::create(builder, loc, ctorStructArrayTy);
3509
3510 for (auto [index, fn] : llvm::enumerate(globalXtors)) {
3511 mlir::Value structInit =
3512 mlir::LLVM::UndefOp::create(builder, loc, ctorStructTy);
3513 mlir::Value initPriority = mlir::LLVM::ConstantOp::create(
3514 builder, loc, ctorStructFields[0], fn.second);
3515 mlir::Value initFuncAddr = mlir::LLVM::AddressOfOp::create(
3516 builder, loc, ctorStructFields[1], fn.first);
3517 mlir::Value initAssociate =
3518 mlir::LLVM::ZeroOp::create(builder, loc, ctorStructFields[2]);
3519 // Literal zero makes the InsertValueOp::create ambiguous.
3520 llvm::SmallVector<int64_t> zero{0};
3521 structInit = mlir::LLVM::InsertValueOp::create(builder, loc, structInit,
3522 initPriority, zero);
3523 structInit = mlir::LLVM::InsertValueOp::create(builder, loc, structInit,
3524 initFuncAddr, 1);
3525 // TODO: handle associated data for initializers.
3526 structInit = mlir::LLVM::InsertValueOp::create(builder, loc, structInit,
3527 initAssociate, 2);
3528 result = mlir::LLVM::InsertValueOp::create(builder, loc, result, structInit,
3529 index);
3530 }
3531
3532 mlir::LLVM::ReturnOp::create(builder, loc, result);
3533}
3534
3535// The applyPartialConversion function traverses blocks in the dominance order,
3536// so it does not lower and operations that are not reachachable from the
3537// operations passed in as arguments. Since we do need to lower such code in
3538// order to avoid verification errors occur, we cannot just pass the module op
3539// to applyPartialConversion. We must build a set of unreachable ops and
3540// explicitly add them, along with the module, to the vector we pass to
3541// applyPartialConversion.
3542//
3543// For instance, this CIR code:
3544//
3545// cir.func @foo(%arg0: !s32i) -> !s32i {
3546// %4 = cir.cast int_to_bool %arg0 : !s32i -> !cir.bool
3547// cir.if %4 {
3548// %5 = cir.const #cir.int<1> : !s32i
3549// cir.return %5 : !s32i
3550// } else {
3551// %5 = cir.const #cir.int<0> : !s32i
3552// cir.return %5 : !s32i
3553// }
3554// cir.return %arg0 : !s32i
3555// }
3556//
3557// contains an unreachable return operation (the last one). After the flattening
3558// pass it will be placed into the unreachable block. The possible error
3559// after the lowering pass is: error: 'cir.return' op expects parent op to be
3560// one of 'cir.func, cir.scope, cir.if ... The reason that this operation was
3561// not lowered and the new parent is llvm.func.
3562//
3563// In the future we may want to get rid of this function and use a DCE pass or
3564// something similar. But for now we need to guarantee the absence of the
3565// dialect verification errors.
3566static void collectUnreachable(mlir::Operation *parent,
3567 llvm::SmallVector<mlir::Operation *> &ops) {
3568
3569 llvm::SmallVector<mlir::Block *> unreachableBlocks;
3570 parent->walk([&](mlir::Block *blk) { // check
3571 if (blk->hasNoPredecessors() && !blk->isEntryBlock())
3572 unreachableBlocks.push_back(blk);
3573 });
3574
3575 std::set<mlir::Block *> visited;
3576 for (mlir::Block *root : unreachableBlocks) {
3577 // We create a work list for each unreachable block.
3578 // Thus we traverse operations in some order.
3579 std::deque<mlir::Block *> workList;
3580 workList.push_back(root);
3581
3582 while (!workList.empty()) {
3583 mlir::Block *blk = workList.back();
3584 workList.pop_back();
3585 if (visited.count(blk))
3586 continue;
3587 visited.emplace(blk);
3588
3589 for (mlir::Operation &op : *blk)
3590 ops.push_back(&op);
3591
3592 for (mlir::Block *succ : blk->getSuccessors())
3593 workList.push_back(succ);
3594 }
3595 }
3596}
3597
3598mlir::LogicalResult CIRToLLVMObjSizeOpLowering::matchAndRewrite(
3599 cir::ObjSizeOp op, OpAdaptor adaptor,
3600 mlir::ConversionPatternRewriter &rewriter) const {
3601 mlir::Type llvmResTy = getTypeConverter()->convertType(op.getType());
3602 mlir::Location loc = op->getLoc();
3603
3604 mlir::IntegerType i1Ty = rewriter.getI1Type();
3605
3606 auto i1Val = [&rewriter, &loc, &i1Ty](bool val) {
3607 return mlir::LLVM::ConstantOp::create(rewriter, loc, i1Ty, val);
3608 };
3609
3610 replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.objectsize", llvmResTy,
3611 {
3612 adaptor.getPtr(),
3613 i1Val(op.getMin()),
3614 i1Val(op.getNullunknown()),
3615 i1Val(op.getDynamic()),
3616 });
3617
3618 return mlir::LogicalResult::success();
3619}
3620
3621void ConvertCIRToLLVMPass::resolveBlockAddressOp(
3622 LLVMBlockAddressInfo &blockInfoAddr) {
3623
3624 mlir::ModuleOp module = getOperation();
3625 mlir::OpBuilder opBuilder(module.getContext());
3626 for (auto &[blockAddOp, blockInfo] :
3627 blockInfoAddr.getUnresolvedBlockAddress()) {
3628 mlir::LLVM::BlockTagOp resolvedLabel =
3629 blockInfoAddr.lookupBlockTag(blockInfo);
3630 assert(resolvedLabel && "expected BlockTagOp to already be emitted");
3631 mlir::FlatSymbolRefAttr fnSym = blockInfo.getFunc();
3632 auto blkAddTag = mlir::LLVM::BlockAddressAttr::get(
3633 opBuilder.getContext(), fnSym, resolvedLabel.getTagAttr());
3634 blockAddOp.setBlockAddrAttr(blkAddTag);
3635 }
3636 blockInfoAddr.clearUnresolvedMap();
3637}
3638
3639void ConvertCIRToLLVMPass::processCIRAttrs(mlir::ModuleOp module) {
3640 // Lower the module attributes to LLVM equivalents.
3641 if (mlir::Attribute tripleAttr =
3642 module->getAttr(cir::CIRDialect::getTripleAttrName()))
3643 module->setAttr(mlir::LLVM::LLVMDialect::getTargetTripleAttrName(),
3644 tripleAttr);
3645
3646 if (mlir::Attribute asmAttr =
3647 module->getAttr(cir::CIRDialect::getModuleLevelAsmAttrName()))
3648 module->setAttr(mlir::LLVM::LLVMDialect::getModuleLevelAsmAttrName(),
3649 asmAttr);
3650}
3651
3652void ConvertCIRToLLVMPass::runOnOperation() {
3653 llvm::TimeTraceScope scope("Convert CIR to LLVM Pass");
3654
3655 mlir::ModuleOp module = getOperation();
3656 mlir::DataLayout dl(module);
3657 mlir::LLVMTypeConverter converter(&getContext());
3658 prepareTypeConverter(converter, dl);
3659
3660 /// Tracks the state required to lower CIR `LabelOp` and `BlockAddressOp`.
3661 /// Maps labels to their corresponding `BlockTagOp` and keeps bookkeeping
3662 /// of unresolved `BlockAddressOp`s until they are matched with the
3663 /// corresponding `BlockTagOp` in `resolveBlockAddressOp`.
3664 LLVMBlockAddressInfo blockInfoAddr;
3665 mlir::RewritePatternSet patterns(&getContext());
3666 patterns.add<CIRToLLVMBlockAddressOpLowering, CIRToLLVMLabelOpLowering>(
3667 converter, patterns.getContext(), dl, blockInfoAddr);
3668
3669 patterns.add<
3670#define GET_LLVM_LOWERING_PATTERNS_LIST
3671#include "clang/CIR/Dialect/IR/CIRLowering.inc"
3672#undef GET_LLVM_LOWERING_PATTERNS_LIST
3673 >(converter, patterns.getContext(), dl);
3674
3675 processCIRAttrs(module);
3676
3677 mlir::ConversionTarget target(getContext());
3678 target.addLegalOp<mlir::ModuleOp>();
3679 target.addLegalDialect<mlir::LLVM::LLVMDialect>();
3680 target.addIllegalDialect<mlir::BuiltinDialect, cir::CIRDialect,
3681 mlir::func::FuncDialect>();
3682
3683 llvm::SmallVector<mlir::Operation *> ops;
3684 ops.push_back(module);
3685 collectUnreachable(module, ops);
3686
3687 if (failed(applyPartialConversion(ops, target, std::move(patterns))))
3688 signalPassFailure();
3689
3690 // Emit the llvm.global_ctors array.
3691 buildCtorDtorList(module, cir::CIRDialect::getGlobalCtorsAttrName(),
3692 "llvm.global_ctors", [](mlir::Attribute attr) {
3693 auto ctorAttr = mlir::cast<cir::GlobalCtorAttr>(attr);
3694 return std::make_pair(ctorAttr.getName(),
3695 ctorAttr.getPriority());
3696 });
3697 // Emit the llvm.global_dtors array.
3698 buildCtorDtorList(module, cir::CIRDialect::getGlobalDtorsAttrName(),
3699 "llvm.global_dtors", [](mlir::Attribute attr) {
3700 auto dtorAttr = mlir::cast<cir::GlobalDtorAttr>(attr);
3701 return std::make_pair(dtorAttr.getName(),
3702 dtorAttr.getPriority());
3703 });
3704 resolveBlockAddressOp(blockInfoAddr);
3705}
3706
3707mlir::LogicalResult CIRToLLVMBrOpLowering::matchAndRewrite(
3708 cir::BrOp op, OpAdaptor adaptor,
3709 mlir::ConversionPatternRewriter &rewriter) const {
3710 rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(op, adaptor.getOperands(),
3711 op.getDest());
3712 return mlir::LogicalResult::success();
3713}
3714
3715mlir::LogicalResult CIRToLLVMGetMemberOpLowering::matchAndRewrite(
3716 cir::GetMemberOp op, OpAdaptor adaptor,
3717 mlir::ConversionPatternRewriter &rewriter) const {
3718 mlir::Type llResTy = getTypeConverter()->convertType(op.getType());
3719 const auto recordTy =
3720 mlir::cast<cir::RecordType>(op.getAddrTy().getPointee());
3721 assert(recordTy && "expected record type");
3722
3723 switch (recordTy.getKind()) {
3724 case cir::RecordType::Class:
3725 case cir::RecordType::Struct: {
3726 // Since the base address is a pointer to an aggregate, the first offset
3727 // is always zero. The second offset tell us which member it will access.
3728 llvm::SmallVector<mlir::LLVM::GEPArg, 2> offset{0, op.getIndex()};
3729 const mlir::Type elementTy = getTypeConverter()->convertType(recordTy);
3730 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, llResTy, elementTy,
3731 adaptor.getAddr(), offset);
3732 return mlir::success();
3733 }
3734 case cir::RecordType::Union:
3735 // Union members share the address space, so we just need a bitcast to
3736 // conform to type-checking.
3737 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(op, llResTy,
3738 adaptor.getAddr());
3739 return mlir::success();
3740 }
3741}
3742
3743mlir::LogicalResult CIRToLLVMExtractMemberOpLowering::matchAndRewrite(
3744 cir::ExtractMemberOp op, OpAdaptor adaptor,
3745 mlir::ConversionPatternRewriter &rewriter) const {
3746 std::int64_t indices[1] = {static_cast<std::int64_t>(op.getIndex())};
3747
3748 mlir::Type recordTy = op.getRecord().getType();
3749 auto cirRecordTy = mlir::cast<cir::RecordType>(recordTy);
3750 switch (cirRecordTy.getKind()) {
3751 case cir::RecordType::Struct:
3752 case cir::RecordType::Class:
3753 rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractValueOp>(
3754 op, adaptor.getRecord(), indices);
3755 return mlir::success();
3756
3757 case cir::RecordType::Union:
3758 op.emitError("cir.extract_member cannot extract member from a union");
3759 return mlir::failure();
3760 }
3761 llvm_unreachable("Unexpected record kind");
3762}
3763
3764mlir::LogicalResult CIRToLLVMInsertMemberOpLowering::matchAndRewrite(
3765 cir::InsertMemberOp op, OpAdaptor adaptor,
3766 mlir::ConversionPatternRewriter &rewriter) const {
3767 std::int64_t indecies[1] = {static_cast<std::int64_t>(op.getIndex())};
3768 mlir::Type recordTy = op.getRecord().getType();
3769
3770 if (auto cirRecordTy = mlir::dyn_cast<cir::RecordType>(recordTy)) {
3771 if (cirRecordTy.getKind() == cir::RecordType::Union) {
3772 op.emitError("cir.update_member cannot update member of a union");
3773 return mlir::failure();
3774 }
3775 }
3776
3777 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(
3778 op, adaptor.getRecord(), adaptor.getValue(), indecies);
3779 return mlir::success();
3780}
3781
3782mlir::LogicalResult CIRToLLVMUnreachableOpLowering::matchAndRewrite(
3783 cir::UnreachableOp op, OpAdaptor adaptor,
3784 mlir::ConversionPatternRewriter &rewriter) const {
3785 rewriter.replaceOpWithNewOp<mlir::LLVM::UnreachableOp>(op);
3786 return mlir::success();
3787}
3788
3789void createLLVMFuncOpIfNotExist(mlir::ConversionPatternRewriter &rewriter,
3790 mlir::Operation *srcOp, llvm::StringRef fnName,
3791 mlir::Type fnTy) {
3792 auto modOp = srcOp->getParentOfType<mlir::ModuleOp>();
3793 auto enclosingFnOp = srcOp->getParentOfType<mlir::LLVM::LLVMFuncOp>();
3794 mlir::Operation *sourceSymbol =
3795 mlir::SymbolTable::lookupSymbolIn(modOp, fnName);
3796 if (!sourceSymbol) {
3797 mlir::OpBuilder::InsertionGuard guard(rewriter);
3798 rewriter.setInsertionPoint(enclosingFnOp);
3799 mlir::LLVM::LLVMFuncOp::create(rewriter, srcOp->getLoc(), fnName, fnTy);
3800 }
3801}
3802
3803mlir::LogicalResult CIRToLLVMThrowOpLowering::matchAndRewrite(
3804 cir::ThrowOp op, OpAdaptor adaptor,
3805 mlir::ConversionPatternRewriter &rewriter) const {
3806 mlir::Location loc = op.getLoc();
3807 auto voidTy = mlir::LLVM::LLVMVoidType::get(getContext());
3808
3809 if (op.rethrows()) {
3810 auto funcTy = mlir::LLVM::LLVMFunctionType::get(voidTy, {});
3811
3812 // Get or create `declare void @__cxa_rethrow()`
3813 const llvm::StringRef functionName = "__cxa_rethrow";
3814 createLLVMFuncOpIfNotExist(rewriter, op, functionName, funcTy);
3815
3816 auto cxaRethrow = mlir::LLVM::CallOp::create(
3817 rewriter, loc, mlir::TypeRange{}, functionName);
3818
3819 rewriter.replaceOp(op, cxaRethrow);
3820 return mlir::success();
3821 }
3822
3823 auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
3824 auto fnTy = mlir::LLVM::LLVMFunctionType::get(
3825 voidTy, {llvmPtrTy, llvmPtrTy, llvmPtrTy});
3826
3827 // Get or create `declare void @__cxa_throw(ptr, ptr, ptr)`
3828 const llvm::StringRef fnName = "__cxa_throw";
3829 createLLVMFuncOpIfNotExist(rewriter, op, fnName, fnTy);
3830
3831 mlir::Value typeInfo = mlir::LLVM::AddressOfOp::create(
3832 rewriter, loc, mlir::LLVM::LLVMPointerType::get(rewriter.getContext()),
3833 adaptor.getTypeInfoAttr());
3834
3835 mlir::Value dtor;
3836 if (op.getDtor()) {
3837 dtor = mlir::LLVM::AddressOfOp::create(rewriter, loc, llvmPtrTy,
3838 adaptor.getDtorAttr());
3839 } else {
3840 dtor = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmPtrTy);
3841 }
3842
3843 auto cxaThrowCall = mlir::LLVM::CallOp::create(
3844 rewriter, loc, mlir::TypeRange{}, fnName,
3845 mlir::ValueRange{adaptor.getExceptionPtr(), typeInfo, dtor});
3846
3847 rewriter.replaceOp(op, cxaThrowCall);
3848 return mlir::success();
3849}
3850
3851mlir::LogicalResult CIRToLLVMAllocExceptionOpLowering::matchAndRewrite(
3852 cir::AllocExceptionOp op, OpAdaptor adaptor,
3853 mlir::ConversionPatternRewriter &rewriter) const {
3854 // Get or create `declare ptr @__cxa_allocate_exception(i64)`
3855 StringRef fnName = "__cxa_allocate_exception";
3856 auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
3857 auto int64Ty = mlir::IntegerType::get(rewriter.getContext(), 64);
3858 auto fnTy = mlir::LLVM::LLVMFunctionType::get(llvmPtrTy, {int64Ty});
3859
3860 createLLVMFuncOpIfNotExist(rewriter, op, fnName, fnTy);
3861 auto exceptionSize = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
3862 adaptor.getSizeAttr());
3863
3864 auto allocaExceptionCall = mlir::LLVM::CallOp::create(
3865 rewriter, op.getLoc(), mlir::TypeRange{llvmPtrTy}, fnName,
3866 mlir::ValueRange{exceptionSize});
3867
3868 rewriter.replaceOp(op, allocaExceptionCall);
3869 return mlir::success();
3870}
3871
3872static mlir::LLVM::LLVMStructType
3873getLLVMLandingPadStructTy(mlir::ConversionPatternRewriter &rewriter) {
3874 // Create the landing pad type: struct { ptr, i32 }
3875 mlir::MLIRContext *ctx = rewriter.getContext();
3876 auto llvmPtr = mlir::LLVM::LLVMPointerType::get(ctx);
3877 llvm::SmallVector<mlir::Type> structFields = {llvmPtr, rewriter.getI32Type()};
3878 return mlir::LLVM::LLVMStructType::getLiteral(ctx, structFields);
3879}
3880
3881mlir::LogicalResult CIRToLLVMEhInflightOpLowering::matchAndRewrite(
3882 cir::EhInflightOp op, OpAdaptor adaptor,
3883 mlir::ConversionPatternRewriter &rewriter) const {
3884 auto llvmFn = op->getParentOfType<mlir::LLVM::LLVMFuncOp>();
3885 assert(llvmFn && "expected LLVM function parent");
3886 mlir::Block *entryBlock = &llvmFn.getRegion().front();
3887 assert(entryBlock->isEntryBlock());
3888
3889 mlir::ArrayAttr catchListAttr = op.getCatchTypeListAttr();
3890 mlir::SmallVector<mlir::Value> catchSymAddrs;
3891
3892 auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
3893 mlir::Location loc = op.getLoc();
3894
3895 // %landingpad = landingpad { ptr, i32 }
3896 // Note that since llvm.landingpad has to be the first operation on the
3897 // block, any needed value for its operands has to be added somewhere else.
3898 if (catchListAttr) {
3899 // catch ptr @_ZTIi
3900 // catch ptr @_ZTIPKc
3901 for (mlir::Attribute catchAttr : catchListAttr) {
3902 auto symAttr = cast<mlir::FlatSymbolRefAttr>(catchAttr);
3903 // Generate `llvm.mlir.addressof` for each symbol, and place those
3904 // operations in the LLVM function entry basic block.
3905 mlir::OpBuilder::InsertionGuard guard(rewriter);
3906 rewriter.setInsertionPointToStart(entryBlock);
3907 mlir::Value addrOp = mlir::LLVM::AddressOfOp::create(
3908 rewriter, loc, llvmPtrTy, symAttr.getValue());
3909 catchSymAddrs.push_back(addrOp);
3910 }
3911 } else if (!op.getCleanup()) {
3912 // We need to emit catch-all only if cleanup is not set, because when we
3913 // have catch-all handler, there is no case when we set would unwind past
3914 // the handler
3915 mlir::OpBuilder::InsertionGuard guard(rewriter);
3916 rewriter.setInsertionPointToStart(entryBlock);
3917 mlir::Value nullOp = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmPtrTy);
3918 catchSymAddrs.push_back(nullOp);
3919 }
3920
3921 // %slot = extractvalue { ptr, i32 } %x, 0
3922 // %selector = extractvalue { ptr, i32 } %x, 1
3923 mlir::LLVM::LLVMStructType llvmLandingPadStructTy =
3924 getLLVMLandingPadStructTy(rewriter);
3925 auto landingPadOp = mlir::LLVM::LandingpadOp::create(
3926 rewriter, loc, llvmLandingPadStructTy, catchSymAddrs);
3927
3928 if (op.getCleanup())
3929 landingPadOp.setCleanup(true);
3930
3931 mlir::Value slot =
3932 mlir::LLVM::ExtractValueOp::create(rewriter, loc, landingPadOp, 0);
3933 mlir::Value selector =
3934 mlir::LLVM::ExtractValueOp::create(rewriter, loc, landingPadOp, 1);
3935 rewriter.replaceOp(op, mlir::ValueRange{slot, selector});
3936
3937 return mlir::success();
3938}
3939
3940mlir::LogicalResult CIRToLLVMResumeFlatOpLowering::matchAndRewrite(
3941 cir::ResumeFlatOp op, OpAdaptor adaptor,
3942 mlir::ConversionPatternRewriter &rewriter) const {
3943 // %lpad.val = insertvalue { ptr, i32 } poison, ptr %exception_ptr, 0
3944 // %lpad.val2 = insertvalue { ptr, i32 } %lpad.val, i32 %selector, 1
3945 // resume { ptr, i32 } %lpad.val2
3946 mlir::Type llvmLandingPadStructTy = getLLVMLandingPadStructTy(rewriter);
3947 mlir::Value poison = mlir::LLVM::PoisonOp::create(rewriter, op.getLoc(),
3948 llvmLandingPadStructTy);
3949
3950 SmallVector<int64_t> slotIdx = {0};
3951 mlir::Value slot = mlir::LLVM::InsertValueOp::create(
3952 rewriter, op.getLoc(), poison, adaptor.getExceptionPtr(), slotIdx);
3953
3954 SmallVector<int64_t> selectorIdx = {1};
3955 mlir::Value selector = mlir::LLVM::InsertValueOp::create(
3956 rewriter, op.getLoc(), slot, adaptor.getTypeId(), selectorIdx);
3957
3958 rewriter.replaceOpWithNewOp<mlir::LLVM::ResumeOp>(op, selector);
3959 return mlir::success();
3960}
3961
3962mlir::LogicalResult CIRToLLVMEhTypeIdOpLowering::matchAndRewrite(
3963 cir::EhTypeIdOp op, OpAdaptor adaptor,
3964 mlir::ConversionPatternRewriter &rewriter) const {
3965 mlir::Value addrOp = mlir::LLVM::AddressOfOp::create(
3966 rewriter, op.getLoc(),
3967 mlir::LLVM::LLVMPointerType::get(rewriter.getContext()),
3968 op.getTypeSymAttr());
3969 rewriter.replaceOpWithNewOp<mlir::LLVM::EhTypeidForOp>(
3970 op, rewriter.getI32Type(), addrOp);
3971 return mlir::success();
3972}
3973
3974mlir::LogicalResult CIRToLLVMEhSetjmpOpLowering::matchAndRewrite(
3975 cir::EhSetjmpOp op, OpAdaptor adaptor,
3976 mlir::ConversionPatternRewriter &rewriter) const {
3977 mlir::Type returnType = typeConverter->convertType(op.getType());
3978 mlir::LLVM::CallIntrinsicOp newOp =
3979 createCallLLVMIntrinsicOp(rewriter, op.getLoc(), "llvm.eh.sjlj.setjmp",
3980 returnType, adaptor.getEnv());
3981 rewriter.replaceOp(op, newOp);
3982 return mlir::success();
3983}
3984
3985mlir::LogicalResult CIRToLLVMEhLongjmpOpLowering::matchAndRewrite(
3986 cir::EhLongjmpOp op, OpAdaptor adaptor,
3987 mlir::ConversionPatternRewriter &rewriter) const {
3988 replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.eh.sjlj.longjmp",
3989 /*resultTy=*/{}, adaptor.getOperands());
3990 return mlir::success();
3991}
3992
3993mlir::LogicalResult CIRToLLVMTrapOpLowering::matchAndRewrite(
3994 cir::TrapOp op, OpAdaptor adaptor,
3995 mlir::ConversionPatternRewriter &rewriter) const {
3996 mlir::Location loc = op->getLoc();
3997 rewriter.eraseOp(op);
3998
3999 mlir::LLVM::Trap::create(rewriter, loc);
4000
4001 // Note that the call to llvm.trap is not a terminator in LLVM dialect.
4002 // So we must emit an additional llvm.unreachable to terminate the current
4003 // block.
4004 mlir::LLVM::UnreachableOp::create(rewriter, loc);
4005
4006 return mlir::success();
4007}
4008
4009static mlir::Value
4010getValueForVTableSymbol(mlir::Operation *op,
4011 mlir::ConversionPatternRewriter &rewriter,
4012 const mlir::TypeConverter *converter,
4013 mlir::FlatSymbolRefAttr nameAttr, mlir::Type &eltType) {
4014 auto module = op->getParentOfType<mlir::ModuleOp>();
4015 mlir::Operation *symbol = mlir::SymbolTable::lookupSymbolIn(module, nameAttr);
4016 if (auto llvmSymbol = mlir::dyn_cast<mlir::LLVM::GlobalOp>(symbol)) {
4017 eltType = llvmSymbol.getType();
4018 } else if (auto cirSymbol = mlir::dyn_cast<cir::GlobalOp>(symbol)) {
4019 eltType = converter->convertType(cirSymbol.getSymType());
4020 } else {
4021 op->emitError() << "unexpected symbol type for " << symbol;
4022 return {};
4023 }
4024
4025 return mlir::LLVM::AddressOfOp::create(
4026 rewriter, op->getLoc(),
4027 mlir::LLVM::LLVMPointerType::get(op->getContext()), nameAttr.getValue());
4028}
4029
4030mlir::LogicalResult CIRToLLVMVTableAddrPointOpLowering::matchAndRewrite(
4031 cir::VTableAddrPointOp op, OpAdaptor adaptor,
4032 mlir::ConversionPatternRewriter &rewriter) const {
4033 const mlir::TypeConverter *converter = getTypeConverter();
4034 mlir::Type targetType = converter->convertType(op.getType());
4035 llvm::SmallVector<mlir::LLVM::GEPArg> offsets;
4036 mlir::Type eltType;
4037 mlir::Value symAddr = getValueForVTableSymbol(op, rewriter, converter,
4038 op.getNameAttr(), eltType);
4039 if (!symAddr)
4040 return op.emitError() << "Unable to get value for vtable symbol";
4041
4042 offsets = llvm::SmallVector<mlir::LLVM::GEPArg>{
4043 0, op.getAddressPointAttr().getIndex(),
4044 op.getAddressPointAttr().getOffset()};
4045
4046 assert(eltType && "Shouldn't ever be missing an eltType here");
4047 mlir::LLVM::GEPNoWrapFlags inboundsNuw =
4048 mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
4049 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, targetType, eltType,
4050 symAddr, offsets, inboundsNuw);
4051 return mlir::success();
4052}
4053
4054mlir::LogicalResult CIRToLLVMVTableGetVPtrOpLowering::matchAndRewrite(
4055 cir::VTableGetVPtrOp op, OpAdaptor adaptor,
4056 mlir::ConversionPatternRewriter &rewriter) const {
4057 // cir.vtable.get_vptr is equivalent to a bitcast from the source object
4058 // pointer to the vptr type. Since the LLVM dialect uses opaque pointers
4059 // we can just replace uses of this operation with the original pointer.
4060 mlir::Value srcVal = adaptor.getSrc();
4061 rewriter.replaceOp(op, srcVal);
4062 return mlir::success();
4063}
4064
4065mlir::LogicalResult CIRToLLVMVTableGetVirtualFnAddrOpLowering::matchAndRewrite(
4066 cir::VTableGetVirtualFnAddrOp op, OpAdaptor adaptor,
4067 mlir::ConversionPatternRewriter &rewriter) const {
4068 mlir::Type targetType = getTypeConverter()->convertType(op.getType());
4069 auto eltType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
4070 llvm::SmallVector<mlir::LLVM::GEPArg> offsets =
4071 llvm::SmallVector<mlir::LLVM::GEPArg>{op.getIndex()};
4072 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
4073 op, targetType, eltType, adaptor.getVptr(), offsets,
4074 mlir::LLVM::GEPNoWrapFlags::inbounds);
4075 return mlir::success();
4076}
4077
4078mlir::LogicalResult CIRToLLVMVTTAddrPointOpLowering::matchAndRewrite(
4079 cir::VTTAddrPointOp op, OpAdaptor adaptor,
4080 mlir::ConversionPatternRewriter &rewriter) const {
4081 const mlir::Type resultType = getTypeConverter()->convertType(op.getType());
4082 llvm::SmallVector<mlir::LLVM::GEPArg> offsets;
4083 mlir::Type eltType;
4084 mlir::Value llvmAddr = adaptor.getSymAddr();
4085
4086 if (op.getSymAddr()) {
4087 if (op.getOffset() == 0) {
4088 rewriter.replaceOp(op, {llvmAddr});
4089 return mlir::success();
4090 }
4091
4092 offsets.push_back(adaptor.getOffset());
4093 eltType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
4094 } else {
4095 llvmAddr = getValueForVTableSymbol(op, rewriter, getTypeConverter(),
4096 op.getNameAttr(), eltType);
4097 assert(eltType && "Shouldn't ever be missing an eltType here");
4098 offsets.push_back(0);
4099 offsets.push_back(adaptor.getOffset());
4100 }
4101 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
4102 op, resultType, eltType, llvmAddr, offsets,
4103 mlir::LLVM::GEPNoWrapFlags::inbounds);
4104 return mlir::success();
4105}
4106
4107mlir::LogicalResult CIRToLLVMStackSaveOpLowering::matchAndRewrite(
4108 cir::StackSaveOp op, OpAdaptor adaptor,
4109 mlir::ConversionPatternRewriter &rewriter) const {
4110 const mlir::Type ptrTy = getTypeConverter()->convertType(op.getType());
4111 rewriter.replaceOpWithNewOp<mlir::LLVM::StackSaveOp>(op, ptrTy);
4112 return mlir::success();
4113}
4114
4115mlir::LogicalResult CIRToLLVMStackRestoreOpLowering::matchAndRewrite(
4116 cir::StackRestoreOp op, OpAdaptor adaptor,
4117 mlir::ConversionPatternRewriter &rewriter) const {
4118 rewriter.replaceOpWithNewOp<mlir::LLVM::StackRestoreOp>(op, adaptor.getPtr());
4119 return mlir::success();
4120}
4121
4122mlir::LogicalResult CIRToLLVMVecCreateOpLowering::matchAndRewrite(
4123 cir::VecCreateOp op, OpAdaptor adaptor,
4124 mlir::ConversionPatternRewriter &rewriter) const {
4125 // Start with an 'undef' value for the vector. Then 'insertelement' for
4126 // each of the vector elements.
4127 const auto vecTy = mlir::cast<cir::VectorType>(op.getType());
4128 const mlir::Type llvmTy = typeConverter->convertType(vecTy);
4129 const mlir::Location loc = op.getLoc();
4130 mlir::Value result = mlir::LLVM::PoisonOp::create(rewriter, loc, llvmTy);
4131 assert(vecTy.getSize() == op.getElements().size() &&
4132 "cir.vec.create op count doesn't match vector type elements count");
4133
4134 for (uint64_t i = 0; i < vecTy.getSize(); ++i) {
4135 const mlir::Value indexValue =
4136 mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i);
4137 result = mlir::LLVM::InsertElementOp::create(
4138 rewriter, loc, result, adaptor.getElements()[i], indexValue);
4139 }
4140
4141 rewriter.replaceOp(op, result);
4142 return mlir::success();
4143}
4144
4145mlir::LogicalResult CIRToLLVMVecExtractOpLowering::matchAndRewrite(
4146 cir::VecExtractOp op, OpAdaptor adaptor,
4147 mlir::ConversionPatternRewriter &rewriter) const {
4148 rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractElementOp>(
4149 op, adaptor.getVec(), adaptor.getIndex());
4150 return mlir::success();
4151}
4152
4153mlir::LogicalResult CIRToLLVMVecInsertOpLowering::matchAndRewrite(
4154 cir::VecInsertOp op, OpAdaptor adaptor,
4155 mlir::ConversionPatternRewriter &rewriter) const {
4156 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertElementOp>(
4157 op, adaptor.getVec(), adaptor.getValue(), adaptor.getIndex());
4158 return mlir::success();
4159}
4160
4161mlir::LogicalResult CIRToLLVMVecCmpOpLowering::matchAndRewrite(
4162 cir::VecCmpOp op, OpAdaptor adaptor,
4163 mlir::ConversionPatternRewriter &rewriter) const {
4164 mlir::Type elementType = elementTypeIfVector(op.getLhs().getType());
4165 mlir::Value bitResult;
4166 if (auto intType = mlir::dyn_cast<cir::IntType>(elementType)) {
4167 bitResult = mlir::LLVM::ICmpOp::create(
4168 rewriter, op.getLoc(),
4169 convertCmpKindToICmpPredicate(op.getKind(), intType.isSigned()),
4170 adaptor.getLhs(), adaptor.getRhs());
4171 } else if (mlir::isa<cir::FPTypeInterface>(elementType)) {
4172 bitResult = mlir::LLVM::FCmpOp::create(
4173 rewriter, op.getLoc(), convertCmpKindToFCmpPredicate(op.getKind()),
4174 adaptor.getLhs(), adaptor.getRhs());
4175 } else {
4176 return op.emitError() << "unsupported type for VecCmpOp: " << elementType;
4177 }
4178
4179 // LLVM IR vector comparison returns a vector of i1. This one-bit vector
4180 // must be sign-extended to the correct result type, unless a vector of i1 is
4181 // the type we need.
4182 if (cast<cir::IntType>(cast<cir::VectorType>(op.getType()).getElementType())
4183 .getWidth() > 1)
4184 rewriter.replaceOpWithNewOp<mlir::LLVM::SExtOp>(
4185 op, typeConverter->convertType(op.getType()), bitResult);
4186 else
4187 rewriter.replaceOp(op, bitResult);
4188 return mlir::success();
4189}
4190
4191mlir::LogicalResult CIRToLLVMVecSplatOpLowering::matchAndRewrite(
4192 cir::VecSplatOp op, OpAdaptor adaptor,
4193 mlir::ConversionPatternRewriter &rewriter) const {
4194 // Vector splat can be implemented with an `insertelement` and a
4195 // `shufflevector`, which is better than an `insertelement` for each
4196 // element in the vector. Start with an undef vector. Insert the value into
4197 // the first element. Then use a `shufflevector` with a mask of all 0 to
4198 // fill out the entire vector with that value.
4199 cir::VectorType vecTy = op.getType();
4200 mlir::Type llvmTy = typeConverter->convertType(vecTy);
4201 mlir::Location loc = op.getLoc();
4202 mlir::Value poison = mlir::LLVM::PoisonOp::create(rewriter, loc, llvmTy);
4203
4204 mlir::Value elementValue = adaptor.getValue();
4205 if (elementValue.getDefiningOp<mlir::LLVM::PoisonOp>()) {
4206 // If the splat value is poison, then we can just use poison value
4207 // for the entire vector.
4208 rewriter.replaceOp(op, poison);
4209 return mlir::success();
4210 }
4211
4212 if (auto constValue = elementValue.getDefiningOp<mlir::LLVM::ConstantOp>()) {
4213 if (auto intAttr = dyn_cast<mlir::IntegerAttr>(constValue.getValue())) {
4214 mlir::DenseIntElementsAttr denseVec = mlir::DenseIntElementsAttr::get(
4215 mlir::cast<mlir::ShapedType>(llvmTy), intAttr.getValue());
4216 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
4217 op, denseVec.getType(), denseVec);
4218 return mlir::success();
4219 }
4220
4221 if (auto fpAttr = dyn_cast<mlir::FloatAttr>(constValue.getValue())) {
4222 mlir::DenseFPElementsAttr denseVec = mlir::DenseFPElementsAttr::get(
4223 mlir::cast<mlir::ShapedType>(llvmTy), fpAttr.getValue());
4224 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
4225 op, denseVec.getType(), denseVec);
4226 return mlir::success();
4227 }
4228 }
4229
4230 mlir::Value indexValue =
4231 mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), 0);
4232 mlir::Value oneElement = mlir::LLVM::InsertElementOp::create(
4233 rewriter, loc, poison, elementValue, indexValue);
4234 SmallVector<int32_t> zeroValues(vecTy.getSize(), 0);
4235 rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(op, oneElement,
4236 poison, zeroValues);
4237 return mlir::success();
4238}
4239
4240mlir::LogicalResult CIRToLLVMVecShuffleOpLowering::matchAndRewrite(
4241 cir::VecShuffleOp op, OpAdaptor adaptor,
4242 mlir::ConversionPatternRewriter &rewriter) const {
4243 // LLVM::ShuffleVectorOp takes an ArrayRef of int for the list of indices.
4244 // Convert the ClangIR ArrayAttr of IntAttr constants into a
4245 // SmallVector<int>.
4246 SmallVector<int, 8> indices;
4247 std::transform(
4248 op.getIndices().begin(), op.getIndices().end(),
4249 std::back_inserter(indices), [](mlir::Attribute intAttr) {
4250 return mlir::cast<cir::IntAttr>(intAttr).getValue().getSExtValue();
4251 });
4252 rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(
4253 op, adaptor.getVec1(), adaptor.getVec2(), indices);
4254 return mlir::success();
4255}
4256
4257mlir::LogicalResult CIRToLLVMVecShuffleDynamicOpLowering::matchAndRewrite(
4258 cir::VecShuffleDynamicOp op, OpAdaptor adaptor,
4259 mlir::ConversionPatternRewriter &rewriter) const {
4260 // LLVM IR does not have an operation that corresponds to this form of
4261 // the built-in.
4262 // __builtin_shufflevector(V, I)
4263 // is implemented as this pseudocode, where the for loop is unrolled
4264 // and N is the number of elements:
4265 //
4266 // result = undef
4267 // maskbits = NextPowerOf2(N - 1)
4268 // masked = I & maskbits
4269 // for (i in 0 <= i < N)
4270 // result[i] = V[masked[i]]
4271 mlir::Location loc = op.getLoc();
4272 mlir::Value input = adaptor.getVec();
4273 mlir::Type llvmIndexVecType =
4274 getTypeConverter()->convertType(op.getIndices().getType());
4275 mlir::Type llvmIndexType = getTypeConverter()->convertType(
4276 elementTypeIfVector(op.getIndices().getType()));
4277 uint64_t numElements =
4278 mlir::cast<cir::VectorType>(op.getVec().getType()).getSize();
4279
4280 uint64_t maskBits = llvm::NextPowerOf2(numElements - 1) - 1;
4281 mlir::Value maskValue = mlir::LLVM::ConstantOp::create(
4282 rewriter, loc, llvmIndexType,
4283 rewriter.getIntegerAttr(llvmIndexType, maskBits));
4284 mlir::Value maskVector =
4285 mlir::LLVM::UndefOp::create(rewriter, loc, llvmIndexVecType);
4286
4287 for (uint64_t i = 0; i < numElements; ++i) {
4288 mlir::Value idxValue =
4289 mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i);
4290 maskVector = mlir::LLVM::InsertElementOp::create(rewriter, loc, maskVector,
4291 maskValue, idxValue);
4292 }
4293
4294 mlir::Value maskedIndices = mlir::LLVM::AndOp::create(
4295 rewriter, loc, llvmIndexVecType, adaptor.getIndices(), maskVector);
4296 mlir::Value result = mlir::LLVM::UndefOp::create(
4297 rewriter, loc, getTypeConverter()->convertType(op.getVec().getType()));
4298 for (uint64_t i = 0; i < numElements; ++i) {
4299 mlir::Value iValue =
4300 mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i);
4301 mlir::Value indexValue = mlir::LLVM::ExtractElementOp::create(
4302 rewriter, loc, maskedIndices, iValue);
4303 mlir::Value valueAtIndex =
4304 mlir::LLVM::ExtractElementOp::create(rewriter, loc, input, indexValue);
4305 result = mlir::LLVM::InsertElementOp::create(rewriter, loc, result,
4306 valueAtIndex, iValue);
4307 }
4308 rewriter.replaceOp(op, result);
4309 return mlir::success();
4310}
4311
4312mlir::LogicalResult CIRToLLVMVecTernaryOpLowering::matchAndRewrite(
4313 cir::VecTernaryOp op, OpAdaptor adaptor,
4314 mlir::ConversionPatternRewriter &rewriter) const {
4315 // Convert `cond` into a vector of i1, then use that in a `select` op.
4316 mlir::Value bitVec = mlir::LLVM::ICmpOp::create(
4317 rewriter, op.getLoc(), mlir::LLVM::ICmpPredicate::ne, adaptor.getCond(),
4318 mlir::LLVM::ZeroOp::create(
4319 rewriter, op.getCond().getLoc(),
4320 typeConverter->convertType(op.getCond().getType())));
4321 rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(
4322 op, bitVec, adaptor.getLhs(), adaptor.getRhs());
4323 return mlir::success();
4324}
4325
4326mlir::LogicalResult CIRToLLVMComplexAddOpLowering::matchAndRewrite(
4327 cir::ComplexAddOp op, OpAdaptor adaptor,
4328 mlir::ConversionPatternRewriter &rewriter) const {
4329 mlir::Value lhs = adaptor.getLhs();
4330 mlir::Value rhs = adaptor.getRhs();
4331 mlir::Location loc = op.getLoc();
4332
4333 auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType());
4334 mlir::Type complexElemTy =
4335 getTypeConverter()->convertType(complexType.getElementType());
4336 auto lhsReal = mlir::LLVM::ExtractValueOp::create(
4337 rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0}));
4338 auto lhsImag = mlir::LLVM::ExtractValueOp::create(
4339 rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1}));
4340 auto rhsReal = mlir::LLVM::ExtractValueOp::create(
4341 rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0}));
4342 auto rhsImag = mlir::LLVM::ExtractValueOp::create(
4343 rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1}));
4344
4345 mlir::Value newReal;
4346 mlir::Value newImag;
4347 if (complexElemTy.isInteger()) {
4348 newReal = mlir::LLVM::AddOp::create(rewriter, loc, complexElemTy, lhsReal,
4349 rhsReal);
4350 newImag = mlir::LLVM::AddOp::create(rewriter, loc, complexElemTy, lhsImag,
4351 rhsImag);
4352 } else {
4353 assert(!cir::MissingFeatures::fastMathFlags());
4354 assert(!cir::MissingFeatures::fpConstraints());
4355 newReal = mlir::LLVM::FAddOp::create(rewriter, loc, complexElemTy, lhsReal,
4356 rhsReal);
4357 newImag = mlir::LLVM::FAddOp::create(rewriter, loc, complexElemTy, lhsImag,
4358 rhsImag);
4359 }
4360
4361 mlir::Type complexLLVMTy =
4362 getTypeConverter()->convertType(op.getResult().getType());
4363 auto initialComplex =
4364 mlir::LLVM::PoisonOp::create(rewriter, op->getLoc(), complexLLVMTy);
4365
4366 auto realComplex = mlir::LLVM::InsertValueOp::create(
4367 rewriter, op->getLoc(), initialComplex, newReal, ArrayRef(int64_t{0}));
4368
4369 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(
4370 op, realComplex, newImag, ArrayRef(int64_t{1}));
4371
4372 return mlir::success();
4373}
4374
4375mlir::LogicalResult CIRToLLVMComplexCreateOpLowering::matchAndRewrite(
4376 cir::ComplexCreateOp op, OpAdaptor adaptor,
4377 mlir::ConversionPatternRewriter &rewriter) const {
4378 mlir::Type complexLLVMTy =
4379 getTypeConverter()->convertType(op.getResult().getType());
4380 auto initialComplex =
4381 mlir::LLVM::UndefOp::create(rewriter, op->getLoc(), complexLLVMTy);
4382
4383 auto realComplex = mlir::LLVM::InsertValueOp::create(
4384 rewriter, op->getLoc(), initialComplex, adaptor.getReal(),
4385 ArrayRef(int64_t{0}));
4386
4387 auto complex = mlir::LLVM::InsertValueOp::create(
4388 rewriter, op->getLoc(), realComplex, adaptor.getImag(),
4389 ArrayRef(int64_t{1}));
4390
4391 rewriter.replaceOp(op, complex);
4392 return mlir::success();
4393}
4394
4395mlir::LogicalResult CIRToLLVMComplexRealOpLowering::matchAndRewrite(
4396 cir::ComplexRealOp op, OpAdaptor adaptor,
4397 mlir::ConversionPatternRewriter &rewriter) const {
4398 mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
4399 mlir::Value operand = adaptor.getOperand();
4400 if (mlir::isa<cir::ComplexType>(op.getOperand().getType())) {
4401 operand = mlir::LLVM::ExtractValueOp::create(
4402 rewriter, op.getLoc(), resultLLVMTy, operand,
4403 llvm::ArrayRef<std::int64_t>{0});
4404 }
4405 rewriter.replaceOp(op, operand);
4406 return mlir::success();
4407}
4408
4409mlir::LogicalResult CIRToLLVMComplexSubOpLowering::matchAndRewrite(
4410 cir::ComplexSubOp op, OpAdaptor adaptor,
4411 mlir::ConversionPatternRewriter &rewriter) const {
4412 mlir::Value lhs = adaptor.getLhs();
4413 mlir::Value rhs = adaptor.getRhs();
4414 mlir::Location loc = op.getLoc();
4415
4416 auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType());
4417 mlir::Type complexElemTy =
4418 getTypeConverter()->convertType(complexType.getElementType());
4419 auto lhsReal = mlir::LLVM::ExtractValueOp::create(
4420 rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0}));
4421 auto lhsImag = mlir::LLVM::ExtractValueOp::create(
4422 rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1}));
4423 auto rhsReal = mlir::LLVM::ExtractValueOp::create(
4424 rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0}));
4425 auto rhsImag = mlir::LLVM::ExtractValueOp::create(
4426 rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1}));
4427
4428 mlir::Value newReal;
4429 mlir::Value newImag;
4430 if (complexElemTy.isInteger()) {
4431 newReal = mlir::LLVM::SubOp::create(rewriter, loc, complexElemTy, lhsReal,
4432 rhsReal);
4433 newImag = mlir::LLVM::SubOp::create(rewriter, loc, complexElemTy, lhsImag,
4434 rhsImag);
4435 } else {
4436 assert(!cir::MissingFeatures::fastMathFlags());
4437 assert(!cir::MissingFeatures::fpConstraints());
4438 newReal = mlir::LLVM::FSubOp::create(rewriter, loc, complexElemTy, lhsReal,
4439 rhsReal);
4440 newImag = mlir::LLVM::FSubOp::create(rewriter, loc, complexElemTy, lhsImag,
4441 rhsImag);
4442 }
4443
4444 mlir::Type complexLLVMTy =
4445 getTypeConverter()->convertType(op.getResult().getType());
4446 auto initialComplex =
4447 mlir::LLVM::PoisonOp::create(rewriter, op->getLoc(), complexLLVMTy);
4448
4449 auto realComplex = mlir::LLVM::InsertValueOp::create(
4450 rewriter, op->getLoc(), initialComplex, newReal, ArrayRef(int64_t{0}));
4451
4452 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(
4453 op, realComplex, newImag, ArrayRef(int64_t{1}));
4454
4455 return mlir::success();
4456}
4457
4458mlir::LogicalResult CIRToLLVMComplexImagOpLowering::matchAndRewrite(
4459 cir::ComplexImagOp op, OpAdaptor adaptor,
4460 mlir::ConversionPatternRewriter &rewriter) const {
4461 mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
4462 mlir::Value operand = adaptor.getOperand();
4463 mlir::Location loc = op.getLoc();
4464
4465 if (mlir::isa<cir::ComplexType>(op.getOperand().getType())) {
4466 operand = mlir::LLVM::ExtractValueOp::create(
4467 rewriter, loc, resultLLVMTy, operand, llvm::ArrayRef<std::int64_t>{1});
4468 } else {
4469 mlir::TypedAttr zeroAttr = rewriter.getZeroAttr(resultLLVMTy);
4470 operand =
4471 mlir::LLVM::ConstantOp::create(rewriter, loc, resultLLVMTy, zeroAttr);
4472 }
4473
4474 rewriter.replaceOp(op, operand);
4475 return mlir::success();
4476}
4477
4478mlir::IntegerType computeBitfieldIntType(mlir::Type storageType,
4479 mlir::MLIRContext *context,
4480 unsigned &storageSize) {
4481 return TypeSwitch<mlir::Type, mlir::IntegerType>(storageType)
4482 .Case<cir::ArrayType>([&](cir::ArrayType atTy) {
4483 storageSize = atTy.getSize() * 8;
4484 return mlir::IntegerType::get(context, storageSize);
4485 })
4486 .Case<cir::IntType>([&](cir::IntType intTy) {
4487 storageSize = intTy.getWidth();
4488 return mlir::IntegerType::get(context, storageSize);
4489 })
4490 .Default([](mlir::Type) -> mlir::IntegerType {
4491 llvm_unreachable(
4492 "Either ArrayType or IntType expected for bitfields storage");
4493 });
4494}
4495
4496mlir::LogicalResult CIRToLLVMSetBitfieldOpLowering::matchAndRewrite(
4497 cir::SetBitfieldOp op, OpAdaptor adaptor,
4498 mlir::ConversionPatternRewriter &rewriter) const {
4499 mlir::OpBuilder::InsertionGuard guard(rewriter);
4500 rewriter.setInsertionPoint(op);
4501
4502 cir::BitfieldInfoAttr info = op.getBitfieldInfo();
4503 uint64_t size = info.getSize();
4504 uint64_t offset = info.getOffset();
4505 mlir::Type storageType = info.getStorageType();
4506 mlir::MLIRContext *context = storageType.getContext();
4507
4508 unsigned storageSize = 0;
4509
4510 mlir::IntegerType intType =
4511 computeBitfieldIntType(storageType, context, storageSize);
4512
4513 mlir::Value srcVal = createIntCast(rewriter, adaptor.getSrc(), intType);
4514 unsigned srcWidth = storageSize;
4515 mlir::Value resultVal = srcVal;
4516
4517 if (storageSize != size) {
4518 assert(storageSize > size && "Invalid bitfield size.");
4519
4520 mlir::Value val = mlir::LLVM::LoadOp::create(
4521 rewriter, op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(),
4522 op.getIsVolatile());
4523
4524 srcVal =
4525 createAnd(rewriter, srcVal, llvm::APInt::getLowBitsSet(srcWidth, size));
4526 resultVal = srcVal;
4527 srcVal = createShL(rewriter, srcVal, offset);
4528
4529 // Mask out the original value.
4530 val = createAnd(rewriter, val,
4531 ~llvm::APInt::getBitsSet(srcWidth, offset, offset + size));
4532
4533 // Or together the unchanged values and the source value.
4534 srcVal = mlir::LLVM::OrOp::create(rewriter, op.getLoc(), val, srcVal);
4535 }
4536
4537 mlir::LLVM::StoreOp::create(rewriter, op.getLoc(), srcVal, adaptor.getAddr(),
4538 op.getAlignment(), op.getIsVolatile());
4539
4540 mlir::Type resultTy = getTypeConverter()->convertType(op.getType());
4541
4542 if (info.getIsSigned()) {
4543 assert(size <= storageSize);
4544 unsigned highBits = storageSize - size;
4545
4546 if (highBits) {
4547 resultVal = createShL(rewriter, resultVal, highBits);
4548 resultVal = createAShR(rewriter, resultVal, highBits);
4549 }
4550 }
4551
4552 resultVal = createIntCast(rewriter, resultVal,
4553 mlir::cast<mlir::IntegerType>(resultTy),
4554 info.getIsSigned());
4555
4556 rewriter.replaceOp(op, resultVal);
4557 return mlir::success();
4558}
4559
4560mlir::LogicalResult CIRToLLVMComplexImagPtrOpLowering::matchAndRewrite(
4561 cir::ComplexImagPtrOp op, OpAdaptor adaptor,
4562 mlir::ConversionPatternRewriter &rewriter) const {
4563 cir::PointerType operandTy = op.getOperand().getType();
4564 mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
4565 mlir::Type elementLLVMTy =
4566 getTypeConverter()->convertType(operandTy.getPointee());
4567
4568 mlir::LLVM::GEPArg gepIndices[2] = {{0}, {1}};
4569 mlir::LLVM::GEPNoWrapFlags inboundsNuw =
4570 mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
4571 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
4572 op, resultLLVMTy, elementLLVMTy, adaptor.getOperand(), gepIndices,
4573 inboundsNuw);
4574 return mlir::success();
4575}
4576
4577mlir::LogicalResult CIRToLLVMComplexRealPtrOpLowering::matchAndRewrite(
4578 cir::ComplexRealPtrOp op, OpAdaptor adaptor,
4579 mlir::ConversionPatternRewriter &rewriter) const {
4580 cir::PointerType operandTy = op.getOperand().getType();
4581 mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
4582 mlir::Type elementLLVMTy =
4583 getTypeConverter()->convertType(operandTy.getPointee());
4584
4585 mlir::LLVM::GEPArg gepIndices[2] = {0, 0};
4586 mlir::LLVM::GEPNoWrapFlags inboundsNuw =
4587 mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
4588 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
4589 op, resultLLVMTy, elementLLVMTy, adaptor.getOperand(), gepIndices,
4590 inboundsNuw);
4591 return mlir::success();
4592}
4593
4594mlir::LogicalResult CIRToLLVMGetBitfieldOpLowering::matchAndRewrite(
4595 cir::GetBitfieldOp op, OpAdaptor adaptor,
4596 mlir::ConversionPatternRewriter &rewriter) const {
4597
4598 mlir::OpBuilder::InsertionGuard guard(rewriter);
4599 rewriter.setInsertionPoint(op);
4600
4601 cir::BitfieldInfoAttr info = op.getBitfieldInfo();
4602 uint64_t size = info.getSize();
4603 uint64_t offset = info.getOffset();
4604 mlir::Type storageType = info.getStorageType();
4605 mlir::MLIRContext *context = storageType.getContext();
4606 unsigned storageSize = 0;
4607
4608 mlir::IntegerType intType =
4609 computeBitfieldIntType(storageType, context, storageSize);
4610
4611 mlir::Value val = mlir::LLVM::LoadOp::create(
4612 rewriter, op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(),
4613 op.getIsVolatile());
4614 val = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), intType, val);
4615
4616 if (info.getIsSigned()) {
4617 assert(static_cast<unsigned>(offset + size) <= storageSize);
4618 unsigned highBits = storageSize - offset - size;
4619 val = createShL(rewriter, val, highBits);
4620 val = createAShR(rewriter, val, offset + highBits);
4621 } else {
4622 val = createLShR(rewriter, val, offset);
4623
4624 if (static_cast<unsigned>(offset) + size < storageSize)
4625 val = createAnd(rewriter, val,
4626 llvm::APInt::getLowBitsSet(storageSize, size));
4627 }
4628
4629 mlir::Type resTy = getTypeConverter()->convertType(op.getType());
4630 mlir::Value newOp = createIntCast(
4631 rewriter, val, mlir::cast<mlir::IntegerType>(resTy), info.getIsSigned());
4632 rewriter.replaceOp(op, newOp);
4633 return mlir::success();
4634}
4635
4636mlir::LogicalResult CIRToLLVMIsConstantOpLowering::matchAndRewrite(
4637 cir::IsConstantOp op, OpAdaptor adaptor,
4638 mlir::ConversionPatternRewriter &rewriter) const {
4639 rewriter.replaceOpWithNewOp<mlir::LLVM::IsConstantOp>(op, adaptor.getVal());
4640 return mlir::success();
4641}
4642
4643mlir::LogicalResult CIRToLLVMInlineAsmOpLowering::matchAndRewrite(
4644 cir::InlineAsmOp op, OpAdaptor adaptor,
4645 mlir::ConversionPatternRewriter &rewriter) const {
4646 mlir::Type llResTy;
4647 if (op.getNumResults())
4648 llResTy = getTypeConverter()->convertType(op.getType(0));
4649
4650 cir::AsmFlavor dialect = op.getAsmFlavor();
4651 mlir::LLVM::AsmDialect llDialect = dialect == cir::AsmFlavor::x86_att
4652 ? mlir::LLVM::AsmDialect::AD_ATT
4653 : mlir::LLVM::AsmDialect::AD_Intel;
4654
4655 SmallVector<mlir::Attribute> opAttrs;
4656 StringRef llvmAttrName = mlir::LLVM::InlineAsmOp::getElementTypeAttrName();
4657
4658 // this is for the lowering to LLVM from LLVM dialect. Otherwise, if we
4659 // don't have the result (i.e. void type as a result of operation), the
4660 // element type attribute will be attached to the whole instruction, but not
4661 // to the operand
4662 if (!op.getNumResults())
4663 opAttrs.push_back(mlir::Attribute());
4664
4665 SmallVector<mlir::Value> llvmOperands;
4666 SmallVector<mlir::Value> cirOperands;
4667 for (auto const &[llvmOp, cirOp] :
4668 zip(adaptor.getAsmOperands(), op.getAsmOperands())) {
4669 append_range(llvmOperands, llvmOp);
4670 append_range(cirOperands, cirOp);
4671 }
4672
4673 // so far we infer the llvm dialect element type attr from
4674 // CIR operand type.
4675 for (auto const &[cirOpAttr, cirOp] :
4676 zip(op.getOperandAttrs(), cirOperands)) {
4677 if (!cirOpAttr) {
4678 opAttrs.push_back(mlir::Attribute());
4679 continue;
4680 }
4681
4682 llvm::SmallVector<mlir::NamedAttribute, 1> attrs;
4683 cir::PointerType typ = mlir::cast<cir::PointerType>(cirOp.getType());
4684 mlir::TypeAttr typAttr = mlir::TypeAttr::get(convertTypeForMemory(
4685 *getTypeConverter(), dataLayout, typ.getPointee()));
4686
4687 attrs.push_back(rewriter.getNamedAttr(llvmAttrName, typAttr));
4688 mlir::DictionaryAttr newDict = rewriter.getDictionaryAttr(attrs);
4689 opAttrs.push_back(newDict);
4690 }
4691
4692 rewriter.replaceOpWithNewOp<mlir::LLVM::InlineAsmOp>(
4693 op, llResTy, llvmOperands, op.getAsmStringAttr(), op.getConstraintsAttr(),
4694 op.getSideEffectsAttr(),
4695 /*is_align_stack*/ mlir::UnitAttr(),
4696 /*tail_call_kind*/
4697 mlir::LLVM::TailCallKindAttr::get(
4698 getContext(), mlir::LLVM::tailcallkind::TailCallKind::None),
4699 mlir::LLVM::AsmDialectAttr::get(getContext(), llDialect),
4700 rewriter.getArrayAttr(opAttrs));
4701
4702 return mlir::success();
4703}
4704
4705mlir::LogicalResult CIRToLLVMVAStartOpLowering::matchAndRewrite(
4706 cir::VAStartOp op, OpAdaptor adaptor,
4707 mlir::ConversionPatternRewriter &rewriter) const {
4708 auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
4709 auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
4710 adaptor.getArgList());
4711 rewriter.replaceOpWithNewOp<mlir::LLVM::VaStartOp>(op, vaList);
4712 return mlir::success();
4713}
4714
4715mlir::LogicalResult CIRToLLVMVAEndOpLowering::matchAndRewrite(
4716 cir::VAEndOp op, OpAdaptor adaptor,
4717 mlir::ConversionPatternRewriter &rewriter) const {
4718 auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
4719 auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
4720 adaptor.getArgList());
4721 rewriter.replaceOpWithNewOp<mlir::LLVM::VaEndOp>(op, vaList);
4722 return mlir::success();
4723}
4724
4725mlir::LogicalResult CIRToLLVMVACopyOpLowering::matchAndRewrite(
4726 cir::VACopyOp op, OpAdaptor adaptor,
4727 mlir::ConversionPatternRewriter &rewriter) const {
4728 auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
4729 auto dstList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
4730 adaptor.getDstList());
4731 auto srcList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
4732 adaptor.getSrcList());
4733 rewriter.replaceOpWithNewOp<mlir::LLVM::VaCopyOp>(op, dstList, srcList);
4734 return mlir::success();
4735}
4736
4737mlir::LogicalResult CIRToLLVMVAArgOpLowering::matchAndRewrite(
4738 cir::VAArgOp op, OpAdaptor adaptor,
4739 mlir::ConversionPatternRewriter &rewriter) const {
4740 assert(!cir::MissingFeatures::vaArgABILowering());
4741 auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
4742 auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
4743 adaptor.getArgList());
4744
4745 mlir::Type llvmType =
4746 getTypeConverter()->convertType(op->getResultTypes().front());
4747 if (!llvmType)
4748 return mlir::failure();
4749
4750 rewriter.replaceOpWithNewOp<mlir::LLVM::VaArgOp>(op, llvmType, vaList);
4751 return mlir::success();
4752}
4753
4754mlir::LogicalResult CIRToLLVMLabelOpLowering::matchAndRewrite(
4755 cir::LabelOp op, OpAdaptor adaptor,
4756 mlir::ConversionPatternRewriter &rewriter) const {
4757 mlir::MLIRContext *ctx = rewriter.getContext();
4758 mlir::Block *block = op->getBlock();
4759 // A BlockTagOp cannot reside in the entry block. The address of the entry
4760 // block cannot be taken
4761 if (block->isEntryBlock()) {
4762 mlir::Block *newBlock =
4763 rewriter.splitBlock(op->getBlock(), mlir::Block::iterator(op));
4764 rewriter.setInsertionPointToEnd(block);
4765 mlir::LLVM::BrOp::create(rewriter, op.getLoc(), newBlock);
4766 }
4767 auto tagAttr =
4768 mlir::LLVM::BlockTagAttr::get(ctx, blockInfoAddr.getTagIndex());
4769 rewriter.setInsertionPoint(op);
4770
4771 auto blockTagOp =
4772 mlir::LLVM::BlockTagOp::create(rewriter, op->getLoc(), tagAttr);
4773 mlir::LLVM::LLVMFuncOp func = op->getParentOfType<mlir::LLVM::LLVMFuncOp>();
4774 auto blockInfoAttr =
4775 cir::BlockAddrInfoAttr::get(ctx, func.getSymName(), op.getLabel());
4776 blockInfoAddr.mapBlockTag(blockInfoAttr, blockTagOp);
4777 rewriter.eraseOp(op);
4778
4779 return mlir::success();
4780}
4781
4782mlir::LogicalResult CIRToLLVMBlockAddressOpLowering::matchAndRewrite(
4783 cir::BlockAddressOp op, OpAdaptor adaptor,
4784 mlir::ConversionPatternRewriter &rewriter) const {
4785 mlir::MLIRContext *ctx = rewriter.getContext();
4786
4787 mlir::LLVM::BlockTagOp matchLabel =
4788 blockInfoAddr.lookupBlockTag(op.getBlockAddrInfoAttr());
4789 mlir::LLVM::BlockTagAttr tagAttr;
4790 if (!matchLabel)
4791 // If the BlockTagOp has not been emitted yet, use a placeholder.
4792 // This will later be replaced with the correct tag index during
4793 // `resolveBlockAddressOp`.
4794 tagAttr = {};
4795 else
4796 tagAttr = matchLabel.getTag();
4797
4798 auto blkAddr = mlir::LLVM::BlockAddressAttr::get(
4799 rewriter.getContext(), op.getBlockAddrInfoAttr().getFunc(), tagAttr);
4800 rewriter.setInsertionPoint(op);
4801 auto newOp = mlir::LLVM::BlockAddressOp::create(
4802 rewriter, op.getLoc(), mlir::LLVM::LLVMPointerType::get(ctx), blkAddr);
4803 if (!matchLabel)
4804 blockInfoAddr.addUnresolvedBlockAddress(newOp, op.getBlockAddrInfoAttr());
4805 rewriter.replaceOp(op, newOp);
4806 return mlir::success();
4807}
4808
4809mlir::LogicalResult CIRToLLVMIndirectBrOpLowering::matchAndRewrite(
4810 cir::IndirectBrOp op, OpAdaptor adaptor,
4811 mlir::ConversionPatternRewriter &rewriter) const {
4812
4813 llvm::SmallVector<mlir::Block *, 8> successors;
4814 llvm::SmallVector<mlir::ValueRange, 8> succOperands;
4815 bool poison = op.getPoison();
4816 for (mlir::Block *succ : op->getSuccessors())
4817 successors.push_back(succ);
4818
4819 for (mlir::ValueRange operand : op.getSuccOperands()) {
4820 succOperands.push_back(operand);
4821 }
4822
4823 auto llvmPtrType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
4824 mlir::Value targetAddr;
4825 if (!poison) {
4826 targetAddr = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(),
4827 llvmPtrType, adaptor.getAddr());
4828 } else {
4829 targetAddr =
4830 mlir::LLVM::PoisonOp::create(rewriter, op->getLoc(), llvmPtrType);
4831 // Remove the block argument to avoid generating an empty PHI during
4832 // lowering.
4833 op->getBlock()->eraseArgument(0);
4834 }
4835
4836 auto newOp = mlir::LLVM::IndirectBrOp::create(
4837 rewriter, op.getLoc(), targetAddr, succOperands, successors);
4838 rewriter.replaceOp(op, newOp);
4839 return mlir::success();
4840}
4841
4842mlir::LogicalResult CIRToLLVMAwaitOpLowering::matchAndRewrite(
4843 cir::AwaitOp op, OpAdaptor adaptor,
4844 mlir::ConversionPatternRewriter &rewriter) const {
4845 return mlir::failure();
4846}
4847
4848mlir::LogicalResult CIRToLLVMCpuIdOpLowering::matchAndRewrite(
4849 cir::CpuIdOp op, OpAdaptor adaptor,
4850 mlir::ConversionPatternRewriter &rewriter) const {
4851 mlir::Type i32Ty = rewriter.getI32Type();
4852 mlir::Type i64Ty = rewriter.getI64Type();
4853 mlir::Type i32PtrTy = mlir::LLVM::LLVMPointerType::get(i32Ty.getContext(), 0);
4854
4855 mlir::Type cpuidRetTy = mlir::LLVM::LLVMStructType::getLiteral(
4856 rewriter.getContext(), {i32Ty, i32Ty, i32Ty, i32Ty});
4857
4858 mlir::Value functionId = adaptor.getFunctionId();
4859 mlir::Value subFunctionId = adaptor.getSubFunctionId();
4860
4861 StringRef asmString, constraints;
4862 mlir::ModuleOp moduleOp = op->getParentOfType<mlir::ModuleOp>();
4863 llvm::Triple triple(
4864 mlir::cast<mlir::StringAttr>(
4865 moduleOp->getAttr(cir::CIRDialect::getTripleAttrName()))
4866 .getValue());
4867 if (triple.getArch() == llvm::Triple::x86) {
4868 asmString = "cpuid";
4869 constraints = "={ax},={bx},={cx},={dx},{ax},{cx}";
4870 } else {
4871 // x86-64 uses %rbx as the base register, so preserve it.
4872 asmString = "xchgq %rbx, ${1:q}\n"
4873 "cpuid\n"
4874 "xchgq %rbx, ${1:q}";
4875 constraints = "={ax},=r,={cx},={dx},0,2";
4876 }
4877
4878 mlir::Value inlineAsm =
4879 mlir::LLVM::InlineAsmOp::create(
4880 rewriter, op.getLoc(), cpuidRetTy, {functionId, subFunctionId},
4881 rewriter.getStringAttr(asmString),
4882 rewriter.getStringAttr(constraints),
4883 /*has_side_effects=*/mlir::UnitAttr{},
4884 /*is_align_stack=*/mlir::UnitAttr{},
4885 /*tail_call_kind=*/mlir::LLVM::TailCallKindAttr{},
4886 /*asm_dialect=*/mlir::LLVM::AsmDialectAttr{},
4887 /*operand_attrs=*/mlir::ArrayAttr{})
4888 .getResult(0);
4889
4890 mlir::Value basePtr = adaptor.getCpuInfo();
4891
4892 mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>());
4893 unsigned alignment = layout.getTypeABIAlignment(i32Ty);
4894 for (unsigned i = 0; i < 4; i++) {
4895 mlir::Value extracted =
4896 mlir::LLVM::ExtractValueOp::create(rewriter, op.getLoc(), inlineAsm, i)
4897 .getResult();
4898 mlir::Value index = mlir::LLVM::ConstantOp::create(
4899 rewriter, op.getLoc(), i64Ty, rewriter.getI64IntegerAttr(i));
4900 llvm::SmallVector<mlir::Value, 1> gepIndices = {index};
4901 mlir::Value storePtr = mlir::LLVM::GEPOp::create(
4902 rewriter, op.getLoc(), i32PtrTy, i32Ty, basePtr,
4903 gepIndices, mlir::LLVM::GEPNoWrapFlags::none)
4904 .getResult();
4905 mlir::LLVM::StoreOp::create(rewriter, op.getLoc(), extracted, storePtr,
4906 alignment);
4907 }
4908
4909 rewriter.eraseOp(op);
4910 return mlir::success();
4911}
4912
4913mlir::LogicalResult CIRToLLVMMemChrOpLowering::matchAndRewrite(
4914 cir::MemChrOp op, OpAdaptor adaptor,
4915 mlir::ConversionPatternRewriter &rewriter) const {
4916 auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
4917 mlir::Type srcTy = getTypeConverter()->convertType(op.getSrc().getType());
4918 mlir::Type patternTy =
4919 getTypeConverter()->convertType(op.getPattern().getType());
4920 mlir::Type lenTy = getTypeConverter()->convertType(op.getLen().getType());
4921 auto fnTy =
4922 mlir::LLVM::LLVMFunctionType::get(llvmPtrTy, {srcTy, patternTy, lenTy},
4923 /*isVarArg=*/false);
4924 llvm::StringRef fnName = "memchr";
4925 createLLVMFuncOpIfNotExist(rewriter, op, fnName, fnTy);
4926 rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
4927 op, mlir::TypeRange{llvmPtrTy}, fnName,
4928 mlir::ValueRange{adaptor.getSrc(), adaptor.getPattern(),
4929 adaptor.getLen()});
4930 return mlir::success();
4931}
4932
4933std::unique_ptr<mlir::Pass> createConvertCIRToLLVMPass() {
4934 return std::make_unique<ConvertCIRToLLVMPass>();
4935}
4936
4937void populateCIRToLLVMPasses(mlir::OpPassManager &pm) {
4938 mlir::populateCIRPreLoweringPasses(pm);
4939 pm.addPass(createConvertCIRToLLVMPass());
4940}
4941
4942std::unique_ptr<llvm::Module>
4943lowerDirectlyFromCIRToLLVMIR(mlir::ModuleOp mlirModule, LLVMContext &llvmCtx,
4944 StringRef mlirSaveTempsOutFile) {
4945 llvm::TimeTraceScope scope("lower from CIR to LLVM directly");
4946
4947 mlir::MLIRContext *mlirCtx = mlirModule.getContext();
4948
4949 mlir::PassManager pm(mlirCtx);
4950 populateCIRToLLVMPasses(pm);
4951
4952 (void)mlir::applyPassManagerCLOptions(pm);
4953
4954 if (mlir::failed(pm.run(mlirModule))) {
4955 // FIXME: Handle any errors where they occurs and return a nullptr here.
4956 report_fatal_error(
4957 "The pass manager failed to lower CIR to LLVMIR dialect!");
4958 }
4959
4960 if (!mlirSaveTempsOutFile.empty()) {
4961 std::error_code ec;
4962 llvm::raw_fd_ostream out(mlirSaveTempsOutFile, ec);
4963 if (!ec)
4964 mlirModule->print(out);
4965 }
4966
4967 mlir::registerBuiltinDialectTranslation(*mlirCtx);
4968 mlir::registerLLVMDialectTranslation(*mlirCtx);
4969 mlir::registerOpenMPDialectTranslation(*mlirCtx);
4970 mlir::registerCIRDialectTranslation(*mlirCtx);
4971
4972 llvm::TimeTraceScope translateScope("translateModuleToLLVMIR");
4973
4974 StringRef moduleName = mlirModule.getName().value_or("CIRToLLVMModule");
4975 std::unique_ptr<llvm::Module> llvmModule =
4976 mlir::translateModuleToLLVMIR(mlirModule, llvmCtx, moduleName);
4977
4978 if (!llvmModule) {
4979 // FIXME: Handle any errors where they occurs and return a nullptr here.
4980 report_fatal_error("Lowering from LLVMIR dialect to llvm IR failed!");
4981 }
4982
4983 return llvmModule;
4984}
4985} // namespace direct
4986} // namespace cir
isUnsigned
static bool isUnsigned(SValBuilder &SVB, NonLoc Value)
Definition ArrayBoundChecker.cpp:307
getLinkageAttrNameString
static llvm::StringRef getLinkageAttrNameString()
Returns the name used for the linkage attribute.
Definition CIRDialect.cpp:2044
createLShR
mlir::Value createLShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs)
Definition LoweringHelpers.cpp:179
createShL
mlir::Value createShL(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs)
Definition LoweringHelpers.cpp:159
lowerConstArrayAttr
std::optional< mlir::Attribute > lowerConstArrayAttr(cir::ConstArrayAttr constArr, const mlir::TypeConverter *converter)
Definition LoweringHelpers.cpp:119
createAShR
mlir::Value createAShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs)
Definition LoweringHelpers.cpp:166
createAnd
mlir::Value createAnd(mlir::OpBuilder &bld, mlir::Value lhs, const llvm::APInt &rhs)
Definition LoweringHelpers.cpp:173
kind
*collection of selector each with an associated kind and an ordered *collection of selectors A selector has a kind
Definition OpenMPClause.h:9714
isVector
static bool isVector(QualType QT, QualType ElementType)
This helper function returns true if QT is a vector type that has element type ElementType.
Definition SemaExpr.cpp:9586
memset
__DEVICE__ void * memset(void *__a, int __b, size_t __c)
Definition __clang_cuda_device_functions.h:1566
bool
#define bool
Definition __clang_spirv_libdevice_declares.h:17
cir::direct::CIRAttrToValue::visit
mlir::Value visit(mlir::Attribute attr)
Definition LowerToLLVM.cpp:388
cir::direct::CIRAttrToValue::visitCirAttr
mlir::Value visitCirAttr(cir::BoolAttr boolAttr)
BoolAttr visitor.
Definition LowerToLLVM.cpp:517
cir::direct::CIRAttrToValue::CIRAttrToValue
CIRAttrToValue(mlir::Operation *parentOp, mlir::ConversionPatternRewriter &rewriter, const mlir::TypeConverter *converter)
Definition LowerToLLVM.cpp:383
cir::direct::GlobalInitAttrRewriter::visit
mlir::Attribute visit(mlir::Attribute attr)
Definition LowerToLLVM.cpp:800
cir::direct::GlobalInitAttrRewriter::visitCirAttr
mlir::Attribute visitCirAttr(cir::FPAttr attr)
Definition LowerToLLVM.cpp:811
cir::direct::GlobalInitAttrRewriter::visitCirAttr
mlir::Attribute visitCirAttr(cir::BoolAttr attr)
Definition LowerToLLVM.cpp:815
cir::direct::GlobalInitAttrRewriter::GlobalInitAttrRewriter
GlobalInitAttrRewriter(mlir::Type type, mlir::ConversionPatternRewriter &rewriter)
Definition LowerToLLVM.cpp:796
cir::direct::GlobalInitAttrRewriter::visitCirAttr
mlir::Attribute visitCirAttr(cir::IntAttr attr)
Definition LowerToLLVM.cpp:807
cir::direct::createLLVMFuncOpIfNotExist
void createLLVMFuncOpIfNotExist(mlir::ConversionPatternRewriter &rewriter, mlir::Operation *srcOp, llvm::StringRef fnName, mlir::Type fnTy)
Definition LowerToLLVM.cpp:3789
cir::direct::collectUnreachable
static void collectUnreachable(mlir::Operation *parent, llvm::SmallVector< mlir::Operation * > &ops)
Definition LowerToLLVM.cpp:3566
cir::direct::getLLVMAtomicBinOp
static mlir::LLVM::AtomicBinOp getLLVMAtomicBinOp(cir::AtomicFetchKind k, bool isInt, bool isSignedInt)
Definition LowerToLLVM.cpp:1071
cir::direct::convertCmpKindToICmpPredicate
static mlir::LLVM::ICmpPredicate convertCmpKindToICmpPredicate(cir::CmpOpKind kind, bool isSigned)
Convert from a CIR comparison kind to an LLVM IR integral comparison kind.
Definition LowerToLLVM.cpp:3022
cir::direct::convertSideEffectForCall
void convertSideEffectForCall(mlir::Operation *callOp, bool isNothrow, cir::SideEffect sideEffect, mlir::LLVM::MemoryEffectsAttr &memoryEffect, bool &noUnwind, bool &willReturn, bool &noReturn)
Definition LowerToLLVM.cpp:430
cir::direct::lowerIncDecOp
static mlir::LogicalResult lowerIncDecOp(CIROp op, typename CIROp::Adaptor adaptor, mlir::ConversionPatternRewriter &rewriter, double fpConstant)
Definition LowerToLLVM.cpp:2767
cir::direct::intOverflowFlag
static mlir::LLVM::IntegerOverflowFlags intOverflowFlag(BinOp op)
Definition LowerToLLVM.cpp:2874
cir::direct::lowerIntFPBinaryOp
static mlir::LogicalResult lowerIntFPBinaryOp(CIROp op, mlir::Value lhs, mlir::Value rhs, mlir::ConversionPatternRewriter &rewriter)
Lower a binary op that maps to unsigned/signed/FP LLVM ops depending on operand type.
Definition LowerToLLVM.cpp:2939
cir::direct::lowerDirectlyFromCIRToLLVMIR
std::unique_ptr< llvm::Module > lowerDirectlyFromCIRToLLVMIR(mlir::ModuleOp mlirModule, llvm::LLVMContext &llvmCtx, llvm::StringRef mlirSaveTempsOutFile={})
cir::direct::getLLVMIntCast
static mlir::Value getLLVMIntCast(mlir::ConversionPatternRewriter &rewriter, mlir::Value llvmSrc, mlir::Type llvmDstIntTy, bool isUnsigned, uint64_t cirSrcWidth, uint64_t cirDstIntWidth)
Definition LowerToLLVM.cpp:363
cir::direct::emitFromMemory
static mlir::Value emitFromMemory(mlir::ConversionPatternRewriter &rewriter, mlir::DataLayout const &dataLayout, cir::LoadOp op, mlir::Value value)
Emits the value from memory as expected by its users.
Definition LowerToLLVM.cpp:122
cir::direct::computeBitfieldIntType
mlir::IntegerType computeBitfieldIntType(mlir::Type storageType, mlir::MLIRContext *context, unsigned &storageSize)
Definition LowerToLLVM.cpp:4478
cir::direct::createCallLLVMIntrinsicOp
static mlir::LLVM::CallIntrinsicOp createCallLLVMIntrinsicOp(mlir::ConversionPatternRewriter &rewriter, mlir::Location loc, const llvm::Twine &intrinsicName, mlir::Type resultTy, mlir::ValueRange operands)
Definition LowerToLLVM.cpp:473
cir::direct::hasTrailingZeros
bool hasTrailingZeros(cir::ConstArrayAttr attr)
Definition LowerToLLVM.cpp:2149
cir::direct::lowerMinMaxOp
static mlir::LogicalResult lowerMinMaxOp(CIROp op, typename CIROp::Adaptor adaptor, mlir::ConversionPatternRewriter &rewriter)
Definition LowerToLLVM.cpp:2995
cir::direct::getLLVMLandingPadStructTy
static mlir::LLVM::LLVMStructType getLLVMLandingPadStructTy(mlir::ConversionPatternRewriter &rewriter)
Definition LowerToLLVM.cpp:3873
cir::direct::lowerCirAttrAsValue
mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp, const mlir::Attribute attr, mlir::ConversionPatternRewriter &rewriter, const mlir::TypeConverter *converter)
Switches on the type of attribute and calls the appropriate conversion.
Definition LowerToLLVM.cpp:419
cir::direct::convertToIndexTy
static mlir::Value convertToIndexTy(mlir::ConversionPatternRewriter &rewriter, mlir::ModuleOp mod, mlir::Value index, mlir::Type baseTy, cir::IntType strideTy)
Definition LowerToLLVM.cpp:1544
cir::direct::lowerSaturatableArithOp
static mlir::LogicalResult lowerSaturatableArithOp(CIROp op, mlir::Value lhs, mlir::Value rhs, mlir::ConversionPatternRewriter &rewriter)
Lower an arithmetic op that supports saturation, overflow flags, and an FP variant.
Definition LowerToLLVM.cpp:2887
cir::direct::replaceOpWithCallLLVMIntrinsicOp
static mlir::LLVM::CallIntrinsicOp replaceOpWithCallLLVMIntrinsicOp(mlir::ConversionPatternRewriter &rewriter, mlir::Operation *op, const llvm::Twine &intrinsicName, mlir::Type resultTy, mlir::ValueRange operands)
Definition LowerToLLVM.cpp:482
cir::direct::prepareTypeConverter
static void prepareTypeConverter(mlir::LLVMTypeConverter &converter, mlir::DataLayout &dataLayout)
Definition LowerToLLVM.cpp:3351
cir::direct::getLLVMMemOrder
static mlir::LLVM::AtomicOrdering getLLVMMemOrder(std::optional< cir::MemOrder > memorder)
Definition LowerToLLVM.cpp:949
cir::direct::createConvertCIRToLLVMPass
std::unique_ptr< mlir::Pass > createConvertCIRToLLVMPass()
Create a pass that fully lowers CIR to the LLVMIR dialect.
Definition LowerToLLVM.cpp:4933
cir::direct::getLLVMSyncScope
static llvm::StringRef getLLVMSyncScope(cir::SyncScopeKind syncScope)
Definition LowerToLLVM.cpp:968
cir::direct::convertCmpKindToFCmpPredicate
static mlir::LLVM::FCmpPredicate convertCmpKindToFCmpPredicate(cir::CmpOpKind kind)
Convert from a CIR comparison kind to an LLVM IR floating-point comparison kind.
Definition LowerToLLVM.cpp:3045
cir::direct::lowerCIRVisibilityToLLVMVisibility
static mlir::LLVM::Visibility lowerCIRVisibilityToLLVMVisibility(cir::VisibilityKind visibilityKind)
Definition LowerToLLVM.cpp:108
cir::direct::lowerCallAttributes
static void lowerCallAttributes(cir::CIRCallOpInterface op, SmallVectorImpl< mlir::NamedAttribute > &result)
Definition LowerToLLVM.cpp:1894
cir::direct::getTypeSize
static uint64_t getTypeSize(mlir::Type type, mlir::Operation &op)
Definition LowerToLLVM.cpp:2294
cir::direct::populateCIRToLLVMPasses
void populateCIRToLLVMPasses(mlir::OpPassManager &pm)
Adds passes that fully lower CIR to the LLVMIR dialect.
Definition LowerToLLVM.cpp:4937
cir::direct::getLLVMBinopForPostAtomic
static llvm::StringLiteral getLLVMBinopForPostAtomic(cir::AtomicFetchKind k, bool isInt)
Definition LowerToLLVM.cpp:1105
cir::direct::convertLinkage
mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage)
Definition LowerToLLVM.cpp:154
cir::direct::buildCtorDtorList
static void buildCtorDtorList(mlir::ModuleOp module, StringRef globalXtorName, StringRef llvmXtorName, llvm::function_ref< std::pair< StringRef, int >(mlir::Attribute)> createXtor)
Definition LowerToLLVM.cpp:3468
cir::direct::lowerBinOpOverflow
static mlir::LogicalResult lowerBinOpOverflow(OpTy op, typename OpTy::Adaptor adaptor, mlir::ConversionPatternRewriter &rewriter, const mlir::TypeConverter *typeConverter, llvm::StringRef opStr)
Shared lowering logic for checked binary arithmetic overflow operations.
Definition LowerToLLVM.cpp:3170
cir::direct::convertTypeForMemory
static mlir::Type convertTypeForMemory(const mlir::TypeConverter &converter, mlir::DataLayout const &dataLayout, mlir::Type type)
Given a type convertor and a data layout, convert the given type to a type that is suitable for memor...
Definition LowerToLLVM.cpp:75
cir::direct::createIntCast
static mlir::Value createIntCast(mlir::OpBuilder &bld, mlir::Value src, mlir::IntegerType dstTy, bool isSigned=false)
Definition LowerToLLVM.cpp:88
cir::direct::nswFlag
static mlir::LLVM::IntegerOverflowFlags nswFlag(bool nsw)
Definition LowerToLLVM.cpp:2760
cir::direct::getValueForVTableSymbol
static mlir::Value getValueForVTableSymbol(mlir::Operation *op, mlir::ConversionPatternRewriter &rewriter, const mlir::TypeConverter *converter, mlir::FlatSymbolRefAttr nameAttr, mlir::Type &eltType)
Definition LowerToLLVM.cpp:4010
cir::direct::emitToMemory
static mlir::Value emitToMemory(mlir::ConversionPatternRewriter &rewriter, mlir::DataLayout const &dataLayout, mlir::Type origType, mlir::Value value)
Emits a value to memory with the expected scalar type.
Definition LowerToLLVM.cpp:139
cir::direct::rewriteCallOrInvoke
static mlir::LogicalResult rewriteCallOrInvoke(mlir::Operation *op, mlir::ValueRange callOperands, mlir::ConversionPatternRewriter &rewriter, const mlir::TypeConverter *converter, mlir::FlatSymbolRefAttr calleeAttr, mlir::Block *continueBlock=nullptr, mlir::Block *landingPadBlock=nullptr)
Definition LowerToLLVM.cpp:1911
cir::direct::isIntTypeUnsigned
static bool isIntTypeUnsigned(mlir::Type type)
Definition LowerToLLVM.cpp:2862
clang::ast_matchers::attr
const internal::VariadicAllOfMatcher< Attr > attr
Definition ASTMatchersInternal.cpp:1088
clang::ast_matchers::type
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
Definition ASTMatchersInternal.cpp:809
clang::ast_matchers::complexType
const AstTypeMatcher< ComplexType > complexType
Definition ASTMatchersInternal.cpp:1093
hlsl::uint64_t
unsigned long uint64_t
Definition hlsl_basic_types.h:42
hlsl::int64_t
long int64_t
Definition hlsl_basic_types.h:43
hlsl::uint32_t
unsigned int uint32_t
Definition hlsl_basic_types.h:36
llvm
Diagnostic wrappers for TextAPI types for error reporting.
Definition Dominators.h:30
mlir::populateCIRPreLoweringPasses
void populateCIRPreLoweringPasses(mlir::OpPassManager &pm)
mlir::registerCIRDialectTranslation
void registerCIRDialectTranslation(mlir::MLIRContext &context)
Definition LowerToLLVMIR.cpp:124
clz
char __ovld __cnfn clz(char)
Returns the number of leading 0-bits in x, starting at the most significant bit position.
ctz
char __ovld __cnfn ctz(char)
Returns the count of trailing 0-bits in x.
sign
float __ovld __cnfn sign(float)
Returns 1.0 if x > 0, -0.0 if x = -0.0, +0.0 if x = +0.0, or -1.0 if x < 0.
length
float __ovld __cnfn length(float)
Return the length of vector p, i.e., sqrt(p.x2 + p.y 2 + ...)
select
char __ovld __cnfn select(char, char, char)
For each component of a vector type, result[i] = if MSB of c[i] is set ?
cir::MissingFeatures::dataMemberType
static bool dataMemberType()
Definition MissingFeatures.h:359
cir::MissingFeatures::addressSpace
static bool addressSpace()
Definition MissingFeatures.h:28
cir::MissingFeatures::globalViewIntLowering
static bool globalViewIntLowering()
Definition MissingFeatures.h:284
cir::MissingFeatures::opAllocaAnnotations
static bool opAllocaAnnotations()
Definition MissingFeatures.h:67
cir::MissingFeatures::opLoadStoreTbaa
static bool opLoadStoreTbaa()
Definition MissingFeatures.h:54
cir::MissingFeatures::optInfoAttr
static bool optInfoAttr()
Definition MissingFeatures.h:377
cir::MissingFeatures::opFuncExtraAttrs
static bool opFuncExtraAttrs()
Definition MissingFeatures.h:78
cir::MissingFeatures::isPPC_FP128Ty
static bool isPPC_FP128Ty()
Definition MissingFeatures.h:270
cir::MissingFeatures::vaArgABILowering
static bool vaArgABILowering()
Definition MissingFeatures.h:346
cir::MissingFeatures::fpConstraints
static bool fpConstraints()
Definition MissingFeatures.h:280
cir::MissingFeatures::intrinsicElementTypeSupport
static bool intrinsicElementTypeSupport()
Definition MissingFeatures.h:291
cir::MissingFeatures::lowerModeOptLevel
static bool lowerModeOptLevel()
Definition MissingFeatures.h:298
cir::MissingFeatures::opCallCallConv
static bool opCallCallConv()
Definition MissingFeatures.h:105
cir::MissingFeatures::opFuncCallingConv
static bool opFuncCallingConv()
Definition MissingFeatures.h:74
cir::MissingFeatures::aggValueSlotVolatile
static bool aggValueSlotVolatile()
Definition MissingFeatures.h:207
cir::MissingFeatures::fastMathFlags
static bool fastMathFlags()
Definition MissingFeatures.h:277
cir::MissingFeatures::llvmLoweringPtrDiffConsidersPointee
static bool llvmLoweringPtrDiffConsidersPointee()
Definition MissingFeatures.h:370
cir::MissingFeatures::opLoadStoreNontemporal
static bool opLoadStoreNontemporal()
Definition MissingFeatures.h:53
cir::MissingFeatures::atomicSyncScopeID
static bool atomicSyncScopeID()
Definition MissingFeatures.h:172
cir::MissingFeatures::opFuncMultipleReturnVals
static bool opFuncMultipleReturnVals()
Definition MissingFeatures.h:81
cir::direct::ConvertCIRToLLVMPass::getDescription
StringRef getDescription() const override
Definition LowerToLLVM.cpp:841
cir::direct::ConvertCIRToLLVMPass::getArgument
StringRef getArgument() const override
Definition LowerToLLVM.cpp:845
cir::direct::ConvertCIRToLLVMPass::getDependentDialects
void getDependentDialects(mlir::DialectRegistry &registry) const override
Definition LowerToLLVM.cpp:831
cir::direct::ConvertCIRToLLVMPass::resolveBlockAddressOp
void resolveBlockAddressOp(LLVMBlockAddressInfo &blockInfoAddr)
Definition LowerToLLVM.cpp:3621
cir::direct::ConvertCIRToLLVMPass::processCIRAttrs
void processCIRAttrs(mlir::ModuleOp module)
Definition LowerToLLVM.cpp:3639
cir::direct::LLVMBlockAddressInfo::lookupBlockTag
mlir::LLVM::BlockTagOp lookupBlockTag(cir::BlockAddrInfoAttr info) const
Definition LowerToLLVM.h:49
cir::direct::LLVMBlockAddressInfo::getUnresolvedBlockAddress
llvm::DenseMap< mlir::LLVM::BlockAddressOp, cir::BlockAddrInfoAttr > & getUnresolvedBlockAddress()
Definition LowerToLLVM.h:62
Generated on for clang by doxygen 1.14.0