18#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
19#include "mlir/Dialect/DLTI/DLTI.h"
20#include "mlir/Dialect/Func/IR/FuncOps.h"
21#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
22#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
23#include "mlir/IR/BuiltinAttributes.h"
24#include "mlir/IR/BuiltinDialect.h"
25#include "mlir/IR/BuiltinOps.h"
26#include "mlir/IR/Types.h"
27#include "mlir/Pass/Pass.h"
28#include "mlir/Pass/PassManager.h"
29#include "mlir/Target/LLVMIR/Dialect/Builtin/BuiltinToLLVMIRTranslation.h"
30#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
31#include "mlir/Target/LLVMIR/Export.h"
32#include "mlir/Transforms/DialectConversion.h"
39#include "llvm/ADT/TypeSwitch.h"
40#include "llvm/IR/Module.h"
41#include "llvm/Support/ErrorHandling.h"
42#include "llvm/Support/TimeProfiler.h"
57mlir::Type elementTypeIfVector(mlir::Type type) {
58 return llvm::TypeSwitch<mlir::Type, mlir::Type>(type)
59 .Case<cir::VectorType, mlir::VectorType>(
60 [](
auto p) {
return p.getElementType(); })
61 .
Default([](mlir::Type p) {
return p; });
69 mlir::DataLayout
const &dataLayout,
73 if (isa<cir::BoolType>(type)) {
74 return mlir::IntegerType::get(type.getContext(),
75 dataLayout.getTypeSizeInBits(type));
78 return converter.convertType(type);
82 mlir::IntegerType dstTy,
83 bool isSigned =
false) {
84 mlir::Type srcTy = src.getType();
85 assert(mlir::isa<mlir::IntegerType>(srcTy));
87 unsigned srcWidth = mlir::cast<mlir::IntegerType>(srcTy).getWidth();
88 unsigned dstWidth = mlir::cast<mlir::IntegerType>(dstTy).getWidth();
89 mlir::Location loc = src.getLoc();
91 if (dstWidth > srcWidth && isSigned)
92 return bld.create<mlir::LLVM::SExtOp>(loc, dstTy, src);
93 if (dstWidth > srcWidth)
94 return bld.create<mlir::LLVM::ZExtOp>(loc, dstTy, src);
95 if (dstWidth < srcWidth)
96 return bld.create<mlir::LLVM::TruncOp>(loc, dstTy, src);
97 return bld.create<mlir::LLVM::BitcastOp>(loc, dstTy, src);
100static mlir::LLVM::Visibility
102 switch (visibilityKind) {
103 case cir::VisibilityKind::Default:
104 return ::mlir::LLVM::Visibility::Default;
105 case cir::VisibilityKind::Hidden:
106 return ::mlir::LLVM::Visibility::Hidden;
107 case cir::VisibilityKind::Protected:
108 return ::mlir::LLVM::Visibility::Protected;
116 mlir::DataLayout
const &dataLayout,
117 cir::LoadOp op, mlir::Value value) {
120 if (
auto boolTy = mlir::dyn_cast<cir::BoolType>(op.getType())) {
122 assert(value.getType().isInteger(dataLayout.getTypeSizeInBits(boolTy)));
132static mlir::Value
emitToMemory(mlir::ConversionPatternRewriter &rewriter,
133 mlir::DataLayout
const &dataLayout,
134 mlir::Type origType, mlir::Value value) {
137 if (
auto boolTy = mlir::dyn_cast<cir::BoolType>(origType)) {
139 mlir::IntegerType memType =
140 rewriter.getIntegerType(dataLayout.getTypeSizeInBits(boolTy));
148 using CIR = cir::GlobalLinkageKind;
149 using LLVM = mlir::LLVM::Linkage;
152 case CIR::AvailableExternallyLinkage:
153 return LLVM::AvailableExternally;
154 case CIR::CommonLinkage:
156 case CIR::ExternalLinkage:
157 return LLVM::External;
158 case CIR::ExternalWeakLinkage:
159 return LLVM::ExternWeak;
160 case CIR::InternalLinkage:
161 return LLVM::Internal;
162 case CIR::LinkOnceAnyLinkage:
163 return LLVM::Linkonce;
164 case CIR::LinkOnceODRLinkage:
165 return LLVM::LinkonceODR;
166 case CIR::PrivateLinkage:
167 return LLVM::Private;
168 case CIR::WeakAnyLinkage:
170 case CIR::WeakODRLinkage:
171 return LLVM::WeakODR;
173 llvm_unreachable(
"Unknown CIR linkage type");
176mlir::LogicalResult CIRToLLVMCopyOpLowering::matchAndRewrite(
177 cir::CopyOp op, OpAdaptor adaptor,
178 mlir::ConversionPatternRewriter &rewriter)
const {
179 mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>());
180 const mlir::Value
length = mlir::LLVM::ConstantOp::create(
181 rewriter, op.getLoc(), rewriter.getI32Type(), op.getLength(layout));
183 rewriter.replaceOpWithNewOp<mlir::LLVM::MemcpyOp>(
184 op, adaptor.getDst(), adaptor.getSrc(),
length,
false);
185 return mlir::success();
188mlir::LogicalResult CIRToLLVMCosOpLowering::matchAndRewrite(
189 cir::CosOp op, OpAdaptor adaptor,
190 mlir::ConversionPatternRewriter &rewriter)
const {
191 mlir::Type resTy = typeConverter->convertType(op.getType());
192 rewriter.replaceOpWithNewOp<mlir::LLVM::CosOp>(op, resTy, adaptor.getSrc());
193 return mlir::success();
197 mlir::Value llvmSrc, mlir::Type llvmDstIntTy,
199 uint64_t cirDstIntWidth) {
200 if (cirSrcWidth == cirDstIntWidth)
203 auto loc = llvmSrc.getLoc();
204 if (cirSrcWidth < cirDstIntWidth) {
206 return rewriter.create<mlir::LLVM::ZExtOp>(loc, llvmDstIntTy, llvmSrc);
207 return rewriter.create<mlir::LLVM::SExtOp>(loc, llvmDstIntTy, llvmSrc);
211 return rewriter.create<mlir::LLVM::TruncOp>(loc, llvmDstIntTy, llvmSrc);
217 mlir::ConversionPatternRewriter &rewriter,
218 const mlir::TypeConverter *converter)
219 : parentOp(parentOp), rewriter(rewriter), converter(converter) {}
221 mlir::Value
visit(mlir::Attribute attr) {
222 return llvm::TypeSwitch<mlir::Attribute, mlir::Value>(attr)
223 .Case<cir::IntAttr, cir::FPAttr, cir::ConstComplexAttr,
224 cir::ConstArrayAttr, cir::ConstRecordAttr, cir::ConstVectorAttr,
225 cir::ConstPtrAttr, cir::GlobalViewAttr, cir::VTableAttr,
226 cir::ZeroAttr>([&](
auto attrT) {
return visitCirAttr(attrT); })
227 .Default([&](
auto attrT) {
return mlir::Value(); });
232 mlir::Value
visitCirAttr(cir::ConstComplexAttr complexAttr);
243 mlir::Operation *parentOp;
244 mlir::ConversionPatternRewriter &rewriter;
245 const mlir::TypeConverter *converter;
250 const mlir::Attribute attr,
251 mlir::ConversionPatternRewriter &rewriter,
252 const mlir::TypeConverter *converter) {
254 mlir::Value value = valueConverter.
visit(attr);
256 llvm_unreachable(
"unhandled attribute type");
261 cir::SideEffect sideEffect,
262 mlir::LLVM::MemoryEffectsAttr &memoryEffect,
263 bool &noUnwind,
bool &willReturn) {
264 using mlir::LLVM::ModRefInfo;
266 switch (sideEffect) {
267 case cir::SideEffect::All:
269 noUnwind = isNothrow;
273 case cir::SideEffect::Pure:
274 memoryEffect = mlir::LLVM::MemoryEffectsAttr::get(
275 callOp->getContext(), ModRefInfo::Ref,
282 case cir::SideEffect::Const:
283 memoryEffect = mlir::LLVM::MemoryEffectsAttr::get(
284 callOp->getContext(), ModRefInfo::NoModRef,
285 ModRefInfo::NoModRef,
286 ModRefInfo::NoModRef);
293static mlir::LLVM::CallIntrinsicOp
295 mlir::Location loc,
const llvm::Twine &intrinsicName,
296 mlir::Type resultTy, mlir::ValueRange operands) {
297 auto intrinsicNameAttr =
298 mlir::StringAttr::get(rewriter.getContext(), intrinsicName);
299 return mlir::LLVM::CallIntrinsicOp::create(rewriter, loc, resultTy,
300 intrinsicNameAttr, operands);
304 mlir::ConversionPatternRewriter &rewriter, mlir::Operation *op,
305 const llvm::Twine &intrinsicName, mlir::Type resultTy,
306 mlir::ValueRange operands) {
308 rewriter, op->getLoc(), intrinsicName, resultTy, operands);
309 rewriter.replaceOp(op, callIntrinOp.getOperation());
315 mlir::Location loc = parentOp->getLoc();
316 return rewriter.create<mlir::LLVM::ConstantOp>(
317 loc, converter->convertType(intAttr.getType()), intAttr.getValue());
322 mlir::Location loc = parentOp->getLoc();
323 return rewriter.create<mlir::LLVM::ConstantOp>(
324 loc, converter->convertType(fltAttr.getType()), fltAttr.getValue());
329 auto complexType = mlir::cast<cir::ComplexType>(complexAttr.getType());
330 mlir::Type complexElemTy = complexType.getElementType();
331 mlir::Type complexElemLLVMTy = converter->convertType(complexElemTy);
333 mlir::Attribute components[2];
334 if (
const auto intType = mlir::dyn_cast<cir::IntType>(complexElemTy)) {
335 components[0] = rewriter.getIntegerAttr(
337 mlir::cast<cir::IntAttr>(complexAttr.getReal()).getValue());
338 components[1] = rewriter.getIntegerAttr(
340 mlir::cast<cir::IntAttr>(complexAttr.getImag()).getValue());
342 components[0] = rewriter.getFloatAttr(
344 mlir::cast<cir::FPAttr>(complexAttr.getReal()).getValue());
345 components[1] = rewriter.getFloatAttr(
347 mlir::cast<cir::FPAttr>(complexAttr.getImag()).getValue());
350 mlir::Location loc = parentOp->getLoc();
351 return rewriter.create<mlir::LLVM::ConstantOp>(
352 loc, converter->convertType(complexAttr.getType()),
353 rewriter.getArrayAttr(components));
358 mlir::Location loc = parentOp->getLoc();
359 if (ptrAttr.isNullValue()) {
360 return rewriter.create<mlir::LLVM::ZeroOp>(
361 loc, converter->convertType(ptrAttr.getType()));
363 mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>());
364 mlir::Value ptrVal = rewriter.create<mlir::LLVM::ConstantOp>(
365 loc, rewriter.getIntegerType(layout.getTypeSizeInBits(ptrAttr.getType())),
366 ptrAttr.getValue().getInt());
367 return rewriter.create<mlir::LLVM::IntToPtrOp>(
368 loc, converter->convertType(ptrAttr.getType()), ptrVal);
373 mlir::Type llvmTy = converter->convertType(attr.getType());
374 mlir::Location loc = parentOp->getLoc();
377 if (attr.hasTrailingZeros()) {
378 mlir::Type arrayTy = attr.getType();
379 result = rewriter.create<mlir::LLVM::ZeroOp>(
380 loc, converter->convertType(arrayTy));
382 result = rewriter.create<mlir::LLVM::UndefOp>(loc, llvmTy);
386 if (
auto arrayAttr = mlir::dyn_cast<mlir::ArrayAttr>(attr.getElts())) {
387 for (
auto [idx, elt] : llvm::enumerate(arrayAttr)) {
388 mlir::DataLayout dataLayout(parentOp->getParentOfType<mlir::ModuleOp>());
389 mlir::Value init =
visit(elt);
391 rewriter.create<mlir::LLVM::InsertValueOp>(loc, result, init, idx);
393 }
else if (
auto strAttr = mlir::dyn_cast<mlir::StringAttr>(attr.getElts())) {
396 auto arrayTy = mlir::dyn_cast<cir::ArrayType>(strAttr.getType());
397 assert(arrayTy &&
"String attribute must have an array type");
398 mlir::Type eltTy = arrayTy.getElementType();
399 for (
auto [idx, elt] : llvm::enumerate(strAttr)) {
400 auto init = rewriter.create<mlir::LLVM::ConstantOp>(
401 loc, converter->convertType(eltTy), elt);
403 rewriter.create<mlir::LLVM::InsertValueOp>(loc, result, init, idx);
406 llvm_unreachable(
"unexpected ConstArrayAttr elements");
414 const mlir::Type llvmTy = converter->convertType(constRecord.getType());
415 const mlir::Location loc = parentOp->getLoc();
416 mlir::Value result = rewriter.create<mlir::LLVM::UndefOp>(loc, llvmTy);
419 for (
auto [idx, elt] : llvm::enumerate(constRecord.getMembers())) {
420 mlir::Value init =
visit(elt);
421 result = rewriter.create<mlir::LLVM::InsertValueOp>(loc, result, init, idx);
429 const mlir::Type llvmTy = converter->convertType(attr.getType());
430 const mlir::Location loc = parentOp->getLoc();
433 for (
const mlir::Attribute elementAttr : attr.getElts()) {
434 mlir::Attribute mlirAttr;
435 if (
auto intAttr = mlir::dyn_cast<cir::IntAttr>(elementAttr)) {
436 mlirAttr = rewriter.getIntegerAttr(
437 converter->convertType(intAttr.getType()), intAttr.getValue());
438 }
else if (
auto floatAttr = mlir::dyn_cast<cir::FPAttr>(elementAttr)) {
439 mlirAttr = rewriter.getFloatAttr(
440 converter->convertType(floatAttr.getType()), floatAttr.getValue());
443 "vector constant with an element that is neither an int nor a float");
445 mlirValues.push_back(mlirAttr);
448 return rewriter.create<mlir::LLVM::ConstantOp>(
450 mlir::DenseElementsAttr::get(mlir::cast<mlir::ShapedType>(llvmTy),
456 auto moduleOp = parentOp->getParentOfType<mlir::ModuleOp>();
457 mlir::DataLayout dataLayout(moduleOp);
458 mlir::Type sourceType;
460 llvm::StringRef symName;
461 mlir::Operation *sourceSymbol =
462 mlir::SymbolTable::lookupSymbolIn(moduleOp, globalAttr.getSymbol());
463 if (
auto llvmSymbol = dyn_cast<mlir::LLVM::GlobalOp>(sourceSymbol)) {
464 sourceType = llvmSymbol.getType();
465 symName = llvmSymbol.getSymName();
466 }
else if (
auto cirSymbol = dyn_cast<cir::GlobalOp>(sourceSymbol)) {
469 symName = cirSymbol.getSymName();
470 }
else if (
auto llvmFun = dyn_cast<mlir::LLVM::LLVMFuncOp>(sourceSymbol)) {
471 sourceType = llvmFun.getFunctionType();
472 symName = llvmFun.getSymName();
473 }
else if (
auto fun = dyn_cast<cir::FuncOp>(sourceSymbol)) {
474 sourceType = converter->convertType(fun.getFunctionType());
475 symName = fun.getSymName();
476 }
else if (
auto alias = dyn_cast<mlir::LLVM::AliasOp>(sourceSymbol)) {
477 sourceType = alias.getType();
478 symName = alias.getSymName();
480 llvm_unreachable(
"Unexpected GlobalOp type");
483 mlir::Location loc = parentOp->getLoc();
484 mlir::Value addrOp = rewriter.create<mlir::LLVM::AddressOfOp>(
485 loc, mlir::LLVM::LLVMPointerType::get(rewriter.getContext()), symName);
487 if (globalAttr.getIndices()) {
490 if (mlir::isa<mlir::LLVM::LLVMArrayType, mlir::LLVM::LLVMStructType>(
492 indices.push_back(0);
494 for (mlir::Attribute idx : globalAttr.getIndices()) {
495 auto intAttr = mlir::cast<mlir::IntegerAttr>(idx);
496 indices.push_back(intAttr.getValue().getSExtValue());
498 mlir::Type resTy = addrOp.getType();
499 mlir::Type eltTy = converter->convertType(sourceType);
500 addrOp = rewriter.create<mlir::LLVM::GEPOp>(
501 loc, resTy, eltTy, addrOp, indices, mlir::LLVM::GEPNoWrapFlags::none);
510 if (
auto ptrTy = mlir::dyn_cast<cir::PointerType>(globalAttr.getType())) {
511 mlir::Type llvmEltTy =
514 if (llvmEltTy == sourceType)
517 mlir::Type llvmDstTy = converter->convertType(globalAttr.getType());
518 return rewriter.create<mlir::LLVM::BitcastOp>(parentOp->getLoc(), llvmDstTy,
522 llvm_unreachable(
"Expecting pointer or integer type for GlobalViewAttr");
527 mlir::Type llvmTy = converter->convertType(typeInfoAttr.getType());
528 mlir::Location loc = parentOp->getLoc();
529 mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);
531 for (
auto [idx, elt] : llvm::enumerate(typeInfoAttr.getData())) {
532 mlir::Value init =
visit(elt);
534 mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);
542 mlir::Type llvmTy = converter->convertType(vtableArr.getType());
543 mlir::Location loc = parentOp->getLoc();
544 mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);
546 for (
auto [idx, elt] : llvm::enumerate(vtableArr.getData())) {
547 mlir::Value init =
visit(elt);
549 mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);
557 mlir::Location loc = parentOp->getLoc();
558 return rewriter.create<mlir::LLVM::ZeroOp>(
559 loc, converter->convertType(attr.getType()));
567 mlir::ConversionPatternRewriter &rewriter)
568 : llvmType(type), rewriter(rewriter) {}
570 mlir::Attribute
visit(mlir::Attribute attr) {
571 return llvm::TypeSwitch<mlir::Attribute, mlir::Attribute>(attr)
572 .Case<cir::IntAttr, cir::FPAttr, cir::BoolAttr>(
574 .Default([&](
auto attrT) {
return mlir::Attribute(); });
578 return rewriter.getIntegerAttr(llvmType, attr.getValue());
582 return rewriter.getFloatAttr(llvmType, attr.getValue());
586 return rewriter.getBoolAttr(attr.getValue());
591 mlir::ConversionPatternRewriter &rewriter;
599 :
public mlir::PassWrapper<ConvertCIRToLLVMPass,
600 mlir::OperationPass<mlir::ModuleOp>> {
602 registry.insert<mlir::BuiltinDialect, mlir::DLTIDialect,
603 mlir::LLVM::LLVMDialect, mlir::func::FuncDialect>();
610 return "Convert the prepared CIR dialect module to LLVM dialect";
613 StringRef
getArgument()
const override {
return "cir-flat-to-llvm"; }
616mlir::LogicalResult CIRToLLVMACosOpLowering::matchAndRewrite(
617 cir::ACosOp op, OpAdaptor adaptor,
618 mlir::ConversionPatternRewriter &rewriter)
const {
619 mlir::Type resTy = typeConverter->convertType(op.getType());
620 rewriter.replaceOpWithNewOp<mlir::LLVM::ACosOp>(op, resTy,
621 adaptor.getOperands()[0]);
622 return mlir::success();
625mlir::LogicalResult CIRToLLVMASinOpLowering::matchAndRewrite(
626 cir::ASinOp op, OpAdaptor adaptor,
627 mlir::ConversionPatternRewriter &rewriter)
const {
628 mlir::Type resTy = typeConverter->convertType(op.getType());
629 rewriter.replaceOpWithNewOp<mlir::LLVM::ASinOp>(op, resTy, adaptor.getSrc());
630 return mlir::success();
633mlir::LogicalResult CIRToLLVMAssumeOpLowering::matchAndRewrite(
634 cir::AssumeOp op, OpAdaptor adaptor,
635 mlir::ConversionPatternRewriter &rewriter)
const {
636 auto cond = adaptor.getPredicate();
637 rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(op, cond);
638 return mlir::success();
641mlir::LogicalResult CIRToLLVMAssumeAlignedOpLowering::matchAndRewrite(
642 cir::AssumeAlignedOp op, OpAdaptor adaptor,
643 mlir::ConversionPatternRewriter &rewriter)
const {
644 SmallVector<mlir::Value, 3> opBundleArgs{adaptor.getPointer()};
646 auto alignment = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
647 adaptor.getAlignmentAttr());
648 opBundleArgs.push_back(alignment);
650 if (mlir::Value offset = adaptor.getOffset())
651 opBundleArgs.push_back(offset);
653 auto cond = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
654 rewriter.getI1Type(), 1);
655 mlir::LLVM::AssumeOp::create(rewriter, op.getLoc(), cond,
"align",
660 rewriter.replaceOp(op, adaptor.getPointer());
661 return mlir::success();
664mlir::LogicalResult CIRToLLVMAssumeSepStorageOpLowering::matchAndRewrite(
665 cir::AssumeSepStorageOp op, OpAdaptor adaptor,
666 mlir::ConversionPatternRewriter &rewriter)
const {
667 auto cond = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(),
668 rewriter.getI1Type(), 1);
669 rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(
670 op, cond, mlir::LLVM::AssumeSeparateStorageTag{}, adaptor.getPtr1(),
672 return mlir::success();
675static mlir::LLVM::AtomicOrdering
678 return mlir::LLVM::AtomicOrdering::not_atomic;
680 case cir::MemOrder::Relaxed:
681 return mlir::LLVM::AtomicOrdering::monotonic;
682 case cir::MemOrder::Consume:
683 case cir::MemOrder::Acquire:
684 return mlir::LLVM::AtomicOrdering::acquire;
685 case cir::MemOrder::Release:
686 return mlir::LLVM::AtomicOrdering::release;
687 case cir::MemOrder::AcquireRelease:
688 return mlir::LLVM::AtomicOrdering::acq_rel;
689 case cir::MemOrder::SequentiallyConsistent:
690 return mlir::LLVM::AtomicOrdering::seq_cst;
692 llvm_unreachable(
"unknown memory order");
695mlir::LogicalResult CIRToLLVMAtomicCmpXchgLowering::matchAndRewrite(
696 cir::AtomicCmpXchg op, OpAdaptor adaptor,
697 mlir::ConversionPatternRewriter &rewriter)
const {
698 mlir::Value expected = adaptor.getExpected();
699 mlir::Value desired = adaptor.getDesired();
701 auto cmpxchg = mlir::LLVM::AtomicCmpXchgOp::create(
702 rewriter, op.getLoc(), adaptor.getPtr(), expected, desired,
706 cmpxchg.setAlignment(adaptor.getAlignment());
707 cmpxchg.setWeak(adaptor.getWeak());
708 cmpxchg.setVolatile_(adaptor.getIsVolatile());
711 auto old = mlir::LLVM::ExtractValueOp::create(rewriter, op.getLoc(),
712 cmpxchg.getResult(), 0);
713 auto cmp = mlir::LLVM::ExtractValueOp::create(rewriter, op.getLoc(),
714 cmpxchg.getResult(), 1);
716 rewriter.replaceOp(op, {old, cmp});
717 return mlir::success();
720mlir::LogicalResult CIRToLLVMAtomicXchgLowering::matchAndRewrite(
721 cir::AtomicXchg op, OpAdaptor adaptor,
722 mlir::ConversionPatternRewriter &rewriter)
const {
724 mlir::LLVM::AtomicOrdering llvmOrder =
getLLVMMemOrder(adaptor.getMemOrder());
725 rewriter.replaceOpWithNewOp<mlir::LLVM::AtomicRMWOp>(
726 op, mlir::LLVM::AtomicBinOp::xchg, adaptor.getPtr(), adaptor.getVal(),
728 return mlir::success();
731mlir::LogicalResult CIRToLLVMBitClrsbOpLowering::matchAndRewrite(
732 cir::BitClrsbOp op, OpAdaptor adaptor,
733 mlir::ConversionPatternRewriter &rewriter)
const {
734 auto zero = rewriter.create<mlir::LLVM::ConstantOp>(
735 op.getLoc(), adaptor.getInput().getType(), 0);
736 auto isNeg = rewriter.create<mlir::LLVM::ICmpOp>(
738 mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(),
739 mlir::LLVM::ICmpPredicate::slt),
740 adaptor.getInput(), zero);
742 auto negOne = rewriter.create<mlir::LLVM::ConstantOp>(
743 op.getLoc(), adaptor.getInput().getType(), -1);
744 auto flipped = rewriter.create<mlir::LLVM::XOrOp>(op.getLoc(),
745 adaptor.getInput(), negOne);
747 auto select = rewriter.create<mlir::LLVM::SelectOp>(
748 op.getLoc(), isNeg, flipped, adaptor.getInput());
750 auto resTy = getTypeConverter()->convertType(op.getType());
751 auto clz = rewriter.create<mlir::LLVM::CountLeadingZerosOp>(
752 op.getLoc(), resTy,
select,
false);
754 auto one = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 1);
755 auto res = rewriter.create<mlir::LLVM::SubOp>(op.getLoc(),
clz, one);
756 rewriter.replaceOp(op, res);
758 return mlir::LogicalResult::success();
761mlir::LogicalResult CIRToLLVMBitClzOpLowering::matchAndRewrite(
762 cir::BitClzOp op, OpAdaptor adaptor,
763 mlir::ConversionPatternRewriter &rewriter)
const {
764 auto resTy = getTypeConverter()->convertType(op.getType());
765 auto llvmOp = rewriter.create<mlir::LLVM::CountLeadingZerosOp>(
766 op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero());
767 rewriter.replaceOp(op, llvmOp);
768 return mlir::LogicalResult::success();
771mlir::LogicalResult CIRToLLVMBitCtzOpLowering::matchAndRewrite(
772 cir::BitCtzOp op, OpAdaptor adaptor,
773 mlir::ConversionPatternRewriter &rewriter)
const {
774 auto resTy = getTypeConverter()->convertType(op.getType());
775 auto llvmOp = rewriter.create<mlir::LLVM::CountTrailingZerosOp>(
776 op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero());
777 rewriter.replaceOp(op, llvmOp);
778 return mlir::LogicalResult::success();
781mlir::LogicalResult CIRToLLVMBitFfsOpLowering::matchAndRewrite(
782 cir::BitFfsOp op, OpAdaptor adaptor,
783 mlir::ConversionPatternRewriter &rewriter)
const {
784 auto resTy = getTypeConverter()->convertType(op.getType());
785 auto ctz = rewriter.create<mlir::LLVM::CountTrailingZerosOp>(
786 op.getLoc(), resTy, adaptor.getInput(),
true);
788 auto one = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 1);
789 auto ctzAddOne = rewriter.create<mlir::LLVM::AddOp>(op.getLoc(),
ctz, one);
791 auto zeroInputTy = rewriter.create<mlir::LLVM::ConstantOp>(
792 op.getLoc(), adaptor.getInput().getType(), 0);
793 auto isZero = rewriter.create<mlir::LLVM::ICmpOp>(
795 mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(),
796 mlir::LLVM::ICmpPredicate::eq),
797 adaptor.getInput(), zeroInputTy);
799 auto zero = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 0);
800 auto res = rewriter.create<mlir::LLVM::SelectOp>(op.getLoc(), isZero, zero,
802 rewriter.replaceOp(op, res);
804 return mlir::LogicalResult::success();
807mlir::LogicalResult CIRToLLVMBitParityOpLowering::matchAndRewrite(
808 cir::BitParityOp op, OpAdaptor adaptor,
809 mlir::ConversionPatternRewriter &rewriter)
const {
810 auto resTy = getTypeConverter()->convertType(op.getType());
811 auto popcnt = rewriter.create<mlir::LLVM::CtPopOp>(op.getLoc(), resTy,
814 auto one = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 1);
816 rewriter.create<mlir::LLVM::AndOp>(op.getLoc(), popcnt, one);
817 rewriter.replaceOp(op, popcntMod2);
819 return mlir::LogicalResult::success();
822mlir::LogicalResult CIRToLLVMBitPopcountOpLowering::matchAndRewrite(
823 cir::BitPopcountOp op, OpAdaptor adaptor,
824 mlir::ConversionPatternRewriter &rewriter)
const {
825 auto resTy = getTypeConverter()->convertType(op.getType());
826 auto llvmOp = rewriter.create<mlir::LLVM::CtPopOp>(op.getLoc(), resTy,
828 rewriter.replaceOp(op, llvmOp);
829 return mlir::LogicalResult::success();
832mlir::LogicalResult CIRToLLVMBitReverseOpLowering::matchAndRewrite(
833 cir::BitReverseOp op, OpAdaptor adaptor,
834 mlir::ConversionPatternRewriter &rewriter)
const {
835 rewriter.replaceOpWithNewOp<mlir::LLVM::BitReverseOp>(op, adaptor.getInput());
836 return mlir::success();
839mlir::LogicalResult CIRToLLVMBrCondOpLowering::matchAndRewrite(
840 cir::BrCondOp brOp, OpAdaptor adaptor,
841 mlir::ConversionPatternRewriter &rewriter)
const {
846 mlir::Value i1Condition = adaptor.getCond();
848 rewriter.replaceOpWithNewOp<mlir::LLVM::CondBrOp>(
849 brOp, i1Condition, brOp.getDestTrue(), adaptor.getDestOperandsTrue(),
850 brOp.getDestFalse(), adaptor.getDestOperandsFalse());
852 return mlir::success();
855mlir::LogicalResult CIRToLLVMByteSwapOpLowering::matchAndRewrite(
856 cir::ByteSwapOp op, OpAdaptor adaptor,
857 mlir::ConversionPatternRewriter &rewriter)
const {
858 rewriter.replaceOpWithNewOp<mlir::LLVM::ByteSwapOp>(op, adaptor.getInput());
859 return mlir::LogicalResult::success();
862mlir::Type CIRToLLVMCastOpLowering::convertTy(mlir::Type ty)
const {
863 return getTypeConverter()->convertType(ty);
866mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
867 cir::CastOp castOp, OpAdaptor adaptor,
868 mlir::ConversionPatternRewriter &rewriter)
const {
873 switch (castOp.getKind()) {
874 case cir::CastKind::array_to_ptrdecay: {
875 const auto ptrTy = mlir::cast<cir::PointerType>(castOp.getType());
876 mlir::Value sourceValue = adaptor.getSrc();
877 mlir::Type targetType = convertTy(ptrTy);
880 llvm::SmallVector<mlir::LLVM::GEPArg> offset{0};
881 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
882 castOp, targetType, elementTy, sourceValue, offset);
885 case cir::CastKind::int_to_bool: {
886 mlir::Value llvmSrcVal = adaptor.getSrc();
887 mlir::Value zeroInt = rewriter.create<mlir::LLVM::ConstantOp>(
888 castOp.getLoc(), llvmSrcVal.getType(), 0);
889 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
890 castOp, mlir::LLVM::ICmpPredicate::ne, llvmSrcVal, zeroInt);
893 case cir::CastKind::integral: {
894 mlir::Type srcType = castOp.getSrc().getType();
895 mlir::Type dstType = castOp.getType();
896 mlir::Value llvmSrcVal = adaptor.getSrc();
897 mlir::Type llvmDstType = getTypeConverter()->convertType(dstType);
898 cir::IntType srcIntType =
899 mlir::cast<cir::IntType>(elementTypeIfVector(srcType));
900 cir::IntType dstIntType =
901 mlir::cast<cir::IntType>(elementTypeIfVector(dstType));
902 rewriter.replaceOp(castOp,
getLLVMIntCast(rewriter, llvmSrcVal, llvmDstType,
903 srcIntType.isUnsigned(),
904 srcIntType.getWidth(),
905 dstIntType.getWidth()));
908 case cir::CastKind::floating: {
909 mlir::Value llvmSrcVal = adaptor.getSrc();
910 mlir::Type llvmDstTy = getTypeConverter()->convertType(castOp.getType());
912 mlir::Type srcTy = elementTypeIfVector(castOp.getSrc().getType());
913 mlir::Type dstTy = elementTypeIfVector(castOp.getType());
915 if (!mlir::isa<cir::FPTypeInterface>(dstTy) ||
916 !mlir::isa<cir::FPTypeInterface>(srcTy))
917 return castOp.emitError() <<
"NYI cast from " << srcTy <<
" to " << dstTy;
919 auto getFloatWidth = [](mlir::Type ty) ->
unsigned {
920 return mlir::cast<cir::FPTypeInterface>(ty).getWidth();
923 if (getFloatWidth(srcTy) > getFloatWidth(dstTy))
924 rewriter.replaceOpWithNewOp<mlir::LLVM::FPTruncOp>(castOp, llvmDstTy,
927 rewriter.replaceOpWithNewOp<mlir::LLVM::FPExtOp>(castOp, llvmDstTy,
929 return mlir::success();
931 case cir::CastKind::int_to_ptr: {
932 auto dstTy = mlir::cast<cir::PointerType>(castOp.getType());
933 mlir::Value llvmSrcVal = adaptor.getSrc();
934 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
935 rewriter.replaceOpWithNewOp<mlir::LLVM::IntToPtrOp>(castOp, llvmDstTy,
937 return mlir::success();
939 case cir::CastKind::ptr_to_int: {
940 auto dstTy = mlir::cast<cir::IntType>(castOp.getType());
941 mlir::Value llvmSrcVal = adaptor.getSrc();
942 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
943 rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(castOp, llvmDstTy,
945 return mlir::success();
947 case cir::CastKind::float_to_bool: {
948 mlir::Value llvmSrcVal = adaptor.getSrc();
949 auto kind = mlir::LLVM::FCmpPredicate::une;
952 auto zeroFloat = rewriter.create<mlir::LLVM::ConstantOp>(
953 castOp.getLoc(), llvmSrcVal.getType(),
954 mlir::FloatAttr::get(llvmSrcVal.getType(), 0.0));
957 rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(castOp,
kind, llvmSrcVal,
960 return mlir::success();
962 case cir::CastKind::bool_to_int: {
963 auto dstTy = mlir::cast<cir::IntType>(castOp.getType());
964 mlir::Value llvmSrcVal = adaptor.getSrc();
965 auto llvmSrcTy = mlir::cast<mlir::IntegerType>(llvmSrcVal.getType());
967 mlir::cast<mlir::IntegerType>(getTypeConverter()->convertType(dstTy));
968 if (llvmSrcTy.getWidth() == llvmDstTy.getWidth())
969 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(castOp, llvmDstTy,
972 rewriter.replaceOpWithNewOp<mlir::LLVM::ZExtOp>(castOp, llvmDstTy,
974 return mlir::success();
976 case cir::CastKind::bool_to_float: {
977 mlir::Type dstTy = castOp.getType();
978 mlir::Value llvmSrcVal = adaptor.getSrc();
979 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
980 rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(castOp, llvmDstTy,
982 return mlir::success();
984 case cir::CastKind::int_to_float: {
985 mlir::Type dstTy = castOp.getType();
986 mlir::Value llvmSrcVal = adaptor.getSrc();
987 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
988 if (mlir::cast<cir::IntType>(elementTypeIfVector(castOp.getSrc().getType()))
990 rewriter.replaceOpWithNewOp<mlir::LLVM::SIToFPOp>(castOp, llvmDstTy,
993 rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(castOp, llvmDstTy,
995 return mlir::success();
997 case cir::CastKind::float_to_int: {
998 mlir::Type dstTy = castOp.getType();
999 mlir::Value llvmSrcVal = adaptor.getSrc();
1000 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
1001 if (mlir::cast<cir::IntType>(elementTypeIfVector(castOp.getType()))
1003 rewriter.replaceOpWithNewOp<mlir::LLVM::FPToSIOp>(castOp, llvmDstTy,
1006 rewriter.replaceOpWithNewOp<mlir::LLVM::FPToUIOp>(castOp, llvmDstTy,
1008 return mlir::success();
1010 case cir::CastKind::bitcast: {
1011 mlir::Type dstTy = castOp.getType();
1012 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
1017 mlir::Value llvmSrcVal = adaptor.getSrc();
1018 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(castOp, llvmDstTy,
1020 return mlir::success();
1022 case cir::CastKind::ptr_to_bool: {
1023 mlir::Value llvmSrcVal = adaptor.getSrc();
1024 mlir::Value zeroPtr = rewriter.create<mlir::LLVM::ZeroOp>(
1025 castOp.getLoc(), llvmSrcVal.getType());
1026 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
1027 castOp, mlir::LLVM::ICmpPredicate::ne, llvmSrcVal, zeroPtr);
1030 case cir::CastKind::address_space: {
1031 mlir::Type dstTy = castOp.getType();
1032 mlir::Value llvmSrcVal = adaptor.getSrc();
1033 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
1034 rewriter.replaceOpWithNewOp<mlir::LLVM::AddrSpaceCastOp>(castOp, llvmDstTy,
1038 case cir::CastKind::member_ptr_to_bool:
1043 return castOp.emitError(
"Unhandled cast kind: ")
1044 << castOp.getKindAttrName();
1048 return mlir::success();
1051mlir::LogicalResult CIRToLLVMPtrStrideOpLowering::matchAndRewrite(
1052 cir::PtrStrideOp ptrStrideOp, OpAdaptor adaptor,
1053 mlir::ConversionPatternRewriter &rewriter)
const {
1055 const mlir::TypeConverter *tc = getTypeConverter();
1056 const mlir::Type resultTy = tc->convertType(ptrStrideOp.getType());
1058 mlir::Type elementTy =
1060 mlir::MLIRContext *ctx = elementTy.getContext();
1064 if (mlir::isa<mlir::LLVM::LLVMVoidType>(elementTy) ||
1065 mlir::isa<mlir::LLVM::LLVMFunctionType>(elementTy))
1066 elementTy = mlir::IntegerType::get(elementTy.getContext(), 8,
1067 mlir::IntegerType::Signless);
1069 mlir::Value index = adaptor.getStride();
1070 const unsigned width =
1071 mlir::cast<mlir::IntegerType>(index.getType()).getWidth();
1072 const std::optional<std::uint64_t> layoutWidth =
1073 dataLayout.getTypeIndexBitwidth(adaptor.getBase().getType());
1075 mlir::Operation *indexOp = index.getDefiningOp();
1076 if (indexOp && layoutWidth && width != *layoutWidth) {
1080 const auto sub = dyn_cast<mlir::LLVM::SubOp>(indexOp);
1081 auto unary = ptrStrideOp.getStride().getDefiningOp<cir::UnaryOp>();
1083 unary && unary.getKind() == cir::UnaryOpKind::Minus && sub;
1085 index = indexOp->getOperand(1);
1088 const auto llvmDstType = mlir::IntegerType::get(ctx, *layoutWidth);
1090 ptrStrideOp.getStride().getType().isUnsigned(),
1091 width, *layoutWidth);
1095 index = rewriter.create<mlir::LLVM::SubOp>(
1096 index.getLoc(), index.getType(),
1097 rewriter.create<mlir::LLVM::ConstantOp>(index.getLoc(),
1098 index.getType(), 0),
1100 rewriter.eraseOp(sub);
1104 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
1105 ptrStrideOp, resultTy, elementTy, adaptor.getBase(), index);
1106 return mlir::success();
1109mlir::LogicalResult CIRToLLVMBaseClassAddrOpLowering::matchAndRewrite(
1110 cir::BaseClassAddrOp baseClassOp, OpAdaptor adaptor,
1111 mlir::ConversionPatternRewriter &rewriter)
const {
1112 const mlir::Type resultType =
1113 getTypeConverter()->convertType(baseClassOp.getType());
1114 mlir::Value derivedAddr = adaptor.getDerivedAddr();
1115 llvm::SmallVector<mlir::LLVM::GEPArg, 1> offset = {
1116 adaptor.getOffset().getZExtValue()};
1117 mlir::Type byteType = mlir::IntegerType::get(resultType.getContext(), 8,
1118 mlir::IntegerType::Signless);
1119 if (adaptor.getOffset().getZExtValue() == 0) {
1120 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(
1121 baseClassOp, resultType, adaptor.getDerivedAddr());
1122 return mlir::success();
1125 if (baseClassOp.getAssumeNotNull()) {
1126 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
1127 baseClassOp, resultType, byteType, derivedAddr, offset);
1129 auto loc = baseClassOp.getLoc();
1130 mlir::Value isNull = rewriter.create<mlir::LLVM::ICmpOp>(
1131 loc, mlir::LLVM::ICmpPredicate::eq, derivedAddr,
1132 rewriter.create<mlir::LLVM::ZeroOp>(loc, derivedAddr.getType()));
1133 mlir::Value adjusted = rewriter.create<mlir::LLVM::GEPOp>(
1134 loc, resultType, byteType, derivedAddr, offset);
1135 rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(baseClassOp, isNull,
1136 derivedAddr, adjusted);
1138 return mlir::success();
1141mlir::LogicalResult CIRToLLVMATanOpLowering::matchAndRewrite(
1142 cir::ATanOp op, OpAdaptor adaptor,
1143 mlir::ConversionPatternRewriter &rewriter)
const {
1144 mlir::Type resTy = typeConverter->convertType(op.getType());
1145 rewriter.replaceOpWithNewOp<mlir::LLVM::ATanOp>(op, resTy, adaptor.getSrc());
1146 return mlir::success();
1149mlir::LogicalResult CIRToLLVMAllocaOpLowering::matchAndRewrite(
1150 cir::AllocaOp op, OpAdaptor adaptor,
1151 mlir::ConversionPatternRewriter &rewriter)
const {
1154 ? adaptor.getDynAllocSize()
1155 : rewriter.create<mlir::LLVM::ConstantOp>(
1157 typeConverter->convertType(rewriter.getIndexType()), 1);
1158 mlir::Type elementTy =
1160 mlir::Type resultTy =
1166 rewriter.replaceOpWithNewOp<mlir::LLVM::AllocaOp>(
1167 op, resultTy, elementTy, size, op.getAlignmentAttr().getInt());
1169 return mlir::success();
1172mlir::LogicalResult CIRToLLVMReturnOpLowering::matchAndRewrite(
1173 cir::ReturnOp op, OpAdaptor adaptor,
1174 mlir::ConversionPatternRewriter &rewriter)
const {
1175 rewriter.replaceOpWithNewOp<mlir::LLVM::ReturnOp>(op, adaptor.getOperands());
1176 return mlir::LogicalResult::success();
1179mlir::LogicalResult CIRToLLVMRotateOpLowering::matchAndRewrite(
1180 cir::RotateOp op, OpAdaptor adaptor,
1181 mlir::ConversionPatternRewriter &rewriter)
const {
1184 mlir::Value input = adaptor.getInput();
1185 if (op.isRotateLeft())
1186 rewriter.replaceOpWithNewOp<mlir::LLVM::FshlOp>(op, input, input,
1187 adaptor.getAmount());
1189 rewriter.replaceOpWithNewOp<mlir::LLVM::FshrOp>(op, input, input,
1190 adaptor.getAmount());
1191 return mlir::LogicalResult::success();
1194static mlir::LogicalResult
1196 mlir::ConversionPatternRewriter &rewriter,
1197 const mlir::TypeConverter *converter,
1198 mlir::FlatSymbolRefAttr calleeAttr) {
1200 mlir::ValueTypeRange<mlir::ResultRange> cirResults = op->getResultTypes();
1201 auto call = cast<cir::CIRCallOpInterface>(op);
1203 if (converter->convertTypes(cirResults, llvmResults).failed())
1204 return mlir::failure();
1208 mlir::LLVM::MemoryEffectsAttr memoryEffects;
1209 bool noUnwind =
false;
1210 bool willReturn =
false;
1212 memoryEffects, noUnwind, willReturn);
1214 mlir::LLVM::LLVMFunctionType llvmFnTy;
1221 mlir::Operation *callee =
1222 mlir::SymbolTable::lookupNearestSymbolFrom(op, calleeAttr);
1223 if (
auto fn = mlir::dyn_cast<mlir::FunctionOpInterface>(callee)) {
1224 llvmFnTy = converter->convertType<mlir::LLVM::LLVMFunctionType>(
1225 fn.getFunctionType());
1226 assert(llvmFnTy &&
"Failed to convert function type");
1227 }
else if (
auto alias = mlir::cast<mlir::LLVM::AliasOp>(callee)) {
1235 auto symAttr = mlir::cast<mlir::FlatSymbolRefAttr>(calleeAttr);
1237 mlir::LLVM::AddressOfOp::create(
1238 rewriter, op->getLoc(),
1239 mlir::LLVM::LLVMPointerType::get(rewriter.getContext()), symAttr)
1241 adjustedCallOperands.push_back(addrOfAlias);
1244 llvm::append_range(adjustedCallOperands, callOperands);
1245 callOperands = adjustedCallOperands;
1249 llvmFnTy = mlir::cast<mlir::LLVM::LLVMFunctionType>(alias.getType());
1252 return op->emitError(
"Unexpected callee type!");
1255 assert(!op->getOperands().empty() &&
1256 "operands list must no be empty for the indirect call");
1257 auto calleeTy = op->getOperands().front().getType();
1258 auto calleePtrTy = cast<cir::PointerType>(calleeTy);
1259 auto calleeFuncTy = cast<cir::FuncType>(calleePtrTy.getPointee());
1260 llvm::append_range(adjustedCallOperands, callOperands);
1261 llvmFnTy = cast<mlir::LLVM::LLVMFunctionType>(
1262 converter->convertType(calleeFuncTy));
1269 auto newOp = rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
1270 op, llvmFnTy, calleeAttr, callOperands);
1272 newOp.setMemoryEffectsAttr(memoryEffects);
1273 newOp.setNoUnwind(noUnwind);
1274 newOp.setWillReturn(willReturn);
1276 return mlir::success();
1279mlir::LogicalResult CIRToLLVMCallOpLowering::matchAndRewrite(
1280 cir::CallOp op, OpAdaptor adaptor,
1281 mlir::ConversionPatternRewriter &rewriter)
const {
1283 getTypeConverter(), op.getCalleeAttr());
1286mlir::LogicalResult CIRToLLVMReturnAddrOpLowering::matchAndRewrite(
1287 cir::ReturnAddrOp op, OpAdaptor adaptor,
1288 mlir::ConversionPatternRewriter &rewriter)
const {
1289 auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
1291 llvmPtrTy, adaptor.getOperands());
1292 return mlir::success();
1295mlir::LogicalResult CIRToLLVMFrameAddrOpLowering::matchAndRewrite(
1296 cir::FrameAddrOp op, OpAdaptor adaptor,
1297 mlir::ConversionPatternRewriter &rewriter)
const {
1298 auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
1300 adaptor.getOperands());
1301 return mlir::success();
1304mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
1305 cir::LoadOp op, OpAdaptor adaptor,
1306 mlir::ConversionPatternRewriter &rewriter)
const {
1307 const mlir::Type llvmTy =
1309 mlir::LLVM::AtomicOrdering ordering =
getLLVMMemOrder(op.getMemOrder());
1310 std::optional<size_t> opAlign = op.getAlignment();
1311 unsigned alignment =
1312 (unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmTy));
1318 mlir::LLVM::LoadOp newLoad = mlir::LLVM::LoadOp::create(
1319 rewriter, op->getLoc(), llvmTy, adaptor.getAddr(), alignment,
1320 op.getIsVolatile(),
false,
1321 false,
false, ordering);
1324 mlir::Value result =
1326 rewriter.replaceOp(op, result);
1328 return mlir::LogicalResult::success();
1331mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite(
1332 cir::StoreOp op, OpAdaptor adaptor,
1333 mlir::ConversionPatternRewriter &rewriter)
const {
1334 mlir::LLVM::AtomicOrdering memorder =
getLLVMMemOrder(op.getMemOrder());
1335 const mlir::Type llvmTy =
1336 getTypeConverter()->convertType(op.getValue().getType());
1337 std::optional<size_t> opAlign = op.getAlignment();
1338 unsigned alignment =
1339 (unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmTy));
1345 op.getValue().getType(), adaptor.getValue());
1349 mlir::LLVM::StoreOp storeOp = mlir::LLVM::StoreOp::create(
1350 rewriter, op->getLoc(), value, adaptor.getAddr(), alignment,
1352 false,
false, memorder);
1353 rewriter.replaceOp(op, storeOp);
1355 return mlir::LogicalResult::success();
1359 auto array = mlir::dyn_cast<mlir::ArrayAttr>(attr.getElts());
1360 return attr.hasTrailingZeros() ||
1361 (array && std::count_if(array.begin(), array.end(), [](
auto elt) {
1362 auto ar = dyn_cast<cir::ConstArrayAttr>(elt);
1363 return ar && hasTrailingZeros(ar);
1367mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
1368 cir::ConstantOp op, OpAdaptor adaptor,
1369 mlir::ConversionPatternRewriter &rewriter)
const {
1370 mlir::Attribute attr = op.getValue();
1372 if (mlir::isa<cir::PoisonAttr>(attr)) {
1373 rewriter.replaceOpWithNewOp<mlir::LLVM::PoisonOp>(
1374 op, getTypeConverter()->convertType(op.getType()));
1375 return mlir::success();
1378 if (mlir::isa<mlir::IntegerType>(op.getType())) {
1383 attr = op.getValue();
1384 }
else if (mlir::isa<cir::BoolType>(op.getType())) {
1385 int value = mlir::cast<cir::BoolAttr>(op.getValue()).getValue();
1386 attr = rewriter.getIntegerAttr(typeConverter->convertType(op.getType()),
1388 }
else if (mlir::isa<cir::IntType>(op.getType())) {
1390 if (
auto ga = mlir::dyn_cast<cir::GlobalViewAttr>(op.getValue())) {
1393 op.emitError() <<
"global view with integer type";
1394 return mlir::failure();
1397 attr = rewriter.getIntegerAttr(
1398 typeConverter->convertType(op.getType()),
1399 mlir::cast<cir::IntAttr>(op.getValue()).getValue());
1400 }
else if (mlir::isa<cir::FPTypeInterface>(op.getType())) {
1401 attr = rewriter.getFloatAttr(
1402 typeConverter->convertType(op.getType()),
1403 mlir::cast<cir::FPAttr>(op.getValue()).getValue());
1404 }
else if (mlir::isa<cir::PointerType>(op.getType())) {
1406 if (mlir::isa<cir::ConstPtrAttr>(op.getValue())) {
1407 if (mlir::cast<cir::ConstPtrAttr>(op.getValue()).isNullValue()) {
1408 rewriter.replaceOpWithNewOp<mlir::LLVM::ZeroOp>(
1409 op, typeConverter->convertType(op.getType()));
1410 return mlir::success();
1414 if (
auto gv = mlir::dyn_cast<cir::GlobalViewAttr>(op.getValue())) {
1416 rewriter.replaceOp(op, newOp);
1417 return mlir::success();
1419 attr = op.getValue();
1420 }
else if (
const auto arrTy = mlir::dyn_cast<cir::ArrayType>(op.getType())) {
1421 const auto constArr = mlir::dyn_cast<cir::ConstArrayAttr>(op.getValue());
1422 if (!constArr && !isa<cir::ZeroAttr, cir::UndefAttr>(op.getValue()))
1423 return op.emitError() <<
"array does not have a constant initializer";
1425 std::optional<mlir::Attribute> denseAttr;
1427 const mlir::Value newOp =
1429 rewriter.replaceOp(op, newOp);
1430 return mlir::success();
1431 }
else if (constArr &&
1433 attr = denseAttr.value();
1435 const mlir::Value initVal =
1437 rewriter.replaceOp(op, initVal);
1438 return mlir::success();
1440 }
else if (
const auto recordAttr =
1441 mlir::dyn_cast<cir::ConstRecordAttr>(op.getValue())) {
1443 rewriter.replaceOp(op, initVal);
1444 return mlir::success();
1445 }
else if (
const auto vecTy = mlir::dyn_cast<cir::VectorType>(op.getType())) {
1447 getTypeConverter()));
1448 return mlir::success();
1449 }
else if (
auto recTy = mlir::dyn_cast<cir::RecordType>(op.getType())) {
1450 if (mlir::isa<cir::ZeroAttr, cir::UndefAttr>(attr)) {
1451 mlir::Value initVal =
1453 rewriter.replaceOp(op, initVal);
1454 return mlir::success();
1456 return op.emitError() <<
"unsupported lowering for record constant type "
1458 }
else if (
auto complexTy = mlir::dyn_cast<cir::ComplexType>(op.getType())) {
1459 mlir::Type complexElemTy = complexTy.getElementType();
1460 mlir::Type complexElemLLVMTy = typeConverter->convertType(complexElemTy);
1462 if (
auto zeroInitAttr = mlir::dyn_cast<cir::ZeroAttr>(op.getValue())) {
1463 mlir::TypedAttr zeroAttr = rewriter.getZeroAttr(complexElemLLVMTy);
1464 mlir::ArrayAttr array = rewriter.getArrayAttr({zeroAttr, zeroAttr});
1465 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
1466 op, getTypeConverter()->convertType(op.getType()), array);
1467 return mlir::success();
1470 auto complexAttr = mlir::cast<cir::ConstComplexAttr>(op.getValue());
1472 mlir::Attribute components[2];
1473 if (mlir::isa<cir::IntType>(complexElemTy)) {
1474 components[0] = rewriter.getIntegerAttr(
1476 mlir::cast<cir::IntAttr>(complexAttr.getReal()).getValue());
1477 components[1] = rewriter.getIntegerAttr(
1479 mlir::cast<cir::IntAttr>(complexAttr.getImag()).getValue());
1481 components[0] = rewriter.getFloatAttr(
1483 mlir::cast<cir::FPAttr>(complexAttr.getReal()).getValue());
1484 components[1] = rewriter.getFloatAttr(
1486 mlir::cast<cir::FPAttr>(complexAttr.getImag()).getValue());
1489 attr = rewriter.getArrayAttr(components);
1491 return op.emitError() <<
"unsupported constant type " << op.getType();
1494 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
1495 op, getTypeConverter()->convertType(op.getType()),
attr);
1497 return mlir::success();
1500mlir::LogicalResult CIRToLLVMExpectOpLowering::matchAndRewrite(
1501 cir::ExpectOp op, OpAdaptor adaptor,
1502 mlir::ConversionPatternRewriter &rewriter)
const {
1506 std::optional<llvm::APFloat> prob = op.getProb();
1508 rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectWithProbabilityOp>(
1509 op, adaptor.getVal(), adaptor.getExpected(), prob.value());
1511 rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectOp>(op, adaptor.getVal(),
1512 adaptor.getExpected());
1513 return mlir::success();
1516mlir::LogicalResult CIRToLLVMFAbsOpLowering::matchAndRewrite(
1517 cir::FAbsOp op, OpAdaptor adaptor,
1518 mlir::ConversionPatternRewriter &rewriter)
const {
1519 mlir::Type resTy = typeConverter->convertType(op.getType());
1520 rewriter.replaceOpWithNewOp<mlir::LLVM::FAbsOp>(op, resTy,
1521 adaptor.getOperands()[0]);
1522 return mlir::success();
1529void CIRToLLVMFuncOpLowering::lowerFuncAttributes(
1530 cir::FuncOp func,
bool filterArgAndResAttrs,
1531 SmallVectorImpl<mlir::NamedAttribute> &result)
const {
1533 for (mlir::NamedAttribute attr : func->getAttrs()) {
1535 if (
attr.getName() == mlir::SymbolTable::getSymbolAttrName() ||
1536 attr.getName() == func.getFunctionTypeAttrName() ||
1538 attr.getName() == func.getGlobalVisibilityAttrName() ||
1539 attr.getName() == func.getDsoLocalAttrName() ||
1540 (filterArgAndResAttrs &&
1541 (
attr.getName() == func.getArgAttrsAttrName() ||
1542 attr.getName() == func.getResAttrsAttrName())))
1546 result.push_back(attr);
1550mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewriteAlias(
1551 cir::FuncOp op, llvm::StringRef aliasee, mlir::Type ty, OpAdaptor adaptor,
1552 mlir::ConversionPatternRewriter &rewriter)
const {
1553 SmallVector<mlir::NamedAttribute, 4> attributes;
1554 lowerFuncAttributes(op,
false, attributes);
1556 mlir::Location loc = op.getLoc();
1557 auto aliasOp = rewriter.replaceOpWithNewOp<mlir::LLVM::AliasOp>(
1558 op, ty,
convertLinkage(op.getLinkage()), op.getName(), op.getDsoLocal(),
1562 mlir::OpBuilder builder(op.getContext());
1563 mlir::Block *block = builder.createBlock(&aliasOp.getInitializerRegion());
1564 builder.setInsertionPointToStart(block);
1567 mlir::Type ptrTy = mlir::LLVM::LLVMPointerType::get(ty.getContext());
1568 auto addrOp = mlir::LLVM::AddressOfOp::create(builder, loc, ptrTy, aliasee);
1569 mlir::LLVM::ReturnOp::create(builder, loc, addrOp);
1571 return mlir::success();
1574mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewrite(
1575 cir::FuncOp op, OpAdaptor adaptor,
1576 mlir::ConversionPatternRewriter &rewriter)
const {
1578 cir::FuncType fnType = op.getFunctionType();
1579 bool isDsoLocal = op.getDsoLocal();
1580 mlir::TypeConverter::SignatureConversion signatureConversion(
1581 fnType.getNumInputs());
1583 for (
const auto &argType : llvm::enumerate(fnType.getInputs())) {
1584 mlir::Type convertedType = typeConverter->convertType(argType.value());
1586 return mlir::failure();
1587 signatureConversion.addInputs(argType.index(), convertedType);
1590 mlir::Type resultType =
1591 getTypeConverter()->convertType(fnType.getReturnType());
1594 mlir::Type llvmFnTy = mlir::LLVM::LLVMFunctionType::get(
1595 resultType ? resultType : mlir::LLVM::LLVMVoidType::get(getContext()),
1596 signatureConversion.getConvertedTypes(),
1600 if (std::optional<llvm::StringRef> aliasee = op.getAliasee())
1601 return matchAndRewriteAlias(op, *aliasee, llvmFnTy, adaptor, rewriter);
1605 mlir::Location loc = op.getLoc();
1606 if (mlir::FusedLoc fusedLoc = mlir::dyn_cast<mlir::FusedLoc>(loc))
1607 loc = fusedLoc.getLocations()[0];
1608 assert((mlir::isa<mlir::FileLineColLoc>(loc) ||
1609 mlir::isa<mlir::UnknownLoc>(loc)) &&
1610 "expected single location or unknown location here");
1614 mlir::LLVM::CConv cconv = mlir::LLVM::CConv::C;
1615 SmallVector<mlir::NamedAttribute, 4> attributes;
1616 lowerFuncAttributes(op,
false, attributes);
1618 mlir::LLVM::LLVMFuncOp fn = rewriter.create<mlir::LLVM::LLVMFuncOp>(
1619 loc, op.getName(), llvmFnTy, linkage, isDsoLocal, cconv,
1620 mlir::SymbolRefAttr(), attributes);
1624 fn.setVisibility_Attr(mlir::LLVM::VisibilityAttr::get(
1626 op.getGlobalVisibilityAttr().getValue())));
1628 rewriter.inlineRegionBefore(op.getBody(), fn.getBody(), fn.end());
1629 if (failed(rewriter.convertRegionTypes(&fn.getBody(), *typeConverter,
1630 &signatureConversion)))
1631 return mlir::failure();
1633 rewriter.eraseOp(op);
1635 return mlir::LogicalResult::success();
1638mlir::LogicalResult CIRToLLVMGetGlobalOpLowering::matchAndRewrite(
1639 cir::GetGlobalOp op, OpAdaptor adaptor,
1640 mlir::ConversionPatternRewriter &rewriter)
const {
1643 if (op->getUses().empty()) {
1644 rewriter.eraseOp(op);
1645 return mlir::success();
1648 mlir::Type
type = getTypeConverter()->convertType(op.getType());
1649 mlir::Operation *newop =
1650 rewriter.create<mlir::LLVM::AddressOfOp>(op.getLoc(), type, op.getName());
1654 rewriter.replaceOp(op, newop);
1655 return mlir::success();
1660void CIRToLLVMGlobalOpLowering::setupRegionInitializedLLVMGlobalOp(
1661 cir::GlobalOp op, mlir::ConversionPatternRewriter &rewriter)
const {
1662 const mlir::Type llvmType =
1670 const bool isConst = op.getConstant();
1672 const unsigned addrSpace = 0;
1673 const bool isDsoLocal = op.getDsoLocal();
1675 const bool isThreadLocal =
false;
1676 const uint64_t alignment = op.getAlignment().value_or(0);
1677 const mlir::LLVM::Linkage linkage =
convertLinkage(op.getLinkage());
1678 const StringRef symbol = op.getSymName();
1679 mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter);
1681 SmallVector<mlir::NamedAttribute> attributes;
1682 mlir::LLVM::GlobalOp newGlobalOp =
1683 rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(
1684 op, llvmType, isConst, linkage, symbol,
nullptr, alignment, addrSpace,
1685 isDsoLocal, isThreadLocal, comdatAttr, attributes);
1686 newGlobalOp.getRegion().emplaceBlock();
1687 rewriter.setInsertionPointToEnd(newGlobalOp.getInitializerBlock());
1691CIRToLLVMGlobalOpLowering::matchAndRewriteRegionInitializedGlobal(
1692 cir::GlobalOp op, mlir::Attribute init,
1693 mlir::ConversionPatternRewriter &rewriter)
const {
1695 assert((isa<cir::ConstArrayAttr, cir::ConstRecordAttr, cir::ConstVectorAttr,
1696 cir::ConstPtrAttr, cir::ConstComplexAttr, cir::GlobalViewAttr,
1697 cir::VTableAttr, cir::ZeroAttr>(init)));
1702 const mlir::Location loc = op.getLoc();
1703 setupRegionInitializedLLVMGlobalOp(op, rewriter);
1705 mlir::Value value = valueConverter.visit(init);
1706 rewriter.create<mlir::LLVM::ReturnOp>(loc, value);
1707 return mlir::success();
1710mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite(
1711 cir::GlobalOp op, OpAdaptor adaptor,
1712 mlir::ConversionPatternRewriter &rewriter)
const {
1714 std::optional<mlir::Attribute> init = op.getInitialValue();
1717 const mlir::Type cirSymType = op.getSymType();
1720 const mlir::Type llvmType =
1725 const bool isConst =
false;
1727 const unsigned addrSpace = 0;
1728 const bool isDsoLocal = op.getDsoLocal();
1730 const bool isThreadLocal =
false;
1731 const uint64_t alignment = op.getAlignment().value_or(0);
1732 const mlir::LLVM::Linkage linkage =
convertLinkage(op.getLinkage());
1733 const StringRef symbol = op.getSymName();
1734 SmallVector<mlir::NamedAttribute> attributes;
1735 mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter);
1737 if (init.has_value()) {
1738 if (mlir::isa<cir::FPAttr, cir::IntAttr, cir::BoolAttr>(init.value())) {
1740 init = initRewriter.visit(init.value());
1745 if (!init.value()) {
1746 op.emitError() <<
"unsupported initializer '" << init.value() <<
"'";
1747 return mlir::failure();
1749 }
else if (mlir::isa<cir::ConstArrayAttr, cir::ConstVectorAttr,
1750 cir::ConstRecordAttr, cir::ConstPtrAttr,
1751 cir::ConstComplexAttr, cir::GlobalViewAttr,
1752 cir::VTableAttr, cir::ZeroAttr>(init.value())) {
1756 return matchAndRewriteRegionInitializedGlobal(op, init.value(), rewriter);
1760 op.emitError() <<
"unsupported initializer '" << init.value() <<
"'";
1761 return mlir::failure();
1766 rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(
1767 op, llvmType, isConst, linkage, symbol, init.value_or(mlir::Attribute()),
1768 alignment, addrSpace, isDsoLocal, isThreadLocal, comdatAttr, attributes);
1769 return mlir::success();
1773CIRToLLVMGlobalOpLowering::getComdatAttr(cir::GlobalOp &op,
1774 mlir::OpBuilder &builder)
const {
1775 if (!op.getComdat())
1776 return mlir::SymbolRefAttr{};
1778 mlir::ModuleOp module = op->getParentOfType<mlir::ModuleOp>();
1779 mlir::OpBuilder::InsertionGuard guard(builder);
1780 StringRef comdatName(
"__llvm_comdat_globals");
1782 builder.setInsertionPointToStart(module.getBody());
1784 builder.create<mlir::LLVM::ComdatOp>(module.getLoc(), comdatName);
1787 builder.setInsertionPointToStart(&comdatOp.getBody().back());
1788 auto selectorOp = builder.create<mlir::LLVM::ComdatSelectorOp>(
1789 comdatOp.getLoc(), op.getSymName(), mlir::LLVM::comdat::Comdat::Any);
1790 return mlir::SymbolRefAttr::get(
1791 builder.getContext(), comdatName,
1792 mlir::FlatSymbolRefAttr::get(selectorOp.getSymNameAttr()));
1795mlir::LogicalResult CIRToLLVMSwitchFlatOpLowering::matchAndRewrite(
1796 cir::SwitchFlatOp op, OpAdaptor adaptor,
1797 mlir::ConversionPatternRewriter &rewriter)
const {
1799 llvm::SmallVector<mlir::APInt, 8> caseValues;
1800 for (mlir::Attribute val : op.getCaseValues()) {
1801 auto intAttr = cast<cir::IntAttr>(val);
1802 caseValues.push_back(intAttr.getValue());
1805 llvm::SmallVector<mlir::Block *, 8> caseDestinations;
1806 llvm::SmallVector<mlir::ValueRange, 8> caseOperands;
1808 for (mlir::Block *x : op.getCaseDestinations())
1809 caseDestinations.push_back(x);
1811 for (mlir::OperandRange x : op.getCaseOperands())
1812 caseOperands.push_back(x);
1815 rewriter.setInsertionPoint(op);
1816 rewriter.replaceOpWithNewOp<mlir::LLVM::SwitchOp>(
1817 op, adaptor.getCondition(), op.getDefaultDestination(),
1818 op.getDefaultOperands(), caseValues, caseDestinations, caseOperands);
1819 return mlir::success();
1822mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite(
1823 cir::UnaryOp op, OpAdaptor adaptor,
1824 mlir::ConversionPatternRewriter &rewriter)
const {
1825 assert(op.getType() == op.getInput().getType() &&
1826 "Unary operation's operand type and result type are different");
1827 mlir::Type
type = op.getType();
1828 mlir::Type elementType = elementTypeIfVector(type);
1829 bool isVector = mlir::isa<cir::VectorType>(type);
1830 mlir::Type llvmType = getTypeConverter()->convertType(type);
1831 mlir::Location loc = op.getLoc();
1834 if (mlir::isa<cir::IntType>(elementType)) {
1835 mlir::LLVM::IntegerOverflowFlags maybeNSW =
1836 op.getNoSignedWrap() ? mlir::LLVM::IntegerOverflowFlags::nsw
1837 : mlir::LLVM::IntegerOverflowFlags::none;
1838 switch (op.getKind()) {
1839 case cir::UnaryOpKind::Inc: {
1840 assert(!
isVector &&
"++ not allowed on vector types");
1841 auto one = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, 1);
1842 rewriter.replaceOpWithNewOp<mlir::LLVM::AddOp>(
1843 op, llvmType, adaptor.getInput(), one, maybeNSW);
1844 return mlir::success();
1846 case cir::UnaryOpKind::Dec: {
1847 assert(!
isVector &&
"-- not allowed on vector types");
1848 auto one = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, 1);
1849 rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>(op, adaptor.getInput(),
1851 return mlir::success();
1853 case cir::UnaryOpKind::Plus:
1854 rewriter.replaceOp(op, adaptor.getInput());
1855 return mlir::success();
1856 case cir::UnaryOpKind::Minus: {
1859 zero = rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmType);
1861 zero = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, 0);
1862 rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>(
1863 op, zero, adaptor.getInput(), maybeNSW);
1864 return mlir::success();
1866 case cir::UnaryOpKind::Not: {
1868 mlir::Value minusOne;
1871 mlir::dyn_cast<cir::VectorType>(type).getSize();
1872 std::vector<int32_t> values(numElements, -1);
1873 mlir::DenseIntElementsAttr denseVec = rewriter.getI32VectorAttr(values);
1875 rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, denseVec);
1877 minusOne = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, -1);
1879 rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getInput(),
1881 return mlir::success();
1884 llvm_unreachable(
"Unexpected unary op for int");
1888 if (mlir::isa<cir::FPTypeInterface>(elementType)) {
1889 switch (op.getKind()) {
1890 case cir::UnaryOpKind::Inc: {
1891 assert(!
isVector &&
"++ not allowed on vector types");
1892 mlir::LLVM::ConstantOp one = rewriter.create<mlir::LLVM::ConstantOp>(
1893 loc, llvmType, rewriter.getFloatAttr(llvmType, 1.0));
1894 rewriter.replaceOpWithNewOp<mlir::LLVM::FAddOp>(op, llvmType, one,
1895 adaptor.getInput());
1896 return mlir::success();
1898 case cir::UnaryOpKind::Dec: {
1899 assert(!
isVector &&
"-- not allowed on vector types");
1900 mlir::LLVM::ConstantOp minusOne = rewriter.create<mlir::LLVM::ConstantOp>(
1901 loc, llvmType, rewriter.getFloatAttr(llvmType, -1.0));
1902 rewriter.replaceOpWithNewOp<mlir::LLVM::FAddOp>(op, llvmType, minusOne,
1903 adaptor.getInput());
1904 return mlir::success();
1906 case cir::UnaryOpKind::Plus:
1907 rewriter.replaceOp(op, adaptor.getInput());
1908 return mlir::success();
1909 case cir::UnaryOpKind::Minus:
1910 rewriter.replaceOpWithNewOp<mlir::LLVM::FNegOp>(op, llvmType,
1911 adaptor.getInput());
1912 return mlir::success();
1913 case cir::UnaryOpKind::Not:
1914 return op.emitError() <<
"Unary not is invalid for floating-point types";
1916 llvm_unreachable(
"Unexpected unary op for float");
1921 if (mlir::isa<cir::BoolType>(elementType)) {
1922 switch (op.getKind()) {
1923 case cir::UnaryOpKind::Inc:
1924 case cir::UnaryOpKind::Dec:
1925 case cir::UnaryOpKind::Plus:
1926 case cir::UnaryOpKind::Minus:
1929 return op.emitError() <<
"Unsupported unary operation on boolean type";
1930 case cir::UnaryOpKind::Not: {
1931 assert(!
isVector &&
"NYI: op! on vector mask");
1932 auto one = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, 1);
1933 rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getInput(),
1935 return mlir::success();
1938 llvm_unreachable(
"Unexpected unary op for bool");
1943 if (mlir::isa<cir::PointerType>(elementType)) {
1944 switch (op.getKind()) {
1945 case cir::UnaryOpKind::Plus:
1946 rewriter.replaceOp(op, adaptor.getInput());
1947 return mlir::success();
1949 op.emitError() <<
"Unknown pointer unary operation during CIR lowering";
1950 return mlir::failure();
1954 return op.emitError() <<
"Unary operation has unsupported type: "
1958mlir::LLVM::IntegerOverflowFlags
1959CIRToLLVMBinOpLowering::getIntOverflowFlag(cir::BinOp op)
const {
1960 if (op.getNoUnsignedWrap())
1961 return mlir::LLVM::IntegerOverflowFlags::nuw;
1963 if (op.getNoSignedWrap())
1964 return mlir::LLVM::IntegerOverflowFlags::nsw;
1966 return mlir::LLVM::IntegerOverflowFlags::none;
1971 return mlir::isa<cir::IntType>(type)
1972 ? mlir::cast<cir::IntType>(type).isUnsigned()
1973 : mlir::cast<mlir::IntegerType>(type).isUnsigned();
1976mlir::LogicalResult CIRToLLVMBinOpLowering::matchAndRewrite(
1977 cir::BinOp op, OpAdaptor adaptor,
1978 mlir::ConversionPatternRewriter &rewriter)
const {
1979 if (adaptor.getLhs().getType() != adaptor.getRhs().getType())
1980 return op.emitError() <<
"inconsistent operands' types not supported yet";
1982 mlir::Type type = op.getRhs().getType();
1983 if (!mlir::isa<cir::IntType, cir::BoolType, cir::FPTypeInterface,
1984 mlir::IntegerType, cir::VectorType>(type))
1985 return op.emitError() <<
"operand type not supported yet";
1987 const mlir::Type llvmTy = getTypeConverter()->convertType(op.getType());
1988 const mlir::Type llvmEltTy = elementTypeIfVector(llvmTy);
1990 const mlir::Value rhs = adaptor.getRhs();
1991 const mlir::Value lhs = adaptor.getLhs();
1992 type = elementTypeIfVector(type);
1994 switch (op.getKind()) {
1995 case cir::BinOpKind::Add:
1996 if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {
1997 if (op.getSaturated()) {
1999 rewriter.replaceOpWithNewOp<mlir::LLVM::UAddSat>(op, lhs, rhs);
2002 rewriter.replaceOpWithNewOp<mlir::LLVM::SAddSat>(op, lhs, rhs);
2005 rewriter.replaceOpWithNewOp<mlir::LLVM::AddOp>(op, llvmTy, lhs, rhs,
2006 getIntOverflowFlag(op));
2008 rewriter.replaceOpWithNewOp<mlir::LLVM::FAddOp>(op, lhs, rhs);
2011 case cir::BinOpKind::Sub:
2012 if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {
2013 if (op.getSaturated()) {
2015 rewriter.replaceOpWithNewOp<mlir::LLVM::USubSat>(op, lhs, rhs);
2018 rewriter.replaceOpWithNewOp<mlir::LLVM::SSubSat>(op, lhs, rhs);
2021 rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>(op, llvmTy, lhs, rhs,
2022 getIntOverflowFlag(op));
2024 rewriter.replaceOpWithNewOp<mlir::LLVM::FSubOp>(op, lhs, rhs);
2027 case cir::BinOpKind::Mul:
2028 if (mlir::isa<mlir::IntegerType>(llvmEltTy))
2029 rewriter.replaceOpWithNewOp<mlir::LLVM::MulOp>(op, llvmTy, lhs, rhs,
2030 getIntOverflowFlag(op));
2032 rewriter.replaceOpWithNewOp<mlir::LLVM::FMulOp>(op, lhs, rhs);
2034 case cir::BinOpKind::Div:
2035 if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {
2038 rewriter.replaceOpWithNewOp<mlir::LLVM::UDivOp>(op, lhs, rhs);
2040 rewriter.replaceOpWithNewOp<mlir::LLVM::SDivOp>(op, lhs, rhs);
2042 rewriter.replaceOpWithNewOp<mlir::LLVM::FDivOp>(op, lhs, rhs);
2045 case cir::BinOpKind::Rem:
2046 if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {
2049 rewriter.replaceOpWithNewOp<mlir::LLVM::URemOp>(op, lhs, rhs);
2051 rewriter.replaceOpWithNewOp<mlir::LLVM::SRemOp>(op, lhs, rhs);
2053 rewriter.replaceOpWithNewOp<mlir::LLVM::FRemOp>(op, lhs, rhs);
2056 case cir::BinOpKind::And:
2057 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op, lhs, rhs);
2059 case cir::BinOpKind::Or:
2060 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(op, lhs, rhs);
2062 case cir::BinOpKind::Xor:
2063 rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, lhs, rhs);
2065 case cir::BinOpKind::Max:
2066 if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {
2069 rewriter.replaceOpWithNewOp<mlir::LLVM::UMaxOp>(op, llvmTy, lhs, rhs);
2071 rewriter.replaceOpWithNewOp<mlir::LLVM::SMaxOp>(op, llvmTy, lhs, rhs);
2075 return mlir::LogicalResult::success();
2079static mlir::LLVM::ICmpPredicate
2081 using CIR = cir::CmpOpKind;
2082 using LLVMICmp = mlir::LLVM::ICmpPredicate;
2085 return LLVMICmp::eq;
2087 return LLVMICmp::ne;
2089 return (isSigned ? LLVMICmp::slt : LLVMICmp::ult);
2091 return (isSigned ? LLVMICmp::sle : LLVMICmp::ule);
2093 return (isSigned ? LLVMICmp::sgt : LLVMICmp::ugt);
2095 return (isSigned ? LLVMICmp::sge : LLVMICmp::uge);
2097 llvm_unreachable(
"Unknown CmpOpKind");
2102static mlir::LLVM::FCmpPredicate
2104 using CIR = cir::CmpOpKind;
2105 using LLVMFCmp = mlir::LLVM::FCmpPredicate;
2108 return LLVMFCmp::oeq;
2110 return LLVMFCmp::une;
2112 return LLVMFCmp::olt;
2114 return LLVMFCmp::ole;
2116 return LLVMFCmp::ogt;
2118 return LLVMFCmp::oge;
2120 llvm_unreachable(
"Unknown CmpOpKind");
2123mlir::LogicalResult CIRToLLVMCmpOpLowering::matchAndRewrite(
2124 cir::CmpOp cmpOp, OpAdaptor adaptor,
2125 mlir::ConversionPatternRewriter &rewriter)
const {
2126 mlir::Type type = cmpOp.getLhs().getType();
2131 if (mlir::isa<cir::IntType, mlir::IntegerType>(type)) {
2132 bool isSigned = mlir::isa<cir::IntType>(type)
2133 ? mlir::cast<cir::IntType>(type).isSigned()
2134 : mlir::cast<mlir::IntegerType>(type).isSigned();
2135 mlir::LLVM::ICmpPredicate kind =
2137 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
2138 cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
2139 return mlir::success();
2142 if (
auto ptrTy = mlir::dyn_cast<cir::PointerType>(type)) {
2143 mlir::LLVM::ICmpPredicate kind =
2146 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
2147 cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
2148 return mlir::success();
2151 if (mlir::isa<cir::FPTypeInterface>(type)) {
2152 mlir::LLVM::FCmpPredicate
kind =
2154 rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(
2155 cmpOp,
kind, adaptor.getLhs(), adaptor.getRhs());
2156 return mlir::success();
2159 if (mlir::isa<cir::ComplexType>(type)) {
2160 mlir::Value lhs = adaptor.getLhs();
2161 mlir::Value rhs = adaptor.getRhs();
2162 mlir::Location loc = cmpOp.getLoc();
2164 auto complexType = mlir::cast<cir::ComplexType>(cmpOp.getLhs().getType());
2165 mlir::Type complexElemTy =
2166 getTypeConverter()->convertType(
complexType.getElementType());
2169 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 0);
2171 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 1);
2173 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 0);
2175 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 1);
2177 if (cmpOp.getKind() == cir::CmpOpKind::eq) {
2178 if (complexElemTy.isInteger()) {
2179 auto realCmp = rewriter.create<mlir::LLVM::ICmpOp>(
2180 loc, mlir::LLVM::ICmpPredicate::eq, lhsReal, rhsReal);
2181 auto imagCmp = rewriter.create<mlir::LLVM::ICmpOp>(
2182 loc, mlir::LLVM::ICmpPredicate::eq, lhsImag, rhsImag);
2183 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp);
2184 return mlir::success();
2187 auto realCmp = rewriter.create<mlir::LLVM::FCmpOp>(
2188 loc, mlir::LLVM::FCmpPredicate::oeq, lhsReal, rhsReal);
2189 auto imagCmp = rewriter.create<mlir::LLVM::FCmpOp>(
2190 loc, mlir::LLVM::FCmpPredicate::oeq, lhsImag, rhsImag);
2191 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp);
2192 return mlir::success();
2195 if (cmpOp.getKind() == cir::CmpOpKind::ne) {
2196 if (complexElemTy.isInteger()) {
2197 auto realCmp = rewriter.create<mlir::LLVM::ICmpOp>(
2198 loc, mlir::LLVM::ICmpPredicate::ne, lhsReal, rhsReal);
2199 auto imagCmp = rewriter.create<mlir::LLVM::ICmpOp>(
2200 loc, mlir::LLVM::ICmpPredicate::ne, lhsImag, rhsImag);
2201 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp);
2202 return mlir::success();
2205 auto realCmp = rewriter.create<mlir::LLVM::FCmpOp>(
2206 loc, mlir::LLVM::FCmpPredicate::une, lhsReal, rhsReal);
2207 auto imagCmp = rewriter.create<mlir::LLVM::FCmpOp>(
2208 loc, mlir::LLVM::FCmpPredicate::une, lhsImag, rhsImag);
2209 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp);
2210 return mlir::success();
2214 return cmpOp.emitError() <<
"unsupported type for CmpOp: " <<
type;
2217mlir::LogicalResult CIRToLLVMShiftOpLowering::matchAndRewrite(
2218 cir::ShiftOp op, OpAdaptor adaptor,
2219 mlir::ConversionPatternRewriter &rewriter)
const {
2220 assert((op.getValue().getType() == op.getType()) &&
2221 "inconsistent operands' types NYI");
2223 const mlir::Type llvmTy = getTypeConverter()->convertType(op.getType());
2224 mlir::Value amt = adaptor.getAmount();
2225 mlir::Value val = adaptor.getValue();
2227 auto cirAmtTy = mlir::dyn_cast<cir::IntType>(op.getAmount().getType());
2230 auto cirValTy = mlir::cast<cir::IntType>(op.getValue().getType());
2238 amt =
getLLVMIntCast(rewriter, amt, llvmTy,
true, cirAmtTy.getWidth(),
2239 cirValTy.getWidth());
2241 auto cirValVTy = mlir::cast<cir::VectorType>(op.getValue().getType());
2243 mlir::cast<cir::IntType>(cirValVTy.getElementType()).isUnsigned();
2247 if (op.getIsShiftleft()) {
2248 rewriter.replaceOpWithNewOp<mlir::LLVM::ShlOp>(op, llvmTy, val, amt);
2249 return mlir::success();
2253 rewriter.replaceOpWithNewOp<mlir::LLVM::LShrOp>(op, llvmTy, val, amt);
2255 rewriter.replaceOpWithNewOp<mlir::LLVM::AShrOp>(op, llvmTy, val, amt);
2256 return mlir::success();
2259mlir::LogicalResult CIRToLLVMSelectOpLowering::matchAndRewrite(
2260 cir::SelectOp op, OpAdaptor adaptor,
2261 mlir::ConversionPatternRewriter &rewriter)
const {
2262 auto getConstantBool = [](mlir::Value value) -> cir::BoolAttr {
2263 auto definingOp = value.getDefiningOp<cir::ConstantOp>();
2267 auto constValue = definingOp.getValueAttr<cir::BoolAttr>();
2277 if (mlir::isa<cir::BoolType>(op.getTrueValue().getType())) {
2278 cir::BoolAttr trueValue = getConstantBool(op.getTrueValue());
2279 cir::BoolAttr falseValue = getConstantBool(op.getFalseValue());
2280 if (falseValue && !falseValue.getValue()) {
2282 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op, adaptor.getCondition(),
2283 adaptor.getTrueValue());
2284 return mlir::success();
2286 if (trueValue && trueValue.getValue()) {
2288 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(op, adaptor.getCondition(),
2289 adaptor.getFalseValue());
2290 return mlir::success();
2294 mlir::Value llvmCondition = adaptor.getCondition();
2295 rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(
2296 op, llvmCondition, adaptor.getTrueValue(), adaptor.getFalseValue());
2298 return mlir::success();
2302 mlir::DataLayout &dataLayout) {
2303 converter.addConversion([&](cir::PointerType type) -> mlir::Type {
2306 unsigned targetAS = 0;
2308 return mlir::LLVM::LLVMPointerType::get(type.getContext(), targetAS);
2310 converter.addConversion([&](cir::VPtrType type) -> mlir::Type {
2312 return mlir::LLVM::LLVMPointerType::get(type.getContext());
2314 converter.addConversion([&](cir::ArrayType type) -> mlir::Type {
2317 return mlir::LLVM::LLVMArrayType::get(ty, type.getSize());
2319 converter.addConversion([&](cir::VectorType type) -> mlir::Type {
2320 const mlir::Type ty = converter.convertType(type.getElementType());
2321 return mlir::VectorType::get(type.getSize(), ty);
2323 converter.addConversion([&](cir::BoolType type) -> mlir::Type {
2324 return mlir::IntegerType::get(type.getContext(), 1,
2325 mlir::IntegerType::Signless);
2327 converter.addConversion([&](cir::IntType type) -> mlir::Type {
2329 return mlir::IntegerType::get(type.getContext(), type.getWidth());
2331 converter.addConversion([&](cir::SingleType type) -> mlir::Type {
2332 return mlir::Float32Type::get(type.getContext());
2334 converter.addConversion([&](cir::DoubleType type) -> mlir::Type {
2335 return mlir::Float64Type::get(type.getContext());
2337 converter.addConversion([&](cir::FP80Type type) -> mlir::Type {
2338 return mlir::Float80Type::get(type.getContext());
2340 converter.addConversion([&](cir::FP128Type type) -> mlir::Type {
2341 return mlir::Float128Type::get(type.getContext());
2343 converter.addConversion([&](cir::LongDoubleType type) -> mlir::Type {
2344 return converter.convertType(type.getUnderlying());
2346 converter.addConversion([&](cir::FP16Type type) -> mlir::Type {
2347 return mlir::Float16Type::get(type.getContext());
2349 converter.addConversion([&](cir::BF16Type type) -> mlir::Type {
2350 return mlir::BFloat16Type::get(type.getContext());
2352 converter.addConversion([&](cir::ComplexType type) -> mlir::Type {
2355 mlir::Type elementTy = converter.convertType(type.getElementType());
2356 mlir::Type structFields[2] = {elementTy, elementTy};
2357 return mlir::LLVM::LLVMStructType::getLiteral(type.getContext(),
2360 converter.addConversion([&](cir::FuncType type) -> std::optional<mlir::Type> {
2361 auto result = converter.convertType(type.getReturnType());
2363 arguments.reserve(type.getNumInputs());
2364 if (converter.convertTypes(type.getInputs(), arguments).failed())
2365 return std::nullopt;
2366 auto varArg = type.isVarArg();
2367 return mlir::LLVM::LLVMFunctionType::get(result, arguments, varArg);
2369 converter.addConversion([&](cir::RecordType type) -> mlir::Type {
2372 switch (type.getKind()) {
2373 case cir::RecordType::Class:
2374 case cir::RecordType::Struct:
2375 for (mlir::Type ty : type.getMembers())
2379 case cir::RecordType::Union:
2380 if (
auto largestMember = type.getLargestMember(dataLayout))
2381 llvmMembers.push_back(
2383 if (type.getPadded()) {
2384 auto last = *type.getMembers().rbegin();
2385 llvmMembers.push_back(
2392 mlir::LLVM::LLVMStructType llvmStruct;
2393 if (type.getName()) {
2394 llvmStruct = mlir::LLVM::LLVMStructType::getIdentified(
2395 type.getContext(), type.getPrefixedName());
2396 if (llvmStruct.setBody(llvmMembers, type.getPacked()).failed())
2397 llvm_unreachable(
"Failed to set body of record");
2399 llvmStruct = mlir::LLVM::LLVMStructType::getLiteral(
2400 type.getContext(), llvmMembers, type.getPacked());
2405 converter.addConversion([&](cir::VoidType type) -> mlir::Type {
2406 return mlir::LLVM::LLVMVoidType::get(type.getContext());
2445 parent->walk([&](mlir::Block *blk) {
2446 if (blk->hasNoPredecessors() && !blk->isEntryBlock())
2447 unreachableBlocks.push_back(blk);
2450 std::set<mlir::Block *> visited;
2451 for (mlir::Block *root : unreachableBlocks) {
2454 std::deque<mlir::Block *> workList;
2455 workList.push_back(root);
2457 while (!workList.empty()) {
2458 mlir::Block *blk = workList.back();
2459 workList.pop_back();
2460 if (visited.count(blk))
2462 visited.emplace(blk);
2464 for (mlir::Operation &op : *blk)
2467 for (mlir::Block *succ : blk->getSuccessors())
2468 workList.push_back(succ);
2475 if (mlir::Attribute tripleAttr =
2476 module->getAttr(cir::CIRDialect::getTripleAttrName()))
2477 module->setAttr(mlir::LLVM::LLVMDialect::getTargetTripleAttrName(),
2480 if (mlir::Attribute asmAttr =
2481 module->getAttr(cir::CIRDialect::getModuleLevelAsmAttrName()))
2482 module->setAttr(mlir::LLVM::LLVMDialect::getModuleLevelAsmAttrName(),
2487 llvm::TimeTraceScope scope(
"Convert CIR to LLVM Pass");
2489 mlir::ModuleOp module = getOperation();
2490 mlir::DataLayout dl(module);
2491 mlir::LLVMTypeConverter converter(&getContext());
2494 mlir::RewritePatternSet patterns(&getContext());
2497#define GET_LLVM_LOWERING_PATTERNS_LIST
2498#include "clang/CIR/Dialect/IR/CIRLowering.inc"
2499#undef GET_LLVM_LOWERING_PATTERNS_LIST
2500 >(converter, patterns.getContext(), dl);
2504 mlir::ConversionTarget target(getContext());
2505 target.addLegalOp<mlir::ModuleOp>();
2506 target.addLegalDialect<mlir::LLVM::LLVMDialect>();
2507 target.addIllegalDialect<mlir::BuiltinDialect, cir::CIRDialect,
2508 mlir::func::FuncDialect>();
2511 ops.push_back(module);
2514 if (failed(applyPartialConversion(ops, target, std::move(patterns))))
2515 signalPassFailure();
2518mlir::LogicalResult CIRToLLVMBrOpLowering::matchAndRewrite(
2519 cir::BrOp op, OpAdaptor adaptor,
2520 mlir::ConversionPatternRewriter &rewriter)
const {
2521 rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(op, adaptor.getOperands(),
2523 return mlir::LogicalResult::success();
2526mlir::LogicalResult CIRToLLVMGetMemberOpLowering::matchAndRewrite(
2527 cir::GetMemberOp op, OpAdaptor adaptor,
2528 mlir::ConversionPatternRewriter &rewriter)
const {
2529 mlir::Type llResTy = getTypeConverter()->convertType(op.getType());
2530 const auto recordTy =
2531 mlir::cast<cir::RecordType>(op.getAddrTy().getPointee());
2532 assert(recordTy &&
"expected record type");
2534 switch (recordTy.getKind()) {
2535 case cir::RecordType::Class:
2536 case cir::RecordType::Struct: {
2540 const mlir::Type elementTy = getTypeConverter()->convertType(recordTy);
2541 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, llResTy, elementTy,
2542 adaptor.getAddr(), offset);
2543 return mlir::success();
2545 case cir::RecordType::Union:
2548 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(op, llResTy,
2550 return mlir::success();
2554mlir::LogicalResult CIRToLLVMUnreachableOpLowering::matchAndRewrite(
2555 cir::UnreachableOp op, OpAdaptor adaptor,
2556 mlir::ConversionPatternRewriter &rewriter)
const {
2557 rewriter.replaceOpWithNewOp<mlir::LLVM::UnreachableOp>(op);
2558 return mlir::success();
2562 mlir::Operation *srcOp, llvm::StringRef fnName,
2564 auto modOp = srcOp->getParentOfType<mlir::ModuleOp>();
2565 auto enclosingFnOp = srcOp->getParentOfType<mlir::LLVM::LLVMFuncOp>();
2566 mlir::Operation *sourceSymbol =
2567 mlir::SymbolTable::lookupSymbolIn(modOp, fnName);
2568 if (!sourceSymbol) {
2569 mlir::OpBuilder::InsertionGuard guard(rewriter);
2570 rewriter.setInsertionPoint(enclosingFnOp);
2571 rewriter.create<mlir::LLVM::LLVMFuncOp>(srcOp->getLoc(), fnName, fnTy);
2575mlir::LogicalResult CIRToLLVMThrowOpLowering::matchAndRewrite(
2576 cir::ThrowOp op, OpAdaptor adaptor,
2577 mlir::ConversionPatternRewriter &rewriter)
const {
2578 if (op.rethrows()) {
2579 auto voidTy = mlir::LLVM::LLVMVoidType::get(getContext());
2581 mlir::LLVM::LLVMFunctionType::get(getContext(), voidTy, {},
false);
2583 auto mlirModule = op->getParentOfType<mlir::ModuleOp>();
2584 rewriter.setInsertionPointToStart(&mlirModule.getBodyRegion().front());
2586 const llvm::StringRef functionName =
"__cxa_rethrow";
2589 rewriter.setInsertionPointAfter(op.getOperation());
2590 rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
2591 op, mlir::TypeRange{}, functionName, mlir::ValueRange{});
2594 return mlir::success();
2597mlir::LogicalResult CIRToLLVMTrapOpLowering::matchAndRewrite(
2598 cir::TrapOp op, OpAdaptor adaptor,
2599 mlir::ConversionPatternRewriter &rewriter)
const {
2600 mlir::Location loc = op->getLoc();
2601 rewriter.eraseOp(op);
2603 rewriter.create<mlir::LLVM::Trap>(loc);
2608 rewriter.create<mlir::LLVM::UnreachableOp>(loc);
2610 return mlir::success();
2615 mlir::ConversionPatternRewriter &rewriter,
2616 const mlir::TypeConverter *converter,
2617 mlir::FlatSymbolRefAttr nameAttr, mlir::Type &eltType) {
2618 auto module = op->getParentOfType<mlir::ModuleOp>();
2619 mlir::Operation *symbol = mlir::SymbolTable::lookupSymbolIn(module, nameAttr);
2620 if (
auto llvmSymbol = mlir::dyn_cast<mlir::LLVM::GlobalOp>(symbol)) {
2621 eltType = llvmSymbol.getType();
2622 }
else if (
auto cirSymbol = mlir::dyn_cast<cir::GlobalOp>(symbol)) {
2623 eltType = converter->convertType(cirSymbol.getSymType());
2625 op->emitError() <<
"unexpected symbol type for " << symbol;
2629 return mlir::LLVM::AddressOfOp::create(
2630 rewriter, op->getLoc(),
2631 mlir::LLVM::LLVMPointerType::get(op->getContext()), nameAttr.getValue());
2634mlir::LogicalResult CIRToLLVMVTableAddrPointOpLowering::matchAndRewrite(
2635 cir::VTableAddrPointOp op, OpAdaptor adaptor,
2636 mlir::ConversionPatternRewriter &rewriter)
const {
2637 const mlir::TypeConverter *converter = getTypeConverter();
2638 mlir::Type targetType = converter->convertType(op.getType());
2642 op.getNameAttr(), eltType);
2644 return op.emitError() <<
"Unable to get value for vtable symbol";
2647 0, op.getAddressPointAttr().getIndex(),
2648 op.getAddressPointAttr().getOffset()};
2650 assert(eltType &&
"Shouldn't ever be missing an eltType here");
2651 mlir::LLVM::GEPNoWrapFlags inboundsNuw =
2652 mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
2653 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, targetType, eltType,
2654 symAddr, offsets, inboundsNuw);
2655 return mlir::success();
2658mlir::LogicalResult CIRToLLVMVTableGetVPtrOpLowering::matchAndRewrite(
2659 cir::VTableGetVPtrOp op, OpAdaptor adaptor,
2660 mlir::ConversionPatternRewriter &rewriter)
const {
2664 mlir::Value srcVal = adaptor.getSrc();
2665 rewriter.replaceOp(op, srcVal);
2666 return mlir::success();
2669mlir::LogicalResult CIRToLLVMVTableGetVirtualFnAddrOpLowering::matchAndRewrite(
2670 cir::VTableGetVirtualFnAddrOp op, OpAdaptor adaptor,
2671 mlir::ConversionPatternRewriter &rewriter)
const {
2672 mlir::Type targetType = getTypeConverter()->convertType(op.getType());
2673 auto eltType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
2674 llvm::SmallVector<mlir::LLVM::GEPArg> offsets =
2675 llvm::SmallVector<mlir::LLVM::GEPArg>{op.getIndex()};
2676 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
2677 op, targetType, eltType, adaptor.getVptr(), offsets,
2678 mlir::LLVM::GEPNoWrapFlags::inbounds);
2679 return mlir::success();
2682mlir::LogicalResult CIRToLLVMVTTAddrPointOpLowering::matchAndRewrite(
2683 cir::VTTAddrPointOp op, OpAdaptor adaptor,
2684 mlir::ConversionPatternRewriter &rewriter)
const {
2685 const mlir::Type resultType = getTypeConverter()->convertType(op.getType());
2686 llvm::SmallVector<mlir::LLVM::GEPArg> offsets;
2688 mlir::Value llvmAddr = adaptor.getSymAddr();
2690 if (op.getSymAddr()) {
2691 if (op.getOffset() == 0) {
2692 rewriter.replaceOp(op, {llvmAddr});
2693 return mlir::success();
2696 offsets.push_back(adaptor.getOffset());
2697 eltType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
2700 op.getNameAttr(), eltType);
2701 assert(eltType &&
"Shouldn't ever be missing an eltType here");
2702 offsets.push_back(0);
2703 offsets.push_back(adaptor.getOffset());
2705 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
2706 op, resultType, eltType, llvmAddr, offsets,
2707 mlir::LLVM::GEPNoWrapFlags::inbounds);
2708 return mlir::success();
2711mlir::LogicalResult CIRToLLVMStackSaveOpLowering::matchAndRewrite(
2712 cir::StackSaveOp op, OpAdaptor adaptor,
2713 mlir::ConversionPatternRewriter &rewriter)
const {
2714 const mlir::Type ptrTy = getTypeConverter()->convertType(op.getType());
2715 rewriter.replaceOpWithNewOp<mlir::LLVM::StackSaveOp>(op, ptrTy);
2716 return mlir::success();
2719mlir::LogicalResult CIRToLLVMStackRestoreOpLowering::matchAndRewrite(
2720 cir::StackRestoreOp op, OpAdaptor adaptor,
2721 mlir::ConversionPatternRewriter &rewriter)
const {
2722 rewriter.replaceOpWithNewOp<mlir::LLVM::StackRestoreOp>(op, adaptor.getPtr());
2723 return mlir::success();
2726mlir::LogicalResult CIRToLLVMVecCreateOpLowering::matchAndRewrite(
2727 cir::VecCreateOp op, OpAdaptor adaptor,
2728 mlir::ConversionPatternRewriter &rewriter)
const {
2731 const auto vecTy = mlir::cast<cir::VectorType>(op.getType());
2732 const mlir::Type llvmTy = typeConverter->convertType(vecTy);
2733 const mlir::Location loc = op.getLoc();
2734 mlir::Value result = rewriter.create<mlir::LLVM::PoisonOp>(loc, llvmTy);
2735 assert(vecTy.getSize() == op.getElements().size() &&
2736 "cir.vec.create op count doesn't match vector type elements count");
2738 for (uint64_t i = 0; i < vecTy.getSize(); ++i) {
2739 const mlir::Value indexValue =
2740 rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI64Type(), i);
2741 result = rewriter.create<mlir::LLVM::InsertElementOp>(
2742 loc, result, adaptor.getElements()[i], indexValue);
2745 rewriter.replaceOp(op, result);
2746 return mlir::success();
2749mlir::LogicalResult CIRToLLVMVecExtractOpLowering::matchAndRewrite(
2750 cir::VecExtractOp op, OpAdaptor adaptor,
2751 mlir::ConversionPatternRewriter &rewriter)
const {
2752 rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractElementOp>(
2753 op, adaptor.getVec(), adaptor.getIndex());
2754 return mlir::success();
2757mlir::LogicalResult CIRToLLVMVecInsertOpLowering::matchAndRewrite(
2758 cir::VecInsertOp op, OpAdaptor adaptor,
2759 mlir::ConversionPatternRewriter &rewriter)
const {
2760 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertElementOp>(
2761 op, adaptor.getVec(), adaptor.getValue(), adaptor.getIndex());
2762 return mlir::success();
2765mlir::LogicalResult CIRToLLVMVecCmpOpLowering::matchAndRewrite(
2766 cir::VecCmpOp op, OpAdaptor adaptor,
2767 mlir::ConversionPatternRewriter &rewriter)
const {
2768 mlir::Type elementType = elementTypeIfVector(op.getLhs().getType());
2769 mlir::Value bitResult;
2770 if (
auto intType = mlir::dyn_cast<cir::IntType>(elementType)) {
2771 bitResult = rewriter.create<mlir::LLVM::ICmpOp>(
2774 adaptor.getLhs(), adaptor.getRhs());
2775 }
else if (mlir::isa<cir::FPTypeInterface>(elementType)) {
2776 bitResult = rewriter.create<mlir::LLVM::FCmpOp>(
2778 adaptor.getLhs(), adaptor.getRhs());
2780 return op.emitError() <<
"unsupported type for VecCmpOp: " << elementType;
2785 rewriter.replaceOpWithNewOp<mlir::LLVM::SExtOp>(
2786 op, typeConverter->convertType(op.getType()), bitResult);
2787 return mlir::success();
2790mlir::LogicalResult CIRToLLVMVecSplatOpLowering::matchAndRewrite(
2791 cir::VecSplatOp op, OpAdaptor adaptor,
2792 mlir::ConversionPatternRewriter &rewriter)
const {
2798 cir::VectorType vecTy = op.getType();
2799 mlir::Type llvmTy = typeConverter->convertType(vecTy);
2800 mlir::Location loc = op.getLoc();
2801 mlir::Value poison = rewriter.create<mlir::LLVM::PoisonOp>(loc, llvmTy);
2803 mlir::Value elementValue = adaptor.getValue();
2804 if (elementValue.getDefiningOp<mlir::LLVM::PoisonOp>()) {
2807 rewriter.replaceOp(op, poison);
2808 return mlir::success();
2811 if (
auto constValue = elementValue.getDefiningOp<mlir::LLVM::ConstantOp>()) {
2812 if (
auto intAttr = dyn_cast<mlir::IntegerAttr>(constValue.getValue())) {
2813 mlir::DenseIntElementsAttr denseVec = mlir::DenseIntElementsAttr::get(
2814 mlir::cast<mlir::ShapedType>(llvmTy), intAttr.getValue());
2815 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
2816 op, denseVec.getType(), denseVec);
2817 return mlir::success();
2820 if (
auto fpAttr = dyn_cast<mlir::FloatAttr>(constValue.getValue())) {
2821 mlir::DenseFPElementsAttr denseVec = mlir::DenseFPElementsAttr::get(
2822 mlir::cast<mlir::ShapedType>(llvmTy), fpAttr.getValue());
2823 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
2824 op, denseVec.getType(), denseVec);
2825 return mlir::success();
2829 mlir::Value indexValue =
2830 rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI64Type(), 0);
2831 mlir::Value oneElement = rewriter.create<mlir::LLVM::InsertElementOp>(
2832 loc, poison, elementValue, indexValue);
2833 SmallVector<int32_t> zeroValues(vecTy.getSize(), 0);
2834 rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(op, oneElement,
2835 poison, zeroValues);
2836 return mlir::success();
2839mlir::LogicalResult CIRToLLVMVecShuffleOpLowering::matchAndRewrite(
2840 cir::VecShuffleOp op, OpAdaptor adaptor,
2841 mlir::ConversionPatternRewriter &rewriter)
const {
2845 SmallVector<int, 8> indices;
2847 op.getIndices().begin(), op.getIndices().end(),
2848 std::back_inserter(indices), [](mlir::Attribute intAttr) {
2849 return mlir::cast<cir::IntAttr>(intAttr).getValue().getSExtValue();
2851 rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(
2852 op, adaptor.getVec1(), adaptor.getVec2(), indices);
2853 return mlir::success();
2856mlir::LogicalResult CIRToLLVMVecShuffleDynamicOpLowering::matchAndRewrite(
2857 cir::VecShuffleDynamicOp op, OpAdaptor adaptor,
2858 mlir::ConversionPatternRewriter &rewriter)
const {
2870 mlir::Location loc = op.getLoc();
2871 mlir::Value input = adaptor.getVec();
2872 mlir::Type llvmIndexVecType =
2873 getTypeConverter()->convertType(op.getIndices().getType());
2874 mlir::Type llvmIndexType = getTypeConverter()->convertType(
2875 elementTypeIfVector(op.getIndices().getType()));
2877 mlir::cast<cir::VectorType>(op.getVec().getType()).getSize();
2879 uint64_t maskBits = llvm::NextPowerOf2(numElements - 1) - 1;
2880 mlir::Value maskValue = rewriter.create<mlir::LLVM::ConstantOp>(
2881 loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, maskBits));
2882 mlir::Value maskVector =
2883 rewriter.create<mlir::LLVM::UndefOp>(loc, llvmIndexVecType);
2885 for (uint64_t i = 0; i < numElements; ++i) {
2886 mlir::Value idxValue =
2887 rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI64Type(), i);
2888 maskVector = rewriter.create<mlir::LLVM::InsertElementOp>(
2889 loc, maskVector, maskValue, idxValue);
2892 mlir::Value maskedIndices = rewriter.create<mlir::LLVM::AndOp>(
2893 loc, llvmIndexVecType, adaptor.getIndices(), maskVector);
2894 mlir::Value result = rewriter.create<mlir::LLVM::UndefOp>(
2895 loc, getTypeConverter()->convertType(op.getVec().getType()));
2896 for (uint64_t i = 0; i < numElements; ++i) {
2897 mlir::Value iValue =
2898 rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI64Type(), i);
2899 mlir::Value indexValue = rewriter.create<mlir::LLVM::ExtractElementOp>(
2900 loc, maskedIndices, iValue);
2901 mlir::Value valueAtIndex =
2902 rewriter.create<mlir::LLVM::ExtractElementOp>(loc, input, indexValue);
2903 result = rewriter.create<mlir::LLVM::InsertElementOp>(loc, result,
2904 valueAtIndex, iValue);
2906 rewriter.replaceOp(op, result);
2907 return mlir::success();
2910mlir::LogicalResult CIRToLLVMVecTernaryOpLowering::matchAndRewrite(
2911 cir::VecTernaryOp op, OpAdaptor adaptor,
2912 mlir::ConversionPatternRewriter &rewriter)
const {
2914 mlir::Value bitVec = rewriter.create<mlir::LLVM::ICmpOp>(
2915 op.getLoc(), mlir::LLVM::ICmpPredicate::ne, adaptor.getCond(),
2916 rewriter.create<mlir::LLVM::ZeroOp>(
2917 op.getCond().getLoc(),
2918 typeConverter->convertType(op.getCond().getType())));
2919 rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(
2920 op, bitVec, adaptor.getLhs(), adaptor.getRhs());
2921 return mlir::success();
2924mlir::LogicalResult CIRToLLVMComplexAddOpLowering::matchAndRewrite(
2925 cir::ComplexAddOp op, OpAdaptor adaptor,
2926 mlir::ConversionPatternRewriter &rewriter)
const {
2927 mlir::Value lhs = adaptor.getLhs();
2928 mlir::Value rhs = adaptor.getRhs();
2929 mlir::Location loc = op.getLoc();
2931 auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType());
2932 mlir::Type complexElemTy =
2933 getTypeConverter()->convertType(
complexType.getElementType());
2935 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 0);
2937 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 1);
2939 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 0);
2941 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 1);
2943 mlir::Value newReal;
2944 mlir::Value newImag;
2945 if (complexElemTy.isInteger()) {
2946 newReal = rewriter.create<mlir::LLVM::AddOp>(loc, complexElemTy, lhsReal,
2948 newImag = rewriter.create<mlir::LLVM::AddOp>(loc, complexElemTy, lhsImag,
2953 newReal = rewriter.create<mlir::LLVM::FAddOp>(loc, complexElemTy, lhsReal,
2955 newImag = rewriter.create<mlir::LLVM::FAddOp>(loc, complexElemTy, lhsImag,
2959 mlir::Type complexLLVMTy =
2960 getTypeConverter()->convertType(op.getResult().getType());
2961 auto initialComplex =
2962 rewriter.create<mlir::LLVM::PoisonOp>(op->getLoc(), complexLLVMTy);
2964 auto realComplex = rewriter.create<mlir::LLVM::InsertValueOp>(
2965 op->getLoc(), initialComplex, newReal, 0);
2967 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(op, realComplex,
2970 return mlir::success();
2973mlir::LogicalResult CIRToLLVMComplexCreateOpLowering::matchAndRewrite(
2974 cir::ComplexCreateOp op, OpAdaptor adaptor,
2975 mlir::ConversionPatternRewriter &rewriter)
const {
2976 mlir::Type complexLLVMTy =
2977 getTypeConverter()->convertType(op.getResult().getType());
2978 auto initialComplex =
2979 rewriter.create<mlir::LLVM::UndefOp>(op->getLoc(), complexLLVMTy);
2981 auto realComplex = rewriter.create<mlir::LLVM::InsertValueOp>(
2982 op->getLoc(), initialComplex, adaptor.getReal(), 0);
2984 auto complex = rewriter.create<mlir::LLVM::InsertValueOp>(
2985 op->getLoc(), realComplex, adaptor.getImag(), 1);
2987 rewriter.replaceOp(op, complex);
2988 return mlir::success();
2991mlir::LogicalResult CIRToLLVMComplexRealOpLowering::matchAndRewrite(
2992 cir::ComplexRealOp op, OpAdaptor adaptor,
2993 mlir::ConversionPatternRewriter &rewriter)
const {
2994 mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
2995 rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractValueOp>(
2996 op, resultLLVMTy, adaptor.getOperand(), llvm::ArrayRef<std::int64_t>{0});
2997 return mlir::success();
3000mlir::LogicalResult CIRToLLVMComplexSubOpLowering::matchAndRewrite(
3001 cir::ComplexSubOp op, OpAdaptor adaptor,
3002 mlir::ConversionPatternRewriter &rewriter)
const {
3003 mlir::Value lhs = adaptor.getLhs();
3004 mlir::Value rhs = adaptor.getRhs();
3005 mlir::Location loc = op.getLoc();
3007 auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType());
3008 mlir::Type complexElemTy =
3009 getTypeConverter()->convertType(
complexType.getElementType());
3011 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 0);
3013 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 1);
3015 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 0);
3017 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 1);
3019 mlir::Value newReal;
3020 mlir::Value newImag;
3021 if (complexElemTy.isInteger()) {
3022 newReal = rewriter.create<mlir::LLVM::SubOp>(loc, complexElemTy, lhsReal,
3024 newImag = rewriter.create<mlir::LLVM::SubOp>(loc, complexElemTy, lhsImag,
3029 newReal = rewriter.create<mlir::LLVM::FSubOp>(loc, complexElemTy, lhsReal,
3031 newImag = rewriter.create<mlir::LLVM::FSubOp>(loc, complexElemTy, lhsImag,
3035 mlir::Type complexLLVMTy =
3036 getTypeConverter()->convertType(op.getResult().getType());
3037 auto initialComplex =
3038 rewriter.create<mlir::LLVM::PoisonOp>(op->getLoc(), complexLLVMTy);
3040 auto realComplex = rewriter.create<mlir::LLVM::InsertValueOp>(
3041 op->getLoc(), initialComplex, newReal, 0);
3043 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(op, realComplex,
3046 return mlir::success();
3049mlir::LogicalResult CIRToLLVMComplexImagOpLowering::matchAndRewrite(
3050 cir::ComplexImagOp op, OpAdaptor adaptor,
3051 mlir::ConversionPatternRewriter &rewriter)
const {
3052 mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
3053 rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractValueOp>(
3054 op, resultLLVMTy, adaptor.getOperand(), llvm::ArrayRef<std::int64_t>{1});
3055 return mlir::success();
3059 mlir::MLIRContext *context,
3060 unsigned &storageSize) {
3061 return TypeSwitch<mlir::Type, mlir::IntegerType>(storageType)
3062 .Case<cir::ArrayType>([&](cir::ArrayType atTy) {
3063 storageSize = atTy.getSize() * 8;
3064 return mlir::IntegerType::get(context, storageSize);
3066 .Case<cir::IntType>([&](cir::IntType intTy) {
3067 storageSize = intTy.getWidth();
3068 return mlir::IntegerType::get(context, storageSize);
3070 .Default([](mlir::Type) -> mlir::IntegerType {
3072 "Either ArrayType or IntType expected for bitfields storage");
3076mlir::LogicalResult CIRToLLVMSetBitfieldOpLowering::matchAndRewrite(
3077 cir::SetBitfieldOp op, OpAdaptor adaptor,
3078 mlir::ConversionPatternRewriter &rewriter)
const {
3079 mlir::OpBuilder::InsertionGuard guard(rewriter);
3080 rewriter.setInsertionPoint(op);
3082 cir::BitfieldInfoAttr info = op.getBitfieldInfo();
3083 uint64_t size = info.getSize();
3084 uint64_t offset = info.getOffset();
3085 mlir::Type storageType = info.getStorageType();
3086 mlir::MLIRContext *context = storageType.getContext();
3088 unsigned storageSize = 0;
3090 mlir::IntegerType intType =
3093 mlir::Value srcVal =
createIntCast(rewriter, adaptor.getSrc(), intType);
3094 unsigned srcWidth = storageSize;
3095 mlir::Value resultVal = srcVal;
3097 if (storageSize != size) {
3098 assert(storageSize > size &&
"Invalid bitfield size.");
3100 mlir::Value val = rewriter.create<mlir::LLVM::LoadOp>(
3101 op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(),
3102 op.getIsVolatile());
3105 createAnd(rewriter, srcVal, llvm::APInt::getLowBitsSet(srcWidth, size));
3107 srcVal =
createShL(rewriter, srcVal, offset);
3111 ~llvm::APInt::getBitsSet(srcWidth, offset, offset + size));
3114 srcVal = rewriter.create<mlir::LLVM::OrOp>(op.getLoc(), val, srcVal);
3117 rewriter.create<mlir::LLVM::StoreOp>(op.getLoc(), srcVal, adaptor.getAddr(),
3118 op.getAlignment(), op.getIsVolatile());
3120 mlir::Type resultTy = getTypeConverter()->convertType(op.getType());
3122 if (info.getIsSigned()) {
3123 assert(size <= storageSize);
3124 unsigned highBits = storageSize - size;
3127 resultVal =
createShL(rewriter, resultVal, highBits);
3128 resultVal =
createAShR(rewriter, resultVal, highBits);
3133 mlir::cast<mlir::IntegerType>(resultTy),
3134 info.getIsSigned());
3136 rewriter.replaceOp(op, resultVal);
3137 return mlir::success();
3140mlir::LogicalResult CIRToLLVMComplexImagPtrOpLowering::matchAndRewrite(
3141 cir::ComplexImagPtrOp op, OpAdaptor adaptor,
3142 mlir::ConversionPatternRewriter &rewriter)
const {
3143 cir::PointerType operandTy = op.getOperand().getType();
3144 mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
3145 mlir::Type elementLLVMTy =
3146 getTypeConverter()->convertType(operandTy.getPointee());
3148 mlir::LLVM::GEPArg gepIndices[2] = {{0}, {1}};
3149 mlir::LLVM::GEPNoWrapFlags inboundsNuw =
3150 mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
3151 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
3152 op, resultLLVMTy, elementLLVMTy, adaptor.getOperand(), gepIndices,
3154 return mlir::success();
3157mlir::LogicalResult CIRToLLVMComplexRealPtrOpLowering::matchAndRewrite(
3158 cir::ComplexRealPtrOp op, OpAdaptor adaptor,
3159 mlir::ConversionPatternRewriter &rewriter)
const {
3160 cir::PointerType operandTy = op.getOperand().getType();
3161 mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
3162 mlir::Type elementLLVMTy =
3163 getTypeConverter()->convertType(operandTy.getPointee());
3165 mlir::LLVM::GEPArg gepIndices[2] = {0, 0};
3166 mlir::LLVM::GEPNoWrapFlags inboundsNuw =
3167 mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
3168 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
3169 op, resultLLVMTy, elementLLVMTy, adaptor.getOperand(), gepIndices,
3171 return mlir::success();
3174mlir::LogicalResult CIRToLLVMGetBitfieldOpLowering::matchAndRewrite(
3175 cir::GetBitfieldOp op, OpAdaptor adaptor,
3176 mlir::ConversionPatternRewriter &rewriter)
const {
3178 mlir::OpBuilder::InsertionGuard guard(rewriter);
3179 rewriter.setInsertionPoint(op);
3181 cir::BitfieldInfoAttr info = op.getBitfieldInfo();
3183 uint64_t offset = info.getOffset();
3184 mlir::Type storageType = info.getStorageType();
3185 mlir::MLIRContext *context = storageType.getContext();
3186 unsigned storageSize = 0;
3188 mlir::IntegerType intType =
3191 mlir::Value val = rewriter.create<mlir::LLVM::LoadOp>(
3192 op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(),
3193 op.getIsVolatile());
3194 val = rewriter.create<mlir::LLVM::BitcastOp>(op.getLoc(), intType, val);
3196 if (info.getIsSigned()) {
3197 assert(
static_cast<unsigned>(offset + size) <= storageSize);
3198 unsigned highBits = storageSize - offset - size;
3199 val =
createShL(rewriter, val, highBits);
3200 val =
createAShR(rewriter, val, offset + highBits);
3204 if (
static_cast<unsigned>(offset) + size < storageSize)
3206 llvm::APInt::getLowBitsSet(storageSize, size));
3209 mlir::Type resTy = getTypeConverter()->convertType(op.getType());
3211 rewriter, val, mlir::cast<mlir::IntegerType>(resTy), info.getIsSigned());
3212 rewriter.replaceOp(op, newOp);
3213 return mlir::success();
3216mlir::LogicalResult CIRToLLVMInlineAsmOpLowering::matchAndRewrite(
3217 cir::InlineAsmOp op, OpAdaptor adaptor,
3218 mlir::ConversionPatternRewriter &rewriter)
const {
3220 if (op.getNumResults())
3221 llResTy = getTypeConverter()->convertType(op.getType(0));
3223 cir::AsmFlavor dialect = op.getAsmFlavor();
3224 mlir::LLVM::AsmDialect llDialect = dialect == cir::AsmFlavor::x86_att
3225 ? mlir::LLVM::AsmDialect::AD_ATT
3226 : mlir::LLVM::AsmDialect::AD_Intel;
3228 SmallVector<mlir::Attribute> opAttrs;
3229 StringRef llvmAttrName = mlir::LLVM::InlineAsmOp::getElementTypeAttrName();
3235 if (!op.getNumResults())
3236 opAttrs.push_back(mlir::Attribute());
3238 SmallVector<mlir::Value> llvmOperands;
3239 SmallVector<mlir::Value> cirOperands;
3240 for (
auto const &[llvmOp, cirOp] :
3241 zip(adaptor.getAsmOperands(), op.getAsmOperands())) {
3242 append_range(llvmOperands, llvmOp);
3243 append_range(cirOperands, cirOp);
3248 for (
auto const &[cirOpAttr, cirOp] :
3249 zip(op.getOperandAttrs(), cirOperands)) {
3251 opAttrs.push_back(mlir::Attribute());
3255 llvm::SmallVector<mlir::NamedAttribute, 1> attrs;
3256 cir::PointerType typ = mlir::cast<cir::PointerType>(cirOp.getType());
3258 *getTypeConverter(), dataLayout, typ.getPointee()));
3260 attrs.push_back(rewriter.getNamedAttr(llvmAttrName, typAttr));
3261 mlir::DictionaryAttr newDict = rewriter.getDictionaryAttr(attrs);
3262 opAttrs.push_back(newDict);
3265 rewriter.replaceOpWithNewOp<mlir::LLVM::InlineAsmOp>(
3266 op, llResTy, llvmOperands, op.getAsmStringAttr(), op.getConstraintsAttr(),
3267 op.getSideEffectsAttr(),
3270 mlir::LLVM::TailCallKindAttr::get(
3271 getContext(), mlir::LLVM::tailcallkind::TailCallKind::None),
3272 mlir::LLVM::AsmDialectAttr::get(getContext(), llDialect),
3273 rewriter.getArrayAttr(opAttrs));
3275 return mlir::success();
3278mlir::LogicalResult CIRToLLVMVAStartOpLowering::matchAndRewrite(
3279 cir::VAStartOp op, OpAdaptor adaptor,
3280 mlir::ConversionPatternRewriter &rewriter)
const {
3281 auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
3282 auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
3283 adaptor.getArgList());
3284 rewriter.replaceOpWithNewOp<mlir::LLVM::VaStartOp>(op, vaList);
3285 return mlir::success();
3288mlir::LogicalResult CIRToLLVMVAEndOpLowering::matchAndRewrite(
3289 cir::VAEndOp op, OpAdaptor adaptor,
3290 mlir::ConversionPatternRewriter &rewriter)
const {
3291 auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
3292 auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
3293 adaptor.getArgList());
3294 rewriter.replaceOpWithNewOp<mlir::LLVM::VaEndOp>(op, vaList);
3295 return mlir::success();
3298mlir::LogicalResult CIRToLLVMVAArgOpLowering::matchAndRewrite(
3299 cir::VAArgOp op, OpAdaptor adaptor,
3300 mlir::ConversionPatternRewriter &rewriter)
const {
3302 auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
3303 auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
3304 adaptor.getArgList());
3306 mlir::Type llvmType =
3307 getTypeConverter()->convertType(op->getResultTypes().front());
3309 return mlir::failure();
3311 rewriter.replaceOpWithNewOp<mlir::LLVM::VaArgOp>(op, llvmType, vaList);
3312 return mlir::success();
3316 return std::make_unique<ConvertCIRToLLVMPass>();
3324std::unique_ptr<llvm::Module>
3326 llvm::TimeTraceScope scope(
"lower from CIR to LLVM directly");
3328 mlir::MLIRContext *mlirCtx = mlirModule.getContext();
3330 mlir::PassManager pm(mlirCtx);
3333 (void)mlir::applyPassManagerCLOptions(pm);
3335 if (mlir::failed(pm.run(mlirModule))) {
3338 "The pass manager failed to lower CIR to LLVMIR dialect!");
3341 mlir::registerBuiltinDialectTranslation(*mlirCtx);
3342 mlir::registerLLVMDialectTranslation(*mlirCtx);
3345 llvm::TimeTraceScope translateScope(
"translateModuleToLLVMIR");
3347 StringRef moduleName = mlirModule.getName().value_or(
"CIRToLLVMModule");
3348 std::unique_ptr<llvm::Module> llvmModule =
3349 mlir::translateModuleToLLVMIR(mlirModule, llvmCtx, moduleName);
3353 report_fatal_error(
"Lowering from LLVMIR dialect to llvm IR failed!");
static bool isUnsigned(SValBuilder &SVB, NonLoc Value)
static llvm::StringRef getLinkageAttrNameString()
Returns the name used for the linkage attribute.
mlir::Value createLShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs)
mlir::Value createShL(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs)
std::optional< mlir::Attribute > lowerConstArrayAttr(cir::ConstArrayAttr constArr, const mlir::TypeConverter *converter)
mlir::Value createAShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs)
mlir::Value createAnd(mlir::OpBuilder &bld, mlir::Value lhs, const llvm::APInt &rhs)
static bool isVector(QualType QT, QualType ElementType)
This helper function returns true if QT is a vector type that has element type ElementType.
mlir::Value visitCirAttr(cir::IntAttr intAttr)
IntAttr visitor.
mlir::Value visit(mlir::Attribute attr)
CIRAttrToValue(mlir::Operation *parentOp, mlir::ConversionPatternRewriter &rewriter, const mlir::TypeConverter *converter)
mlir::Attribute visit(mlir::Attribute attr)
mlir::Attribute visitCirAttr(cir::FPAttr attr)
mlir::Attribute visitCirAttr(cir::BoolAttr attr)
GlobalInitAttrRewriter(mlir::Type type, mlir::ConversionPatternRewriter &rewriter)
mlir::Attribute visitCirAttr(cir::IntAttr attr)
void createLLVMFuncOpIfNotExist(mlir::ConversionPatternRewriter &rewriter, mlir::Operation *srcOp, llvm::StringRef fnName, mlir::Type fnTy)
static void collectUnreachable(mlir::Operation *parent, llvm::SmallVector< mlir::Operation * > &ops)
static mlir::LLVM::ICmpPredicate convertCmpKindToICmpPredicate(cir::CmpOpKind kind, bool isSigned)
Convert from a CIR comparison kind to an LLVM IR integral comparison kind.
static mlir::Value getLLVMIntCast(mlir::ConversionPatternRewriter &rewriter, mlir::Value llvmSrc, mlir::Type llvmDstIntTy, bool isUnsigned, uint64_t cirSrcWidth, uint64_t cirDstIntWidth)
void convertSideEffectForCall(mlir::Operation *callOp, bool isNothrow, cir::SideEffect sideEffect, mlir::LLVM::MemoryEffectsAttr &memoryEffect, bool &noUnwind, bool &willReturn)
static mlir::Value emitFromMemory(mlir::ConversionPatternRewriter &rewriter, mlir::DataLayout const &dataLayout, cir::LoadOp op, mlir::Value value)
Emits the value from memory as expected by its users.
mlir::IntegerType computeBitfieldIntType(mlir::Type storageType, mlir::MLIRContext *context, unsigned &storageSize)
static mlir::LLVM::CallIntrinsicOp createCallLLVMIntrinsicOp(mlir::ConversionPatternRewriter &rewriter, mlir::Location loc, const llvm::Twine &intrinsicName, mlir::Type resultTy, mlir::ValueRange operands)
bool hasTrailingZeros(cir::ConstArrayAttr attr)
static mlir::LogicalResult rewriteCallOrInvoke(mlir::Operation *op, mlir::ValueRange callOperands, mlir::ConversionPatternRewriter &rewriter, const mlir::TypeConverter *converter, mlir::FlatSymbolRefAttr calleeAttr)
std::unique_ptr< llvm::Module > lowerDirectlyFromCIRToLLVMIR(mlir::ModuleOp mlirModule, llvm::LLVMContext &llvmCtx)
mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp, const mlir::Attribute attr, mlir::ConversionPatternRewriter &rewriter, const mlir::TypeConverter *converter)
Switches on the type of attribute and calls the appropriate conversion.
static mlir::LLVM::CallIntrinsicOp replaceOpWithCallLLVMIntrinsicOp(mlir::ConversionPatternRewriter &rewriter, mlir::Operation *op, const llvm::Twine &intrinsicName, mlir::Type resultTy, mlir::ValueRange operands)
static void prepareTypeConverter(mlir::LLVMTypeConverter &converter, mlir::DataLayout &dataLayout)
static mlir::LLVM::AtomicOrdering getLLVMMemOrder(std::optional< cir::MemOrder > memorder)
std::unique_ptr< mlir::Pass > createConvertCIRToLLVMPass()
Create a pass that fully lowers CIR to the LLVMIR dialect.
static mlir::LLVM::FCmpPredicate convertCmpKindToFCmpPredicate(cir::CmpOpKind kind)
Convert from a CIR comparison kind to an LLVM IR floating-point comparison kind.
static mlir::LLVM::Visibility lowerCIRVisibilityToLLVMVisibility(cir::VisibilityKind visibilityKind)
void populateCIRToLLVMPasses(mlir::OpPassManager &pm)
Adds passes that fully lower CIR to the LLVMIR dialect.
mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage)
static mlir::Type convertTypeForMemory(const mlir::TypeConverter &converter, mlir::DataLayout const &dataLayout, mlir::Type type)
Given a type convertor and a data layout, convert the given type to a type that is suitable for memor...
static mlir::Value createIntCast(mlir::OpBuilder &bld, mlir::Value src, mlir::IntegerType dstTy, bool isSigned=false)
static mlir::Value getValueForVTableSymbol(mlir::Operation *op, mlir::ConversionPatternRewriter &rewriter, const mlir::TypeConverter *converter, mlir::FlatSymbolRefAttr nameAttr, mlir::Type &eltType)
static mlir::Value emitToMemory(mlir::ConversionPatternRewriter &rewriter, mlir::DataLayout const &dataLayout, mlir::Type origType, mlir::Value value)
Emits a value to memory with the expected scalar type.
static bool isIntTypeUnsigned(mlir::Type type)
const internal::VariadicAllOfMatcher< Attr > attr
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
const AstTypeMatcher< ComplexType > complexType
unsigned kind
All of the diagnostics that can be emitted by the frontend.
Diagnostic wrappers for TextAPI types for error reporting.
void populateCIRPreLoweringPasses(mlir::OpPassManager &pm)
void registerCIRDialectTranslation(mlir::MLIRContext &context)
char __ovld __cnfn clz(char)
Returns the number of leading 0-bits in x, starting at the most significant bit position.
char __ovld __cnfn ctz(char)
Returns the count of trailing 0-bits in x.
float __ovld __cnfn length(float)
Return the length of vector p, i.e., sqrt(p.x2 + p.y 2 + ...)
char __ovld __cnfn select(char, char, char)
For each component of a vector type, result[i] = if MSB of c[i] is set ?
static bool dataMemberType()
static bool opGlobalConstant()
static bool addressSpace()
static bool opGlobalThreadLocal()
static bool globalViewIntLowering()
static bool opAllocaAnnotations()
static bool atomicScope()
static bool opLoadStoreTbaa()
static bool optInfoAttr()
static bool opFuncExtraAttrs()
static bool opCallLandingPad()
static bool vaArgABILowering()
static bool fpConstraints()
static bool lowerModeOptLevel()
static bool opCallCallConv()
static bool opFuncCallingConv()
static bool aggValueSlotVolatile()
static bool fastMathFlags()
static bool opCallContinueBlock()
static bool opLoadStoreNontemporal()
static bool atomicSyncScopeID()
static bool opFuncMultipleReturnVals()
void runOnOperation() final
StringRef getDescription() const override
StringRef getArgument() const override
void getDependentDialects(mlir::DialectRegistry ®istry) const override
void processCIRAttrs(mlir::ModuleOp module)