clang 22.0.0git
LowerToLLVM.cpp
Go to the documentation of this file.
1//====- LowerToLLVM.cpp - Lowering from CIR to LLVMIR ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements lowering of CIR operations to LLVMIR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "LowerToLLVM.h"
14
15#include <deque>
16#include <optional>
17
18#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
19#include "mlir/Dialect/DLTI/DLTI.h"
20#include "mlir/Dialect/Func/IR/FuncOps.h"
21#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
22#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
23#include "mlir/IR/BuiltinAttributes.h"
24#include "mlir/IR/BuiltinDialect.h"
25#include "mlir/IR/BuiltinOps.h"
26#include "mlir/IR/Types.h"
27#include "mlir/Pass/Pass.h"
28#include "mlir/Pass/PassManager.h"
29#include "mlir/Target/LLVMIR/Dialect/Builtin/BuiltinToLLVMIRTranslation.h"
30#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
31#include "mlir/Target/LLVMIR/Export.h"
32#include "mlir/Transforms/DialectConversion.h"
38#include "clang/CIR/Passes.h"
39#include "llvm/ADT/TypeSwitch.h"
40#include "llvm/IR/Module.h"
41#include "llvm/Support/ErrorHandling.h"
42#include "llvm/Support/TimeProfiler.h"
43
44using namespace cir;
45using namespace llvm;
46
47namespace cir {
48namespace direct {
49
50//===----------------------------------------------------------------------===//
51// Helper Methods
52//===----------------------------------------------------------------------===//
53
54namespace {
55/// If the given type is a vector type, return the vector's element type.
56/// Otherwise return the given type unchanged.
57mlir::Type elementTypeIfVector(mlir::Type type) {
58 return llvm::TypeSwitch<mlir::Type, mlir::Type>(type)
59 .Case<cir::VectorType, mlir::VectorType>(
60 [](auto p) { return p.getElementType(); })
61 .Default([](mlir::Type p) { return p; });
62}
63} // namespace
64
65/// Given a type convertor and a data layout, convert the given type to a type
66/// that is suitable for memory operations. For example, this can be used to
67/// lower cir.bool accesses to i8.
68static mlir::Type convertTypeForMemory(const mlir::TypeConverter &converter,
69 mlir::DataLayout const &dataLayout,
70 mlir::Type type) {
71 // TODO(cir): Handle other types similarly to clang's codegen
72 // convertTypeForMemory
73 if (isa<cir::BoolType>(type)) {
74 return mlir::IntegerType::get(type.getContext(),
75 dataLayout.getTypeSizeInBits(type));
76 }
77
78 return converter.convertType(type);
79}
80
81static mlir::Value createIntCast(mlir::OpBuilder &bld, mlir::Value src,
82 mlir::IntegerType dstTy,
83 bool isSigned = false) {
84 mlir::Type srcTy = src.getType();
85 assert(mlir::isa<mlir::IntegerType>(srcTy));
86
87 unsigned srcWidth = mlir::cast<mlir::IntegerType>(srcTy).getWidth();
88 unsigned dstWidth = mlir::cast<mlir::IntegerType>(dstTy).getWidth();
89 mlir::Location loc = src.getLoc();
90
91 if (dstWidth > srcWidth && isSigned)
92 return bld.create<mlir::LLVM::SExtOp>(loc, dstTy, src);
93 if (dstWidth > srcWidth)
94 return bld.create<mlir::LLVM::ZExtOp>(loc, dstTy, src);
95 if (dstWidth < srcWidth)
96 return bld.create<mlir::LLVM::TruncOp>(loc, dstTy, src);
97 return bld.create<mlir::LLVM::BitcastOp>(loc, dstTy, src);
98}
99
100static mlir::LLVM::Visibility
101lowerCIRVisibilityToLLVMVisibility(cir::VisibilityKind visibilityKind) {
102 switch (visibilityKind) {
103 case cir::VisibilityKind::Default:
104 return ::mlir::LLVM::Visibility::Default;
105 case cir::VisibilityKind::Hidden:
106 return ::mlir::LLVM::Visibility::Hidden;
107 case cir::VisibilityKind::Protected:
108 return ::mlir::LLVM::Visibility::Protected;
109 }
110}
111
112/// Emits the value from memory as expected by its users. Should be called when
113/// the memory represetnation of a CIR type is not equal to its scalar
114/// representation.
115static mlir::Value emitFromMemory(mlir::ConversionPatternRewriter &rewriter,
116 mlir::DataLayout const &dataLayout,
117 cir::LoadOp op, mlir::Value value) {
118
119 // TODO(cir): Handle other types similarly to clang's codegen EmitFromMemory
120 if (auto boolTy = mlir::dyn_cast<cir::BoolType>(op.getType())) {
121 // Create a cast value from specified size in datalayout to i1
122 assert(value.getType().isInteger(dataLayout.getTypeSizeInBits(boolTy)));
123 return createIntCast(rewriter, value, rewriter.getI1Type());
124 }
125
126 return value;
127}
128
129/// Emits a value to memory with the expected scalar type. Should be called when
130/// the memory represetnation of a CIR type is not equal to its scalar
131/// representation.
132static mlir::Value emitToMemory(mlir::ConversionPatternRewriter &rewriter,
133 mlir::DataLayout const &dataLayout,
134 mlir::Type origType, mlir::Value value) {
135
136 // TODO(cir): Handle other types similarly to clang's codegen EmitToMemory
137 if (auto boolTy = mlir::dyn_cast<cir::BoolType>(origType)) {
138 // Create zext of value from i1 to i8
139 mlir::IntegerType memType =
140 rewriter.getIntegerType(dataLayout.getTypeSizeInBits(boolTy));
141 return createIntCast(rewriter, value, memType);
142 }
143
144 return value;
145}
146
147mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage) {
148 using CIR = cir::GlobalLinkageKind;
149 using LLVM = mlir::LLVM::Linkage;
150
151 switch (linkage) {
152 case CIR::AvailableExternallyLinkage:
153 return LLVM::AvailableExternally;
154 case CIR::CommonLinkage:
155 return LLVM::Common;
156 case CIR::ExternalLinkage:
157 return LLVM::External;
158 case CIR::ExternalWeakLinkage:
159 return LLVM::ExternWeak;
160 case CIR::InternalLinkage:
161 return LLVM::Internal;
162 case CIR::LinkOnceAnyLinkage:
163 return LLVM::Linkonce;
164 case CIR::LinkOnceODRLinkage:
165 return LLVM::LinkonceODR;
166 case CIR::PrivateLinkage:
167 return LLVM::Private;
168 case CIR::WeakAnyLinkage:
169 return LLVM::Weak;
170 case CIR::WeakODRLinkage:
171 return LLVM::WeakODR;
172 };
173 llvm_unreachable("Unknown CIR linkage type");
174}
175
176static mlir::Value getLLVMIntCast(mlir::ConversionPatternRewriter &rewriter,
177 mlir::Value llvmSrc, mlir::Type llvmDstIntTy,
178 bool isUnsigned, uint64_t cirSrcWidth,
179 uint64_t cirDstIntWidth) {
180 if (cirSrcWidth == cirDstIntWidth)
181 return llvmSrc;
182
183 auto loc = llvmSrc.getLoc();
184 if (cirSrcWidth < cirDstIntWidth) {
185 if (isUnsigned)
186 return rewriter.create<mlir::LLVM::ZExtOp>(loc, llvmDstIntTy, llvmSrc);
187 return rewriter.create<mlir::LLVM::SExtOp>(loc, llvmDstIntTy, llvmSrc);
188 }
189
190 // Otherwise truncate
191 return rewriter.create<mlir::LLVM::TruncOp>(loc, llvmDstIntTy, llvmSrc);
192}
193
195public:
196 CIRAttrToValue(mlir::Operation *parentOp,
197 mlir::ConversionPatternRewriter &rewriter,
198 const mlir::TypeConverter *converter)
199 : parentOp(parentOp), rewriter(rewriter), converter(converter) {}
200
201 mlir::Value visit(mlir::Attribute attr) {
202 return llvm::TypeSwitch<mlir::Attribute, mlir::Value>(attr)
203 .Case<cir::IntAttr, cir::FPAttr, cir::ConstComplexAttr,
204 cir::ConstArrayAttr, cir::ConstRecordAttr, cir::ConstVectorAttr,
205 cir::ConstPtrAttr, cir::GlobalViewAttr, cir::VTableAttr,
206 cir::ZeroAttr>([&](auto attrT) { return visitCirAttr(attrT); })
207 .Default([&](auto attrT) { return mlir::Value(); });
208 }
209
210 mlir::Value visitCirAttr(cir::IntAttr intAttr);
211 mlir::Value visitCirAttr(cir::FPAttr fltAttr);
212 mlir::Value visitCirAttr(cir::ConstComplexAttr complexAttr);
213 mlir::Value visitCirAttr(cir::ConstPtrAttr ptrAttr);
214 mlir::Value visitCirAttr(cir::ConstArrayAttr attr);
215 mlir::Value visitCirAttr(cir::ConstRecordAttr attr);
216 mlir::Value visitCirAttr(cir::ConstVectorAttr attr);
217 mlir::Value visitCirAttr(cir::GlobalViewAttr attr);
218 mlir::Value visitCirAttr(cir::VTableAttr attr);
219 mlir::Value visitCirAttr(cir::ZeroAttr attr);
220
221private:
222 mlir::Operation *parentOp;
223 mlir::ConversionPatternRewriter &rewriter;
224 const mlir::TypeConverter *converter;
225};
226
227/// Switches on the type of attribute and calls the appropriate conversion.
228mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp,
229 const mlir::Attribute attr,
230 mlir::ConversionPatternRewriter &rewriter,
231 const mlir::TypeConverter *converter) {
232 CIRAttrToValue valueConverter(parentOp, rewriter, converter);
233 mlir::Value value = valueConverter.visit(attr);
234 if (!value)
235 llvm_unreachable("unhandled attribute type");
236 return value;
237}
238
239void convertSideEffectForCall(mlir::Operation *callOp, bool isNothrow,
240 cir::SideEffect sideEffect,
241 mlir::LLVM::MemoryEffectsAttr &memoryEffect,
242 bool &noUnwind, bool &willReturn) {
243 using mlir::LLVM::ModRefInfo;
244
245 switch (sideEffect) {
246 case cir::SideEffect::All:
247 memoryEffect = {};
248 noUnwind = isNothrow;
249 willReturn = false;
250 break;
251
252 case cir::SideEffect::Pure:
253 memoryEffect = mlir::LLVM::MemoryEffectsAttr::get(
254 callOp->getContext(), /*other=*/ModRefInfo::Ref,
255 /*argMem=*/ModRefInfo::Ref,
256 /*inaccessibleMem=*/ModRefInfo::Ref);
257 noUnwind = true;
258 willReturn = true;
259 break;
260
261 case cir::SideEffect::Const:
262 memoryEffect = mlir::LLVM::MemoryEffectsAttr::get(
263 callOp->getContext(), /*other=*/ModRefInfo::NoModRef,
264 /*argMem=*/ModRefInfo::NoModRef,
265 /*inaccessibleMem=*/ModRefInfo::NoModRef);
266 noUnwind = true;
267 willReturn = true;
268 break;
269 }
270}
271
272static mlir::LLVM::CallIntrinsicOp
273createCallLLVMIntrinsicOp(mlir::ConversionPatternRewriter &rewriter,
274 mlir::Location loc, const llvm::Twine &intrinsicName,
275 mlir::Type resultTy, mlir::ValueRange operands) {
276 auto intrinsicNameAttr =
277 mlir::StringAttr::get(rewriter.getContext(), intrinsicName);
278 return mlir::LLVM::CallIntrinsicOp::create(rewriter, loc, resultTy,
279 intrinsicNameAttr, operands);
280}
281
282static mlir::LLVM::CallIntrinsicOp replaceOpWithCallLLVMIntrinsicOp(
283 mlir::ConversionPatternRewriter &rewriter, mlir::Operation *op,
284 const llvm::Twine &intrinsicName, mlir::Type resultTy,
285 mlir::ValueRange operands) {
286 mlir::LLVM::CallIntrinsicOp callIntrinOp = createCallLLVMIntrinsicOp(
287 rewriter, op->getLoc(), intrinsicName, resultTy, operands);
288 rewriter.replaceOp(op, callIntrinOp.getOperation());
289 return callIntrinOp;
290}
291
292/// IntAttr visitor.
293mlir::Value CIRAttrToValue::visitCirAttr(cir::IntAttr intAttr) {
294 mlir::Location loc = parentOp->getLoc();
295 return rewriter.create<mlir::LLVM::ConstantOp>(
296 loc, converter->convertType(intAttr.getType()), intAttr.getValue());
297}
298
299/// FPAttr visitor.
300mlir::Value CIRAttrToValue::visitCirAttr(cir::FPAttr fltAttr) {
301 mlir::Location loc = parentOp->getLoc();
302 return rewriter.create<mlir::LLVM::ConstantOp>(
303 loc, converter->convertType(fltAttr.getType()), fltAttr.getValue());
304}
305
306/// ConstComplexAttr visitor.
307mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstComplexAttr complexAttr) {
308 auto complexType = mlir::cast<cir::ComplexType>(complexAttr.getType());
309 mlir::Type complexElemTy = complexType.getElementType();
310 mlir::Type complexElemLLVMTy = converter->convertType(complexElemTy);
311
312 mlir::Attribute components[2];
313 if (const auto intType = mlir::dyn_cast<cir::IntType>(complexElemTy)) {
314 components[0] = rewriter.getIntegerAttr(
315 complexElemLLVMTy,
316 mlir::cast<cir::IntAttr>(complexAttr.getReal()).getValue());
317 components[1] = rewriter.getIntegerAttr(
318 complexElemLLVMTy,
319 mlir::cast<cir::IntAttr>(complexAttr.getImag()).getValue());
320 } else {
321 components[0] = rewriter.getFloatAttr(
322 complexElemLLVMTy,
323 mlir::cast<cir::FPAttr>(complexAttr.getReal()).getValue());
324 components[1] = rewriter.getFloatAttr(
325 complexElemLLVMTy,
326 mlir::cast<cir::FPAttr>(complexAttr.getImag()).getValue());
327 }
328
329 mlir::Location loc = parentOp->getLoc();
330 return rewriter.create<mlir::LLVM::ConstantOp>(
331 loc, converter->convertType(complexAttr.getType()),
332 rewriter.getArrayAttr(components));
333}
334
335/// ConstPtrAttr visitor.
336mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstPtrAttr ptrAttr) {
337 mlir::Location loc = parentOp->getLoc();
338 if (ptrAttr.isNullValue()) {
339 return rewriter.create<mlir::LLVM::ZeroOp>(
340 loc, converter->convertType(ptrAttr.getType()));
341 }
342 mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>());
343 mlir::Value ptrVal = rewriter.create<mlir::LLVM::ConstantOp>(
344 loc, rewriter.getIntegerType(layout.getTypeSizeInBits(ptrAttr.getType())),
345 ptrAttr.getValue().getInt());
346 return rewriter.create<mlir::LLVM::IntToPtrOp>(
347 loc, converter->convertType(ptrAttr.getType()), ptrVal);
348}
349
350// ConstArrayAttr visitor
351mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstArrayAttr attr) {
352 mlir::Type llvmTy = converter->convertType(attr.getType());
353 mlir::Location loc = parentOp->getLoc();
354 mlir::Value result;
355
356 if (attr.hasTrailingZeros()) {
357 mlir::Type arrayTy = attr.getType();
358 result = rewriter.create<mlir::LLVM::ZeroOp>(
359 loc, converter->convertType(arrayTy));
360 } else {
361 result = rewriter.create<mlir::LLVM::UndefOp>(loc, llvmTy);
362 }
363
364 // Iteratively lower each constant element of the array.
365 if (auto arrayAttr = mlir::dyn_cast<mlir::ArrayAttr>(attr.getElts())) {
366 for (auto [idx, elt] : llvm::enumerate(arrayAttr)) {
367 mlir::DataLayout dataLayout(parentOp->getParentOfType<mlir::ModuleOp>());
368 mlir::Value init = visit(elt);
369 result =
370 rewriter.create<mlir::LLVM::InsertValueOp>(loc, result, init, idx);
371 }
372 } else if (auto strAttr = mlir::dyn_cast<mlir::StringAttr>(attr.getElts())) {
373 // TODO(cir): this diverges from traditional lowering. Normally the string
374 // would be a global constant that is memcopied.
375 auto arrayTy = mlir::dyn_cast<cir::ArrayType>(strAttr.getType());
376 assert(arrayTy && "String attribute must have an array type");
377 mlir::Type eltTy = arrayTy.getElementType();
378 for (auto [idx, elt] : llvm::enumerate(strAttr)) {
379 auto init = rewriter.create<mlir::LLVM::ConstantOp>(
380 loc, converter->convertType(eltTy), elt);
381 result =
382 rewriter.create<mlir::LLVM::InsertValueOp>(loc, result, init, idx);
383 }
384 } else {
385 llvm_unreachable("unexpected ConstArrayAttr elements");
386 }
387
388 return result;
389}
390
391/// ConstRecord visitor.
392mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstRecordAttr constRecord) {
393 const mlir::Type llvmTy = converter->convertType(constRecord.getType());
394 const mlir::Location loc = parentOp->getLoc();
395 mlir::Value result = rewriter.create<mlir::LLVM::UndefOp>(loc, llvmTy);
396
397 // Iteratively lower each constant element of the record.
398 for (auto [idx, elt] : llvm::enumerate(constRecord.getMembers())) {
399 mlir::Value init = visit(elt);
400 result = rewriter.create<mlir::LLVM::InsertValueOp>(loc, result, init, idx);
401 }
402
403 return result;
404}
405
406/// ConstVectorAttr visitor.
407mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstVectorAttr attr) {
408 const mlir::Type llvmTy = converter->convertType(attr.getType());
409 const mlir::Location loc = parentOp->getLoc();
410
412 for (const mlir::Attribute elementAttr : attr.getElts()) {
413 mlir::Attribute mlirAttr;
414 if (auto intAttr = mlir::dyn_cast<cir::IntAttr>(elementAttr)) {
415 mlirAttr = rewriter.getIntegerAttr(
416 converter->convertType(intAttr.getType()), intAttr.getValue());
417 } else if (auto floatAttr = mlir::dyn_cast<cir::FPAttr>(elementAttr)) {
418 mlirAttr = rewriter.getFloatAttr(
419 converter->convertType(floatAttr.getType()), floatAttr.getValue());
420 } else {
421 llvm_unreachable(
422 "vector constant with an element that is neither an int nor a float");
423 }
424 mlirValues.push_back(mlirAttr);
425 }
426
427 return rewriter.create<mlir::LLVM::ConstantOp>(
428 loc, llvmTy,
429 mlir::DenseElementsAttr::get(mlir::cast<mlir::ShapedType>(llvmTy),
430 mlirValues));
431}
432
433// GlobalViewAttr visitor.
434mlir::Value CIRAttrToValue::visitCirAttr(cir::GlobalViewAttr globalAttr) {
435 auto moduleOp = parentOp->getParentOfType<mlir::ModuleOp>();
436 mlir::DataLayout dataLayout(moduleOp);
437 mlir::Type sourceType;
439 llvm::StringRef symName;
440 mlir::Operation *sourceSymbol =
441 mlir::SymbolTable::lookupSymbolIn(moduleOp, globalAttr.getSymbol());
442 if (auto llvmSymbol = dyn_cast<mlir::LLVM::GlobalOp>(sourceSymbol)) {
443 sourceType = llvmSymbol.getType();
444 symName = llvmSymbol.getSymName();
445 } else if (auto cirSymbol = dyn_cast<cir::GlobalOp>(sourceSymbol)) {
446 sourceType =
447 convertTypeForMemory(*converter, dataLayout, cirSymbol.getSymType());
448 symName = cirSymbol.getSymName();
449 } else if (auto llvmFun = dyn_cast<mlir::LLVM::LLVMFuncOp>(sourceSymbol)) {
450 sourceType = llvmFun.getFunctionType();
451 symName = llvmFun.getSymName();
452 } else if (auto fun = dyn_cast<cir::FuncOp>(sourceSymbol)) {
453 sourceType = converter->convertType(fun.getFunctionType());
454 symName = fun.getSymName();
455 } else if (auto alias = dyn_cast<mlir::LLVM::AliasOp>(sourceSymbol)) {
456 sourceType = alias.getType();
457 symName = alias.getSymName();
458 } else {
459 llvm_unreachable("Unexpected GlobalOp type");
460 }
461
462 mlir::Location loc = parentOp->getLoc();
463 mlir::Value addrOp = rewriter.create<mlir::LLVM::AddressOfOp>(
464 loc, mlir::LLVM::LLVMPointerType::get(rewriter.getContext()), symName);
465
466 if (globalAttr.getIndices()) {
468
469 if (mlir::isa<mlir::LLVM::LLVMArrayType, mlir::LLVM::LLVMStructType>(
470 sourceType))
471 indices.push_back(0);
472
473 for (mlir::Attribute idx : globalAttr.getIndices()) {
474 auto intAttr = mlir::cast<mlir::IntegerAttr>(idx);
475 indices.push_back(intAttr.getValue().getSExtValue());
476 }
477 mlir::Type resTy = addrOp.getType();
478 mlir::Type eltTy = converter->convertType(sourceType);
479 addrOp = rewriter.create<mlir::LLVM::GEPOp>(
480 loc, resTy, eltTy, addrOp, indices, mlir::LLVM::GEPNoWrapFlags::none);
481 }
482
483 // The incubator has handling here for the attribute having integer type, but
484 // the only test case I could find that reaches it is a direct CIR-to-LLVM IR
485 // lowering with no clear indication of how the CIR might have been generated.
486 // We'll hit the unreachable below if this happens.
488
489 if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(globalAttr.getType())) {
490 mlir::Type llvmEltTy =
491 convertTypeForMemory(*converter, dataLayout, ptrTy.getPointee());
492
493 if (llvmEltTy == sourceType)
494 return addrOp;
495
496 mlir::Type llvmDstTy = converter->convertType(globalAttr.getType());
497 return rewriter.create<mlir::LLVM::BitcastOp>(parentOp->getLoc(), llvmDstTy,
498 addrOp);
499 }
500
501 llvm_unreachable("Expecting pointer or integer type for GlobalViewAttr");
502}
503
504// VTableAttr visitor.
505mlir::Value CIRAttrToValue::visitCirAttr(cir::VTableAttr vtableArr) {
506 mlir::Type llvmTy = converter->convertType(vtableArr.getType());
507 mlir::Location loc = parentOp->getLoc();
508 mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);
509
510 for (auto [idx, elt] : llvm::enumerate(vtableArr.getData())) {
511 mlir::Value init = visit(elt);
512 result =
513 mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);
514 }
515
516 return result;
517}
518
519/// ZeroAttr visitor.
520mlir::Value CIRAttrToValue::visitCirAttr(cir::ZeroAttr attr) {
521 mlir::Location loc = parentOp->getLoc();
522 return rewriter.create<mlir::LLVM::ZeroOp>(
523 loc, converter->convertType(attr.getType()));
524}
525
526// This class handles rewriting initializer attributes for types that do not
527// require region initialization.
529public:
530 GlobalInitAttrRewriter(mlir::Type type,
531 mlir::ConversionPatternRewriter &rewriter)
532 : llvmType(type), rewriter(rewriter) {}
533
534 mlir::Attribute visit(mlir::Attribute attr) {
535 return llvm::TypeSwitch<mlir::Attribute, mlir::Attribute>(attr)
536 .Case<cir::IntAttr, cir::FPAttr, cir::BoolAttr>(
537 [&](auto attrT) { return visitCirAttr(attrT); })
538 .Default([&](auto attrT) { return mlir::Attribute(); });
539 }
540
541 mlir::Attribute visitCirAttr(cir::IntAttr attr) {
542 return rewriter.getIntegerAttr(llvmType, attr.getValue());
543 }
544
545 mlir::Attribute visitCirAttr(cir::FPAttr attr) {
546 return rewriter.getFloatAttr(llvmType, attr.getValue());
547 }
548
549 mlir::Attribute visitCirAttr(cir::BoolAttr attr) {
550 return rewriter.getBoolAttr(attr.getValue());
551 }
552
553private:
554 mlir::Type llvmType;
555 mlir::ConversionPatternRewriter &rewriter;
556};
557
558// This pass requires the CIR to be in a "flat" state. All blocks in each
559// function must belong to the parent region. Once scopes and control flow
560// are implemented in CIR, a pass will be run before this one to flatten
561// the CIR and get it into the state that this pass requires.
563 : public mlir::PassWrapper<ConvertCIRToLLVMPass,
564 mlir::OperationPass<mlir::ModuleOp>> {
565 void getDependentDialects(mlir::DialectRegistry &registry) const override {
566 registry.insert<mlir::BuiltinDialect, mlir::DLTIDialect,
567 mlir::LLVM::LLVMDialect, mlir::func::FuncDialect>();
568 }
569 void runOnOperation() final;
570
571 void processCIRAttrs(mlir::ModuleOp module);
572
573 StringRef getDescription() const override {
574 return "Convert the prepared CIR dialect module to LLVM dialect";
575 }
576
577 StringRef getArgument() const override { return "cir-flat-to-llvm"; }
578};
579
581 cir::AssumeOp op, OpAdaptor adaptor,
582 mlir::ConversionPatternRewriter &rewriter) const {
583 auto cond = adaptor.getPredicate();
584 rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(op, cond);
585 return mlir::success();
586}
587
589 cir::AssumeAlignedOp op, OpAdaptor adaptor,
590 mlir::ConversionPatternRewriter &rewriter) const {
591 SmallVector<mlir::Value, 3> opBundleArgs{adaptor.getPointer()};
592
593 auto alignment = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
594 adaptor.getAlignmentAttr());
595 opBundleArgs.push_back(alignment);
596
597 if (mlir::Value offset = adaptor.getOffset())
598 opBundleArgs.push_back(offset);
599
600 auto cond = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
601 rewriter.getI1Type(), 1);
602 mlir::LLVM::AssumeOp::create(rewriter, op.getLoc(), cond, "align",
603 opBundleArgs);
604
605 // The llvm.assume operation does not have a result, so we need to replace
606 // all uses of this cir.assume_aligned operation with the input ptr itself.
607 rewriter.replaceOp(op, adaptor.getPointer());
608 return mlir::success();
609}
610
612 cir::AssumeSepStorageOp op, OpAdaptor adaptor,
613 mlir::ConversionPatternRewriter &rewriter) const {
614 auto cond = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(),
615 rewriter.getI1Type(), 1);
616 rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(
617 op, cond, mlir::LLVM::AssumeSeparateStorageTag{}, adaptor.getPtr1(),
618 adaptor.getPtr2());
619 return mlir::success();
620}
621
623 cir::BitClrsbOp op, OpAdaptor adaptor,
624 mlir::ConversionPatternRewriter &rewriter) const {
625 auto zero = rewriter.create<mlir::LLVM::ConstantOp>(
626 op.getLoc(), adaptor.getInput().getType(), 0);
627 auto isNeg = rewriter.create<mlir::LLVM::ICmpOp>(
628 op.getLoc(),
629 mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(),
630 mlir::LLVM::ICmpPredicate::slt),
631 adaptor.getInput(), zero);
632
633 auto negOne = rewriter.create<mlir::LLVM::ConstantOp>(
634 op.getLoc(), adaptor.getInput().getType(), -1);
635 auto flipped = rewriter.create<mlir::LLVM::XOrOp>(op.getLoc(),
636 adaptor.getInput(), negOne);
637
638 auto select = rewriter.create<mlir::LLVM::SelectOp>(
639 op.getLoc(), isNeg, flipped, adaptor.getInput());
640
641 auto resTy = getTypeConverter()->convertType(op.getType());
642 auto clz = rewriter.create<mlir::LLVM::CountLeadingZerosOp>(
643 op.getLoc(), resTy, select, /*is_zero_poison=*/false);
644
645 auto one = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 1);
646 auto res = rewriter.create<mlir::LLVM::SubOp>(op.getLoc(), clz, one);
647 rewriter.replaceOp(op, res);
648
649 return mlir::LogicalResult::success();
650}
651
653 cir::BitClzOp op, OpAdaptor adaptor,
654 mlir::ConversionPatternRewriter &rewriter) const {
655 auto resTy = getTypeConverter()->convertType(op.getType());
656 auto llvmOp = rewriter.create<mlir::LLVM::CountLeadingZerosOp>(
657 op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero());
658 rewriter.replaceOp(op, llvmOp);
659 return mlir::LogicalResult::success();
660}
661
663 cir::BitCtzOp op, OpAdaptor adaptor,
664 mlir::ConversionPatternRewriter &rewriter) const {
665 auto resTy = getTypeConverter()->convertType(op.getType());
666 auto llvmOp = rewriter.create<mlir::LLVM::CountTrailingZerosOp>(
667 op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero());
668 rewriter.replaceOp(op, llvmOp);
669 return mlir::LogicalResult::success();
670}
671
673 cir::BitFfsOp op, OpAdaptor adaptor,
674 mlir::ConversionPatternRewriter &rewriter) const {
675 auto resTy = getTypeConverter()->convertType(op.getType());
676 auto ctz = rewriter.create<mlir::LLVM::CountTrailingZerosOp>(
677 op.getLoc(), resTy, adaptor.getInput(), /*is_zero_poison=*/true);
678
679 auto one = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 1);
680 auto ctzAddOne = rewriter.create<mlir::LLVM::AddOp>(op.getLoc(), ctz, one);
681
682 auto zeroInputTy = rewriter.create<mlir::LLVM::ConstantOp>(
683 op.getLoc(), adaptor.getInput().getType(), 0);
684 auto isZero = rewriter.create<mlir::LLVM::ICmpOp>(
685 op.getLoc(),
686 mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(),
687 mlir::LLVM::ICmpPredicate::eq),
688 adaptor.getInput(), zeroInputTy);
689
690 auto zero = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 0);
691 auto res = rewriter.create<mlir::LLVM::SelectOp>(op.getLoc(), isZero, zero,
692 ctzAddOne);
693 rewriter.replaceOp(op, res);
694
695 return mlir::LogicalResult::success();
696}
697
699 cir::BitParityOp op, OpAdaptor adaptor,
700 mlir::ConversionPatternRewriter &rewriter) const {
701 auto resTy = getTypeConverter()->convertType(op.getType());
702 auto popcnt = rewriter.create<mlir::LLVM::CtPopOp>(op.getLoc(), resTy,
703 adaptor.getInput());
704
705 auto one = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 1);
706 auto popcntMod2 =
707 rewriter.create<mlir::LLVM::AndOp>(op.getLoc(), popcnt, one);
708 rewriter.replaceOp(op, popcntMod2);
709
710 return mlir::LogicalResult::success();
711}
712
714 cir::BitPopcountOp op, OpAdaptor adaptor,
715 mlir::ConversionPatternRewriter &rewriter) const {
716 auto resTy = getTypeConverter()->convertType(op.getType());
717 auto llvmOp = rewriter.create<mlir::LLVM::CtPopOp>(op.getLoc(), resTy,
718 adaptor.getInput());
719 rewriter.replaceOp(op, llvmOp);
720 return mlir::LogicalResult::success();
721}
722
724 cir::BitReverseOp op, OpAdaptor adaptor,
725 mlir::ConversionPatternRewriter &rewriter) const {
726 rewriter.replaceOpWithNewOp<mlir::LLVM::BitReverseOp>(op, adaptor.getInput());
727 return mlir::success();
728}
729
731 cir::BrCondOp brOp, OpAdaptor adaptor,
732 mlir::ConversionPatternRewriter &rewriter) const {
733 // When ZExtOp is implemented, we'll need to check if the condition is a
734 // ZExtOp and if so, delete it if it has a single use.
736
737 mlir::Value i1Condition = adaptor.getCond();
738
739 rewriter.replaceOpWithNewOp<mlir::LLVM::CondBrOp>(
740 brOp, i1Condition, brOp.getDestTrue(), adaptor.getDestOperandsTrue(),
741 brOp.getDestFalse(), adaptor.getDestOperandsFalse());
742
743 return mlir::success();
744}
745
747 cir::ByteSwapOp op, OpAdaptor adaptor,
748 mlir::ConversionPatternRewriter &rewriter) const {
749 rewriter.replaceOpWithNewOp<mlir::LLVM::ByteSwapOp>(op, adaptor.getInput());
750 return mlir::LogicalResult::success();
751}
752
753mlir::Type CIRToLLVMCastOpLowering::convertTy(mlir::Type ty) const {
754 return getTypeConverter()->convertType(ty);
755}
756
758 cir::CastOp castOp, OpAdaptor adaptor,
759 mlir::ConversionPatternRewriter &rewriter) const {
760 // For arithmetic conversions, LLVM IR uses the same instruction to convert
761 // both individual scalars and entire vectors. This lowering pass handles
762 // both situations.
763
764 switch (castOp.getKind()) {
765 case cir::CastKind::array_to_ptrdecay: {
766 const auto ptrTy = mlir::cast<cir::PointerType>(castOp.getType());
767 mlir::Value sourceValue = adaptor.getSrc();
768 mlir::Type targetType = convertTy(ptrTy);
769 mlir::Type elementTy = convertTypeForMemory(*getTypeConverter(), dataLayout,
770 ptrTy.getPointee());
772 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
773 castOp, targetType, elementTy, sourceValue, offset);
774 break;
775 }
776 case cir::CastKind::int_to_bool: {
777 mlir::Value llvmSrcVal = adaptor.getSrc();
778 mlir::Value zeroInt = rewriter.create<mlir::LLVM::ConstantOp>(
779 castOp.getLoc(), llvmSrcVal.getType(), 0);
780 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
781 castOp, mlir::LLVM::ICmpPredicate::ne, llvmSrcVal, zeroInt);
782 break;
783 }
784 case cir::CastKind::integral: {
785 mlir::Type srcType = castOp.getSrc().getType();
786 mlir::Type dstType = castOp.getType();
787 mlir::Value llvmSrcVal = adaptor.getSrc();
788 mlir::Type llvmDstType = getTypeConverter()->convertType(dstType);
789 cir::IntType srcIntType =
790 mlir::cast<cir::IntType>(elementTypeIfVector(srcType));
791 cir::IntType dstIntType =
792 mlir::cast<cir::IntType>(elementTypeIfVector(dstType));
793 rewriter.replaceOp(castOp, getLLVMIntCast(rewriter, llvmSrcVal, llvmDstType,
794 srcIntType.isUnsigned(),
795 srcIntType.getWidth(),
796 dstIntType.getWidth()));
797 break;
798 }
799 case cir::CastKind::floating: {
800 mlir::Value llvmSrcVal = adaptor.getSrc();
801 mlir::Type llvmDstTy = getTypeConverter()->convertType(castOp.getType());
802
803 mlir::Type srcTy = elementTypeIfVector(castOp.getSrc().getType());
804 mlir::Type dstTy = elementTypeIfVector(castOp.getType());
805
806 if (!mlir::isa<cir::FPTypeInterface>(dstTy) ||
807 !mlir::isa<cir::FPTypeInterface>(srcTy))
808 return castOp.emitError() << "NYI cast from " << srcTy << " to " << dstTy;
809
810 auto getFloatWidth = [](mlir::Type ty) -> unsigned {
811 return mlir::cast<cir::FPTypeInterface>(ty).getWidth();
812 };
813
814 if (getFloatWidth(srcTy) > getFloatWidth(dstTy))
815 rewriter.replaceOpWithNewOp<mlir::LLVM::FPTruncOp>(castOp, llvmDstTy,
816 llvmSrcVal);
817 else
818 rewriter.replaceOpWithNewOp<mlir::LLVM::FPExtOp>(castOp, llvmDstTy,
819 llvmSrcVal);
820 return mlir::success();
821 }
822 case cir::CastKind::int_to_ptr: {
823 auto dstTy = mlir::cast<cir::PointerType>(castOp.getType());
824 mlir::Value llvmSrcVal = adaptor.getSrc();
825 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
826 rewriter.replaceOpWithNewOp<mlir::LLVM::IntToPtrOp>(castOp, llvmDstTy,
827 llvmSrcVal);
828 return mlir::success();
829 }
830 case cir::CastKind::ptr_to_int: {
831 auto dstTy = mlir::cast<cir::IntType>(castOp.getType());
832 mlir::Value llvmSrcVal = adaptor.getSrc();
833 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
834 rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(castOp, llvmDstTy,
835 llvmSrcVal);
836 return mlir::success();
837 }
838 case cir::CastKind::float_to_bool: {
839 mlir::Value llvmSrcVal = adaptor.getSrc();
840 auto kind = mlir::LLVM::FCmpPredicate::une;
841
842 // Check if float is not equal to zero.
843 auto zeroFloat = rewriter.create<mlir::LLVM::ConstantOp>(
844 castOp.getLoc(), llvmSrcVal.getType(),
845 mlir::FloatAttr::get(llvmSrcVal.getType(), 0.0));
846
847 // Extend comparison result to either bool (C++) or int (C).
848 rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(castOp, kind, llvmSrcVal,
849 zeroFloat);
850
851 return mlir::success();
852 }
853 case cir::CastKind::bool_to_int: {
854 auto dstTy = mlir::cast<cir::IntType>(castOp.getType());
855 mlir::Value llvmSrcVal = adaptor.getSrc();
856 auto llvmSrcTy = mlir::cast<mlir::IntegerType>(llvmSrcVal.getType());
857 auto llvmDstTy =
858 mlir::cast<mlir::IntegerType>(getTypeConverter()->convertType(dstTy));
859 if (llvmSrcTy.getWidth() == llvmDstTy.getWidth())
860 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(castOp, llvmDstTy,
861 llvmSrcVal);
862 else
863 rewriter.replaceOpWithNewOp<mlir::LLVM::ZExtOp>(castOp, llvmDstTy,
864 llvmSrcVal);
865 return mlir::success();
866 }
867 case cir::CastKind::bool_to_float: {
868 mlir::Type dstTy = castOp.getType();
869 mlir::Value llvmSrcVal = adaptor.getSrc();
870 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
871 rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(castOp, llvmDstTy,
872 llvmSrcVal);
873 return mlir::success();
874 }
875 case cir::CastKind::int_to_float: {
876 mlir::Type dstTy = castOp.getType();
877 mlir::Value llvmSrcVal = adaptor.getSrc();
878 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
879 if (mlir::cast<cir::IntType>(elementTypeIfVector(castOp.getSrc().getType()))
880 .isSigned())
881 rewriter.replaceOpWithNewOp<mlir::LLVM::SIToFPOp>(castOp, llvmDstTy,
882 llvmSrcVal);
883 else
884 rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(castOp, llvmDstTy,
885 llvmSrcVal);
886 return mlir::success();
887 }
888 case cir::CastKind::float_to_int: {
889 mlir::Type dstTy = castOp.getType();
890 mlir::Value llvmSrcVal = adaptor.getSrc();
891 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
892 if (mlir::cast<cir::IntType>(elementTypeIfVector(castOp.getType()))
893 .isSigned())
894 rewriter.replaceOpWithNewOp<mlir::LLVM::FPToSIOp>(castOp, llvmDstTy,
895 llvmSrcVal);
896 else
897 rewriter.replaceOpWithNewOp<mlir::LLVM::FPToUIOp>(castOp, llvmDstTy,
898 llvmSrcVal);
899 return mlir::success();
900 }
901 case cir::CastKind::bitcast: {
902 mlir::Type dstTy = castOp.getType();
903 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
904
905 assert(!MissingFeatures::cxxABI());
907
908 mlir::Value llvmSrcVal = adaptor.getSrc();
909 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(castOp, llvmDstTy,
910 llvmSrcVal);
911 return mlir::success();
912 }
913 case cir::CastKind::ptr_to_bool: {
914 mlir::Value llvmSrcVal = adaptor.getSrc();
915 mlir::Value zeroPtr = rewriter.create<mlir::LLVM::ZeroOp>(
916 castOp.getLoc(), llvmSrcVal.getType());
917 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
918 castOp, mlir::LLVM::ICmpPredicate::ne, llvmSrcVal, zeroPtr);
919 break;
920 }
921 case cir::CastKind::address_space: {
922 mlir::Type dstTy = castOp.getType();
923 mlir::Value llvmSrcVal = adaptor.getSrc();
924 mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
925 rewriter.replaceOpWithNewOp<mlir::LLVM::AddrSpaceCastOp>(castOp, llvmDstTy,
926 llvmSrcVal);
927 break;
928 }
929 case cir::CastKind::member_ptr_to_bool:
930 assert(!MissingFeatures::cxxABI());
932 break;
933 default: {
934 return castOp.emitError("Unhandled cast kind: ")
935 << castOp.getKindAttrName();
936 }
937 }
938
939 return mlir::success();
940}
941
943 cir::PtrStrideOp ptrStrideOp, OpAdaptor adaptor,
944 mlir::ConversionPatternRewriter &rewriter) const {
945
946 const mlir::TypeConverter *tc = getTypeConverter();
947 const mlir::Type resultTy = tc->convertType(ptrStrideOp.getType());
948
949 mlir::Type elementTy =
950 convertTypeForMemory(*tc, dataLayout, ptrStrideOp.getElementTy());
951 mlir::MLIRContext *ctx = elementTy.getContext();
952
953 // void and function types doesn't really have a layout to use in GEPs,
954 // make it i8 instead.
955 if (mlir::isa<mlir::LLVM::LLVMVoidType>(elementTy) ||
956 mlir::isa<mlir::LLVM::LLVMFunctionType>(elementTy))
957 elementTy = mlir::IntegerType::get(elementTy.getContext(), 8,
958 mlir::IntegerType::Signless);
959 // Zero-extend, sign-extend or trunc the pointer value.
960 mlir::Value index = adaptor.getStride();
961 const unsigned width =
962 mlir::cast<mlir::IntegerType>(index.getType()).getWidth();
963 const std::optional<std::uint64_t> layoutWidth =
964 dataLayout.getTypeIndexBitwidth(adaptor.getBase().getType());
965
966 mlir::Operation *indexOp = index.getDefiningOp();
967 if (indexOp && layoutWidth && width != *layoutWidth) {
968 // If the index comes from a subtraction, make sure the extension happens
969 // before it. To achieve that, look at unary minus, which already got
970 // lowered to "sub 0, x".
971 const auto sub = dyn_cast<mlir::LLVM::SubOp>(indexOp);
972 auto unary = ptrStrideOp.getStride().getDefiningOp<cir::UnaryOp>();
973 bool rewriteSub =
974 unary && unary.getKind() == cir::UnaryOpKind::Minus && sub;
975 if (rewriteSub)
976 index = indexOp->getOperand(1);
977
978 // Handle the cast
979 const auto llvmDstType = mlir::IntegerType::get(ctx, *layoutWidth);
980 index = getLLVMIntCast(rewriter, index, llvmDstType,
981 ptrStrideOp.getStride().getType().isUnsigned(),
982 width, *layoutWidth);
983
984 // Rewrite the sub in front of extensions/trunc
985 if (rewriteSub) {
986 index = rewriter.create<mlir::LLVM::SubOp>(
987 index.getLoc(), index.getType(),
988 rewriter.create<mlir::LLVM::ConstantOp>(index.getLoc(),
989 index.getType(), 0),
990 index);
991 rewriter.eraseOp(sub);
992 }
993 }
994
995 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
996 ptrStrideOp, resultTy, elementTy, adaptor.getBase(), index);
997 return mlir::success();
998}
999
1001 cir::BaseClassAddrOp baseClassOp, OpAdaptor adaptor,
1002 mlir::ConversionPatternRewriter &rewriter) const {
1003 const mlir::Type resultType =
1004 getTypeConverter()->convertType(baseClassOp.getType());
1005 mlir::Value derivedAddr = adaptor.getDerivedAddr();
1007 adaptor.getOffset().getZExtValue()};
1008 mlir::Type byteType = mlir::IntegerType::get(resultType.getContext(), 8,
1009 mlir::IntegerType::Signless);
1010 if (adaptor.getOffset().getZExtValue() == 0) {
1011 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(
1012 baseClassOp, resultType, adaptor.getDerivedAddr());
1013 return mlir::success();
1014 }
1015
1016 if (baseClassOp.getAssumeNotNull()) {
1017 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
1018 baseClassOp, resultType, byteType, derivedAddr, offset);
1019 } else {
1020 auto loc = baseClassOp.getLoc();
1021 mlir::Value isNull = rewriter.create<mlir::LLVM::ICmpOp>(
1022 loc, mlir::LLVM::ICmpPredicate::eq, derivedAddr,
1023 rewriter.create<mlir::LLVM::ZeroOp>(loc, derivedAddr.getType()));
1024 mlir::Value adjusted = rewriter.create<mlir::LLVM::GEPOp>(
1025 loc, resultType, byteType, derivedAddr, offset);
1026 rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(baseClassOp, isNull,
1027 derivedAddr, adjusted);
1028 }
1029 return mlir::success();
1030}
1031
1033 cir::AllocaOp op, OpAdaptor adaptor,
1034 mlir::ConversionPatternRewriter &rewriter) const {
1036 mlir::Value size = rewriter.create<mlir::LLVM::ConstantOp>(
1037 op.getLoc(), typeConverter->convertType(rewriter.getIndexType()), 1);
1038 mlir::Type elementTy =
1039 convertTypeForMemory(*getTypeConverter(), dataLayout, op.getAllocaType());
1040 mlir::Type resultTy =
1041 convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType());
1042
1045
1046 rewriter.replaceOpWithNewOp<mlir::LLVM::AllocaOp>(
1047 op, resultTy, elementTy, size, op.getAlignmentAttr().getInt());
1048
1049 return mlir::success();
1050}
1051
1053 cir::ReturnOp op, OpAdaptor adaptor,
1054 mlir::ConversionPatternRewriter &rewriter) const {
1055 rewriter.replaceOpWithNewOp<mlir::LLVM::ReturnOp>(op, adaptor.getOperands());
1056 return mlir::LogicalResult::success();
1057}
1058
1060 cir::RotateOp op, OpAdaptor adaptor,
1061 mlir::ConversionPatternRewriter &rewriter) const {
1062 // Note that LLVM intrinsic calls to @llvm.fsh{r,l}.i* have the same type as
1063 // the operand.
1064 mlir::Value input = adaptor.getInput();
1065 if (op.isRotateLeft())
1066 rewriter.replaceOpWithNewOp<mlir::LLVM::FshlOp>(op, input, input,
1067 adaptor.getAmount());
1068 else
1069 rewriter.replaceOpWithNewOp<mlir::LLVM::FshrOp>(op, input, input,
1070 adaptor.getAmount());
1071 return mlir::LogicalResult::success();
1072}
1073
1074static mlir::LogicalResult
1075rewriteCallOrInvoke(mlir::Operation *op, mlir::ValueRange callOperands,
1076 mlir::ConversionPatternRewriter &rewriter,
1077 const mlir::TypeConverter *converter,
1078 mlir::FlatSymbolRefAttr calleeAttr) {
1080 mlir::ValueTypeRange<mlir::ResultRange> cirResults = op->getResultTypes();
1081 auto call = cast<cir::CIRCallOpInterface>(op);
1082
1083 if (converter->convertTypes(cirResults, llvmResults).failed())
1084 return mlir::failure();
1085
1087
1088 mlir::LLVM::MemoryEffectsAttr memoryEffects;
1089 bool noUnwind = false;
1090 bool willReturn = false;
1091 convertSideEffectForCall(op, call.getNothrow(), call.getSideEffect(),
1092 memoryEffects, noUnwind, willReturn);
1093
1094 mlir::LLVM::LLVMFunctionType llvmFnTy;
1095
1096 // Temporary to handle the case where we need to prepend an operand if the
1097 // callee is an alias.
1098 SmallVector<mlir::Value> adjustedCallOperands;
1099
1100 if (calleeAttr) { // direct call
1101 mlir::Operation *callee =
1102 mlir::SymbolTable::lookupNearestSymbolFrom(op, calleeAttr);
1103 if (auto fn = mlir::dyn_cast<mlir::FunctionOpInterface>(callee)) {
1104 llvmFnTy = converter->convertType<mlir::LLVM::LLVMFunctionType>(
1105 fn.getFunctionType());
1106 assert(llvmFnTy && "Failed to convert function type");
1107 } else if (auto alias = mlir::cast<mlir::LLVM::AliasOp>(callee)) {
1108 // If the callee was an alias. In that case,
1109 // we need to prepend the address of the alias to the operands. The
1110 // way aliases work in the LLVM dialect is a little counter-intuitive.
1111 // The AliasOp itself is a pseudo-function that returns the address of
1112 // the global value being aliased, but when we generate the call we
1113 // need to insert an operation that gets the address of the AliasOp.
1114 // This all gets sorted out when the LLVM dialect is lowered to LLVM IR.
1115 auto symAttr = mlir::cast<mlir::FlatSymbolRefAttr>(calleeAttr);
1116 auto addrOfAlias =
1117 mlir::LLVM::AddressOfOp::create(
1118 rewriter, op->getLoc(),
1119 mlir::LLVM::LLVMPointerType::get(rewriter.getContext()), symAttr)
1120 .getResult();
1121 adjustedCallOperands.push_back(addrOfAlias);
1122
1123 // Now add the regular operands and assign this to the range value.
1124 llvm::append_range(adjustedCallOperands, callOperands);
1125 callOperands = adjustedCallOperands;
1126
1127 // Clear the callee attribute because we're calling an alias.
1128 calleeAttr = {};
1129 llvmFnTy = mlir::cast<mlir::LLVM::LLVMFunctionType>(alias.getType());
1130 } else {
1131 // Was this an ifunc?
1132 return op->emitError("Unexpected callee type!");
1133 }
1134 } else { // indirect call
1135 assert(!op->getOperands().empty() &&
1136 "operands list must no be empty for the indirect call");
1137 auto calleeTy = op->getOperands().front().getType();
1138 auto calleePtrTy = cast<cir::PointerType>(calleeTy);
1139 auto calleeFuncTy = cast<cir::FuncType>(calleePtrTy.getPointee());
1140 llvm::append_range(adjustedCallOperands, callOperands);
1141 llvmFnTy = cast<mlir::LLVM::LLVMFunctionType>(
1142 converter->convertType(calleeFuncTy));
1143 }
1144
1148
1149 auto newOp = rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
1150 op, llvmFnTy, calleeAttr, callOperands);
1151 if (memoryEffects)
1152 newOp.setMemoryEffectsAttr(memoryEffects);
1153 newOp.setNoUnwind(noUnwind);
1154 newOp.setWillReturn(willReturn);
1155
1156 return mlir::success();
1157}
1158
1160 cir::CallOp op, OpAdaptor adaptor,
1161 mlir::ConversionPatternRewriter &rewriter) const {
1162 return rewriteCallOrInvoke(op.getOperation(), adaptor.getOperands(), rewriter,
1163 getTypeConverter(), op.getCalleeAttr());
1164}
1165
1167 cir::ReturnAddrOp op, OpAdaptor adaptor,
1168 mlir::ConversionPatternRewriter &rewriter) const {
1169 auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
1170 replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.returnaddress",
1171 llvmPtrTy, adaptor.getOperands());
1172 return mlir::success();
1173}
1174
1176 cir::FrameAddrOp op, OpAdaptor adaptor,
1177 mlir::ConversionPatternRewriter &rewriter) const {
1178 auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
1179 replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.frameaddress", llvmPtrTy,
1180 adaptor.getOperands());
1181 return mlir::success();
1182}
1183
1184static mlir::LLVM::AtomicOrdering
1185getLLVMMemOrder(std::optional<cir::MemOrder> memorder) {
1186 if (!memorder)
1187 return mlir::LLVM::AtomicOrdering::not_atomic;
1188 switch (*memorder) {
1189 case cir::MemOrder::Relaxed:
1190 return mlir::LLVM::AtomicOrdering::monotonic;
1191 case cir::MemOrder::Consume:
1192 case cir::MemOrder::Acquire:
1193 return mlir::LLVM::AtomicOrdering::acquire;
1194 case cir::MemOrder::Release:
1195 return mlir::LLVM::AtomicOrdering::release;
1196 case cir::MemOrder::AcquireRelease:
1197 return mlir::LLVM::AtomicOrdering::acq_rel;
1198 case cir::MemOrder::SequentiallyConsistent:
1199 return mlir::LLVM::AtomicOrdering::seq_cst;
1200 }
1201 llvm_unreachable("unknown memory order");
1202}
1203
1205 cir::LoadOp op, OpAdaptor adaptor,
1206 mlir::ConversionPatternRewriter &rewriter) const {
1207 const mlir::Type llvmTy =
1208 convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType());
1209 mlir::LLVM::AtomicOrdering ordering = getLLVMMemOrder(op.getMemOrder());
1210 std::optional<size_t> opAlign = op.getAlignment();
1211 unsigned alignment =
1212 (unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmTy));
1213
1215
1216 // TODO: nontemporal, syncscope.
1218 mlir::LLVM::LoadOp newLoad = mlir::LLVM::LoadOp::create(
1219 rewriter, op->getLoc(), llvmTy, adaptor.getAddr(), alignment,
1220 op.getIsVolatile(), /*isNonTemporal=*/false,
1221 /*isInvariant=*/false, /*isInvariantGroup=*/false, ordering);
1222
1223 // Convert adapted result to its original type if needed.
1224 mlir::Value result =
1225 emitFromMemory(rewriter, dataLayout, op, newLoad.getResult());
1226 rewriter.replaceOp(op, result);
1228 return mlir::LogicalResult::success();
1229}
1230
1232 cir::StoreOp op, OpAdaptor adaptor,
1233 mlir::ConversionPatternRewriter &rewriter) const {
1234 mlir::LLVM::AtomicOrdering memorder = getLLVMMemOrder(op.getMemOrder());
1235 const mlir::Type llvmTy =
1236 getTypeConverter()->convertType(op.getValue().getType());
1237 std::optional<size_t> opAlign = op.getAlignment();
1238 unsigned alignment =
1239 (unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmTy));
1240
1242
1243 // Convert adapted value to its memory type if needed.
1244 mlir::Value value = emitToMemory(rewriter, dataLayout,
1245 op.getValue().getType(), adaptor.getValue());
1246 // TODO: nontemporal, syncscope.
1249 mlir::LLVM::StoreOp storeOp = mlir::LLVM::StoreOp::create(
1250 rewriter, op->getLoc(), value, adaptor.getAddr(), alignment,
1251 op.getIsVolatile(),
1252 /*isNonTemporal=*/false, /*isInvariantGroup=*/false, memorder);
1253 rewriter.replaceOp(op, storeOp);
1255 return mlir::LogicalResult::success();
1256}
1257
1258bool hasTrailingZeros(cir::ConstArrayAttr attr) {
1259 auto array = mlir::dyn_cast<mlir::ArrayAttr>(attr.getElts());
1260 return attr.hasTrailingZeros() ||
1261 (array && std::count_if(array.begin(), array.end(), [](auto elt) {
1262 auto ar = dyn_cast<cir::ConstArrayAttr>(elt);
1263 return ar && hasTrailingZeros(ar);
1264 }));
1265}
1266
1268 cir::ConstantOp op, OpAdaptor adaptor,
1269 mlir::ConversionPatternRewriter &rewriter) const {
1270 mlir::Attribute attr = op.getValue();
1271
1272 if (mlir::isa<cir::PoisonAttr>(attr)) {
1273 rewriter.replaceOpWithNewOp<mlir::LLVM::PoisonOp>(
1274 op, getTypeConverter()->convertType(op.getType()));
1275 return mlir::success();
1276 }
1277
1278 if (mlir::isa<mlir::IntegerType>(op.getType())) {
1279 // Verified cir.const operations cannot actually be of these types, but the
1280 // lowering pass may generate temporary cir.const operations with these
1281 // types. This is OK since MLIR allows unverified operations to be alive
1282 // during a pass as long as they don't live past the end of the pass.
1283 attr = op.getValue();
1284 } else if (mlir::isa<cir::BoolType>(op.getType())) {
1285 int value = mlir::cast<cir::BoolAttr>(op.getValue()).getValue();
1286 attr = rewriter.getIntegerAttr(typeConverter->convertType(op.getType()),
1287 value);
1288 } else if (mlir::isa<cir::IntType>(op.getType())) {
1289 // Lower GlobalViewAttr to llvm.mlir.addressof + llvm.mlir.ptrtoint
1290 if (auto ga = mlir::dyn_cast<cir::GlobalViewAttr>(op.getValue())) {
1291 // See the comment in visitCirAttr for why this isn't implemented.
1293 op.emitError() << "global view with integer type";
1294 return mlir::failure();
1295 }
1296
1297 attr = rewriter.getIntegerAttr(
1298 typeConverter->convertType(op.getType()),
1299 mlir::cast<cir::IntAttr>(op.getValue()).getValue());
1300 } else if (mlir::isa<cir::FPTypeInterface>(op.getType())) {
1301 attr = rewriter.getFloatAttr(
1302 typeConverter->convertType(op.getType()),
1303 mlir::cast<cir::FPAttr>(op.getValue()).getValue());
1304 } else if (mlir::isa<cir::PointerType>(op.getType())) {
1305 // Optimize with dedicated LLVM op for null pointers.
1306 if (mlir::isa<cir::ConstPtrAttr>(op.getValue())) {
1307 if (mlir::cast<cir::ConstPtrAttr>(op.getValue()).isNullValue()) {
1308 rewriter.replaceOpWithNewOp<mlir::LLVM::ZeroOp>(
1309 op, typeConverter->convertType(op.getType()));
1310 return mlir::success();
1311 }
1312 }
1313 // Lower GlobalViewAttr to llvm.mlir.addressof
1314 if (auto gv = mlir::dyn_cast<cir::GlobalViewAttr>(op.getValue())) {
1315 auto newOp = lowerCirAttrAsValue(op, gv, rewriter, getTypeConverter());
1316 rewriter.replaceOp(op, newOp);
1317 return mlir::success();
1318 }
1319 attr = op.getValue();
1320 } else if (const auto arrTy = mlir::dyn_cast<cir::ArrayType>(op.getType())) {
1321 const auto constArr = mlir::dyn_cast<cir::ConstArrayAttr>(op.getValue());
1322 if (!constArr && !isa<cir::ZeroAttr, cir::UndefAttr>(op.getValue()))
1323 return op.emitError() << "array does not have a constant initializer";
1324
1325 std::optional<mlir::Attribute> denseAttr;
1326 if (constArr && hasTrailingZeros(constArr)) {
1327 const mlir::Value newOp =
1328 lowerCirAttrAsValue(op, constArr, rewriter, getTypeConverter());
1329 rewriter.replaceOp(op, newOp);
1330 return mlir::success();
1331 } else if (constArr &&
1332 (denseAttr = lowerConstArrayAttr(constArr, typeConverter))) {
1333 attr = denseAttr.value();
1334 } else {
1335 const mlir::Value initVal =
1336 lowerCirAttrAsValue(op, op.getValue(), rewriter, typeConverter);
1337 rewriter.replaceAllUsesWith(op, initVal);
1338 rewriter.eraseOp(op);
1339 return mlir::success();
1340 }
1341 } else if (const auto recordAttr =
1342 mlir::dyn_cast<cir::ConstRecordAttr>(op.getValue())) {
1343 auto initVal = lowerCirAttrAsValue(op, recordAttr, rewriter, typeConverter);
1344 rewriter.replaceOp(op, initVal);
1345 return mlir::success();
1346 } else if (const auto vecTy = mlir::dyn_cast<cir::VectorType>(op.getType())) {
1347 rewriter.replaceOp(op, lowerCirAttrAsValue(op, op.getValue(), rewriter,
1348 getTypeConverter()));
1349 return mlir::success();
1350 } else if (auto complexTy = mlir::dyn_cast<cir::ComplexType>(op.getType())) {
1351 mlir::Type complexElemTy = complexTy.getElementType();
1352 mlir::Type complexElemLLVMTy = typeConverter->convertType(complexElemTy);
1353
1354 if (auto zeroInitAttr = mlir::dyn_cast<cir::ZeroAttr>(op.getValue())) {
1355 mlir::TypedAttr zeroAttr = rewriter.getZeroAttr(complexElemLLVMTy);
1356 mlir::ArrayAttr array = rewriter.getArrayAttr({zeroAttr, zeroAttr});
1357 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
1358 op, getTypeConverter()->convertType(op.getType()), array);
1359 return mlir::success();
1360 }
1361
1362 auto complexAttr = mlir::cast<cir::ConstComplexAttr>(op.getValue());
1363
1364 mlir::Attribute components[2];
1365 if (mlir::isa<cir::IntType>(complexElemTy)) {
1366 components[0] = rewriter.getIntegerAttr(
1367 complexElemLLVMTy,
1368 mlir::cast<cir::IntAttr>(complexAttr.getReal()).getValue());
1369 components[1] = rewriter.getIntegerAttr(
1370 complexElemLLVMTy,
1371 mlir::cast<cir::IntAttr>(complexAttr.getImag()).getValue());
1372 } else {
1373 components[0] = rewriter.getFloatAttr(
1374 complexElemLLVMTy,
1375 mlir::cast<cir::FPAttr>(complexAttr.getReal()).getValue());
1376 components[1] = rewriter.getFloatAttr(
1377 complexElemLLVMTy,
1378 mlir::cast<cir::FPAttr>(complexAttr.getImag()).getValue());
1379 }
1380
1381 attr = rewriter.getArrayAttr(components);
1382 } else {
1383 return op.emitError() << "unsupported constant type " << op.getType();
1384 }
1385
1386 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
1387 op, getTypeConverter()->convertType(op.getType()), attr);
1388
1389 return mlir::success();
1390}
1391
1393 cir::ExpectOp op, OpAdaptor adaptor,
1394 mlir::ConversionPatternRewriter &rewriter) const {
1395 // TODO(cir): do not generate LLVM intrinsics under -O0
1397
1398 std::optional<llvm::APFloat> prob = op.getProb();
1399 if (prob)
1400 rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectWithProbabilityOp>(
1401 op, adaptor.getVal(), adaptor.getExpected(), prob.value());
1402 else
1403 rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectOp>(op, adaptor.getVal(),
1404 adaptor.getExpected());
1405 return mlir::success();
1406}
1407
1409 cir::FAbsOp op, OpAdaptor adaptor,
1410 mlir::ConversionPatternRewriter &rewriter) const {
1411 mlir::Type resTy = typeConverter->convertType(op.getType());
1412 rewriter.replaceOpWithNewOp<mlir::LLVM::FAbsOp>(op, resTy,
1413 adaptor.getOperands()[0]);
1414 return mlir::success();
1415}
1416
1417/// Convert the `cir.func` attributes to `llvm.func` attributes.
1418/// Only retain those attributes that are not constructed by
1419/// `LLVMFuncOp::build`. If `filterArgAttrs` is set, also filter out
1420/// argument attributes.
1421void CIRToLLVMFuncOpLowering::lowerFuncAttributes(
1422 cir::FuncOp func, bool filterArgAndResAttrs,
1425 for (mlir::NamedAttribute attr : func->getAttrs()) {
1427 if (attr.getName() == mlir::SymbolTable::getSymbolAttrName() ||
1428 attr.getName() == func.getFunctionTypeAttrName() ||
1429 attr.getName() == getLinkageAttrNameString() ||
1430 attr.getName() == func.getGlobalVisibilityAttrName() ||
1431 attr.getName() == func.getDsoLocalAttrName() ||
1432 (filterArgAndResAttrs &&
1433 (attr.getName() == func.getArgAttrsAttrName() ||
1434 attr.getName() == func.getResAttrsAttrName())))
1435 continue;
1436
1438 result.push_back(attr);
1439 }
1440}
1441
1442mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewriteAlias(
1443 cir::FuncOp op, llvm::StringRef aliasee, mlir::Type ty, OpAdaptor adaptor,
1444 mlir::ConversionPatternRewriter &rewriter) const {
1446 lowerFuncAttributes(op, /*filterArgAndResAttrs=*/false, attributes);
1447
1448 mlir::Location loc = op.getLoc();
1449 auto aliasOp = rewriter.replaceOpWithNewOp<mlir::LLVM::AliasOp>(
1450 op, ty, convertLinkage(op.getLinkage()), op.getName(), op.getDsoLocal(),
1451 /*threadLocal=*/false, attributes);
1452
1453 // Create the alias body
1454 mlir::OpBuilder builder(op.getContext());
1455 mlir::Block *block = builder.createBlock(&aliasOp.getInitializerRegion());
1456 builder.setInsertionPointToStart(block);
1457 // The type of AddressOfOp is always a pointer.
1459 mlir::Type ptrTy = mlir::LLVM::LLVMPointerType::get(ty.getContext());
1460 auto addrOp = mlir::LLVM::AddressOfOp::create(builder, loc, ptrTy, aliasee);
1461 mlir::LLVM::ReturnOp::create(builder, loc, addrOp);
1462
1463 return mlir::success();
1464}
1465
1467 cir::FuncOp op, OpAdaptor adaptor,
1468 mlir::ConversionPatternRewriter &rewriter) const {
1469
1470 cir::FuncType fnType = op.getFunctionType();
1471 bool isDsoLocal = op.getDsoLocal();
1472 mlir::TypeConverter::SignatureConversion signatureConversion(
1473 fnType.getNumInputs());
1474
1475 for (const auto &argType : llvm::enumerate(fnType.getInputs())) {
1476 mlir::Type convertedType = typeConverter->convertType(argType.value());
1477 if (!convertedType)
1478 return mlir::failure();
1479 signatureConversion.addInputs(argType.index(), convertedType);
1480 }
1481
1482 mlir::Type resultType =
1483 getTypeConverter()->convertType(fnType.getReturnType());
1484
1485 // Create the LLVM function operation.
1486 mlir::Type llvmFnTy = mlir::LLVM::LLVMFunctionType::get(
1487 resultType ? resultType : mlir::LLVM::LLVMVoidType::get(getContext()),
1488 signatureConversion.getConvertedTypes(),
1489 /*isVarArg=*/fnType.isVarArg());
1490
1491 // If this is an alias, it needs to be lowered to llvm::AliasOp.
1492 if (std::optional<llvm::StringRef> aliasee = op.getAliasee())
1493 return matchAndRewriteAlias(op, *aliasee, llvmFnTy, adaptor, rewriter);
1494
1495 // LLVMFuncOp expects a single FileLine Location instead of a fused
1496 // location.
1497 mlir::Location loc = op.getLoc();
1498 if (mlir::FusedLoc fusedLoc = mlir::dyn_cast<mlir::FusedLoc>(loc))
1499 loc = fusedLoc.getLocations()[0];
1500 assert((mlir::isa<mlir::FileLineColLoc>(loc) ||
1501 mlir::isa<mlir::UnknownLoc>(loc)) &&
1502 "expected single location or unknown location here");
1503
1504 mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());
1506 mlir::LLVM::CConv cconv = mlir::LLVM::CConv::C;
1508 lowerFuncAttributes(op, /*filterArgAndResAttrs=*/false, attributes);
1509
1510 mlir::LLVM::LLVMFuncOp fn = rewriter.create<mlir::LLVM::LLVMFuncOp>(
1511 loc, op.getName(), llvmFnTy, linkage, isDsoLocal, cconv,
1512 mlir::SymbolRefAttr(), attributes);
1513
1515
1516 fn.setVisibility_Attr(mlir::LLVM::VisibilityAttr::get(
1518 op.getGlobalVisibilityAttr().getValue())));
1519
1520 rewriter.inlineRegionBefore(op.getBody(), fn.getBody(), fn.end());
1521 if (failed(rewriter.convertRegionTypes(&fn.getBody(), *typeConverter,
1522 &signatureConversion)))
1523 return mlir::failure();
1524
1525 rewriter.eraseOp(op);
1526
1527 return mlir::LogicalResult::success();
1528}
1529
1531 cir::GetGlobalOp op, OpAdaptor adaptor,
1532 mlir::ConversionPatternRewriter &rewriter) const {
1533 // FIXME(cir): Premature DCE to avoid lowering stuff we're not using.
1534 // CIRGen should mitigate this and not emit the get_global.
1535 if (op->getUses().empty()) {
1536 rewriter.eraseOp(op);
1537 return mlir::success();
1538 }
1539
1540 mlir::Type type = getTypeConverter()->convertType(op.getType());
1541 mlir::Operation *newop =
1542 rewriter.create<mlir::LLVM::AddressOfOp>(op.getLoc(), type, op.getName());
1543
1545
1546 rewriter.replaceOp(op, newop);
1547 return mlir::success();
1548}
1549
1550/// Replace CIR global with a region initialized LLVM global and update
1551/// insertion point to the end of the initializer block.
1552void CIRToLLVMGlobalOpLowering::setupRegionInitializedLLVMGlobalOp(
1553 cir::GlobalOp op, mlir::ConversionPatternRewriter &rewriter) const {
1554 const mlir::Type llvmType =
1555 convertTypeForMemory(*getTypeConverter(), dataLayout, op.getSymType());
1556
1557 // FIXME: These default values are placeholders until the the equivalent
1558 // attributes are available on cir.global ops. This duplicates code
1559 // in CIRToLLVMGlobalOpLowering::matchAndRewrite() but that will go
1560 // away when the placeholders are no longer needed.
1562 const bool isConst = op.getConstant();
1564 const unsigned addrSpace = 0;
1565 const bool isDsoLocal = op.getDsoLocal();
1567 const bool isThreadLocal = false;
1568 const uint64_t alignment = op.getAlignment().value_or(0);
1569 const mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());
1570 const StringRef symbol = op.getSymName();
1571 mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter);
1572
1574 mlir::LLVM::GlobalOp newGlobalOp =
1575 rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(
1576 op, llvmType, isConst, linkage, symbol, nullptr, alignment, addrSpace,
1577 isDsoLocal, isThreadLocal, comdatAttr, attributes);
1578 newGlobalOp.getRegion().emplaceBlock();
1579 rewriter.setInsertionPointToEnd(newGlobalOp.getInitializerBlock());
1580}
1581
1582mlir::LogicalResult
1583CIRToLLVMGlobalOpLowering::matchAndRewriteRegionInitializedGlobal(
1584 cir::GlobalOp op, mlir::Attribute init,
1585 mlir::ConversionPatternRewriter &rewriter) const {
1586 // TODO: Generalize this handling when more types are needed here.
1587 assert((isa<cir::ConstArrayAttr, cir::ConstRecordAttr, cir::ConstVectorAttr,
1588 cir::ConstPtrAttr, cir::ConstComplexAttr, cir::GlobalViewAttr,
1589 cir::VTableAttr, cir::ZeroAttr>(init)));
1590
1591 // TODO(cir): once LLVM's dialect has proper equivalent attributes this
1592 // should be updated. For now, we use a custom op to initialize globals
1593 // to the appropriate value.
1594 const mlir::Location loc = op.getLoc();
1595 setupRegionInitializedLLVMGlobalOp(op, rewriter);
1596 CIRAttrToValue valueConverter(op, rewriter, typeConverter);
1597 mlir::Value value = valueConverter.visit(init);
1598 rewriter.create<mlir::LLVM::ReturnOp>(loc, value);
1599 return mlir::success();
1600}
1601
1603 cir::GlobalOp op, OpAdaptor adaptor,
1604 mlir::ConversionPatternRewriter &rewriter) const {
1605
1606 std::optional<mlir::Attribute> init = op.getInitialValue();
1607
1608 // Fetch required values to create LLVM op.
1609 const mlir::Type cirSymType = op.getSymType();
1610
1611 // This is the LLVM dialect type.
1612 const mlir::Type llvmType =
1613 convertTypeForMemory(*getTypeConverter(), dataLayout, cirSymType);
1614 // FIXME: These default values are placeholders until the the equivalent
1615 // attributes are available on cir.global ops.
1617 const bool isConst = false;
1619 const unsigned addrSpace = 0;
1620 const bool isDsoLocal = op.getDsoLocal();
1622 const bool isThreadLocal = false;
1623 const uint64_t alignment = op.getAlignment().value_or(0);
1624 const mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());
1625 const StringRef symbol = op.getSymName();
1627 mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter);
1628
1629 if (init.has_value()) {
1630 if (mlir::isa<cir::FPAttr, cir::IntAttr, cir::BoolAttr>(init.value())) {
1631 GlobalInitAttrRewriter initRewriter(llvmType, rewriter);
1632 init = initRewriter.visit(init.value());
1633 // If initRewriter returned a null attribute, init will have a value but
1634 // the value will be null. If that happens, initRewriter didn't handle the
1635 // attribute type. It probably needs to be added to
1636 // GlobalInitAttrRewriter.
1637 if (!init.value()) {
1638 op.emitError() << "unsupported initializer '" << init.value() << "'";
1639 return mlir::failure();
1640 }
1641 } else if (mlir::isa<cir::ConstArrayAttr, cir::ConstVectorAttr,
1642 cir::ConstRecordAttr, cir::ConstPtrAttr,
1643 cir::ConstComplexAttr, cir::GlobalViewAttr,
1644 cir::VTableAttr, cir::ZeroAttr>(init.value())) {
1645 // TODO(cir): once LLVM's dialect has proper equivalent attributes this
1646 // should be updated. For now, we use a custom op to initialize globals
1647 // to the appropriate value.
1648 return matchAndRewriteRegionInitializedGlobal(op, init.value(), rewriter);
1649 } else {
1650 // We will only get here if new initializer types are added and this
1651 // code is not updated to handle them.
1652 op.emitError() << "unsupported initializer '" << init.value() << "'";
1653 return mlir::failure();
1654 }
1655 }
1656
1657 // Rewrite op.
1658 rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(
1659 op, llvmType, isConst, linkage, symbol, init.value_or(mlir::Attribute()),
1660 alignment, addrSpace, isDsoLocal, isThreadLocal, comdatAttr, attributes);
1661 return mlir::success();
1662}
1663
1664mlir::SymbolRefAttr
1665CIRToLLVMGlobalOpLowering::getComdatAttr(cir::GlobalOp &op,
1666 mlir::OpBuilder &builder) const {
1667 if (!op.getComdat())
1668 return mlir::SymbolRefAttr{};
1669
1670 mlir::ModuleOp module = op->getParentOfType<mlir::ModuleOp>();
1671 mlir::OpBuilder::InsertionGuard guard(builder);
1672 StringRef comdatName("__llvm_comdat_globals");
1673 if (!comdatOp) {
1674 builder.setInsertionPointToStart(module.getBody());
1675 comdatOp =
1676 builder.create<mlir::LLVM::ComdatOp>(module.getLoc(), comdatName);
1677 }
1678
1679 builder.setInsertionPointToStart(&comdatOp.getBody().back());
1680 auto selectorOp = builder.create<mlir::LLVM::ComdatSelectorOp>(
1681 comdatOp.getLoc(), op.getSymName(), mlir::LLVM::comdat::Comdat::Any);
1682 return mlir::SymbolRefAttr::get(
1683 builder.getContext(), comdatName,
1684 mlir::FlatSymbolRefAttr::get(selectorOp.getSymNameAttr()));
1685}
1686
1688 cir::SwitchFlatOp op, OpAdaptor adaptor,
1689 mlir::ConversionPatternRewriter &rewriter) const {
1690
1692 for (mlir::Attribute val : op.getCaseValues()) {
1693 auto intAttr = cast<cir::IntAttr>(val);
1694 caseValues.push_back(intAttr.getValue());
1695 }
1696
1697 llvm::SmallVector<mlir::Block *, 8> caseDestinations;
1699
1700 for (mlir::Block *x : op.getCaseDestinations())
1701 caseDestinations.push_back(x);
1702
1703 for (mlir::OperandRange x : op.getCaseOperands())
1704 caseOperands.push_back(x);
1705
1706 // Set switch op to branch to the newly created blocks.
1707 rewriter.setInsertionPoint(op);
1708 rewriter.replaceOpWithNewOp<mlir::LLVM::SwitchOp>(
1709 op, adaptor.getCondition(), op.getDefaultDestination(),
1710 op.getDefaultOperands(), caseValues, caseDestinations, caseOperands);
1711 return mlir::success();
1712}
1713
1715 cir::UnaryOp op, OpAdaptor adaptor,
1716 mlir::ConversionPatternRewriter &rewriter) const {
1717 assert(op.getType() == op.getInput().getType() &&
1718 "Unary operation's operand type and result type are different");
1719 mlir::Type type = op.getType();
1720 mlir::Type elementType = elementTypeIfVector(type);
1721 bool isVector = mlir::isa<cir::VectorType>(type);
1722 mlir::Type llvmType = getTypeConverter()->convertType(type);
1723 mlir::Location loc = op.getLoc();
1724
1725 // Integer unary operations: + - ~ ++ --
1726 if (mlir::isa<cir::IntType>(elementType)) {
1727 mlir::LLVM::IntegerOverflowFlags maybeNSW =
1728 op.getNoSignedWrap() ? mlir::LLVM::IntegerOverflowFlags::nsw
1729 : mlir::LLVM::IntegerOverflowFlags::none;
1730 switch (op.getKind()) {
1731 case cir::UnaryOpKind::Inc: {
1732 assert(!isVector && "++ not allowed on vector types");
1733 auto one = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, 1);
1734 rewriter.replaceOpWithNewOp<mlir::LLVM::AddOp>(
1735 op, llvmType, adaptor.getInput(), one, maybeNSW);
1736 return mlir::success();
1737 }
1738 case cir::UnaryOpKind::Dec: {
1739 assert(!isVector && "-- not allowed on vector types");
1740 auto one = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, 1);
1741 rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>(op, adaptor.getInput(),
1742 one, maybeNSW);
1743 return mlir::success();
1744 }
1745 case cir::UnaryOpKind::Plus:
1746 rewriter.replaceOp(op, adaptor.getInput());
1747 return mlir::success();
1748 case cir::UnaryOpKind::Minus: {
1749 mlir::Value zero;
1750 if (isVector)
1751 zero = rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmType);
1752 else
1753 zero = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, 0);
1754 rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>(
1755 op, zero, adaptor.getInput(), maybeNSW);
1756 return mlir::success();
1757 }
1758 case cir::UnaryOpKind::Not: {
1759 // bit-wise compliment operator, implemented as an XOR with -1.
1760 mlir::Value minusOne;
1761 if (isVector) {
1762 const uint64_t numElements =
1763 mlir::dyn_cast<cir::VectorType>(type).getSize();
1764 std::vector<int32_t> values(numElements, -1);
1765 mlir::DenseIntElementsAttr denseVec = rewriter.getI32VectorAttr(values);
1766 minusOne =
1767 rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, denseVec);
1768 } else {
1769 minusOne = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, -1);
1770 }
1771 rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getInput(),
1772 minusOne);
1773 return mlir::success();
1774 }
1775 }
1776 llvm_unreachable("Unexpected unary op for int");
1777 }
1778
1779 // Floating point unary operations: + - ++ --
1780 if (mlir::isa<cir::FPTypeInterface>(elementType)) {
1781 switch (op.getKind()) {
1782 case cir::UnaryOpKind::Inc: {
1783 assert(!isVector && "++ not allowed on vector types");
1784 mlir::LLVM::ConstantOp one = rewriter.create<mlir::LLVM::ConstantOp>(
1785 loc, llvmType, rewriter.getFloatAttr(llvmType, 1.0));
1786 rewriter.replaceOpWithNewOp<mlir::LLVM::FAddOp>(op, llvmType, one,
1787 adaptor.getInput());
1788 return mlir::success();
1789 }
1790 case cir::UnaryOpKind::Dec: {
1791 assert(!isVector && "-- not allowed on vector types");
1792 mlir::LLVM::ConstantOp minusOne = rewriter.create<mlir::LLVM::ConstantOp>(
1793 loc, llvmType, rewriter.getFloatAttr(llvmType, -1.0));
1794 rewriter.replaceOpWithNewOp<mlir::LLVM::FAddOp>(op, llvmType, minusOne,
1795 adaptor.getInput());
1796 return mlir::success();
1797 }
1798 case cir::UnaryOpKind::Plus:
1799 rewriter.replaceOp(op, adaptor.getInput());
1800 return mlir::success();
1801 case cir::UnaryOpKind::Minus:
1802 rewriter.replaceOpWithNewOp<mlir::LLVM::FNegOp>(op, llvmType,
1803 adaptor.getInput());
1804 return mlir::success();
1805 case cir::UnaryOpKind::Not:
1806 return op.emitError() << "Unary not is invalid for floating-point types";
1807 }
1808 llvm_unreachable("Unexpected unary op for float");
1809 }
1810
1811 // Boolean unary operations: ! only. (For all others, the operand has
1812 // already been promoted to int.)
1813 if (mlir::isa<cir::BoolType>(elementType)) {
1814 switch (op.getKind()) {
1815 case cir::UnaryOpKind::Inc:
1816 case cir::UnaryOpKind::Dec:
1817 case cir::UnaryOpKind::Plus:
1818 case cir::UnaryOpKind::Minus:
1819 // Some of these are allowed in source code, but we shouldn't get here
1820 // with a boolean type.
1821 return op.emitError() << "Unsupported unary operation on boolean type";
1822 case cir::UnaryOpKind::Not: {
1823 assert(!isVector && "NYI: op! on vector mask");
1824 auto one = rewriter.create<mlir::LLVM::ConstantOp>(loc, llvmType, 1);
1825 rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getInput(),
1826 one);
1827 return mlir::success();
1828 }
1829 }
1830 llvm_unreachable("Unexpected unary op for bool");
1831 }
1832
1833 // Pointer unary operations: + only. (++ and -- of pointers are implemented
1834 // with cir.ptr_stride, not cir.unary.)
1835 if (mlir::isa<cir::PointerType>(elementType)) {
1836 return op.emitError()
1837 << "Unary operation on pointer types is not yet implemented";
1838 }
1839
1840 return op.emitError() << "Unary operation has unsupported type: "
1841 << elementType;
1842}
1843
1844mlir::LLVM::IntegerOverflowFlags
1845CIRToLLVMBinOpLowering::getIntOverflowFlag(cir::BinOp op) const {
1846 if (op.getNoUnsignedWrap())
1847 return mlir::LLVM::IntegerOverflowFlags::nuw;
1848
1849 if (op.getNoSignedWrap())
1850 return mlir::LLVM::IntegerOverflowFlags::nsw;
1851
1852 return mlir::LLVM::IntegerOverflowFlags::none;
1853}
1854
1855static bool isIntTypeUnsigned(mlir::Type type) {
1856 // TODO: Ideally, we should only need to check cir::IntType here.
1857 return mlir::isa<cir::IntType>(type)
1858 ? mlir::cast<cir::IntType>(type).isUnsigned()
1859 : mlir::cast<mlir::IntegerType>(type).isUnsigned();
1860}
1861
1863 cir::BinOp op, OpAdaptor adaptor,
1864 mlir::ConversionPatternRewriter &rewriter) const {
1865 if (adaptor.getLhs().getType() != adaptor.getRhs().getType())
1866 return op.emitError() << "inconsistent operands' types not supported yet";
1867
1868 mlir::Type type = op.getRhs().getType();
1869 if (!mlir::isa<cir::IntType, cir::BoolType, cir::FPTypeInterface,
1870 mlir::IntegerType, cir::VectorType>(type))
1871 return op.emitError() << "operand type not supported yet";
1872
1873 const mlir::Type llvmTy = getTypeConverter()->convertType(op.getType());
1874 const mlir::Type llvmEltTy = elementTypeIfVector(llvmTy);
1875
1876 const mlir::Value rhs = adaptor.getRhs();
1877 const mlir::Value lhs = adaptor.getLhs();
1878 type = elementTypeIfVector(type);
1879
1880 switch (op.getKind()) {
1881 case cir::BinOpKind::Add:
1882 if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {
1883 if (op.getSaturated()) {
1884 if (isIntTypeUnsigned(type)) {
1885 rewriter.replaceOpWithNewOp<mlir::LLVM::UAddSat>(op, lhs, rhs);
1886 break;
1887 }
1888 rewriter.replaceOpWithNewOp<mlir::LLVM::SAddSat>(op, lhs, rhs);
1889 break;
1890 }
1891 rewriter.replaceOpWithNewOp<mlir::LLVM::AddOp>(op, llvmTy, lhs, rhs,
1892 getIntOverflowFlag(op));
1893 } else {
1894 rewriter.replaceOpWithNewOp<mlir::LLVM::FAddOp>(op, lhs, rhs);
1895 }
1896 break;
1897 case cir::BinOpKind::Sub:
1898 if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {
1899 if (op.getSaturated()) {
1900 if (isIntTypeUnsigned(type)) {
1901 rewriter.replaceOpWithNewOp<mlir::LLVM::USubSat>(op, lhs, rhs);
1902 break;
1903 }
1904 rewriter.replaceOpWithNewOp<mlir::LLVM::SSubSat>(op, lhs, rhs);
1905 break;
1906 }
1907 rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>(op, llvmTy, lhs, rhs,
1908 getIntOverflowFlag(op));
1909 } else {
1910 rewriter.replaceOpWithNewOp<mlir::LLVM::FSubOp>(op, lhs, rhs);
1911 }
1912 break;
1913 case cir::BinOpKind::Mul:
1914 if (mlir::isa<mlir::IntegerType>(llvmEltTy))
1915 rewriter.replaceOpWithNewOp<mlir::LLVM::MulOp>(op, llvmTy, lhs, rhs,
1916 getIntOverflowFlag(op));
1917 else
1918 rewriter.replaceOpWithNewOp<mlir::LLVM::FMulOp>(op, lhs, rhs);
1919 break;
1920 case cir::BinOpKind::Div:
1921 if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {
1922 auto isUnsigned = isIntTypeUnsigned(type);
1923 if (isUnsigned)
1924 rewriter.replaceOpWithNewOp<mlir::LLVM::UDivOp>(op, lhs, rhs);
1925 else
1926 rewriter.replaceOpWithNewOp<mlir::LLVM::SDivOp>(op, lhs, rhs);
1927 } else {
1928 rewriter.replaceOpWithNewOp<mlir::LLVM::FDivOp>(op, lhs, rhs);
1929 }
1930 break;
1931 case cir::BinOpKind::Rem:
1932 if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {
1933 auto isUnsigned = isIntTypeUnsigned(type);
1934 if (isUnsigned)
1935 rewriter.replaceOpWithNewOp<mlir::LLVM::URemOp>(op, lhs, rhs);
1936 else
1937 rewriter.replaceOpWithNewOp<mlir::LLVM::SRemOp>(op, lhs, rhs);
1938 } else {
1939 rewriter.replaceOpWithNewOp<mlir::LLVM::FRemOp>(op, lhs, rhs);
1940 }
1941 break;
1942 case cir::BinOpKind::And:
1943 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op, lhs, rhs);
1944 break;
1945 case cir::BinOpKind::Or:
1946 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(op, lhs, rhs);
1947 break;
1948 case cir::BinOpKind::Xor:
1949 rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, lhs, rhs);
1950 break;
1951 case cir::BinOpKind::Max:
1952 if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {
1953 auto isUnsigned = isIntTypeUnsigned(type);
1954 if (isUnsigned)
1955 rewriter.replaceOpWithNewOp<mlir::LLVM::UMaxOp>(op, llvmTy, lhs, rhs);
1956 else
1957 rewriter.replaceOpWithNewOp<mlir::LLVM::SMaxOp>(op, llvmTy, lhs, rhs);
1958 }
1959 break;
1960 }
1961 return mlir::LogicalResult::success();
1962}
1963
1964/// Convert from a CIR comparison kind to an LLVM IR integral comparison kind.
1965static mlir::LLVM::ICmpPredicate
1966convertCmpKindToICmpPredicate(cir::CmpOpKind kind, bool isSigned) {
1967 using CIR = cir::CmpOpKind;
1968 using LLVMICmp = mlir::LLVM::ICmpPredicate;
1969 switch (kind) {
1970 case CIR::eq:
1971 return LLVMICmp::eq;
1972 case CIR::ne:
1973 return LLVMICmp::ne;
1974 case CIR::lt:
1975 return (isSigned ? LLVMICmp::slt : LLVMICmp::ult);
1976 case CIR::le:
1977 return (isSigned ? LLVMICmp::sle : LLVMICmp::ule);
1978 case CIR::gt:
1979 return (isSigned ? LLVMICmp::sgt : LLVMICmp::ugt);
1980 case CIR::ge:
1981 return (isSigned ? LLVMICmp::sge : LLVMICmp::uge);
1982 }
1983 llvm_unreachable("Unknown CmpOpKind");
1984}
1985
1986/// Convert from a CIR comparison kind to an LLVM IR floating-point comparison
1987/// kind.
1988static mlir::LLVM::FCmpPredicate
1989convertCmpKindToFCmpPredicate(cir::CmpOpKind kind) {
1990 using CIR = cir::CmpOpKind;
1991 using LLVMFCmp = mlir::LLVM::FCmpPredicate;
1992 switch (kind) {
1993 case CIR::eq:
1994 return LLVMFCmp::oeq;
1995 case CIR::ne:
1996 return LLVMFCmp::une;
1997 case CIR::lt:
1998 return LLVMFCmp::olt;
1999 case CIR::le:
2000 return LLVMFCmp::ole;
2001 case CIR::gt:
2002 return LLVMFCmp::ogt;
2003 case CIR::ge:
2004 return LLVMFCmp::oge;
2005 }
2006 llvm_unreachable("Unknown CmpOpKind");
2007}
2008
2010 cir::CmpOp cmpOp, OpAdaptor adaptor,
2011 mlir::ConversionPatternRewriter &rewriter) const {
2012 mlir::Type type = cmpOp.getLhs().getType();
2013
2016
2017 if (mlir::isa<cir::IntType, mlir::IntegerType>(type)) {
2018 bool isSigned = mlir::isa<cir::IntType>(type)
2019 ? mlir::cast<cir::IntType>(type).isSigned()
2020 : mlir::cast<mlir::IntegerType>(type).isSigned();
2021 mlir::LLVM::ICmpPredicate kind =
2022 convertCmpKindToICmpPredicate(cmpOp.getKind(), isSigned);
2023 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
2024 cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
2025 return mlir::success();
2026 }
2027
2028 if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(type)) {
2029 mlir::LLVM::ICmpPredicate kind =
2030 convertCmpKindToICmpPredicate(cmpOp.getKind(),
2031 /* isSigned=*/false);
2032 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
2033 cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
2034 return mlir::success();
2035 }
2036
2037 if (mlir::isa<cir::FPTypeInterface>(type)) {
2038 mlir::LLVM::FCmpPredicate kind =
2039 convertCmpKindToFCmpPredicate(cmpOp.getKind());
2040 rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(
2041 cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
2042 return mlir::success();
2043 }
2044
2045 if (mlir::isa<cir::ComplexType>(type)) {
2046 mlir::Value lhs = adaptor.getLhs();
2047 mlir::Value rhs = adaptor.getRhs();
2048 mlir::Location loc = cmpOp.getLoc();
2049
2050 auto complexType = mlir::cast<cir::ComplexType>(cmpOp.getLhs().getType());
2051 mlir::Type complexElemTy =
2052 getTypeConverter()->convertType(complexType.getElementType());
2053
2054 auto lhsReal =
2055 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 0);
2056 auto lhsImag =
2057 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 1);
2058 auto rhsReal =
2059 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 0);
2060 auto rhsImag =
2061 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 1);
2062
2063 if (cmpOp.getKind() == cir::CmpOpKind::eq) {
2064 if (complexElemTy.isInteger()) {
2065 auto realCmp = rewriter.create<mlir::LLVM::ICmpOp>(
2066 loc, mlir::LLVM::ICmpPredicate::eq, lhsReal, rhsReal);
2067 auto imagCmp = rewriter.create<mlir::LLVM::ICmpOp>(
2068 loc, mlir::LLVM::ICmpPredicate::eq, lhsImag, rhsImag);
2069 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp);
2070 return mlir::success();
2071 }
2072
2073 auto realCmp = rewriter.create<mlir::LLVM::FCmpOp>(
2074 loc, mlir::LLVM::FCmpPredicate::oeq, lhsReal, rhsReal);
2075 auto imagCmp = rewriter.create<mlir::LLVM::FCmpOp>(
2076 loc, mlir::LLVM::FCmpPredicate::oeq, lhsImag, rhsImag);
2077 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp);
2078 return mlir::success();
2079 }
2080
2081 if (cmpOp.getKind() == cir::CmpOpKind::ne) {
2082 if (complexElemTy.isInteger()) {
2083 auto realCmp = rewriter.create<mlir::LLVM::ICmpOp>(
2084 loc, mlir::LLVM::ICmpPredicate::ne, lhsReal, rhsReal);
2085 auto imagCmp = rewriter.create<mlir::LLVM::ICmpOp>(
2086 loc, mlir::LLVM::ICmpPredicate::ne, lhsImag, rhsImag);
2087 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp);
2088 return mlir::success();
2089 }
2090
2091 auto realCmp = rewriter.create<mlir::LLVM::FCmpOp>(
2092 loc, mlir::LLVM::FCmpPredicate::une, lhsReal, rhsReal);
2093 auto imagCmp = rewriter.create<mlir::LLVM::FCmpOp>(
2094 loc, mlir::LLVM::FCmpPredicate::une, lhsImag, rhsImag);
2095 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp);
2096 return mlir::success();
2097 }
2098 }
2099
2100 return cmpOp.emitError() << "unsupported type for CmpOp: " << type;
2101}
2102
2104 cir::ShiftOp op, OpAdaptor adaptor,
2105 mlir::ConversionPatternRewriter &rewriter) const {
2106 assert((op.getValue().getType() == op.getType()) &&
2107 "inconsistent operands' types NYI");
2108
2109 const mlir::Type llvmTy = getTypeConverter()->convertType(op.getType());
2110 mlir::Value amt = adaptor.getAmount();
2111 mlir::Value val = adaptor.getValue();
2112
2113 auto cirAmtTy = mlir::dyn_cast<cir::IntType>(op.getAmount().getType());
2114 bool isUnsigned;
2115 if (cirAmtTy) {
2116 auto cirValTy = mlir::cast<cir::IntType>(op.getValue().getType());
2117 isUnsigned = cirValTy.isUnsigned();
2118
2119 // Ensure shift amount is the same type as the value. Some undefined
2120 // behavior might occur in the casts below as per [C99 6.5.7.3].
2121 // Vector type shift amount needs no cast as type consistency is expected to
2122 // be already be enforced at CIRGen.
2123 if (cirAmtTy)
2124 amt = getLLVMIntCast(rewriter, amt, llvmTy, true, cirAmtTy.getWidth(),
2125 cirValTy.getWidth());
2126 } else {
2127 auto cirValVTy = mlir::cast<cir::VectorType>(op.getValue().getType());
2128 isUnsigned =
2129 mlir::cast<cir::IntType>(cirValVTy.getElementType()).isUnsigned();
2130 }
2131
2132 // Lower to the proper LLVM shift operation.
2133 if (op.getIsShiftleft()) {
2134 rewriter.replaceOpWithNewOp<mlir::LLVM::ShlOp>(op, llvmTy, val, amt);
2135 return mlir::success();
2136 }
2137
2138 if (isUnsigned)
2139 rewriter.replaceOpWithNewOp<mlir::LLVM::LShrOp>(op, llvmTy, val, amt);
2140 else
2141 rewriter.replaceOpWithNewOp<mlir::LLVM::AShrOp>(op, llvmTy, val, amt);
2142 return mlir::success();
2143}
2144
2146 cir::SelectOp op, OpAdaptor adaptor,
2147 mlir::ConversionPatternRewriter &rewriter) const {
2148 auto getConstantBool = [](mlir::Value value) -> cir::BoolAttr {
2149 auto definingOp = value.getDefiningOp<cir::ConstantOp>();
2150 if (!definingOp)
2151 return {};
2152
2153 auto constValue = definingOp.getValueAttr<cir::BoolAttr>();
2154 if (!constValue)
2155 return {};
2156
2157 return constValue;
2158 };
2159
2160 // Two special cases in the LLVMIR codegen of select op:
2161 // - select %0, %1, false => and %0, %1
2162 // - select %0, true, %1 => or %0, %1
2163 if (mlir::isa<cir::BoolType>(op.getTrueValue().getType())) {
2164 cir::BoolAttr trueValue = getConstantBool(op.getTrueValue());
2165 cir::BoolAttr falseValue = getConstantBool(op.getFalseValue());
2166 if (falseValue && !falseValue.getValue()) {
2167 // select %0, %1, false => and %0, %1
2168 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op, adaptor.getCondition(),
2169 adaptor.getTrueValue());
2170 return mlir::success();
2171 }
2172 if (trueValue && trueValue.getValue()) {
2173 // select %0, true, %1 => or %0, %1
2174 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(op, adaptor.getCondition(),
2175 adaptor.getFalseValue());
2176 return mlir::success();
2177 }
2178 }
2179
2180 mlir::Value llvmCondition = adaptor.getCondition();
2181 rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(
2182 op, llvmCondition, adaptor.getTrueValue(), adaptor.getFalseValue());
2183
2184 return mlir::success();
2185}
2186
2187static void prepareTypeConverter(mlir::LLVMTypeConverter &converter,
2188 mlir::DataLayout &dataLayout) {
2189 converter.addConversion([&](cir::PointerType type) -> mlir::Type {
2190 // Drop pointee type since LLVM dialect only allows opaque pointers.
2192 unsigned targetAS = 0;
2193
2194 return mlir::LLVM::LLVMPointerType::get(type.getContext(), targetAS);
2195 });
2196 converter.addConversion([&](cir::VPtrType type) -> mlir::Type {
2198 return mlir::LLVM::LLVMPointerType::get(type.getContext());
2199 });
2200 converter.addConversion([&](cir::ArrayType type) -> mlir::Type {
2201 mlir::Type ty =
2202 convertTypeForMemory(converter, dataLayout, type.getElementType());
2203 return mlir::LLVM::LLVMArrayType::get(ty, type.getSize());
2204 });
2205 converter.addConversion([&](cir::VectorType type) -> mlir::Type {
2206 const mlir::Type ty = converter.convertType(type.getElementType());
2207 return mlir::VectorType::get(type.getSize(), ty);
2208 });
2209 converter.addConversion([&](cir::BoolType type) -> mlir::Type {
2210 return mlir::IntegerType::get(type.getContext(), 1,
2211 mlir::IntegerType::Signless);
2212 });
2213 converter.addConversion([&](cir::IntType type) -> mlir::Type {
2214 // LLVM doesn't work with signed types, so we drop the CIR signs here.
2215 return mlir::IntegerType::get(type.getContext(), type.getWidth());
2216 });
2217 converter.addConversion([&](cir::SingleType type) -> mlir::Type {
2218 return mlir::Float32Type::get(type.getContext());
2219 });
2220 converter.addConversion([&](cir::DoubleType type) -> mlir::Type {
2221 return mlir::Float64Type::get(type.getContext());
2222 });
2223 converter.addConversion([&](cir::FP80Type type) -> mlir::Type {
2224 return mlir::Float80Type::get(type.getContext());
2225 });
2226 converter.addConversion([&](cir::FP128Type type) -> mlir::Type {
2227 return mlir::Float128Type::get(type.getContext());
2228 });
2229 converter.addConversion([&](cir::LongDoubleType type) -> mlir::Type {
2230 return converter.convertType(type.getUnderlying());
2231 });
2232 converter.addConversion([&](cir::FP16Type type) -> mlir::Type {
2233 return mlir::Float16Type::get(type.getContext());
2234 });
2235 converter.addConversion([&](cir::BF16Type type) -> mlir::Type {
2236 return mlir::BFloat16Type::get(type.getContext());
2237 });
2238 converter.addConversion([&](cir::ComplexType type) -> mlir::Type {
2239 // A complex type is lowered to an LLVM struct that contains the real and
2240 // imaginary part as data fields.
2241 mlir::Type elementTy = converter.convertType(type.getElementType());
2242 mlir::Type structFields[2] = {elementTy, elementTy};
2243 return mlir::LLVM::LLVMStructType::getLiteral(type.getContext(),
2244 structFields);
2245 });
2246 converter.addConversion([&](cir::FuncType type) -> std::optional<mlir::Type> {
2247 auto result = converter.convertType(type.getReturnType());
2249 arguments.reserve(type.getNumInputs());
2250 if (converter.convertTypes(type.getInputs(), arguments).failed())
2251 return std::nullopt;
2252 auto varArg = type.isVarArg();
2253 return mlir::LLVM::LLVMFunctionType::get(result, arguments, varArg);
2254 });
2255 converter.addConversion([&](cir::RecordType type) -> mlir::Type {
2256 // Convert struct members.
2258 switch (type.getKind()) {
2259 case cir::RecordType::Class:
2260 case cir::RecordType::Struct:
2261 for (mlir::Type ty : type.getMembers())
2262 llvmMembers.push_back(convertTypeForMemory(converter, dataLayout, ty));
2263 break;
2264 // Unions are lowered as only the largest member.
2265 case cir::RecordType::Union:
2266 if (auto largestMember = type.getLargestMember(dataLayout))
2267 llvmMembers.push_back(
2268 convertTypeForMemory(converter, dataLayout, largestMember));
2269 if (type.getPadded()) {
2270 auto last = *type.getMembers().rbegin();
2271 llvmMembers.push_back(
2272 convertTypeForMemory(converter, dataLayout, last));
2273 }
2274 break;
2275 }
2276 converter.addConversion([&](cir::VoidType type) -> mlir::Type {
2277 return mlir::LLVM::LLVMVoidType::get(type.getContext());
2278 });
2279
2280 // Record has a name: lower as an identified record.
2281 mlir::LLVM::LLVMStructType llvmStruct;
2282 if (type.getName()) {
2283 llvmStruct = mlir::LLVM::LLVMStructType::getIdentified(
2284 type.getContext(), type.getPrefixedName());
2285 if (llvmStruct.setBody(llvmMembers, type.getPacked()).failed())
2286 llvm_unreachable("Failed to set body of record");
2287 } else { // Record has no name: lower as literal record.
2288 llvmStruct = mlir::LLVM::LLVMStructType::getLiteral(
2289 type.getContext(), llvmMembers, type.getPacked());
2290 }
2291
2292 return llvmStruct;
2293 });
2294 converter.addConversion([&](cir::VoidType type) -> mlir::Type {
2295 return mlir::LLVM::LLVMVoidType::get(type.getContext());
2296 });
2297}
2298
2299// The applyPartialConversion function traverses blocks in the dominance order,
2300// so it does not lower and operations that are not reachachable from the
2301// operations passed in as arguments. Since we do need to lower such code in
2302// order to avoid verification errors occur, we cannot just pass the module op
2303// to applyPartialConversion. We must build a set of unreachable ops and
2304// explicitly add them, along with the module, to the vector we pass to
2305// applyPartialConversion.
2306//
2307// For instance, this CIR code:
2308//
2309// cir.func @foo(%arg0: !s32i) -> !s32i {
2310// %4 = cir.cast(int_to_bool, %arg0 : !s32i), !cir.bool
2311// cir.if %4 {
2312// %5 = cir.const #cir.int<1> : !s32i
2313// cir.return %5 : !s32i
2314// } else {
2315// %5 = cir.const #cir.int<0> : !s32i
2316// cir.return %5 : !s32i
2317// }
2318// cir.return %arg0 : !s32i
2319// }
2320//
2321// contains an unreachable return operation (the last one). After the flattening
2322// pass it will be placed into the unreachable block. The possible error
2323// after the lowering pass is: error: 'cir.return' op expects parent op to be
2324// one of 'cir.func, cir.scope, cir.if ... The reason that this operation was
2325// not lowered and the new parent is llvm.func.
2326//
2327// In the future we may want to get rid of this function and use a DCE pass or
2328// something similar. But for now we need to guarantee the absence of the
2329// dialect verification errors.
2330static void collectUnreachable(mlir::Operation *parent,
2332
2333 llvm::SmallVector<mlir::Block *> unreachableBlocks;
2334 parent->walk([&](mlir::Block *blk) { // check
2335 if (blk->hasNoPredecessors() && !blk->isEntryBlock())
2336 unreachableBlocks.push_back(blk);
2337 });
2338
2339 std::set<mlir::Block *> visited;
2340 for (mlir::Block *root : unreachableBlocks) {
2341 // We create a work list for each unreachable block.
2342 // Thus we traverse operations in some order.
2343 std::deque<mlir::Block *> workList;
2344 workList.push_back(root);
2345
2346 while (!workList.empty()) {
2347 mlir::Block *blk = workList.back();
2348 workList.pop_back();
2349 if (visited.count(blk))
2350 continue;
2351 visited.emplace(blk);
2352
2353 for (mlir::Operation &op : *blk)
2354 ops.push_back(&op);
2355
2356 for (mlir::Block *succ : blk->getSuccessors())
2357 workList.push_back(succ);
2358 }
2359 }
2360}
2361
2362void ConvertCIRToLLVMPass::processCIRAttrs(mlir::ModuleOp module) {
2363 // Lower the module attributes to LLVM equivalents.
2364 if (mlir::Attribute tripleAttr =
2365 module->getAttr(cir::CIRDialect::getTripleAttrName()))
2366 module->setAttr(mlir::LLVM::LLVMDialect::getTargetTripleAttrName(),
2367 tripleAttr);
2368
2369 if (mlir::Attribute asmAttr =
2370 module->getAttr(cir::CIRDialect::getModuleLevelAsmAttrName()))
2371 module->setAttr(mlir::LLVM::LLVMDialect::getModuleLevelAsmAttrName(),
2372 asmAttr);
2373}
2374
2376 llvm::TimeTraceScope scope("Convert CIR to LLVM Pass");
2377
2378 mlir::ModuleOp module = getOperation();
2379 mlir::DataLayout dl(module);
2380 mlir::LLVMTypeConverter converter(&getContext());
2381 prepareTypeConverter(converter, dl);
2382
2383 mlir::RewritePatternSet patterns(&getContext());
2384
2385 patterns.add<CIRToLLVMReturnOpLowering>(patterns.getContext());
2386 // This could currently be merged with the group below, but it will get more
2387 // arguments later, so we'll keep it separate for now.
2388 patterns.add<CIRToLLVMAllocaOpLowering>(converter, patterns.getContext(), dl);
2389 patterns.add<CIRToLLVMLoadOpLowering>(converter, patterns.getContext(), dl);
2390 patterns.add<CIRToLLVMStoreOpLowering>(converter, patterns.getContext(), dl);
2391 patterns.add<CIRToLLVMGlobalOpLowering>(converter, patterns.getContext(), dl);
2392 patterns.add<CIRToLLVMCastOpLowering>(converter, patterns.getContext(), dl);
2393 patterns.add<CIRToLLVMPtrStrideOpLowering>(converter, patterns.getContext(),
2394 dl);
2395 patterns.add<CIRToLLVMInlineAsmOpLowering>(converter, patterns.getContext(),
2396 dl);
2397 patterns.add<
2398 // clang-format off
2458 // clang-format on
2459 >(converter, patterns.getContext());
2460
2461 processCIRAttrs(module);
2462
2463 mlir::ConversionTarget target(getContext());
2464 target.addLegalOp<mlir::ModuleOp>();
2465 target.addLegalDialect<mlir::LLVM::LLVMDialect>();
2466 target.addIllegalDialect<mlir::BuiltinDialect, cir::CIRDialect,
2467 mlir::func::FuncDialect>();
2468
2470 ops.push_back(module);
2471 collectUnreachable(module, ops);
2472
2473 if (failed(applyPartialConversion(ops, target, std::move(patterns))))
2474 signalPassFailure();
2475}
2476
2478 cir::BrOp op, OpAdaptor adaptor,
2479 mlir::ConversionPatternRewriter &rewriter) const {
2480 rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(op, adaptor.getOperands(),
2481 op.getDest());
2482 return mlir::LogicalResult::success();
2483}
2484
2486 cir::GetMemberOp op, OpAdaptor adaptor,
2487 mlir::ConversionPatternRewriter &rewriter) const {
2488 mlir::Type llResTy = getTypeConverter()->convertType(op.getType());
2489 const auto recordTy =
2490 mlir::cast<cir::RecordType>(op.getAddrTy().getPointee());
2491 assert(recordTy && "expected record type");
2492
2493 switch (recordTy.getKind()) {
2494 case cir::RecordType::Class:
2495 case cir::RecordType::Struct: {
2496 // Since the base address is a pointer to an aggregate, the first offset
2497 // is always zero. The second offset tell us which member it will access.
2498 llvm::SmallVector<mlir::LLVM::GEPArg, 2> offset{0, op.getIndex()};
2499 const mlir::Type elementTy = getTypeConverter()->convertType(recordTy);
2500 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, llResTy, elementTy,
2501 adaptor.getAddr(), offset);
2502 return mlir::success();
2503 }
2504 case cir::RecordType::Union:
2505 // Union members share the address space, so we just need a bitcast to
2506 // conform to type-checking.
2507 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(op, llResTy,
2508 adaptor.getAddr());
2509 return mlir::success();
2510 }
2511}
2512
2514 cir::UnreachableOp op, OpAdaptor adaptor,
2515 mlir::ConversionPatternRewriter &rewriter) const {
2516 rewriter.replaceOpWithNewOp<mlir::LLVM::UnreachableOp>(op);
2517 return mlir::success();
2518}
2519
2520void createLLVMFuncOpIfNotExist(mlir::ConversionPatternRewriter &rewriter,
2521 mlir::Operation *srcOp, llvm::StringRef fnName,
2522 mlir::Type fnTy) {
2523 auto modOp = srcOp->getParentOfType<mlir::ModuleOp>();
2524 auto enclosingFnOp = srcOp->getParentOfType<mlir::LLVM::LLVMFuncOp>();
2525 mlir::Operation *sourceSymbol =
2526 mlir::SymbolTable::lookupSymbolIn(modOp, fnName);
2527 if (!sourceSymbol) {
2528 mlir::OpBuilder::InsertionGuard guard(rewriter);
2529 rewriter.setInsertionPoint(enclosingFnOp);
2530 rewriter.create<mlir::LLVM::LLVMFuncOp>(srcOp->getLoc(), fnName, fnTy);
2531 }
2532}
2533
2535 cir::ThrowOp op, OpAdaptor adaptor,
2536 mlir::ConversionPatternRewriter &rewriter) const {
2537 if (op.rethrows()) {
2538 auto voidTy = mlir::LLVM::LLVMVoidType::get(getContext());
2539 auto funcTy =
2540 mlir::LLVM::LLVMFunctionType::get(getContext(), voidTy, {}, false);
2541
2542 auto mlirModule = op->getParentOfType<mlir::ModuleOp>();
2543 rewriter.setInsertionPointToStart(&mlirModule.getBodyRegion().front());
2544
2545 const llvm::StringRef functionName = "__cxa_rethrow";
2546 createLLVMFuncOpIfNotExist(rewriter, op, functionName, funcTy);
2547
2548 rewriter.setInsertionPointAfter(op.getOperation());
2549 rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
2550 op, mlir::TypeRange{}, functionName, mlir::ValueRange{});
2551 }
2552
2553 return mlir::success();
2554}
2555
2557 cir::TrapOp op, OpAdaptor adaptor,
2558 mlir::ConversionPatternRewriter &rewriter) const {
2559 mlir::Location loc = op->getLoc();
2560 rewriter.eraseOp(op);
2561
2562 rewriter.create<mlir::LLVM::Trap>(loc);
2563
2564 // Note that the call to llvm.trap is not a terminator in LLVM dialect.
2565 // So we must emit an additional llvm.unreachable to terminate the current
2566 // block.
2567 rewriter.create<mlir::LLVM::UnreachableOp>(loc);
2568
2569 return mlir::success();
2570}
2571
2572static mlir::Value
2573getValueForVTableSymbol(mlir::Operation *op,
2574 mlir::ConversionPatternRewriter &rewriter,
2575 const mlir::TypeConverter *converter,
2576 mlir::FlatSymbolRefAttr nameAttr, mlir::Type &eltType) {
2577 auto module = op->getParentOfType<mlir::ModuleOp>();
2578 mlir::Operation *symbol = mlir::SymbolTable::lookupSymbolIn(module, nameAttr);
2579 if (auto llvmSymbol = mlir::dyn_cast<mlir::LLVM::GlobalOp>(symbol)) {
2580 eltType = llvmSymbol.getType();
2581 } else if (auto cirSymbol = mlir::dyn_cast<cir::GlobalOp>(symbol)) {
2582 eltType = converter->convertType(cirSymbol.getSymType());
2583 } else {
2584 op->emitError() << "unexpected symbol type for " << symbol;
2585 return {};
2586 }
2587
2588 return mlir::LLVM::AddressOfOp::create(
2589 rewriter, op->getLoc(),
2590 mlir::LLVM::LLVMPointerType::get(op->getContext()), nameAttr.getValue());
2591}
2592
2594 cir::VTableAddrPointOp op, OpAdaptor adaptor,
2595 mlir::ConversionPatternRewriter &rewriter) const {
2596 const mlir::TypeConverter *converter = getTypeConverter();
2597 mlir::Type targetType = converter->convertType(op.getType());
2599 mlir::Type eltType;
2600 mlir::Value symAddr = getValueForVTableSymbol(op, rewriter, converter,
2601 op.getNameAttr(), eltType);
2602 if (!symAddr)
2603 return op.emitError() << "Unable to get value for vtable symbol";
2604
2606 0, op.getAddressPointAttr().getIndex(),
2607 op.getAddressPointAttr().getOffset()};
2608
2609 assert(eltType && "Shouldn't ever be missing an eltType here");
2610 mlir::LLVM::GEPNoWrapFlags inboundsNuw =
2611 mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
2612 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, targetType, eltType,
2613 symAddr, offsets, inboundsNuw);
2614 return mlir::success();
2615}
2616
2618 cir::VTableGetVPtrOp op, OpAdaptor adaptor,
2619 mlir::ConversionPatternRewriter &rewriter) const {
2620 // cir.vtable.get_vptr is equivalent to a bitcast from the source object
2621 // pointer to the vptr type. Since the LLVM dialect uses opaque pointers
2622 // we can just replace uses of this operation with the original pointer.
2623 mlir::Value srcVal = adaptor.getSrc();
2624 rewriter.replaceAllUsesWith(op, srcVal);
2625 rewriter.eraseOp(op);
2626 return mlir::success();
2627}
2628
2630 cir::VTableGetVirtualFnAddrOp op, OpAdaptor adaptor,
2631 mlir::ConversionPatternRewriter &rewriter) const {
2632 mlir::Type targetType = getTypeConverter()->convertType(op.getType());
2633 auto eltType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
2636 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
2637 op, targetType, eltType, adaptor.getVptr(), offsets,
2638 mlir::LLVM::GEPNoWrapFlags::inbounds);
2639 return mlir::success();
2640}
2641
2643 cir::VTTAddrPointOp op, OpAdaptor adaptor,
2644 mlir::ConversionPatternRewriter &rewriter) const {
2645 const mlir::Type resultType = getTypeConverter()->convertType(op.getType());
2647 mlir::Type eltType;
2648 mlir::Value llvmAddr = adaptor.getSymAddr();
2649
2650 if (op.getSymAddr()) {
2651 if (op.getOffset() == 0) {
2652 rewriter.replaceOp(op, {llvmAddr});
2653 return mlir::success();
2654 }
2655
2656 offsets.push_back(adaptor.getOffset());
2657 eltType = mlir::IntegerType::get(resultType.getContext(), 8,
2658 mlir::IntegerType::Signless);
2659 } else {
2660 llvmAddr = getValueForVTableSymbol(op, rewriter, getTypeConverter(),
2661 op.getNameAttr(), eltType);
2662 assert(eltType && "Shouldn't ever be missing an eltType here");
2663 offsets.push_back(0);
2664 offsets.push_back(adaptor.getOffset());
2665 }
2666 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
2667 op, resultType, eltType, llvmAddr, offsets,
2668 mlir::LLVM::GEPNoWrapFlags::inbounds);
2669 return mlir::success();
2670}
2671
2673 cir::StackSaveOp op, OpAdaptor adaptor,
2674 mlir::ConversionPatternRewriter &rewriter) const {
2675 const mlir::Type ptrTy = getTypeConverter()->convertType(op.getType());
2676 rewriter.replaceOpWithNewOp<mlir::LLVM::StackSaveOp>(op, ptrTy);
2677 return mlir::success();
2678}
2679
2681 cir::StackRestoreOp op, OpAdaptor adaptor,
2682 mlir::ConversionPatternRewriter &rewriter) const {
2683 rewriter.replaceOpWithNewOp<mlir::LLVM::StackRestoreOp>(op, adaptor.getPtr());
2684 return mlir::success();
2685}
2686
2688 cir::VecCreateOp op, OpAdaptor adaptor,
2689 mlir::ConversionPatternRewriter &rewriter) const {
2690 // Start with an 'undef' value for the vector. Then 'insertelement' for
2691 // each of the vector elements.
2692 const auto vecTy = mlir::cast<cir::VectorType>(op.getType());
2693 const mlir::Type llvmTy = typeConverter->convertType(vecTy);
2694 const mlir::Location loc = op.getLoc();
2695 mlir::Value result = rewriter.create<mlir::LLVM::PoisonOp>(loc, llvmTy);
2696 assert(vecTy.getSize() == op.getElements().size() &&
2697 "cir.vec.create op count doesn't match vector type elements count");
2698
2699 for (uint64_t i = 0; i < vecTy.getSize(); ++i) {
2700 const mlir::Value indexValue =
2701 rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI64Type(), i);
2702 result = rewriter.create<mlir::LLVM::InsertElementOp>(
2703 loc, result, adaptor.getElements()[i], indexValue);
2704 }
2705
2706 rewriter.replaceOp(op, result);
2707 return mlir::success();
2708}
2709
2711 cir::VecExtractOp op, OpAdaptor adaptor,
2712 mlir::ConversionPatternRewriter &rewriter) const {
2713 rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractElementOp>(
2714 op, adaptor.getVec(), adaptor.getIndex());
2715 return mlir::success();
2716}
2717
2719 cir::VecInsertOp op, OpAdaptor adaptor,
2720 mlir::ConversionPatternRewriter &rewriter) const {
2721 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertElementOp>(
2722 op, adaptor.getVec(), adaptor.getValue(), adaptor.getIndex());
2723 return mlir::success();
2724}
2725
2727 cir::VecCmpOp op, OpAdaptor adaptor,
2728 mlir::ConversionPatternRewriter &rewriter) const {
2729 mlir::Type elementType = elementTypeIfVector(op.getLhs().getType());
2730 mlir::Value bitResult;
2731 if (auto intType = mlir::dyn_cast<cir::IntType>(elementType)) {
2732 bitResult = rewriter.create<mlir::LLVM::ICmpOp>(
2733 op.getLoc(),
2734 convertCmpKindToICmpPredicate(op.getKind(), intType.isSigned()),
2735 adaptor.getLhs(), adaptor.getRhs());
2736 } else if (mlir::isa<cir::FPTypeInterface>(elementType)) {
2737 bitResult = rewriter.create<mlir::LLVM::FCmpOp>(
2738 op.getLoc(), convertCmpKindToFCmpPredicate(op.getKind()),
2739 adaptor.getLhs(), adaptor.getRhs());
2740 } else {
2741 return op.emitError() << "unsupported type for VecCmpOp: " << elementType;
2742 }
2743
2744 // LLVM IR vector comparison returns a vector of i1. This one-bit vector
2745 // must be sign-extended to the correct result type.
2746 rewriter.replaceOpWithNewOp<mlir::LLVM::SExtOp>(
2747 op, typeConverter->convertType(op.getType()), bitResult);
2748 return mlir::success();
2749}
2750
2752 cir::VecSplatOp op, OpAdaptor adaptor,
2753 mlir::ConversionPatternRewriter &rewriter) const {
2754 // Vector splat can be implemented with an `insertelement` and a
2755 // `shufflevector`, which is better than an `insertelement` for each
2756 // element in the vector. Start with an undef vector. Insert the value into
2757 // the first element. Then use a `shufflevector` with a mask of all 0 to
2758 // fill out the entire vector with that value.
2759 cir::VectorType vecTy = op.getType();
2760 mlir::Type llvmTy = typeConverter->convertType(vecTy);
2761 mlir::Location loc = op.getLoc();
2762 mlir::Value poison = rewriter.create<mlir::LLVM::PoisonOp>(loc, llvmTy);
2763
2764 mlir::Value elementValue = adaptor.getValue();
2765 if (elementValue.getDefiningOp<mlir::LLVM::PoisonOp>()) {
2766 // If the splat value is poison, then we can just use poison value
2767 // for the entire vector.
2768 rewriter.replaceOp(op, poison);
2769 return mlir::success();
2770 }
2771
2772 if (auto constValue = elementValue.getDefiningOp<mlir::LLVM::ConstantOp>()) {
2773 if (auto intAttr = dyn_cast<mlir::IntegerAttr>(constValue.getValue())) {
2774 mlir::DenseIntElementsAttr denseVec = mlir::DenseIntElementsAttr::get(
2775 mlir::cast<mlir::ShapedType>(llvmTy), intAttr.getValue());
2776 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
2777 op, denseVec.getType(), denseVec);
2778 return mlir::success();
2779 }
2780
2781 if (auto fpAttr = dyn_cast<mlir::FloatAttr>(constValue.getValue())) {
2782 mlir::DenseFPElementsAttr denseVec = mlir::DenseFPElementsAttr::get(
2783 mlir::cast<mlir::ShapedType>(llvmTy), fpAttr.getValue());
2784 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
2785 op, denseVec.getType(), denseVec);
2786 return mlir::success();
2787 }
2788 }
2789
2790 mlir::Value indexValue =
2791 rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI64Type(), 0);
2792 mlir::Value oneElement = rewriter.create<mlir::LLVM::InsertElementOp>(
2793 loc, poison, elementValue, indexValue);
2794 SmallVector<int32_t> zeroValues(vecTy.getSize(), 0);
2795 rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(op, oneElement,
2796 poison, zeroValues);
2797 return mlir::success();
2798}
2799
2801 cir::VecShuffleOp op, OpAdaptor adaptor,
2802 mlir::ConversionPatternRewriter &rewriter) const {
2803 // LLVM::ShuffleVectorOp takes an ArrayRef of int for the list of indices.
2804 // Convert the ClangIR ArrayAttr of IntAttr constants into a
2805 // SmallVector<int>.
2806 SmallVector<int, 8> indices;
2807 std::transform(
2808 op.getIndices().begin(), op.getIndices().end(),
2809 std::back_inserter(indices), [](mlir::Attribute intAttr) {
2810 return mlir::cast<cir::IntAttr>(intAttr).getValue().getSExtValue();
2811 });
2812 rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(
2813 op, adaptor.getVec1(), adaptor.getVec2(), indices);
2814 return mlir::success();
2815}
2816
2818 cir::VecShuffleDynamicOp op, OpAdaptor adaptor,
2819 mlir::ConversionPatternRewriter &rewriter) const {
2820 // LLVM IR does not have an operation that corresponds to this form of
2821 // the built-in.
2822 // __builtin_shufflevector(V, I)
2823 // is implemented as this pseudocode, where the for loop is unrolled
2824 // and N is the number of elements:
2825 //
2826 // result = undef
2827 // maskbits = NextPowerOf2(N - 1)
2828 // masked = I & maskbits
2829 // for (i in 0 <= i < N)
2830 // result[i] = V[masked[i]]
2831 mlir::Location loc = op.getLoc();
2832 mlir::Value input = adaptor.getVec();
2833 mlir::Type llvmIndexVecType =
2834 getTypeConverter()->convertType(op.getIndices().getType());
2835 mlir::Type llvmIndexType = getTypeConverter()->convertType(
2836 elementTypeIfVector(op.getIndices().getType()));
2837 uint64_t numElements =
2838 mlir::cast<cir::VectorType>(op.getVec().getType()).getSize();
2839
2840 uint64_t maskBits = llvm::NextPowerOf2(numElements - 1) - 1;
2841 mlir::Value maskValue = rewriter.create<mlir::LLVM::ConstantOp>(
2842 loc, llvmIndexType, rewriter.getIntegerAttr(llvmIndexType, maskBits));
2843 mlir::Value maskVector =
2844 rewriter.create<mlir::LLVM::UndefOp>(loc, llvmIndexVecType);
2845
2846 for (uint64_t i = 0; i < numElements; ++i) {
2847 mlir::Value idxValue =
2848 rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI64Type(), i);
2849 maskVector = rewriter.create<mlir::LLVM::InsertElementOp>(
2850 loc, maskVector, maskValue, idxValue);
2851 }
2852
2853 mlir::Value maskedIndices = rewriter.create<mlir::LLVM::AndOp>(
2854 loc, llvmIndexVecType, adaptor.getIndices(), maskVector);
2855 mlir::Value result = rewriter.create<mlir::LLVM::UndefOp>(
2856 loc, getTypeConverter()->convertType(op.getVec().getType()));
2857 for (uint64_t i = 0; i < numElements; ++i) {
2858 mlir::Value iValue =
2859 rewriter.create<mlir::LLVM::ConstantOp>(loc, rewriter.getI64Type(), i);
2860 mlir::Value indexValue = rewriter.create<mlir::LLVM::ExtractElementOp>(
2861 loc, maskedIndices, iValue);
2862 mlir::Value valueAtIndex =
2863 rewriter.create<mlir::LLVM::ExtractElementOp>(loc, input, indexValue);
2864 result = rewriter.create<mlir::LLVM::InsertElementOp>(loc, result,
2865 valueAtIndex, iValue);
2866 }
2867 rewriter.replaceOp(op, result);
2868 return mlir::success();
2869}
2870
2872 cir::VecTernaryOp op, OpAdaptor adaptor,
2873 mlir::ConversionPatternRewriter &rewriter) const {
2874 // Convert `cond` into a vector of i1, then use that in a `select` op.
2875 mlir::Value bitVec = rewriter.create<mlir::LLVM::ICmpOp>(
2876 op.getLoc(), mlir::LLVM::ICmpPredicate::ne, adaptor.getCond(),
2877 rewriter.create<mlir::LLVM::ZeroOp>(
2878 op.getCond().getLoc(),
2879 typeConverter->convertType(op.getCond().getType())));
2880 rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(
2881 op, bitVec, adaptor.getLhs(), adaptor.getRhs());
2882 return mlir::success();
2883}
2884
2886 cir::ComplexAddOp op, OpAdaptor adaptor,
2887 mlir::ConversionPatternRewriter &rewriter) const {
2888 mlir::Value lhs = adaptor.getLhs();
2889 mlir::Value rhs = adaptor.getRhs();
2890 mlir::Location loc = op.getLoc();
2891
2892 auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType());
2893 mlir::Type complexElemTy =
2894 getTypeConverter()->convertType(complexType.getElementType());
2895 auto lhsReal =
2896 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 0);
2897 auto lhsImag =
2898 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 1);
2899 auto rhsReal =
2900 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 0);
2901 auto rhsImag =
2902 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 1);
2903
2904 mlir::Value newReal;
2905 mlir::Value newImag;
2906 if (complexElemTy.isInteger()) {
2907 newReal = rewriter.create<mlir::LLVM::AddOp>(loc, complexElemTy, lhsReal,
2908 rhsReal);
2909 newImag = rewriter.create<mlir::LLVM::AddOp>(loc, complexElemTy, lhsImag,
2910 rhsImag);
2911 } else {
2914 newReal = rewriter.create<mlir::LLVM::FAddOp>(loc, complexElemTy, lhsReal,
2915 rhsReal);
2916 newImag = rewriter.create<mlir::LLVM::FAddOp>(loc, complexElemTy, lhsImag,
2917 rhsImag);
2918 }
2919
2920 mlir::Type complexLLVMTy =
2921 getTypeConverter()->convertType(op.getResult().getType());
2922 auto initialComplex =
2923 rewriter.create<mlir::LLVM::PoisonOp>(op->getLoc(), complexLLVMTy);
2924
2925 auto realComplex = rewriter.create<mlir::LLVM::InsertValueOp>(
2926 op->getLoc(), initialComplex, newReal, 0);
2927
2928 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(op, realComplex,
2929 newImag, 1);
2930
2931 return mlir::success();
2932}
2933
2935 cir::ComplexCreateOp op, OpAdaptor adaptor,
2936 mlir::ConversionPatternRewriter &rewriter) const {
2937 mlir::Type complexLLVMTy =
2938 getTypeConverter()->convertType(op.getResult().getType());
2939 auto initialComplex =
2940 rewriter.create<mlir::LLVM::UndefOp>(op->getLoc(), complexLLVMTy);
2941
2942 auto realComplex = rewriter.create<mlir::LLVM::InsertValueOp>(
2943 op->getLoc(), initialComplex, adaptor.getReal(), 0);
2944
2945 auto complex = rewriter.create<mlir::LLVM::InsertValueOp>(
2946 op->getLoc(), realComplex, adaptor.getImag(), 1);
2947
2948 rewriter.replaceOp(op, complex);
2949 return mlir::success();
2950}
2951
2953 cir::ComplexRealOp op, OpAdaptor adaptor,
2954 mlir::ConversionPatternRewriter &rewriter) const {
2955 mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
2956 rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractValueOp>(
2957 op, resultLLVMTy, adaptor.getOperand(), llvm::ArrayRef<std::int64_t>{0});
2958 return mlir::success();
2959}
2960
2962 cir::ComplexSubOp op, OpAdaptor adaptor,
2963 mlir::ConversionPatternRewriter &rewriter) const {
2964 mlir::Value lhs = adaptor.getLhs();
2965 mlir::Value rhs = adaptor.getRhs();
2966 mlir::Location loc = op.getLoc();
2967
2968 auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType());
2969 mlir::Type complexElemTy =
2970 getTypeConverter()->convertType(complexType.getElementType());
2971 auto lhsReal =
2972 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 0);
2973 auto lhsImag =
2974 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 1);
2975 auto rhsReal =
2976 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 0);
2977 auto rhsImag =
2978 rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 1);
2979
2980 mlir::Value newReal;
2981 mlir::Value newImag;
2982 if (complexElemTy.isInteger()) {
2983 newReal = rewriter.create<mlir::LLVM::SubOp>(loc, complexElemTy, lhsReal,
2984 rhsReal);
2985 newImag = rewriter.create<mlir::LLVM::SubOp>(loc, complexElemTy, lhsImag,
2986 rhsImag);
2987 } else {
2990 newReal = rewriter.create<mlir::LLVM::FSubOp>(loc, complexElemTy, lhsReal,
2991 rhsReal);
2992 newImag = rewriter.create<mlir::LLVM::FSubOp>(loc, complexElemTy, lhsImag,
2993 rhsImag);
2994 }
2995
2996 mlir::Type complexLLVMTy =
2997 getTypeConverter()->convertType(op.getResult().getType());
2998 auto initialComplex =
2999 rewriter.create<mlir::LLVM::PoisonOp>(op->getLoc(), complexLLVMTy);
3000
3001 auto realComplex = rewriter.create<mlir::LLVM::InsertValueOp>(
3002 op->getLoc(), initialComplex, newReal, 0);
3003
3004 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(op, realComplex,
3005 newImag, 1);
3006
3007 return mlir::success();
3008}
3009
3011 cir::ComplexImagOp op, OpAdaptor adaptor,
3012 mlir::ConversionPatternRewriter &rewriter) const {
3013 mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
3014 rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractValueOp>(
3015 op, resultLLVMTy, adaptor.getOperand(), llvm::ArrayRef<std::int64_t>{1});
3016 return mlir::success();
3017}
3018
3019mlir::IntegerType computeBitfieldIntType(mlir::Type storageType,
3020 mlir::MLIRContext *context,
3021 unsigned &storageSize) {
3022 return TypeSwitch<mlir::Type, mlir::IntegerType>(storageType)
3023 .Case<cir::ArrayType>([&](cir::ArrayType atTy) {
3024 storageSize = atTy.getSize() * 8;
3025 return mlir::IntegerType::get(context, storageSize);
3026 })
3027 .Case<cir::IntType>([&](cir::IntType intTy) {
3028 storageSize = intTy.getWidth();
3029 return mlir::IntegerType::get(context, storageSize);
3030 })
3031 .Default([](mlir::Type) -> mlir::IntegerType {
3032 llvm_unreachable(
3033 "Either ArrayType or IntType expected for bitfields storage");
3034 });
3035}
3036
3038 cir::SetBitfieldOp op, OpAdaptor adaptor,
3039 mlir::ConversionPatternRewriter &rewriter) const {
3040 mlir::OpBuilder::InsertionGuard guard(rewriter);
3041 rewriter.setInsertionPoint(op);
3042
3043 cir::BitfieldInfoAttr info = op.getBitfieldInfo();
3044 uint64_t size = info.getSize();
3045 uint64_t offset = info.getOffset();
3046 mlir::Type storageType = info.getStorageType();
3047 mlir::MLIRContext *context = storageType.getContext();
3048
3049 unsigned storageSize = 0;
3050
3051 mlir::IntegerType intType =
3052 computeBitfieldIntType(storageType, context, storageSize);
3053
3054 mlir::Value srcVal = createIntCast(rewriter, adaptor.getSrc(), intType);
3055 unsigned srcWidth = storageSize;
3056 mlir::Value resultVal = srcVal;
3057
3058 if (storageSize != size) {
3059 assert(storageSize > size && "Invalid bitfield size.");
3060
3061 mlir::Value val = rewriter.create<mlir::LLVM::LoadOp>(
3062 op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(),
3063 op.getIsVolatile());
3064
3065 srcVal =
3066 createAnd(rewriter, srcVal, llvm::APInt::getLowBitsSet(srcWidth, size));
3067 resultVal = srcVal;
3068 srcVal = createShL(rewriter, srcVal, offset);
3069
3070 // Mask out the original value.
3071 val = createAnd(rewriter, val,
3072 ~llvm::APInt::getBitsSet(srcWidth, offset, offset + size));
3073
3074 // Or together the unchanged values and the source value.
3075 srcVal = rewriter.create<mlir::LLVM::OrOp>(op.getLoc(), val, srcVal);
3076 }
3077
3078 rewriter.create<mlir::LLVM::StoreOp>(op.getLoc(), srcVal, adaptor.getAddr(),
3079 op.getAlignment(), op.getIsVolatile());
3080
3081 mlir::Type resultTy = getTypeConverter()->convertType(op.getType());
3082
3083 if (info.getIsSigned()) {
3084 assert(size <= storageSize);
3085 unsigned highBits = storageSize - size;
3086
3087 if (highBits) {
3088 resultVal = createShL(rewriter, resultVal, highBits);
3089 resultVal = createAShR(rewriter, resultVal, highBits);
3090 }
3091 }
3092
3093 resultVal = createIntCast(rewriter, resultVal,
3094 mlir::cast<mlir::IntegerType>(resultTy),
3095 info.getIsSigned());
3096
3097 rewriter.replaceOp(op, resultVal);
3098 return mlir::success();
3099}
3100
3102 cir::ComplexImagPtrOp op, OpAdaptor adaptor,
3103 mlir::ConversionPatternRewriter &rewriter) const {
3104 cir::PointerType operandTy = op.getOperand().getType();
3105 mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
3106 mlir::Type elementLLVMTy =
3107 getTypeConverter()->convertType(operandTy.getPointee());
3108
3109 mlir::LLVM::GEPArg gepIndices[2] = {{0}, {1}};
3110 mlir::LLVM::GEPNoWrapFlags inboundsNuw =
3111 mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
3112 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
3113 op, resultLLVMTy, elementLLVMTy, adaptor.getOperand(), gepIndices,
3114 inboundsNuw);
3115 return mlir::success();
3116}
3117
3119 cir::ComplexRealPtrOp op, OpAdaptor adaptor,
3120 mlir::ConversionPatternRewriter &rewriter) const {
3121 cir::PointerType operandTy = op.getOperand().getType();
3122 mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
3123 mlir::Type elementLLVMTy =
3124 getTypeConverter()->convertType(operandTy.getPointee());
3125
3126 mlir::LLVM::GEPArg gepIndices[2] = {0, 0};
3127 mlir::LLVM::GEPNoWrapFlags inboundsNuw =
3128 mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
3129 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
3130 op, resultLLVMTy, elementLLVMTy, adaptor.getOperand(), gepIndices,
3131 inboundsNuw);
3132 return mlir::success();
3133}
3134
3136 cir::GetBitfieldOp op, OpAdaptor adaptor,
3137 mlir::ConversionPatternRewriter &rewriter) const {
3138
3139 mlir::OpBuilder::InsertionGuard guard(rewriter);
3140 rewriter.setInsertionPoint(op);
3141
3142 cir::BitfieldInfoAttr info = op.getBitfieldInfo();
3143 uint64_t size = info.getSize();
3144 uint64_t offset = info.getOffset();
3145 mlir::Type storageType = info.getStorageType();
3146 mlir::MLIRContext *context = storageType.getContext();
3147 unsigned storageSize = 0;
3148
3149 mlir::IntegerType intType =
3150 computeBitfieldIntType(storageType, context, storageSize);
3151
3152 mlir::Value val = rewriter.create<mlir::LLVM::LoadOp>(
3153 op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(),
3154 op.getIsVolatile());
3155 val = rewriter.create<mlir::LLVM::BitcastOp>(op.getLoc(), intType, val);
3156
3157 if (info.getIsSigned()) {
3158 assert(static_cast<unsigned>(offset + size) <= storageSize);
3159 unsigned highBits = storageSize - offset - size;
3160 val = createShL(rewriter, val, highBits);
3161 val = createAShR(rewriter, val, offset + highBits);
3162 } else {
3163 val = createLShR(rewriter, val, offset);
3164
3165 if (static_cast<unsigned>(offset) + size < storageSize)
3166 val = createAnd(rewriter, val,
3167 llvm::APInt::getLowBitsSet(storageSize, size));
3168 }
3169
3170 mlir::Type resTy = getTypeConverter()->convertType(op.getType());
3171 mlir::Value newOp = createIntCast(
3172 rewriter, val, mlir::cast<mlir::IntegerType>(resTy), info.getIsSigned());
3173 rewriter.replaceOp(op, newOp);
3174 return mlir::success();
3175}
3176
3178 cir::InlineAsmOp op, OpAdaptor adaptor,
3179 mlir::ConversionPatternRewriter &rewriter) const {
3180 mlir::Type llResTy;
3181 if (op.getNumResults())
3182 llResTy = getTypeConverter()->convertType(op.getType(0));
3183
3184 cir::AsmFlavor dialect = op.getAsmFlavor();
3185 mlir::LLVM::AsmDialect llDialect = dialect == cir::AsmFlavor::x86_att
3186 ? mlir::LLVM::AsmDialect::AD_ATT
3187 : mlir::LLVM::AsmDialect::AD_Intel;
3188
3190 StringRef llvmAttrName = mlir::LLVM::InlineAsmOp::getElementTypeAttrName();
3191
3192 // this is for the lowering to LLVM from LLVM dialect. Otherwise, if we
3193 // don't have the result (i.e. void type as a result of operation), the
3194 // element type attribute will be attached to the whole instruction, but not
3195 // to the operand
3196 if (!op.getNumResults())
3197 opAttrs.push_back(mlir::Attribute());
3198
3199 SmallVector<mlir::Value> llvmOperands;
3200 SmallVector<mlir::Value> cirOperands;
3201 for (auto const &[llvmOp, cirOp] :
3202 zip(adaptor.getAsmOperands(), op.getAsmOperands())) {
3203 append_range(llvmOperands, llvmOp);
3204 append_range(cirOperands, cirOp);
3205 }
3206
3207 // so far we infer the llvm dialect element type attr from
3208 // CIR operand type.
3209 for (auto const &[cirOpAttr, cirOp] :
3210 zip(op.getOperandAttrs(), cirOperands)) {
3211 if (!cirOpAttr) {
3212 opAttrs.push_back(mlir::Attribute());
3213 continue;
3214 }
3215
3217 cir::PointerType typ = mlir::cast<cir::PointerType>(cirOp.getType());
3218 mlir::TypeAttr typAttr = mlir::TypeAttr::get(convertTypeForMemory(
3219 *getTypeConverter(), dataLayout, typ.getPointee()));
3220
3221 attrs.push_back(rewriter.getNamedAttr(llvmAttrName, typAttr));
3222 mlir::DictionaryAttr newDict = rewriter.getDictionaryAttr(attrs);
3223 opAttrs.push_back(newDict);
3224 }
3225
3226 rewriter.replaceOpWithNewOp<mlir::LLVM::InlineAsmOp>(
3227 op, llResTy, llvmOperands, op.getAsmStringAttr(), op.getConstraintsAttr(),
3228 op.getSideEffectsAttr(),
3229 /*is_align_stack*/ mlir::UnitAttr(),
3230 /*tail_call_kind*/
3231 mlir::LLVM::TailCallKindAttr::get(
3232 getContext(), mlir::LLVM::tailcallkind::TailCallKind::None),
3233 mlir::LLVM::AsmDialectAttr::get(getContext(), llDialect),
3234 rewriter.getArrayAttr(opAttrs));
3235
3236 return mlir::success();
3237}
3238
3240 cir::VAStartOp op, OpAdaptor adaptor,
3241 mlir::ConversionPatternRewriter &rewriter) const {
3242 auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
3243 auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
3244 adaptor.getArgList());
3245 rewriter.replaceOpWithNewOp<mlir::LLVM::VaStartOp>(op, vaList);
3246 return mlir::success();
3247}
3248
3250 cir::VAEndOp op, OpAdaptor adaptor,
3251 mlir::ConversionPatternRewriter &rewriter) const {
3252 auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
3253 auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
3254 adaptor.getArgList());
3255 rewriter.replaceOpWithNewOp<mlir::LLVM::VaEndOp>(op, vaList);
3256 return mlir::success();
3257}
3258
3260 cir::VAArgOp op, OpAdaptor adaptor,
3261 mlir::ConversionPatternRewriter &rewriter) const {
3263 auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
3264 auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
3265 adaptor.getArgList());
3266
3267 mlir::Type llvmType =
3268 getTypeConverter()->convertType(op->getResultTypes().front());
3269 if (!llvmType)
3270 return mlir::failure();
3271
3272 rewriter.replaceOpWithNewOp<mlir::LLVM::VaArgOp>(op, llvmType, vaList);
3273 return mlir::success();
3274}
3275
3276std::unique_ptr<mlir::Pass> createConvertCIRToLLVMPass() {
3277 return std::make_unique<ConvertCIRToLLVMPass>();
3278}
3279
3280void populateCIRToLLVMPasses(mlir::OpPassManager &pm) {
3282 pm.addPass(createConvertCIRToLLVMPass());
3283}
3284
3285std::unique_ptr<llvm::Module>
3286lowerDirectlyFromCIRToLLVMIR(mlir::ModuleOp mlirModule, LLVMContext &llvmCtx) {
3287 llvm::TimeTraceScope scope("lower from CIR to LLVM directly");
3288
3289 mlir::MLIRContext *mlirCtx = mlirModule.getContext();
3290
3291 mlir::PassManager pm(mlirCtx);
3293
3294 (void)mlir::applyPassManagerCLOptions(pm);
3295
3296 if (mlir::failed(pm.run(mlirModule))) {
3297 // FIXME: Handle any errors where they occurs and return a nullptr here.
3298 report_fatal_error(
3299 "The pass manager failed to lower CIR to LLVMIR dialect!");
3300 }
3301
3302 mlir::registerBuiltinDialectTranslation(*mlirCtx);
3303 mlir::registerLLVMDialectTranslation(*mlirCtx);
3305
3306 llvm::TimeTraceScope translateScope("translateModuleToLLVMIR");
3307
3308 StringRef moduleName = mlirModule.getName().value_or("CIRToLLVMModule");
3309 std::unique_ptr<llvm::Module> llvmModule =
3310 mlir::translateModuleToLLVMIR(mlirModule, llvmCtx, moduleName);
3311
3312 if (!llvmModule) {
3313 // FIXME: Handle any errors where they occurs and return a nullptr here.
3314 report_fatal_error("Lowering from LLVMIR dialect to llvm IR failed!");
3315 }
3316
3317 return llvmModule;
3318}
3319} // namespace direct
3320} // namespace cir
static bool isUnsigned(SValBuilder &SVB, NonLoc Value)
mlir::Value createLShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs)
mlir::Value createShL(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs)
std::optional< mlir::Attribute > lowerConstArrayAttr(cir::ConstArrayAttr constArr, const mlir::TypeConverter *converter)
mlir::Value createAShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs)
mlir::Value createAnd(mlir::OpBuilder &bld, mlir::Value lhs, const llvm::APInt &rhs)
static bool isVector(QualType QT, QualType ElementType)
This helper function returns true if QT is a vector type that has element type ElementType.
Definition: SemaExpr.cpp:9302
mlir::Value visitCirAttr(cir::IntAttr intAttr)
IntAttr visitor.
mlir::Value visit(mlir::Attribute attr)
CIRAttrToValue(mlir::Operation *parentOp, mlir::ConversionPatternRewriter &rewriter, const mlir::TypeConverter *converter)
mlir::LogicalResult matchAndRewrite(cir::AllocaOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::AssumeAlignedOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::AssumeOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::AssumeSepStorageOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::BaseClassAddrOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::BinOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::BitClrsbOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::BitClzOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::BitCtzOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::BitFfsOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::BitParityOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::BitPopcountOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::BitReverseOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::BrCondOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::BrOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::ByteSwapOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::CallOp op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) const override
mlir::LogicalResult matchAndRewrite(cir::CastOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::CmpOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::ComplexAddOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::ComplexCreateOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::ComplexImagOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::ComplexImagPtrOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::ComplexRealOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::ComplexRealPtrOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::ComplexSubOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::ConstantOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::ExpectOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::FAbsOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::FrameAddrOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::FuncOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::GetBitfieldOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::GetGlobalOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::GetMemberOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::GlobalOp op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) const override
mlir::LogicalResult matchAndRewrite(cir::InlineAsmOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::LoadOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::PtrStrideOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::ReturnAddrOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::ReturnOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::RotateOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::SelectOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::SetBitfieldOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::ShiftOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::StackRestoreOp op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) const override
mlir::LogicalResult matchAndRewrite(cir::StackSaveOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::StoreOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::SwitchFlatOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::ThrowOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::TrapOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::UnaryOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::UnreachableOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VAArgOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VAEndOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VAStartOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VTTAddrPointOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VTableAddrPointOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VTableGetVPtrOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VTableGetVirtualFnAddrOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VecCmpOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VecCreateOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VecExtractOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VecInsertOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VecShuffleDynamicOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VecShuffleOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VecSplatOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::LogicalResult matchAndRewrite(cir::VecTernaryOp op, OpAdaptor, mlir::ConversionPatternRewriter &) const override
mlir::Attribute visit(mlir::Attribute attr)
mlir::Attribute visitCirAttr(cir::FPAttr attr)
mlir::Attribute visitCirAttr(cir::BoolAttr attr)
GlobalInitAttrRewriter(mlir::Type type, mlir::ConversionPatternRewriter &rewriter)
mlir::Attribute visitCirAttr(cir::IntAttr attr)
void createLLVMFuncOpIfNotExist(mlir::ConversionPatternRewriter &rewriter, mlir::Operation *srcOp, llvm::StringRef fnName, mlir::Type fnTy)
static void collectUnreachable(mlir::Operation *parent, llvm::SmallVector< mlir::Operation * > &ops)
static mlir::LLVM::ICmpPredicate convertCmpKindToICmpPredicate(cir::CmpOpKind kind, bool isSigned)
Convert from a CIR comparison kind to an LLVM IR integral comparison kind.
static mlir::Value getLLVMIntCast(mlir::ConversionPatternRewriter &rewriter, mlir::Value llvmSrc, mlir::Type llvmDstIntTy, bool isUnsigned, uint64_t cirSrcWidth, uint64_t cirDstIntWidth)
void convertSideEffectForCall(mlir::Operation *callOp, bool isNothrow, cir::SideEffect sideEffect, mlir::LLVM::MemoryEffectsAttr &memoryEffect, bool &noUnwind, bool &willReturn)
static mlir::Value emitFromMemory(mlir::ConversionPatternRewriter &rewriter, mlir::DataLayout const &dataLayout, cir::LoadOp op, mlir::Value value)
Emits the value from memory as expected by its users.
mlir::IntegerType computeBitfieldIntType(mlir::Type storageType, mlir::MLIRContext *context, unsigned &storageSize)
static mlir::LLVM::CallIntrinsicOp createCallLLVMIntrinsicOp(mlir::ConversionPatternRewriter &rewriter, mlir::Location loc, const llvm::Twine &intrinsicName, mlir::Type resultTy, mlir::ValueRange operands)
bool hasTrailingZeros(cir::ConstArrayAttr attr)
static mlir::LogicalResult rewriteCallOrInvoke(mlir::Operation *op, mlir::ValueRange callOperands, mlir::ConversionPatternRewriter &rewriter, const mlir::TypeConverter *converter, mlir::FlatSymbolRefAttr calleeAttr)
std::unique_ptr< llvm::Module > lowerDirectlyFromCIRToLLVMIR(mlir::ModuleOp mlirModule, llvm::LLVMContext &llvmCtx)
mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp, const mlir::Attribute attr, mlir::ConversionPatternRewriter &rewriter, const mlir::TypeConverter *converter)
Switches on the type of attribute and calls the appropriate conversion.
static mlir::LLVM::CallIntrinsicOp replaceOpWithCallLLVMIntrinsicOp(mlir::ConversionPatternRewriter &rewriter, mlir::Operation *op, const llvm::Twine &intrinsicName, mlir::Type resultTy, mlir::ValueRange operands)
static void prepareTypeConverter(mlir::LLVMTypeConverter &converter, mlir::DataLayout &dataLayout)
static mlir::LLVM::AtomicOrdering getLLVMMemOrder(std::optional< cir::MemOrder > memorder)
std::unique_ptr< mlir::Pass > createConvertCIRToLLVMPass()
Create a pass that fully lowers CIR to the LLVMIR dialect.
static mlir::LLVM::FCmpPredicate convertCmpKindToFCmpPredicate(cir::CmpOpKind kind)
Convert from a CIR comparison kind to an LLVM IR floating-point comparison kind.
static mlir::LLVM::Visibility lowerCIRVisibilityToLLVMVisibility(cir::VisibilityKind visibilityKind)
void populateCIRToLLVMPasses(mlir::OpPassManager &pm)
Adds passes that fully lower CIR to the LLVMIR dialect.
mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage)
static mlir::Type convertTypeForMemory(const mlir::TypeConverter &converter, mlir::DataLayout const &dataLayout, mlir::Type type)
Given a type convertor and a data layout, convert the given type to a type that is suitable for memor...
Definition: LowerToLLVM.cpp:68
static mlir::Value createIntCast(mlir::OpBuilder &bld, mlir::Value src, mlir::IntegerType dstTy, bool isSigned=false)
Definition: LowerToLLVM.cpp:81
static mlir::Value getValueForVTableSymbol(mlir::Operation *op, mlir::ConversionPatternRewriter &rewriter, const mlir::TypeConverter *converter, mlir::FlatSymbolRefAttr nameAttr, mlir::Type &eltType)
static mlir::Value emitToMemory(mlir::ConversionPatternRewriter &rewriter, mlir::DataLayout const &dataLayout, mlir::Type origType, mlir::Value value)
Emits a value to memory with the expected scalar type.
static bool isIntTypeUnsigned(mlir::Type type)
Definition: ABIArgInfo.h:22
Diagnostic wrappers for TextAPI types for error reporting.
Definition: Dominators.h:30
void populateCIRPreLoweringPasses(mlir::OpPassManager &pm)
void registerCIRDialectTranslation(mlir::MLIRContext &context)
char __ovld __cnfn clz(char)
Returns the number of leading 0-bits in x, starting at the most significant bit position.
char __ovld __cnfn ctz(char)
Returns the count of trailing 0-bits in x.
char __ovld __cnfn select(char, char, char)
For each component of a vector type, result[i] = if MSB of c[i] is set ? b[i] : a[i].
static bool dataMemberType()
static bool opGlobalConstant()
static bool addressSpace()
static bool opGlobalThreadLocal()
static bool globalViewIntLowering()
static bool opAllocaAnnotations()
static bool opLoadStoreTbaa()
static bool optInfoAttr()
static bool opFuncExtraAttrs()
static bool opCallLandingPad()
static bool vaArgABILowering()
static bool fpConstraints()
static bool lowerModeOptLevel()
static bool opCallCallConv()
static bool opFuncCallingConv()
static bool fastMathFlags()
static bool opCallContinueBlock()
static bool opLoadStoreNontemporal()
static bool opFuncMultipleReturnVals()
static bool methodType()
static bool opAllocaDynAllocSize()
StringRef getDescription() const override
StringRef getArgument() const override
void getDependentDialects(mlir::DialectRegistry &registry) const override
void processCIRAttrs(mlir::ModuleOp module)