10#include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h"
11#include "mlir/IR/Builders.h"
12#include "mlir/IR/Dominance.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/IR/Attributes.h"
21#include "llvm/Support/MathExtras.h"
28using namespace mlir::abi;
71cir::PointerType getPtrTyInAddrSpace(mlir::Type pointee,
unsigned addrSpace) {
72 mlir::ptr::MemorySpaceAttrInterface asAttr;
74 asAttr = cir::TargetAddressSpaceAttr::get(pointee.getContext(), addrSpace);
75 return cir::PointerType::get(pointee, asAttr);
80cir::PointerType getIndirectPtrTy(mlir::Type pointee,
81 const ArgClassification &ac) {
83 (ac.kind == ArgKind::Indirect || ac.kind == ArgKind::IndirectAliased) &&
84 "expected an indirect classification");
85 return getPtrTyInAddrSpace(pointee, ac.indirectAddrSpace);
90cir::StructType getUnionFlattenType(cir::UnionType unionTy,
91 const mlir::DataLayout &dl) {
93 if (mlir::Type storage = unionTy.getUnionStorageType(dl))
94 fields.push_back(storage);
95 if (mlir::Type padding = unionTy.getPadding())
96 fields.push_back(padding);
97 return cir::StructType::get(unionTy.getContext(), fields, unionTy.getPacked(),
114 const mlir::DataLayout &dl) {
115 if (ac.kind != ArgKind::Direct || !ac.coercedType || !ac.canFlatten)
117 auto recTy = dyn_cast<cir::RecordType>(ac.coercedType);
118 if (
auto unionTy = dyn_cast_if_present<cir::UnionType>(recTy))
119 recTy = getUnionFlattenType(unionTy, dl);
120 if (!recTy || !recTy.isStruct() || recTy.getMembers().empty())
128bool isRecordExtend(
const ArgClassification &ac, mlir::Type origTy) {
129 if (ac.kind != ArgKind::Extend || !mlir::isa<cir::RecordType>(origTy))
131 assert(ac.coercedType &&
"an Extend of a record names its first member type");
141mlir::LogicalResult buildNewArgTypes(
143 SmallVectorImpl<mlir::Type> &newArgTypes,
const mlir::DataLayout &dl,
144 function_ref<mlir::InFlightDiagnostic()> emitError) {
145 assert(newArgTypes.empty() &&
"expected an empty output vector");
146 newArgTypes.reserve(oldArgTypes.size());
147 for (
auto [idx, ac] : llvm::enumerate(fc.argInfos)) {
148 mlir::Type origTy = oldArgTypes[idx];
150 case ArgKind::Direct:
154 llvm::append_range(newArgTypes, flatTy.getMembers());
159 newArgTypes.push_back(ac.coercedType ? ac.coercedType : origTy);
162 case ArgKind::Ignore:
164 case ArgKind::Expand: {
168 auto recTy = cast<cir::RecordType>(origTy);
169 assert(recTy.isStruct() &&
170 "Expand classification requires a struct type, not a union");
171 assert(!recTy.getMembers().empty() &&
172 "Expand classification requires at least one struct field");
173 llvm::append_range(newArgTypes, recTy.getMembers());
176 case ArgKind::Extend:
184 newArgTypes.push_back(isRecordExtend(ac, origTy) ? ac.coercedType
187 case ArgKind::Indirect:
188 case ArgKind::IndirectAliased:
189 newArgTypes.push_back(getIndirectPtrTy(origTy, ac));
193 return mlir::success();
201computeNewReturnType(mlir::Type origRetTy,
const ArgClassification &retInfo,
202 mlir::MLIRContext *ctx,
203 function_ref<mlir::InFlightDiagnostic()> emitError) {
204 switch (retInfo.kind) {
205 case ArgKind::Direct:
208 return retInfo.coercedType ? retInfo.coercedType : origRetTy;
209 case ArgKind::Ignore:
210 return cir::VoidType::get(ctx);
211 case ArgKind::Expand:
212 emitError() <<
"Expand return is not allowed (classic codegen rejects "
213 <<
"it in EmitFunctionEpilog)";
215 case ArgKind::IndirectAliased:
216 emitError() <<
"IndirectAliased return is not supported";
218 case ArgKind::Extend:
223 case ArgKind::Indirect:
228 return cir::VoidType::get(ctx);
230 llvm_unreachable(
"all ArgKind cases handled");
238mlir::Value createIgnoredValue(mlir::OpBuilder &builder, mlir::Location loc,
240 return cir::ConstantOp::create(builder, loc, ty, cir::PoisonAttr::get(ty));
248static bool isCoercionWiderThanMemory(mlir::Type origTy, mlir::Type coercedTy,
249 const mlir::DataLayout &dl) {
250 uint64_t memoryBits = dl.getTypeSizeInBits(origTy).getFixedValue();
251 if (
auto vecTy = mlir::dyn_cast<cir::VectorType>(origTy);
252 vecTy && mlir::isa<cir::BoolType>(vecTy.getElementType()))
253 memoryBits = vecTy.getBoolStorageWidth();
254 return dl.getTypeSizeInBits(coercedTy).getFixedValue() > memoryBits;
263static bool needsArgAttrUpdate(
const FunctionClassification &fc) {
264 return llvm::any_of(fc.argInfos, [](
const ArgClassification &ac) {
265 return ac.kind == ArgKind::Ignore || ac.kind == ArgKind::Extend ||
266 ac.kind == ArgKind::Indirect ||
267 ac.kind == ArgKind::IndirectAliased || ac.kind == ArgKind::Expand ||
268 (ac.kind == ArgKind::Direct && (ac.coercedType || ac.directAlign));
282mlir::ArrayAttr updateArgAttrs(mlir::MLIRContext *ctx, mlir::ModuleOp module,
284 mlir::ArrayAttr existingArgAttrs,
285 const FunctionClassification &fc,
286 const mlir::DataLayout &dl,
287 bool enableNoundefAttrs,
bool isCallSite) {
288 mlir::Builder builder(ctx);
290 newArgAttrs.reserve(fc.argInfos.size());
291 for (
auto [oldIdx, ac] : llvm::enumerate(fc.argInfos)) {
292 if (ac.kind == ArgKind::Ignore)
294 mlir::DictionaryAttr existing = builder.getDictionaryAttr({});
295 if (existingArgAttrs && oldIdx < existingArgAttrs.size())
296 existing = mlir::cast<mlir::DictionaryAttr>(existingArgAttrs[oldIdx]);
297 if (ac.kind == ArgKind::Direct && ac.coercedType &&
298 isCoercionWiderThanMemory(origArgTypes[oldIdx], ac.coercedType, dl)) {
299 mlir::NamedAttrList attrs(existing);
300 attrs.erase(mlir::LLVM::LLVMDialect::getNoUndefAttrName());
301 existing = attrs.getDictionary(ctx);
303 if (ac.kind == ArgKind::Direct && ac.directAlign) {
304 mlir::NamedAttrList attrs(existing);
305 attrs.set(mlir::LLVM::LLVMDialect::getStackAlignmentAttrName(),
306 builder.getI64IntegerAttr(ac.directAlign->value()));
307 existing = attrs.getDictionary(ctx);
312 newArgAttrs.append(flatTy.getNumElements(), existing);
313 }
else if (ac.kind == ArgKind::Expand) {
316 auto recTy = cast<cir::RecordType>(origArgTypes[oldIdx]);
317 newArgAttrs.append(recTy.getNumElements(), builder.getDictionaryAttr({}));
318 }
else if (ac.kind == ArgKind::Extend) {
322 StringRef attrName = ac.signExtend
323 ? mlir::LLVM::LLVMDialect::getSExtAttrName()
324 :
mlir::LLVM::LLVMDialect::getZExtAttrName();
325 mlir::NamedAttrList attrs(existing);
327 if (isCallSite || !ac.conservative)
328 attrs.set(attrName, builder.getUnitAttr());
329 if (enableNoundefAttrs)
330 attrs.set(mlir::LLVM::LLVMDialect::getNoUndefAttrName(),
331 builder.getUnitAttr());
332 newArgAttrs.push_back(attrs.getDictionary(ctx));
333 }
else if (ac.kind == ArgKind::Indirect) {
341 mlir::Type pointeeTy = origArgTypes[oldIdx];
342 mlir::NamedAttrList attrs(existing);
343 attrs.erase(mlir::LLVM::LLVMDialect::getNoFPClassAttrName());
344 attrs.set(mlir::LLVM::LLVMDialect::getAlignAttrName(),
345 builder.getI64IntegerAttr(ac.indirectAlign.value()));
346 if (enableNoundefAttrs)
347 attrs.set(mlir::LLVM::LLVMDialect::getNoUndefAttrName(),
348 builder.getUnitAttr());
354 attrs.set(mlir::LLVM::LLVMDialect::getByValAttrName(),
355 mlir::TypeAttr::get(pointeeTy));
359 attrs.set(mlir::LLVM::LLVMDialect::getNoFreeObjAttrName(),
360 builder.getUnitAttr());
364 auto recTy = mlir::dyn_cast<cir::RecordType>(pointeeTy);
365 cir::RecordLayoutAttr layout =
367 :
cir::RecordLayoutAttr();
368 if (!layout || layout.getHasTrivialDtor())
369 attrs.set(mlir::LLVM::LLVMDialect::getDeadOnReturnAttrName(),
370 builder.getI64IntegerAttr(
static_cast<int64_t>(
371 llvm::DeadOnReturnInfo().toIntValue())));
372 attrs.set(mlir::LLVM::LLVMDialect::getDereferenceableAttrName(),
373 builder.getI64IntegerAttr(
374 dl.getTypeAllocSize(pointeeTy).getFixedValue()));
376 newArgAttrs.push_back(attrs.getDictionary(ctx));
377 }
else if (ac.kind == ArgKind::IndirectAliased) {
380 mlir::NamedAttrList attrs(existing);
381 attrs.erase(mlir::LLVM::LLVMDialect::getNoFPClassAttrName());
382 attrs.set(mlir::LLVM::LLVMDialect::getAlignAttrName(),
383 builder.getI64IntegerAttr(ac.indirectAlign.value()));
384 if (enableNoundefAttrs)
385 attrs.set(mlir::LLVM::LLVMDialect::getNoUndefAttrName(),
386 builder.getUnitAttr());
387 attrs.set(mlir::LLVM::LLVMDialect::getByRefAttrName(),
388 mlir::TypeAttr::get(origArgTypes[oldIdx]));
389 newArgAttrs.push_back(attrs.getDictionary(ctx));
391 newArgAttrs.push_back(existing);
394 return builder.getArrayAttr(newArgAttrs);
403mlir::ArrayAttr updateResAttrs(mlir::MLIRContext *ctx,
404 mlir::ArrayAttr existingResAttrs,
405 const ArgClassification &retInfo,
406 mlir::Type origRetTy,
407 const mlir::DataLayout &dl) {
408 bool needsNoUndefDrop =
409 retInfo.kind == ArgKind::Direct && retInfo.coercedType &&
410 isCoercionWiderThanMemory(origRetTy, retInfo.coercedType, dl);
411 if (retInfo.kind != ArgKind::Extend && !needsNoUndefDrop)
412 return existingResAttrs;
414 mlir::NamedAttrList attrs;
415 if (existingResAttrs && !existingResAttrs.empty())
417 mlir::cast<mlir::DictionaryAttr>(existingResAttrs[0]).getValue());
418 if (retInfo.kind == ArgKind::Extend)
419 attrs.set(retInfo.signExtend ?
"llvm.signext" :
"llvm.zeroext",
420 mlir::UnitAttr::get(ctx));
421 if (needsNoUndefDrop)
422 attrs.erase(mlir::LLVM::LLVMDialect::getNoUndefAttrName());
423 return mlir::ArrayAttr::get(ctx, {attrs.getDictionary(ctx)});
436static uint64_t coercionByteSize(mlir::Type ty,
const mlir::DataLayout &dl) {
437 if (
auto intTy = mlir::dyn_cast<cir::IntType>(ty))
438 return llvm::divideCeil(intTy.getWidth(), 8);
439 return dl.getTypeSize(ty);
448emitViewAtOffset(mlir::OpBuilder &builder, mlir::Location loc,
449 mlir::Type wantTy, mlir::Value base,
450 SmallPtrSetImpl<mlir::Operation *> &createdOps,
452 auto basePtrTy = mlir::cast<cir::PointerType>(base.getType());
453 if (wantTy == basePtrTy.getPointee()) {
454 assert(offset == 0 &&
"a view of the storage's own type has no offset");
457 mlir::ptr::MemorySpaceAttrInterface addrSpace = basePtrTy.getAddrSpace();
458 mlir::Value view = base;
460 auto u8Ty = cir::IntType::get(builder.getContext(), 8,
false);
461 auto u8PtrTy = cir::PointerType::get(u8Ty, addrSpace);
462 auto u8Base = cir::CastOp::create(builder, loc, u8PtrTy,
463 cir::CastKind::bitcast, base);
464 createdOps.insert(u8Base);
466 cir::IntType::get(builder.getContext(), 64,
true);
467 auto strideVal = cir::ConstantOp::create(
468 builder, loc, cir::IntAttr::get(strideTy, offset));
469 createdOps.insert(strideVal);
470 view = cir::PtrStrideOp::create(builder, loc, u8PtrTy, u8Base, strideVal);
471 createdOps.insert(view.getDefiningOp());
473 auto typedView = cir::CastOp::create(builder, loc,
474 cir::PointerType::get(wantTy, addrSpace),
475 cir::CastKind::bitcast, view);
476 createdOps.insert(typedView);
482static cir::PointerType getAllocaPtrTy(mlir::Type pointee,
483 const mlir::DataLayout &dl) {
484 mlir::ptr::MemorySpaceAttrInterface addrSpace;
485 if (
auto asAttr = mlir::dyn_cast_if_present<mlir::IntegerAttr>(
486 dl.getAllocaMemorySpace()))
487 addrSpace = cir::TargetAddressSpaceAttr::get(pointee.getContext(),
489 return cir::PointerType::get(pointee, addrSpace);
497static cir::AllocaOp createCoercionSlot(
498 mlir::OpBuilder &builder, mlir::Location loc, mlir::Type srcTy,
499 mlir::Type dstTy, mlir::Block *slotBlock,
const mlir::DataLayout &dl,
500 SmallPtrSetImpl<mlir::Operation *> &createdOps,
unsigned offset) {
501 assert(srcTy != dstTy &&
"callers must pre-check that the types differ");
504 slotBlock->getParentOp()->getParentOfType<mlir::ModuleOp>());
505 uint64_t srcAlign = cirDataLayout.getTypeLanguageAlignment(srcTy);
506 uint64_t dstAlign = cirDataLayout.getTypeLanguageAlignment(dstTy);
507 uint64_t allocaAlign = std::max(srcAlign, dstAlign);
508 mlir::Type slotTy = coercionByteSize(srcTy, dl) >= coercionByteSize(dstTy, dl)
515 [[maybe_unused]] mlir::Type coercedTy = ((slotTy == srcTy) ? dstTy : srcTy);
516 assert((offset == 0 ||
517 coercionByteSize(coercedTy, dl) < coercionByteSize(slotTy, dl)) &&
518 "a direct offset must land on the coerced side, the smaller one");
519 assert((offset == 0 ||
520 offset + dl.getTypeSize(coercedTy) <= dl.getTypeSize(slotTy)) &&
521 "coerce slot too small for offset access");
522 assert((offset == 0 ||
523 offset % cirDataLayout.getTypeLanguageAlignment(coercedTy) == 0) &&
524 "a direct offset must be aligned for the coerced access");
526 mlir::OpBuilder::InsertionGuard guard(builder);
527 builder.setInsertionPointToStart(slotBlock);
528 auto alloca = cir::AllocaOp::create(builder, loc, getAllocaPtrTy(slotTy, dl),
529 builder.getStringAttr(
"coerce"),
530 builder.getI64IntegerAttr(allocaAlign));
531 createdOps.insert(alloca);
565mlir::Value emitCoercionToMemory(mlir::OpBuilder &builder, mlir::Location loc,
566 mlir::Type dstTy, mlir::Value src,
567 mlir::Block *slotBlock,
568 const mlir::DataLayout &dl,
569 SmallPtrSetImpl<mlir::Operation *> &createdOps,
571 mlir::Type srcTy = src.getType();
572 cir::AllocaOp alloca = createCoercionSlot(builder, loc, srcTy, dstTy,
573 slotBlock, dl, createdOps, offset);
574 mlir::Type slotTy = alloca.getAllocaType();
577 mlir::Value srcSlot = emitViewAtOffset(
578 builder, loc, srcTy, alloca, createdOps, srcTy == slotTy ? 0 : offset);
579 auto store = cir::StoreOp::create(builder, loc, src, srcSlot);
580 createdOps.insert(store);
583 return emitViewAtOffset(builder, loc, dstTy, alloca, createdOps,
584 dstTy == slotTy ? 0 : offset);
590emitAddrSpaceCoercion(mlir::OpBuilder &builder, mlir::Location loc,
591 mlir::Type dstTy, mlir::Value src,
592 SmallPtrSetImpl<mlir::Operation *> &createdOps,
594 auto srcPtrTy = dyn_cast<cir::PointerType>(src.getType());
595 auto dstPtrTy = dyn_cast<cir::PointerType>(dstTy);
596 if (offset || !srcPtrTy || !dstPtrTy ||
597 srcPtrTy.getPointee() != dstPtrTy.getPointee())
599 assert(srcPtrTy.getAddrSpace() != dstPtrTy.getAddrSpace() &&
600 "pointers to the same pointee must differ in address space");
601 auto cast = cir::CastOp::create(builder, loc, dstTy,
602 cir::CastKind::address_space, src);
603 createdOps.insert(cast);
611mlir::Value emitCoercion(mlir::OpBuilder &builder, mlir::Location loc,
612 mlir::Type dstTy, mlir::Value src,
613 mlir::Block *slotBlock,
const mlir::DataLayout &dl,
614 SmallPtrSetImpl<mlir::Operation *> &createdOps,
616 if (mlir::Value cast =
617 emitAddrSpaceCoercion(builder, loc, dstTy, src, createdOps, offset))
620 mlir::Value dstSlot = emitCoercionToMemory(builder, loc, dstTy, src,
621 slotBlock, dl, createdOps, offset);
622 auto load = cir::LoadOp::create(builder, loc, dstSlot);
623 createdOps.insert(load);
629mlir::Value emitCoercion(mlir::OpBuilder &builder, mlir::Location loc,
630 mlir::Type dstTy, mlir::Value src,
631 mlir::Block *slotBlock,
const mlir::DataLayout &dl,
633 SmallPtrSet<mlir::Operation *, 4> ignored;
634 return emitCoercion(builder, loc, dstTy, src, slotBlock, dl, ignored, offset);
646mlir::Block *coercionSlotBlock(mlir::Operation *op) {
647 if (
auto funcOp = op->getParentOfType<mlir::FunctionOpInterface>())
648 return &funcOp->getRegion(0).front();
649 mlir::Region *region = op->getParentRegion();
650 while (mlir::Region *outer = region->getParentRegion()) {
651 if (mlir::isa<mlir::ModuleOp>(outer->getParentOp()))
655 assert(!region->empty() &&
"coercion slot needs a block to hold the alloca");
656 return ®ion->front();
663static cir::LoadOp maybeGetSimpleLoad(mlir::Value val) {
664 cir::LoadOp load = val.getDefiningOp<cir::LoadOp>();
665 if (!load || load.isVolatileOrAtomic())
673static cir::LoadOp maybeGetCopyableRecordLoad(mlir::Value val) {
674 cir::LoadOp load = val.getDefiningOp<cir::LoadOp>();
675 if (!load || load.getMemOrder() ||
676 !mlir::isa<cir::RecordType>(load.getType()))
684static mlir::LogicalResult rejectAtomicRecordLoad(mlir::Operation *user,
686 auto load = val.getDefiningOp<cir::LoadOp>();
687 if (!load || !load.getMemOrder() ||
688 !mlir::isa<cir::RecordType>(load.getType()))
689 return mlir::success();
690 return user->emitOpError()
691 <<
"atomic load of a record operand not yet implemented in "
699static bool canReadInPlace(cir::LoadOp load, mlir::Type coercedTy,
713static mlir::Value emitInPlaceCoercedLoad(mlir::OpBuilder &builder,
714 mlir::Location loc, cir::LoadOp load,
715 mlir::Type coercedTy,
718 assert(src &&
"the record load must read an alloca");
719 mlir::OpBuilder::InsertionGuard guard(builder);
720 builder.setInsertionPoint(load);
721 SmallPtrSet<mlir::Operation *, 4> ignored;
722 mlir::Value view = emitViewAtOffset(builder, loc, coercedTy, load.getAddr(),
724 auto coerced = cir::LoadOp::create(builder, loc, view);
725 coerced.setAlignment(llvm::MinAlign(src.getAlignment(), offset));
735emitCopiedCoercionSlot(mlir::OpBuilder &builder, mlir::Location loc,
736 cir::LoadOp load, mlir::Type coercedTy,
737 mlir::Block *slotBlock,
const mlir::DataLayout &dl,
739 auto recTy = cast<cir::RecordType>(load.getType());
742 srcAlign = src.getAlignment();
744 srcAlign = load.getAlignment().value_or(
747 SmallPtrSet<mlir::Operation *, 4> ignored;
748 cir::AllocaOp slot = createCoercionSlot(builder, loc, recTy, coercedTy,
749 slotBlock, dl, ignored, offset);
750 slot.setAlignment(std::max(slot.getAlignment(), srcAlign));
751 assert((offset == 0 || slot.getAllocaType() == recTy) &&
752 "a direct offset must leave the record as the slot's type");
753 mlir::OpBuilder::InsertionGuard guard(builder);
754 builder.setInsertionPoint(load);
755 mlir::Value recView =
756 emitViewAtOffset(builder, loc, recTy, slot, ignored, 0);
758 cir::CopyOp::create(builder, loc, recView, load.getAddr(),
759 builder.getI64IntegerAttr(slot.getAlignment()),
760 builder.getI64IntegerAttr(srcAlign));
761 copy.setIsVolatile(load.getIsVolatile());
768static mlir::Value emitCopiedCoercedLoad(mlir::OpBuilder &builder,
769 mlir::Location loc, cir::LoadOp load,
770 mlir::Type coercedTy,
771 mlir::Block *slotBlock,
772 const mlir::DataLayout &dl,
774 cir::AllocaOp slot = emitCopiedCoercionSlot(builder, loc, load, coercedTy,
775 slotBlock, dl, offset);
776 mlir::OpBuilder::InsertionGuard guard(builder);
777 builder.setInsertionPoint(load);
778 SmallPtrSet<mlir::Operation *, 4> ignored;
779 mlir::Value coercedView =
780 emitViewAtOffset(builder, loc, coercedTy, slot, ignored, offset);
781 auto coerced = cir::LoadOp::create(builder, loc, coercedView);
782 coerced.setAlignment(llvm::MinAlign(slot.getAlignment(), offset));
793static mlir::Value emitCoercedRecordLoad(
794 mlir::OpBuilder &builder, mlir::Location loc, cir::LoadOp load,
795 mlir::Type coercedTy, mlir::Block *slotBlock,
const mlir::DataLayout &dl,
797 if (!load.getIsVolatile() &&
798 canReadInPlace(load, coercedTy, cirDataLayout, offset))
799 return emitInPlaceCoercedLoad(builder, loc, load, coercedTy, offset);
800 return emitCopiedCoercedLoad(builder, loc, load, coercedTy, slotBlock, dl,
814static bool shouldSkipTailPadding(mlir::Value dst, mlir::Type ty) {
815 auto structTy = mlir::dyn_cast<cir::StructType>(ty);
820 for (
auto [member,
kind] : llvm::reverse(llvm::zip_equal(members, kinds)))
822 return kind == cir::RecordMemberKind::Pad;
844static mlir::LogicalResult
845rewireRecordUses(mlir::OpBuilder &builder, mlir::Location loc, mlir::Value val,
846 mlir::Value addr,
uint64_t addrAlign,
847 const mlir::DataLayout &dl,
848 const SmallPtrSetImpl<mlir::Operation *> &exceptions) {
849 mlir::Type recTy = mlir::cast<cir::PointerType>(addr.getType()).getPointee();
850 assert(mlir::isa<cir::RecordType>(recTy) &&
"only a record is rewired");
853 for (mlir::OpOperand &use : val.getUses()) {
854 mlir::Operation *user = use.getOwner();
855 if (exceptions.contains(user))
857 if (
auto store = mlir::dyn_cast<cir::StoreOp>(user)) {
858 if (store.getMemOrder())
859 return store.emitOpError()
860 <<
"atomic store of a record argument or result not yet "
861 "implemented in CallConvLowering";
862 stores.push_back(store);
865 auto member = mlir::dyn_cast<cir::ExtractMemberOp>(user);
866 bool isFieldRead =
member && !mlir::isa<cir::RecordType>(
member.getType());
868 !mlir::isa<cir::CIRCallOpInterface, cir::ReturnOp>(user))
869 return user->emitOpError()
870 <<
"use of a record argument or result other than by a store, "
871 "call, return or member read not yet implemented in "
873 otherUses.push_back(&use);
876 mlir::IntegerAttr srcAlign = builder.getI64IntegerAttr(addrAlign);
878 val.getParentRegion()->getParentOfType<mlir::ModuleOp>());
879 for (cir::StoreOp store : stores) {
880 mlir::OpBuilder::InsertionGuard guard(builder);
881 builder.setInsertionPoint(store);
882 uint64_t dstAlign = store.getAlignment().value_or(
885 cir::CopyOp::create(builder, store.getLoc(), store.getAddr(), addr,
886 builder.getI64IntegerAttr(dstAlign), srcAlign);
887 copy.setIsVolatile(store.getIsVolatile());
888 copy.setSkipTailPadding(shouldSkipTailPadding(store.getAddr(), recTy));
892 if (otherUses.empty())
893 return mlir::success();
894 auto load = cir::LoadOp::create(builder, loc, addr);
895 load.setAlignment(addrAlign);
896 for (mlir::OpOperand *use : otherUses)
898 return mlir::success();
905static mlir::LogicalResult
906replaceUsesFromSlot(mlir::OpBuilder &builder, mlir::Location loc,
907 mlir::Value val, mlir::Value view,
908 const mlir::DataLayout &dl,
909 const SmallPtrSetImpl<mlir::Operation *> &exceptions) {
910 if (!mlir::isa<cir::RecordType>(
911 mlir::cast<cir::PointerType>(view.getType()).getPointee())) {
912 auto load = cir::LoadOp::create(builder, loc, view);
913 val.replaceAllUsesExcept(load, exceptions);
914 return mlir::success();
917 assert(slot &&
"the view must start at the coercion slot");
918 return rewireRecordUses(builder, loc, val, view, slot.getAlignment(), dl,
929static mlir::LogicalResult
930emitCoercionAndReplaceUses(mlir::OpBuilder &builder, mlir::Location loc,
931 mlir::Type origTy, mlir::Value src, mlir::Value val,
932 mlir::Block *slotBlock,
const mlir::DataLayout &dl,
933 SmallPtrSetImpl<mlir::Operation *> &createdOps,
935 if (mlir::Value cast = emitAddrSpaceCoercion(builder, loc, origTy, src,
936 createdOps, offset)) {
937 val.replaceAllUsesExcept(cast, createdOps);
938 return mlir::success();
940 mlir::Value view = emitCoercionToMemory(builder, loc, origTy, src, slotBlock,
941 dl, createdOps, offset);
942 return replaceUsesFromSlot(builder, loc, val, view, dl, createdOps);
955insertReturnCoercion(mlir::FunctionOpInterface funcOp, mlir::Type origRetTy,
956 mlir::Type coercedRetTy, mlir::OpBuilder &builder,
957 const mlir::DataLayout &dl,
960 funcOp.walk([&](cir::ReturnOp r) { returns.push_back(r); });
961 for (cir::ReturnOp r : returns) {
962 if (r.getInput().empty())
964 mlir::Value origVal = r.getInput()[0];
965 if (origVal.getType() == coercedRetTy)
968 if (failed(rejectAtomicRecordLoad(r, origVal)))
969 return mlir::failure();
970 if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(origVal)) {
971 mlir::Value coerced = emitCoercedRecordLoad(
972 builder, r.getLoc(), recLoad, coercedRetTy,
973 &funcOp->getRegion(0).front(), dl, cirDataLayout, offset);
974 r->setOperand(0, coerced);
975 if (recLoad->use_empty())
980 builder.setInsertionPoint(r);
981 mlir::Value coerced =
982 emitCoercion(builder, r.getLoc(), coercedRetTy, origVal,
983 &funcOp->getRegion(0).front(), dl, offset);
984 r->setOperand(0, coerced);
986 return mlir::success();
991static cir::LoadOp getWholeRecordLoad(mlir::Value recordVal) {
992 cir::LoadOp load = maybeGetSimpleLoad(recordVal);
1019static mlir::Value forwardableIndirectAddr(cir::LoadOp load,
1020 cir::PointerType paramTy,
1024 mlir::Value addr = load.getAddr();
1026 if (
auto view = addr.getDefiningOp<cir::CastOp>();
1027 view && view.getKind() == cir::CastKind::address_space &&
1028 view.getSrc().getType() == paramTy)
1029 addr = view.getSrc();
1030 if (addr.getType() != paramTy)
1033 return slot && slot.getAlignment() >= minAlign ? addr : mlir::Value();
1043static bool passesCallerStorage(mlir::ModuleOp module, mlir::Type ty) {
1044 auto recTy = mlir::dyn_cast<cir::RecordType>(ty);
1047 cir::RecordLayoutAttr layout =
1050 layout.getArgPassingKind() != cir::ArgPassingKind::CanPassInRegs;
1063static bool storageUnwrittenBetween(cir::LoadOp load, mlir::Operation *call,
1064 mlir::AliasAnalysis &aliasAnalysis) {
1065 if (load->getBlock() != call->getBlock())
1067 for (mlir::Operation *op = load->getNextNode(); op != call;
1068 op = op->getNextNode()) {
1069 assert(op &&
"the load must precede the call in the block they share");
1070 if (op->getNumRegions() != 0 ||
1071 aliasAnalysis.getModRef(op, load.getAddr()).isMod())
1081static mlir::IntegerAttr loadSourceAlignment(cir::LoadOp load,
1082 mlir::OpBuilder &builder) {
1083 if (mlir::IntegerAttr stated = load.getAlignmentAttr())
1086 return builder.getI64IntegerAttr(slot.getAlignment());
1087 return builder.getI64IntegerAttr(1);
1101static void emitStructFieldArgs(mlir::OpBuilder &builder, mlir::Location loc,
1103 SmallVectorImpl<mlir::Value> &newArgs,
1104 SmallVectorImpl<cir::LoadOp> &deadRecordLoads) {
1105 cir::LoadOp srcLoad = getWholeRecordLoad(structVal);
1108 mlir::OpBuilder::InsertionGuard guard(builder);
1109 builder.setInsertionPoint(srcLoad);
1110 cir::PointerType baseTy = srcLoad.getAddr().getType();
1111 for (
auto [f, fieldTy] : llvm::enumerate(recTy.
getMembers())) {
1112 mlir::Type fieldPtrTy =
1113 cir::PointerType::get(fieldTy, baseTy.getAddrSpace());
1114 mlir::Value fieldPtr = cir::GetMemberOp::create(
1115 builder, loc, fieldPtrTy, srcLoad.getAddr(),
"",
1117 newArgs.push_back(cir::LoadOp::create(builder, loc, fieldPtr));
1119 deadRecordLoads.push_back(srcLoad);
1123 cir::ExtractMemberOp::create(builder, loc, structVal, f));
1134 for (mlir::Operation *load : uniqueLoads)
1135 if (load->use_empty())
1141 for (cir::FreezeOp freeze : freezes)
1147static cir::StoreOp maybeFindParamSpill(mlir::BlockArgument blockArg) {
1148 if (!blockArg.hasOneUse())
1150 auto store = dyn_cast<cir::StoreOp>(*blockArg.user_begin());
1151 if (!store || store.getValue() != blockArg)
1158static cir::AllocaOp getSpillSlot(mlir::Value addr) {
1159 if (
auto view = addr.getDefiningOp<cir::CastOp>();
1160 view && view.getKind() == cir::CastKind::address_space)
1161 addr = view.getSrc();
1162 return addr.getDefiningOp<cir::AllocaOp>();
1171static std::pair<cir::StoreOp, cir::AllocaOp>
1172findParamSpill(mlir::BlockArgument blockArg) {
1173 if (blockArg.use_empty())
1175 cir::StoreOp store = maybeFindParamSpill(blockArg);
1176 assert(store &&
"non-byval arg's only use must be its CIRGen param spill");
1177 cir::AllocaOp slot = getSpillSlot(store.getAddr());
1178 assert(slot &&
"non-byval arg's spill must store to an alloca");
1179 return {store, slot};
1204insertArgCoercion(mlir::FunctionOpInterface funcOp,
1205 const FunctionClassification &fc, mlir::OpBuilder &builder,
1206 const mlir::DataLayout &dl,
bool hasSRetArg,
1207 SmallVectorImpl<std::pair<cir::AllocaOp, mlir::BlockArgument>>
1208 &pendingParamSlots) {
1209 mlir::Region &body = funcOp->getRegion(0);
1211 return mlir::success();
1212 mlir::Block &entry = body.front();
1218 unsigned blockArgIdx = hasSRetArg ? 1 : 0;
1220 for (
const ArgClassification &ac : fc.argInfos) {
1221 assert(blockArgIdx < entry.getNumArguments() &&
1222 "classification count must not exceed entry block arguments");
1224 if (ac.kind == ArgKind::Expand) {
1228 mlir::BlockArgument origArg = entry.getArgument(blockArgIdx);
1229 auto recTy = cast<cir::RecordType>(origArg.getType());
1231 "Expand classification requires a struct type, not a union");
1233 assert(numFields > 0 &&
1234 "Expand classification requires at least one struct field");
1235 mlir::Location loc = funcOp.getLoc();
1245 cir::StoreOp paramStore;
1246 cir::AllocaOp destAlloca;
1247 if (!origArg.use_empty()) {
1248 assert(origArg.hasOneUse() &&
1249 "Expand arg must have exactly one use (the CIRGen param spill)");
1250 paramStore = cast<cir::StoreOp>(*origArg.user_begin());
1251 assert(paramStore.getValue() == origArg &&
1252 "Expand arg's use must be the value operand of its store");
1253 destAlloca = cast<cir::AllocaOp>(paramStore.getAddr().getDefiningOp());
1260 mlir::Operation *fieldStoreInsertPt =
nullptr;
1262 fieldStoreInsertPt = paramStore->getNextNode();
1263 assert(fieldStoreInsertPt &&
1264 "param spill must be followed by a block terminator");
1265 paramStore->erase();
1276 builder.setInsertionPoint(fieldStoreInsertPt);
1277 for (
auto [f, fieldTy] : llvm::enumerate(recTy.
getMembers())) {
1279 origArg.setType(fieldTy);
1281 entry.insertArgument(blockArgIdx + f, fieldTy, loc);
1284 mlir::Type fieldPtrTy = cir::PointerType::get(
1286 mlir::cast<cir::PointerType>(destAlloca.getType()).getAddrSpace());
1287 auto fieldPtr = cir::GetMemberOp::create(builder, loc, fieldPtrTy,
1290 cir::StoreOp::create(builder, loc, entry.getArgument(blockArgIdx + f),
1294 blockArgIdx += numFields;
1298 mlir::BlockArgument blockArg = entry.getArgument(blockArgIdx);
1306 unsigned numFields = flatTy.getNumElements();
1307 Type origTy = blockArg.getType();
1308 Location loc = funcOp.getLoc();
1311 blockArg.setType(flatTy.getElementType(0));
1312 for (
unsigned f = 1; f < numFields; ++f)
1313 entry.insertArgument(blockArgIdx + f, flatTy.getElementType(f), loc);
1316 builder.setInsertionPointToStart(&entry);
1317 assert(!ac.directOffset &&
1318 "the coerced struct starts at slot offset 0 here, so a flattened "
1319 "coercion cannot honor a direct offset");
1320 SmallPtrSet<Operation *, 8> flattenOps;
1322 if (origTy == flatTy) {
1323 slot = cir::AllocaOp::create(
1324 builder, loc, getAllocaPtrTy(flatTy, dl),
1325 builder.getStringAttr(
"coerce"),
1326 builder.getI64IntegerAttr(
1329 flattenOps.insert(slot);
1331 slot = createCoercionSlot(builder, loc, flatTy, origTy, &entry, dl,
1334 Value flatView = emitViewAtOffset(builder, loc, flatTy, slot, flattenOps,
1336 auto flatPtrTy = cast<cir::PointerType>(flatView.getType());
1337 for (
auto [f, fieldTy] : llvm::enumerate(flatTy.getMembers())) {
1339 cir::PointerType::get(fieldTy, flatPtrTy.getAddrSpace());
1340 auto fieldPtr = cir::GetMemberOp::create(builder, loc, fieldPtrTy,
1343 flattenOps.insert(fieldPtr);
1344 auto storeOp = cir::StoreOp::create(
1345 builder, loc, entry.getArgument(blockArgIdx + f), fieldPtr);
1348 llvm::MinAlign(slot.getAlignment(), flatTy.getElementOffset(dl, f));
1349 if (fieldAlign < dl.getTypeABIAlignment(fieldTy))
1350 storeOp.setAlignment(fieldAlign);
1351 flattenOps.insert(storeOp);
1356 Value origView = emitViewAtOffset(builder, loc, origTy, slot, flattenOps,
1358 if (failed(replaceUsesFromSlot(builder, loc, blockArg, origView, dl,
1360 return mlir::failure();
1362 blockArgIdx += numFields;
1366 if ((ac.kind == ArgKind::Direct && ac.coercedType) ||
1367 isRecordExtend(ac, blockArg.getType())) {
1368 mlir::Type oldArgTy = blockArg.getType();
1369 mlir::Type newArgTy = ac.coercedType;
1370 if (oldArgTy == newArgTy) {
1374 blockArg.setType(newArgTy);
1378 builder.setInsertionPointToStart(&entry);
1379 SmallPtrSet<mlir::Operation *, 4> coercionOps;
1380 if (failed(emitCoercionAndReplaceUses(builder, funcOp.getLoc(), oldArgTy,
1381 blockArg, blockArg, &entry, dl,
1382 coercionOps, ac.directOffset)))
1383 return mlir::failure();
1384 }
else if (ac.kind == ArgKind::Indirect ||
1385 ac.kind == ArgKind::IndirectAliased) {
1393 bool byRef = ac.kind == ArgKind::IndirectAliased;
1394 cir::PointerType ptrTy = getIndirectPtrTy(blockArg.getType(), ac);
1396 if (!byRef && !ac.byVal) {
1402 auto [paramStore, destAlloca] = findParamSpill(blockArg);
1405 paramStore->erase();
1408 blockArg.setType(ptrTy);
1417 pendingParamSlots.emplace_back(destAlloca, blockArg);
1424 cir::StoreOp paramStore;
1425 if (mlir::isa<cir::RecordType>(blockArg.getType()))
1426 paramStore = maybeFindParamSpill(blockArg);
1430 mlir::Value destAddr;
1431 mlir::IntegerAttr destAlign;
1432 mlir::Operation *spillPos =
nullptr;
1433 bool spillIsVolatile =
false;
1435 if (paramStore.getMemOrder())
1436 return paramStore.emitOpError()
1437 <<
"atomic store of a record argument or result not yet "
1438 "implemented in CallConvLowering";
1439 spillIsVolatile = paramStore.getIsVolatile();
1440 destAddr = paramStore.getAddr();
1442 destAlign = paramStore.getAlignmentAttr();
1443 if (!destAlign && destAlloca)
1444 destAlign = destAlloca.getAlignmentAttr();
1445 spillPos = paramStore->getNextNode();
1446 assert(spillPos &&
"param spill must be followed by a block "
1448 paramStore->erase();
1451 blockArg.setType(ptrTy);
1454 mlir::OpBuilder::InsertionGuard guard(builder);
1455 builder.setInsertionPoint(spillPos);
1456 auto copy = cir::CopyOp::create(
1457 builder, funcOp.getLoc(), destAddr, blockArg, destAlign,
1458 builder.getI64IntegerAttr(ac.indirectAlign.value()));
1459 copy.setIsVolatile(spillIsVolatile);
1460 copy.setSkipTailPadding(
1461 shouldSkipTailPadding(destAddr, ptrTy.getPointee()));
1462 }
else if (!blockArg.use_empty()) {
1463 builder.setInsertionPointToStart(&entry);
1464 if (mlir::isa<cir::RecordType>(ptrTy.getPointee())) {
1465 SmallPtrSet<mlir::Operation *, 1> noExceptions;
1466 if (failed(rewireRecordUses(builder, funcOp.getLoc(), blockArg,
1467 blockArg, ac.indirectAlign.value(), dl,
1469 return mlir::failure();
1472 cir::LoadOp::create(builder, funcOp.getLoc(), blockArg);
1473 SmallPtrSet<mlir::Operation *, 1> loadOps = {loadOp};
1474 blockArg.replaceAllUsesExcept(loadOp.getResult(), loadOps);
1483 return mlir::success();
1488void replaceAddrSpaceViews(cir::AllocaOp slot, mlir::Value ptr) {
1489 for (mlir::Operation *user : llvm::make_early_inc_range(slot->getUsers())) {
1490 auto view = mlir::dyn_cast<cir::CastOp>(user);
1491 if (view && view.getKind() == cir::CastKind::address_space &&
1492 view.getType() == ptr.getType()) {
1493 view.getResult().replaceAllUsesWith(ptr);
1530void insertSRetStores(mlir::FunctionOpInterface funcOp, mlir::Type origRetTy,
1531 mlir::OpBuilder &builder) {
1532 mlir::Value sretPtr = funcOp.getArguments()[0];
1535 funcOp->walk([&](cir::ReturnOp retOp) { returnOps.push_back(retOp); });
1537 cir::AllocaOp retAlloca =
nullptr;
1538 for (cir::ReturnOp retOp : returnOps) {
1541 assert(!retOp.getInput().empty() &&
1542 "cir.return in sret function must have an operand");
1544 cir::LoadOp retLoad =
1545 mlir::cast<cir::LoadOp>(retOp.getInput()[0].getDefiningOp());
1552 retAlloca = mlir::cast<cir::AllocaOp>(retLoad.getAddr().getDefiningOp());
1553 if (retAlloca.getType() == sretPtr.getType())
1554 retAlloca.getResult().replaceAllUsesWith(sretPtr);
1556 replaceAddrSpaceViews(retAlloca, sretPtr);
1561 builder.setInsertionPoint(retOp);
1562 cir::ReturnOp::create(builder, retOp.getLoc());
1564 if (retLoad.use_empty())
1569 assert(retAlloca->use_empty() &&
1570 "return slot used other than by its returns and address_space "
1590 builder.getNamedAttr(
"llvm.sret", mlir::TypeAttr::get(retTy)));
1592 builder.getNamedAttr(
"llvm.align", builder.getI64IntegerAttr(align)));
1595 builder.getNamedAttr(
"llvm.noalias", builder.getUnitAttr()));
1596 attrs.push_back(builder.getNamedAttr(
"llvm.writable", builder.getUnitAttr()));
1598 builder.getNamedAttr(
"llvm.dead_on_unwind", builder.getUnitAttr()));
1608void applySretSlotAttrs(cir::CallOp newCall, mlir::ArrayAttr argAttrs,
1610 mlir::OpBuilder &builder) {
1611 mlir::MLIRContext *ctx = newCall->getContext();
1613 buildSretSlotAttrs(builder, retTy, align,
false);
1616 newArgAttrs.reserve(newCall.getArgOperands().size());
1617 newArgAttrs.push_back(mlir::DictionaryAttr::get(ctx, sretAttrs));
1619 llvm::append_range(newArgAttrs, argAttrs);
1620 assert(newArgAttrs.size() <= newCall.getArgOperands().size() &&
1621 "arg_attrs wider than the rewritten call's operand list");
1622 newArgAttrs.resize(newCall.getArgOperands().size(),
1623 mlir::DictionaryAttr::get(ctx));
1624 newCall->setAttr(
"arg_attrs", mlir::ArrayAttr::get(ctx, newArgAttrs));
1631static void copyCallAttributes(cir::CallOp source, cir::CallOp target) {
1632 source->getName().walkInherentAttrs(source, [&](llvm::StringRef name,
1633 mlir::Attribute &attr) {
1634 if (name != cir::CIRDialect::getCalleeAttrName())
1635 target->setInherentAttr(mlir::StringAttr::get(source->getContext(), name),
1638 for (mlir::NamedAttribute attr : source->getDiscardableAttrs())
1639 if (!target->hasDiscardableAttr(
attr.getName()))
1640 target->setDiscardableAttr(
attr.getName(),
attr.getValue());
1651static void prependIndirectCallee(cir::CallOp call,
1652 SmallVectorImpl<mlir::Value> &args,
1653 mlir::Type retTy,
unsigned numEllipsisArgs,
1654 mlir::OpBuilder &builder) {
1655 if (!call.isIndirect())
1657 assert(numEllipsisArgs <= args.size() &&
1658 "more ellipsis arguments than arguments");
1659 mlir::Value calleePtr = call.getIndirectCall();
1661 paramTypes.reserve(args.size() - numEllipsisArgs);
1662 llvm::transform(
ArrayRef(args).drop_back(numEllipsisArgs),
1663 std::back_inserter(paramTypes),
1664 [](mlir::Value v) {
return v.getType(); });
1674 assert((calleeFnTy.isVarArg() || numEllipsisArgs == 0) &&
1675 "only a variadic callee takes arguments through an ellipsis");
1676 auto newPtrTy = cir::PointerType::get(
1677 cir::FuncType::get(paramTypes, retTy, calleeFnTy.isVarArg()));
1678 if (calleePtr.getType() != newPtrTy)
1679 calleePtr = cir::CastOp::create(builder, call.getLoc(), newPtrTy,
1680 cir::CastKind::bitcast, calleePtr);
1681 args.insert(args.begin(), calleePtr);
1695rewriteIndirectReturnCall(cir::CallOp call,
const FunctionClassification &fc,
1697 unsigned numEllipsisArgs, mlir::Type origRetTy,
1699 mlir::OpBuilder &builder,
const mlir::DataLayout &dl,
1700 bool enableNoundefAttrs) {
1701 mlir::MLIRContext *ctx = call->getContext();
1702 cir::PointerType ptrTy = getIndirectPtrTy(origRetTy, fc.returnInfo);
1703 builder.setInsertionPoint(call);
1704 uint64_t sretAlign = fc.returnInfo.indirectAlign.value();
1718 mlir::Value sretSlot =
nullptr;
1719 cir::StoreOp reuseStore =
nullptr;
1720 if (call.getResult().hasOneUse()) {
1721 mlir::Operation *user = *call.getResult().getUsers().begin();
1722 auto store = mlir::dyn_cast<cir::StoreOp>(user);
1723 if (store && store.getValue() == call.getResult()) {
1724 mlir::Value dest = store.getAddr();
1725 if (
auto view = dest.getDefiningOp<cir::CastOp>();
1726 view && view.getKind() == cir::CastKind::address_space &&
1727 view.getSrc().getType() == ptrTy)
1728 dest = view.getSrc();
1729 if (dest.getType() == ptrTy &&
1730 mlir::DominanceInfo().properlyDominates(dest, call)) {
1737 cir::PointerType allocaPtrTy = getAllocaPtrTy(origRetTy, dl);
1738 auto slot = cir::AllocaOp::create(
1739 builder, call.getLoc(), allocaPtrTy,
1740 builder.getStringAttr(
""),
1741 builder.getI64IntegerAttr(sretAlign));
1742 slot.setReturnSlot(
true);
1745 if (allocaPtrTy != ptrTy)
1746 sretSlot = cir::CastOp::create(builder, call.getLoc(), ptrTy,
1747 cir::CastKind::address_space, sretSlot);
1751 sretArgs.push_back(sretSlot);
1752 sretArgs.append(newArgs.begin(), newArgs.end());
1756 mlir::Type sretVoidTy = cir::VoidType::get(ctx);
1757 prependIndirectCallee(call, sretArgs, sretVoidTy, numEllipsisArgs, builder);
1758 auto newCall = cir::CallOp::create(
1759 builder, call.getLoc(), call.getCalleeAttr(), sretVoidTy, sretArgs);
1760 copyCallAttributes(call, newCall);
1761 newCall->removeAttr(
"res_attrs");
1766 mlir::ArrayAttr argAttrs = call->getAttrOfType<mlir::ArrayAttr>(
"arg_attrs");
1767 if (needsArgAttrUpdate(fc))
1769 updateArgAttrs(ctx, call->getParentOfType<mlir::ModuleOp>(),
1770 origCallArgTypes, argAttrs, fc, dl, enableNoundefAttrs,
1772 applySretSlotAttrs(newCall, argAttrs, origRetTy, sretAlign, builder);
1778 reuseStore->erase();
1779 }
else if (mlir::isa<cir::RecordType>(origRetTy)) {
1780 builder.setInsertionPointAfter(newCall);
1781 SmallPtrSet<mlir::Operation *, 1> noExceptions;
1782 if (failed(rewireRecordUses(builder, call.getLoc(), call.getResult(),
1783 sretSlot, sretAlign, dl, noExceptions)))
1784 return mlir::failure();
1786 builder.setInsertionPointAfter(newCall);
1787 auto load = cir::LoadOp::create(builder, call.getLoc(), origRetTy, sretSlot,
1791 mlir::IntegerAttr(),
1792 cir::SyncScopeKindAttr(),
1793 cir::MemOrderAttr(),
1795 call.getResult().replaceAllUsesWith(load);
1798 return mlir::success();
1803bool isSSERegisterClass(mlir::Type ty) {
1804 return mlir::isa<cir::VectorType, cir::FPTypeInterface>(ty);
1810 assert(call.isIndirect() &&
"expected an indirect call");
1811 return cast<cir::FuncType>(
1812 cast<cir::PointerType>(call.getIndirectCall().getType()).getPointee());
1821 cir::FuncOp funcOp,
unsigned argNo,
const ArgClassification &ac,
1822 mlir::BlockArgument blockArg, mlir::DominanceInfo &dom,
1823 SmallPtrSetImpl<mlir::Operation *> &claimedSlots) {
1824 cir::PointerType paramPtrTy = getIndirectPtrTy(blockArg.getType(), ac);
1831 cir::StoreOp extraSpill;
1832 mlir::Operation *otherUse =
nullptr;
1834 for (mlir::OpOperand &use : blockArg.getUses()) {
1835 auto store = dyn_cast<cir::StoreOp>(use.getOwner());
1836 if (store && store.getValue() == blockArg) {
1850 auto call = dyn_cast<cir::CIRCallOpInterface>(use.getOwner());
1851 if (call && llvm::is_contained(call.getArgOperands(), blockArg))
1852 callArgs.push_back(&use);
1854 otherUse = use.getOwner();
1858 return extraSpill->emitOpError()
1859 <<
"non-byval parameter " << argNo
1860 <<
" spilled more than once is not yet implemented in "
1864 return otherUse->emitOpError()
1865 <<
"non-byval parameter " << argNo
1866 <<
" consumed other than as a call argument is not yet implemented "
1867 "in CallConvLowering";
1872 if (callArgs.empty())
1873 return mlir::success();
1880 mlir::OpBuilder builder(funcOp.getContext());
1881 builder.setInsertionPointToStart(blockArg.getOwner());
1882 auto synthesized = cir::AllocaOp::create(
1883 builder, funcOp.getLoc(), paramPtrTy,
1884 builder.getStringAttr(
"nonbyval.param"),
1885 builder.getI64IntegerAttr(ac.indirectAlign.value()));
1887 cir::StoreOp::create(builder, funcOp.getLoc(), blockArg, synthesized);
1896 cir::AllocaOp slot = getSpillSlot(spill.getAddr());
1897 if (!slot || slot.getType() != paramPtrTy)
1898 return spill->emitOpError()
1899 <<
"non-byval parameter " << argNo
1900 <<
" spilled to storage that cannot take the incoming pointer is "
1901 "not yet implemented in CallConvLowering";
1907 if (!claimedSlots.insert(slot).second)
1908 return spill->emitOpError()
1909 <<
"non-byval parameter " << argNo
1910 <<
" sharing its spill slot with another parameter is not yet "
1911 "implemented in CallConvLowering";
1916 slot.setAlignment(ac.indirectAlign.value());
1918 if (callArgs.empty())
1919 return mlir::success();
1923 for (mlir::OpOperand *callArg : callArgs)
1924 if (!dom.properlyDominates(spill, callArg->getOwner()))
1925 return callArg->getOwner()->emitOpError()
1926 <<
"non-byval parameter " << argNo
1927 <<
" read before its spill is not yet implemented in "
1940 mlir::OpBuilder builder(spill);
1941 builder.setInsertionPointAfter(spill);
1942 auto reload = cir::LoadOp::create(builder, spill.getLoc(), slot);
1943 reload.setAlignment(ac.indirectAlign.value());
1944 for (mlir::OpOperand *callArg : callArgs)
1945 callArg->set(reload.getResult());
1946 return mlir::success();
1950 cir::FuncOp funcOp,
const FunctionClassification &fc) {
1951 if (!funcOp.isDefinition())
1952 return mlir::success();
1953 mlir::Region &body = funcOp->getRegion(0);
1955 return mlir::success();
1956 mlir::Block &entry = body.front();
1957 mlir::DominanceInfo dom;
1958 SmallPtrSet<mlir::Operation *, 4> claimedSlots;
1964 for (
auto [idx, ac] : llvm::enumerate(fc.argInfos)) {
1965 if (ac.kind != ArgKind::Indirect || ac.byVal)
1967 assert(idx < entry.getNumArguments() &&
1968 "classification count must not exceed entry block arguments");
1970 funcOp, idx, ac, entry.getArgument(idx), dom, claimedSlots)))
1971 return mlir::failure();
1973 return mlir::success();
1977 for (
auto [slot, incoming] : pendingParamSlots) {
1978 slot.getResult().replaceAllUsesWith(incoming);
1981 pendingParamSlots.clear();
1985 mlir::FunctionOpInterface funcOpInterface,
const FunctionClassification &fc,
1986 mlir::OpBuilder &builder) {
1992 cir::FuncOp funcOp = mlir::cast<cir::FuncOp>(funcOpInterface);
1994 if (!fc.needsRewrite())
1995 return mlir::success();
1999 mlir::MLIRContext *ctx = funcOp->getContext();
2004 assert(oldResultTypes.size() <= 1 &&
2005 "CIR functions return zero or one value");
2008 if (mlir::failed(buildNewArgTypes(oldArgTypes, fc, newArgTypes, dl,
2009 [&]() {
return funcOp.emitOpError(); })))
2010 return mlir::failure();
2012 mlir::Type voidTy = cir::VoidType::get(ctx);
2013 mlir::Type origRetTy = oldResultTypes.empty() ? voidTy : oldResultTypes[0];
2014 mlir::Type newRetTy = computeNewReturnType(
2015 origRetTy, fc.returnInfo, ctx, [&]() { return funcOp.emitOpError(); });
2017 return mlir::failure();
2027 fc.returnInfo.kind == ArgKind::Indirect && !oldResultTypes.empty();
2028 cir::PointerType sretPtrTy;
2030 sretPtrTy = getIndirectPtrTy(origRetTy, fc.returnInfo);
2031 newArgTypes.insert(newArgTypes.begin(), sretPtrTy);
2034 if (funcOp.isDefinition()) {
2035 mlir::Region &body = funcOp->getRegion(0);
2036 if (!body.empty()) {
2041 body.front().insertArgument(0u, sretPtrTy, funcOp.getLoc());
2042 insertSRetStores(funcOp, origRetTy, builder);
2051 if (failed(insertArgCoercion(funcOp, fc, builder, dl, hasSRet,
2052 pendingParamSlots)))
2053 return mlir::failure();
2058 if (fc.returnInfo.kind == ArgKind::Direct && fc.returnInfo.coercedType &&
2059 !oldResultTypes.empty() && fc.returnInfo.coercedType != origRetTy &&
2060 failed(insertReturnCoercion(
2061 funcOp, origRetTy, fc.returnInfo.coercedType, builder, dl,
2062 cirDataLayout, fc.returnInfo.directOffset)))
2063 return mlir::failure();
2065 mlir::Block &entry = body.front();
2074 unsigned blockArgIdx = hasSRet ? 1 : 0;
2075 for (
auto [i, ac] : llvm::enumerate(fc.argInfos)) {
2076 if (blockArgIdx >= entry.getNumArguments())
2078 if (ac.kind == ArgKind::Ignore) {
2079 mlir::BlockArgument arg = entry.getArgument(blockArgIdx);
2080 if (!arg.use_empty()) {
2081 builder.setInsertionPointToStart(&entry);
2082 mlir::Value poison =
2083 createIgnoredValue(builder, funcOp.getLoc(), arg.getType());
2084 arg.replaceAllUsesWith(poison);
2086 entry.eraseArgument(blockArgIdx);
2090 blockArgIdx += flatTy.getNumElements();
2091 else if (ac.kind == ArgKind::Expand)
2092 blockArgIdx += cast<cir::RecordType>(oldArgTypes[i]).getNumElements();
2104 if (fc.returnInfo.kind == ArgKind::Ignore && !oldResultTypes.empty()) {
2105 assert(mlir::isa<cir::VoidType>(newRetTy) &&
2106 "Ignore-return path requires the new return type to be void");
2108 funcOp.walk([&](cir::ReturnOp r) { returns.push_back(r); });
2109 for (cir::ReturnOp r : returns) {
2110 if (r.getNumOperands() == 0)
2112 builder.setInsertionPoint(r);
2113 cir::ReturnOp::create(builder, r.getLoc());
2119 mlir::Type newFnTy = funcOp.cloneTypeWith(newArgTypes, newResultTypes);
2120 funcOp.setFunctionTypeAttr(mlir::TypeAttr::get(newFnTy));
2124 if (hasSRet || needsArgAttrUpdate(fc)) {
2125 auto existing = funcOp->getAttrOfType<mlir::ArrayAttr>(
"arg_attrs");
2126 mlir::ArrayAttr updated = updateArgAttrs(ctx, module, oldArgTypes, existing,
2127 fc, dl, enableNoundefAttrs,
2134 builder, origRetTy, fc.returnInfo.indirectAlign.value(),
2135 funcOp.isDefinition());
2137 withSret.push_back(mlir::DictionaryAttr::get(ctx, sretAttrs));
2138 llvm::append_range(withSret, updated);
2139 funcOp->setAttr(
"arg_attrs", mlir::ArrayAttr::get(ctx, withSret));
2141 funcOp->setAttr(
"arg_attrs", updated);
2145 if (mlir::isa<cir::VoidType>(newRetTy)) {
2146 funcOp->removeAttr(
"res_attrs");
2147 }
else if (fc.returnInfo.kind == ArgKind::Extend ||
2148 fc.returnInfo.kind == ArgKind::Direct) {
2149 auto existing = funcOp->getAttrOfType<mlir::ArrayAttr>(
"res_attrs");
2150 if (mlir::ArrayAttr updated =
2151 updateResAttrs(ctx, existing, fc.returnInfo, origRetTy, dl))
2152 funcOp->setAttr(
"res_attrs", updated);
2155 return mlir::success();
2160 const FunctionClassification &fc,
2161 mlir::OpBuilder &builder) {
2173 unsigned numOperands =
2174 mlir::cast<cir::CIRCallOpInterface>(callOp).getNumArgOperands();
2175 if (numOperands > fc.argInfos.size())
2176 return callOp->emitOpError()
2177 <<
"variadic arguments not yet implemented in CallConvLowering";
2178 if (numOperands < fc.argInfos.size())
2179 return callOp->emitOpError()
2180 <<
"call passes fewer arguments than the callee declares, which is "
2181 "not yet implemented in CallConvLowering";
2183 if (!fc.needsRewrite())
2184 return mlir::success();
2186 if (mlir::isa<cir::TryCallOp>(callOp))
2187 return callOp->emitOpError()
2188 <<
"TryCallOp not yet implemented in CallConvLowering";
2190 auto call = mlir::cast<cir::CallOp>(callOp);
2191 mlir::MLIRContext *ctx = callOp->getContext();
2192 mlir::Block *slotBlock = coercionSlotBlock(call);
2194 builder.setInsertionPoint(call);
2197 mlir::ValueRange argOperands = call.getArgOperands();
2198 newArgs.reserve(argOperands.size());
2212 llvm::append_range(origCallArgTypes, argOperands.getTypes());
2218 unsigned numDeclared = fc.argInfos.size();
2219 if (call.isIndirect()) {
2221 assert(numDeclared <= fc.argInfos.size() &&
2222 "the classification covers at least the declared parameters");
2226 std::optional<unsigned> ellipsisStart;
2231 for (
auto [idx, ac] : llvm::enumerate(fc.argInfos)) {
2232 if (idx == numDeclared)
2233 ellipsisStart = newArgs.size();
2234 argBegin.push_back(newArgs.size());
2235 mlir::Value arg = argOperands[idx];
2236 auto freeze = arg.getDefiningOp<cir::FreezeOp>();
2237 if (freeze && !freeze->hasOneUse())
2239 if (ac.kind == ArgKind::Ignore) {
2241 deadFreezes.push_back(freeze);
2247 if (freeze && ((ac.kind != ArgKind::Indirect &&
2248 ac.kind != ArgKind::IndirectAliased) ||
2249 freeze.getInput().getDefiningOp<cir::LoadOp>())) {
2250 arg = freeze.getInput();
2251 deadFreezes.push_back(freeze);
2252 refreezes.push_back({idx, freeze});
2261 if (arg.getType() != flatTy) {
2262 SmallPtrSet<mlir::Operation *, 4> coercionOps;
2263 assert(!ac.directOffset &&
2264 "the coerced struct starts at slot offset 0 here, so a "
2265 "flattened coercion cannot honor a direct offset");
2266 if (failed(rejectAtomicRecordLoad(call, arg)))
2267 return mlir::failure();
2268 mlir::OpBuilder::InsertionGuard guard(builder);
2269 mlir::Value coercedPtr;
2270 uint64_t baseAlign = 0;
2271 if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(arg)) {
2272 mlir::Value base = recLoad.getAddr();
2273 if (!recLoad.getIsVolatile() &&
2274 canReadInPlace(recLoad, flatTy, cirDataLayout, 0)) {
2276 assert(baseAlloca &&
"canReadInPlace requires an alloca");
2277 baseAlign = baseAlloca.getAlignment();
2279 base = emitCopiedCoercionSlot(builder, call.getLoc(), recLoad,
2280 flatTy, slotBlock, dl, 0);
2282 builder.setInsertionPoint(recLoad);
2283 coercedPtr = emitViewAtOffset(builder, call.getLoc(), flatTy, base,
2285 deadRecordLoads.push_back(recLoad);
2288 emitCoercionToMemory(builder, call.getLoc(), flatTy, arg,
2289 slotBlock, dl, coercionOps, 0);
2291 auto coercedPtrTy = mlir::cast<cir::PointerType>(coercedPtr.getType());
2292 for (
auto [f, fieldTy] : llvm::enumerate(flatTy.getMembers())) {
2293 mlir::Type fieldPtrTy =
2294 cir::PointerType::get(fieldTy, coercedPtrTy.getAddrSpace());
2296 cir::GetMemberOp::create(builder, call.getLoc(), fieldPtrTy,
2298 auto field = cir::LoadOp::create(builder, call.getLoc(), fieldTy,
2299 fieldPtr.getResult());
2304 llvm::MinAlign(baseAlign, flatTy.getElementOffset(dl, f)));
2305 newArgs.push_back(field);
2308 emitStructFieldArgs(builder, call.getLoc(), arg, flatTy, newArgs,
2311 }
else if (ac.kind == ArgKind::Expand) {
2314 auto recTy = cast<cir::RecordType>(arg.getType());
2316 "Expand classification requires a struct type, not a union");
2317 emitStructFieldArgs(builder, call.getLoc(), arg, recTy, newArgs,
2319 }
else if ((ac.kind == ArgKind::Direct && ac.coercedType &&
2320 arg.getType() != ac.coercedType) ||
2321 isRecordExtend(ac, arg.getType())) {
2322 if (failed(rejectAtomicRecordLoad(call, arg)))
2323 return mlir::failure();
2324 if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(arg)) {
2325 newArgs.push_back(emitCoercedRecordLoad(
2326 builder, call.getLoc(), recLoad, ac.coercedType, slotBlock, dl,
2327 cirDataLayout, ac.directOffset));
2328 deadRecordLoads.push_back(recLoad);
2330 newArgs.push_back(emitCoercion(builder, call.getLoc(), ac.coercedType,
2331 arg, slotBlock, dl, ac.directOffset));
2333 }
else if (ac.kind == ArgKind::Indirect ||
2334 ac.kind == ArgKind::IndirectAliased) {
2335 uint64_t minAlign = ac.indirectAlign.value();
2338 bool byRef = ac.kind == ArgKind::IndirectAliased;
2339 bool byCopy = byRef || ac.byVal;
2340 bool fromCallerStorage =
2341 !byCopy && passesCallerStorage(module, arg.getType());
2342 cir::PointerType ptrTy = getIndirectPtrTy(arg.getType(), ac);
2350 cir::LoadOp srcLoad;
2351 if (mlir::isa<cir::RecordType>(arg.getType()))
2352 srcLoad = maybeGetSimpleLoad(arg);
2359 if (fromCallerStorage) {
2360 mlir::Value fwdAddr = forwardableIndirectAddr(srcLoad, ptrTy, minAlign);
2362 return call->emitOpError()
2363 <<
"non-byval indirect argument that does not name the "
2364 "caller's storage is not yet implemented in "
2366 newArgs.push_back(fwdAddr);
2367 deadRecordLoads.push_back(srcLoad);
2378 if (mlir::Value fwdAddr =
2379 byCopy ? forwardableIndirectAddr(srcLoad, ptrTy, minAlign)
2381 fwdAddr && storageUnwrittenBetween(srcLoad, call.getOperation(),
2383 newArgs.push_back(fwdAddr);
2384 deadRecordLoads.push_back(srcLoad);
2390 if (byRef && ptrTy != getAllocaPtrTy(arg.getType(), dl))
2391 return call->emitOpError()
2392 <<
"byref argument outside the alloca address space that does "
2393 "not name forwardable storage is not yet implemented in "
2398 if (failed(rejectAtomicRecordLoad(call, arg)))
2399 return mlir::failure();
2401 srcLoad = maybeGetCopyableRecordLoad(arg);
2402 mlir::IntegerAttr slotAlign = builder.getI64IntegerAttr(minAlign);
2403 mlir::OpBuilder::InsertionGuard guard(builder);
2405 builder.setInsertionPoint(srcLoad);
2407 llvm::StringRef slotName =
2408 byRef ?
"byref" : (ac.byVal ?
"byval" :
"indirect.tmp");
2410 cir::AllocaOp::create(builder, call.getLoc(), ptrTy,
2411 builder.getStringAttr(slotName), slotAlign);
2413 auto copy = cir::CopyOp::create(builder, call.getLoc(), slot,
2414 srcLoad.getAddr(), slotAlign,
2415 loadSourceAlignment(srcLoad, builder));
2416 copy.setIsVolatile(srcLoad.getIsVolatile());
2417 deadRecordLoads.push_back(srcLoad);
2422 cir::StoreOp::create(builder, call.getLoc(), arg, slot);
2424 newArgs.push_back(slot);
2426 newArgs.push_back(arg);
2429 argBegin.push_back(newArgs.size());
2430 builder.setInsertionPoint(call);
2431 for (
auto [idx, freeze] : refreezes)
2432 for (
unsigned i = argBegin[idx]; i != argBegin[idx + 1]; ++i)
2433 newArgs[i] = cir::FreezeOp::create(builder, freeze.getLoc(), newArgs[i]);
2434 unsigned numEllipsisArgs =
2435 ellipsisStart ? newArgs.size() - *ellipsisStart : 0;
2437 bool hasResult = call.getNumResults() > 0;
2438 mlir::Type origRetTy =
2439 hasResult ? call.getResult().getType() : cir::VoidType::get(ctx);
2445 if (fc.returnInfo.kind == ArgKind::Indirect && hasResult) {
2446 if (failed(rewriteIndirectReturnCall(call, fc, newArgs, numEllipsisArgs,
2447 origRetTy, origCallArgTypes, builder,
2448 dl, enableNoundefAttrs)))
2449 return mlir::failure();
2450 eraseDeadFreezes(deadFreezes);
2451 eraseDeadRecordLoads(deadRecordLoads);
2452 return mlir::success();
2455 mlir::Type callRetTy = origRetTy;
2456 if (fc.returnInfo.kind == ArgKind::Ignore && hasResult)
2457 callRetTy = cir::VoidType::get(ctx);
2458 bool returnNeedsCoercion =
2459 hasResult && fc.returnInfo.kind == ArgKind::Direct &&
2460 fc.returnInfo.coercedType && fc.returnInfo.coercedType != origRetTy;
2461 if (returnNeedsCoercion)
2462 callRetTy = fc.returnInfo.coercedType;
2464 builder.setInsertionPoint(call);
2465 prependIndirectCallee(call, newArgs, callRetTy, numEllipsisArgs, builder);
2466 auto newCall = cir::CallOp::create(builder, call.getLoc(),
2467 call.getCalleeAttr(), callRetTy, newArgs);
2468 copyCallAttributes(call, newCall);
2473 if (returnNeedsCoercion) {
2474 builder.setInsertionPointAfter(newCall);
2475 SmallPtrSet<mlir::Operation *, 4> coercionOps;
2476 if (failed(emitCoercionAndReplaceUses(builder, call.getLoc(), origRetTy,
2477 newCall.getResult(), call.getResult(),
2478 slotBlock, dl, coercionOps,
2479 fc.returnInfo.directOffset)))
2480 return mlir::failure();
2483 if (needsArgAttrUpdate(fc)) {
2484 auto existing = call->getAttrOfType<mlir::ArrayAttr>(
"arg_attrs");
2485 newCall->setAttr(
"arg_attrs",
2486 updateArgAttrs(ctx, module, origCallArgTypes, existing, fc,
2487 dl, enableNoundefAttrs,
2492 if (fc.returnInfo.kind == ArgKind::Extend ||
2493 (hasResult && fc.returnInfo.kind == ArgKind::Direct)) {
2494 auto existing = call->getAttrOfType<mlir::ArrayAttr>(
"res_attrs");
2495 if (mlir::ArrayAttr updated =
2496 updateResAttrs(ctx, existing, fc.returnInfo, origRetTy, dl))
2497 newCall->setAttr(
"res_attrs", updated);
2498 }
else if (hasResult && mlir::isa<cir::VoidType>(callRetTy)) {
2499 newCall->removeAttr(
"res_attrs");
2502 if (hasResult && fc.returnInfo.kind == ArgKind::Ignore) {
2507 if (!call.getResult().use_empty()) {
2508 builder.setInsertionPointAfter(newCall);
2509 mlir::Value poison =
2510 createIgnoredValue(builder, call.getLoc(), origRetTy);
2511 call.getResult().replaceAllUsesWith(poison);
2513 }
else if (hasResult && !returnNeedsCoercion) {
2515 call.getResult().replaceAllUsesWith(newCall.getResult());
2519 eraseDeadFreezes(deadFreezes);
2520 eraseDeadRecordLoads(deadRecordLoads);
2522 return mlir::success();
2527 mlir::OpBuilder &builder) {
2528 auto oldPtrTy = mlir::cast<cir::PointerType>(addrOp.getAddr().getType());
2529 cir::FuncType newFuncTy = funcOp.getFunctionType();
2532 if (newFuncTy == oldPtrTy.getPointee())
2536 addrOp.getAddr().setType(cir::PointerType::get(newFuncTy));
2537 if (addrOp.getAddr().use_empty())
2542 mlir::OpBuilder::InsertionGuard guard(builder);
2543 builder.setInsertionPointAfter(addrOp);
2544 auto bitcast = cir::CastOp::create(builder, addrOp.getLoc(), oldPtrTy,
2545 cir::CastKind::bitcast, addrOp.getAddr());
2546 addrOp.getAddr().replaceAllUsesExcept(bitcast.getResult(), bitcast);
2555 VAArgFetch(mlir::Location loc, mlir::Value valist,
2557 const ArgClassification &ac,
const mlir::DataLayout &dl,
2558 mlir::ModuleOp module)
2559 : loc(loc), valist(valist), vaFields(vaFields), resultTy(resultTy),
2560 ac(ac), dl(dl), module(module) {
2561 assert(vaFields.size() == 4 &&
2562 "the x86-64 va_list is a four-field cursor, checked by the caller");
2568 mlir::Type resultTy;
2569 const ArgClassification ∾
2570 const mlir::DataLayout &dl;
2571 mlir::ModuleOp module;
2577struct RegisterCursor {
2578 mlir::Value gpOffsetP;
2579 mlir::Value fpOffsetP;
2580 mlir::Value gpOffset;
2581 mlir::Value fpOffset;
2585mlir::LogicalResult reportVAArgNYI(cir::VAArgOp op, llvm::StringRef what) {
2586 op->emitOpError() <<
"va_arg of " << what
2587 <<
" not yet implemented in CallConvLowering";
2588 return mlir::failure();
2595mlir::LogicalResult classifyRegisterPair(cir::VAArgOp op,
2596 const ArgClassification &ac,
2597 std::array<bool, 2> &pairIsSse,
2599 auto pairTy = mlir::dyn_cast<cir::RecordType>(ac.coercedType);
2601 return mlir::success();
2603 if (pairTy.getNumElements() != 2)
2604 return reportVAArgNYI(op,
"a register coercion that is not two eightbytes");
2606 assert(!ac.directOffset &&
2607 "a pair already spans both eightbytes, so it cannot also start "
2608 "partway into the value");
2609 for (
auto [i, memberTy] : llvm::enumerate(pairTy.getMembers()))
2610 pairIsSse[i] = isSSERegisterClass(memberTy);
2612 return mlir::success();
2617uint64_t argumentAreaAlign(mlir::Type ty, mlir::ModuleOp modOp) {
2618 return cir::CIRDataLayout(modOp).getTypeLanguageAlignment(ty);
2621mlir::Value roundPointerUpToAlignment(
CIRBaseBuilderTy &b, mlir::Location loc,
2622 mlir::Value bytePtr,
uint64_t align,
2623 const mlir::DataLayout &dl) {
2624 assert(mlir::cast<cir::PointerType>(bytePtr.getType()).getPointee() ==
2626 "the bump strides in bytes, so the pointee must be u8");
2627 assert(llvm::isPowerOf2_64(align) &&
2628 "mask rounding needs a power-of-two alignment");
2629 mlir::Value bumped =
2631 std::optional<uint64_t> indexWidth =
2632 dl.getTypeIndexBitwidth(bytePtr.getType());
2633 assert(indexWidth &&
"a pointer in the argument area has an index width");
2635 static_cast<unsigned>(*indexWidth));
2636 return cir::PtrMaskOp::create(b, loc, bytePtr.getType(), bumped, mask);
2642 const VAArgFetch &f) {
2645 f.loc, b.
getPointerTo(f.vaFields[2]), f.valist,
"overflow_arg_area", 2);
2646 mlir::Value overflow = b.
createLoad(f.loc, overflowP);
2649 uint64_t tyAlign = argumentAreaAlign(f.resultTy, f.module);
2651 bytePtr = roundPointerUpToAlignment(b, f.loc, bytePtr, tyAlign, f.dl);
2654 llvm::alignTo(f.dl.getTypeAllocSize(f.resultTy).getFixedValue(), 8);
2655 mlir::Value strideVal = b.
getSignedInt(f.loc, stride, 32);
2666 unsigned neededInt,
unsigned neededSse) {
2667 RegisterCursor cursor;
2670 f.valist,
"gp_offset", 0);
2671 cursor.gpOffset = b.
createLoad(f.loc, cursor.gpOffsetP);
2673 b.
getConstantInt(f.loc, cursor.gpOffset.getType(), 48 - neededInt * 8);
2675 b.
createCompare(f.loc, cir::CmpOpKind::le, cursor.gpOffset, limit);
2679 f.valist,
"fp_offset", 1);
2680 cursor.fpOffset = b.
createLoad(f.loc, cursor.fpOffsetP);
2681 mlir::Value limit = b.
getConstantInt(f.loc, cursor.fpOffset.getType(),
2682 176 - neededSse * 16);
2683 mlir::Value fitsInFp =
2684 b.
createCompare(f.loc, cir::CmpOpKind::le, cursor.fpOffset, limit);
2685 cursor.inRegs = cursor.inRegs
2695 const ArgClassification &ac,
2696 const RegisterCursor &cursor,
2697 const std::array<bool, 2> &pairIsSse,
2698 mlir::Value regSaveArea, mlir::Value regPairTemp) {
2699 auto pairTy = mlir::cast<cir::RecordType>(ac.coercedType);
2703 bool bothSse = pairIsSse[0] && pairIsSse[1];
2706 unsigned seenOfClass[2] = {0, 0};
2707 for (
unsigned i = 0; i < 2; ++i) {
2708 bool isSse = pairIsSse[i];
2709 mlir::Value base = isSse ? cursor.fpOffset : cursor.gpOffset;
2710 unsigned regSize = isSse ? 16 : 8;
2711 unsigned prior = seenOfClass[isSse];
2712 ++seenOfClass[isSse];
2713 mlir::Value off = base;
2719 mlir::Type elemTy = pairTy.getElementType(i);
2730uint64_t registerSlotSize(
unsigned neededInt,
unsigned neededSse) {
2731 assert(!(neededInt && neededSse) &&
2732 "a fetch needing both classes is a pair, reassembled elsewhere");
2733 return neededSse ? neededSse * 16 : neededInt * 8;
2738bool needsTempCopy(
const VAArgFetch &f,
unsigned neededInt,
2739 unsigned neededSse) {
2740 uint64_t tySize = f.dl.getTypeSize(f.resultTy).getFixedValue();
2741 if (f.ac.coercedType &&
2742 (f.ac.directOffset || registerSlotSize(neededInt, neededSse) < tySize))
2746 uint64_t slotAlign = neededSse ? 16 : 8;
2747 return argumentAreaAlign(f.resultTy, f.module) > slotAlign;
2753 const VAArgFetch &f, mlir::Value regAddr,
2754 mlir::Value temp,
unsigned neededInt,
2755 unsigned neededSse) {
2757 uint64_t tySize = f.dl.getTypeSize(f.resultTy).getFixedValue();
2759 if (f.ac.coercedType &&
2760 (f.ac.directOffset || registerSlotSize(neededInt, neededSse) < tySize)) {
2767 mlir::Value dst = temp;
2768 if (f.ac.directOffset) {
2770 mlir::Value offset = b.
getSignedInt(loc, f.ac.directOffset, 32);
2778 if (mlir::isa<cir::RecordType>(f.resultTy))
2779 cir::CopyOp::create(
2780 b, loc, temp, regView,
2781 b.getI64IntegerAttr(argumentAreaAlign(f.resultTy, f.module)),
2782 b.getI64IntegerAttr(8));
2789 const RegisterCursor &cursor,
unsigned neededInt,
2790 unsigned neededSse) {
2809mlir::Value buildSlotAddrAndAdvance(
CIRBaseBuilderTy &b, mlir::Location loc,
2810 mlir::Value valist, mlir::Type resultTy,
2812 const mlir::DataLayout &dl) {
2816 llvm::alignTo(dl.getTypeAllocSize(resultTy).getFixedValue(), slotSize);
2825mlir::LogicalResult replaceVAArgResult(
CIRBaseBuilderTy &b, cir::VAArgOp op,
2827 const mlir::DataLayout &dl) {
2828 mlir::Location loc = op.getLoc();
2829 mlir::Type resultTy = op.getType();
2831 if (mlir::isa<cir::RecordType>(resultTy)) {
2832 SmallPtrSet<mlir::Operation *, 1> noExceptions;
2833 if (failed(rewireRecordUses(b, loc, op.getResult(), resultAddr, align, dl,
2835 return mlir::failure();
2837 op.getResult().replaceAllUsesWith(
2841 return mlir::success();
2848 const ArgClassification &ac,
2849 mlir::OpBuilder &opBuilder) {
2850 auto op = mlir::cast<cir::VAArgOp>(vaArgOp);
2856 if (ac.kind == ArgKind::Ignore) {
2857 builder.setInsertionPoint(op);
2858 op.getResult().replaceAllUsesWith(
2859 createIgnoredValue(builder, op.getLoc(), op.getType()));
2861 return mlir::success();
2866 return rewriteVAArgX86_64(op, ac, builder);
2868 return rewriteVAArgAMDGPU(op, ac, builder);
2875 return reportVAArgNYI(op,
"a type on this target");
2878mlir::LogicalResult CIRABIRewriteContext::rewriteVAArgAMDGPU(
2880 assert(ac.kind == ArgKind::Direct && !ac.coercedType && ac.directAlign &&
2881 "AMDGPU reads every variadic argument in place from its slots");
2882 builder.setInsertionPoint(op);
2883 uint64_t slotSize = ac.directAlign->value();
2884 mlir::Value addr = buildSlotAddrAndAdvance(
2885 builder, op.getLoc(), op.getArgList(), op.getType(), slotSize, dl);
2886 return replaceVAArgResult(builder, op, addr, slotSize, dl);
2889mlir::LogicalResult CIRABIRewriteContext::rewriteVAArgX86_64(
2891 mlir::Location loc = op.getLoc();
2892 mlir::Type resultTy = op.getType();
2893 mlir::Value valist = op.getArgList();
2895 if (ac.kind == ArgKind::Indirect && !ac.byVal)
2896 return reportVAArgNYI(op,
"a non-trivially-copyable type");
2897 if (ac.kind == ArgKind::IndirectAliased)
2898 return reportVAArgNYI(op,
"a type passed byref");
2903 unsigned neededInt = ac.neededIntRegs;
2904 unsigned neededSse = ac.neededSseRegs;
2908 std::array<bool, 2> pairIsSse = {
false,
false};
2909 bool isRegPair =
false;
2910 if (ac.kind == ArgKind::Direct && neededInt + neededSse == 2 &&
2912 if (classifyRegisterPair(op, ac, pairIsSse, isRegPair).failed())
2913 return mlir::failure();
2916 auto vaListRecTy = mlir::dyn_cast<cir::RecordType>(
2917 mlir::cast<cir::PointerType>(valist.getType()).getPointee());
2918 if (!vaListRecTy || vaListRecTy.getNumElements() != 4) {
2919 return reportVAArgNYI(op,
2920 "a va_list that is not the four-field gp_offset / "
2921 "fp_offset / overflow_arg_area / reg_save_area "
2925 const VAArgFetch fetch{loc, valist, vaListRecTy.getMembers(), resultTy, ac,
2929 builder.setInsertionPoint(op);
2932 if (neededInt == 0 && neededSse == 0) {
2933 addr = buildOverflowAddrAndAdvance(builder, fetch);
2935 RegisterCursor cursor =
2936 buildRegisterGate(builder, fetch, neededInt, neededSse);
2944 bool pairNeedsReassembly = isRegPair && neededSse != 0;
2947 mlir::Value regPairTemp;
2948 if (pairNeedsReassembly) {
2953 loc, builder.
getPointerTo(ac.coercedType),
"vaarg.reg",
2955 std::max(cirDataLayout.getTypeLanguageAlignment(ac.coercedType),
2956 argumentAreaAlign(resultTy, module))));
2959 mlir::Value regTemp;
2960 bool copyThroughTemp =
2961 !pairNeedsReassembly && needsTempCopy(fetch, neededInt, neededSse);
2962 if (copyThroughTemp) {
2968 addr = cir::TernaryOp::create(
2969 builder, loc, cursor.inRegs,
2971 [&](mlir::OpBuilder &ob, mlir::Location l) {
2972 CIRBaseBuilderTy b(ob);
2973 mlir::Value regSaveArea = b.createLoad(
2974 l, b.createGetMember(l, b.getPointerTo(fetch.vaFields[3]),
2975 valist,
"reg_save_area", 3));
2976 regSaveArea = b.createPtrBitcast(regSaveArea, byteTy);
2978 mlir::Value regAddr;
2979 if (pairNeedsReassembly) {
2980 reassembleRegisterPair(b, l, ac, cursor, pairIsSse,
2981 regSaveArea, regPairTemp);
2982 regAddr = b.createPtrBitcast(regPairTemp, byteTy);
2985 neededSse ? cursor.fpOffset : cursor.gpOffset;
2986 regAddr = b.createPtrStride(l, regSaveArea, off);
2987 if (copyThroughTemp) {
2988 regAddr = copyRegisterToTemp(b, l, fetch, regAddr, regTemp,
2989 neededInt, neededSse);
2993 advanceRegisterCursors(b, l, cursor, neededInt, neededSse);
2994 cir::YieldOp::create(b, l, regAddr);
2997 [&](mlir::OpBuilder &ob, mlir::Location l) {
2999 mlir::Value memAddr = buildOverflowAddrAndAdvance(b, fetch);
3000 cir::YieldOp::create(b, l, memAddr);
3006 return replaceVAArgResult(builder, op, addr,
3007 argumentAreaAlign(resultTy, module), dl);
static mlir::LogicalResult prepareOneNonByvalParameter(cir::FuncOp funcOp, unsigned argNo, const ArgClassification &ac, mlir::BlockArgument blockArg, mlir::DominanceInfo &dom, SmallPtrSetImpl< mlir::Operation * > &claimedSlots)
Bring funcOp's non-byval indirect parameter argNo into the shape the rest of the rewrite assumes.
*collection of selector each with an associated kind and an ordered *collection of selectors A selector has a kind
void rewriteFunctionAddress(cir::GetGlobalOp addrOp, cir::FuncOp funcOp, mlir::OpBuilder &builder)
Retype addrOp, which holds the address of funcOp, to the signature funcOp was rewritten to,...
mlir::LogicalResult rewriteFunctionDefinition(mlir::FunctionOpInterface funcOp, const mlir::abi::FunctionClassification &fc, mlir::OpBuilder &builder) override
void finalizeParameterSlots()
Replace each non-byval indirect parameter's spill slot with the incoming pointer, so the body operate...
mlir::LogicalResult prepareNonByvalParameters(cir::FuncOp funcOp, const mlir::abi::FunctionClassification &fc)
Bring each non-byval indirect parameter of funcOp into the shape the rest of the rewrite assumes: the...
mlir::LogicalResult rewriteVAArg(mlir::Operation *vaArgOp, const mlir::abi::ArgClassification &ac, mlir::OpBuilder &builder) override
Expand a cir.va_arg with the va_arg rules of the target this context lowers for.
mlir::LogicalResult rewriteCallSite(mlir::Operation *callOp, const mlir::abi::FunctionClassification &fc, mlir::OpBuilder &builder) override
mlir::Value createAdd(mlir::Location loc, mlir::Value lhs, mlir::Value rhs, OverflowFlags flags=OverflowFlags::None)
cir::PtrStrideOp createPtrStride(mlir::Location loc, mlir::Value base, mlir::Value stride)
cir::GetMemberOp createGetMember(mlir::Location loc, mlir::Type resultTy, mlir::Value base, llvm::StringRef name, unsigned index)
cir::PointerType getPointerTo(mlir::Type ty)
cir::IntType getUIntNTy(int n)
mlir::Value createPtrBitcast(mlir::Value src, mlir::Type newPointeeTy)
mlir::Value createAlloca(mlir::Location loc, cir::PointerType addrType, llvm::StringRef name, mlir::IntegerAttr alignment, mlir::Value dynAllocSize)
cir::LoadOp createLoad(mlir::Location loc, mlir::Value ptr, bool isVolatile=false, uint64_t alignment=0, bool isNontemporal=false)
mlir::Value getSignedInt(mlir::Location loc, int64_t val, unsigned numBits)
mlir::Value createBitcast(mlir::Value src, mlir::Type newTy)
cir::CmpOp createCompare(mlir::Location loc, cir::CmpOpKind kind, mlir::Value lhs, mlir::Value rhs)
cir::ConstantOp getConstantInt(mlir::Location loc, mlir::Type ty, int64_t value)
mlir::Value createAlignedLoad(mlir::Location loc, mlir::Value ptr, uint64_t alignment)
mlir::Value createLogicalAnd(mlir::Location loc, mlir::Value lhs, mlir::Value rhs)
cir::StoreOp createStore(mlir::Location loc, mlir::Value val, mlir::Value dst, bool isVolatile=false, bool isNontemporal=false, mlir::IntegerAttr align={}, cir::SyncScopeKindAttr scope={}, cir::MemOrderAttr order={})
uint64_t getTypeLanguageAlignment(mlir::Type ty) const
Returns the alignment, in bytes, that the source language requires for an object of type ty.
llvm::TypeSize getTypeAllocSize(mlir::Type ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
llvm::TypeSize getTypeStoreSize(mlir::Type ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
C++ view class that accepts both !cir.struct and !cir.union types.
llvm::ArrayRef< mlir::Type > getMembers() const
static llvm::SmallVector< RecordMemberKind > getAllDataKinds(llvm::ArrayRef< mlir::Type > members)
One Data kind per member.
size_t getNumElements() const
static CharUnits fromQuantity(QuantityType Quantity)
Construct a CharUnits quantity from a raw integer type.
RecordLayoutAttr tryGetRecordLayout(mlir::ModuleOp mod, mlir::StringAttr name)
Same lookup as getRecordLayout, but returns a null attribute instead of asserting when the record has...
bool memberOwnsBytes(mlir::Type memberTy)
Whether a record member occupies bytes of its record.
cir::FuncType getIndirectCalleeType(cir::CIRCallOpInterface call)
The signature the indirect call call reaches its callee through.
AllocaOp getUnderlyingAlloca(mlir::Value addr)
The alloca that defines addr, looking through casts that preserve the underlying storage.
const internal::VariadicAllOfMatcher< Attr > attr
U cast(CodeGen::Address addr)
__builtin_elementwise_add_sat __builtin_elementwise_sub_sat uint32_t int64_t
__builtin_elementwise_add_sat __builtin_elementwise_sub_sat uint32_t uint64_t __builtin_riscv_ssha_i32 __builtin_riscv_sshl_u32 uint64_t
static bool noaliasOnByvalAttr()