clang 24.0.0git
CIRABIRewriteContext.cpp
Go to the documentation of this file.
1//===- CIRABIRewriteContext.cpp - CIR ABI rewrite context ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
10#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
11#include "mlir/IR/Builders.h"
12#include "mlir/IR/Dominance.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/IR/Attributes.h"
21#include "llvm/Support/MathExtras.h"
22#include <algorithm>
23#include <array>
24#include <utility>
25
26using namespace cir;
27using namespace mlir;
28using namespace mlir::abi;
29
30// This rewrite context supports the Direct (with or without coercion),
31// Extend, Ignore, Indirect-return (sret), Indirect-argument (byval and
32// non-byval), IndirectAliased-argument (byref), and Expand (struct
33// flattening) classifications.
34//
35// An Indirect argument is byval or not, following its classification's
36// byVal flag. byval is a by-value parameter the ABI passes in memory rather
37// than registers, usually for its size. Non-byval is a by-value parameter
38// whose type cannot be copied freely, because it has a non-trivial copy
39// constructor, move constructor, or destructor, so the callee works on the
40// caller's own object rather than a copy.
41//
42// At the call site both forward the caller's storage, since byval is copied
43// on the caller's side of the call. byval falls back to a slot of its own
44// when the operand's storage cannot be handed on. At the callee, byval fills
45// the CIRGen param slot with a copy of the incoming object, while non-byval
46// rewires that slot to the incoming pointer so the body mutates the caller's
47// storage in place.
48//
49// A byref argument is lowered like byval, apart from its attribute and the
50// address space of its pointer. The callee must not write through that
51// pointer, so it fills the CIRGen param slot with a copy just as byval does.
52//
53// For Expand, the single struct argument is replaced by N scalar arguments
54// (one per field). At the callee, the N field block arguments are stored
55// directly into the parameter's own alloca (the CIRGen spill slot). At the
56// call site, the struct operand is decomposed into its fields by reading
57// each member from the source alloca (get_member + load) when the operand is
58// a load of an alloca, or via cir.extract_member otherwise.
59//
60// For Direct + canFlatten (where the coerced type is a multi-field struct), the
61// coerced struct is similarly passed as N separate arguments, one per field.
62// The callee stores the N scalar block args into one slot sized and aligned for
63// both the coerced struct and the original argument type, which the body reads.
64// The call site reads each field of the coerced struct out of memory holding
65// the operand as a separate call argument.
66
67namespace {
68
69/// A pointer to \p pointee in target address space \p addrSpace. Zero is the
70/// default address space, which carries no attribute.
71cir::PointerType getPtrTyInAddrSpace(mlir::Type pointee, unsigned addrSpace) {
72 mlir::ptr::MemorySpaceAttrInterface asAttr;
73 if (addrSpace)
74 asAttr = cir::TargetAddressSpaceAttr::get(pointee.getContext(), addrSpace);
75 return cir::PointerType::get(pointee, asAttr);
76}
77
78/// The pointer passed in place of an Indirect or byref value, or an sret
79/// return, of type \p pointee.
80cir::PointerType getIndirectPtrTy(mlir::Type pointee,
81 const ArgClassification &ac) {
82 assert(
83 (ac.kind == ArgKind::Indirect || ac.kind == ArgKind::IndirectAliased) &&
84 "expected an indirect classification");
85 return getPtrTyInAddrSpace(pointee, ac.indirectAddrSpace);
86}
87
88/// The struct a union is flattened as: its storage member, then its tail
89/// padding if it has any. This matches the union's LLVM type.
90cir::StructType getUnionFlattenType(cir::UnionType unionTy,
91 const mlir::DataLayout &dl) {
93 if (mlir::Type storage = unionTy.getUnionStorageType(dl))
94 fields.push_back(storage);
95 if (mlir::Type padding = unionTy.getPadding())
96 fields.push_back(padding);
97 return cir::StructType::get(unionTy.getContext(), fields, unionTy.getPacked(),
98 /*is_class=*/false,
100}
101
102/// Return the coerced RecordType for a Direct classification that should be
103/// flattened into individual scalar arguments, or a null type if the
104/// classification does not call for flattening.
105///
106/// Flattening applies when all four conditions hold:
107/// 1. The classification is Direct with a non-null coercedType.
108/// 2. canFlatten is set.
109/// 3. The coercedType is a struct, or a union, which is flattened as the
110/// struct getUnionFlattenType returns.
111/// 4. The struct has at least one field. A struct of one field is passed as
112/// that field.
113cir::RecordType getFlattenedCoercedType(const ArgClassification &ac,
114 const mlir::DataLayout &dl) {
115 if (ac.kind != ArgKind::Direct || !ac.coercedType || !ac.canFlatten)
116 return {};
117 auto recTy = dyn_cast<cir::RecordType>(ac.coercedType);
118 if (auto unionTy = dyn_cast_if_present<cir::UnionType>(recTy))
119 recTy = getUnionFlattenType(unionTy, dl);
120 if (!recTy || !recTy.isStruct() || recTy.getMembers().empty())
121 return {};
122 return recTy;
123}
124
125/// Whether \p ac is an Extend of a record argument of type \p origTy. Such an
126/// argument is a transparent union, passed as its first member, which
127/// \p ac.coercedType holds.
128bool isRecordExtend(const ArgClassification &ac, mlir::Type origTy) {
129 if (ac.kind != ArgKind::Extend || !mlir::isa<cir::RecordType>(origTy))
130 return false;
131 assert(ac.coercedType && "an Extend of a record names its first member type");
132 return true;
133}
134
135/// Build the new argument-type list for a function whose ABI classification
136/// is \p fc. Handles Direct (with or without coercion), Extend, Ignore,
137/// Indirect (byval and non-byval), IndirectAliased (byref), and Expand
138/// (struct flattening) arguments.
139/// The sret return pointer, when present, is prepended by
140/// rewriteFunctionDefinition rather than here.
141mlir::LogicalResult buildNewArgTypes(
142 ArrayRef<mlir::Type> oldArgTypes, const FunctionClassification &fc,
143 SmallVectorImpl<mlir::Type> &newArgTypes, const mlir::DataLayout &dl,
144 function_ref<mlir::InFlightDiagnostic()> emitError) {
145 assert(newArgTypes.empty() && "expected an empty output vector");
146 newArgTypes.reserve(oldArgTypes.size());
147 for (auto [idx, ac] : llvm::enumerate(fc.argInfos)) {
148 mlir::Type origTy = oldArgTypes[idx];
149 switch (ac.kind) {
150 case ArgKind::Direct:
151 // Direct with canFlatten and a struct coerced type: push one wire type
152 // per field of the coerced struct rather than the struct itself.
153 if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl)) {
154 llvm::append_range(newArgTypes, flatTy.getMembers());
155 } else {
156 // Direct with a coerced type: the lowered signature uses the coerced
157 // type; the body still expects origTy and insertArgCoercion recovers it
158 // from a coercion slot. Direct without coercion is a pass-through.
159 newArgTypes.push_back(ac.coercedType ? ac.coercedType : origTy);
160 }
161 break;
162 case ArgKind::Ignore:
163 break;
164 case ArgKind::Expand: {
165 // Flatten the struct into one wire argument per field. The
166 // reassembly in the callee body and the decomposition at the call
167 // site are handled by insertArgCoercion and rewriteCallSite.
168 auto recTy = cast<cir::RecordType>(origTy);
169 assert(recTy.isStruct() &&
170 "Expand classification requires a struct type, not a union");
171 assert(!recTy.getMembers().empty() &&
172 "Expand classification requires at least one struct field");
173 llvm::append_range(newArgTypes, recTy.getMembers());
174 break;
175 }
176 case ArgKind::Extend:
177 // Extend keeps the original (narrow) type in the signature; the
178 // sign/zero extension is communicated to LLVM via the llvm.signext /
179 // llvm.zeroext arg attribute, attached separately below. For an integer
180 // the coercedType the classifier set is informational (typically the
181 // register-width type the value gets extended to in registers) and does
182 // not change the CIR signature. A transparent union is passed as the
183 // type of its first member, which coercedType holds.
184 newArgTypes.push_back(isRecordExtend(ac, origTy) ? ac.coercedType
185 : origTy);
186 break;
187 case ArgKind::Indirect:
188 case ArgKind::IndirectAliased:
189 newArgTypes.push_back(getIndirectPtrTy(origTy, ac));
190 break;
191 }
192 }
193 return mlir::success();
194}
195
196/// Compute the new return type for a function whose return classification
197/// is \p retInfo. Direct returns keep (or coerce to) their type, Ignore and
198/// Indirect (sret) returns become void, Extend keeps its type; Expand and
199/// IndirectAliased emit an error.
200mlir::Type
201computeNewReturnType(mlir::Type origRetTy, const ArgClassification &retInfo,
202 mlir::MLIRContext *ctx,
203 function_ref<mlir::InFlightDiagnostic()> emitError) {
204 switch (retInfo.kind) {
205 case ArgKind::Direct:
206 // Direct return with a coerced type uses the coerced type on the wire;
207 // the rewriter inserts a coercion before each cir.return.
208 return retInfo.coercedType ? retInfo.coercedType : origRetTy;
209 case ArgKind::Ignore:
210 return cir::VoidType::get(ctx);
211 case ArgKind::Expand:
212 emitError() << "Expand return is not allowed (classic codegen rejects "
213 << "it in EmitFunctionEpilog)";
214 return nullptr;
215 case ArgKind::IndirectAliased:
216 emitError() << "IndirectAliased return is not supported";
217 return nullptr;
218 case ArgKind::Extend:
219 // Same convention as Extend args: keep the original return type in the
220 // signature; the sign/zero extension is communicated via the
221 // llvm.signext / llvm.zeroext res attribute attached separately below.
222 return origRetTy;
223 case ArgKind::Indirect:
224 // sret: the value is returned through a pointer argument that the ABI
225 // synthesizes (rewriteFunctionDefinition prepends it to the argument
226 // list); it is not part of the source-level signature, so the wire
227 // return type becomes void.
228 return cir::VoidType::get(ctx);
229 }
230 llvm_unreachable("all ArgKind cases handled");
231}
232
233/// Create a typed poison constant to stand in for a value the body of a
234/// function (or the result of a call) still references but whose ABI
235/// classification is Ignore. Using poison is honest -- the value is
236/// genuinely unused at the ABI boundary -- and avoids a fake alloca+load
237/// pattern that would suggest we have a value when we don't.
238mlir::Value createIgnoredValue(mlir::OpBuilder &builder, mlir::Location loc,
239 mlir::Type ty) {
240 return cir::ConstantOp::create(builder, loc, ty, cir::PoisonAttr::get(ty));
241}
242
243/// Whether \p coercedTy is wider than the type a value of type \p origTy has
244/// in memory, so the coerced value passes bits the value does not define and
245/// cannot be noundef. This is the comparison classic CodeGen makes. A bool
246/// vector's memory type is its storage integer, one bit per element and at
247/// least a byte wide, not its power-of-two size.
248static bool isCoercionWiderThanMemory(mlir::Type origTy, mlir::Type coercedTy,
249 const mlir::DataLayout &dl) {
250 uint64_t memoryBits = dl.getTypeSizeInBits(origTy).getFixedValue();
251 if (auto vecTy = mlir::dyn_cast<cir::VectorType>(origTy);
252 vecTy && mlir::isa<cir::BoolType>(vecTy.getElementType()))
253 memoryBits = vecTy.getBoolStorageWidth();
254 return dl.getTypeSizeInBits(coercedTy).getFixedValue() > memoryBits;
255}
256
257/// Whether updateArgAttrs has anything to change: an arg that is Ignore (its
258/// slot is dropped), Extend (needs llvm.signext / llvm.zeroext), Indirect or
259/// IndirectAliased (needs the pointer attributes), Expand (changes the argument
260/// count), or a coerced Direct, which may be flattened or may lose
261/// llvm.noundef, or that has a stack-slot alignment, which needs
262/// llvm.alignstack.
263static bool needsArgAttrUpdate(const FunctionClassification &fc) {
264 return llvm::any_of(fc.argInfos, [](const ArgClassification &ac) {
265 return ac.kind == ArgKind::Ignore || ac.kind == ArgKind::Extend ||
266 ac.kind == ArgKind::Indirect ||
267 ac.kind == ArgKind::IndirectAliased || ac.kind == ArgKind::Expand ||
268 (ac.kind == ArgKind::Direct && (ac.coercedType || ac.directAlign));
269 });
270}
271
272/// Build an updated arg_attrs ArrayAttr that drops Ignore'd args, adds
273/// llvm.signext / llvm.zeroext on Extend args, adds the pointer attributes
274/// for Indirect and IndirectAliased args in place of llvm.nofpclass, drops
275/// llvm.noundef from a coerced Direct arg that passes undefined bits,
276/// adds llvm.alignstack on a Direct arg with a stack-slot alignment, gives each
277/// field of a Direct+canFlatten argument the argument's attributes, and gives
278/// each field of an Expand argument an empty dict. When \p enableNoundefAttrs
279/// is set, Extend, Indirect and IndirectAliased args are also marked
280/// llvm.noundef. Direct and Extend slots keep their other existing attributes.
281/// \p origArgTypes provides the pre-rewrite type for each arg slot.
282mlir::ArrayAttr updateArgAttrs(mlir::MLIRContext *ctx, mlir::ModuleOp module,
283 ArrayRef<mlir::Type> origArgTypes,
284 mlir::ArrayAttr existingArgAttrs,
285 const FunctionClassification &fc,
286 const mlir::DataLayout &dl,
287 bool enableNoundefAttrs) {
288 mlir::Builder builder(ctx);
290 newArgAttrs.reserve(fc.argInfos.size());
291 for (auto [oldIdx, ac] : llvm::enumerate(fc.argInfos)) {
292 if (ac.kind == ArgKind::Ignore)
293 continue;
294 mlir::DictionaryAttr existing = builder.getDictionaryAttr({});
295 if (existingArgAttrs && oldIdx < existingArgAttrs.size())
296 existing = mlir::cast<mlir::DictionaryAttr>(existingArgAttrs[oldIdx]);
297 if (ac.kind == ArgKind::Direct && ac.coercedType &&
298 isCoercionWiderThanMemory(origArgTypes[oldIdx], ac.coercedType, dl)) {
299 mlir::NamedAttrList attrs(existing);
300 attrs.erase(mlir::LLVM::LLVMDialect::getNoUndefAttrName());
301 existing = attrs.getDictionary(ctx);
302 }
303 if (ac.kind == ArgKind::Direct && ac.directAlign) {
304 mlir::NamedAttrList attrs(existing);
305 attrs.set(mlir::LLVM::LLVMDialect::getStackAlignmentAttrName(),
306 builder.getI64IntegerAttr(ac.directAlign->value()));
307 existing = attrs.getDictionary(ctx);
308 }
309 if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl)) {
310 // Direct + canFlatten: as in classic CodeGen, each flattened field
311 // carries the attributes of the argument it came from.
312 newArgAttrs.append(flatTy.getNumElements(), existing);
313 } else if (ac.kind == ArgKind::Expand) {
314 // An expanded record has no attributes classic CodeGen would copy to its
315 // fields, so each field gets an empty dict.
316 auto recTy = cast<cir::RecordType>(origArgTypes[oldIdx]);
317 newArgAttrs.append(recTy.getNumElements(), builder.getDictionaryAttr({}));
318 } else if (ac.kind == ArgKind::Extend) {
319 // An extended value has no undefined bits. This covers a transparent
320 // union passed as its extended first member, which CIRGen does not
321 // mark noundef because it is a record.
322 StringRef attrName = ac.signExtend
323 ? mlir::LLVM::LLVMDialect::getSExtAttrName()
324 : mlir::LLVM::LLVMDialect::getZExtAttrName();
325 mlir::NamedAttrList attrs(existing);
326 attrs.set(attrName, builder.getUnitAttr());
327 if (enableNoundefAttrs)
328 attrs.set(mlir::LLVM::LLVMDialect::getNoUndefAttrName(),
329 builder.getUnitAttr());
330 newArgAttrs.push_back(attrs.getDictionary(ctx));
331 } else if (ac.kind == ArgKind::Indirect) {
332 // Indirect lowering hands the callee a pointer. llvm.align and
333 // llvm.noundef describe that pointer, not the bytes it points to, and
334 // llvm.nofpclass, which describes the value, does not apply to a
335 // pointer. Without byval the pointer points to the caller's own object.
336 // With byval the backend copies the pointee on the caller's side of the
337 // call, so the callee gets an object of its own even when the pointer
338 // points to the caller's own object.
339 mlir::Type pointeeTy = origArgTypes[oldIdx];
340 mlir::NamedAttrList attrs(existing);
341 attrs.erase(mlir::LLVM::LLVMDialect::getNoFPClassAttrName());
342 attrs.set(mlir::LLVM::LLVMDialect::getAlignAttrName(),
343 builder.getI64IntegerAttr(ac.indirectAlign.value()));
344 if (enableNoundefAttrs)
345 attrs.set(mlir::LLVM::LLVMDialect::getNoUndefAttrName(),
346 builder.getUnitAttr());
347 if (ac.byVal) {
348 // llvm.byval(T) records the pre-rewrite arg type because the opaque
349 // LLVM pointer cannot carry it. Classic adds llvm.noalias under
350 // -fpass-by-value-is-noalias, which CIR does not plumb through.
352 attrs.set(mlir::LLVM::LLVMDialect::getByValAttrName(),
353 mlir::TypeAttr::get(pointeeTy));
354 } else {
355 // llvm.nofreeobj says the object cannot be freed while the callee
356 // runs, which holds because the caller owns it across the call.
357 attrs.set(mlir::LLVM::LLVMDialect::getNoFreeObjAttrName(),
358 builder.getUnitAttr());
359 // llvm.dead_on_return says the object's lifetime ends in the callee,
360 // which holds unless the caller still has a destructor to run on it.
361 // A record with no layout entry has no destructor.
362 auto recTy = mlir::dyn_cast<cir::RecordType>(pointeeTy);
363 cir::RecordLayoutAttr layout =
364 recTy ? cir::tryGetRecordLayout(module, recTy.getName())
365 : cir::RecordLayoutAttr();
366 if (!layout || layout.getHasTrivialDtor())
367 attrs.set(mlir::LLVM::LLVMDialect::getDeadOnReturnAttrName(),
368 builder.getI64IntegerAttr(static_cast<int64_t>(
369 llvm::DeadOnReturnInfo().toIntValue())));
370 attrs.set(mlir::LLVM::LLVMDialect::getDereferenceableAttrName(),
371 builder.getI64IntegerAttr(
372 dl.getTypeAllocSize(pointeeTy).getFixedValue()));
373 }
374 newArgAttrs.push_back(attrs.getDictionary(ctx));
375 } else if (ac.kind == ArgKind::IndirectAliased) {
376 // llvm.byref(T) records the pre-rewrite arg type, as llvm.byval does.
377 // llvm.nofpclass describes the value and does not apply to a pointer.
378 mlir::NamedAttrList attrs(existing);
379 attrs.erase(mlir::LLVM::LLVMDialect::getNoFPClassAttrName());
380 attrs.set(mlir::LLVM::LLVMDialect::getAlignAttrName(),
381 builder.getI64IntegerAttr(ac.indirectAlign.value()));
382 if (enableNoundefAttrs)
383 attrs.set(mlir::LLVM::LLVMDialect::getNoUndefAttrName(),
384 builder.getUnitAttr());
385 attrs.set(mlir::LLVM::LLVMDialect::getByRefAttrName(),
386 mlir::TypeAttr::get(origArgTypes[oldIdx]));
387 newArgAttrs.push_back(attrs.getDictionary(ctx));
388 } else {
389 newArgAttrs.push_back(existing);
390 }
391 }
392 return builder.getArrayAttr(newArgAttrs);
393}
394
395/// Build an updated res_attrs ArrayAttr (single entry, since CIR funcs have
396/// at most one result) that adds llvm.signext / llvm.zeroext on an Extend
397/// return and drops llvm.noundef from a coerced return that passes undefined
398/// bits. Preserves any other existing res attributes. Returns
399/// \p existingResAttrs unchanged, which may be null, when there is nothing to
400/// add or drop.
401mlir::ArrayAttr updateResAttrs(mlir::MLIRContext *ctx,
402 mlir::ArrayAttr existingResAttrs,
403 const ArgClassification &retInfo,
404 mlir::Type origRetTy,
405 const mlir::DataLayout &dl) {
406 bool needsNoUndefDrop =
407 retInfo.kind == ArgKind::Direct && retInfo.coercedType &&
408 isCoercionWiderThanMemory(origRetTy, retInfo.coercedType, dl);
409 if (retInfo.kind != ArgKind::Extend && !needsNoUndefDrop)
410 return existingResAttrs;
411
412 mlir::NamedAttrList attrs;
413 if (existingResAttrs && !existingResAttrs.empty())
414 attrs.append(
415 mlir::cast<mlir::DictionaryAttr>(existingResAttrs[0]).getValue());
416 if (retInfo.kind == ArgKind::Extend)
417 attrs.set(retInfo.signExtend ? "llvm.signext" : "llvm.zeroext",
418 mlir::UnitAttr::get(ctx));
419 if (needsNoUndefDrop)
420 attrs.erase(mlir::LLVM::LLVMDialect::getNoUndefAttrName());
421 return mlir::ArrayAttr::get(ctx, {attrs.getDictionary(ctx)});
422}
423
424/// The number of bytes a coercion memory slot needs to hold a value of type
425/// \p ty without truncating it. For most types this is the ordinary storage
426/// size. For a _BitInt it is deliberately the value's own literal byte
427/// footprint (ceil(width/8)) rather than the wider, ABI-alignment-padded
428/// footprint a _BitInt gets as a record member (see
429/// cir::IntType::getStorageTypeWidth): this coercion is about how many bytes
430/// the *value* needs to round-trip, not how a record would lay it out, and
431/// those are genuinely different questions for a _BitInt (e.g. _BitInt(33)
432/// only needs 5 bytes here, even though it occupies 8 padded bytes as a
433/// record member).
434static uint64_t coercionByteSize(mlir::Type ty, const mlir::DataLayout &dl) {
435 if (auto intTy = mlir::dyn_cast<cir::IntType>(ty))
436 return llvm::divideCeil(intTy.getWidth(), 8);
437 return dl.getTypeSize(ty);
438}
439
440/// A \p wantTy-typed view of the storage \p base points at. When \p wantTy is
441/// \p base's own pointee type, \p base is returned unchanged and \p offset must
442/// be 0. Any other type is reached by a bitcast, preceded by a byte stride
443/// of \p offset bytes when \p offset is nonzero. The view keeps \p base's
444/// address space. Any operations created are added to \p createdOps.
445static mlir::Value
446emitViewAtOffset(mlir::OpBuilder &builder, mlir::Location loc,
447 mlir::Type wantTy, mlir::Value base,
448 SmallPtrSetImpl<mlir::Operation *> &createdOps,
449 unsigned offset) {
450 auto basePtrTy = mlir::cast<cir::PointerType>(base.getType());
451 if (wantTy == basePtrTy.getPointee()) {
452 assert(offset == 0 && "a view of the storage's own type has no offset");
453 return base;
454 }
455 mlir::ptr::MemorySpaceAttrInterface addrSpace = basePtrTy.getAddrSpace();
456 mlir::Value view = base;
457 if (offset != 0) {
458 auto u8Ty = cir::IntType::get(builder.getContext(), 8, /*isSigned=*/false);
459 auto u8PtrTy = cir::PointerType::get(u8Ty, addrSpace);
460 auto u8Base = cir::CastOp::create(builder, loc, u8PtrTy,
461 cir::CastKind::bitcast, base);
462 createdOps.insert(u8Base);
463 auto strideTy =
464 cir::IntType::get(builder.getContext(), 64, /*isSigned=*/true);
465 auto strideVal = cir::ConstantOp::create(
466 builder, loc, cir::IntAttr::get(strideTy, offset));
467 createdOps.insert(strideVal);
468 view = cir::PtrStrideOp::create(builder, loc, u8PtrTy, u8Base, strideVal);
469 createdOps.insert(view.getDefiningOp());
470 }
471 auto typedView = cir::CastOp::create(builder, loc,
472 cir::PointerType::get(wantTy, addrSpace),
473 cir::CastKind::bitcast, view);
474 createdOps.insert(typedView);
475 return typedView;
476}
477
478/// A pointer to \p pointee in the target's alloca address space, which every
479/// temporary this pass allocates has to live in.
480static cir::PointerType getAllocaPtrTy(mlir::Type pointee,
481 const mlir::DataLayout &dl) {
482 mlir::ptr::MemorySpaceAttrInterface addrSpace;
483 if (auto asAttr = mlir::dyn_cast_if_present<mlir::IntegerAttr>(
484 dl.getAllocaMemorySpace()))
485 addrSpace = cir::TargetAddressSpaceAttr::get(pointee.getContext(),
486 asAttr.getUInt());
487 return cir::PointerType::get(pointee, addrSpace);
488}
489
490/// A coercion slot for \p srcTy and \p dstTy, which must differ. It is typed
491/// as \p srcTy unless coercionByteSize says \p dstTy is larger, aligned for
492/// both, and placed at the start of \p slotBlock. \p offset, where the
493/// coerced side sits within the slot, is only checked by assertions. The
494/// alloca is added to \p createdOps.
495static cir::AllocaOp createCoercionSlot(
496 mlir::OpBuilder &builder, mlir::Location loc, mlir::Type srcTy,
497 mlir::Type dstTy, mlir::Block *slotBlock, const mlir::DataLayout &dl,
498 SmallPtrSetImpl<mlir::Operation *> &createdOps, unsigned offset) {
499 assert(srcTy != dstTy && "callers must pre-check that the types differ");
500
501 uint64_t srcAlign = dl.getTypeABIAlignment(srcTy);
502 uint64_t dstAlign = dl.getTypeABIAlignment(dstTy);
503 uint64_t allocaAlign = std::max(srcAlign, dstAlign);
504 mlir::Type slotTy = coercionByteSize(srcTy, dl) >= coercionByteSize(dstTy, dl)
505 ? srcTy
506 : dstTy;
507
508 // Sizes are compared two ways on purpose: relative size in
509 // coercionByteSize terms, which is how slotTy was picked, and capacity in
510 // getTypeSize terms, which is what the alloca is given.
511 [[maybe_unused]] mlir::Type coercedTy = ((slotTy == srcTy) ? dstTy : srcTy);
512 assert((offset == 0 ||
513 coercionByteSize(coercedTy, dl) < coercionByteSize(slotTy, dl)) &&
514 "a direct offset must land on the coerced side, the smaller one");
515 assert((offset == 0 ||
516 offset + dl.getTypeSize(coercedTy) <= dl.getTypeSize(slotTy)) &&
517 "coerce slot too small for offset access");
518 assert((offset == 0 || offset % dl.getTypeABIAlignment(coercedTy) == 0) &&
519 "a direct offset must be aligned for the coerced access");
520
521 mlir::OpBuilder::InsertionGuard guard(builder);
522 builder.setInsertionPointToStart(slotBlock);
523 auto alloca = cir::AllocaOp::create(builder, loc, getAllocaPtrTy(slotTy, dl),
524 builder.getStringAttr("coerce"),
525 builder.getI64IntegerAttr(allocaAlign));
526 createdOps.insert(alloca);
527 return alloca;
528}
529
530/// Coerce \p src into a temporary memory slot typed for \p dstTy at the
531/// current builder insertion point, and return the destination-typed pointer
532/// to that slot without loading the value back out. This is the shared
533/// memory half of emitCoercion: callers that want the whole coerced value use
534/// emitCoercion (below); callers that want to read individual members of a
535/// coerced struct (the call-site struct flattening) take the returned pointer
536/// and emit their own cir.get_member + cir.load per field. Lowers uniformly
537/// for scalar, vector, and record types.
538///
539/// The slot is sized to the larger of the two types so that neither the store
540/// nor a later load ever runs past it: the coerced ABI type can be larger
541/// than the original (e.g. a 12-byte aggregate passed as `{i64, i64}`), so
542/// accessing the destination through a source-sized slot would over-read.
543/// Alignment is max(srcAlign, dstAlign) to satisfy both accesses. The slot
544/// is written through a source-typed view and returned as a destination-typed
545/// view.
546///
547/// The temporary alloca is placed at the start of \p slotBlock, which must
548/// dominate every use of the coerced value and must be a block that ends up
549/// inside the enclosing function's entry block after any later outlining.
550///
551/// \p offset is where the coerced value sits within the larger of the two
552/// types, non-zero when the ABI passes a value in a register read from
553/// partway into its storage. The coerced type must be the smaller of the
554/// two, so one that can exceed the value it coerces, such as a multi-field
555/// register tuple, must not be given an offset.
556///
557/// Any operations the helper creates are appended to \p createdOps so the
558/// caller can leave them out when it later rewires the source value's uses,
559/// which would otherwise clobber the store's value operand.
560mlir::Value emitCoercionToMemory(mlir::OpBuilder &builder, mlir::Location loc,
561 mlir::Type dstTy, mlir::Value src,
562 mlir::Block *slotBlock,
563 const mlir::DataLayout &dl,
564 SmallPtrSetImpl<mlir::Operation *> &createdOps,
565 unsigned offset) {
566 mlir::Type srcTy = src.getType();
567 cir::AllocaOp alloca = createCoercionSlot(builder, loc, srcTy, dstTy,
568 slotBlock, dl, createdOps, offset);
569 mlir::Type slotTy = alloca.getAllocaType();
570
571 // Store through a source-typed view of the slot.
572 mlir::Value srcSlot = emitViewAtOffset(
573 builder, loc, srcTy, alloca, createdOps, srcTy == slotTy ? 0 : offset);
574 auto store = cir::StoreOp::create(builder, loc, src, srcSlot);
575 createdOps.insert(store);
576
577 // Return a destination-typed view of the slot.
578 return emitViewAtOffset(builder, loc, dstTy, alloca, createdOps,
579 dstTy == slotTy ? 0 : offset);
580}
581
582/// Cast \p src to \p dstTy when both are pointers to the same pointee, so only
583/// the address space changes. Returns null when a real coercion is needed.
584static mlir::Value
585emitAddrSpaceCoercion(mlir::OpBuilder &builder, mlir::Location loc,
586 mlir::Type dstTy, mlir::Value src,
587 SmallPtrSetImpl<mlir::Operation *> &createdOps,
588 unsigned offset) {
589 auto srcPtrTy = dyn_cast<cir::PointerType>(src.getType());
590 auto dstPtrTy = dyn_cast<cir::PointerType>(dstTy);
591 if (offset || !srcPtrTy || !dstPtrTy ||
592 srcPtrTy.getPointee() != dstPtrTy.getPointee())
593 return {};
594 assert(srcPtrTy.getAddrSpace() != dstPtrTy.getAddrSpace() &&
595 "pointers to the same pointee must differ in address space");
596 auto cast = cir::CastOp::create(builder, loc, dstTy,
597 cir::CastKind::address_space, src);
598 createdOps.insert(cast);
599 return cast;
600}
601
602/// Coerce \p src to type \p dstTy by going through memory and load the whole
603/// coerced value back out. Builds on emitCoercionToMemory, adding the final
604/// load of the destination-typed view. A pointer that only changes address
605/// space is cast instead.
606mlir::Value emitCoercion(mlir::OpBuilder &builder, mlir::Location loc,
607 mlir::Type dstTy, mlir::Value src,
608 mlir::Block *slotBlock, const mlir::DataLayout &dl,
609 SmallPtrSetImpl<mlir::Operation *> &createdOps,
610 unsigned offset) {
611 if (mlir::Value cast =
612 emitAddrSpaceCoercion(builder, loc, dstTy, src, createdOps, offset))
613 return cast;
614
615 mlir::Value dstSlot = emitCoercionToMemory(builder, loc, dstTy, src,
616 slotBlock, dl, createdOps, offset);
617 auto load = cir::LoadOp::create(builder, loc, dstSlot);
618 createdOps.insert(load);
619 return load;
620}
621
622/// Convenience overload for callers that don't need the createdOps set
623/// (e.g. call-site coercion where we don't replaceAllUsesExcept).
624mlir::Value emitCoercion(mlir::OpBuilder &builder, mlir::Location loc,
625 mlir::Type dstTy, mlir::Value src,
626 mlir::Block *slotBlock, const mlir::DataLayout &dl,
627 unsigned offset) {
628 SmallPtrSet<mlir::Operation *, 4> ignored;
629 return emitCoercion(builder, loc, dstTy, src, slotBlock, dl, ignored, offset);
630}
631
632/// The block a coercion slot's alloca belongs at the start of.
633///
634/// Normally the enclosing function's entry block, where HoistAllocas expects
635/// allocas to be. A body carrying a call is not always inside a function
636/// when this pass runs, though, because LoweringPrepare runs after it: a
637/// namespace-scope `T g = makeT();` is still in its cir.global ctor region,
638/// and an OpenACC recipe's init and destroy bodies are in regions the module
639/// owns. Those take the outermost region below the module, which dominates
640/// the whole body and travels with it when the body is outlined.
641mlir::Block *coercionSlotBlock(mlir::Operation *op) {
642 if (auto funcOp = op->getParentOfType<mlir::FunctionOpInterface>())
643 return &funcOp->getRegion(0).front();
644 mlir::Region *region = op->getParentRegion();
645 while (mlir::Region *outer = region->getParentRegion()) {
646 if (mlir::isa<mlir::ModuleOp>(outer->getParentOp()))
647 break;
648 region = outer;
649 }
650 assert(!region->empty() && "coercion slot needs a block to hold the alloca");
651 return &region->front();
652}
653
654/// \p val's defining load, if it is simple, meaning neither volatile nor
655/// atomic. Null otherwise: a non-simple load's access has to survive as
656/// written, and a call result or any other first-class value has no defining
657/// load at all.
658static cir::LoadOp maybeGetSimpleLoad(mlir::Value val) {
659 cir::LoadOp load = val.getDefiningOp<cir::LoadOp>();
660 if (!load || load.getIsVolatile() || load.getMemOrder())
661 return {};
662 return load;
663}
664
665/// \p val's defining load, if it loads a record and is not atomic. Null
666/// otherwise. The load may be volatile, and then its bytes may only be read
667/// through a volatile copy, never in place.
668static cir::LoadOp maybeGetCopyableRecordLoad(mlir::Value val) {
669 cir::LoadOp load = val.getDefiningOp<cir::LoadOp>();
670 if (!load || load.getMemOrder() ||
671 !mlir::isa<cir::RecordType>(load.getType()))
672 return {};
673 return load;
674}
675
676/// Emits an error and returns failure if \p val, an operand of \p user, is an
677/// atomic load of a record. The record's bytes are read through a copy or in
678/// place, neither of which keeps the load atomic.
679static mlir::LogicalResult rejectAtomicRecordLoad(mlir::Operation *user,
680 mlir::Value val) {
681 auto load = val.getDefiningOp<cir::LoadOp>();
682 if (!load || !load.getMemOrder() ||
683 !mlir::isa<cir::RecordType>(load.getType()))
684 return mlir::success();
685 return user->emitOpError()
686 << "atomic load of a record operand not yet implemented in "
687 "CallConvLowering";
688}
689
690/// Whether the storage the record load \p load reads is an alloca that holds
691/// \p coercedTy at \p offset. With no offset, the coerced type's alloc size
692/// has to fit in the record's. At an offset, the coerced value has to end
693/// within the record's alloc size.
694static bool canReadInPlace(cir::LoadOp load, mlir::Type coercedTy,
695 const cir::CIRDataLayout &dl, unsigned offset) {
696 if (!cir::getUnderlyingAlloca(load.getAddr()))
697 return false;
698 uint64_t readEnd =
699 offset == 0 ? dl.getTypeAllocSize(coercedTy).getFixedValue()
700 : offset + dl.getTypeStoreSize(coercedTy).getFixedValue();
701 return readEnd <= dl.getTypeAllocSize(load.getType()).getFixedValue();
702}
703
704/// Read \p coercedTy at \p offset straight out of the storage the record load
705/// \p load reads, which must be an alloca. The read goes just before \p load,
706/// so it reads the bytes \p load reads, and takes the alloca's alignment at
707/// \p offset.
708static mlir::Value emitInPlaceCoercedLoad(mlir::OpBuilder &builder,
709 mlir::Location loc, cir::LoadOp load,
710 mlir::Type coercedTy,
711 unsigned offset) {
712 cir::AllocaOp src = cir::getUnderlyingAlloca(load.getAddr());
713 assert(src && "the record load must read an alloca");
714 mlir::OpBuilder::InsertionGuard guard(builder);
715 builder.setInsertionPoint(load);
716 SmallPtrSet<mlir::Operation *, 4> ignored;
717 mlir::Value view = emitViewAtOffset(builder, loc, coercedTy, load.getAddr(),
718 ignored, offset);
719 auto coerced = cir::LoadOp::create(builder, loc, view);
720 coerced.setAlignment(llvm::MinAlign(src.getAlignment(), offset));
721 return coerced;
722}
723
724/// Copy the bytes of the record \p load reads into a new coercion slot for
725/// \p coercedTy at \p offset, placed at the start of \p slotBlock, and return
726/// the slot. The slot is at least as aligned as the storage copied from. The
727/// copy goes just before \p load, so it reads the bytes \p load reads, and is
728/// volatile if \p load is.
729static cir::AllocaOp
730emitCopiedCoercionSlot(mlir::OpBuilder &builder, mlir::Location loc,
731 cir::LoadOp load, mlir::Type coercedTy,
732 mlir::Block *slotBlock, const mlir::DataLayout &dl,
733 unsigned offset) {
734 auto recTy = cast<cir::RecordType>(load.getType());
735 uint64_t srcAlign;
736 if (cir::AllocaOp src = cir::getUnderlyingAlloca(load.getAddr()))
737 srcAlign = src.getAlignment();
738 else
739 srcAlign = load.getAlignment().value_or(dl.getTypeABIAlignment(recTy));
740 SmallPtrSet<mlir::Operation *, 4> ignored;
741 cir::AllocaOp slot = createCoercionSlot(builder, loc, recTy, coercedTy,
742 slotBlock, dl, ignored, offset);
743 slot.setAlignment(std::max(slot.getAlignment(), srcAlign));
744 assert((offset == 0 || slot.getAllocaType() == recTy) &&
745 "a direct offset must leave the record as the slot's type");
746 mlir::OpBuilder::InsertionGuard guard(builder);
747 builder.setInsertionPoint(load);
748 mlir::Value recView =
749 emitViewAtOffset(builder, loc, recTy, slot, ignored, /*offset=*/0);
750 auto copy =
751 cir::CopyOp::create(builder, loc, recView, load.getAddr(),
752 builder.getI64IntegerAttr(slot.getAlignment()),
753 builder.getI64IntegerAttr(srcAlign));
754 copy.setIsVolatile(load.getIsVolatile());
755 return slot;
756}
757
758/// Copy the bytes of the record \p load reads into a coercion slot, as
759/// emitCopiedCoercionSlot does, then read \p coercedTy out of the slot at
760/// \p offset. The read takes the slot's alignment at \p offset.
761static mlir::Value emitCopiedCoercedLoad(mlir::OpBuilder &builder,
762 mlir::Location loc, cir::LoadOp load,
763 mlir::Type coercedTy,
764 mlir::Block *slotBlock,
765 const mlir::DataLayout &dl,
766 unsigned offset) {
767 cir::AllocaOp slot = emitCopiedCoercionSlot(builder, loc, load, coercedTy,
768 slotBlock, dl, offset);
769 mlir::OpBuilder::InsertionGuard guard(builder);
770 builder.setInsertionPoint(load);
771 SmallPtrSet<mlir::Operation *, 4> ignored;
772 mlir::Value coercedView =
773 emitViewAtOffset(builder, loc, coercedTy, slot, ignored, offset);
774 auto coerced = cir::LoadOp::create(builder, loc, coercedView);
775 coerced.setAlignment(llvm::MinAlign(slot.getAlignment(), offset));
776 return coerced;
777}
778
779/// Read \p coercedTy at \p offset out of the bytes the record load \p load
780/// reads: straight out of its storage if \p load is not volatile and
781/// canReadInPlace accepts it, and otherwise out of a coercion slot at the start
782/// of \p slotBlock that those bytes are copied into. emitInPlaceCoercedLoad
783/// does not mark its read volatile, so a volatile load is read through the
784/// volatile copy instead. Either way no new load of the record type is
785/// emitted.
786static mlir::Value emitCoercedRecordLoad(
787 mlir::OpBuilder &builder, mlir::Location loc, cir::LoadOp load,
788 mlir::Type coercedTy, mlir::Block *slotBlock, const mlir::DataLayout &dl,
789 const cir::CIRDataLayout &cirDataLayout, unsigned offset) {
790 if (!load.getIsVolatile() &&
791 canReadInPlace(load, coercedTy, cirDataLayout, offset))
792 return emitInPlaceCoercedLoad(builder, loc, load, coercedTy, offset);
793 return emitCopiedCoercedLoad(builder, loc, load, coercedTy, slotBlock, dl,
794 offset);
795}
796
797/// Whether a cir.copy of \p ty into \p dst that stands in for a store of it
798/// skips tail padding. Unless \p dst is an alloca, which holds a complete
799/// object, it can be a potentially-overlapping subobject, such as a
800/// [[no_unique_address]] member or a base, whose tail padding holds another
801/// object. Classic CodeGen writes only the data size into a destination it is
802/// handed (EmitCall) and into one that may overlap (EmitAggregateCopy).
803/// Copying the whole object into an alloca is equivalent, since nothing else
804/// lives in its tail padding. A struct has tail padding when its last members
805/// are pad members, which CIRGen emits for padding another object may reuse and
806/// for some other structs, such as an over-aligned one.
807static bool shouldSkipTailPadding(mlir::Value dst, mlir::Type ty) {
808 auto structTy = mlir::dyn_cast<cir::StructType>(ty);
809 if (!structTy || cir::getUnderlyingAlloca(dst))
810 return false;
811 llvm::ArrayRef<mlir::Type> members = structTy.getMembers();
812 llvm::ArrayRef<cir::RecordMemberKind> kinds = structTy.getMemberKinds();
813 for (auto [member, kind] : llvm::reverse(llvm::zip_equal(members, kinds)))
814 if (cir::memberOwnsBytes(member))
815 return kind == cir::RecordMemberKind::Pad;
816 return false;
817}
818
819/// Rewire the uses of the record value \p val, whose bytes sit at \p addr,
820/// which is aligned to \p addrAlign, so that each one finds the record in
821/// memory. \p val may already have been retyped to the type it is passed as, a
822/// coerced type or a pointer, but its uses still expect the record.
823///
824/// A store of \p val becomes a cir.copy from \p addr at the store, volatile if
825/// the store is and skipping tail padding as shouldSkipTailPadding says. A
826/// store of the record value writes only what its lowered type stores, so
827/// another union member's data past those bytes would be lost. A consumer may
828/// be rewritten before \p val's defining op, and has then already stored \p val
829/// into a slot of its own. That store becomes a copy here too.
830///
831/// Every other use is pointed at one cir.load of \p addr created at the
832/// builder's insertion point. Only a call argument, a return and a read of a
833/// member that is not itself a record may use it, since those either read
834/// \p addr when they are rewritten or read a field. Uses by an operation in
835/// \p exceptions are left alone. Fails, after reporting it, on an atomic store
836/// of \p val or a use by anything else, before anything is rewired.
837static mlir::LogicalResult
838rewireRecordUses(mlir::OpBuilder &builder, mlir::Location loc, mlir::Value val,
839 mlir::Value addr, uint64_t addrAlign,
840 const mlir::DataLayout &dl,
841 const SmallPtrSetImpl<mlir::Operation *> &exceptions) {
842 mlir::Type recTy = mlir::cast<cir::PointerType>(addr.getType()).getPointee();
843 assert(mlir::isa<cir::RecordType>(recTy) && "only a record is rewired");
846 for (mlir::OpOperand &use : val.getUses()) {
847 mlir::Operation *user = use.getOwner();
848 if (exceptions.contains(user))
849 continue;
850 if (auto store = mlir::dyn_cast<cir::StoreOp>(user)) {
851 if (store.getMemOrder())
852 return store.emitOpError()
853 << "atomic store of a record argument or result not yet "
854 "implemented in CallConvLowering";
855 stores.push_back(store);
856 continue;
857 }
858 auto member = mlir::dyn_cast<cir::ExtractMemberOp>(user);
859 bool isFieldRead = member && !mlir::isa<cir::RecordType>(member.getType());
860 if (!isFieldRead &&
861 !mlir::isa<cir::CIRCallOpInterface, cir::ReturnOp>(user))
862 return user->emitOpError()
863 << "use of a record argument or result other than by a store, "
864 "call, return or member read not yet implemented in "
865 "CallConvLowering";
866 otherUses.push_back(&use);
867 }
868
869 mlir::IntegerAttr srcAlign = builder.getI64IntegerAttr(addrAlign);
870 for (cir::StoreOp store : stores) {
871 mlir::OpBuilder::InsertionGuard guard(builder);
872 builder.setInsertionPoint(store);
873 uint64_t dstAlign =
874 store.getAlignment().value_or(dl.getTypeABIAlignment(recTy));
875 auto copy =
876 cir::CopyOp::create(builder, store.getLoc(), store.getAddr(), addr,
877 builder.getI64IntegerAttr(dstAlign), srcAlign);
878 copy.setIsVolatile(store.getIsVolatile());
879 copy.setSkipTailPadding(shouldSkipTailPadding(store.getAddr(), recTy));
880 store->erase();
881 }
882
883 if (otherUses.empty())
884 return mlir::success();
885 auto load = cir::LoadOp::create(builder, loc, addr);
886 load.setAlignment(addrAlign);
887 for (mlir::OpOperand *use : otherUses)
888 use->set(load);
889 return mlir::success();
890}
891
892/// Point the uses of \p val outside \p exceptions at the value of \p val's
893/// original type that \p view, a view of the start of a coercion slot, holds.
894/// For a record, rewireRecordUses points the uses at the slot. Any other type
895/// is loaded out of it at the builder's insertion point.
896static mlir::LogicalResult
897replaceUsesFromSlot(mlir::OpBuilder &builder, mlir::Location loc,
898 mlir::Value val, mlir::Value view,
899 const mlir::DataLayout &dl,
900 const SmallPtrSetImpl<mlir::Operation *> &exceptions) {
901 if (!mlir::isa<cir::RecordType>(
902 mlir::cast<cir::PointerType>(view.getType()).getPointee())) {
903 auto load = cir::LoadOp::create(builder, loc, view);
904 val.replaceAllUsesExcept(load, exceptions);
905 return mlir::success();
906 }
907 cir::AllocaOp slot = cir::getUnderlyingAlloca(view);
908 assert(slot && "the view must start at the coercion slot");
909 return rewireRecordUses(builder, loc, val, view, slot.getAlignment(), dl,
910 exceptions);
911}
912
913/// Coerce \p src, which is \p val retyped or the value replacing it, back to
914/// \p origTy through a coercion slot at the start of \p slotBlock, and point
915/// the uses of \p val outside \p createdOps at the result with
916/// replaceUsesFromSlot. A direct \p offset lands on the coerced side, so the
917/// view of \p origTy starts at the slot. A pointer that only changes address
918/// space is cast instead. The coercion's operations are added to
919/// \p createdOps.
920static mlir::LogicalResult
921emitCoercionAndReplaceUses(mlir::OpBuilder &builder, mlir::Location loc,
922 mlir::Type origTy, mlir::Value src, mlir::Value val,
923 mlir::Block *slotBlock, const mlir::DataLayout &dl,
924 SmallPtrSetImpl<mlir::Operation *> &createdOps,
925 unsigned offset) {
926 if (mlir::Value cast = emitAddrSpaceCoercion(builder, loc, origTy, src,
927 createdOps, offset)) {
928 val.replaceAllUsesExcept(cast, createdOps);
929 return mlir::success();
930 }
931 mlir::Value view = emitCoercionToMemory(builder, loc, origTy, src, slotBlock,
932 dl, createdOps, offset);
933 return replaceUsesFromSlot(builder, loc, val, view, dl, createdOps);
934}
935
936/// Coerce the value each cir.return returns to the new (coerced) return type.
937///
938/// When the returned value is a non-atomic load of a record, the coerced value
939/// is read by emitCoercedRecordLoad, and the record load is erased once nothing
940/// else uses it. Any other value is stored into a coercion slot and read back.
941/// A record with no storage behind it, such as a constant, defines no bytes
942/// beyond its fields, so the store writes every byte it defines. A call
943/// result's store is turned into a copy when its call is rewritten later.
944/// Fails, after reporting it, on an atomic load of a record.
945mlir::LogicalResult
946insertReturnCoercion(mlir::FunctionOpInterface funcOp, mlir::Type origRetTy,
947 mlir::Type coercedRetTy, mlir::OpBuilder &builder,
948 const mlir::DataLayout &dl,
949 const cir::CIRDataLayout &cirDataLayout, unsigned offset) {
951 funcOp.walk([&](cir::ReturnOp r) { returns.push_back(r); });
952 for (cir::ReturnOp r : returns) {
953 if (r.getInput().empty())
954 continue;
955 mlir::Value origVal = r.getInput()[0];
956 if (origVal.getType() == coercedRetTy)
957 continue;
958
959 if (failed(rejectAtomicRecordLoad(r, origVal)))
960 return mlir::failure();
961 if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(origVal)) {
962 mlir::Value coerced = emitCoercedRecordLoad(
963 builder, r.getLoc(), recLoad, coercedRetTy,
964 &funcOp->getRegion(0).front(), dl, cirDataLayout, offset);
965 r->setOperand(0, coerced);
966 if (recLoad->use_empty())
967 recLoad->erase();
968 continue;
969 }
970
971 builder.setInsertionPoint(r);
972 mlir::Value coerced =
973 emitCoercion(builder, r.getLoc(), coercedRetTy, origVal,
974 &funcOp->getRegion(0).front(), dl, offset);
975 r->setOperand(0, coerced);
976 }
977 return mlir::success();
978}
979
980/// \p recordVal's defining load, if it is simple and its address resolves to
981/// an alloca. Null otherwise.
982static cir::LoadOp getWholeRecordLoad(mlir::Value recordVal) {
983 cir::LoadOp load = maybeGetSimpleLoad(recordVal);
984 if (!load || !cir::getUnderlyingAlloca(load.getAddr()))
985 return {};
986 return load;
987}
988
989/// Returns the address \p load read from, as a pointer of type \p paramTy, if
990/// it can be passed in place of the loaded value for an indirect argument that
991/// needs \p minAlign alignment, and null otherwise, including when \p load is
992/// null. A byval or byref argument also needs storageUnwrittenBetween. On
993/// null the caller reports a non-byval argument or fills a slot of its own.
994///
995/// The address has to have \p paramTy, the rewritten parameter's type, since a
996/// cir.load pins the pointee type without pinning the address space. An
997/// address_space cast of a \p paramTy pointer qualifies through its source.
998/// The storage it names has to be a slot allocated here, reached through
999/// storage-preserving casts, already stating at least \p minAlign. The
1000/// enclosing function's own non-byval parameter qualifies as well, since its
1001/// slot stands until finalizeParameterSlots and prepareNonByvalParameters has
1002/// restated it with the alignment that parameter promises rather than the one
1003/// CIRGen chose for a local copy. A byval or byref parameter's slot keeps
1004/// CIRGen's alignment, so it qualifies only where that already covers
1005/// \p minAlign.
1006///
1007/// Raising a slot's alignment here would not hold for one that stands in
1008/// for a parameter, since finalizeParameterSlots replaces it with the
1009/// incoming pointer, which promises only what the caller gave it.
1010static mlir::Value forwardableIndirectAddr(cir::LoadOp load,
1011 cir::PointerType paramTy,
1012 uint64_t minAlign) {
1013 if (!load)
1014 return {};
1015 mlir::Value addr = load.getAddr();
1016 // CIRGen reads an alloca outside the default address space through a cast.
1017 if (auto view = addr.getDefiningOp<cir::CastOp>();
1018 view && view.getKind() == cir::CastKind::address_space &&
1019 view.getSrc().getType() == paramTy)
1020 addr = view.getSrc();
1021 if (addr.getType() != paramTy)
1022 return {};
1023 cir::AllocaOp slot = cir::getUnderlyingAlloca(addr);
1024 return slot && slot.getAlignment() >= minAlign ? addr : mlir::Value();
1025}
1026
1027/// True when \p aliasAnalysis shows that nothing between \p load and \p call
1028/// can write the storage \p load read, so a byval argument may name that
1029/// storage rather than a copy of it. byval takes the callee's copy at the
1030/// call, while the operand carries the value as of the load, so a write in
1031/// between would reach the callee that the operand does not hold.
1032/// prepareNonByvalParameters reads a parameter at its spill, which leaves the
1033/// body's own stores in between.
1034///
1035/// A load in another block is not forwardable here, since the paths from it
1036/// to the call are not walked.
1037static bool storageUnwrittenBetween(cir::LoadOp load, mlir::Operation *call,
1038 mlir::AliasAnalysis &aliasAnalysis) {
1039 if (load->getBlock() != call->getBlock())
1040 return false;
1041 for (mlir::Operation *op = load->getNextNode(); op != call;
1042 op = op->getNextNode()) {
1043 assert(op && "the load must precede the call in the block they share");
1044 if (op->getNumRegions() != 0 ||
1045 aliasAnalysis.getModRef(op, load.getAddr()).isMod())
1046 return false;
1047 }
1048 return true;
1049}
1050
1051/// The alignment a copy may claim for the storage \p load read, which is the
1052/// one \p load states, or the underlying slot's when it states none. A load
1053/// with neither names storage this pass cannot reason about, so the copy is
1054/// left to assume nothing.
1055static mlir::IntegerAttr loadSourceAlignment(cir::LoadOp load,
1056 mlir::OpBuilder &builder) {
1057 if (mlir::IntegerAttr stated = load.getAlignmentAttr())
1058 return stated;
1059 if (cir::AllocaOp slot = cir::getUnderlyingAlloca(load.getAddr()))
1060 return builder.getI64IntegerAttr(slot.getAlignment());
1061 return builder.getI64IntegerAttr(1);
1062}
1063
1064/// Decompose a struct value into one scalar call argument per field of \p
1065/// recTy, appending the field values to \p newArgs. When \p structVal is a
1066/// simple load from an alloca, read each field with cir.get_member +
1067/// cir.load from the address the load used, emitted at the original load's
1068/// position so they observe the same memory state, and record the now-dead
1069/// whole-struct load in \p deadRecordLoads for later erasure. Otherwise (a
1070/// call result, compound literal, or a volatile or atomic load) extract each
1071/// field from the value with cir.extract_member. Loading the members from
1072/// memory rather than extracting from a whole-struct value keeps the result in
1073/// a form SROA can promote (it does not reason about extractvalue). Shared by
1074/// the Expand and Direct+canFlatten argument paths.
1075static void emitStructFieldArgs(mlir::OpBuilder &builder, mlir::Location loc,
1076 mlir::Value structVal, cir::RecordType recTy,
1077 SmallVectorImpl<mlir::Value> &newArgs,
1078 SmallVectorImpl<cir::LoadOp> &deadRecordLoads) {
1079 cir::LoadOp srcLoad = getWholeRecordLoad(structVal);
1080
1081 if (srcLoad) {
1082 mlir::OpBuilder::InsertionGuard guard(builder);
1083 builder.setInsertionPoint(srcLoad);
1084 cir::PointerType baseTy = srcLoad.getAddr().getType();
1085 for (auto [f, fieldTy] : llvm::enumerate(recTy.getMembers())) {
1086 mlir::Type fieldPtrTy =
1087 cir::PointerType::get(fieldTy, baseTy.getAddrSpace());
1088 mlir::Value fieldPtr = cir::GetMemberOp::create(
1089 builder, loc, fieldPtrTy, srcLoad.getAddr(), /*name=*/"",
1090 /*index=*/f);
1091 newArgs.push_back(cir::LoadOp::create(builder, loc, fieldPtr));
1092 }
1093 deadRecordLoads.push_back(srcLoad);
1094 } else {
1095 for (unsigned f = 0; f < recTy.getNumElements(); ++f)
1096 newArgs.push_back(
1097 cir::ExtractMemberOp::create(builder, loc, structVal, f));
1098 }
1099}
1100
1101/// Erase the loads that a rewritten call left unused. The old call must
1102/// already be erased, since until then it still counts as a user. One load can
1103/// feed two operands of the same call, as in f(s, s), so \p loads can hold the
1104/// same load twice. A load that another op still reads is left alone.
1105static void eraseDeadRecordLoads(ArrayRef<cir::LoadOp> loads) {
1106 llvm::SmallSetVector<mlir::Operation *, 4> uniqueLoads(llvm::from_range,
1107 loads);
1108 for (mlir::Operation *load : uniqueLoads)
1109 if (load->use_empty())
1110 load->erase();
1111}
1112
1113/// The store that spills indirect parameter \p blockArg, if the parameter's
1114/// only use is that store. Null otherwise.
1115static cir::StoreOp maybeFindParamSpill(mlir::BlockArgument blockArg) {
1116 if (!blockArg.hasOneUse())
1117 return {};
1118 auto store = dyn_cast<cir::StoreOp>(*blockArg.user_begin());
1119 if (!store || store.getValue() != blockArg)
1120 return {};
1121 return store;
1122}
1123
1124/// The alloca a parameter spill writes to through \p addr, either directly or
1125/// through an address_space cast of it. Null otherwise.
1126static cir::AllocaOp getSpillSlot(mlir::Value addr) {
1127 if (auto view = addr.getDefiningOp<cir::CastOp>();
1128 view && view.getKind() == cir::CastKind::address_space)
1129 addr = view.getSrc();
1130 return addr.getDefiningOp<cir::AllocaOp>();
1131}
1132
1133/// The store that spills non-byval indirect parameter \p blockArg, and the
1134/// slot it spills into. prepareNonByvalParameters has already established
1135/// that the spill is the block argument's only use and that it stores into
1136/// an alloca, which is what the assertions below rest on. Both results are
1137/// null when the parameter has no spill, which is so only when nothing uses
1138/// it at all.
1139static std::pair<cir::StoreOp, cir::AllocaOp>
1140findParamSpill(mlir::BlockArgument blockArg) {
1141 if (blockArg.use_empty())
1142 return {};
1143 cir::StoreOp store = maybeFindParamSpill(blockArg);
1144 assert(store && "non-byval arg's only use must be its CIRGen param spill");
1145 cir::AllocaOp slot = getSpillSlot(store.getAddr());
1146 assert(slot && "non-byval arg's spill must store to an alloca");
1147 return {store, slot};
1148}
1149
1150/// For each Direct arg with a coerced type, change the block argument's type to
1151/// the coerced type and insert a coercion at function entry that maps it back
1152/// to the original type for body uses, which find a record in the coercion slot
1153/// through rewireRecordUses. For each Indirect byval arg and each
1154/// IndirectAliased arg, change the block argument's type to a pointer, then
1155/// replace a record's single param spill with a copy from that pointer.
1156/// Without such a spill, a record's uses find it at the pointer through
1157/// rewireRecordUses, and a value of any other type is loaded from the pointer
1158/// at entry. For each Indirect non-byval arg, change the block argument to a
1159/// pointer and queue the param-slot alloca to be replaced by it (no entry load
1160/// or byte-copy) so the body operates on the caller's storage in place. For
1161/// each Expand arg, replace the single struct block argument with N scalar
1162/// block arguments (one per field) and store each field directly into the
1163/// parameter's own alloca (the CIRGen spill slot), erasing the original
1164/// whole-struct store.
1165///
1166/// \p hasSRetArg is true when the function has an sret return (a hidden return
1167/// pointer is prepended as block argument 0). Expand arguments expand the
1168/// block argument count, so a running index tracks the current block argument
1169/// position rather than computing the classification index + \p hasSRetArg
1170/// directly. Fails, after reporting it, when rewireRecordUses does.
1171mlir::LogicalResult
1172insertArgCoercion(mlir::FunctionOpInterface funcOp,
1173 const FunctionClassification &fc, mlir::OpBuilder &builder,
1174 const mlir::DataLayout &dl, bool hasSRetArg,
1175 SmallVectorImpl<std::pair<cir::AllocaOp, mlir::BlockArgument>>
1176 &pendingParamSlots) {
1177 mlir::Region &body = funcOp->getRegion(0);
1178 if (body.empty())
1179 return mlir::success();
1180 mlir::Block &entry = body.front();
1181
1182 // Running block argument index. Each non-Expand classification occupies
1183 // one block argument slot; each Expand classification occupies N slots
1184 // (one per struct field), so the running index must be incremented by N
1185 // rather than 1 after processing an Expand arg.
1186 unsigned blockArgIdx = hasSRetArg ? 1 : 0;
1187
1188 for (const ArgClassification &ac : fc.argInfos) {
1189 assert(blockArgIdx < entry.getNumArguments() &&
1190 "classification count must not exceed entry block arguments");
1191
1192 if (ac.kind == ArgKind::Expand) {
1193 // The block arg at blockArgIdx currently has the original struct type.
1194 // Replace it with N scalar args (one per field) and store each field
1195 // directly into the parameter's own alloca.
1196 mlir::BlockArgument origArg = entry.getArgument(blockArgIdx);
1197 auto recTy = cast<cir::RecordType>(origArg.getType());
1198 assert(recTy.isStruct() &&
1199 "Expand classification requires a struct type, not a union");
1200 unsigned numFields = recTy.getNumElements();
1201 assert(numFields > 0 &&
1202 "Expand classification requires at least one struct field");
1203 mlir::Location loc = funcOp.getLoc();
1204
1205 // CIRGen spills every by-value struct parameter into its local alloca
1206 // with a single store before any other use, so the struct block arg's
1207 // only use is that spill. Capture it and the destination alloca so the
1208 // expanded fields can be stored straight into that alloca, preserving
1209 // the alloca's variable name and `init` flag and avoiding a
1210 // reassemble-then-reload roundtrip. DCE may have run earlier and
1211 // removed the spill (leaving the block arg unused); tolerate that by
1212 // only flattening the signature and emitting no field stores.
1213 cir::StoreOp paramStore;
1214 cir::AllocaOp destAlloca;
1215 if (!origArg.use_empty()) {
1216 assert(origArg.hasOneUse() &&
1217 "Expand arg must have exactly one use (the CIRGen param spill)");
1218 paramStore = cast<cir::StoreOp>(*origArg.user_begin());
1219 assert(paramStore.getValue() == origArg &&
1220 "Expand arg's use must be the value operand of its store");
1221 destAlloca = cast<cir::AllocaOp>(paramStore.getAddr().getDefiningOp());
1222 }
1223
1224 // Erase the original whole-struct spill before retyping the block
1225 // argument, so the store is never left feeding a type-mismatched value.
1226 // The field stores take its place, just before the following operation
1227 // (the spill always precedes the entry block's terminator).
1228 mlir::Operation *fieldStoreInsertPt = nullptr;
1229 if (paramStore) {
1230 fieldStoreInsertPt = paramStore->getNextNode();
1231 assert(fieldStoreInsertPt &&
1232 "param spill must be followed by a block terminator");
1233 paramStore->erase();
1234 }
1235
1236 // Split the single struct block arg into N scalar field block args (slot
1237 // 0 reuses the original; slots 1..N-1 are inserted after it). The
1238 // reshape needs no insertion point. The field stores are gated on the
1239 // same destAlloca condition: when the spill survived we set the insert
1240 // point to its old slot (which sits after the CIRGen allocas) and store
1241 // each field there; when DCE removed the spill the parameter is dead, so
1242 // we only reshape the signature and emit no stores.
1243 if (destAlloca)
1244 builder.setInsertionPoint(fieldStoreInsertPt);
1245 for (auto [f, fieldTy] : llvm::enumerate(recTy.getMembers())) {
1246 if (f == 0)
1247 origArg.setType(fieldTy);
1248 else
1249 entry.insertArgument(blockArgIdx + f, fieldTy, loc);
1250 if (!destAlloca)
1251 continue;
1252 mlir::Type fieldPtrTy = cir::PointerType::get(
1253 fieldTy,
1254 mlir::cast<cir::PointerType>(destAlloca.getType()).getAddrSpace());
1255 auto fieldPtr = cir::GetMemberOp::create(builder, loc, fieldPtrTy,
1256 destAlloca, /*name=*/"",
1257 /*index=*/f);
1258 cir::StoreOp::create(builder, loc, entry.getArgument(blockArgIdx + f),
1259 fieldPtr);
1260 }
1261
1262 blockArgIdx += numFields;
1263 continue;
1264 }
1265
1266 mlir::BlockArgument blockArg = entry.getArgument(blockArgIdx);
1267
1268 if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl)) {
1269 // Direct + canFlatten: the coerced type is a struct whose fields are
1270 // passed as separate arguments. As in the Expand path, the single block
1271 // arg becomes N scalar block args, and each is stored straight into a
1272 // slot sized and aligned for both the coerced struct and the original
1273 // argument type, which the body then reads.
1274 unsigned numFields = flatTy.getNumElements();
1275 Type origTy = blockArg.getType();
1276 Location loc = funcOp.getLoc();
1277
1278 // Change slot 0 to field 0's type; insert slots 1..N-1 after it.
1279 blockArg.setType(flatTy.getElementType(0));
1280 for (unsigned f = 1; f < numFields; ++f)
1281 entry.insertArgument(blockArgIdx + f, flatTy.getElementType(f), loc);
1282
1283 // setInsertionPointToStart: see comment in the Expand arm above.
1284 builder.setInsertionPointToStart(&entry);
1285 assert(!ac.directOffset &&
1286 "the coerced struct starts at slot offset 0 here, so a flattened "
1287 "coercion cannot honor a direct offset");
1288 SmallPtrSet<Operation *, 8> flattenOps;
1289 cir::AllocaOp slot;
1290 if (origTy == flatTy) {
1291 slot = cir::AllocaOp::create(
1292 builder, loc, getAllocaPtrTy(flatTy, dl),
1293 builder.getStringAttr("coerce"),
1294 builder.getI64IntegerAttr(dl.getTypeABIAlignment(flatTy)));
1295 flattenOps.insert(slot);
1296 } else {
1297 slot = createCoercionSlot(builder, loc, flatTy, origTy, &entry, dl,
1298 flattenOps, /*offset=*/0);
1299 }
1300 Value flatView = emitViewAtOffset(builder, loc, flatTy, slot, flattenOps,
1301 /*offset=*/0);
1302 auto flatPtrTy = cast<cir::PointerType>(flatView.getType());
1303 for (auto [f, fieldTy] : llvm::enumerate(flatTy.getMembers())) {
1304 Type fieldPtrTy =
1305 cir::PointerType::get(fieldTy, flatPtrTy.getAddrSpace());
1306 auto fieldPtr = cir::GetMemberOp::create(builder, loc, fieldPtrTy,
1307 flatView, /*name=*/"",
1308 /*index=*/f);
1309 flattenOps.insert(fieldPtr);
1310 auto storeOp = cir::StoreOp::create(
1311 builder, loc, entry.getArgument(blockArgIdx + f), fieldPtr);
1312 // A packed slot can be less aligned than the field's own type.
1313 uint64_t fieldAlign =
1314 llvm::MinAlign(slot.getAlignment(), flatTy.getElementOffset(dl, f));
1315 if (fieldAlign < dl.getTypeABIAlignment(fieldTy))
1316 storeOp.setAlignment(fieldAlign);
1317 flattenOps.insert(storeOp);
1318 }
1319
1320 // Point the original body uses of the struct block arg, now field 0, at
1321 // the original-type view of the slot.
1322 Value origView = emitViewAtOffset(builder, loc, origTy, slot, flattenOps,
1323 /*offset=*/0);
1324 if (failed(replaceUsesFromSlot(builder, loc, blockArg, origView, dl,
1325 flattenOps)))
1326 return mlir::failure();
1327
1328 blockArgIdx += numFields;
1329 continue;
1330 }
1331
1332 if ((ac.kind == ArgKind::Direct && ac.coercedType) ||
1333 isRecordExtend(ac, blockArg.getType())) {
1334 mlir::Type oldArgTy = blockArg.getType();
1335 mlir::Type newArgTy = ac.coercedType;
1336 if (oldArgTy == newArgTy) {
1337 ++blockArgIdx;
1338 continue;
1339 }
1340 blockArg.setType(newArgTy);
1341
1342 // The coercion's own operations are left out of the rewiring, since they
1343 // read blockArg as the coerced type.
1344 builder.setInsertionPointToStart(&entry);
1345 SmallPtrSet<mlir::Operation *, 4> coercionOps;
1346 if (failed(emitCoercionAndReplaceUses(builder, funcOp.getLoc(), oldArgTy,
1347 blockArg, blockArg, &entry, dl,
1348 coercionOps, ac.directOffset)))
1349 return mlir::failure();
1350 } else if (ac.kind == ArgKind::Indirect ||
1351 ac.kind == ArgKind::IndirectAliased) {
1352 // byval and non-byval both lower to a pointer to T in the classified
1353 // address space, and which it is shows up only in the attrs
1354 // updateArgAttrs applies. Body lowering differs: the body addresses a
1355 // byval parameter through the CIRGen slot it was spilled into, so that
1356 // slot has to be filled from the incoming pointer, while a non-byval
1357 // parameter's slot is replaced by the pointer so the body works on the
1358 // caller's storage in place. A byref parameter takes the byval path.
1359 bool byRef = ac.kind == ArgKind::IndirectAliased;
1360 cir::PointerType ptrTy = getIndirectPtrTy(blockArg.getType(), ac);
1361
1362 if (!byRef && !ac.byVal) {
1363 // Without byval, drop the spill store and let the slot's uses read the
1364 // incoming pointer, so the body operates on the caller's storage in
1365 // place. A byte-copy would be wrong for non-trivially-copyable
1366 // aggregates (e.g. libstdc++ SSO std::string, where it would leave
1367 // `_M_p` aliasing the source's `_M_local_buf`).
1368 auto [paramStore, destAlloca] = findParamSpill(blockArg);
1369
1370 if (paramStore)
1371 paramStore->erase();
1372
1373 // Update the block argument to point to its original type.
1374 blockArg.setType(ptrTy);
1375
1376 // Pointing the slot's uses at the incoming pointer waits until every
1377 // call site has been rewritten. A call that hands this parameter
1378 // straight on recognizes it by the slot its operand was loaded from,
1379 // and collapsing the slot here would leave that call reading a block
1380 // argument with no defining operation to inspect. A dead spill DCE
1381 // already removed leaves nothing to collapse.
1382 if (destAlloca)
1383 pendingParamSlots.emplace_back(destAlloca, blockArg);
1384 } else {
1385 // A record's spill slot is filled by a byte copy of the incoming
1386 // object rather than by a store of the loaded value: its padding
1387 // bytes can hold another union member's live data, which a value
1388 // store would leave unwritten. A _BitInt keeps its load and spill
1389 // store, which are its conversion between value and in-memory form.
1390 cir::StoreOp paramStore;
1391 if (mlir::isa<cir::RecordType>(blockArg.getType()))
1392 paramStore = maybeFindParamSpill(blockArg);
1393
1394 // Erasing the spill before the block argument is retyped keeps the
1395 // store from being left with a value operand of the wrong type.
1396 mlir::Value destAddr;
1397 mlir::IntegerAttr destAlign;
1398 mlir::Operation *spillPos = nullptr;
1399 bool spillIsVolatile = false;
1400 if (paramStore) {
1401 if (paramStore.getMemOrder())
1402 return paramStore.emitOpError()
1403 << "atomic store of a record argument or result not yet "
1404 "implemented in CallConvLowering";
1405 spillIsVolatile = paramStore.getIsVolatile();
1406 destAddr = paramStore.getAddr();
1407 cir::AllocaOp destAlloca = cir::getUnderlyingAlloca(destAddr);
1408 destAlign = paramStore.getAlignmentAttr();
1409 if (!destAlign && destAlloca)
1410 destAlign = destAlloca.getAlignmentAttr();
1411 spillPos = paramStore->getNextNode();
1412 assert(spillPos && "param spill must be followed by a block "
1413 "terminator");
1414 paramStore->erase();
1415 }
1416
1417 blockArg.setType(ptrTy);
1418
1419 if (destAddr) {
1420 mlir::OpBuilder::InsertionGuard guard(builder);
1421 builder.setInsertionPoint(spillPos);
1422 auto copy = cir::CopyOp::create(
1423 builder, funcOp.getLoc(), destAddr, blockArg, destAlign,
1424 builder.getI64IntegerAttr(ac.indirectAlign.value()));
1425 copy.setIsVolatile(spillIsVolatile);
1426 copy.setSkipTailPadding(
1427 shouldSkipTailPadding(destAddr, ptrTy.getPointee()));
1428 } else if (!blockArg.use_empty()) {
1429 builder.setInsertionPointToStart(&entry);
1430 if (mlir::isa<cir::RecordType>(ptrTy.getPointee())) {
1431 SmallPtrSet<mlir::Operation *, 1> noExceptions;
1432 if (failed(rewireRecordUses(builder, funcOp.getLoc(), blockArg,
1433 blockArg, ac.indirectAlign.value(), dl,
1434 noExceptions)))
1435 return mlir::failure();
1436 } else {
1437 auto loadOp =
1438 cir::LoadOp::create(builder, funcOp.getLoc(), blockArg);
1439 SmallPtrSet<mlir::Operation *, 1> loadOps = {loadOp};
1440 blockArg.replaceAllUsesExcept(loadOp.getResult(), loadOps);
1441 }
1442 }
1443 }
1444 }
1445 // Ignore, Extend, and Direct-without-coerce need no block-level changes.
1446
1447 ++blockArgIdx;
1448 }
1449 return mlir::success();
1450}
1451
1452/// Replace each address_space cast of \p slot to the type of \p ptr with
1453/// \p ptr.
1454void replaceAddrSpaceViews(cir::AllocaOp slot, mlir::Value ptr) {
1455 for (mlir::Operation *user : llvm::make_early_inc_range(slot->getUsers())) {
1456 auto view = mlir::dyn_cast<cir::CastOp>(user);
1457 if (view && view.getKind() == cir::CastKind::address_space &&
1458 view.getType() == ptr.getType()) {
1459 view.getResult().replaceAllUsesWith(ptr);
1460 view->erase();
1461 }
1462 }
1463}
1464
1465/// Rewrite each cir.return so the return value flows through the sret
1466/// pointer (the prepended first block argument) and the function returns
1467/// void.
1468///
1469/// CIRGen emits a local `__retval` alloca and emits `cir.return %loaded`
1470/// where `%loaded = cir.load __retval`. The naive lowering -- store the
1471/// loaded SSA value through the sret pointer -- byte-copies the record,
1472/// which is wrong for non-trivially-copyable types: e.g. libstdc++'s SSO
1473/// `std::string` has a `_M_p` pointer that aliases the source's internal
1474/// `_M_local_buf`, so a byte-copy leaves the destination pointing at the
1475/// source's (now-dying) stack storage and the destination's destructor
1476/// later `free()`s a stack pointer.
1477///
1478/// Instead, route construction directly into the sret slot: find the
1479/// `__retval` alloca, replace its uses with the sret pointer, and drop the
1480/// trailing `cir.load __retval` so the rewritten return has no operand.
1481/// The CIRGen-emitted constructor / store-into-`__retval` then targets the
1482/// sret slot uniformly, matching classic CodeGen's "construct directly into
1483/// `%agg.result`" pattern.
1484///
1485/// CIRGen emits one `%v = cir.load %__retval` / `cir.return %v` pair per
1486/// return statement, and every such load reads the single `__retval`
1487/// alloca (CIR does not merge returns into a shared epilogue block). The
1488/// alloca is therefore rewired to the sret pointer once; each cir.return is
1489/// then collapsed to a bare return and its now-dead load erased. This
1490/// `cir.return (cir.load <alloca>)` shape is an invariant guaranteed by
1491/// CIRGen, so it is asserted via `cast<>` rather than guarded with a
1492/// fallback.
1493///
1494/// A `__retval` outside the sret pointer's address space is not replaced
1495/// itself. Its address_space casts to the sret pointer's type are.
1496void insertSRetStores(mlir::FunctionOpInterface funcOp, mlir::Type origRetTy,
1497 mlir::OpBuilder &builder) {
1498 mlir::Value sretPtr = funcOp.getArguments()[0];
1499
1501 funcOp->walk([&](cir::ReturnOp retOp) { returnOps.push_back(retOp); });
1502
1503 cir::AllocaOp retAlloca = nullptr;
1504 for (cir::ReturnOp retOp : returnOps) {
1505 // Every cir.return in an sret function must carry the loaded return
1506 // value -- a bare return would mean the sret slot was never written.
1507 assert(!retOp.getInput().empty() &&
1508 "cir.return in sret function must have an operand");
1509
1510 cir::LoadOp retLoad =
1511 mlir::cast<cir::LoadOp>(retOp.getInput()[0].getDefiningOp());
1512
1513 // Rewire the shared `__retval` alloca to the sret pointer once.
1514 // replaceAllUsesWith updates every load of the alloca (including those
1515 // feeding the other cir.return ops) to read from sretPtr instead, so
1516 // all returns are covered by this single rewiring.
1517 if (!retAlloca) {
1518 retAlloca = mlir::cast<cir::AllocaOp>(retLoad.getAddr().getDefiningOp());
1519 if (retAlloca.getType() == sretPtr.getType())
1520 retAlloca.getResult().replaceAllUsesWith(sretPtr);
1521 else
1522 replaceAddrSpaceViews(retAlloca, sretPtr);
1523 }
1524
1525 // The sret slot now holds the return value directly; replace the
1526 // value-carrying return with a void return (no operand).
1527 builder.setInsertionPoint(retOp);
1528 cir::ReturnOp::create(builder, retOp.getLoc());
1529 retOp->erase();
1530 if (retLoad.use_empty())
1531 retLoad->erase();
1532 }
1533
1534 if (retAlloca) {
1535 assert(retAlloca->use_empty() &&
1536 "__retval used other than by its returns and address_space views");
1537 retAlloca->erase();
1538 }
1539}
1540
1541/// Build the attribute dictionary for the sret slot (slot 0 of an
1542/// sret-returning function or call). Matches classic CodeGen's
1543/// `sret(T) align A [noalias] writable dead_on_unwind`. noalias is only
1544/// valid on the callee's parameter, not at the call site, so it is gated by
1545/// \p withNoalias. Key order is irrelevant: DictionaryAttr sorts by name.
1546SmallVector<mlir::NamedAttribute> buildSretSlotAttrs(mlir::OpBuilder &builder,
1547 mlir::Type retTy,
1548 uint64_t align,
1549 bool withNoalias) {
1551 // The sret type must be carried explicitly: LLVM's sret attribute requires
1552 // it, and once the CIR `!cir.ptr<retTy>` lowers to an opaque LLVM `ptr` the
1553 // pointee type can no longer be recovered from the pointer.
1554 attrs.push_back(
1555 builder.getNamedAttr("llvm.sret", mlir::TypeAttr::get(retTy)));
1556 attrs.push_back(
1557 builder.getNamedAttr("llvm.align", builder.getI64IntegerAttr(align)));
1558 if (withNoalias)
1559 attrs.push_back(
1560 builder.getNamedAttr("llvm.noalias", builder.getUnitAttr()));
1561 attrs.push_back(builder.getNamedAttr("llvm.writable", builder.getUnitAttr()));
1562 attrs.push_back(
1563 builder.getNamedAttr("llvm.dead_on_unwind", builder.getUnitAttr()));
1564 return attrs;
1565}
1566
1567/// Prepend the sret slot's attrs at position 0 of newCall's arg_attrs.
1568/// Called after the call has been rewritten with the sret pointer at
1569/// operand 0, so the operand count now includes the sret slot. \p argAttrs
1570/// must already be shaped for the rewritten argument list (Extend slots
1571/// carry signext/zeroext, Ignore slots dropped); it is shifted to slots
1572/// 1..N behind the sret slot.
1573void applySretSlotAttrs(cir::CallOp newCall, mlir::ArrayAttr argAttrs,
1574 mlir::Type retTy, uint64_t align,
1575 mlir::OpBuilder &builder) {
1576 mlir::MLIRContext *ctx = newCall->getContext();
1578 buildSretSlotAttrs(builder, retTy, align, /*withNoalias=*/false);
1579
1580 SmallVector<mlir::Attribute> newArgAttrs;
1581 newArgAttrs.reserve(newCall.getArgOperands().size());
1582 newArgAttrs.push_back(mlir::DictionaryAttr::get(ctx, sretAttrs));
1583 if (argAttrs)
1584 llvm::append_range(newArgAttrs, argAttrs);
1585 assert(newArgAttrs.size() <= newCall.getArgOperands().size() &&
1586 "arg_attrs wider than the rewritten call's operand list");
1587 newArgAttrs.resize(newCall.getArgOperands().size(),
1588 mlir::DictionaryAttr::get(ctx));
1589 newCall->setAttr("arg_attrs", mlir::ArrayAttr::get(ctx, newArgAttrs));
1590}
1591
1592/// Copy the call attributes from \p source to the rebuilt \p target. The
1593/// callee is already established by CallOp::create and may have been rewritten,
1594/// so it is not copied. Existing discardable attributes on \p target take
1595/// precedence over attributes from \p source.
1596static void copyCallAttributes(cir::CallOp source, cir::CallOp target) {
1597 source->getName().walkInherentAttrs(source, [&](llvm::StringRef name,
1598 mlir::Attribute &attr) {
1599 if (name != cir::CIRDialect::getCalleeAttrName())
1600 target->setInherentAttr(mlir::StringAttr::get(source->getContext(), name),
1601 attr);
1602 });
1603 for (mlir::NamedAttribute attr : source->getDiscardableAttrs())
1604 if (!target->hasDiscardableAttr(attr.getName()))
1605 target->setDiscardableAttr(attr.getName(), attr.getValue());
1606}
1607
1608/// For an indirect call, prepend the callee function pointer as operand 0 so
1609/// CallOp::create rebuilds it as an indirect call, bitcasting it to a function
1610/// pointer that takes \p args, less the last \p numEllipsisArgs, and returns
1611/// \p retTy. Those last arguments were passed through the callee's ellipsis
1612/// and stay out of the function type, as in classic CodeGen's
1613/// GetFunctionType: LLVM treats operands past a call's function type as
1614/// variadic, and some backends pass variadic arguments differently. No-op
1615/// for direct calls.
1616static void prependIndirectCallee(cir::CallOp call,
1617 SmallVectorImpl<mlir::Value> &args,
1618 mlir::Type retTy, unsigned numEllipsisArgs,
1619 mlir::OpBuilder &builder) {
1620 if (!call.isIndirect())
1621 return;
1622 assert(numEllipsisArgs <= args.size() &&
1623 "more ellipsis arguments than arguments");
1624 mlir::Value calleePtr = call.getIndirectCall();
1625 SmallVector<mlir::Type> paramTypes;
1626 paramTypes.reserve(args.size() - numEllipsisArgs);
1627 llvm::transform(ArrayRef(args).drop_back(numEllipsisArgs),
1628 std::back_inserter(paramTypes),
1629 [](mlir::Value v) { return v.getType(); });
1630 // Lowering builds an indirect call's LLVM function type from the callee
1631 // pointer's pointee and takes the call's result from that type, so the
1632 // pointee's return type has to track the rewrite: an sret return would
1633 // leave a result the call no longer produces, and a coerced return one of
1634 // the wrong type. The ellipsis has to survive for the same reason: the
1635 // rebuilt pointee is what makes the lowered call variadic, and only a
1636 // variadic call gets the vector-register count that the x86_64 SysV ABI
1637 // passes in AL and that the callee's va_arg reads back.
1638 cir::FuncType calleeFnTy = getIndirectCalleeType(call);
1639 assert((calleeFnTy.isVarArg() || numEllipsisArgs == 0) &&
1640 "only a variadic callee takes arguments through an ellipsis");
1641 auto newPtrTy = cir::PointerType::get(
1642 cir::FuncType::get(paramTypes, retTy, calleeFnTy.isVarArg()));
1643 if (calleePtr.getType() != newPtrTy)
1644 calleePtr = cir::CastOp::create(builder, call.getLoc(), newPtrTy,
1645 cir::CastKind::bitcast, calleePtr);
1646 args.insert(args.begin(), calleePtr);
1647}
1648
1649/// Rewrite an indirect-return (sret) call site: prepend a return-slot pointer
1650/// as operand 0, make the call return void, and either reuse a dominating
1651/// single-use store destination as the slot (so construction flows directly
1652/// into it) or allocate a fresh slot and point the result's uses at it, a
1653/// record's through rewireRecordUses and any other type's through a load of the
1654/// slot after the call. \p newArgs is the already-shaped (Ignore-dropped,
1655/// coercion-applied) non-sret argument list, the last \p numEllipsisArgs of
1656/// which were passed through the callee's ellipsis. The caller guarantees the
1657/// call has a result and an indirect-return classification. Fails, after
1658/// reporting it, when rewireRecordUses does.
1659mlir::LogicalResult
1660rewriteIndirectReturnCall(cir::CallOp call, const FunctionClassification &fc,
1661 ArrayRef<mlir::Value> newArgs,
1662 unsigned numEllipsisArgs, mlir::Type origRetTy,
1663 ArrayRef<mlir::Type> origCallArgTypes,
1664 mlir::OpBuilder &builder, const mlir::DataLayout &dl,
1665 bool enableNoundefAttrs) {
1666 mlir::MLIRContext *ctx = call->getContext();
1667 cir::PointerType ptrTy = getIndirectPtrTy(origRetTy, fc.returnInfo);
1668 builder.setInsertionPoint(call);
1669 uint64_t sretAlign = fc.returnInfo.indirectAlign.value();
1670
1671 // CIRGen emits `cir.store %callResult, %dest` when the call's result is
1672 // bound to a local (e.g. `T s = make();`). Allocating a fresh sret slot
1673 // and copying into %dest would byte-copy the record, which is wrong for
1674 // non-trivially-copyable types (the libstdc++ SSO `_M_p` pointer
1675 // survives a byte-copy but ends up pointing at the dying temp's local
1676 // buffer, so the destination's destructor later `free()`s a stack
1677 // pointer). When the result has a single store-into-%dest use, use
1678 // %dest as the sret slot directly so construction flows into it,
1679 // matching classic CodeGen's "pass %s as sret" pattern. %dest must
1680 // dominate the call so the rewritten call (which takes it as operand 0)
1681 // does not use a value before its definition. An address_space cast of a
1682 // slot that already has the sret pointer's type stands for that slot.
1683 mlir::Value sretSlot = nullptr;
1684 cir::StoreOp reuseStore = nullptr;
1685 if (call.getResult().hasOneUse()) {
1686 mlir::Operation *user = *call.getResult().getUsers().begin();
1687 auto store = mlir::dyn_cast<cir::StoreOp>(user);
1688 if (store && store.getValue() == call.getResult()) {
1689 mlir::Value dest = store.getAddr();
1690 if (auto view = dest.getDefiningOp<cir::CastOp>();
1691 view && view.getKind() == cir::CastKind::address_space &&
1692 view.getSrc().getType() == ptrTy)
1693 dest = view.getSrc();
1694 if (dest.getType() == ptrTy &&
1695 mlir::DominanceInfo().properlyDominates(dest, call)) {
1696 sretSlot = dest;
1697 reuseStore = store;
1698 }
1699 }
1700 }
1701 if (!sretSlot) {
1702 cir::PointerType allocaPtrTy = getAllocaPtrTy(origRetTy, dl);
1703 sretSlot = cir::AllocaOp::create(
1704 builder, call.getLoc(), allocaPtrTy,
1705 /*name=*/builder.getStringAttr("sret"),
1706 /*alignment=*/builder.getI64IntegerAttr(sretAlign));
1707 // The sret pointer can be outside the alloca address space.
1708 if (allocaPtrTy != ptrTy)
1709 sretSlot = cir::CastOp::create(builder, call.getLoc(), ptrTy,
1710 cir::CastKind::address_space, sretSlot);
1711 }
1712
1713 SmallVector<mlir::Value> sretArgs;
1714 sretArgs.push_back(sretSlot);
1715 sretArgs.append(newArgs.begin(), newArgs.end());
1716
1717 // numEllipsisArgs counts from the end, so the sret slot prepended here stays
1718 // in the retyped signature, as it does in classic CodeGen.
1719 mlir::Type sretVoidTy = cir::VoidType::get(ctx);
1720 prependIndirectCallee(call, sretArgs, sretVoidTy, numEllipsisArgs, builder);
1721 auto newCall = cir::CallOp::create(
1722 builder, call.getLoc(), call.getCalleeAttr(), sretVoidTy, sretArgs);
1723 copyCallAttributes(call, newCall);
1724 newCall->removeAttr("res_attrs");
1725
1726 // Shape the per-argument attrs exactly as the non-sret path does, through
1727 // updateArgAttrs, before prepending the sret slot, so sret composes with
1728 // every argument classification.
1729 mlir::ArrayAttr argAttrs = call->getAttrOfType<mlir::ArrayAttr>("arg_attrs");
1730 if (needsArgAttrUpdate(fc))
1731 argAttrs =
1732 updateArgAttrs(ctx, call->getParentOfType<mlir::ModuleOp>(),
1733 origCallArgTypes, argAttrs, fc, dl, enableNoundefAttrs);
1734 applySretSlotAttrs(newCall, argAttrs, origRetTy, sretAlign, builder);
1735
1736 if (reuseStore) {
1737 // The callee now constructs directly into the destination slot, so the
1738 // original store-from-result is redundant; dropping it avoids a
1739 // byte-copy of the record.
1740 reuseStore->erase();
1741 } else if (mlir::isa<cir::RecordType>(origRetTy)) {
1742 builder.setInsertionPointAfter(newCall);
1743 SmallPtrSet<mlir::Operation *, 1> noExceptions;
1744 if (failed(rewireRecordUses(builder, call.getLoc(), call.getResult(),
1745 sretSlot, sretAlign, dl, noExceptions)))
1746 return mlir::failure();
1747 } else {
1748 builder.setInsertionPointAfter(newCall);
1749 auto load = cir::LoadOp::create(builder, call.getLoc(), origRetTy, sretSlot,
1750 /*isDeref=*/mlir::UnitAttr(),
1751 /*isVolatile=*/mlir::UnitAttr(),
1752 /*is_nontemporal=*/mlir::UnitAttr(),
1753 /*alignment=*/mlir::IntegerAttr(),
1754 /*sync_scope=*/cir::SyncScopeKindAttr(),
1755 /*mem_order=*/cir::MemOrderAttr(),
1756 /*invariant=*/mlir::UnitAttr());
1757 call.getResult().replaceAllUsesWith(load);
1758 }
1759 call->erase();
1760 return mlir::success();
1761}
1762
1763/// Whether \p ty, a type the classifier named for one register of a
1764/// coercion, is carried in a vector register.
1765bool isSSERegisterClass(mlir::Type ty) {
1766 return mlir::isa<cir::VectorType, cir::FPTypeInterface>(ty);
1767}
1768
1769} // namespace
1770
1771cir::FuncType cir::getIndirectCalleeType(cir::CIRCallOpInterface call) {
1772 assert(call.isIndirect() && "expected an indirect call");
1773 return cast<cir::FuncType>(
1774 cast<cir::PointerType>(call.getIndirectCall().getType()).getPointee());
1775}
1776
1777/// Bring \p funcOp's non-byval indirect parameter \p argNo into the shape the
1778/// rest of the rewrite assumes. \p claimedSlots carries the slots \p funcOp's
1779/// earlier non-byval indirect parameters took. The shape itself, and the
1780/// cases reported rather than repaired, are documented on
1781/// CIRABIRewriteContext::prepareNonByvalParameters in the header.
1782static mlir::LogicalResult prepareOneNonByvalParameter(
1783 cir::FuncOp funcOp, unsigned argNo, const ArgClassification &ac,
1784 mlir::BlockArgument blockArg, mlir::DominanceInfo &dom,
1785 SmallPtrSetImpl<mlir::Operation *> &claimedSlots) {
1786 cir::PointerType paramPtrTy = getIndirectPtrTy(blockArg.getType(), ac);
1787
1788 // The spill is the store that writes the parameter itself. Any other use
1789 // consumes the record value. At -O1 and above such a use comes from
1790 // cir-simplify: CIRGen marks a const-qualified parameter's slot const, so
1791 // the load of it folds to the stored parameter.
1792 cir::StoreOp spill;
1793 cir::StoreOp extraSpill;
1794 mlir::Operation *otherUse = nullptr;
1796 for (mlir::OpOperand &use : blockArg.getUses()) {
1797 auto store = dyn_cast<cir::StoreOp>(use.getOwner());
1798 if (store && store.getValue() == blockArg) {
1799 // Which of two stores is the spill decides which slot stands in for the
1800 // parameter, and the use list is in no particular order, so there is
1801 // nothing to prefer between them.
1802 if (spill)
1803 extraSpill = store;
1804 else
1805 spill = store;
1806 continue;
1807 }
1808 // Only an argument is served by a load of the slot. Any other consumer
1809 // belongs to a rewrite that reads the parameter its own way: a returned
1810 // record, for one, is rewritten through the sret slot, which assumes the
1811 // returned load names the return slot and not this one.
1812 auto call = dyn_cast<cir::CIRCallOpInterface>(use.getOwner());
1813 if (call && llvm::is_contained(call.getArgOperands(), blockArg))
1814 callArgs.push_back(&use);
1815 else if (!otherUse)
1816 otherUse = use.getOwner();
1817 }
1818
1819 if (extraSpill)
1820 return extraSpill->emitOpError()
1821 << "non-byval parameter " << argNo
1822 << " spilled more than once is not yet implemented in "
1823 "CallConvLowering";
1824
1825 if (otherUse)
1826 return otherUse->emitOpError()
1827 << "non-byval parameter " << argNo
1828 << " consumed other than as a call argument is not yet implemented "
1829 "in CallConvLowering";
1830
1831 if (!spill) {
1832 // Every other kind of use was reported above, so the parameter has no
1833 // uses at all and needs neither a slot nor a read.
1834 if (callArgs.empty())
1835 return mlir::success();
1836
1837 // Without a spill the parameter becomes the incoming pointer directly,
1838 // which is the storage an argument taken from it has to name. Giving it
1839 // the spill it lacks lets the checks and the read below apply unchanged,
1840 // and neither survives the pass: insertArgCoercion erases the store and
1841 // finalizeParameterSlots replaces the slot.
1842 mlir::OpBuilder builder(funcOp.getContext());
1843 builder.setInsertionPointToStart(blockArg.getOwner());
1844 auto synthesized = cir::AllocaOp::create(
1845 builder, funcOp.getLoc(), paramPtrTy,
1846 builder.getStringAttr("nonbyval.param"),
1847 builder.getI64IntegerAttr(ac.indirectAlign.value()));
1848 spill =
1849 cir::StoreOp::create(builder, funcOp.getLoc(), blockArg, synthesized);
1850 }
1851
1852 // The incoming pointer replaces the slot itself, which also covers a spill
1853 // through an address_space cast of it. Retargeting only the cast would
1854 // leave the allocation's other views reading storage nothing writes.
1855 // Storage in another address space than the incoming pointer, or storage
1856 // that is not a local alloca at all, is not something the incoming pointer
1857 // can be substituted for.
1858 cir::AllocaOp slot = getSpillSlot(spill.getAddr());
1859 if (!slot || slot.getType() != paramPtrTy)
1860 return spill->emitOpError()
1861 << "non-byval parameter " << argNo
1862 << " spilled to storage that cannot take the incoming pointer is "
1863 "not yet implemented in CallConvLowering";
1864
1865 // One slot stands in for one non-byval parameter, since the incoming
1866 // pointer replaces it. Two of them spilled to the same slot are each
1867 // spilled once, so the check above cannot see the collision and only
1868 // comparing the slots can.
1869 if (!claimedSlots.insert(slot).second)
1870 return spill->emitOpError()
1871 << "non-byval parameter " << argNo
1872 << " sharing its spill slot with another parameter is not yet "
1873 "implemented in CallConvLowering";
1874
1875 // CIRGen picked the slot's alignment for a local copy of the record, but the
1876 // slot is about to stand in for the parameter, and a call forwarding it may
1877 // only promise what the incoming pointer does.
1878 slot.setAlignment(ac.indirectAlign.value());
1879
1880 if (callArgs.empty())
1881 return mlir::success();
1882
1883 // A reader the spill does not dominate could read the slot before anything
1884 // wrote it, and a load placed at the spill would not dominate it either.
1885 for (mlir::OpOperand *callArg : callArgs)
1886 if (!dom.properlyDominates(spill, callArg->getOwner()))
1887 return callArg->getOwner()->emitOpError()
1888 << "non-byval parameter " << argNo
1889 << " read before its spill is not yet implemented in "
1890 "CallConvLowering";
1891
1892 // Read the record back out of the slot instead, so every reader sees the
1893 // shape CIRGen emits without cir-simplify: a load naming the storage the
1894 // argument would have to name anyway. Reading at the spill, not at the
1895 // reader, keeps the value the parameter's own whatever the body later
1896 // stores into the slot, and the load promises only the alignment the
1897 // classification gives the incoming pointer.
1898 //
1899 // A call that forwards the argument consumes the load once it recognizes
1900 // the slot. One that wants a copy keeps it, reading the incoming pointer
1901 // once finalizeParameterSlots replaces the slot.
1902 mlir::OpBuilder builder(spill);
1903 builder.setInsertionPointAfter(spill);
1904 auto reload = cir::LoadOp::create(builder, spill.getLoc(), slot);
1905 reload.setAlignment(ac.indirectAlign.value());
1906 for (mlir::OpOperand *callArg : callArgs)
1907 callArg->set(reload.getResult());
1908 return mlir::success();
1909}
1910
1912 cir::FuncOp funcOp, const FunctionClassification &fc) {
1913 if (!funcOp.isDefinition())
1914 return mlir::success();
1915 mlir::Region &body = funcOp->getRegion(0);
1916 if (body.empty())
1917 return mlir::success();
1918 mlir::Block &entry = body.front();
1919 mlir::DominanceInfo dom;
1920 SmallPtrSet<mlir::Operation *, 4> claimedSlots;
1921
1922 // No signature has been rewritten yet, so no sret pointer has been prepended
1923 // and no Expand argument has been split into its fields. Every
1924 // classification therefore still maps to the entry block argument at its own
1925 // index.
1926 for (auto [idx, ac] : llvm::enumerate(fc.argInfos)) {
1927 if (ac.kind != ArgKind::Indirect || ac.byVal)
1928 continue;
1929 assert(idx < entry.getNumArguments() &&
1930 "classification count must not exceed entry block arguments");
1931 if (failed(prepareOneNonByvalParameter(
1932 funcOp, idx, ac, entry.getArgument(idx), dom, claimedSlots)))
1933 return mlir::failure();
1934 }
1935 return mlir::success();
1936}
1937
1939 for (auto [slot, incoming] : pendingParamSlots) {
1940 slot.getResult().replaceAllUsesWith(incoming);
1941 slot->erase();
1942 }
1943 pendingParamSlots.clear();
1944}
1945
1947 mlir::FunctionOpInterface funcOpInterface, const FunctionClassification &fc,
1948 mlir::OpBuilder &builder) {
1949 // The pass driver (CallConvLoweringPass) only ever hands us cir.func ops.
1950 // Cast once at the top so the rest of the function reads in CIR's own
1951 // vocabulary, and so we can dispatch to the CIRGlobalValueInterface for
1952 // isDefinition() (FunctionOpInterface alone does not inherit from
1953 // CIRGlobalValueInterface).
1954 cir::FuncOp funcOp = mlir::cast<cir::FuncOp>(funcOpInterface);
1955
1956 if (!fc.needsRewrite())
1957 return mlir::success();
1958
1959 ArrayRef<mlir::Type> oldArgTypes = funcOp.getArgumentTypes();
1960 ArrayRef<mlir::Type> oldResultTypes = funcOp.getResultTypes();
1961 mlir::MLIRContext *ctx = funcOp->getContext();
1962
1963 // CIR follows LLVM IR's single-result rule: a function returns either
1964 // zero or one value. Document the invariant so a future multi-result
1965 // change forces us to revisit the return-handling below.
1966 assert(oldResultTypes.size() <= 1 &&
1967 "CIR functions return zero or one value");
1968
1969 SmallVector<mlir::Type> newArgTypes;
1970 if (mlir::failed(buildNewArgTypes(oldArgTypes, fc, newArgTypes, dl,
1971 [&]() { return funcOp.emitOpError(); })))
1972 return mlir::failure();
1973
1974 mlir::Type voidTy = cir::VoidType::get(ctx);
1975 mlir::Type origRetTy = oldResultTypes.empty() ? voidTy : oldResultTypes[0];
1976 mlir::Type newRetTy = computeNewReturnType(
1977 origRetTy, fc.returnInfo, ctx, [&]() { return funcOp.emitOpError(); });
1978 if (!newRetTy)
1979 return mlir::failure();
1980 SmallVector<mlir::Type> newResultTypes = {newRetTy};
1981
1982 // sret return: the value is returned through a pointer the ABI inserts as
1983 // argument 0. This pointer is not part of the function's source-level
1984 // signature -- it is synthesized here -- and the wire return type was
1985 // already set to void by computeNewReturnType. Every classification index
1986 // therefore maps to a block argument shifted by one in the body handling
1987 // below.
1988 bool hasSRet =
1989 fc.returnInfo.kind == ArgKind::Indirect && !oldResultTypes.empty();
1990 cir::PointerType sretPtrTy;
1991 if (hasSRet) {
1992 sretPtrTy = getIndirectPtrTy(origRetTy, fc.returnInfo);
1993 newArgTypes.insert(newArgTypes.begin(), sretPtrTy);
1994 }
1995
1996 if (funcOp.isDefinition()) {
1997 mlir::Region &body = funcOp->getRegion(0);
1998 if (!body.empty()) {
1999 // Prepend the sret pointer block argument and route every cir.return
2000 // through it before any index-based argument handling below (which
2001 // then accounts for the +1 offset).
2002 if (hasSRet) {
2003 body.front().insertArgument(0u, sretPtrTy, funcOp.getLoc());
2004 insertSRetStores(funcOp, origRetTy, builder);
2005 }
2006
2007 // Reshape the entry block for the classifications that change what a
2008 // parameter looks like to the body: a Direct arg with a coerced type
2009 // gets a memory roundtrip back to its original type, an Indirect arg
2010 // becomes a pointer, and an Expand arg becomes one block argument per
2011 // field. Done before the Ignore-drop below so the entry block
2012 // argument indices used here still refer to the original positions.
2013 if (failed(insertArgCoercion(funcOp, fc, builder, dl, hasSRet,
2014 pendingParamSlots)))
2015 return mlir::failure();
2016
2017 // Direct return with coerced type: insert a coercion at every
2018 // cir.return so the returned value matches the (coerced) return
2019 // type in the new function signature set below.
2020 if (fc.returnInfo.kind == ArgKind::Direct && fc.returnInfo.coercedType &&
2021 !oldResultTypes.empty() && fc.returnInfo.coercedType != origRetTy &&
2022 failed(insertReturnCoercion(
2023 funcOp, origRetTy, fc.returnInfo.coercedType, builder, dl,
2024 cirDataLayout, fc.returnInfo.directOffset)))
2025 return mlir::failure();
2026
2027 mlir::Block &entry = body.front();
2028
2029 // Drop each Ignored argument's block argument, replacing any remaining
2030 // body uses with a poison constant (an Ignore arg is not passed at the
2031 // ABI level, so any use is vacuous; poison says exactly that). Walk
2032 // forward with a running block-argument index that mirrors
2033 // insertArgCoercion: an Expand arg or a Direct+canFlatten arg occupies N
2034 // slots, every other kept kind one. On erase, do not advance the index
2035 // -- the next block argument shifts into the vacated slot.
2036 unsigned blockArgIdx = hasSRet ? 1 : 0;
2037 for (auto [i, ac] : llvm::enumerate(fc.argInfos)) {
2038 if (blockArgIdx >= entry.getNumArguments())
2039 break;
2040 if (ac.kind == ArgKind::Ignore) {
2041 mlir::BlockArgument arg = entry.getArgument(blockArgIdx);
2042 if (!arg.use_empty()) {
2043 builder.setInsertionPointToStart(&entry);
2044 mlir::Value poison =
2045 createIgnoredValue(builder, funcOp.getLoc(), arg.getType());
2046 arg.replaceAllUsesWith(poison);
2047 }
2048 entry.eraseArgument(blockArgIdx);
2049 continue;
2050 }
2051 if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl))
2052 blockArgIdx += flatTy.getNumElements();
2053 else if (ac.kind == ArgKind::Expand)
2054 blockArgIdx += cast<cir::RecordType>(oldArgTypes[i]).getNumElements();
2055 else
2056 ++blockArgIdx;
2057 }
2058 }
2059
2060 // When the return is classified Ignore but the original function had
2061 // a non-void return type, every cir.return becomes a naked return.
2062 // This relies on the invariant that computeNewReturnType has set
2063 // newRetTy = void for Ignore above, and that the function type is
2064 // updated below to match. Asserting this keeps the dependency
2065 // explicit.
2066 if (fc.returnInfo.kind == ArgKind::Ignore && !oldResultTypes.empty()) {
2067 assert(mlir::isa<cir::VoidType>(newRetTy) &&
2068 "Ignore-return path requires the new return type to be void");
2070 funcOp.walk([&](cir::ReturnOp r) { returns.push_back(r); });
2071 for (cir::ReturnOp r : returns) {
2072 if (r.getNumOperands() == 0)
2073 continue;
2074 builder.setInsertionPoint(r);
2075 cir::ReturnOp::create(builder, r.getLoc());
2076 r.erase();
2077 }
2078 }
2079 }
2080
2081 mlir::Type newFnTy = funcOp.cloneTypeWith(newArgTypes, newResultTypes);
2082 funcOp.setFunctionTypeAttr(mlir::TypeAttr::get(newFnTy));
2083
2084 // Rebuild arg_attrs when the function has an sret slot (slot 0 needs the
2085 // sret attribute set) or any arg needs updateArgAttrs.
2086 if (hasSRet || needsArgAttrUpdate(fc)) {
2087 auto existing = funcOp->getAttrOfType<mlir::ArrayAttr>("arg_attrs");
2088 mlir::ArrayAttr updated = updateArgAttrs(ctx, module, oldArgTypes, existing,
2089 fc, dl, enableNoundefAttrs);
2090 if (hasSRet) {
2091 // Prepend the sret slot's attribute dict (slot 0); the per-argument
2092 // dicts shift to slots 1..N. noalias is valid only on the callee's
2093 // parameter, so it is added only for definitions.
2094 SmallVector<mlir::NamedAttribute> sretAttrs = buildSretSlotAttrs(
2095 builder, origRetTy, fc.returnInfo.indirectAlign.value(),
2096 /*withNoalias=*/funcOp.isDefinition());
2098 withSret.push_back(mlir::DictionaryAttr::get(ctx, sretAttrs));
2099 llvm::append_range(withSret, updated);
2100 funcOp->setAttr("arg_attrs", mlir::ArrayAttr::get(ctx, withSret));
2101 } else {
2102 funcOp->setAttr("arg_attrs", updated);
2103 }
2104 }
2105
2106 if (mlir::isa<cir::VoidType>(newRetTy)) {
2107 funcOp->removeAttr("res_attrs");
2108 } else if (fc.returnInfo.kind == ArgKind::Extend ||
2109 fc.returnInfo.kind == ArgKind::Direct) {
2110 auto existing = funcOp->getAttrOfType<mlir::ArrayAttr>("res_attrs");
2111 if (mlir::ArrayAttr updated =
2112 updateResAttrs(ctx, existing, fc.returnInfo, origRetTy, dl))
2113 funcOp->setAttr("res_attrs", updated);
2114 }
2115
2116 return mlir::success();
2117}
2118
2119mlir::LogicalResult
2121 const FunctionClassification &fc,
2122 mlir::OpBuilder &builder) {
2123 // The rewrite below pairs the classification with the call's operands one
2124 // for one, so a call passing arguments through an ellipsis needs a
2125 // classification built from its own operands rather than from the callee's
2126 // declared parameters. Both directions of a mismatch have to be reported
2127 // before the pass-through early return, or a call whose declared parameters
2128 // happen to be pass-through is left as written with its surplus operands
2129 // never classified.
2130 //
2131 // A surplus operand means fc was built from the callee's declared
2132 // parameters alone. A shortfall means the callee was declared no_proto,
2133 // which turns off the verifier's argument-count check altogether.
2134 unsigned numOperands =
2135 mlir::cast<cir::CIRCallOpInterface>(callOp).getNumArgOperands();
2136 if (numOperands > fc.argInfos.size())
2137 return callOp->emitOpError()
2138 << "variadic arguments not yet implemented in CallConvLowering";
2139 if (numOperands < fc.argInfos.size())
2140 return callOp->emitOpError()
2141 << "call passes fewer arguments than the callee declares, which is "
2142 "not yet implemented in CallConvLowering";
2143
2144 if (!fc.needsRewrite())
2145 return mlir::success();
2146
2147 if (mlir::isa<cir::TryCallOp>(callOp))
2148 return callOp->emitOpError()
2149 << "TryCallOp not yet implemented in CallConvLowering";
2150
2151 auto call = mlir::cast<cir::CallOp>(callOp);
2152 mlir::MLIRContext *ctx = callOp->getContext();
2153 mlir::Block *slotBlock = coercionSlotBlock(call);
2154
2155 builder.setInsertionPoint(call);
2156
2158 mlir::ValueRange argOperands = call.getArgOperands();
2159 newArgs.reserve(argOperands.size());
2160
2161 // Loads that the new call leaves unused: a Direct (coerced or flattened) or
2162 // Expand record is read out of its load's storage or a copy of it, and an
2163 // Indirect argument passes or copies from the address the load read from.
2164 // The old call still uses them, so erase them only after it is gone.
2165 SmallVector<cir::LoadOp> deadRecordLoads;
2166
2167 // Capture original arg types before building newArgs (an Indirect argument
2168 // changes the wire argument from T to !cir.ptr<T>, so we save the
2169 // pre-rewrite types here for use in updateArgAttrs).
2170 SmallVector<mlir::Type> origCallArgTypes;
2171 llvm::append_range(origCallArgTypes, argOperands.getTypes());
2172
2173 // Operands past an indirect callee's declared parameters were passed
2174 // through its ellipsis. Note where they begin in newArgs, so
2175 // prependIndirectCallee can leave them out of the retyped callee type. A
2176 // direct call has no callee pointer to retype, so it records no boundary.
2177 unsigned numDeclared = fc.argInfos.size();
2178 if (call.isIndirect()) {
2179 numDeclared = getIndirectCalleeType(call).getNumInputs();
2180 assert(numDeclared <= fc.argInfos.size() &&
2181 "the classification covers at least the declared parameters");
2182 }
2183 // Recorded before the Ignore skip below, so an ignored first ellipsis
2184 // argument still marks where the ellipsis arguments begin in newArgs.
2185 std::optional<unsigned> ellipsisStart;
2186 for (auto [idx, ac] : llvm::enumerate(fc.argInfos)) {
2187 if (idx == numDeclared)
2188 ellipsisStart = newArgs.size();
2189 if (ac.kind == ArgKind::Ignore)
2190 continue;
2191 mlir::Value arg = argOperands[idx];
2192 if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl)) {
2193 // Direct + canFlatten: pass one scalar call argument per field of the
2194 // ABI-coerced struct. A non-atomic record load is read in place or
2195 // through a copy of its storage, as emitCoercedRecordLoad decides, and
2196 // any other operand is stored into a coercion slot first. When the types
2197 // already match, the fields come from the struct value directly (reading
2198 // from its source alloca when possible).
2199 if (arg.getType() != flatTy) {
2200 SmallPtrSet<mlir::Operation *, 4> coercionOps;
2201 assert(!ac.directOffset &&
2202 "the coerced struct starts at slot offset 0 here, so a "
2203 "flattened coercion cannot honor a direct offset");
2204 if (failed(rejectAtomicRecordLoad(call, arg)))
2205 return mlir::failure();
2206 mlir::OpBuilder::InsertionGuard guard(builder);
2207 mlir::Value coercedPtr;
2208 uint64_t baseAlign = 0;
2209 if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(arg)) {
2210 mlir::Value base = recLoad.getAddr();
2211 if (!recLoad.getIsVolatile() &&
2212 canReadInPlace(recLoad, flatTy, cirDataLayout, /*offset=*/0)) {
2213 cir::AllocaOp baseAlloca = cir::getUnderlyingAlloca(base);
2214 assert(baseAlloca && "canReadInPlace requires an alloca");
2215 baseAlign = baseAlloca.getAlignment();
2216 } else {
2217 base = emitCopiedCoercionSlot(builder, call.getLoc(), recLoad,
2218 flatTy, slotBlock, dl, /*offset=*/0);
2219 }
2220 builder.setInsertionPoint(recLoad);
2221 coercedPtr = emitViewAtOffset(builder, call.getLoc(), flatTy, base,
2222 coercionOps, /*offset=*/0);
2223 deadRecordLoads.push_back(recLoad);
2224 } else {
2225 coercedPtr =
2226 emitCoercionToMemory(builder, call.getLoc(), flatTy, arg,
2227 slotBlock, dl, coercionOps, /*offset=*/0);
2228 }
2229 auto coercedPtrTy = mlir::cast<cir::PointerType>(coercedPtr.getType());
2230 for (auto [f, fieldTy] : llvm::enumerate(flatTy.getMembers())) {
2231 mlir::Type fieldPtrTy =
2232 cir::PointerType::get(fieldTy, coercedPtrTy.getAddrSpace());
2233 auto fieldPtr =
2234 cir::GetMemberOp::create(builder, call.getLoc(), fieldPtrTy,
2235 coercedPtr, /*name=*/"", /*index=*/f);
2236 auto field = cir::LoadOp::create(builder, call.getLoc(), fieldTy,
2237 fieldPtr.getResult());
2238 // Storage read in place may be less aligned than the coerced struct,
2239 // so each field claims only what its offset in it allows.
2240 if (baseAlign)
2241 field.setAlignment(
2242 llvm::MinAlign(baseAlign, flatTy.getElementOffset(dl, f)));
2243 newArgs.push_back(field);
2244 }
2245 } else {
2246 emitStructFieldArgs(builder, call.getLoc(), arg, flatTy, newArgs,
2247 deadRecordLoads);
2248 }
2249 } else if (ac.kind == ArgKind::Expand) {
2250 // Decompose the struct value into its constituent scalar fields and
2251 // pass each as a separate argument.
2252 auto recTy = cast<cir::RecordType>(arg.getType());
2253 assert(recTy.isStruct() &&
2254 "Expand classification requires a struct type, not a union");
2255 emitStructFieldArgs(builder, call.getLoc(), arg, recTy, newArgs,
2256 deadRecordLoads);
2257 } else if ((ac.kind == ArgKind::Direct && ac.coercedType &&
2258 arg.getType() != ac.coercedType) ||
2259 isRecordExtend(ac, arg.getType())) {
2260 if (failed(rejectAtomicRecordLoad(call, arg)))
2261 return mlir::failure();
2262 if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(arg)) {
2263 newArgs.push_back(emitCoercedRecordLoad(
2264 builder, call.getLoc(), recLoad, ac.coercedType, slotBlock, dl,
2265 cirDataLayout, ac.directOffset));
2266 deadRecordLoads.push_back(recLoad);
2267 } else {
2268 newArgs.push_back(emitCoercion(builder, call.getLoc(), ac.coercedType,
2269 arg, slotBlock, dl, ac.directOffset));
2270 }
2271 } else if (ac.kind == ArgKind::Indirect ||
2272 ac.kind == ArgKind::IndirectAliased) {
2273 uint64_t minAlign = ac.indirectAlign.value();
2274 // The callee copies a byref object at entry, so the caller treats it
2275 // like byval.
2276 bool byRef = ac.kind == ArgKind::IndirectAliased;
2277 bool byCopy = byRef || ac.byVal;
2278 cir::PointerType ptrTy = getIndirectPtrTy(arg.getType(), ac);
2279
2280 // A record operand that names storage is passed from that storage
2281 // rather than from the loaded value: a record value carries only the
2282 // fields of its type, so a store leaves the type's padding bytes
2283 // unwritten, and for a union those can be another member's live data.
2284 // A byval _BitInt is passed from its value, since the load and the
2285 // store below are its conversion between value and in-memory form.
2286 cir::LoadOp srcLoad;
2287 if (!byCopy || mlir::isa<cir::RecordType>(arg.getType()))
2288 srcLoad = maybeGetSimpleLoad(arg);
2289
2290 // A non-byval argument must name the caller's storage, so that the
2291 // object the callee operates on is the one the caller destroys. That
2292 // means passing the address the operand was loaded from rather than the
2293 // loaded value, so a store to that storage after the load is visible to
2294 // the callee. byval fills a slot below, this cannot.
2295 if (!byCopy) {
2296 mlir::Value fwdAddr = forwardableIndirectAddr(srcLoad, ptrTy, minAlign);
2297 if (!fwdAddr)
2298 return call->emitOpError()
2299 << "non-byval indirect argument that does not name the "
2300 "caller's storage is not yet implemented in "
2301 "CallConvLowering";
2302 newArgs.push_back(fwdAddr);
2303 deadRecordLoads.push_back(srcLoad);
2304 continue;
2305 }
2306
2307 // byval hands the callee a copy taken at the call, so naming the object's
2308 // own storage leaves it with the same copy and saves this one. That
2309 // holds only while nothing writes that storage between the load and the
2310 // call, since the operand carries the value as of the load. A volatile
2311 // load is not forwarded, since the copy byval takes at the call is not
2312 // volatile.
2313 if (mlir::Value fwdAddr =
2314 forwardableIndirectAddr(srcLoad, ptrTy, minAlign);
2315 fwdAddr && storageUnwrittenBetween(srcLoad, call.getOperation(),
2316 aliasAnalysis)) {
2317 newArgs.push_back(fwdAddr);
2318 deadRecordLoads.push_back(srcLoad);
2319 continue;
2320 }
2321
2322 // A byref slot has to be an alloca of the parameter's own type, which
2323 // only the alloca address space can hold.
2324 if (byRef && ptrTy != getAllocaPtrTy(arg.getType(), dl))
2325 return call->emitOpError()
2326 << "byref argument outside the alloca address space that does "
2327 "not name forwardable storage is not yet implemented in "
2328 "CallConvLowering";
2329
2330 // Otherwise fill a slot, at the load's position when there is one so
2331 // the copy reads the memory state the load read.
2332 if (failed(rejectAtomicRecordLoad(call, arg)))
2333 return mlir::failure();
2334 if (!srcLoad)
2335 srcLoad = maybeGetCopyableRecordLoad(arg);
2336 mlir::IntegerAttr slotAlign = builder.getI64IntegerAttr(minAlign);
2337 mlir::OpBuilder::InsertionGuard guard(builder);
2338 if (srcLoad)
2339 builder.setInsertionPoint(srcLoad);
2340
2341 auto slot = cir::AllocaOp::create(
2342 builder, call.getLoc(), ptrTy,
2343 builder.getStringAttr(byRef ? "byref" : "byval"), slotAlign);
2344 if (srcLoad) {
2345 auto copy = cir::CopyOp::create(builder, call.getLoc(), slot,
2346 srcLoad.getAddr(), slotAlign,
2347 loadSourceAlignment(srcLoad, builder));
2348 copy.setIsVolatile(srcLoad.getIsVolatile());
2349 deadRecordLoads.push_back(srcLoad);
2350 } else {
2351 // A record with no storage behind it, such as a constant, defines no
2352 // bytes beyond its fields. A producer rewritten later turns this store
2353 // into a copy through rewireRecordUses.
2354 cir::StoreOp::create(builder, call.getLoc(), arg, slot);
2355 }
2356 newArgs.push_back(slot);
2357 } else {
2358 newArgs.push_back(arg);
2359 }
2360 }
2361 unsigned numEllipsisArgs =
2362 ellipsisStart ? newArgs.size() - *ellipsisStart : 0;
2363
2364 bool hasResult = call.getNumResults() > 0;
2365 mlir::Type origRetTy =
2366 hasResult ? call.getResult().getType() : cir::VoidType::get(ctx);
2367
2368 // An indirect (sret) return has a different call shape than the coerce /
2369 // extend / ignore return handling further down (the value is returned
2370 // through a prepended pointer slot, not as a result), so dispatch to a
2371 // dedicated helper for it; everything below handles the by-value returns.
2372 if (fc.returnInfo.kind == ArgKind::Indirect && hasResult) {
2373 if (failed(rewriteIndirectReturnCall(call, fc, newArgs, numEllipsisArgs,
2374 origRetTy, origCallArgTypes, builder,
2375 dl, enableNoundefAttrs)))
2376 return mlir::failure();
2377 eraseDeadRecordLoads(deadRecordLoads);
2378 return mlir::success();
2379 }
2380
2381 mlir::Type callRetTy = origRetTy;
2382 if (fc.returnInfo.kind == ArgKind::Ignore && hasResult)
2383 callRetTy = cir::VoidType::get(ctx);
2384 bool returnNeedsCoercion =
2385 hasResult && fc.returnInfo.kind == ArgKind::Direct &&
2386 fc.returnInfo.coercedType && fc.returnInfo.coercedType != origRetTy;
2387 if (returnNeedsCoercion)
2388 callRetTy = fc.returnInfo.coercedType;
2389
2390 builder.setInsertionPoint(call);
2391 prependIndirectCallee(call, newArgs, callRetTy, numEllipsisArgs, builder);
2392 auto newCall = cir::CallOp::create(builder, call.getLoc(),
2393 call.getCalleeAttr(), callRetTy, newArgs);
2394 copyCallAttributes(call, newCall);
2395
2396 // Direct return with coercion: the new call returns the coerced type, which
2397 // is stored into a coercion slot where the old call's uses find the original
2398 // type.
2399 if (returnNeedsCoercion) {
2400 builder.setInsertionPointAfter(newCall);
2401 SmallPtrSet<mlir::Operation *, 4> coercionOps;
2402 if (failed(emitCoercionAndReplaceUses(builder, call.getLoc(), origRetTy,
2403 newCall.getResult(), call.getResult(),
2404 slotBlock, dl, coercionOps,
2405 fc.returnInfo.directOffset)))
2406 return mlir::failure();
2407 }
2408
2409 if (needsArgAttrUpdate(fc)) {
2410 auto existing = call->getAttrOfType<mlir::ArrayAttr>("arg_attrs");
2411 newCall->setAttr("arg_attrs",
2412 updateArgAttrs(ctx, module, origCallArgTypes, existing, fc,
2413 dl, enableNoundefAttrs));
2414 }
2415 // Add llvm.signext / llvm.zeroext to an Extend return, and drop llvm.noundef
2416 // from a coerced Direct return that passes undefined bits.
2417 if (fc.returnInfo.kind == ArgKind::Extend ||
2418 (hasResult && fc.returnInfo.kind == ArgKind::Direct)) {
2419 auto existing = call->getAttrOfType<mlir::ArrayAttr>("res_attrs");
2420 if (mlir::ArrayAttr updated =
2421 updateResAttrs(ctx, existing, fc.returnInfo, origRetTy, dl))
2422 newCall->setAttr("res_attrs", updated);
2423 } else if (hasResult && mlir::isa<cir::VoidType>(callRetTy)) {
2424 newCall->removeAttr("res_attrs");
2425 }
2426
2427 if (hasResult && fc.returnInfo.kind == ArgKind::Ignore) {
2428 // The new call returns void, but the original call's result may still
2429 // have uses. Substitute a poison constant of the original type so
2430 // those uses remain well-formed without pretending we have a real
2431 // value at the ABI boundary.
2432 if (!call.getResult().use_empty()) {
2433 builder.setInsertionPointAfter(newCall);
2434 mlir::Value poison =
2435 createIgnoredValue(builder, call.getLoc(), origRetTy);
2436 call.getResult().replaceAllUsesWith(poison);
2437 }
2438 } else if (hasResult && !returnNeedsCoercion) {
2439 // returnNeedsCoercion already wired up the coerced result above.
2440 call.getResult().replaceAllUsesWith(newCall.getResult());
2441 }
2442
2443 call->erase();
2444 eraseDeadRecordLoads(deadRecordLoads);
2445
2446 return mlir::success();
2447}
2448
2450 cir::FuncOp funcOp,
2451 mlir::OpBuilder &builder) {
2452 auto oldPtrTy = mlir::cast<cir::PointerType>(addrOp.getAddr().getType());
2453 cir::FuncType newFuncTy = funcOp.getFunctionType();
2454 // An extension rides on an argument attribute and leaves the signature
2455 // alone, so such a callee still matches the written type.
2456 if (newFuncTy == oldPtrTy.getPointee())
2457 return;
2458
2459 // The verifier requires the retype even when nothing reads the address.
2460 addrOp.getAddr().setType(cir::PointerType::get(newFuncTy));
2461 if (addrOp.getAddr().use_empty())
2462 return;
2463
2464 // A later indirect call through the written type is classified and coerced
2465 // on its own, so casting back leaves it to that rewrite.
2466 mlir::OpBuilder::InsertionGuard guard(builder);
2467 builder.setInsertionPointAfter(addrOp);
2468 auto bitcast = cir::CastOp::create(builder, addrOp.getLoc(), oldPtrTy,
2469 cir::CastKind::bitcast, addrOp.getAddr());
2470 addrOp.getAddr().replaceAllUsesExcept(bitcast.getResult(), bitcast);
2471}
2472
2473namespace {
2474
2475/// What one `va_arg` expansion needs from the op it rewrites. The x86-64
2476/// cursor fields are reached through `vaFields` by index: 0 gp_offset,
2477/// 1 fp_offset, 2 overflow_arg_area, 3 reg_save_area.
2478struct VAArgFetch {
2479 VAArgFetch(mlir::Location loc, mlir::Value valist,
2480 llvm::ArrayRef<mlir::Type> vaFields, mlir::Type resultTy,
2481 const ArgClassification &ac, const mlir::DataLayout &dl,
2482 mlir::ModuleOp module)
2483 : loc(loc), valist(valist), vaFields(vaFields), resultTy(resultTy),
2484 ac(ac), dl(dl), module(module) {
2485 assert(vaFields.size() == 4 &&
2486 "the x86-64 va_list is a four-field cursor, checked by the caller");
2487 }
2488
2489 mlir::Location loc;
2490 mlir::Value valist;
2492 mlir::Type resultTy;
2493 const ArgClassification &ac;
2494 const mlir::DataLayout &dl;
2495 mlir::ModuleOp module;
2496};
2497
2498/// The cursor fields a register fetch reads, and the predicate saying every
2499/// class it needs still has room. An offset is null when the fetch needs no
2500/// register of that class.
2501struct RegisterCursor {
2502 mlir::Value gpOffsetP;
2503 mlir::Value fpOffsetP;
2504 mlir::Value gpOffset;
2505 mlir::Value fpOffset;
2506 mlir::Value inRegs;
2507};
2508
2509mlir::LogicalResult reportVAArgNYI(cir::VAArgOp op, llvm::StringRef what) {
2510 op->emitOpError() << "va_arg of " << what
2511 << " not yet implemented in CallConvLowering";
2512 return mlir::failure();
2513}
2514
2515/// Sets \p isRegPair when a two-register argument is a coerced pair, one
2516/// register per member, rather than a single wide scalar, and records in
2517/// \p pairIsSse which element is SSE class. Fails on a coercion that is not
2518/// two eightbytes.
2519mlir::LogicalResult classifyRegisterPair(cir::VAArgOp op,
2520 const ArgClassification &ac,
2521 std::array<bool, 2> &pairIsSse,
2522 bool &isRegPair) {
2523 auto pairTy = mlir::dyn_cast<cir::RecordType>(ac.coercedType);
2524 if (!pairTy)
2525 return mlir::success();
2526
2527 if (pairTy.getNumElements() != 2)
2528 return reportVAArgNYI(op, "a register coercion that is not two eightbytes");
2529
2530 assert(!ac.directOffset &&
2531 "a pair already spans both eightbytes, so it cannot also start "
2532 "partway into the value");
2533 for (auto [i, memberTy] : llvm::enumerate(pairTy.getMembers()))
2534 pairIsSse[i] = isSSERegisterClass(memberTy);
2535 isRegPair = true;
2536 return mlir::success();
2537}
2538
2539/// The alignment an argument is placed at. A record can require more than
2540/// the types of its members imply, from an `aligned` attribute on the record
2541/// or on one of its fields, and neither raises the alignment of any type.
2542/// Only the record layout knows, so the member-derived value alone can be too
2543/// small.
2544uint64_t argumentAreaAlign(mlir::Type ty, mlir::ModuleOp modOp,
2545 const mlir::DataLayout &dl) {
2546 uint64_t align = dl.getTypeABIAlignment(ty);
2547 if (auto recTy = mlir::dyn_cast<cir::RecordType>(ty))
2548 if (auto layout = cir::tryGetRecordLayout(modOp, recTy.getName()))
2549 align = std::max<uint64_t>(align, layout.getRecordAlign());
2550 return align;
2551}
2552
2553mlir::Value roundPointerUpToAlignment(CIRBaseBuilderTy &b, mlir::Location loc,
2554 mlir::Value bytePtr, uint64_t align,
2555 const mlir::DataLayout &dl) {
2556 assert(mlir::cast<cir::PointerType>(bytePtr.getType()).getPointee() ==
2557 b.getUIntNTy(8) &&
2558 "the bump strides in bytes, so the pointee must be u8");
2559 assert(llvm::isPowerOf2_64(align) &&
2560 "mask rounding needs a power-of-two alignment");
2561 mlir::Value bumped =
2562 b.createPtrStride(loc, bytePtr, b.getSignedInt(loc, align - 1, 32));
2563 std::optional<uint64_t> indexWidth =
2564 dl.getTypeIndexBitwidth(bytePtr.getType());
2565 assert(indexWidth && "a pointer in the argument area has an index width");
2566 mlir::Value mask = b.getSignedInt(loc, -static_cast<int64_t>(align),
2567 static_cast<unsigned>(*indexWidth));
2568 return cir::PtrMaskOp::create(b, loc, bytePtr.getType(), bumped, mask);
2569}
2570
2571/// Reads the argument's address out of the overflow area, which also advances
2572/// the cursor past the argument.
2573mlir::Value buildOverflowAddrAndAdvance(CIRBaseBuilderTy &b,
2574 const VAArgFetch &f) {
2575 cir::IntType byteTy = b.getUIntNTy(8);
2576 mlir::Value overflowP = b.createGetMember(
2577 f.loc, b.getPointerTo(f.vaFields[2]), f.valist, "overflow_arg_area", 2);
2578 mlir::Value overflow = b.createLoad(f.loc, overflowP);
2579 mlir::Value bytePtr = b.createPtrBitcast(overflow, byteTy);
2580
2581 uint64_t tyAlign = argumentAreaAlign(f.resultTy, f.module, f.dl);
2582 if (tyAlign > 8)
2583 bytePtr = roundPointerUpToAlignment(b, f.loc, bytePtr, tyAlign, f.dl);
2584
2585 uint64_t stride =
2586 llvm::alignTo(f.dl.getTypeAllocSize(f.resultTy).getFixedValue(), 8);
2587 mlir::Value strideVal = b.getSignedInt(f.loc, stride, 32);
2588 mlir::Value next = b.createPtrStride(f.loc, bytePtr, strideVal);
2589 b.createStore(f.loc, next, overflowP);
2590 return bytePtr;
2591}
2592
2593/// Loads the cursor offsets and builds the predicate that sends the fetch to
2594/// the register-save area. Both offsets count from the start of that area, so
2595/// the integer limit is the six GP registers at 48 and the vector limit is
2596/// those plus the eight SSE registers at 176.
2597RegisterCursor buildRegisterGate(CIRBaseBuilderTy &b, const VAArgFetch &f,
2598 unsigned neededInt, unsigned neededSse) {
2599 RegisterCursor cursor;
2600 if (neededInt) {
2601 cursor.gpOffsetP = b.createGetMember(f.loc, b.getPointerTo(f.vaFields[0]),
2602 f.valist, "gp_offset", 0);
2603 cursor.gpOffset = b.createLoad(f.loc, cursor.gpOffsetP);
2604 mlir::Value limit =
2605 b.getConstantInt(f.loc, cursor.gpOffset.getType(), 48 - neededInt * 8);
2606 cursor.inRegs =
2607 b.createCompare(f.loc, cir::CmpOpKind::le, cursor.gpOffset, limit);
2608 }
2609 if (neededSse) {
2610 cursor.fpOffsetP = b.createGetMember(f.loc, b.getPointerTo(f.vaFields[1]),
2611 f.valist, "fp_offset", 1);
2612 cursor.fpOffset = b.createLoad(f.loc, cursor.fpOffsetP);
2613 mlir::Value limit = b.getConstantInt(f.loc, cursor.fpOffset.getType(),
2614 176 - neededSse * 16);
2615 mlir::Value fitsInFp =
2616 b.createCompare(f.loc, cir::CmpOpKind::le, cursor.fpOffset, limit);
2617 cursor.inRegs = cursor.inRegs
2618 ? b.createLogicalAnd(f.loc, cursor.inRegs, fitsInFp)
2619 : fitsInFp;
2620 }
2621 return cursor;
2622}
2623
2624/// Copies each half of a non-contiguous pair out of the register-save area
2625/// into \p regPairTemp, laid out as the coerced pair.
2626void reassembleRegisterPair(CIRBaseBuilderTy &b, mlir::Location loc,
2627 const ArgClassification &ac,
2628 const RegisterCursor &cursor,
2629 const std::array<bool, 2> &pairIsSse,
2630 mlir::Value regSaveArea, mlir::Value regPairTemp) {
2631 auto pairTy = mlir::cast<cir::RecordType>(ac.coercedType);
2632 // Both halves of an all-SSE pair sit in 16-byte slots, which classic
2633 // CodeGen tells the load about. It leaves a mixed pair to the element's
2634 // own alignment, so match that rather than claiming the slot there.
2635 bool bothSse = pairIsSse[0] && pairIsSse[1];
2636 // Track how many of each class came before, since a slot is reached from
2637 // its class's own cursor.
2638 unsigned seenOfClass[2] = {0, 0};
2639 for (unsigned i = 0; i < 2; ++i) {
2640 bool isSse = pairIsSse[i];
2641 mlir::Value base = isSse ? cursor.fpOffset : cursor.gpOffset;
2642 unsigned regSize = isSse ? 16 : 8;
2643 unsigned prior = seenOfClass[isSse];
2644 ++seenOfClass[isSse];
2645 mlir::Value off = base;
2646 if (prior) {
2647 off = b.createAdd(loc, base,
2648 b.getConstantInt(loc, base.getType(), prior * regSize));
2649 }
2650 mlir::Value src = b.createPtrStride(loc, regSaveArea, off);
2651 mlir::Type elemTy = pairTy.getElementType(i);
2652 mlir::Value elemPtr = b.createPtrBitcast(src, elemTy);
2653 mlir::Value val = bothSse ? b.createAlignedLoad(loc, elemPtr, 16)
2654 : b.createLoad(loc, elemPtr);
2655 b.createStore(
2656 loc, val,
2657 b.createGetMember(loc, b.getPointerTo(elemTy), regPairTemp, "", i));
2658 }
2659}
2660
2661/// The bytes carried by the registers of this fetch's one class.
2662uint64_t registerSlotSize(unsigned neededInt, unsigned neededSse) {
2663 assert(!(neededInt && neededSse) &&
2664 "a fetch needing both classes is a pair, reassembled elsewhere");
2665 return neededSse ? neededSse * 16 : neededInt * 8;
2666}
2667
2668/// Whether the register cannot be read in place, either because it carries
2669/// less than the whole result or because its slot is under-aligned for it.
2670bool needsTempCopy(const VAArgFetch &f, unsigned neededInt,
2671 unsigned neededSse) {
2672 uint64_t tySize = f.dl.getTypeSize(f.resultTy).getFixedValue();
2673 if (f.ac.coercedType &&
2674 (f.ac.directOffset || registerSlotSize(neededInt, neededSse) < tySize))
2675 return true;
2676 // A slot is only as aligned as its class, 8 for a GP register and 16 for an
2677 // SSE one, so a fetch the ABI places more strictly is copied through a temp.
2678 uint64_t slotAlign = neededSse ? 16 : 8;
2679 return argumentAreaAlign(f.resultTy, f.module, f.dl) > slotAlign;
2680}
2681
2682/// Copies what the registers carry into \p temp, which is the size of the
2683/// whole result, and returns the address to read the result from.
2684mlir::Value copyRegisterToTemp(CIRBaseBuilderTy &b, mlir::Location loc,
2685 const VAArgFetch &f, mlir::Value regAddr,
2686 mlir::Value temp, unsigned neededInt,
2687 unsigned neededSse) {
2688 cir::IntType byteTy = b.getUIntNTy(8);
2689 uint64_t tySize = f.dl.getTypeSize(f.resultTy).getFixedValue();
2690
2691 if (f.ac.coercedType &&
2692 (f.ac.directOffset || registerSlotSize(neededInt, neededSse) < tySize)) {
2693 // The registers carry less than the whole result, either because the
2694 // eightbytes below directOffset hold no field or because the result is
2695 // wider than the registers carrying it. Copy only what they carry, so
2696 // that reading the temp cannot run on into the neighboring slot.
2697 mlir::Value val = b.createAlignedLoad(
2698 loc, b.createPtrBitcast(regAddr, f.ac.coercedType), 8);
2699 mlir::Value dst = temp;
2700 if (f.ac.directOffset) {
2701 mlir::Value tempByte = b.createPtrBitcast(temp, byteTy);
2702 mlir::Value offset = b.getSignedInt(loc, f.ac.directOffset, 32);
2703 dst = b.createPtrStride(loc, tempByte, offset);
2704 }
2705 b.createStore(loc, val, b.createPtrBitcast(dst, f.ac.coercedType));
2706 return b.createPtrBitcast(temp, byteTy);
2707 }
2708
2709 mlir::Value regView = b.createPtrBitcast(regAddr, f.resultTy);
2710 if (mlir::isa<cir::RecordType>(f.resultTy))
2711 cir::CopyOp::create(
2712 b, loc, temp, regView,
2713 b.getI64IntegerAttr(argumentAreaAlign(f.resultTy, f.module, f.dl)),
2714 b.getI64IntegerAttr(8));
2715 else
2716 b.createStore(loc, b.createAlignedLoad(loc, regView, 8), temp);
2717 return b.createPtrBitcast(temp, byteTy);
2718}
2719
2720void advanceRegisterCursors(CIRBaseBuilderTy &b, mlir::Location loc,
2721 const RegisterCursor &cursor, unsigned neededInt,
2722 unsigned neededSse) {
2723 if (neededInt) {
2724 b.createStore(loc,
2725 b.createAdd(loc, cursor.gpOffset,
2726 b.getConstantInt(loc, cursor.gpOffset.getType(),
2727 neededInt * 8)),
2728 cursor.gpOffsetP);
2729 }
2730 if (neededSse) {
2731 b.createStore(loc,
2732 b.createAdd(loc, cursor.fpOffset,
2733 b.getConstantInt(loc, cursor.fpOffset.getType(),
2734 neededSse * 16)),
2735 cursor.fpOffsetP);
2736 }
2737}
2738
2739} // namespace
2740
2741mlir::LogicalResult
2742CIRABIRewriteContext::rewriteVAArg(mlir::Operation *vaArgOp,
2743 const ArgClassification &ac,
2744 mlir::OpBuilder &opBuilder) {
2745 auto op = mlir::cast<cir::VAArgOp>(vaArgOp);
2746 CIRBaseBuilderTy builder(opBuilder);
2747 mlir::Location loc = op.getLoc();
2748 mlir::Type resultTy = op.getType();
2749 mlir::Value valist = op.getArgList();
2750
2751 // An ignored type travels in no register and no stack slot, so the fetch
2752 // reads nothing and must leave the cursor unchanged. The value holds no
2753 // bytes, so poison stands in for it.
2754 if (ac.kind == ArgKind::Ignore) {
2755 builder.setInsertionPoint(op);
2756 op.getResult().replaceAllUsesWith(
2757 createIgnoredValue(builder, loc, resultTy));
2758 op->erase();
2759 return mlir::success();
2760 }
2761
2762 if (ac.kind == ArgKind::Indirect && !ac.byVal)
2763 return reportVAArgNYI(op, "a non-trivially-copyable type");
2764 if (ac.kind == ArgKind::IndirectAliased)
2765 return reportVAArgNYI(op, "a type passed byref");
2766
2767 // neededInt counts 8-byte integer slots and neededSse counts 16-byte vector
2768 // slots. Zero of both means the type travels in memory and is read
2769 // straight from the overflow area.
2770 unsigned neededInt = ac.neededIntRegs;
2771 unsigned neededSse = ac.neededSseRegs;
2772
2773 // Which coerced-pair element (0 = low eightbyte, 1 = high) is SSE rather
2774 // than INTEGER class. Only meaningful when isRegPair is set.
2775 std::array<bool, 2> pairIsSse = {false, false};
2776 bool isRegPair = false;
2777 if (ac.kind == ArgKind::Direct && neededInt + neededSse == 2 &&
2778 ac.coercedType) {
2779 if (classifyRegisterPair(op, ac, pairIsSse, isRegPair).failed())
2780 return mlir::failure();
2781 }
2782
2783 auto vaListRecTy = mlir::dyn_cast<cir::RecordType>(
2784 mlir::cast<cir::PointerType>(valist.getType()).getPointee());
2785 if (!vaListRecTy || vaListRecTy.getNumElements() != 4) {
2786 return reportVAArgNYI(op,
2787 "a va_list that is not the four-field gp_offset / "
2788 "fp_offset / overflow_arg_area / reg_save_area "
2789 "cursor");
2790 }
2791
2792 const VAArgFetch fetch{loc, valist, vaListRecTy.getMembers(), resultTy, ac,
2793 dl, module};
2794 cir::IntType byteTy = builder.getUIntNTy(8);
2795
2796 builder.setInsertionPoint(op);
2797
2798 mlir::Value addr;
2799 if (neededInt == 0 && neededSse == 0) {
2800 addr = buildOverflowAddrAndAdvance(builder, fetch);
2801 } else {
2802 RegisterCursor cursor =
2803 buildRegisterGate(builder, fetch, neededInt, neededSse);
2804
2805 // A two-eightbyte pair that is purely INTEGER class is contiguous in the
2806 // register-save area, since GP slots are 8-byte packed, so the address of
2807 // its low eightbyte is already the address of the whole value. A pure
2808 // SSE pair or a mixed pair is not contiguous, since SSE slots are 16-byte
2809 // spaced and a mixed pair's halves live in disjoint areas, so each half is
2810 // copied into a temp laid out as the coerced pair.
2811 bool pairNeedsReassembly = isRegPair && neededSse != 0;
2812
2813 // Both temps are allocated here, since a ternary arm yields their address.
2814 mlir::Value regPairTemp;
2815 if (pairNeedsReassembly) {
2816 // The temp is written one member at a time through the coerced pair, so
2817 // it has to meet that type's alignment, and read back as the result, so
2818 // it has to meet the result's alignment too.
2819 regPairTemp = builder.createAlloca(
2820 loc, builder.getPointerTo(ac.coercedType), "vaarg.reg",
2822 std::max(dl.getTypeABIAlignment(ac.coercedType),
2823 argumentAreaAlign(resultTy, module, dl))));
2824 }
2825
2826 mlir::Value regTemp;
2827 bool copyThroughTemp =
2828 !pairNeedsReassembly && needsTempCopy(fetch, neededInt, neededSse);
2829 if (copyThroughTemp) {
2830 regTemp =
2831 builder.createAlloca(loc, builder.getPointerTo(resultTy), "vaarg.reg",
2833 argumentAreaAlign(resultTy, module, dl)));
2834 }
2835
2836 addr = cir::TernaryOp::create(
2837 builder, loc, cursor.inRegs,
2838 /*trueBuilder=*/
2839 [&](mlir::OpBuilder &ob, mlir::Location l) {
2840 CIRBaseBuilderTy b(ob);
2841 mlir::Value regSaveArea = b.createLoad(
2842 l, b.createGetMember(l, b.getPointerTo(fetch.vaFields[3]),
2843 valist, "reg_save_area", 3));
2844 regSaveArea = b.createPtrBitcast(regSaveArea, byteTy);
2845
2846 mlir::Value regAddr;
2847 if (pairNeedsReassembly) {
2848 reassembleRegisterPair(b, l, ac, cursor, pairIsSse,
2849 regSaveArea, regPairTemp);
2850 regAddr = b.createPtrBitcast(regPairTemp, byteTy);
2851 } else {
2852 mlir::Value off =
2853 neededSse ? cursor.fpOffset : cursor.gpOffset;
2854 regAddr = b.createPtrStride(l, regSaveArea, off);
2855 if (copyThroughTemp) {
2856 regAddr = copyRegisterToTemp(b, l, fetch, regAddr, regTemp,
2857 neededInt, neededSse);
2858 }
2859 }
2860
2861 advanceRegisterCursors(b, l, cursor, neededInt, neededSse);
2862 cir::YieldOp::create(b, l, regAddr);
2863 },
2864 /*falseBuilder=*/
2865 [&](mlir::OpBuilder &ob, mlir::Location l) {
2866 CIRBaseBuilderTy b(ob);
2867 mlir::Value memAddr = buildOverflowAddrAndAdvance(b, fetch);
2868 cir::YieldOp::create(b, l, memAddr);
2869 })
2870 .getResult();
2871 }
2872
2873 // Every path above places the result at least this well aligned.
2874 mlir::Value resultAddr = builder.createPtrBitcast(addr, resultTy);
2875 uint64_t resultAlign = argumentAreaAlign(resultTy, module, dl);
2876 if (mlir::isa<cir::RecordType>(resultTy)) {
2877 SmallPtrSet<mlir::Operation *, 1> noExceptions;
2878 if (failed(rewireRecordUses(builder, loc, op.getResult(), resultAddr,
2879 resultAlign, dl, noExceptions)))
2880 return mlir::failure();
2881 } else {
2882 op.getResult().replaceAllUsesWith(
2883 builder.createAlignedLoad(loc, resultAddr, resultAlign));
2884 }
2885 op->erase();
2886 return mlir::success();
2887}
static mlir::LogicalResult prepareOneNonByvalParameter(cir::FuncOp funcOp, unsigned argNo, const ArgClassification &ac, mlir::BlockArgument blockArg, mlir::DominanceInfo &dom, SmallPtrSetImpl< mlir::Operation * > &claimedSlots)
Bring funcOp's non-byval indirect parameter argNo into the shape the rest of the rewrite assumes.
*collection of selector each with an associated kind and an ordered *collection of selectors A selector has a kind
void rewriteFunctionAddress(cir::GetGlobalOp addrOp, cir::FuncOp funcOp, mlir::OpBuilder &builder)
Retype addrOp, which holds the address of funcOp, to the signature funcOp was rewritten to,...
mlir::LogicalResult rewriteFunctionDefinition(mlir::FunctionOpInterface funcOp, const mlir::abi::FunctionClassification &fc, mlir::OpBuilder &builder) override
void finalizeParameterSlots()
Replace each non-byval indirect parameter's spill slot with the incoming pointer, so the body operate...
mlir::LogicalResult prepareNonByvalParameters(cir::FuncOp funcOp, const mlir::abi::FunctionClassification &fc)
Bring each non-byval indirect parameter of funcOp into the shape the rest of the rewrite assumes: the...
mlir::LogicalResult rewriteVAArg(mlir::Operation *vaArgOp, const mlir::abi::ArgClassification &ac, mlir::OpBuilder &builder) override
Expand a cir.va_arg into the x86-64 SysV register-save-area / overflow-area sequence.
mlir::LogicalResult rewriteCallSite(mlir::Operation *callOp, const mlir::abi::FunctionClassification &fc, mlir::OpBuilder &builder) override
mlir::Value createAdd(mlir::Location loc, mlir::Value lhs, mlir::Value rhs, OverflowFlags flags=OverflowFlags::None)
cir::PtrStrideOp createPtrStride(mlir::Location loc, mlir::Value base, mlir::Value stride)
cir::GetMemberOp createGetMember(mlir::Location loc, mlir::Type resultTy, mlir::Value base, llvm::StringRef name, unsigned index)
cir::PointerType getPointerTo(mlir::Type ty)
cir::IntType getUIntNTy(int n)
mlir::Value createPtrBitcast(mlir::Value src, mlir::Type newPointeeTy)
mlir::Value createAlloca(mlir::Location loc, cir::PointerType addrType, llvm::StringRef name, mlir::IntegerAttr alignment, mlir::Value dynAllocSize)
cir::LoadOp createLoad(mlir::Location loc, mlir::Value ptr, bool isVolatile=false, uint64_t alignment=0, bool isNontemporal=false)
mlir::Value getSignedInt(mlir::Location loc, int64_t val, unsigned numBits)
cir::CmpOp createCompare(mlir::Location loc, cir::CmpOpKind kind, mlir::Value lhs, mlir::Value rhs)
cir::ConstantOp getConstantInt(mlir::Location loc, mlir::Type ty, int64_t value)
mlir::Value createAlignedLoad(mlir::Location loc, mlir::Value ptr, uint64_t alignment)
mlir::Value createLogicalAnd(mlir::Location loc, mlir::Value lhs, mlir::Value rhs)
cir::StoreOp createStore(mlir::Location loc, mlir::Value val, mlir::Value dst, bool isVolatile=false, bool isNontemporal=false, mlir::IntegerAttr align={}, cir::SyncScopeKindAttr scope={}, cir::MemOrderAttr order={})
llvm::TypeSize getTypeAllocSize(mlir::Type ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
llvm::TypeSize getTypeStoreSize(mlir::Type ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
C++ view class that accepts both !cir.struct and !cir.union types.
Definition CIRTypes.h:150
llvm::ArrayRef< mlir::Type > getMembers() const
Definition CIRTypes.cpp:648
static llvm::SmallVector< RecordMemberKind > getAllDataKinds(llvm::ArrayRef< mlir::Type > members)
One Data kind per member.
Definition CIRTypes.cpp:151
mlir::StringAttr getName() const
Definition CIRTypes.cpp:653
bool isStruct() const
Definition CIRTypes.cpp:683
size_t getNumElements() const
Definition CIRTypes.h:188
static CharUnits fromQuantity(QuantityType Quantity)
Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:58
RecordLayoutAttr tryGetRecordLayout(mlir::ModuleOp mod, mlir::StringAttr name)
Same lookup as getRecordLayout, but returns a null attribute instead of asserting when the record has...
bool memberOwnsBytes(mlir::Type memberTy)
Whether a record member occupies bytes of its record.
Definition CIRTypes.h:126
cir::FuncType getIndirectCalleeType(cir::CIRCallOpInterface call)
The signature the indirect call call reaches its callee through.
AllocaOp getUnderlyingAlloca(mlir::Value addr)
The alloca that defines addr, looking through casts that preserve the underlying storage.
const internal::VariadicAllOfMatcher< Attr > attr
constexpr size_t align(size_t Size)
Aligns a size to the pointer alignment.
Definition PrimType.h:218
RangeSelector member(std::string ID)
Given a MemberExpr, selects the member token. ID is the node's binding in the match result.
U cast(CodeGen::Address addr)
Definition Address.h:327
__builtin_riscv_nclipru_u32 int64_t
static bool noaliasOnByvalAttr()