clang 22.0.0git
CGExprAgg.cpp
Go to the documentation of this file.
1//===--- CGExprAgg.cpp - Emit LLVM Code from Aggregate Expressions --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Aggregate Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCXXABI.h"
14#include "CGDebugInfo.h"
15#include "CGHLSLRuntime.h"
16#include "CGObjCRuntime.h"
17#include "CGRecordLayout.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "ConstantEmitter.h"
21#include "EHScopeStack.h"
22#include "TargetInfo.h"
24#include "clang/AST/Attr.h"
25#include "clang/AST/DeclCXX.h"
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/GlobalVariable.h"
31#include "llvm/IR/Instruction.h"
32#include "llvm/IR/IntrinsicInst.h"
33#include "llvm/IR/Intrinsics.h"
34using namespace clang;
35using namespace CodeGen;
36
37//===----------------------------------------------------------------------===//
38// Aggregate Expression Emitter
39//===----------------------------------------------------------------------===//
40
41namespace llvm {
42extern cl::opt<bool> EnableSingleByteCoverage;
43} // namespace llvm
44
45namespace {
46class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
47 CodeGenFunction &CGF;
48 CGBuilderTy &Builder;
49 AggValueSlot Dest;
50 bool IsResultUnused;
51
52 AggValueSlot EnsureSlot(QualType T) {
53 if (!Dest.isIgnored()) return Dest;
54 return CGF.CreateAggTemp(T, "agg.tmp.ensured");
55 }
56 void EnsureDest(QualType T) {
57 if (!Dest.isIgnored()) return;
58 Dest = CGF.CreateAggTemp(T, "agg.tmp.ensured");
59 }
60
61 // Calls `Fn` with a valid return value slot, potentially creating a temporary
62 // to do so. If a temporary is created, an appropriate copy into `Dest` will
63 // be emitted, as will lifetime markers.
64 //
65 // The given function should take a ReturnValueSlot, and return an RValue that
66 // points to said slot.
67 void withReturnValueSlot(const Expr *E,
68 llvm::function_ref<RValue(ReturnValueSlot)> Fn);
69
70 void DoZeroInitPadding(uint64_t &PaddingStart, uint64_t PaddingEnd,
71 const FieldDecl *NextField);
72
73public:
74 AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused)
75 : CGF(cgf), Builder(CGF.Builder), Dest(Dest),
76 IsResultUnused(IsResultUnused) { }
77
78 //===--------------------------------------------------------------------===//
79 // Utilities
80 //===--------------------------------------------------------------------===//
81
82 /// EmitAggLoadOfLValue - Given an expression with aggregate type that
83 /// represents a value lvalue, this method emits the address of the lvalue,
84 /// then loads the result into DestPtr.
85 void EmitAggLoadOfLValue(const Expr *E);
86
87 /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
88 /// SrcIsRValue is true if source comes from an RValue.
89 void EmitFinalDestCopy(QualType type, const LValue &src,
92 void EmitFinalDestCopy(QualType type, RValue src);
93 void EmitCopy(QualType type, const AggValueSlot &dest,
94 const AggValueSlot &src);
95
96 void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, QualType ArrayQTy,
97 Expr *ExprToVisit, ArrayRef<Expr *> Args,
98 Expr *ArrayFiller);
99
100 AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) {
101 if (CGF.getLangOpts().getGC() && TypeRequiresGCollection(T))
104 }
105
106 bool TypeRequiresGCollection(QualType T);
107
108 //===--------------------------------------------------------------------===//
109 // Visitor Methods
110 //===--------------------------------------------------------------------===//
111
112 void Visit(Expr *E) {
113 ApplyDebugLocation DL(CGF, E);
114 StmtVisitor<AggExprEmitter>::Visit(E);
115 }
116
117 void VisitStmt(Stmt *S) {
118 CGF.ErrorUnsupported(S, "aggregate expression");
119 }
120 void VisitParenExpr(ParenExpr *PE) { Visit(PE->getSubExpr()); }
121 void VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
122 Visit(GE->getResultExpr());
123 }
124 void VisitCoawaitExpr(CoawaitExpr *E) {
125 CGF.EmitCoawaitExpr(*E, Dest, IsResultUnused);
126 }
127 void VisitCoyieldExpr(CoyieldExpr *E) {
128 CGF.EmitCoyieldExpr(*E, Dest, IsResultUnused);
129 }
130 void VisitUnaryCoawait(UnaryOperator *E) { Visit(E->getSubExpr()); }
131 void VisitUnaryExtension(UnaryOperator *E) { Visit(E->getSubExpr()); }
132 void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) {
133 return Visit(E->getReplacement());
134 }
135
136 void VisitConstantExpr(ConstantExpr *E) {
137 EnsureDest(E->getType());
138
139 if (llvm::Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(E)) {
141 Result, Dest.getAddress(),
142 llvm::TypeSize::getFixed(
143 Dest.getPreferredSize(CGF.getContext(), E->getType())
144 .getQuantity()),
146 return;
147 }
148 return Visit(E->getSubExpr());
149 }
150
151 // l-values.
152 void VisitDeclRefExpr(DeclRefExpr *E) { EmitAggLoadOfLValue(E); }
153 void VisitMemberExpr(MemberExpr *ME) { EmitAggLoadOfLValue(ME); }
154 void VisitUnaryDeref(UnaryOperator *E) { EmitAggLoadOfLValue(E); }
155 void VisitStringLiteral(StringLiteral *E) { EmitAggLoadOfLValue(E); }
156 void VisitCompoundLiteralExpr(CompoundLiteralExpr *E);
157 void VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
158 EmitAggLoadOfLValue(E);
159 }
160 void VisitPredefinedExpr(const PredefinedExpr *E) {
161 EmitAggLoadOfLValue(E);
162 }
163
164 // Operators.
165 void VisitCastExpr(CastExpr *E);
166 void VisitCallExpr(const CallExpr *E);
167 void VisitStmtExpr(const StmtExpr *E);
168 void VisitBinaryOperator(const BinaryOperator *BO);
169 void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *BO);
170 void VisitBinAssign(const BinaryOperator *E);
171 void VisitBinComma(const BinaryOperator *E);
172 void VisitBinCmp(const BinaryOperator *E);
173 void VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *E) {
174 Visit(E->getSemanticForm());
175 }
176
177 void VisitObjCMessageExpr(ObjCMessageExpr *E);
178 void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
179 EmitAggLoadOfLValue(E);
180 }
181
182 void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E);
183 void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);
184 void VisitChooseExpr(const ChooseExpr *CE);
185 void VisitInitListExpr(InitListExpr *E);
186 void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,
187 FieldDecl *InitializedFieldInUnion,
188 Expr *ArrayFiller);
189 void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
190 llvm::Value *outerBegin = nullptr);
191 void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E);
192 void VisitNoInitExpr(NoInitExpr *E) { } // Do nothing.
193 void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
194 CodeGenFunction::CXXDefaultArgExprScope Scope(CGF, DAE);
195 Visit(DAE->getExpr());
196 }
197 void VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) {
198 CodeGenFunction::CXXDefaultInitExprScope Scope(CGF, DIE);
199 Visit(DIE->getExpr());
200 }
201 void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E);
202 void VisitCXXConstructExpr(const CXXConstructExpr *E);
203 void VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *E);
204 void VisitLambdaExpr(LambdaExpr *E);
205 void VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E);
206 void VisitExprWithCleanups(ExprWithCleanups *E);
207 void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E);
208 void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); }
209 void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E);
210 void VisitOpaqueValueExpr(OpaqueValueExpr *E);
211
212 void VisitPseudoObjectExpr(PseudoObjectExpr *E) {
213 if (E->isGLValue()) {
214 LValue LV = CGF.EmitPseudoObjectLValue(E);
215 return EmitFinalDestCopy(E->getType(), LV);
216 }
217
218 AggValueSlot Slot = EnsureSlot(E->getType());
219 bool NeedsDestruction =
220 !Slot.isExternallyDestructed() &&
222 if (NeedsDestruction)
224 CGF.EmitPseudoObjectRValue(E, Slot);
225 if (NeedsDestruction)
227 E->getType());
228 }
229
230 void VisitVAArgExpr(VAArgExpr *E);
231 void VisitCXXParenListInitExpr(CXXParenListInitExpr *E);
232 void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,
233 Expr *ArrayFiller);
234
235 void EmitInitializationToLValue(Expr *E, LValue Address);
236 void EmitNullInitializationToLValue(LValue Address);
237 // case Expr::ChooseExprClass:
238 void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); }
239 void VisitAtomicExpr(AtomicExpr *E) {
240 RValue Res = CGF.EmitAtomicExpr(E);
241 EmitFinalDestCopy(E->getType(), Res);
242 }
243 void VisitPackIndexingExpr(PackIndexingExpr *E) {
244 Visit(E->getSelectedExpr());
245 }
246};
247} // end anonymous namespace.
248
249//===----------------------------------------------------------------------===//
250// Utilities
251//===----------------------------------------------------------------------===//
252
253/// EmitAggLoadOfLValue - Given an expression with aggregate type that
254/// represents a value lvalue, this method emits the address of the lvalue,
255/// then loads the result into DestPtr.
256void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) {
257 LValue LV = CGF.EmitLValue(E);
258
259 // If the type of the l-value is atomic, then do an atomic load.
260 if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(LV)) {
261 CGF.EmitAtomicLoad(LV, E->getExprLoc(), Dest);
262 return;
263 }
264
265 EmitFinalDestCopy(E->getType(), LV);
266}
267
268/// True if the given aggregate type requires special GC API calls.
269bool AggExprEmitter::TypeRequiresGCollection(QualType T) {
270 // Only record types have members that might require garbage collection.
271 const auto *Record = T->getAsRecordDecl();
272 if (!Record)
273 return false;
274
275 // Don't mess with non-trivial C++ types.
277 (cast<CXXRecordDecl>(Record)->hasNonTrivialCopyConstructor() ||
278 !cast<CXXRecordDecl>(Record)->hasTrivialDestructor()))
279 return false;
280
281 // Check whether the type has an object member.
282 return Record->hasObjectMember();
283}
284
285void AggExprEmitter::withReturnValueSlot(
286 const Expr *E, llvm::function_ref<RValue(ReturnValueSlot)> EmitCall) {
287 QualType RetTy = E->getType();
288 bool RequiresDestruction =
289 !Dest.isExternallyDestructed() &&
291
292 // If it makes no observable difference, save a memcpy + temporary.
293 //
294 // We need to always provide our own temporary if destruction is required.
295 // Otherwise, EmitCall will emit its own, notice that it's "unused", and end
296 // its lifetime before we have the chance to emit a proper destructor call.
297 bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() ||
298 (RequiresDestruction && Dest.isIgnored());
299
300 Address RetAddr = Address::invalid();
301
302 EHScopeStack::stable_iterator LifetimeEndBlock;
303 llvm::IntrinsicInst *LifetimeStartInst = nullptr;
304 if (!UseTemp) {
305 RetAddr = Dest.getAddress();
306 } else {
307 RetAddr = CGF.CreateMemTempWithoutCast(RetTy, "tmp");
308 if (CGF.EmitLifetimeStart(RetAddr.getBasePointer())) {
309 LifetimeStartInst =
310 cast<llvm::IntrinsicInst>(std::prev(Builder.GetInsertPoint()));
311 assert(LifetimeStartInst->getIntrinsicID() ==
312 llvm::Intrinsic::lifetime_start &&
313 "Last insertion wasn't a lifetime.start?");
314
315 CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
316 NormalEHLifetimeMarker, RetAddr);
317 LifetimeEndBlock = CGF.EHStack.stable_begin();
318 }
319 }
320
321 RValue Src =
322 EmitCall(ReturnValueSlot(RetAddr, Dest.isVolatile(), IsResultUnused,
323 Dest.isExternallyDestructed()));
324
325 if (!UseTemp)
326 return;
327
328 assert(Dest.isIgnored() || Dest.emitRawPointer(CGF) !=
329 Src.getAggregatePointer(E->getType(), CGF));
330 EmitFinalDestCopy(E->getType(), Src);
331
332 if (!RequiresDestruction && LifetimeStartInst) {
333 // If there's no dtor to run, the copy was the last use of our temporary.
334 // Since we're not guaranteed to be in an ExprWithCleanups, clean up
335 // eagerly.
336 CGF.DeactivateCleanupBlock(LifetimeEndBlock, LifetimeStartInst);
337 CGF.EmitLifetimeEnd(RetAddr.getBasePointer());
338 }
339}
340
341/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
342void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) {
343 assert(src.isAggregate() && "value must be aggregate value!");
344 LValue srcLV = CGF.MakeAddrLValue(src.getAggregateAddress(), type);
345 EmitFinalDestCopy(type, srcLV, CodeGenFunction::EVK_RValue);
346}
347
348/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
349void AggExprEmitter::EmitFinalDestCopy(
350 QualType type, const LValue &src,
351 CodeGenFunction::ExprValueKind SrcValueKind) {
352 // If Dest is ignored, then we're evaluating an aggregate expression
353 // in a context that doesn't care about the result. Note that loads
354 // from volatile l-values force the existence of a non-ignored
355 // destination.
356 if (Dest.isIgnored())
357 return;
358
359 // Copy non-trivial C structs here.
360 LValue DstLV = CGF.MakeAddrLValue(
361 Dest.getAddress(), Dest.isVolatile() ? type.withVolatile() : type);
362
363 if (SrcValueKind == CodeGenFunction::EVK_RValue) {
364 if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) {
365 if (Dest.isPotentiallyAliased())
366 CGF.callCStructMoveAssignmentOperator(DstLV, src);
367 else
368 CGF.callCStructMoveConstructor(DstLV, src);
369 return;
370 }
371 } else {
372 if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) {
373 if (Dest.isPotentiallyAliased())
374 CGF.callCStructCopyAssignmentOperator(DstLV, src);
375 else
376 CGF.callCStructCopyConstructor(DstLV, src);
377 return;
378 }
379 }
380
381 AggValueSlot srcAgg = AggValueSlot::forLValue(
384 EmitCopy(type, Dest, srcAgg);
385}
386
387/// Perform a copy from the source into the destination.
388///
389/// \param type - the type of the aggregate being copied; qualifiers are
390/// ignored
391void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest,
392 const AggValueSlot &src) {
393 if (dest.requiresGCollection()) {
394 CharUnits sz = dest.getPreferredSize(CGF.getContext(), type);
395 llvm::Value *size = llvm::ConstantInt::get(CGF.SizeTy, sz.getQuantity());
397 dest.getAddress(),
398 src.getAddress(),
399 size);
400 return;
401 }
402
403 // If the result of the assignment is used, copy the LHS there also.
404 // It's volatile if either side is. Use the minimum alignment of
405 // the two sides.
406 LValue DestLV = CGF.MakeAddrLValue(dest.getAddress(), type);
407 LValue SrcLV = CGF.MakeAddrLValue(src.getAddress(), type);
408 CGF.EmitAggregateCopy(DestLV, SrcLV, type, dest.mayOverlap(),
409 dest.isVolatile() || src.isVolatile());
410}
411
412/// Emit the initializer for a std::initializer_list initialized with a
413/// real initializer list.
414void
415AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) {
416 // Emit an array containing the elements. The array is externally destructed
417 // if the std::initializer_list object is.
418 ASTContext &Ctx = CGF.getContext();
419 LValue Array = CGF.EmitLValue(E->getSubExpr());
420 assert(Array.isSimple() && "initializer_list array not a simple lvalue");
421 Address ArrayPtr = Array.getAddress();
422
423 const ConstantArrayType *ArrayType =
425 assert(ArrayType && "std::initializer_list constructed from non-array");
426
427 auto *Record = E->getType()->castAsRecordDecl();
428 RecordDecl::field_iterator Field = Record->field_begin();
429 assert(Field != Record->field_end() &&
430 Ctx.hasSameType(Field->getType()->getPointeeType(),
431 ArrayType->getElementType()) &&
432 "Expected std::initializer_list first field to be const E *");
433
434 // Start pointer.
435 AggValueSlot Dest = EnsureSlot(E->getType());
436 LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
437 LValue Start = CGF.EmitLValueForFieldInitialization(DestLV, *Field);
438 llvm::Value *ArrayStart = ArrayPtr.emitRawPointer(CGF);
439 CGF.EmitStoreThroughLValue(RValue::get(ArrayStart), Start);
440 ++Field;
441 assert(Field != Record->field_end() &&
442 "Expected std::initializer_list to have two fields");
443
444 llvm::Value *Size = Builder.getInt(ArrayType->getSize());
445 LValue EndOrLength = CGF.EmitLValueForFieldInitialization(DestLV, *Field);
446 if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) {
447 // Length.
448 CGF.EmitStoreThroughLValue(RValue::get(Size), EndOrLength);
449
450 } else {
451 // End pointer.
452 assert(Field->getType()->isPointerType() &&
453 Ctx.hasSameType(Field->getType()->getPointeeType(),
454 ArrayType->getElementType()) &&
455 "Expected std::initializer_list second field to be const E *");
456 llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0);
457 llvm::Value *IdxEnd[] = { Zero, Size };
458 llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP(
459 ArrayPtr.getElementType(), ArrayPtr.emitRawPointer(CGF), IdxEnd,
460 "arrayend");
461 CGF.EmitStoreThroughLValue(RValue::get(ArrayEnd), EndOrLength);
462 }
463
464 assert(++Field == Record->field_end() &&
465 "Expected std::initializer_list to only have two fields");
466}
467
468/// Determine if E is a trivial array filler, that is, one that is
469/// equivalent to zero-initialization.
470static bool isTrivialFiller(Expr *E) {
471 if (!E)
472 return true;
473
475 return true;
476
477 if (auto *ILE = dyn_cast<InitListExpr>(E)) {
478 if (ILE->getNumInits())
479 return false;
480 return isTrivialFiller(ILE->getArrayFiller());
481 }
482
483 if (auto *Cons = dyn_cast_or_null<CXXConstructExpr>(E))
484 return Cons->getConstructor()->isDefaultConstructor() &&
485 Cons->getConstructor()->isTrivial();
486
487 // FIXME: Are there other cases where we can avoid emitting an initializer?
488 return false;
489}
490
491// emit an elementwise cast where the RHS is a scalar or vector
492// or emit an aggregate splat cast
494 LValue DestVal,
495 llvm::Value *SrcVal,
496 QualType SrcTy,
497 SourceLocation Loc) {
498 // Flatten our destination
499 SmallVector<LValue, 16> StoreList;
500 CGF.FlattenAccessAndTypeLValue(DestVal, StoreList);
501
502 bool isVector = false;
503 if (auto *VT = SrcTy->getAs<VectorType>()) {
504 isVector = true;
505 SrcTy = VT->getElementType();
506 assert(StoreList.size() <= VT->getNumElements() &&
507 "Cannot perform HLSL flat cast when vector source \
508 object has less elements than flattened destination \
509 object.");
510 }
511
512 for (unsigned I = 0, Size = StoreList.size(); I < Size; I++) {
513 LValue DestLVal = StoreList[I];
514 llvm::Value *Load =
515 isVector ? CGF.Builder.CreateExtractElement(SrcVal, I, "vec.load")
516 : SrcVal;
517 llvm::Value *Cast =
518 CGF.EmitScalarConversion(Load, SrcTy, DestLVal.getType(), Loc);
519 CGF.EmitStoreThroughLValue(RValue::get(Cast), DestLVal);
520 }
521}
522
523// emit a flat cast where the RHS is an aggregate
524static void EmitHLSLElementwiseCast(CodeGenFunction &CGF, LValue DestVal,
525 LValue SrcVal, SourceLocation Loc) {
526 // Flatten our destination
527 SmallVector<LValue, 16> StoreList;
528 CGF.FlattenAccessAndTypeLValue(DestVal, StoreList);
529 // Flatten our src
531 CGF.FlattenAccessAndTypeLValue(SrcVal, LoadList);
532
533 assert(StoreList.size() <= LoadList.size() &&
534 "Cannot perform HLSL elementwise cast when flattened source object \
535 has less elements than flattened destination object.");
536 // apply casts to what we load from LoadList
537 // and store result in Dest
538 for (unsigned I = 0, E = StoreList.size(); I < E; I++) {
539 LValue DestLVal = StoreList[I];
540 LValue SrcLVal = LoadList[I];
541 RValue RVal = CGF.EmitLoadOfLValue(SrcLVal, Loc);
542 assert(RVal.isScalar() && "All flattened source values should be scalars");
543 llvm::Value *Val = RVal.getScalarVal();
544 llvm::Value *Cast = CGF.EmitScalarConversion(Val, SrcLVal.getType(),
545 DestLVal.getType(), Loc);
546 CGF.EmitStoreThroughLValue(RValue::get(Cast), DestLVal);
547 }
548}
549
550/// Emit initialization of an array from an initializer list. ExprToVisit must
551/// be either an InitListEpxr a CXXParenInitListExpr.
552void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
553 QualType ArrayQTy, Expr *ExprToVisit,
554 ArrayRef<Expr *> Args, Expr *ArrayFiller) {
555 uint64_t NumInitElements = Args.size();
556
557 uint64_t NumArrayElements = AType->getNumElements();
558 for (const auto *Init : Args) {
559 if (const auto *Embed = dyn_cast<EmbedExpr>(Init->IgnoreParenImpCasts())) {
560 NumInitElements += Embed->getDataElementCount() - 1;
561 if (NumInitElements > NumArrayElements) {
562 NumInitElements = NumArrayElements;
563 break;
564 }
565 }
566 }
567
568 assert(NumInitElements <= NumArrayElements);
569
570 QualType elementType =
571 CGF.getContext().getAsArrayType(ArrayQTy)->getElementType();
572 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);
573 CharUnits elementAlign =
574 DestPtr.getAlignment().alignmentOfArrayElement(elementSize);
575 llvm::Type *llvmElementType = CGF.ConvertTypeForMem(elementType);
576
577 // Consider initializing the array by copying from a global. For this to be
578 // more efficient than per-element initialization, the size of the elements
579 // with explicit initializers should be large enough.
580 if (NumInitElements * elementSize.getQuantity() > 16 &&
581 elementType.isTriviallyCopyableType(CGF.getContext())) {
582 CodeGen::CodeGenModule &CGM = CGF.CGM;
583 ConstantEmitter Emitter(CGF);
584 QualType GVArrayQTy = CGM.getContext().getAddrSpaceQualType(
585 CGM.getContext().removeAddrSpaceQualType(ArrayQTy),
587 LangAS AS = GVArrayQTy.getAddressSpace();
588 if (llvm::Constant *C =
589 Emitter.tryEmitForInitializer(ExprToVisit, AS, GVArrayQTy)) {
590 auto GV = new llvm::GlobalVariable(
591 CGM.getModule(), C->getType(),
592 /* isConstant= */ true, llvm::GlobalValue::PrivateLinkage, C,
593 "constinit",
594 /* InsertBefore= */ nullptr, llvm::GlobalVariable::NotThreadLocal,
596 Emitter.finalize(GV);
597 CharUnits Align = CGM.getContext().getTypeAlignInChars(GVArrayQTy);
598 GV->setAlignment(Align.getAsAlign());
599 Address GVAddr(GV, GV->getValueType(), Align);
600 EmitFinalDestCopy(ArrayQTy, CGF.MakeAddrLValue(GVAddr, GVArrayQTy));
601 return;
602 }
603 }
604
605 // Exception safety requires us to destroy all the
606 // already-constructed members if an initializer throws.
607 // For that, we'll need an EH cleanup.
608 QualType::DestructionKind dtorKind = elementType.isDestructedType();
609 Address endOfInit = Address::invalid();
610 CodeGenFunction::CleanupDeactivationScope deactivation(CGF);
611
612 llvm::Value *begin = DestPtr.emitRawPointer(CGF);
613 if (dtorKind) {
614 CodeGenFunction::AllocaTrackerRAII allocaTracker(CGF);
615 // In principle we could tell the cleanup where we are more
616 // directly, but the control flow can get so varied here that it
617 // would actually be quite complex. Therefore we go through an
618 // alloca.
619 llvm::Instruction *dominatingIP =
620 Builder.CreateFlagLoad(llvm::ConstantInt::getNullValue(CGF.Int8PtrTy));
621 endOfInit = CGF.CreateTempAlloca(begin->getType(), CGF.getPointerAlign(),
622 "arrayinit.endOfInit");
623 Builder.CreateStore(begin, endOfInit);
624 CGF.pushIrregularPartialArrayCleanup(begin, endOfInit, elementType,
625 elementAlign,
626 CGF.getDestroyer(dtorKind));
628 .AddAuxAllocas(allocaTracker.Take());
629
631 {CGF.EHStack.stable_begin(), dominatingIP});
632 }
633
634 llvm::Value *one = llvm::ConstantInt::get(CGF.SizeTy, 1);
635
636 auto Emit = [&](Expr *Init, uint64_t ArrayIndex) {
637 llvm::Value *element = begin;
638 if (ArrayIndex > 0) {
639 element = Builder.CreateInBoundsGEP(
640 llvmElementType, begin,
641 llvm::ConstantInt::get(CGF.SizeTy, ArrayIndex), "arrayinit.element");
642
643 // Tell the cleanup that it needs to destroy up to this
644 // element. TODO: some of these stores can be trivially
645 // observed to be unnecessary.
646 if (endOfInit.isValid())
647 Builder.CreateStore(element, endOfInit);
648 }
649
650 LValue elementLV = CGF.MakeAddrLValue(
651 Address(element, llvmElementType, elementAlign), elementType);
652 EmitInitializationToLValue(Init, elementLV);
653 return true;
654 };
655
656 unsigned ArrayIndex = 0;
657 // Emit the explicit initializers.
658 for (uint64_t i = 0; i != NumInitElements; ++i) {
659 if (ArrayIndex >= NumInitElements)
660 break;
661 if (auto *EmbedS = dyn_cast<EmbedExpr>(Args[i]->IgnoreParenImpCasts())) {
662 EmbedS->doForEachDataElement(Emit, ArrayIndex);
663 } else {
664 Emit(Args[i], ArrayIndex);
665 ArrayIndex++;
666 }
667 }
668
669 // Check whether there's a non-trivial array-fill expression.
670 bool hasTrivialFiller = isTrivialFiller(ArrayFiller);
671
672 // Any remaining elements need to be zero-initialized, possibly
673 // using the filler expression. We can skip this if the we're
674 // emitting to zeroed memory.
675 if (NumInitElements != NumArrayElements &&
676 !(Dest.isZeroed() && hasTrivialFiller &&
677 CGF.getTypes().isZeroInitializable(elementType))) {
678
679 // Use an actual loop. This is basically
680 // do { *array++ = filler; } while (array != end);
681
682 // Advance to the start of the rest of the array.
683 llvm::Value *element = begin;
684 if (NumInitElements) {
685 element = Builder.CreateInBoundsGEP(
686 llvmElementType, element,
687 llvm::ConstantInt::get(CGF.SizeTy, NumInitElements),
688 "arrayinit.start");
689 if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit);
690 }
691
692 // Compute the end of the array.
693 llvm::Value *end = Builder.CreateInBoundsGEP(
694 llvmElementType, begin,
695 llvm::ConstantInt::get(CGF.SizeTy, NumArrayElements), "arrayinit.end");
696
697 llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
698 llvm::BasicBlock *bodyBB = CGF.createBasicBlock("arrayinit.body");
699
700 // Jump into the body.
701 CGF.EmitBlock(bodyBB);
702 llvm::PHINode *currentElement =
703 Builder.CreatePHI(element->getType(), 2, "arrayinit.cur");
704 currentElement->addIncoming(element, entryBB);
705
706 // Emit the actual filler expression.
707 {
708 // C++1z [class.temporary]p5:
709 // when a default constructor is called to initialize an element of
710 // an array with no corresponding initializer [...] the destruction of
711 // every temporary created in a default argument is sequenced before
712 // the construction of the next array element, if any
713 CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
714 LValue elementLV = CGF.MakeAddrLValue(
715 Address(currentElement, llvmElementType, elementAlign), elementType);
716 if (ArrayFiller)
717 EmitInitializationToLValue(ArrayFiller, elementLV);
718 else
719 EmitNullInitializationToLValue(elementLV);
720 }
721
722 // Move on to the next element.
723 llvm::Value *nextElement = Builder.CreateInBoundsGEP(
724 llvmElementType, currentElement, one, "arrayinit.next");
725
726 // Tell the EH cleanup that we finished with the last element.
727 if (endOfInit.isValid()) Builder.CreateStore(nextElement, endOfInit);
728
729 // Leave the loop if we're done.
730 llvm::Value *done = Builder.CreateICmpEQ(nextElement, end,
731 "arrayinit.done");
732 llvm::BasicBlock *endBB = CGF.createBasicBlock("arrayinit.end");
733 Builder.CreateCondBr(done, endBB, bodyBB);
734 currentElement->addIncoming(nextElement, Builder.GetInsertBlock());
735
736 CGF.EmitBlock(endBB);
737 }
738}
739
740//===----------------------------------------------------------------------===//
741// Visitor Methods
742//===----------------------------------------------------------------------===//
743
744void AggExprEmitter::VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E){
745 Visit(E->getSubExpr());
746}
747
748void AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) {
749 // If this is a unique OVE, just visit its source expression.
750 if (e->isUnique())
751 Visit(e->getSourceExpr());
752 else
753 EmitFinalDestCopy(e->getType(), CGF.getOrCreateOpaqueLValueMapping(e));
754}
755
756void
757AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
758 if (Dest.isPotentiallyAliased() &&
759 E->getType().isPODType(CGF.getContext())) {
760 // For a POD type, just emit a load of the lvalue + a copy, because our
761 // compound literal might alias the destination.
762 EmitAggLoadOfLValue(E);
763 return;
764 }
765
766 AggValueSlot Slot = EnsureSlot(E->getType());
767
768 // Block-scope compound literals are destroyed at the end of the enclosing
769 // scope in C.
770 bool Destruct =
771 !CGF.getLangOpts().CPlusPlus && !Slot.isExternallyDestructed();
772 if (Destruct)
774
775 CGF.EmitAggExpr(E->getInitializer(), Slot);
776
777 if (Destruct)
780 CGF.getCleanupKind(DtorKind), Slot.getAddress(), E->getType(),
781 CGF.getDestroyer(DtorKind), DtorKind & EHCleanup);
782}
783
784/// Attempt to look through various unimportant expressions to find a
785/// cast of the given kind.
786static Expr *findPeephole(Expr *op, CastKind kind, const ASTContext &ctx) {
787 op = op->IgnoreParenNoopCasts(ctx);
788 if (auto castE = dyn_cast<CastExpr>(op)) {
789 if (castE->getCastKind() == kind)
790 return castE->getSubExpr();
791 }
792 return nullptr;
793}
794
795void AggExprEmitter::VisitCastExpr(CastExpr *E) {
796 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E))
797 CGF.CGM.EmitExplicitCastExprType(ECE, &CGF);
798 switch (E->getCastKind()) {
799 case CK_Dynamic: {
800 // FIXME: Can this actually happen? We have no test coverage for it.
801 assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?");
802 LValue LV = CGF.EmitCheckedLValue(E->getSubExpr(),
804 // FIXME: Do we also need to handle property references here?
805 if (LV.isSimple())
806 CGF.EmitDynamicCast(LV.getAddress(), cast<CXXDynamicCastExpr>(E));
807 else
808 CGF.CGM.ErrorUnsupported(E, "non-simple lvalue dynamic_cast");
809
810 if (!Dest.isIgnored())
811 CGF.CGM.ErrorUnsupported(E, "lvalue dynamic_cast with a destination");
812 break;
813 }
814
815 case CK_ToUnion: {
816 // Evaluate even if the destination is ignored.
817 if (Dest.isIgnored()) {
819 /*ignoreResult=*/true);
820 break;
821 }
822
823 // GCC union extension
824 QualType Ty = E->getSubExpr()->getType();
825 Address CastPtr = Dest.getAddress().withElementType(CGF.ConvertType(Ty));
826 EmitInitializationToLValue(E->getSubExpr(),
827 CGF.MakeAddrLValue(CastPtr, Ty));
828 break;
829 }
830
831 case CK_LValueToRValueBitCast: {
832 if (Dest.isIgnored()) {
834 /*ignoreResult=*/true);
835 break;
836 }
837
838 LValue SourceLV = CGF.EmitLValue(E->getSubExpr());
839 Address SourceAddress = SourceLV.getAddress().withElementType(CGF.Int8Ty);
840 Address DestAddress = Dest.getAddress().withElementType(CGF.Int8Ty);
841 llvm::Value *SizeVal = llvm::ConstantInt::get(
842 CGF.SizeTy,
844 Builder.CreateMemCpy(DestAddress, SourceAddress, SizeVal);
845 break;
846 }
847
848 case CK_DerivedToBase:
849 case CK_BaseToDerived:
850 case CK_UncheckedDerivedToBase: {
851 llvm_unreachable("cannot perform hierarchy conversion in EmitAggExpr: "
852 "should have been unpacked before we got here");
853 }
854
855 case CK_NonAtomicToAtomic:
856 case CK_AtomicToNonAtomic: {
857 bool isToAtomic = (E->getCastKind() == CK_NonAtomicToAtomic);
858
859 // Determine the atomic and value types.
860 QualType atomicType = E->getSubExpr()->getType();
861 QualType valueType = E->getType();
862 if (isToAtomic) std::swap(atomicType, valueType);
863
864 assert(atomicType->isAtomicType());
865 assert(CGF.getContext().hasSameUnqualifiedType(valueType,
866 atomicType->castAs<AtomicType>()->getValueType()));
867
868 // Just recurse normally if we're ignoring the result or the
869 // atomic type doesn't change representation.
870 if (Dest.isIgnored() || !CGF.CGM.isPaddedAtomicType(atomicType)) {
871 return Visit(E->getSubExpr());
872 }
873
874 CastKind peepholeTarget =
875 (isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic);
876
877 // These two cases are reverses of each other; try to peephole them.
878 if (Expr *op =
879 findPeephole(E->getSubExpr(), peepholeTarget, CGF.getContext())) {
880 assert(CGF.getContext().hasSameUnqualifiedType(op->getType(),
881 E->getType()) &&
882 "peephole significantly changed types?");
883 return Visit(op);
884 }
885
886 // If we're converting an r-value of non-atomic type to an r-value
887 // of atomic type, just emit directly into the relevant sub-object.
888 if (isToAtomic) {
889 AggValueSlot valueDest = Dest;
890 if (!valueDest.isIgnored() && CGF.CGM.isPaddedAtomicType(atomicType)) {
891 // Zero-initialize. (Strictly speaking, we only need to initialize
892 // the padding at the end, but this is simpler.)
893 if (!Dest.isZeroed())
895
896 // Build a GEP to refer to the subobject.
897 Address valueAddr =
898 CGF.Builder.CreateStructGEP(valueDest.getAddress(), 0);
899 valueDest = AggValueSlot::forAddr(valueAddr,
900 valueDest.getQualifiers(),
901 valueDest.isExternallyDestructed(),
902 valueDest.requiresGCollection(),
903 valueDest.isPotentiallyAliased(),
906 }
907
908 CGF.EmitAggExpr(E->getSubExpr(), valueDest);
909 return;
910 }
911
912 // Otherwise, we're converting an atomic type to a non-atomic type.
913 // Make an atomic temporary, emit into that, and then copy the value out.
914 AggValueSlot atomicSlot =
915 CGF.CreateAggTemp(atomicType, "atomic-to-nonatomic.temp");
916 CGF.EmitAggExpr(E->getSubExpr(), atomicSlot);
917
918 Address valueAddr = Builder.CreateStructGEP(atomicSlot.getAddress(), 0);
919 RValue rvalue = RValue::getAggregate(valueAddr, atomicSlot.isVolatile());
920 return EmitFinalDestCopy(valueType, rvalue);
921 }
922 case CK_AddressSpaceConversion:
923 return Visit(E->getSubExpr());
924
925 case CK_LValueToRValue:
926 // If we're loading from a volatile type, force the destination
927 // into existence.
928 if (E->getSubExpr()->getType().isVolatileQualified()) {
929 bool Destruct =
930 !Dest.isExternallyDestructed() &&
932 if (Destruct)
934 EnsureDest(E->getType());
935 Visit(E->getSubExpr());
936
937 if (Destruct)
939 E->getType());
940
941 return;
942 }
943
944 [[fallthrough]];
945
946 case CK_HLSLArrayRValue:
947 Visit(E->getSubExpr());
948 break;
949 case CK_HLSLAggregateSplatCast: {
950 Expr *Src = E->getSubExpr();
951 QualType SrcTy = Src->getType();
952 RValue RV = CGF.EmitAnyExpr(Src);
953 LValue DestLVal = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
954 SourceLocation Loc = E->getExprLoc();
955
956 assert(RV.isScalar() && SrcTy->isScalarType() &&
957 "RHS of HLSL splat cast must be a scalar.");
958 llvm::Value *SrcVal = RV.getScalarVal();
959 EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestLVal, SrcVal, SrcTy, Loc);
960 break;
961 }
962 case CK_HLSLElementwiseCast: {
963 Expr *Src = E->getSubExpr();
964 QualType SrcTy = Src->getType();
965 RValue RV = CGF.EmitAnyExpr(Src);
966 LValue DestLVal = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
967 SourceLocation Loc = E->getExprLoc();
968
969 if (RV.isScalar()) {
970 llvm::Value *SrcVal = RV.getScalarVal();
971 assert(SrcTy->isVectorType() &&
972 "HLSL Elementwise cast doesn't handle splatting.");
973 EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestLVal, SrcVal, SrcTy, Loc);
974 } else {
975 assert(RV.isAggregate() &&
976 "Can't perform HLSL Aggregate cast on a complex type.");
977 Address SrcVal = RV.getAggregateAddress();
978 EmitHLSLElementwiseCast(CGF, DestLVal, CGF.MakeAddrLValue(SrcVal, SrcTy),
979 Loc);
980 }
981 break;
982 }
983 case CK_NoOp:
984 case CK_UserDefinedConversion:
985 case CK_ConstructorConversion:
987 E->getType()) &&
988 "Implicit cast types must be compatible");
989 Visit(E->getSubExpr());
990 break;
991
992 case CK_LValueBitCast:
993 llvm_unreachable("should not be emitting lvalue bitcast as rvalue");
994
995 case CK_Dependent:
996 case CK_BitCast:
997 case CK_ArrayToPointerDecay:
998 case CK_FunctionToPointerDecay:
999 case CK_NullToPointer:
1000 case CK_NullToMemberPointer:
1001 case CK_BaseToDerivedMemberPointer:
1002 case CK_DerivedToBaseMemberPointer:
1003 case CK_MemberPointerToBoolean:
1004 case CK_ReinterpretMemberPointer:
1005 case CK_IntegralToPointer:
1006 case CK_PointerToIntegral:
1007 case CK_PointerToBoolean:
1008 case CK_ToVoid:
1009 case CK_VectorSplat:
1010 case CK_IntegralCast:
1011 case CK_BooleanToSignedIntegral:
1012 case CK_IntegralToBoolean:
1013 case CK_IntegralToFloating:
1014 case CK_FloatingToIntegral:
1015 case CK_FloatingToBoolean:
1016 case CK_FloatingCast:
1017 case CK_CPointerToObjCPointerCast:
1018 case CK_BlockPointerToObjCPointerCast:
1019 case CK_AnyPointerToBlockPointerCast:
1020 case CK_ObjCObjectLValueCast:
1021 case CK_FloatingRealToComplex:
1022 case CK_FloatingComplexToReal:
1023 case CK_FloatingComplexToBoolean:
1024 case CK_FloatingComplexCast:
1025 case CK_FloatingComplexToIntegralComplex:
1026 case CK_IntegralRealToComplex:
1027 case CK_IntegralComplexToReal:
1028 case CK_IntegralComplexToBoolean:
1029 case CK_IntegralComplexCast:
1030 case CK_IntegralComplexToFloatingComplex:
1031 case CK_ARCProduceObject:
1032 case CK_ARCConsumeObject:
1033 case CK_ARCReclaimReturnedObject:
1034 case CK_ARCExtendBlockObject:
1035 case CK_CopyAndAutoreleaseBlockObject:
1036 case CK_BuiltinFnToFnPtr:
1037 case CK_ZeroToOCLOpaqueType:
1038 case CK_MatrixCast:
1039 case CK_HLSLVectorTruncation:
1040
1041 case CK_IntToOCLSampler:
1042 case CK_FloatingToFixedPoint:
1043 case CK_FixedPointToFloating:
1044 case CK_FixedPointCast:
1045 case CK_FixedPointToBoolean:
1046 case CK_FixedPointToIntegral:
1047 case CK_IntegralToFixedPoint:
1048 llvm_unreachable("cast kind invalid for aggregate types");
1049 }
1050}
1051
1052void AggExprEmitter::VisitCallExpr(const CallExpr *E) {
1053 if (E->getCallReturnType(CGF.getContext())->isReferenceType()) {
1054 EmitAggLoadOfLValue(E);
1055 return;
1056 }
1057
1058 withReturnValueSlot(E, [&](ReturnValueSlot Slot) {
1059 return CGF.EmitCallExpr(E, Slot);
1060 });
1061}
1062
1063void AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
1064 withReturnValueSlot(E, [&](ReturnValueSlot Slot) {
1065 return CGF.EmitObjCMessageExpr(E, Slot);
1066 });
1067}
1068
1069void AggExprEmitter::VisitBinComma(const BinaryOperator *E) {
1070 CGF.EmitIgnoredExpr(E->getLHS());
1071 Visit(E->getRHS());
1072}
1073
1074void AggExprEmitter::VisitStmtExpr(const StmtExpr *E) {
1075 CodeGenFunction::StmtExprEvaluation eval(CGF);
1076 CGF.EmitCompoundStmt(*E->getSubStmt(), true, Dest);
1077}
1078
1084
1085static llvm::Value *EmitCompare(CGBuilderTy &Builder, CodeGenFunction &CGF,
1086 const BinaryOperator *E, llvm::Value *LHS,
1087 llvm::Value *RHS, CompareKind Kind,
1088 const char *NameSuffix = "") {
1089 QualType ArgTy = E->getLHS()->getType();
1090 if (const ComplexType *CT = ArgTy->getAs<ComplexType>())
1091 ArgTy = CT->getElementType();
1092
1093 if (const auto *MPT = ArgTy->getAs<MemberPointerType>()) {
1094 assert(Kind == CK_Equal &&
1095 "member pointers may only be compared for equality");
1097 CGF, LHS, RHS, MPT, /*IsInequality*/ false);
1098 }
1099
1100 // Compute the comparison instructions for the specified comparison kind.
1101 struct CmpInstInfo {
1102 const char *Name;
1103 llvm::CmpInst::Predicate FCmp;
1104 llvm::CmpInst::Predicate SCmp;
1105 llvm::CmpInst::Predicate UCmp;
1106 };
1107 CmpInstInfo InstInfo = [&]() -> CmpInstInfo {
1108 using FI = llvm::FCmpInst;
1109 using II = llvm::ICmpInst;
1110 switch (Kind) {
1111 case CK_Less:
1112 return {"cmp.lt", FI::FCMP_OLT, II::ICMP_SLT, II::ICMP_ULT};
1113 case CK_Greater:
1114 return {"cmp.gt", FI::FCMP_OGT, II::ICMP_SGT, II::ICMP_UGT};
1115 case CK_Equal:
1116 return {"cmp.eq", FI::FCMP_OEQ, II::ICMP_EQ, II::ICMP_EQ};
1117 }
1118 llvm_unreachable("Unrecognised CompareKind enum");
1119 }();
1120
1121 if (ArgTy->hasFloatingRepresentation())
1122 return Builder.CreateFCmp(InstInfo.FCmp, LHS, RHS,
1123 llvm::Twine(InstInfo.Name) + NameSuffix);
1124 if (ArgTy->isIntegralOrEnumerationType() || ArgTy->isPointerType()) {
1125 auto Inst =
1126 ArgTy->hasSignedIntegerRepresentation() ? InstInfo.SCmp : InstInfo.UCmp;
1127 return Builder.CreateICmp(Inst, LHS, RHS,
1128 llvm::Twine(InstInfo.Name) + NameSuffix);
1129 }
1130
1131 llvm_unreachable("unsupported aggregate binary expression should have "
1132 "already been handled");
1133}
1134
1135void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) {
1136 using llvm::BasicBlock;
1137 using llvm::PHINode;
1138 using llvm::Value;
1139 assert(CGF.getContext().hasSameType(E->getLHS()->getType(),
1140 E->getRHS()->getType()));
1141 const ComparisonCategoryInfo &CmpInfo =
1143 assert(CmpInfo.Record->isTriviallyCopyable() &&
1144 "cannot copy non-trivially copyable aggregate");
1145
1146 QualType ArgTy = E->getLHS()->getType();
1147
1148 if (!ArgTy->isIntegralOrEnumerationType() && !ArgTy->isRealFloatingType() &&
1149 !ArgTy->isNullPtrType() && !ArgTy->isPointerType() &&
1150 !ArgTy->isMemberPointerType() && !ArgTy->isAnyComplexType()) {
1151 return CGF.ErrorUnsupported(E, "aggregate three-way comparison");
1152 }
1153 bool IsComplex = ArgTy->isAnyComplexType();
1154
1155 // Evaluate the operands to the expression and extract their values.
1156 auto EmitOperand = [&](Expr *E) -> std::pair<Value *, Value *> {
1157 RValue RV = CGF.EmitAnyExpr(E);
1158 if (RV.isScalar())
1159 return {RV.getScalarVal(), nullptr};
1160 if (RV.isAggregate())
1161 return {RV.getAggregatePointer(E->getType(), CGF), nullptr};
1162 assert(RV.isComplex());
1163 return RV.getComplexVal();
1164 };
1165 auto LHSValues = EmitOperand(E->getLHS()),
1166 RHSValues = EmitOperand(E->getRHS());
1167
1168 auto EmitCmp = [&](CompareKind K) {
1169 Value *Cmp = EmitCompare(Builder, CGF, E, LHSValues.first, RHSValues.first,
1170 K, IsComplex ? ".r" : "");
1171 if (!IsComplex)
1172 return Cmp;
1173 assert(K == CompareKind::CK_Equal);
1174 Value *CmpImag = EmitCompare(Builder, CGF, E, LHSValues.second,
1175 RHSValues.second, K, ".i");
1176 return Builder.CreateAnd(Cmp, CmpImag, "and.eq");
1177 };
1178 auto EmitCmpRes = [&](const ComparisonCategoryInfo::ValueInfo *VInfo) {
1179 return Builder.getInt(VInfo->getIntValue());
1180 };
1181
1182 Value *Select;
1183 if (ArgTy->isNullPtrType()) {
1184 Select = EmitCmpRes(CmpInfo.getEqualOrEquiv());
1185 } else if (!CmpInfo.isPartial()) {
1186 Value *SelectOne =
1187 Builder.CreateSelect(EmitCmp(CK_Less), EmitCmpRes(CmpInfo.getLess()),
1188 EmitCmpRes(CmpInfo.getGreater()), "sel.lt");
1189 Select = Builder.CreateSelect(EmitCmp(CK_Equal),
1190 EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1191 SelectOne, "sel.eq");
1192 } else {
1193 Value *SelectEq = Builder.CreateSelect(
1194 EmitCmp(CK_Equal), EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1195 EmitCmpRes(CmpInfo.getUnordered()), "sel.eq");
1196 Value *SelectGT = Builder.CreateSelect(EmitCmp(CK_Greater),
1197 EmitCmpRes(CmpInfo.getGreater()),
1198 SelectEq, "sel.gt");
1199 Select = Builder.CreateSelect(
1200 EmitCmp(CK_Less), EmitCmpRes(CmpInfo.getLess()), SelectGT, "sel.lt");
1201 }
1202 // Create the return value in the destination slot.
1203 EnsureDest(E->getType());
1204 LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
1205
1206 // Emit the address of the first (and only) field in the comparison category
1207 // type, and initialize it from the constant integer value selected above.
1208 LValue FieldLV = CGF.EmitLValueForFieldInitialization(
1209 DestLV, *CmpInfo.Record->field_begin());
1210 CGF.EmitStoreThroughLValue(RValue::get(Select), FieldLV, /*IsInit*/ true);
1211
1212 // All done! The result is in the Dest slot.
1213}
1214
1215void AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) {
1216 if (E->getOpcode() == BO_PtrMemD || E->getOpcode() == BO_PtrMemI)
1217 VisitPointerToDataMemberBinaryOperator(E);
1218 else
1219 CGF.ErrorUnsupported(E, "aggregate binary expression");
1220}
1221
1222void AggExprEmitter::VisitPointerToDataMemberBinaryOperator(
1223 const BinaryOperator *E) {
1224 LValue LV = CGF.EmitPointerToDataMemberBinaryExpr(E);
1225 EmitFinalDestCopy(E->getType(), LV);
1226}
1227
1228/// Is the value of the given expression possibly a reference to or
1229/// into a __block variable?
1230static bool isBlockVarRef(const Expr *E) {
1231 // Make sure we look through parens.
1232 E = E->IgnoreParens();
1233
1234 // Check for a direct reference to a __block variable.
1235 if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E)) {
1236 const VarDecl *var = dyn_cast<VarDecl>(DRE->getDecl());
1237 return (var && var->hasAttr<BlocksAttr>());
1238 }
1239
1240 // More complicated stuff.
1241
1242 // Binary operators.
1243 if (const BinaryOperator *op = dyn_cast<BinaryOperator>(E)) {
1244 // For an assignment or pointer-to-member operation, just care
1245 // about the LHS.
1246 if (op->isAssignmentOp() || op->isPtrMemOp())
1247 return isBlockVarRef(op->getLHS());
1248
1249 // For a comma, just care about the RHS.
1250 if (op->getOpcode() == BO_Comma)
1251 return isBlockVarRef(op->getRHS());
1252
1253 // FIXME: pointer arithmetic?
1254 return false;
1255
1256 // Check both sides of a conditional operator.
1257 } else if (const AbstractConditionalOperator *op
1258 = dyn_cast<AbstractConditionalOperator>(E)) {
1259 return isBlockVarRef(op->getTrueExpr())
1260 || isBlockVarRef(op->getFalseExpr());
1261
1262 // OVEs are required to support BinaryConditionalOperators.
1263 } else if (const OpaqueValueExpr *op
1264 = dyn_cast<OpaqueValueExpr>(E)) {
1265 if (const Expr *src = op->getSourceExpr())
1266 return isBlockVarRef(src);
1267
1268 // Casts are necessary to get things like (*(int*)&var) = foo().
1269 // We don't really care about the kind of cast here, except
1270 // we don't want to look through l2r casts, because it's okay
1271 // to get the *value* in a __block variable.
1272 } else if (const CastExpr *cast = dyn_cast<CastExpr>(E)) {
1273 if (cast->getCastKind() == CK_LValueToRValue)
1274 return false;
1275 return isBlockVarRef(cast->getSubExpr());
1276
1277 // Handle unary operators. Again, just aggressively look through
1278 // it, ignoring the operation.
1279 } else if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E)) {
1280 return isBlockVarRef(uop->getSubExpr());
1281
1282 // Look into the base of a field access.
1283 } else if (const MemberExpr *mem = dyn_cast<MemberExpr>(E)) {
1284 return isBlockVarRef(mem->getBase());
1285
1286 // Look into the base of a subscript.
1287 } else if (const ArraySubscriptExpr *sub = dyn_cast<ArraySubscriptExpr>(E)) {
1288 return isBlockVarRef(sub->getBase());
1289 }
1290
1291 return false;
1292}
1293
1294void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
1295 ApplyAtomGroup Grp(CGF.getDebugInfo());
1296 // For an assignment to work, the value on the right has
1297 // to be compatible with the value on the left.
1298 assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(),
1299 E->getRHS()->getType())
1300 && "Invalid assignment");
1301
1302 // If the LHS might be a __block variable, and the RHS can
1303 // potentially cause a block copy, we need to evaluate the RHS first
1304 // so that the assignment goes the right place.
1305 // This is pretty semantically fragile.
1306 if (isBlockVarRef(E->getLHS()) &&
1307 E->getRHS()->HasSideEffects(CGF.getContext())) {
1308 // Ensure that we have a destination, and evaluate the RHS into that.
1309 EnsureDest(E->getRHS()->getType());
1310 Visit(E->getRHS());
1311
1312 // Now emit the LHS and copy into it.
1313 LValue LHS = CGF.EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store);
1314
1315 // That copy is an atomic copy if the LHS is atomic.
1316 if (LHS.getType()->isAtomicType() ||
1318 CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false);
1319 return;
1320 }
1321
1322 EmitCopy(E->getLHS()->getType(),
1324 needsGC(E->getLHS()->getType()),
1327 Dest);
1328 return;
1329 }
1330
1331 LValue LHS = CGF.EmitLValue(E->getLHS());
1332
1333 // If we have an atomic type, evaluate into the destination and then
1334 // do an atomic copy.
1335 if (LHS.getType()->isAtomicType() ||
1337 EnsureDest(E->getRHS()->getType());
1338 Visit(E->getRHS());
1339 CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false);
1340 return;
1341 }
1342
1343 // Codegen the RHS so that it stores directly into the LHS.
1344 AggValueSlot LHSSlot = AggValueSlot::forLValue(
1345 LHS, AggValueSlot::IsDestructed, needsGC(E->getLHS()->getType()),
1347 // A non-volatile aggregate destination might have volatile member.
1348 if (!LHSSlot.isVolatile() &&
1349 CGF.hasVolatileMember(E->getLHS()->getType()))
1350 LHSSlot.setVolatile(true);
1351
1352 CGF.EmitAggExpr(E->getRHS(), LHSSlot);
1353
1354 // Copy into the destination if the assignment isn't ignored.
1355 EmitFinalDestCopy(E->getType(), LHS);
1356
1357 if (!Dest.isIgnored() && !Dest.isExternallyDestructed() &&
1360 E->getType());
1361}
1362
1363void AggExprEmitter::
1364VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
1365 llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true");
1366 llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("cond.false");
1367 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");
1368
1369 // Bind the common expression if necessary.
1370 CodeGenFunction::OpaqueValueMapping binding(CGF, E);
1371
1372 CodeGenFunction::ConditionalEvaluation eval(CGF);
1373 CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock,
1374 CGF.getProfileCount(E));
1375
1376 // Save whether the destination's lifetime is externally managed.
1377 bool isExternallyDestructed = Dest.isExternallyDestructed();
1378 bool destructNonTrivialCStruct =
1379 !isExternallyDestructed &&
1381 isExternallyDestructed |= destructNonTrivialCStruct;
1382 Dest.setExternallyDestructed(isExternallyDestructed);
1383
1384 eval.begin(CGF);
1385 CGF.EmitBlock(LHSBlock);
1388 else
1390 Visit(E->getTrueExpr());
1391 eval.end(CGF);
1392
1393 assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!");
1394 CGF.Builder.CreateBr(ContBlock);
1395
1396 // If the result of an agg expression is unused, then the emission
1397 // of the LHS might need to create a destination slot. That's fine
1398 // with us, and we can safely emit the RHS into the same slot, but
1399 // we shouldn't claim that it's already being destructed.
1400 Dest.setExternallyDestructed(isExternallyDestructed);
1401
1402 eval.begin(CGF);
1403 CGF.EmitBlock(RHSBlock);
1406 Visit(E->getFalseExpr());
1407 eval.end(CGF);
1408
1409 if (destructNonTrivialCStruct)
1411 E->getType());
1412
1413 CGF.EmitBlock(ContBlock);
1416}
1417
1418void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) {
1419 Visit(CE->getChosenSubExpr());
1420}
1421
1422void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
1423 Address ArgValue = Address::invalid();
1424 CGF.EmitVAArg(VE, ArgValue, Dest);
1425
1426 // If EmitVAArg fails, emit an error.
1427 if (!ArgValue.isValid()) {
1428 CGF.ErrorUnsupported(VE, "aggregate va_arg expression");
1429 return;
1430 }
1431}
1432
1433void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
1434 // Ensure that we have a slot, but if we already do, remember
1435 // whether it was externally destructed.
1436 bool wasExternallyDestructed = Dest.isExternallyDestructed();
1437 EnsureDest(E->getType());
1438
1439 // We're going to push a destructor if there isn't already one.
1441
1442 Visit(E->getSubExpr());
1443
1444 // Push that destructor we promised.
1445 if (!wasExternallyDestructed)
1446 CGF.EmitCXXTemporary(E->getTemporary(), E->getType(), Dest.getAddress());
1447}
1448
1449void
1450AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) {
1451 AggValueSlot Slot = EnsureSlot(E->getType());
1452 CGF.EmitCXXConstructExpr(E, Slot);
1453}
1454
1455void AggExprEmitter::VisitCXXInheritedCtorInitExpr(
1456 const CXXInheritedCtorInitExpr *E) {
1457 AggValueSlot Slot = EnsureSlot(E->getType());
1459 E->getConstructor(), E->constructsVBase(), Slot.getAddress(),
1460 E->inheritedFromVBase(), E);
1461}
1462
1463void
1464AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) {
1465 AggValueSlot Slot = EnsureSlot(E->getType());
1466 LValue SlotLV = CGF.MakeAddrLValue(Slot.getAddress(), E->getType());
1467
1468 // We'll need to enter cleanup scopes in case any of the element
1469 // initializers throws an exception or contains branch out of the expressions.
1470 CodeGenFunction::CleanupDeactivationScope scope(CGF);
1471
1472 CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin();
1474 e = E->capture_init_end();
1475 i != e; ++i, ++CurField) {
1476 // Emit initialization
1477 LValue LV = CGF.EmitLValueForFieldInitialization(SlotLV, *CurField);
1478 if (CurField->hasCapturedVLAType()) {
1479 CGF.EmitLambdaVLACapture(CurField->getCapturedVLAType(), LV);
1480 continue;
1481 }
1482
1483 EmitInitializationToLValue(*i, LV);
1484
1485 // Push a destructor if necessary.
1486 if (QualType::DestructionKind DtorKind =
1487 CurField->getType().isDestructedType()) {
1488 assert(LV.isSimple());
1489 if (DtorKind)
1491 CurField->getType(),
1492 CGF.getDestroyer(DtorKind), false);
1493 }
1494 }
1495}
1496
1497void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) {
1498 CodeGenFunction::RunCleanupsScope cleanups(CGF);
1499 Visit(E->getSubExpr());
1500}
1501
1502void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
1503 QualType T = E->getType();
1504 AggValueSlot Slot = EnsureSlot(T);
1505 EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T));
1506}
1507
1508void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
1509 QualType T = E->getType();
1510 AggValueSlot Slot = EnsureSlot(T);
1511 EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T));
1512}
1513
1514/// Determine whether the given cast kind is known to always convert values
1515/// with all zero bits in their value representation to values with all zero
1516/// bits in their value representation.
1517static bool castPreservesZero(const CastExpr *CE) {
1518 switch (CE->getCastKind()) {
1519 // No-ops.
1520 case CK_NoOp:
1521 case CK_UserDefinedConversion:
1522 case CK_ConstructorConversion:
1523 case CK_BitCast:
1524 case CK_ToUnion:
1525 case CK_ToVoid:
1526 // Conversions between (possibly-complex) integral, (possibly-complex)
1527 // floating-point, and bool.
1528 case CK_BooleanToSignedIntegral:
1529 case CK_FloatingCast:
1530 case CK_FloatingComplexCast:
1531 case CK_FloatingComplexToBoolean:
1532 case CK_FloatingComplexToIntegralComplex:
1533 case CK_FloatingComplexToReal:
1534 case CK_FloatingRealToComplex:
1535 case CK_FloatingToBoolean:
1536 case CK_FloatingToIntegral:
1537 case CK_IntegralCast:
1538 case CK_IntegralComplexCast:
1539 case CK_IntegralComplexToBoolean:
1540 case CK_IntegralComplexToFloatingComplex:
1541 case CK_IntegralComplexToReal:
1542 case CK_IntegralRealToComplex:
1543 case CK_IntegralToBoolean:
1544 case CK_IntegralToFloating:
1545 // Reinterpreting integers as pointers and vice versa.
1546 case CK_IntegralToPointer:
1547 case CK_PointerToIntegral:
1548 // Language extensions.
1549 case CK_VectorSplat:
1550 case CK_MatrixCast:
1551 case CK_NonAtomicToAtomic:
1552 case CK_AtomicToNonAtomic:
1553 case CK_HLSLVectorTruncation:
1554 case CK_HLSLElementwiseCast:
1555 case CK_HLSLAggregateSplatCast:
1556 return true;
1557
1558 case CK_BaseToDerivedMemberPointer:
1559 case CK_DerivedToBaseMemberPointer:
1560 case CK_MemberPointerToBoolean:
1561 case CK_NullToMemberPointer:
1562 case CK_ReinterpretMemberPointer:
1563 // FIXME: ABI-dependent.
1564 return false;
1565
1566 case CK_AnyPointerToBlockPointerCast:
1567 case CK_BlockPointerToObjCPointerCast:
1568 case CK_CPointerToObjCPointerCast:
1569 case CK_ObjCObjectLValueCast:
1570 case CK_IntToOCLSampler:
1571 case CK_ZeroToOCLOpaqueType:
1572 // FIXME: Check these.
1573 return false;
1574
1575 case CK_FixedPointCast:
1576 case CK_FixedPointToBoolean:
1577 case CK_FixedPointToFloating:
1578 case CK_FixedPointToIntegral:
1579 case CK_FloatingToFixedPoint:
1580 case CK_IntegralToFixedPoint:
1581 // FIXME: Do all fixed-point types represent zero as all 0 bits?
1582 return false;
1583
1584 case CK_AddressSpaceConversion:
1585 case CK_BaseToDerived:
1586 case CK_DerivedToBase:
1587 case CK_Dynamic:
1588 case CK_NullToPointer:
1589 case CK_PointerToBoolean:
1590 // FIXME: Preserves zeroes only if zero pointers and null pointers have the
1591 // same representation in all involved address spaces.
1592 return false;
1593
1594 case CK_ARCConsumeObject:
1595 case CK_ARCExtendBlockObject:
1596 case CK_ARCProduceObject:
1597 case CK_ARCReclaimReturnedObject:
1598 case CK_CopyAndAutoreleaseBlockObject:
1599 case CK_ArrayToPointerDecay:
1600 case CK_FunctionToPointerDecay:
1601 case CK_BuiltinFnToFnPtr:
1602 case CK_Dependent:
1603 case CK_LValueBitCast:
1604 case CK_LValueToRValue:
1605 case CK_LValueToRValueBitCast:
1606 case CK_UncheckedDerivedToBase:
1607 case CK_HLSLArrayRValue:
1608 return false;
1609 }
1610 llvm_unreachable("Unhandled clang::CastKind enum");
1611}
1612
1613/// isSimpleZero - If emitting this value will obviously just cause a store of
1614/// zero to memory, return true. This can return false if uncertain, so it just
1615/// handles simple cases.
1616static bool isSimpleZero(const Expr *E, CodeGenFunction &CGF) {
1617 E = E->IgnoreParens();
1618 while (auto *CE = dyn_cast<CastExpr>(E)) {
1619 if (!castPreservesZero(CE))
1620 break;
1621 E = CE->getSubExpr()->IgnoreParens();
1622 }
1623
1624 // 0
1625 if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(E))
1626 return IL->getValue() == 0;
1627 // +0.0
1628 if (const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(E))
1629 return FL->getValue().isPosZero();
1630 // int()
1633 return true;
1634 // (int*)0 - Null pointer expressions.
1635 if (const CastExpr *ICE = dyn_cast<CastExpr>(E))
1636 return ICE->getCastKind() == CK_NullToPointer &&
1638 !E->HasSideEffects(CGF.getContext());
1639 // '\0'
1640 if (const CharacterLiteral *CL = dyn_cast<CharacterLiteral>(E))
1641 return CL->getValue() == 0;
1642
1643 // Otherwise, hard case: conservatively return false.
1644 return false;
1645}
1646
1647
1648void
1649AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) {
1650 QualType type = LV.getType();
1651 // FIXME: Ignore result?
1652 // FIXME: Are initializers affected by volatile?
1653 if (Dest.isZeroed() && isSimpleZero(E, CGF)) {
1654 // Storing "i32 0" to a zero'd memory location is a noop.
1655 return;
1657 return EmitNullInitializationToLValue(LV);
1658 } else if (isa<NoInitExpr>(E)) {
1659 // Do nothing.
1660 return;
1661 } else if (type->isReferenceType()) {
1662 RValue RV = CGF.EmitReferenceBindingToExpr(E);
1663 return CGF.EmitStoreThroughLValue(RV, LV);
1664 }
1665
1666 CGF.EmitInitializationToLValue(E, LV, Dest.isZeroed());
1667}
1668
1669void AggExprEmitter::EmitNullInitializationToLValue(LValue lv) {
1670 QualType type = lv.getType();
1671
1672 // If the destination slot is already zeroed out before the aggregate is
1673 // copied into it, we don't have to emit any zeros here.
1674 if (Dest.isZeroed() && CGF.getTypes().isZeroInitializable(type))
1675 return;
1676
1677 if (CGF.hasScalarEvaluationKind(type)) {
1678 // For non-aggregates, we can store the appropriate null constant.
1679 llvm::Value *null = CGF.CGM.EmitNullConstant(type);
1680 // Note that the following is not equivalent to
1681 // EmitStoreThroughBitfieldLValue for ARC types.
1682 if (lv.isBitField()) {
1684 } else {
1685 assert(lv.isSimple());
1686 CGF.EmitStoreOfScalar(null, lv, /* isInitialization */ true);
1687 }
1688 } else {
1689 // There's a potential optimization opportunity in combining
1690 // memsets; that would be easy for arrays, but relatively
1691 // difficult for structures with the current code.
1692 CGF.EmitNullInitialization(lv.getAddress(), lv.getType());
1693 }
1694}
1695
1696void AggExprEmitter::VisitCXXParenListInitExpr(CXXParenListInitExpr *E) {
1697 VisitCXXParenListOrInitListExpr(E, E->getInitExprs(),
1699 E->getArrayFiller());
1700}
1701
1702void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
1703 if (E->hadArrayRangeDesignator())
1704 CGF.ErrorUnsupported(E, "GNU array range designator extension");
1705
1706 if (E->isTransparent())
1707 return Visit(E->getInit(0));
1708
1709 VisitCXXParenListOrInitListExpr(
1710 E, E->inits(), E->getInitializedFieldInUnion(), E->getArrayFiller());
1711}
1712
1713void AggExprEmitter::VisitCXXParenListOrInitListExpr(
1714 Expr *ExprToVisit, ArrayRef<Expr *> InitExprs,
1715 FieldDecl *InitializedFieldInUnion, Expr *ArrayFiller) {
1716#if 0
1717 // FIXME: Assess perf here? Figure out what cases are worth optimizing here
1718 // (Length of globals? Chunks of zeroed-out space?).
1719 //
1720 // If we can, prefer a copy from a global; this is a lot less code for long
1721 // globals, and it's easier for the current optimizers to analyze.
1722 if (llvm::Constant *C =
1723 CGF.CGM.EmitConstantExpr(ExprToVisit, ExprToVisit->getType(), &CGF)) {
1724 llvm::GlobalVariable* GV =
1725 new llvm::GlobalVariable(CGF.CGM.getModule(), C->getType(), true,
1726 llvm::GlobalValue::InternalLinkage, C, "");
1727 EmitFinalDestCopy(ExprToVisit->getType(),
1728 CGF.MakeAddrLValue(GV, ExprToVisit->getType()));
1729 return;
1730 }
1731#endif
1732
1733 // HLSL initialization lists in the AST are an expansion which can contain
1734 // side-effecting expressions wrapped in opaque value expressions. To properly
1735 // emit these we need to emit the opaque values before we emit the argument
1736 // expressions themselves. This is a little hacky, but it prevents us needing
1737 // to do a bigger AST-level change for a language feature that we need
1738 // deprecate in the near future. See related HLSL language proposals:
1739 // * 0005-strict-initializer-lists.md
1740 // * https://github.com/microsoft/hlsl-specs/pull/325
1741 if (CGF.getLangOpts().HLSL && isa<InitListExpr>(ExprToVisit))
1743 CGF, cast<InitListExpr>(ExprToVisit));
1744
1745 AggValueSlot Dest = EnsureSlot(ExprToVisit->getType());
1746
1747 LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), ExprToVisit->getType());
1748
1749 // Handle initialization of an array.
1750 if (ExprToVisit->getType()->isConstantArrayType()) {
1751 auto AType = cast<llvm::ArrayType>(Dest.getAddress().getElementType());
1752 EmitArrayInit(Dest.getAddress(), AType, ExprToVisit->getType(), ExprToVisit,
1753 InitExprs, ArrayFiller);
1754 return;
1755 } else if (ExprToVisit->getType()->isVariableArrayType()) {
1756 // A variable array type that has an initializer can only do empty
1757 // initialization. And because this feature is not exposed as an extension
1758 // in C++, we can safely memset the array memory to zero.
1759 assert(InitExprs.size() == 0 &&
1760 "you can only use an empty initializer with VLAs");
1761 CGF.EmitNullInitialization(Dest.getAddress(), ExprToVisit->getType());
1762 return;
1763 }
1764
1765 assert(ExprToVisit->getType()->isRecordType() &&
1766 "Only support structs/unions here!");
1767
1768 // Do struct initialization; this code just sets each individual member
1769 // to the approprate value. This makes bitfield support automatic;
1770 // the disadvantage is that the generated code is more difficult for
1771 // the optimizer, especially with bitfields.
1772 unsigned NumInitElements = InitExprs.size();
1773 RecordDecl *record = ExprToVisit->getType()->castAsRecordDecl();
1774
1775 // We'll need to enter cleanup scopes in case any of the element
1776 // initializers throws an exception.
1777 CodeGenFunction::CleanupDeactivationScope DeactivateCleanups(CGF);
1778
1779 unsigned curInitIndex = 0;
1780
1781 // Emit initialization of base classes.
1782 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(record)) {
1783 assert(NumInitElements >= CXXRD->getNumBases() &&
1784 "missing initializer for base class");
1785 for (auto &Base : CXXRD->bases()) {
1786 assert(!Base.isVirtual() && "should not see vbases here");
1787 auto *BaseRD = Base.getType()->getAsCXXRecordDecl();
1789 Dest.getAddress(), CXXRD, BaseRD,
1790 /*isBaseVirtual*/ false);
1791 AggValueSlot AggSlot = AggValueSlot::forAddr(
1792 V, Qualifiers(),
1796 CGF.getOverlapForBaseInit(CXXRD, BaseRD, Base.isVirtual()));
1797 CGF.EmitAggExpr(InitExprs[curInitIndex++], AggSlot);
1798
1799 if (QualType::DestructionKind dtorKind =
1800 Base.getType().isDestructedType())
1801 CGF.pushDestroyAndDeferDeactivation(dtorKind, V, Base.getType());
1802 }
1803 }
1804
1805 // Prepare a 'this' for CXXDefaultInitExprs.
1806 CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress());
1807
1808 const bool ZeroInitPadding =
1809 CGF.CGM.shouldZeroInitPadding() && !Dest.isZeroed();
1810
1811 if (record->isUnion()) {
1812 // Only initialize one field of a union. The field itself is
1813 // specified by the initializer list.
1814 if (!InitializedFieldInUnion) {
1815 // Empty union; we have nothing to do.
1816
1817#ifndef NDEBUG
1818 // Make sure that it's really an empty and not a failure of
1819 // semantic analysis.
1820 for (const auto *Field : record->fields())
1821 assert(
1822 (Field->isUnnamedBitField() || Field->isAnonymousStructOrUnion()) &&
1823 "Only unnamed bitfields or anonymous class allowed");
1824#endif
1825 return;
1826 }
1827
1828 // FIXME: volatility
1829 FieldDecl *Field = InitializedFieldInUnion;
1830
1831 LValue FieldLoc = CGF.EmitLValueForFieldInitialization(DestLV, Field);
1832 if (NumInitElements) {
1833 // Store the initializer into the field
1834 EmitInitializationToLValue(InitExprs[0], FieldLoc);
1835 if (ZeroInitPadding) {
1836 uint64_t TotalSize = CGF.getContext().toBits(
1837 Dest.getPreferredSize(CGF.getContext(), DestLV.getType()));
1838 uint64_t FieldSize = CGF.getContext().getTypeSize(FieldLoc.getType());
1839 DoZeroInitPadding(FieldSize, TotalSize, nullptr);
1840 }
1841 } else {
1842 // Default-initialize to null.
1843 if (ZeroInitPadding)
1844 EmitNullInitializationToLValue(DestLV);
1845 else
1846 EmitNullInitializationToLValue(FieldLoc);
1847 }
1848 return;
1849 }
1850
1851 // Here we iterate over the fields; this makes it simpler to both
1852 // default-initialize fields and skip over unnamed fields.
1853 const ASTRecordLayout &Layout = CGF.getContext().getASTRecordLayout(record);
1854 uint64_t PaddingStart = 0;
1855
1856 for (const auto *field : record->fields()) {
1857 // We're done once we hit the flexible array member.
1858 if (field->getType()->isIncompleteArrayType())
1859 break;
1860
1861 // Always skip anonymous bitfields.
1862 if (field->isUnnamedBitField())
1863 continue;
1864
1865 // We're done if we reach the end of the explicit initializers, we
1866 // have a zeroed object, and the rest of the fields are
1867 // zero-initializable.
1868 if (curInitIndex == NumInitElements && Dest.isZeroed() &&
1869 CGF.getTypes().isZeroInitializable(ExprToVisit->getType()))
1870 break;
1871
1872 if (ZeroInitPadding)
1873 DoZeroInitPadding(PaddingStart,
1874 Layout.getFieldOffset(field->getFieldIndex()), field);
1875
1876 LValue LV = CGF.EmitLValueForFieldInitialization(DestLV, field);
1877 // We never generate write-barries for initialized fields.
1878 LV.setNonGC(true);
1879
1880 if (curInitIndex < NumInitElements) {
1881 // Store the initializer into the field.
1882 EmitInitializationToLValue(InitExprs[curInitIndex++], LV);
1883 } else {
1884 // We're out of initializers; default-initialize to null
1885 EmitNullInitializationToLValue(LV);
1886 }
1887
1888 // Push a destructor if necessary.
1889 // FIXME: if we have an array of structures, all explicitly
1890 // initialized, we can end up pushing a linear number of cleanups.
1891 if (QualType::DestructionKind dtorKind
1892 = field->getType().isDestructedType()) {
1893 assert(LV.isSimple());
1894 if (dtorKind) {
1896 field->getType(),
1897 CGF.getDestroyer(dtorKind), false);
1898 }
1899 }
1900 }
1901 if (ZeroInitPadding) {
1902 uint64_t TotalSize = CGF.getContext().toBits(
1903 Dest.getPreferredSize(CGF.getContext(), DestLV.getType()));
1904 DoZeroInitPadding(PaddingStart, TotalSize, nullptr);
1905 }
1906}
1907
1908void AggExprEmitter::DoZeroInitPadding(uint64_t &PaddingStart,
1909 uint64_t PaddingEnd,
1910 const FieldDecl *NextField) {
1911
1912 auto InitBytes = [&](uint64_t StartBit, uint64_t EndBit) {
1913 CharUnits Start = CGF.getContext().toCharUnitsFromBits(StartBit);
1914 CharUnits End = CGF.getContext().toCharUnitsFromBits(EndBit);
1915 Address Addr = Dest.getAddress().withElementType(CGF.CharTy);
1916 if (!Start.isZero())
1917 Addr = Builder.CreateConstGEP(Addr, Start.getQuantity());
1918 llvm::Constant *SizeVal = Builder.getInt64((End - Start).getQuantity());
1919 CGF.Builder.CreateMemSet(Addr, Builder.getInt8(0), SizeVal, false);
1920 };
1921
1922 if (NextField != nullptr && NextField->isBitField()) {
1923 // For bitfield, zero init StorageSize before storing the bits. So we don't
1924 // need to handle big/little endian.
1925 const CGRecordLayout &RL =
1926 CGF.getTypes().getCGRecordLayout(NextField->getParent());
1927 const CGBitFieldInfo &Info = RL.getBitFieldInfo(NextField);
1928 uint64_t StorageStart = CGF.getContext().toBits(Info.StorageOffset);
1929 if (StorageStart + Info.StorageSize > PaddingStart) {
1930 if (StorageStart > PaddingStart)
1931 InitBytes(PaddingStart, StorageStart);
1932 Address Addr = Dest.getAddress();
1933 if (!Info.StorageOffset.isZero())
1934 Addr = Builder.CreateConstGEP(Addr.withElementType(CGF.CharTy),
1935 Info.StorageOffset.getQuantity());
1936 Addr = Addr.withElementType(
1937 llvm::Type::getIntNTy(CGF.getLLVMContext(), Info.StorageSize));
1938 Builder.CreateStore(Builder.getIntN(Info.StorageSize, 0), Addr);
1939 PaddingStart = StorageStart + Info.StorageSize;
1940 }
1941 return;
1942 }
1943
1944 if (PaddingStart < PaddingEnd)
1945 InitBytes(PaddingStart, PaddingEnd);
1946 if (NextField != nullptr)
1947 PaddingStart =
1948 PaddingEnd + CGF.getContext().getTypeSize(NextField->getType());
1949}
1950
1951void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
1952 llvm::Value *outerBegin) {
1953 // Emit the common subexpression.
1954 CodeGenFunction::OpaqueValueMapping binding(CGF, E->getCommonExpr());
1955
1956 Address destPtr = EnsureSlot(E->getType()).getAddress();
1957 uint64_t numElements = E->getArraySize().getZExtValue();
1958
1959 if (!numElements)
1960 return;
1961
1962 // destPtr is an array*. Construct an elementType* by drilling down a level.
1963 llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
1964 llvm::Value *indices[] = {zero, zero};
1965 llvm::Value *begin = Builder.CreateInBoundsGEP(destPtr.getElementType(),
1966 destPtr.emitRawPointer(CGF),
1967 indices, "arrayinit.begin");
1968
1969 // Prepare to special-case multidimensional array initialization: we avoid
1970 // emitting multiple destructor loops in that case.
1971 if (!outerBegin)
1972 outerBegin = begin;
1973 ArrayInitLoopExpr *InnerLoop = dyn_cast<ArrayInitLoopExpr>(E->getSubExpr());
1974
1975 QualType elementType =
1977 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);
1978 CharUnits elementAlign =
1979 destPtr.getAlignment().alignmentOfArrayElement(elementSize);
1980 llvm::Type *llvmElementType = CGF.ConvertTypeForMem(elementType);
1981
1982 llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
1983 llvm::BasicBlock *bodyBB = CGF.createBasicBlock("arrayinit.body");
1984
1985 // Jump into the body.
1986 CGF.EmitBlock(bodyBB);
1987 llvm::PHINode *index =
1988 Builder.CreatePHI(zero->getType(), 2, "arrayinit.index");
1989 index->addIncoming(zero, entryBB);
1990 llvm::Value *element =
1991 Builder.CreateInBoundsGEP(llvmElementType, begin, index);
1992
1993 // Prepare for a cleanup.
1994 QualType::DestructionKind dtorKind = elementType.isDestructedType();
1995 EHScopeStack::stable_iterator cleanup;
1996 if (CGF.needsEHCleanup(dtorKind) && !InnerLoop) {
1997 if (outerBegin->getType() != element->getType())
1998 outerBegin = Builder.CreateBitCast(outerBegin, element->getType());
1999 CGF.pushRegularPartialArrayCleanup(outerBegin, element, elementType,
2000 elementAlign,
2001 CGF.getDestroyer(dtorKind));
2003 } else {
2004 dtorKind = QualType::DK_none;
2005 }
2006
2007 // Emit the actual filler expression.
2008 {
2009 // Temporaries created in an array initialization loop are destroyed
2010 // at the end of each iteration.
2011 CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
2012 CodeGenFunction::ArrayInitLoopExprScope Scope(CGF, index);
2013 LValue elementLV = CGF.MakeAddrLValue(
2014 Address(element, llvmElementType, elementAlign), elementType);
2015
2016 if (InnerLoop) {
2017 // If the subexpression is an ArrayInitLoopExpr, share its cleanup.
2018 auto elementSlot = AggValueSlot::forLValue(
2019 elementLV, AggValueSlot::IsDestructed,
2022 AggExprEmitter(CGF, elementSlot, false)
2023 .VisitArrayInitLoopExpr(InnerLoop, outerBegin);
2024 } else
2025 EmitInitializationToLValue(E->getSubExpr(), elementLV);
2026 }
2027
2028 // Move on to the next element.
2029 llvm::Value *nextIndex = Builder.CreateNUWAdd(
2030 index, llvm::ConstantInt::get(CGF.SizeTy, 1), "arrayinit.next");
2031 index->addIncoming(nextIndex, Builder.GetInsertBlock());
2032
2033 // Leave the loop if we're done.
2034 llvm::Value *done = Builder.CreateICmpEQ(
2035 nextIndex, llvm::ConstantInt::get(CGF.SizeTy, numElements),
2036 "arrayinit.done");
2037 llvm::BasicBlock *endBB = CGF.createBasicBlock("arrayinit.end");
2038 Builder.CreateCondBr(done, endBB, bodyBB);
2039
2040 CGF.EmitBlock(endBB);
2041
2042 // Leave the partial-array cleanup if we entered one.
2043 if (dtorKind)
2044 CGF.DeactivateCleanupBlock(cleanup, index);
2045}
2046
2047void AggExprEmitter::VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E) {
2048 AggValueSlot Dest = EnsureSlot(E->getType());
2049
2050 LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
2051 EmitInitializationToLValue(E->getBase(), DestLV);
2052 VisitInitListExpr(E->getUpdater());
2053}
2054
2055//===----------------------------------------------------------------------===//
2056// Entry Points into this File
2057//===----------------------------------------------------------------------===//
2058
2059/// GetNumNonZeroBytesInInit - Get an approximate count of the number of
2060/// non-zero bytes that will be stored when outputting the initializer for the
2061/// specified initializer expression.
2063 if (auto *MTE = dyn_cast<MaterializeTemporaryExpr>(E))
2064 E = MTE->getSubExpr();
2065 E = E->IgnoreParenNoopCasts(CGF.getContext());
2066
2067 // 0 and 0.0 won't require any non-zero stores!
2068 if (isSimpleZero(E, CGF)) return CharUnits::Zero();
2069
2070 // If this is an initlist expr, sum up the size of sizes of the (present)
2071 // elements. If this is something weird, assume the whole thing is non-zero.
2072 const InitListExpr *ILE = dyn_cast<InitListExpr>(E);
2073 while (ILE && ILE->isTransparent())
2074 ILE = dyn_cast<InitListExpr>(ILE->getInit(0));
2075 if (!ILE || !CGF.getTypes().isZeroInitializable(ILE->getType()))
2076 return CGF.getContext().getTypeSizeInChars(E->getType());
2077
2078 // InitListExprs for structs have to be handled carefully. If there are
2079 // reference members, we need to consider the size of the reference, not the
2080 // referencee. InitListExprs for unions and arrays can't have references.
2081 if (const RecordType *RT = E->getType()->getAsCanonical<RecordType>()) {
2082 if (!RT->isUnionType()) {
2083 RecordDecl *SD = RT->getDecl()->getDefinitionOrSelf();
2084 CharUnits NumNonZeroBytes = CharUnits::Zero();
2085
2086 unsigned ILEElement = 0;
2087 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(SD))
2088 while (ILEElement != CXXRD->getNumBases())
2089 NumNonZeroBytes +=
2090 GetNumNonZeroBytesInInit(ILE->getInit(ILEElement++), CGF);
2091 for (const auto *Field : SD->fields()) {
2092 // We're done once we hit the flexible array member or run out of
2093 // InitListExpr elements.
2094 if (Field->getType()->isIncompleteArrayType() ||
2095 ILEElement == ILE->getNumInits())
2096 break;
2097 if (Field->isUnnamedBitField())
2098 continue;
2099
2100 const Expr *E = ILE->getInit(ILEElement++);
2101
2102 // Reference values are always non-null and have the width of a pointer.
2103 if (Field->getType()->isReferenceType())
2104 NumNonZeroBytes += CGF.getContext().toCharUnitsFromBits(
2106 else
2107 NumNonZeroBytes += GetNumNonZeroBytesInInit(E, CGF);
2108 }
2109
2110 return NumNonZeroBytes;
2111 }
2112 }
2113
2114 // FIXME: This overestimates the number of non-zero bytes for bit-fields.
2115 CharUnits NumNonZeroBytes = CharUnits::Zero();
2116 for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i)
2117 NumNonZeroBytes += GetNumNonZeroBytesInInit(ILE->getInit(i), CGF);
2118 return NumNonZeroBytes;
2119}
2120
2121/// CheckAggExprForMemSetUse - If the initializer is large and has a lot of
2122/// zeros in it, emit a memset and avoid storing the individual zeros.
2123///
2124static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,
2125 CodeGenFunction &CGF) {
2126 // If the slot is already known to be zeroed, nothing to do. Don't mess with
2127 // volatile stores.
2128 if (Slot.isZeroed() || Slot.isVolatile() || !Slot.getAddress().isValid())
2129 return;
2130
2131 // C++ objects with a user-declared constructor don't need zero'ing.
2132 if (CGF.getLangOpts().CPlusPlus)
2133 if (const RecordType *RT = CGF.getContext()
2135 ->getAsCanonical<RecordType>()) {
2136 const auto *RD = cast<CXXRecordDecl>(RT->getDecl());
2137 if (RD->hasUserDeclaredConstructor())
2138 return;
2139 }
2140
2141 // If the type is 16-bytes or smaller, prefer individual stores over memset.
2142 CharUnits Size = Slot.getPreferredSize(CGF.getContext(), E->getType());
2143 if (Size <= CharUnits::fromQuantity(16))
2144 return;
2145
2146 // Check to see if over 3/4 of the initializer are known to be zero. If so,
2147 // we prefer to emit memset + individual stores for the rest.
2148 CharUnits NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF);
2149 if (NumNonZeroBytes*4 > Size)
2150 return;
2151
2152 // Okay, it seems like a good idea to use an initial memset, emit the call.
2153 llvm::Constant *SizeVal = CGF.Builder.getInt64(Size.getQuantity());
2154
2155 Address Loc = Slot.getAddress().withElementType(CGF.Int8Ty);
2156 CGF.Builder.CreateMemSet(Loc, CGF.Builder.getInt8(0), SizeVal, false);
2157
2158 // Tell the AggExprEmitter that the slot is known zero.
2159 Slot.setZeroed();
2160}
2161
2162
2163
2164
2165/// EmitAggExpr - Emit the computation of the specified expression of aggregate
2166/// type. The result is computed into DestPtr. Note that if DestPtr is null,
2167/// the value of the aggregate expression is not needed. If VolatileDest is
2168/// true, DestPtr cannot be 0.
2170 assert(E && hasAggregateEvaluationKind(E->getType()) &&
2171 "Invalid aggregate expression to emit");
2172 assert((Slot.getAddress().isValid() || Slot.isIgnored()) &&
2173 "slot has bits but no address");
2174
2175 // Optimize the slot if possible.
2176 CheckAggExprForMemSetUse(Slot, E, *this);
2177
2178 AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(const_cast<Expr*>(E));
2179}
2180
2191
2193 const LValue &Src,
2194 ExprValueKind SrcKind) {
2195 return AggExprEmitter(*this, Dest, Dest.isIgnored())
2196 .EmitFinalDestCopy(Type, Src, SrcKind);
2197}
2198
2201 if (!FD->hasAttr<NoUniqueAddressAttr>() || !FD->getType()->isRecordType())
2203
2204 // Empty fields can overlap earlier fields.
2205 if (FD->getType()->getAsCXXRecordDecl()->isEmpty())
2207
2208 // If the field lies entirely within the enclosing class's nvsize, its tail
2209 // padding cannot overlap any already-initialized object. (The only subobjects
2210 // with greater addresses that might already be initialized are vbases.)
2211 const RecordDecl *ClassRD = FD->getParent();
2212 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(ClassRD);
2213 if (Layout.getFieldOffset(FD->getFieldIndex()) +
2214 getContext().getTypeSize(FD->getType()) <=
2215 (uint64_t)getContext().toBits(Layout.getNonVirtualSize()))
2217
2218 // The tail padding may contain values we need to preserve.
2220}
2221
2223 const CXXRecordDecl *RD, const CXXRecordDecl *BaseRD, bool IsVirtual) {
2224 // If the most-derived object is a field declared with [[no_unique_address]],
2225 // the tail padding of any virtual base could be reused for other subobjects
2226 // of that field's class.
2227 if (IsVirtual)
2229
2230 // Empty bases can overlap earlier bases.
2231 if (BaseRD->isEmpty())
2233
2234 // If the base class is laid out entirely within the nvsize of the derived
2235 // class, its tail padding cannot yet be initialized, so we can issue
2236 // stores at the full width of the base class.
2237 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2238 if (Layout.getBaseClassOffset(BaseRD) +
2239 getContext().getASTRecordLayout(BaseRD).getSize() <=
2240 Layout.getNonVirtualSize())
2242
2243 // The tail padding may contain values we need to preserve.
2245}
2246
2248 AggValueSlot::Overlap_t MayOverlap,
2249 bool isVolatile) {
2250 assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");
2251
2252 Address DestPtr = Dest.getAddress();
2253 Address SrcPtr = Src.getAddress();
2254
2255 if (getLangOpts().CPlusPlus) {
2256 if (const auto *Record = Ty->getAsCXXRecordDecl()) {
2257 assert((Record->hasTrivialCopyConstructor() ||
2258 Record->hasTrivialCopyAssignment() ||
2259 Record->hasTrivialMoveConstructor() ||
2260 Record->hasTrivialMoveAssignment() ||
2261 Record->hasAttr<TrivialABIAttr>() || Record->isUnion()) &&
2262 "Trying to aggregate-copy a type without a trivial copy/move "
2263 "constructor or assignment operator");
2264 // Ignore empty classes in C++.
2265 if (Record->isEmpty())
2266 return;
2267 }
2268 }
2269
2270 if (getLangOpts().CUDAIsDevice) {
2272 if (getTargetHooks().emitCUDADeviceBuiltinSurfaceDeviceCopy(*this, Dest,
2273 Src))
2274 return;
2275 } else if (Ty->isCUDADeviceBuiltinTextureType()) {
2276 if (getTargetHooks().emitCUDADeviceBuiltinTextureDeviceCopy(*this, Dest,
2277 Src))
2278 return;
2279 }
2280 }
2281
2282 // Aggregate assignment turns into llvm.memcpy. This is almost valid per
2283 // C99 6.5.16.1p3, which states "If the value being stored in an object is
2284 // read from another object that overlaps in anyway the storage of the first
2285 // object, then the overlap shall be exact and the two objects shall have
2286 // qualified or unqualified versions of a compatible type."
2287 //
2288 // memcpy is not defined if the source and destination pointers are exactly
2289 // equal, but other compilers do this optimization, and almost every memcpy
2290 // implementation handles this case safely. If there is a libc that does not
2291 // safely handle this, we can add a target hook.
2292
2293 // Get data size info for this aggregate. Don't copy the tail padding if this
2294 // might be a potentially-overlapping subobject, since the tail padding might
2295 // be occupied by a different object. Otherwise, copying it is fine.
2297 if (MayOverlap)
2298 TypeInfo = getContext().getTypeInfoDataSizeInChars(Ty);
2299 else
2300 TypeInfo = getContext().getTypeInfoInChars(Ty);
2301
2302 llvm::Value *SizeVal = nullptr;
2303 if (TypeInfo.Width.isZero()) {
2304 // But note that getTypeInfo returns 0 for a VLA.
2305 if (auto *VAT = dyn_cast_or_null<VariableArrayType>(
2306 getContext().getAsArrayType(Ty))) {
2307 QualType BaseEltTy;
2308 SizeVal = emitArrayLength(VAT, BaseEltTy, DestPtr);
2309 TypeInfo = getContext().getTypeInfoInChars(BaseEltTy);
2310 assert(!TypeInfo.Width.isZero());
2311 SizeVal = Builder.CreateNUWMul(
2312 SizeVal,
2313 llvm::ConstantInt::get(SizeTy, TypeInfo.Width.getQuantity()));
2314 }
2315 }
2316 if (!SizeVal) {
2317 SizeVal = llvm::ConstantInt::get(SizeTy, TypeInfo.Width.getQuantity());
2318 }
2319
2320 // FIXME: If we have a volatile struct, the optimizer can remove what might
2321 // appear to be `extra' memory ops:
2322 //
2323 // volatile struct { int i; } a, b;
2324 //
2325 // int main() {
2326 // a = b;
2327 // a = b;
2328 // }
2329 //
2330 // we need to use a different call here. We use isVolatile to indicate when
2331 // either the source or the destination is volatile.
2332
2333 DestPtr = DestPtr.withElementType(Int8Ty);
2334 SrcPtr = SrcPtr.withElementType(Int8Ty);
2335
2336 // Don't do any of the memmove_collectable tests if GC isn't set.
2337 if (CGM.getLangOpts().getGC() == LangOptions::NonGC) {
2338 // fall through
2339 } else if (const auto *Record = Ty->getAsRecordDecl()) {
2340 if (Record->hasObjectMember()) {
2341 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr,
2342 SizeVal);
2343 return;
2344 }
2345 } else if (Ty->isArrayType()) {
2346 QualType BaseType = getContext().getBaseElementType(Ty);
2347 if (const auto *Record = BaseType->getAsRecordDecl()) {
2348 if (Record->hasObjectMember()) {
2349 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr,
2350 SizeVal);
2351 return;
2352 }
2353 }
2354 }
2355
2356 auto *Inst = Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, isVolatile);
2357 addInstToCurrentSourceAtom(Inst, nullptr);
2358
2359 // Determine the metadata to describe the position of any padding in this
2360 // memcpy, as well as the TBAA tags for the members of the struct, in case
2361 // the optimizer wishes to expand it in to scalar memory operations.
2362 if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty))
2363 Inst->setMetadata(llvm::LLVMContext::MD_tbaa_struct, TBAAStructTag);
2364
2365 if (CGM.getCodeGenOpts().NewStructPathTBAA) {
2366 TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForMemoryTransfer(
2367 Dest.getTBAAInfo(), Src.getTBAAInfo());
2368 CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo);
2369 }
2370}
Defines the clang::ASTContext interface.
#define V(N, I)
CompareKind
@ CK_Greater
@ CK_Less
@ CK_Equal
static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF)
GetNumNonZeroBytesInInit - Get an approximate count of the number of non-zero bytes that will be stor...
static Expr * findPeephole(Expr *op, CastKind kind, const ASTContext &ctx)
Attempt to look through various unimportant expressions to find a cast of the given kind.
static bool isBlockVarRef(const Expr *E)
Is the value of the given expression possibly a reference to or into a __block variable?
static bool isSimpleZero(const Expr *E, CodeGenFunction &CGF)
isSimpleZero - If emitting this value will obviously just cause a store of zero to memory,...
static llvm::Value * EmitCompare(CGBuilderTy &Builder, CodeGenFunction &CGF, const BinaryOperator *E, llvm::Value *LHS, llvm::Value *RHS, CompareKind Kind, const char *NameSuffix="")
static void EmitHLSLElementwiseCast(CodeGenFunction &CGF, LValue DestVal, LValue SrcVal, SourceLocation Loc)
static bool castPreservesZero(const CastExpr *CE)
Determine whether the given cast kind is known to always convert values with all zero bits in their v...
static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E, CodeGenFunction &CGF)
CheckAggExprForMemSetUse - If the initializer is large and has a lot of zeros in it,...
static void EmitHLSLScalarElementwiseAndSplatCasts(CodeGenFunction &CGF, LValue DestVal, llvm::Value *SrcVal, QualType SrcTy, SourceLocation Loc)
static bool isTrivialFiller(Expr *e)
Defines the C++ Decl subclasses, other than those for templates (found in DeclTemplate....
Defines the C++ template declaration subclasses.
tooling::Replacements cleanup(const FormatStyle &Style, StringRef Code, ArrayRef< tooling::Range > Ranges, StringRef FileName="<stdin>")
Clean up any erroneous/redundant code in the given Ranges in Code.
llvm::MachO::Record Record
Definition MachO.h:31
static bool isVector(QualType QT, QualType ElementType)
This helper function returns true if QT is a vector type that has element type ElementType.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition ASTContext.h:220
const ConstantArrayType * getAsConstantArrayType(QualType T) const
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
ComparisonCategories CompCategories
Types and expressions required to build C++2a three-way comparisons using operator<=>,...
QualType removeAddrSpaceQualType(QualType T) const
Remove any existing address space on the type and returns the type with qualifiers intact (or that's ...
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
static bool hasSameType(QualType T1, QualType T2)
Determine whether the given types T1 and T2 are equivalent.
QualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
QualType getAddrSpaceQualType(QualType T, LangAS AddressSpace) const
Return the uniqued reference to the type for an address space qualified type with the specified type ...
unsigned getTargetAddressSpace(LangAS AS) const
static bool hasSameUnqualifiedType(QualType T1, QualType T2)
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
CharUnits getBaseClassOffset(const CXXRecordDecl *Base) const
getBaseClassOffset - Get the offset, in chars, for the given base class.
CharUnits getNonVirtualSize() const
getNonVirtualSize - Get the non-virtual size (in chars) of an object, which is the size of the object...
AbstractConditionalOperator - An abstract base class for ConditionalOperator and BinaryConditionalOpe...
Definition Expr.h:4287
Expr * getCond() const
getCond - Return the expression representing the condition for the ?
Definition Expr.h:4465
Expr * getTrueExpr() const
getTrueExpr - Return the subexpression representing the value of the expression if the condition eval...
Definition Expr.h:4471
Expr * getFalseExpr() const
getFalseExpr - Return the subexpression representing the value of the expression if the condition eva...
Definition Expr.h:4477
llvm::APInt getArraySize() const
Definition Expr.h:5924
OpaqueValueExpr * getCommonExpr() const
Get the common subexpression shared by all initializations (the source array).
Definition Expr.h:5917
Expr * getSubExpr() const
Get the initializer to use for each array element.
Definition Expr.h:5922
ArraySubscriptExpr - [C99 6.5.2.1] Array Subscripting.
Definition Expr.h:2721
QualType getElementType() const
Definition TypeBase.h:3734
A builtin binary operation expression such as "x + y" or "x <= y".
Definition Expr.h:3972
Expr * getLHS() const
Definition Expr.h:4022
Expr * getRHS() const
Definition Expr.h:4024
Opcode getOpcode() const
Definition Expr.h:4017
CXXTemporary * getTemporary()
Definition ExprCXX.h:1512
const Expr * getSubExpr() const
Definition ExprCXX.h:1516
Expr * getExpr()
Get the initialization expression that will be used.
Definition ExprCXX.cpp:1105
bool constructsVBase() const
Determine whether this constructor is actually constructing a base class (rather than a complete obje...
Definition ExprCXX.h:1794
CXXConstructorDecl * getConstructor() const
Get the constructor that this expression will call.
Definition ExprCXX.h:1790
bool inheritedFromVBase() const
Determine whether the inherited constructor is inherited from a virtual base of the object we constru...
Definition ExprCXX.h:1804
MutableArrayRef< Expr * > getInitExprs()
Definition ExprCXX.h:5183
FieldDecl * getInitializedFieldInUnion()
Definition ExprCXX.h:5221
Represents a C++ struct/union/class.
Definition DeclCXX.h:258
bool isTriviallyCopyable() const
Determine whether this class is considered trivially copyable per (C++11 [class]p6).
Definition DeclCXX.cpp:607
bool isEmpty() const
Determine whether this is an empty class in the sense of (C++11 [meta.unary.prop]).
Definition DeclCXX.h:1186
Expr * getSemanticForm()
Get an equivalent semantic form for this expression.
Definition ExprCXX.h:304
QualType getCallReturnType(const ASTContext &Ctx) const
getCallReturnType - Get the return type of the call expr.
Definition Expr.cpp:1599
CastExpr - Base class for type casts, including both implicit casts (ImplicitCastExpr) and explicit c...
Definition Expr.h:3610
CastKind getCastKind() const
Definition Expr.h:3654
Expr * getSubExpr()
Definition Expr.h:3660
CharUnits - This is an opaque type for sizes expressed in character units.
Definition CharUnits.h:38
bool isZero() const
isZero - Test whether the quantity equals zero.
Definition CharUnits.h:122
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
Definition CharUnits.h:189
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition CharUnits.h:185
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
Definition CharUnits.h:214
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:63
static CharUnits Zero()
Zero - Construct a CharUnits quantity of zero.
Definition CharUnits.h:53
Expr * getChosenSubExpr() const
getChosenSubExpr - Return the subexpression chosen according to the condition.
Definition Expr.h:4818
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
Definition Address.h:128
llvm::Value * getBasePointer() const
Definition Address.h:198
static Address invalid()
Definition Address.h:176
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
Definition Address.h:253
CharUnits getAlignment() const
Definition Address.h:194
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition Address.h:209
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Definition Address.h:276
bool isValid() const
Definition Address.h:177
An aggregate value slot.
Definition CGValue.h:504
void setVolatile(bool flag)
Definition CGValue.h:623
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
Definition CGValue.h:572
Address getAddress() const
Definition CGValue.h:644
CharUnits getPreferredSize(ASTContext &Ctx, QualType Type) const
Get the preferred size to use when storing a value to this slot.
Definition CGValue.h:682
NeedsGCBarriers_t requiresGCollection() const
Definition CGValue.h:634
void setExternallyDestructed(bool destructed=true)
Definition CGValue.h:613
void setZeroed(bool V=true)
Definition CGValue.h:674
IsZeroed_t isZeroed() const
Definition CGValue.h:675
Qualifiers getQualifiers() const
Definition CGValue.h:617
static AggValueSlot forLValue(const LValue &LV, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, Overlap_t mayOverlap, IsZeroed_t isZeroed=IsNotZeroed, IsSanitizerChecked_t isChecked=IsNotSanitizerChecked)
Definition CGValue.h:602
IsAliased_t isPotentiallyAliased() const
Definition CGValue.h:654
static AggValueSlot forAddr(Address addr, Qualifiers quals, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, Overlap_t mayOverlap, IsZeroed_t isZeroed=IsNotZeroed, IsSanitizerChecked_t isChecked=IsNotSanitizerChecked)
forAddr - Make a slot for an aggregate value.
Definition CGValue.h:587
IsDestructed_t isExternallyDestructed() const
Definition CGValue.h:610
Overlap_t mayOverlap() const
Definition CGValue.h:658
RValue asRValue() const
Definition CGValue.h:666
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Definition CGValue.h:640
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
Definition CGBuilder.h:402
Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name="")
Definition CGBuilder.h:223
virtual llvm::Value * EmitMemberPointerComparison(CodeGenFunction &CGF, llvm::Value *L, llvm::Value *R, const MemberPointerType *MPT, bool Inequality)
Emit a comparison between two member pointers. Returns an i1.
Definition CGCXXABI.cpp:85
void emitInitListOpaqueValues(CodeGenFunction &CGF, InitListExpr *E)
virtual void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *Size)=0
const CGBitFieldInfo & getBitFieldInfo(const FieldDecl *FD) const
Return the BitFieldInfo that corresponds to the field FD.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void CreateCoercedStore(llvm::Value *Src, Address Dst, llvm::TypeSize DstSize, bool DstIsVolatile)
Create a store to.
Definition CGCall.cpp:1398
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr, const VarDecl *ConditionalDecl=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
RValue EmitObjCMessageExpr(const ObjCMessageExpr *E, ReturnValueSlot Return=ReturnValueSlot())
Definition CGObjC.cpp:573
void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest)
AggValueSlot::Overlap_t getOverlapForFieldInit(const FieldDecl *FD)
Determine whether a field initialization may overlap some other object.
void callCStructMoveConstructor(LValue Dst, LValue Src)
void EmitNullInitialization(Address DestPtr, QualType Ty)
EmitNullInitialization - Generate code to set a value of the given type to null, If the type contains...
static bool hasScalarEvaluationKind(QualType T)
llvm::Type * ConvertType(QualType T)
void EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest, const LValue &Src, ExprValueKind SrcKind)
EmitAggFinalDestCopy - Emit copy of the specified aggregate into destination address.
void pushRegularPartialArrayCleanup(llvm::Value *arrayBegin, llvm::Value *arrayEnd, QualType elementType, CharUnits elementAlignment, Destroyer *destroyer)
pushRegularPartialArrayCleanup - Push an EH cleanup to destroy already-constructed elements of the gi...
Definition CGDecl.cpp:2594
void EmitCXXThrowExpr(const CXXThrowExpr *E, bool KeepInsertionPoint=true)
void EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst, llvm::Value **Result=nullptr)
EmitStoreThroughBitfieldLValue - Store Src into Dst with same constraints as EmitStoreThroughLValue.
Definition CGExpr.cpp:2871
bool hasVolatileMember(QualType T)
hasVolatileMember - returns true if aggregate type has a volatile member.
llvm::SmallVector< DeferredDeactivateCleanup > DeferredDeactivationCleanupStack
RValue EmitVAArg(VAArgExpr *VE, Address &VAListAddr, AggValueSlot Slot=AggValueSlot::ignored())
Generate code to get an argument from the passed in pointer and update it accordingly.
Definition CGCall.cpp:6296
RValue EmitPseudoObjectRValue(const PseudoObjectExpr *e, AggValueSlot slot=AggValueSlot::ignored())
Definition CGExpr.cpp:6980
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void addInstToCurrentSourceAtom(llvm::Instruction *KeyInstruction, llvm::Value *Backup)
See CGDebugInfo::addInstToCurrentSourceAtom.
AggValueSlot::Overlap_t getOverlapForBaseInit(const CXXRecordDecl *RD, const CXXRecordDecl *BaseRD, bool IsVirtual)
Determine whether a base class initialization may overlap some other object.
const LangOptions & getLangOpts() const
RValue EmitReferenceBindingToExpr(const Expr *E)
Emits a reference binding to the passed in expression.
Definition CGExpr.cpp:685
LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E)
Definition CGExpr.cpp:6810
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
pushDestroy - Push the standard destructor for the given type as at least a normal cleanup.
Definition CGDecl.cpp:2278
@ TCK_Store
Checking the destination of a store. Must be suitably sized and aligned.
@ TCK_Load
Checking the operand of a load. Must be suitably sized and aligned.
void pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin, Address arrayEndPointer, QualType elementType, CharUnits elementAlignment, Destroyer *destroyer)
pushIrregularPartialArrayCleanup - Push a NormalAndEHCleanup to destroy already-constructed elements ...
Definition CGDecl.cpp:2578
Destroyer * getDestroyer(QualType::DestructionKind destructionKind)
Definition CGDecl.cpp:2251
LValue EmitPseudoObjectLValue(const PseudoObjectExpr *e)
Definition CGExpr.cpp:6985
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
const TargetInfo & getTarget() const
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
Definition CGExpr.cpp:243
RValue EmitCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue=ReturnValueSlot(), llvm::CallBase **CallOrInvoke=nullptr)
Definition CGExpr.cpp:6125
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition CGExpr.cpp:2527
void pushDestroyAndDeferDeactivation(QualType::DestructionKind dtorKind, Address addr, QualType type)
Definition CGDecl.cpp:2303
void DeactivateCleanupBlock(EHScopeStack::stable_iterator Cleanup, llvm::Instruction *DominatingIP)
DeactivateCleanupBlock - Deactivates the given cleanup block.
void callCStructCopyAssignmentOperator(LValue Dst, LValue Src)
void pushFullExprCleanup(CleanupKind kind, As... A)
pushFullExprCleanup - Push a cleanup to be run at the end of the current full-expression.
LValue EmitAggExprToLValue(const Expr *E)
EmitAggExprToLValue - Emit the computation of the specified expression of aggregate type into a tempo...
RValue EmitCoyieldExpr(const CoyieldExpr &E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
AggValueSlot CreateAggTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateAggTemp - Create a temporary memory object for the given aggregate type.
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
void callCStructCopyConstructor(LValue Dst, LValue Src)
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
bool EmitLifetimeStart(llvm::Value *Addr)
Emit a lifetime.begin marker if some criteria are satisfied.
Definition CGDecl.cpp:1356
LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field)
EmitLValueForFieldInitialization - Like EmitLValueForField, except that if the Field is a reference,...
Definition CGExpr.cpp:5617
Address GetAddressOfDirectBaseInCompleteClass(Address Value, const CXXRecordDecl *Derived, const CXXRecordDecl *Base, bool BaseIsVirtual)
GetAddressOfBaseOfCompleteClass - Convert the given pointer to a complete class to the given direct b...
Definition CGClass.cpp:216
llvm::AllocaInst * CreateTempAlloca(llvm::Type *Ty, const Twine &Name="tmp", llvm::Value *ArraySize=nullptr)
CreateTempAlloca - This creates an alloca and inserts it into the entry block if ArraySize is nullptr...
Definition CGExpr.cpp:152
LValue getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e)
Given an opaque value expression, return its LValue mapping if it exists, otherwise create one.
Definition CGExpr.cpp:6064
const TargetCodeGenInfo & getTargetHooks() const
void EmitLifetimeEnd(llvm::Value *Addr)
Definition CGDecl.cpp:1368
RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen without...
Definition CGExpr.cpp:216
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
void callCStructMoveAssignmentOperator(LValue Dst, LValue Src)
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition CGExpr.cpp:2724
void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
Definition CGDecl.cpp:2331
Address EmitCompoundStmt(const CompoundStmt &S, bool GetLast=false, AggValueSlot AVS=AggValueSlot::ignored())
EmitCompoundStmt - Emit a compound statement {..} node.
Definition CGStmt.cpp:569
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
Definition CGExpr.cpp:265
bool needsEHCleanup(QualType::DestructionKind kind)
Determines whether an EH cleanup is required to destroy a type with the given destruction kind.
CleanupKind getCleanupKind(QualType::DestructionKind kind)
llvm::Type * ConvertTypeForMem(QualType T)
RValue EmitAtomicExpr(AtomicExpr *E)
Definition CGAtomic.cpp:892
CodeGenTypes & getTypes() const
void FlattenAccessAndTypeLValue(LValue LVal, SmallVectorImpl< LValue > &AccessList)
Definition CGExpr.cpp:6989
RValue EmitCoawaitExpr(const CoawaitExpr &E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType, Address Ptr)
Emits all the code to cause the given temporary to be cleaned up.
bool LValueIsSuitableForInlineAtomic(LValue Src)
An LValue is a candidate for having its loads and stores be made atomic if we are operating under /vo...
LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK)
Same as EmitLValue but additionally we generate checking code to guard against undefined behavior.
Definition CGExpr.cpp:1787
void EmitInheritedCXXConstructorCall(const CXXConstructorDecl *D, bool ForVirtualBase, Address This, bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E)
Emit a call to a constructor inherited from a base class, passing the current constructor's arguments...
Definition CGClass.cpp:2337
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition CGExpr.cpp:187
void EmitInitializationToLValue(const Expr *E, LValue LV, AggValueSlot::IsZeroed_t IsZeroed=AggValueSlot::IsNotZeroed)
EmitInitializationToLValue - Emit an initializer to an LValue.
Definition CGExpr.cpp:324
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
static bool hasAggregateEvaluationKind(QualType T)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitLambdaVLACapture(const VariableArrayType *VAT, LValue LV)
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
void ErrorUnsupported(const Stmt *S, const char *Type)
ErrorUnsupported - Print out an error that codegen doesn't support the specified stmt yet.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
Definition CGExpr.cpp:1822
llvm::LLVMContext & getLLVMContext()
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
llvm::Value * EmitDynamicCast(Address V, const CXXDynamicCastExpr *DCE)
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition CGStmt.cpp:655
void EmitExplicitCastExprType(const ExplicitCastExpr *E, CodeGenFunction *CGF=nullptr)
Emit type info if type of an expression is a variably modified type.
Definition CGExpr.cpp:1502
CGHLSLRuntime & getHLSLRuntime()
Return a reference to the configured HLSL runtime.
llvm::Module & getModule() const
bool isPaddedAtomicType(QualType type)
void ErrorUnsupported(const Stmt *S, const char *Type)
Print out an error that codegen doesn't support the specified stmt yet.
ASTContext & getContext() const
CGObjCRuntime & getObjCRuntime()
Return a reference to the configured Objective-C runtime.
llvm::Constant * EmitNullConstant(QualType T)
Return the result of value-initializing the given type, i.e.
LangAS GetGlobalConstantAddressSpace() const
Return the AST address space of constant literal, which is used to emit the constant literal as globa...
bool isPointerZeroInitializable(QualType T)
Check if the pointer type can be zero-initialized (in the C++ sense) with an LLVM zeroinitializer.
const CGRecordLayout & getCGRecordLayout(const RecordDecl *)
getCGRecordLayout - Return record layout info for the given record decl.
bool isZeroInitializable(QualType T)
IsZeroInitializable - Return whether a type can be zero-initialized (in the C++ sense) with an LLVM z...
stable_iterator stable_begin() const
Create a stable reference to the top of the EH stack.
iterator find(stable_iterator save) const
Turn a stable reference to a scope depth into a unstable pointer to the EH stack.
Definition CGCleanup.h:647
LValue - This represents an lvalue references.
Definition CGValue.h:182
Address getAddress() const
Definition CGValue.h:361
TBAAAccessInfo getTBAAInfo() const
Definition CGValue.h:335
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
Definition CGValue.h:42
llvm::Value * getAggregatePointer(QualType PointeeType, CodeGenFunction &CGF) const
Definition CGValue.h:88
bool isScalar() const
Definition CGValue.h:64
static RValue get(llvm::Value *V)
Definition CGValue.h:98
static RValue getAggregate(Address addr, bool isVolatile=false)
Convert an Address to an RValue.
Definition CGValue.h:125
bool isAggregate() const
Definition CGValue.h:66
Address getAggregateAddress() const
getAggregateAddr() - Return the Value* of the address of the aggregate.
Definition CGValue.h:83
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition CGValue.h:71
bool isComplex() const
Definition CGValue.h:65
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition CGValue.h:78
const ComparisonCategoryInfo & getInfoForType(QualType Ty) const
Return the comparison category information as specified by getCategoryForType(Ty).
bool isPartial() const
True iff the comparison is not totally ordered.
const ValueInfo * getLess() const
const ValueInfo * getUnordered() const
const CXXRecordDecl * Record
The declaration for the comparison category type from the standard library.
const ValueInfo * getGreater() const
const ValueInfo * getEqualOrEquiv() const
Complex values, per C99 6.2.5p11.
Definition TypeBase.h:3275
const Expr * getInitializer() const
Definition Expr.h:3567
llvm::APInt getSize() const
Return the constant array size as an APInt.
Definition TypeBase.h:3816
A reference to a declared variable, function, enum, etc.
Definition Expr.h:1270
bool hasAttr() const
Definition DeclBase.h:577
InitListExpr * getUpdater() const
Definition Expr.h:5870
This represents one expression.
Definition Expr.h:112
bool isGLValue() const
Definition Expr.h:287
Expr * IgnoreParenNoopCasts(const ASTContext &Ctx) LLVM_READONLY
Skip past any parentheses and casts which do not change the value (including ptr->int casts of the sa...
Definition Expr.cpp:3112
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
Definition Expr.cpp:3081
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
Definition Expr.cpp:3665
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition Expr.cpp:273
QualType getType() const
Definition Expr.h:144
Represents a member of a struct/union/class.
Definition Decl.h:3160
bool isBitField() const
Determines whether this field is a bitfield.
Definition Decl.h:3263
unsigned getFieldIndex() const
Returns the index of this field within its record, as appropriate for passing to ASTRecordLayout::get...
Definition Decl.h:3245
const RecordDecl * getParent() const
Returns the parent of this field declaration, which is the struct in which this field is defined.
Definition Decl.h:3396
const Expr * getSubExpr() const
Definition Expr.h:1062
Describes an C or C++ initializer list.
Definition Expr.h:5233
bool isTransparent() const
Is this a transparent initializer list (that is, an InitListExpr that is purely syntactic,...
Definition Expr.cpp:2457
FieldDecl * getInitializedFieldInUnion()
If this initializes a union, specifies which field in the union to initialize.
Definition Expr.h:5359
unsigned getNumInits() const
Definition Expr.h:5263
bool hadArrayRangeDesignator() const
Definition Expr.h:5417
Expr * getArrayFiller()
If this initializer list initializes an array with more elements than there are initializers in the l...
Definition Expr.h:5335
const Expr * getInit(unsigned Init) const
Definition Expr.h:5287
ArrayRef< Expr * > inits()
Definition Expr.h:5283
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument for this lambda expression.
Definition ExprCXX.h:2108
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
Definition ExprCXX.h:2082
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument for this lambda expression (which initializes the first ca...
Definition ExprCXX.h:2096
CXXRecordDecl * getLambdaClass() const
Retrieve the class that corresponds to the lambda.
Definition ExprCXX.cpp:1400
Expr * getSubExpr() const
Retrieve the temporary-generating subexpression whose value will be materialized into a glvalue.
Definition ExprCXX.h:4939
MemberExpr - [C99 6.5.2.3] Structure and Union Members.
Definition Expr.h:3298
A pointer to member type per C++ 8.3.3 - Pointers to members.
Definition TypeBase.h:3653
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
Definition Expr.h:1178
Expr * getSourceExpr() const
The source expression of an opaque value expression is the expression which originally generated the ...
Definition Expr.h:1228
bool isUnique() const
Definition Expr.h:1236
Expr * getSelectedExpr() const
Definition ExprCXX.h:4641
const Expr * getSubExpr() const
Definition Expr.h:2199
A (possibly-)qualified type.
Definition TypeBase.h:937
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
Definition TypeBase.h:8362
bool isTriviallyCopyableType(const ASTContext &Context) const
Return true if this is a trivially copyable type (C++0x [basic.types]p9)
Definition Type.cpp:2866
LangAS getAddressSpace() const
Return the address space of this type.
Definition TypeBase.h:8404
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after.
Definition TypeBase.h:1545
bool isPODType(const ASTContext &Context) const
Determine whether this is a Plain Old Data (POD) type (C++ 3.9p10).
Definition Type.cpp:2694
@ PCK_Struct
The type is a struct containing a field whose type is neither PCK_Trivial nor PCK_VolatileTrivial.
Definition TypeBase.h:1517
Represents a struct/union/class.
Definition Decl.h:4312
bool hasObjectMember() const
Definition Decl.h:4372
field_range fields() const
Definition Decl.h:4515
specific_decl_iterator< FieldDecl > field_iterator
Definition Decl.h:4512
RecordDecl * getDefinitionOrSelf() const
Definition Decl.h:4500
field_iterator field_begin() const
Definition Decl.cpp:5202
Encodes a location in the source.
CompoundStmt * getSubStmt()
Definition Expr.h:4546
StmtVisitor - This class implements a simple visitor for Stmt subclasses.
bool isUnion() const
Definition Decl.h:3922
uint64_t getPointerWidth(LangAS AddrSpace) const
Return the width of pointers on this target, for the specified address space.
Definition TargetInfo.h:486
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
Definition Type.h:26
bool isConstantArrayType() const
Definition TypeBase.h:8618
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Definition Type.h:41
bool isArrayType() const
Definition TypeBase.h:8614
bool isPointerType() const
Definition TypeBase.h:8515
bool isReferenceType() const
Definition TypeBase.h:8539
bool isScalarType() const
Definition TypeBase.h:8973
bool isVariableArrayType() const
Definition TypeBase.h:8626
bool isCUDADeviceBuiltinSurfaceType() const
Check if the type is the CUDA device builtin surface type.
Definition Type.cpp:5326
bool isIntegralOrEnumerationType() const
Determine whether this type is an integral or enumeration type.
Definition TypeBase.h:8989
RecordDecl * castAsRecordDecl() const
Definition Type.h:48
bool isAnyComplexType() const
Definition TypeBase.h:8650
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
Definition Type.cpp:2243
bool isMemberPointerType() const
Definition TypeBase.h:8596
bool isCUDADeviceBuiltinTextureType() const
Check if the type is the CUDA device builtin texture type.
Definition Type.cpp:5335
bool hasFloatingRepresentation() const
Determine whether this type has a floating-point representation of some sort, e.g....
Definition Type.cpp:2312
bool isVectorType() const
Definition TypeBase.h:8654
bool isRealFloatingType() const
Floating point categories.
Definition Type.cpp:2320
const T * getAsCanonical() const
If this type is canonically the specified type, return its canonical type cast to that specified type...
Definition TypeBase.h:2921
const T * getAs() const
Member-template getAs<specific type>'.
Definition TypeBase.h:9091
bool isNullPtrType() const
Definition TypeBase.h:8908
bool isRecordType() const
Definition TypeBase.h:8642
UnaryOperator - This represents the unary-expression's (except sizeof and alignof),...
Definition Expr.h:2244
Expr * getSubExpr() const
Definition Expr.h:2285
QualType getType() const
Definition Decl.h:723
Represents a variable declaration or definition.
Definition Decl.h:926
Represents a GCC generic vector type.
Definition TypeBase.h:4175
Definition SPIR.cpp:35
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
Definition CGValue.h:154
@ EHCleanup
Denotes a cleanup that should run when a scope is exited using exceptional control flow (a throw stat...
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
const AstTypeMatcher< AtomicType > atomicType
bool GE(InterpState &S, CodePtr OpPC)
Definition Interp.h:1262
The JSON file list parser is used to communicate input to InstallAPI.
bool isa(CodeGen::Address addr)
Definition Address.h:330
@ CPlusPlus
@ Result
The result type of a method or function.
Definition TypeBase.h:905
const FunctionProtoType * T
LangAS
Defines the address space values used by the address space qualifier of QualType.
CastKind
CastKind - The kind of operation required for a conversion.
U cast(CodeGen::Address addr)
Definition Address.h:327
unsigned long uint64_t
Diagnostic wrappers for TextAPI types for error reporting.
Definition Dominators.h:30
cl::opt< bool > EnableSingleByteCoverage
CharUnits StorageOffset
The offset of the bitfield storage from the start of the struct.
unsigned StorageSize
The storage size in bits which should be used when accessing this bitfield.
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::IntegerType * CharTy
char