clang 17.0.0git
CGExpr.cpp
Go to the documentation of this file.
1//===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCUDARuntime.h"
14#include "CGCXXABI.h"
15#include "CGCall.h"
16#include "CGCleanup.h"
17#include "CGDebugInfo.h"
18#include "CGObjCRuntime.h"
19#include "CGOpenMPRuntime.h"
20#include "CGRecordLayout.h"
21#include "CodeGenFunction.h"
22#include "CodeGenModule.h"
23#include "ConstantEmitter.h"
24#include "TargetInfo.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/DeclObjC.h"
28#include "clang/AST/NSAPI.h"
32#include "llvm/ADT/Hashing.h"
33#include "llvm/ADT/StringExtras.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/Intrinsics.h"
36#include "llvm/IR/LLVMContext.h"
37#include "llvm/IR/MDBuilder.h"
38#include "llvm/IR/MatrixBuilder.h"
39#include "llvm/Support/ConvertUTF.h"
40#include "llvm/Support/MathExtras.h"
41#include "llvm/Support/Path.h"
42#include "llvm/Support/SaveAndRestore.h"
43#include "llvm/Transforms/Utils/SanitizerStats.h"
44
45#include <optional>
46#include <string>
47
48using namespace clang;
49using namespace CodeGen;
50
51//===--------------------------------------------------------------------===//
52// Miscellaneous Helper Methods
53//===--------------------------------------------------------------------===//
54
55llvm::Value *CodeGenFunction::EmitCastToVoidPtr(llvm::Value *value) {
56 unsigned addressSpace =
57 cast<llvm::PointerType>(value->getType())->getAddressSpace();
58
59 llvm::PointerType *destType = Int8PtrTy;
60 if (addressSpace)
61 destType = llvm::Type::getInt8PtrTy(getLLVMContext(), addressSpace);
62
63 if (value->getType() == destType) return value;
64 return Builder.CreateBitCast(value, destType);
65}
66
67/// CreateTempAlloca - This creates a alloca and inserts it into the entry
68/// block.
70 CharUnits Align,
71 const Twine &Name,
72 llvm::Value *ArraySize) {
73 auto Alloca = CreateTempAlloca(Ty, Name, ArraySize);
74 Alloca->setAlignment(Align.getAsAlign());
75 return Address(Alloca, Ty, Align, KnownNonNull);
76}
77
78/// CreateTempAlloca - This creates a alloca and inserts it into the entry
79/// block. The alloca is casted to default address space if necessary.
81 const Twine &Name,
82 llvm::Value *ArraySize,
83 Address *AllocaAddr) {
84 auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
85 if (AllocaAddr)
86 *AllocaAddr = Alloca;
87 llvm::Value *V = Alloca.getPointer();
88 // Alloca always returns a pointer in alloca address space, which may
89 // be different from the type defined by the language. For example,
90 // in C++ the auto variables are in the default address space. Therefore
91 // cast alloca to the default address space when necessary.
93 auto DestAddrSpace = getContext().getTargetAddressSpace(LangAS::Default);
94 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
95 // When ArraySize is nullptr, alloca is inserted at AllocaInsertPt,
96 // otherwise alloca is inserted at the current insertion point of the
97 // builder.
98 if (!ArraySize)
99 Builder.SetInsertPoint(getPostAllocaInsertPoint());
102 Ty->getPointerTo(DestAddrSpace), /*non-null*/ true);
103 }
104
105 return Address(V, Ty, Align, KnownNonNull);
106}
107
108/// CreateTempAlloca - This creates an alloca and inserts it into the entry
109/// block if \p ArraySize is nullptr, otherwise inserts it at the current
110/// insertion point of the builder.
111llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
112 const Twine &Name,
113 llvm::Value *ArraySize) {
114 if (ArraySize)
115 return Builder.CreateAlloca(Ty, ArraySize, Name);
116 return new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),
117 ArraySize, Name, AllocaInsertPt);
118}
119
120/// CreateDefaultAlignTempAlloca - This creates an alloca with the
121/// default alignment of the corresponding LLVM type, which is *not*
122/// guaranteed to be related in any way to the expected alignment of
123/// an AST type that might have been lowered to Ty.
125 const Twine &Name) {
126 CharUnits Align =
127 CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlign(Ty));
128 return CreateTempAlloca(Ty, Align, Name);
129}
130
131Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) {
133 return CreateTempAlloca(ConvertType(Ty), Align, Name);
134}
135
137 Address *Alloca) {
138 // FIXME: Should we prefer the preferred type alignment here?
139 return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca);
140}
141
143 const Twine &Name, Address *Alloca) {
145 /*ArraySize=*/nullptr, Alloca);
146
147 if (Ty->isConstantMatrixType()) {
148 auto *ArrayTy = cast<llvm::ArrayType>(Result.getElementType());
149 auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
150 ArrayTy->getNumElements());
151
152 Result = Address(
153 Builder.CreateBitCast(Result.getPointer(), VectorTy->getPointerTo()),
154 VectorTy, Result.getAlignment(), KnownNonNull);
155 }
156 return Result;
157}
158
160 const Twine &Name) {
161 return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name);
162}
163
165 const Twine &Name) {
166 return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty),
167 Name);
168}
169
170/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
171/// expression and compare the result against zero, returning an Int1Ty value.
172llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
173 PGO.setCurrentStmt(E);
174 if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
175 llvm::Value *MemPtr = EmitScalarExpr(E);
176 return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT);
177 }
178
179 QualType BoolTy = getContext().BoolTy;
180 SourceLocation Loc = E->getExprLoc();
181 CGFPOptionsRAII FPOptsRAII(*this, E);
182 if (!E->getType()->isAnyComplexType())
183 return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy, Loc);
184
186 Loc);
187}
188
189/// EmitIgnoredExpr - Emit code to compute the specified expression,
190/// ignoring the result.
192 if (E->isPRValue())
193 return (void)EmitAnyExpr(E, AggValueSlot::ignored(), true);
194
195 // if this is a bitfield-resulting conditional operator, we can special case
196 // emit this. The normal 'EmitLValue' version of this is particularly
197 // difficult to codegen for, since creating a single "LValue" for two
198 // different sized arguments here is not particularly doable.
199 if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
201 if (CondOp->getObjectKind() == OK_BitField)
202 return EmitIgnoredConditionalOperator(CondOp);
203 }
204
205 // Just emit it as an l-value and drop the result.
206 EmitLValue(E);
207}
208
209/// EmitAnyExpr - Emit code to compute the specified expression which
210/// can have any type. The result is returned as an RValue struct.
211/// If this is an aggregate expression, AggSlot indicates where the
212/// result should be returned.
214 AggValueSlot aggSlot,
215 bool ignoreResult) {
216 switch (getEvaluationKind(E->getType())) {
217 case TEK_Scalar:
218 return RValue::get(EmitScalarExpr(E, ignoreResult));
219 case TEK_Complex:
220 return RValue::getComplex(EmitComplexExpr(E, ignoreResult, ignoreResult));
221 case TEK_Aggregate:
222 if (!ignoreResult && aggSlot.isIgnored())
223 aggSlot = CreateAggTemp(E->getType(), "agg-temp");
224 EmitAggExpr(E, aggSlot);
225 return aggSlot.asRValue();
226 }
227 llvm_unreachable("bad evaluation kind");
228}
229
230/// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will
231/// always be accessible even if no aggregate location is provided.
234
236 AggSlot = CreateAggTemp(E->getType(), "agg.tmp");
237 return EmitAnyExpr(E, AggSlot);
238}
239
240/// EmitAnyExprToMem - Evaluate an expression into a given memory
241/// location.
243 Address Location,
244 Qualifiers Quals,
245 bool IsInit) {
246 // FIXME: This function should take an LValue as an argument.
247 switch (getEvaluationKind(E->getType())) {
248 case TEK_Complex:
250 /*isInit*/ false);
251 return;
252
253 case TEK_Aggregate: {
254 EmitAggExpr(E, AggValueSlot::forAddr(Location, Quals,
259 return;
260 }
261
262 case TEK_Scalar: {
263 RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false));
264 LValue LV = MakeAddrLValue(Location, E->getType());
266 return;
267 }
268 }
269 llvm_unreachable("bad evaluation kind");
270}
271
272static void
274 const Expr *E, Address ReferenceTemporary) {
275 // Objective-C++ ARC:
276 // If we are binding a reference to a temporary that has ownership, we
277 // need to perform retain/release operations on the temporary.
278 //
279 // FIXME: This should be looking at E, not M.
280 if (auto Lifetime = M->getType().getObjCLifetime()) {
281 switch (Lifetime) {
284 // Carry on to normal cleanup handling.
285 break;
286
288 // Nothing to do; cleaned up by an autorelease pool.
289 return;
290
293 switch (StorageDuration Duration = M->getStorageDuration()) {
294 case SD_Static:
295 // Note: we intentionally do not register a cleanup to release
296 // the object on program termination.
297 return;
298
299 case SD_Thread:
300 // FIXME: We should probably register a cleanup in this case.
301 return;
302
303 case SD_Automatic:
307 if (Lifetime == Qualifiers::OCL_Strong) {
308 const ValueDecl *VD = M->getExtendingDecl();
309 bool Precise =
310 VD && isa<VarDecl>(VD) && VD->hasAttr<ObjCPreciseLifetimeAttr>();
314 } else {
315 // __weak objects always get EH cleanups; otherwise, exceptions
316 // could cause really nasty crashes instead of mere leaks.
319 }
320 if (Duration == SD_FullExpression)
321 CGF.pushDestroy(CleanupKind, ReferenceTemporary,
322 M->getType(), *Destroy,
324 else
325 CGF.pushLifetimeExtendedDestroy(CleanupKind, ReferenceTemporary,
326 M->getType(),
327 *Destroy, CleanupKind & EHCleanup);
328 return;
329
330 case SD_Dynamic:
331 llvm_unreachable("temporary cannot have dynamic storage duration");
332 }
333 llvm_unreachable("unknown storage duration");
334 }
335 }
336
337 CXXDestructorDecl *ReferenceTemporaryDtor = nullptr;
338 if (const RecordType *RT =
340 // Get the destructor for the reference temporary.
341 auto *ClassDecl = cast<CXXRecordDecl>(RT->getDecl());
342 if (!ClassDecl->hasTrivialDestructor())
343 ReferenceTemporaryDtor = ClassDecl->getDestructor();
344 }
345
346 if (!ReferenceTemporaryDtor)
347 return;
348
349 // Call the destructor for the temporary.
350 switch (M->getStorageDuration()) {
351 case SD_Static:
352 case SD_Thread: {
353 llvm::FunctionCallee CleanupFn;
354 llvm::Constant *CleanupArg;
355 if (E->getType()->isArrayType()) {
357 ReferenceTemporary, E->getType(),
359 dyn_cast_or_null<VarDecl>(M->getExtendingDecl()));
360 CleanupArg = llvm::Constant::getNullValue(CGF.Int8PtrTy);
361 } else {
362 CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(
363 GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));
364 CleanupArg = cast<llvm::Constant>(ReferenceTemporary.getPointer());
365 }
367 CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg);
368 break;
369 }
370
372 CGF.pushDestroy(NormalAndEHCleanup, ReferenceTemporary, E->getType(),
374 CGF.getLangOpts().Exceptions);
375 break;
376
377 case SD_Automatic:
379 ReferenceTemporary, E->getType(),
381 CGF.getLangOpts().Exceptions);
382 break;
383
384 case SD_Dynamic:
385 llvm_unreachable("temporary cannot have dynamic storage duration");
386 }
387}
388
391 const Expr *Inner,
392 Address *Alloca = nullptr) {
393 auto &TCG = CGF.getTargetHooks();
394 switch (M->getStorageDuration()) {
396 case SD_Automatic: {
397 // If we have a constant temporary array or record try to promote it into a
398 // constant global under the same rules a normal constant would've been
399 // promoted. This is easier on the optimizer and generally emits fewer
400 // instructions.
401 QualType Ty = Inner->getType();
402 if (CGF.CGM.getCodeGenOpts().MergeAllConstants &&
403 (Ty->isArrayType() || Ty->isRecordType()) &&
404 CGF.CGM.isTypeConstant(Ty, true, false))
405 if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(Inner, Ty)) {
406 auto AS = CGF.CGM.GetGlobalConstantAddressSpace();
407 auto *GV = new llvm::GlobalVariable(
408 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
409 llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr,
410 llvm::GlobalValue::NotThreadLocal,
412 CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty);
413 GV->setAlignment(alignment.getAsAlign());
414 llvm::Constant *C = GV;
415 if (AS != LangAS::Default)
416 C = TCG.performAddrSpaceCast(
417 CGF.CGM, GV, AS, LangAS::Default,
418 GV->getValueType()->getPointerTo(
420 // FIXME: Should we put the new global into a COMDAT?
421 return Address(C, GV->getValueType(), alignment);
422 }
423 return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca);
424 }
425 case SD_Thread:
426 case SD_Static:
427 return CGF.CGM.GetAddrOfGlobalTemporary(M, Inner);
428
429 case SD_Dynamic:
430 llvm_unreachable("temporary can't have dynamic storage duration");
431 }
432 llvm_unreachable("unknown storage duration");
433}
434
435/// Helper method to check if the underlying ABI is AAPCS
436static bool isAAPCS(const TargetInfo &TargetInfo) {
437 return TargetInfo.getABI().startswith("aapcs");
438}
439
442 const Expr *E = M->getSubExpr();
443
444 assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) ||
445 !cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) &&
446 "Reference should never be pseudo-strong!");
447
448 // FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so
449 // as that will cause the lifetime adjustment to be lost for ARC
450 auto ownership = M->getType().getObjCLifetime();
451 if (ownership != Qualifiers::OCL_None &&
452 ownership != Qualifiers::OCL_ExplicitNone) {
454 if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) {
455 llvm::Type *Ty = ConvertTypeForMem(E->getType());
456 Object = Address(llvm::ConstantExpr::getBitCast(
457 Var, Ty->getPointerTo(Object.getAddressSpace())),
458 Ty, Object.getAlignment());
459
460 // createReferenceTemporary will promote the temporary to a global with a
461 // constant initializer if it can. It can only do this to a value of
462 // ARC-manageable type if the value is global and therefore "immune" to
463 // ref-counting operations. Therefore we have no need to emit either a
464 // dynamic initialization or a cleanup and we can just return the address
465 // of the temporary.
466 if (Var->hasInitializer())
467 return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
468
469 Var->setInitializer(CGM.EmitNullConstant(E->getType()));
470 }
471 LValue RefTempDst = MakeAddrLValue(Object, M->getType(),
473
474 switch (getEvaluationKind(E->getType())) {
475 default: llvm_unreachable("expected scalar or aggregate expression");
476 case TEK_Scalar:
477 EmitScalarInit(E, M->getExtendingDecl(), RefTempDst, false);
478 break;
479 case TEK_Aggregate: {
481 E->getType().getQualifiers(),
486 break;
487 }
488 }
489
490 pushTemporaryCleanup(*this, M, E, Object);
491 return RefTempDst;
492 }
493
496 E = E->skipRValueSubobjectAdjustments(CommaLHSs, Adjustments);
497
498 for (const auto &Ignored : CommaLHSs)
499 EmitIgnoredExpr(Ignored);
500
501 if (const auto *opaque = dyn_cast<OpaqueValueExpr>(E)) {
502 if (opaque->getType()->isRecordType()) {
503 assert(Adjustments.empty());
504 return EmitOpaqueValueLValue(opaque);
505 }
506 }
507
508 // Create and initialize the reference temporary.
509 Address Alloca = Address::invalid();
510 Address Object = createReferenceTemporary(*this, M, E, &Alloca);
511 if (auto *Var = dyn_cast<llvm::GlobalVariable>(
512 Object.getPointer()->stripPointerCasts())) {
513 llvm::Type *TemporaryType = ConvertTypeForMem(E->getType());
514 Object = Address(llvm::ConstantExpr::getBitCast(
515 cast<llvm::Constant>(Object.getPointer()),
516 TemporaryType->getPointerTo()),
517 TemporaryType,
518 Object.getAlignment());
519 // If the temporary is a global and has a constant initializer or is a
520 // constant temporary that we promoted to a global, we may have already
521 // initialized it.
522 if (!Var->hasInitializer()) {
523 Var->setInitializer(CGM.EmitNullConstant(E->getType()));
524 EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
525 }
526 } else {
527 switch (M->getStorageDuration()) {
528 case SD_Automatic:
529 if (auto *Size = EmitLifetimeStart(
530 CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()),
531 Alloca.getPointer())) {
532 pushCleanupAfterFullExpr<CallLifetimeEnd>(NormalEHLifetimeMarker,
533 Alloca, Size);
534 }
535 break;
536
537 case SD_FullExpression: {
538 if (!ShouldEmitLifetimeMarkers)
539 break;
540
541 // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end
542 // marker. Instead, start the lifetime of a conditional temporary earlier
543 // so that it's unconditional. Don't do this with sanitizers which need
544 // more precise lifetime marks. However when inside an "await.suspend"
545 // block, we should always avoid conditional cleanup because it creates
546 // boolean marker that lives across await_suspend, which can destroy coro
547 // frame.
548 ConditionalEvaluation *OldConditional = nullptr;
549 CGBuilderTy::InsertPoint OldIP;
551 ((!SanOpts.has(SanitizerKind::HWAddress) &&
552 !SanOpts.has(SanitizerKind::Memory) &&
553 !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) ||
554 inSuspendBlock())) {
555 OldConditional = OutermostConditional;
556 OutermostConditional = nullptr;
557
558 OldIP = Builder.saveIP();
559 llvm::BasicBlock *Block = OldConditional->getStartingBlock();
560 Builder.restoreIP(CGBuilderTy::InsertPoint(
561 Block, llvm::BasicBlock::iterator(Block->back())));
562 }
563
564 if (auto *Size = EmitLifetimeStart(
565 CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()),
566 Alloca.getPointer())) {
567 pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, Alloca,
568 Size);
569 }
570
571 if (OldConditional) {
572 OutermostConditional = OldConditional;
573 Builder.restoreIP(OldIP);
574 }
575 break;
576 }
577
578 default:
579 break;
580 }
581 EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
582 }
583 pushTemporaryCleanup(*this, M, E, Object);
584
585 // Perform derived-to-base casts and/or field accesses, to get from the
586 // temporary object we created (and, potentially, for which we extended
587 // the lifetime) to the subobject we're binding the reference to.
588 for (SubobjectAdjustment &Adjustment : llvm::reverse(Adjustments)) {
589 switch (Adjustment.Kind) {
591 Object =
592 GetAddressOfBaseClass(Object, Adjustment.DerivedToBase.DerivedClass,
593 Adjustment.DerivedToBase.BasePath->path_begin(),
594 Adjustment.DerivedToBase.BasePath->path_end(),
595 /*NullCheckValue=*/ false, E->getExprLoc());
596 break;
597
600 LV = EmitLValueForField(LV, Adjustment.Field);
601 assert(LV.isSimple() &&
602 "materialized temporary field is not a simple lvalue");
603 Object = LV.getAddress(*this);
604 break;
605 }
606
608 llvm::Value *Ptr = EmitScalarExpr(Adjustment.Ptr.RHS);
610 Adjustment.Ptr.MPT);
611 break;
612 }
613 }
614 }
615
616 return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
617}
618
619RValue
621 // Emit the expression as an lvalue.
622 LValue LV = EmitLValue(E);
623 assert(LV.isSimple());
624 llvm::Value *Value = LV.getPointer(*this);
625
627 // C++11 [dcl.ref]p5 (as amended by core issue 453):
628 // If a glvalue to which a reference is directly bound designates neither
629 // an existing object or function of an appropriate type nor a region of
630 // storage of suitable size and alignment to contain an object of the
631 // reference's type, the behavior is undefined.
632 QualType Ty = E->getType();
634 }
635
636 return RValue::get(Value);
637}
638
639
640/// getAccessedFieldNo - Given an encoded value and a result number, return the
641/// input field number being accessed.
642unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
643 const llvm::Constant *Elts) {
644 return cast<llvm::ConstantInt>(Elts->getAggregateElement(Idx))
645 ->getZExtValue();
646}
647
648/// Emit the hash_16_bytes function from include/llvm/ADT/Hashing.h.
649static llvm::Value *emitHash16Bytes(CGBuilderTy &Builder, llvm::Value *Low,
650 llvm::Value *High) {
651 llvm::Value *KMul = Builder.getInt64(0x9ddfea08eb382d69ULL);
652 llvm::Value *K47 = Builder.getInt64(47);
653 llvm::Value *A0 = Builder.CreateMul(Builder.CreateXor(Low, High), KMul);
654 llvm::Value *A1 = Builder.CreateXor(Builder.CreateLShr(A0, K47), A0);
655 llvm::Value *B0 = Builder.CreateMul(Builder.CreateXor(High, A1), KMul);
656 llvm::Value *B1 = Builder.CreateXor(Builder.CreateLShr(B0, K47), B0);
657 return Builder.CreateMul(B1, KMul);
658}
659
660bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) {
661 return TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||
663}
664
665bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {
667 return (RD && RD->hasDefinition() && RD->isDynamicClass()) &&
668 (TCK == TCK_MemberAccess || TCK == TCK_MemberCall ||
671}
672
674 return SanOpts.has(SanitizerKind::Null) ||
675 SanOpts.has(SanitizerKind::Alignment) ||
676 SanOpts.has(SanitizerKind::ObjectSize) ||
677 SanOpts.has(SanitizerKind::Vptr);
678}
679
680void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
681 llvm::Value *Ptr, QualType Ty,
682 CharUnits Alignment,
683 SanitizerSet SkippedChecks,
684 llvm::Value *ArraySize) {
686 return;
687
688 // Don't check pointers outside the default address space. The null check
689 // isn't correct, the object-size check isn't supported by LLVM, and we can't
690 // communicate the addresses to the runtime handler for the vptr check.
691 if (Ptr->getType()->getPointerAddressSpace())
692 return;
693
694 // Don't check pointers to volatile data. The behavior here is implementation-
695 // defined.
696 if (Ty.isVolatileQualified())
697 return;
698
699 SanitizerScope SanScope(this);
700
702 llvm::BasicBlock *Done = nullptr;
703
704 // Quickly determine whether we have a pointer to an alloca. It's possible
705 // to skip null checks, and some alignment checks, for these pointers. This
706 // can reduce compile-time significantly.
707 auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Ptr->stripPointerCasts());
708
709 llvm::Value *True = llvm::ConstantInt::getTrue(getLLVMContext());
710 llvm::Value *IsNonNull = nullptr;
711 bool IsGuaranteedNonNull =
712 SkippedChecks.has(SanitizerKind::Null) || PtrToAlloca;
713 bool AllowNullPointers = isNullPointerAllowed(TCK);
714 if ((SanOpts.has(SanitizerKind::Null) || AllowNullPointers) &&
715 !IsGuaranteedNonNull) {
716 // The glvalue must not be an empty glvalue.
717 IsNonNull = Builder.CreateIsNotNull(Ptr);
718
719 // The IR builder can constant-fold the null check if the pointer points to
720 // a constant.
721 IsGuaranteedNonNull = IsNonNull == True;
722
723 // Skip the null check if the pointer is known to be non-null.
724 if (!IsGuaranteedNonNull) {
725 if (AllowNullPointers) {
726 // When performing pointer casts, it's OK if the value is null.
727 // Skip the remaining checks in that case.
728 Done = createBasicBlock("null");
729 llvm::BasicBlock *Rest = createBasicBlock("not.null");
730 Builder.CreateCondBr(IsNonNull, Rest, Done);
731 EmitBlock(Rest);
732 } else {
733 Checks.push_back(std::make_pair(IsNonNull, SanitizerKind::Null));
734 }
735 }
736 }
737
738 if (SanOpts.has(SanitizerKind::ObjectSize) &&
739 !SkippedChecks.has(SanitizerKind::ObjectSize) &&
740 !Ty->isIncompleteType()) {
742 llvm::Value *Size = llvm::ConstantInt::get(IntPtrTy, TySize);
743 if (ArraySize)
744 Size = Builder.CreateMul(Size, ArraySize);
745
746 // Degenerate case: new X[0] does not need an objectsize check.
747 llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Size);
748 if (!ConstantSize || !ConstantSize->isNullValue()) {
749 // The glvalue must refer to a large enough storage region.
750 // FIXME: If Address Sanitizer is enabled, insert dynamic instrumentation
751 // to check this.
752 // FIXME: Get object address space
753 llvm::Type *Tys[2] = { IntPtrTy, Int8PtrTy };
754 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, Tys);
755 llvm::Value *Min = Builder.getFalse();
756 llvm::Value *NullIsUnknown = Builder.getFalse();
757 llvm::Value *Dynamic = Builder.getFalse();
758 llvm::Value *CastAddr = Builder.CreateBitCast(Ptr, Int8PtrTy);
759 llvm::Value *LargeEnough = Builder.CreateICmpUGE(
760 Builder.CreateCall(F, {CastAddr, Min, NullIsUnknown, Dynamic}), Size);
761 Checks.push_back(std::make_pair(LargeEnough, SanitizerKind::ObjectSize));
762 }
763 }
764
765 llvm::MaybeAlign AlignVal;
766 llvm::Value *PtrAsInt = nullptr;
767
768 if (SanOpts.has(SanitizerKind::Alignment) &&
769 !SkippedChecks.has(SanitizerKind::Alignment)) {
770 AlignVal = Alignment.getAsMaybeAlign();
771 if (!Ty->isIncompleteType() && !AlignVal)
772 AlignVal = CGM.getNaturalTypeAlignment(Ty, nullptr, nullptr,
773 /*ForPointeeType=*/true)
775
776 // The glvalue must be suitably aligned.
777 if (AlignVal && *AlignVal > llvm::Align(1) &&
778 (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
779 PtrAsInt = Builder.CreatePtrToInt(Ptr, IntPtrTy);
780 llvm::Value *Align = Builder.CreateAnd(
781 PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal->value() - 1));
782 llvm::Value *Aligned =
783 Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0));
784 if (Aligned != True)
785 Checks.push_back(std::make_pair(Aligned, SanitizerKind::Alignment));
786 }
787 }
788
789 if (Checks.size() > 0) {
790 llvm::Constant *StaticData[] = {
792 llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2(*AlignVal) : 1),
793 llvm::ConstantInt::get(Int8Ty, TCK)};
794 EmitCheck(Checks, SanitizerHandler::TypeMismatch, StaticData,
795 PtrAsInt ? PtrAsInt : Ptr);
796 }
797
798 // If possible, check that the vptr indicates that there is a subobject of
799 // type Ty at offset zero within this object.
800 //
801 // C++11 [basic.life]p5,6:
802 // [For storage which does not refer to an object within its lifetime]
803 // The program has undefined behavior if:
804 // -- the [pointer or glvalue] is used to access a non-static data member
805 // or call a non-static member function
806 if (SanOpts.has(SanitizerKind::Vptr) &&
807 !SkippedChecks.has(SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) {
808 // Ensure that the pointer is non-null before loading it. If there is no
809 // compile-time guarantee, reuse the run-time null check or emit a new one.
810 if (!IsGuaranteedNonNull) {
811 if (!IsNonNull)
812 IsNonNull = Builder.CreateIsNotNull(Ptr);
813 if (!Done)
814 Done = createBasicBlock("vptr.null");
815 llvm::BasicBlock *VptrNotNull = createBasicBlock("vptr.not.null");
816 Builder.CreateCondBr(IsNonNull, VptrNotNull, Done);
817 EmitBlock(VptrNotNull);
818 }
819
820 // Compute a hash of the mangled name of the type.
821 //
822 // FIXME: This is not guaranteed to be deterministic! Move to a
823 // fingerprinting mechanism once LLVM provides one. For the time
824 // being the implementation happens to be deterministic.
825 SmallString<64> MangledName;
826 llvm::raw_svector_ostream Out(MangledName);
828 Out);
829
830 // Contained in NoSanitizeList based on the mangled type.
831 if (!CGM.getContext().getNoSanitizeList().containsType(SanitizerKind::Vptr,
832 Out.str())) {
833 llvm::hash_code TypeHash = hash_value(Out.str());
834
835 // Load the vptr, and compute hash_16_bytes(TypeHash, vptr).
836 llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash);
837 llvm::Type *VPtrTy = llvm::PointerType::get(IntPtrTy, 0);
838 Address VPtrAddr(Builder.CreateBitCast(Ptr, VPtrTy), IntPtrTy,
840 llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr);
841 llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty);
842
843 llvm::Value *Hash = emitHash16Bytes(Builder, Low, High);
844 Hash = Builder.CreateTrunc(Hash, IntPtrTy);
845
846 // Look the hash up in our cache.
847 const int CacheSize = 128;
848 llvm::Type *HashTable = llvm::ArrayType::get(IntPtrTy, CacheSize);
849 llvm::Value *Cache = CGM.CreateRuntimeVariable(HashTable,
850 "__ubsan_vptr_type_cache");
851 llvm::Value *Slot = Builder.CreateAnd(Hash,
852 llvm::ConstantInt::get(IntPtrTy,
853 CacheSize-1));
854 llvm::Value *Indices[] = { Builder.getInt32(0), Slot };
855 llvm::Value *CacheVal = Builder.CreateAlignedLoad(
856 IntPtrTy, Builder.CreateInBoundsGEP(HashTable, Cache, Indices),
858
859 // If the hash isn't in the cache, call a runtime handler to perform the
860 // hard work of checking whether the vptr is for an object of the right
861 // type. This will either fill in the cache and return, or produce a
862 // diagnostic.
863 llvm::Value *EqualHash = Builder.CreateICmpEQ(CacheVal, Hash);
864 llvm::Constant *StaticData[] = {
868 llvm::ConstantInt::get(Int8Ty, TCK)
869 };
870 llvm::Value *DynamicData[] = { Ptr, Hash };
871 EmitCheck(std::make_pair(EqualHash, SanitizerKind::Vptr),
872 SanitizerHandler::DynamicTypeCacheMiss, StaticData,
873 DynamicData);
874 }
875 }
876
877 if (Done) {
878 Builder.CreateBr(Done);
879 EmitBlock(Done);
880 }
881}
882
883llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E,
884 QualType EltTy) {
886 uint64_t EltSize = C.getTypeSizeInChars(EltTy).getQuantity();
887 if (!EltSize)
888 return nullptr;
889
890 auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts());
891 if (!ArrayDeclRef)
892 return nullptr;
893
894 auto *ParamDecl = dyn_cast<ParmVarDecl>(ArrayDeclRef->getDecl());
895 if (!ParamDecl)
896 return nullptr;
897
898 auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();
899 if (!POSAttr)
900 return nullptr;
901
902 // Don't load the size if it's a lower bound.
903 int POSType = POSAttr->getType();
904 if (POSType != 0 && POSType != 1)
905 return nullptr;
906
907 // Find the implicit size parameter.
908 auto PassedSizeIt = SizeArguments.find(ParamDecl);
909 if (PassedSizeIt == SizeArguments.end())
910 return nullptr;
911
912 const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second;
913 assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable");
914 Address AddrOfSize = LocalDeclMap.find(PassedSizeDecl)->second;
915 llvm::Value *SizeInBytes = EmitLoadOfScalar(AddrOfSize, /*Volatile=*/false,
916 C.getSizeType(), E->getExprLoc());
917 llvm::Value *SizeOfElement =
918 llvm::ConstantInt::get(SizeInBytes->getType(), EltSize);
919 return Builder.CreateUDiv(SizeInBytes, SizeOfElement);
920}
921
922/// If Base is known to point to the start of an array, return the length of
923/// that array. Return 0 if the length cannot be determined.
924static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
925 const Expr *Base,
926 QualType &IndexedType,
928 StrictFlexArraysLevel) {
929 // For the vector indexing extension, the bound is the number of elements.
930 if (const VectorType *VT = Base->getType()->getAs<VectorType>()) {
931 IndexedType = Base->getType();
932 return CGF.Builder.getInt32(VT->getNumElements());
933 }
934
935 Base = Base->IgnoreParens();
936
937 if (const auto *CE = dyn_cast<CastExpr>(Base)) {
938 if (CE->getCastKind() == CK_ArrayToPointerDecay &&
939 !CE->getSubExpr()->isFlexibleArrayMemberLike(CGF.getContext(),
940 StrictFlexArraysLevel)) {
941 IndexedType = CE->getSubExpr()->getType();
942 const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();
943 if (const auto *CAT = dyn_cast<ConstantArrayType>(AT))
944 return CGF.Builder.getInt(CAT->getSize());
945 else if (const auto *VAT = dyn_cast<VariableArrayType>(AT))
946 return CGF.getVLASize(VAT).NumElts;
947 // Ignore pass_object_size here. It's not applicable on decayed pointers.
948 }
949 }
950
951 QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};
952 if (llvm::Value *POS = CGF.LoadPassedObjectSize(Base, EltTy)) {
953 IndexedType = Base->getType();
954 return POS;
955 }
956
957 return nullptr;
958}
959
960void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base,
961 llvm::Value *Index, QualType IndexType,
962 bool Accessed) {
963 assert(SanOpts.has(SanitizerKind::ArrayBounds) &&
964 "should not be called unless adding bounds checks");
965 SanitizerScope SanScope(this);
966
967 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
968 getLangOpts().getStrictFlexArraysLevel();
969
970 QualType IndexedType;
971 llvm::Value *Bound =
972 getArrayIndexingBound(*this, Base, IndexedType, StrictFlexArraysLevel);
973 if (!Bound)
974 return;
975
976 bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
977 llvm::Value *IndexVal = Builder.CreateIntCast(Index, SizeTy, IndexSigned);
978 llvm::Value *BoundVal = Builder.CreateIntCast(Bound, SizeTy, false);
979
980 llvm::Constant *StaticData[] = {
982 EmitCheckTypeDescriptor(IndexedType),
983 EmitCheckTypeDescriptor(IndexType)
984 };
985 llvm::Value *Check = Accessed ? Builder.CreateICmpULT(IndexVal, BoundVal)
986 : Builder.CreateICmpULE(IndexVal, BoundVal);
987 EmitCheck(std::make_pair(Check, SanitizerKind::ArrayBounds),
988 SanitizerHandler::OutOfBounds, StaticData, Index);
989}
990
991
994 bool isInc, bool isPre) {
996
997 llvm::Value *NextVal;
998 if (isa<llvm::IntegerType>(InVal.first->getType())) {
999 uint64_t AmountVal = isInc ? 1 : -1;
1000 NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal, true);
1001
1002 // Add the inc/dec to the real part.
1003 NextVal = Builder.CreateAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
1004 } else {
1005 QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType();
1006 llvm::APFloat FVal(getContext().getFloatTypeSemantics(ElemTy), 1);
1007 if (!isInc)
1008 FVal.changeSign();
1009 NextVal = llvm::ConstantFP::get(getLLVMContext(), FVal);
1010
1011 // Add the inc/dec to the real part.
1012 NextVal = Builder.CreateFAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
1013 }
1014
1015 ComplexPairTy IncVal(NextVal, InVal.second);
1016
1017 // Store the updated result through the lvalue.
1018 EmitStoreOfComplex(IncVal, LV, /*init*/ false);
1019 if (getLangOpts().OpenMP)
1021 E->getSubExpr());
1022
1023 // If this is a postinc, return the value read from memory, otherwise use the
1024 // updated value.
1025 return isPre ? IncVal : InVal;
1026}
1027
1029 CodeGenFunction *CGF) {
1030 // Bind VLAs in the cast type.
1031 if (CGF && E->getType()->isVariablyModifiedType())
1033
1034 if (CGDebugInfo *DI = getModuleDebugInfo())
1035 DI->EmitExplicitCastType(E->getType());
1036}
1037
1038//===----------------------------------------------------------------------===//
1039// LValue Expression Emission
1040//===----------------------------------------------------------------------===//
1041
1043 TBAAAccessInfo *TBAAInfo,
1044 KnownNonNull_t IsKnownNonNull,
1045 CodeGenFunction &CGF) {
1046 // We allow this with ObjC object pointers because of fragile ABIs.
1047 assert(E->getType()->isPointerType() ||
1049 E = E->IgnoreParens();
1050
1051 // Casts:
1052 if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {
1053 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(CE))
1054 CGF.CGM.EmitExplicitCastExprType(ECE, &CGF);
1055
1056 switch (CE->getCastKind()) {
1057 // Non-converting casts (but not C's implicit conversion from void*).
1058 case CK_BitCast:
1059 case CK_NoOp:
1060 case CK_AddressSpaceConversion:
1061 if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) {
1062 if (PtrTy->getPointeeType()->isVoidType())
1063 break;
1064
1065 LValueBaseInfo InnerBaseInfo;
1066 TBAAAccessInfo InnerTBAAInfo;
1068 CE->getSubExpr(), &InnerBaseInfo, &InnerTBAAInfo, IsKnownNonNull);
1069 if (BaseInfo) *BaseInfo = InnerBaseInfo;
1070 if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;
1071
1072 if (isa<ExplicitCastExpr>(CE)) {
1073 LValueBaseInfo TargetTypeBaseInfo;
1074 TBAAAccessInfo TargetTypeTBAAInfo;
1076 E->getType(), &TargetTypeBaseInfo, &TargetTypeTBAAInfo);
1077 if (TBAAInfo)
1078 *TBAAInfo =
1079 CGF.CGM.mergeTBAAInfoForCast(*TBAAInfo, TargetTypeTBAAInfo);
1080 // If the source l-value is opaque, honor the alignment of the
1081 // casted-to type.
1082 if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
1083 if (BaseInfo)
1084 BaseInfo->mergeForCast(TargetTypeBaseInfo);
1085 Addr = Address(Addr.getPointer(), Addr.getElementType(), Align,
1086 IsKnownNonNull);
1087 }
1088 }
1089
1090 if (CGF.SanOpts.has(SanitizerKind::CFIUnrelatedCast) &&
1091 CE->getCastKind() == CK_BitCast) {
1092 if (auto PT = E->getType()->getAs<PointerType>())
1093 CGF.EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr,
1094 /*MayBeNull=*/true,
1096 CE->getBeginLoc());
1097 }
1098
1099 llvm::Type *ElemTy =
1101 Addr = CGF.Builder.CreateElementBitCast(Addr, ElemTy);
1102 if (CE->getCastKind() == CK_AddressSpaceConversion)
1103 Addr = CGF.Builder.CreateAddrSpaceCast(Addr,
1104 CGF.ConvertType(E->getType()));
1105 return Addr;
1106 }
1107 break;
1108
1109 // Array-to-pointer decay.
1110 case CK_ArrayToPointerDecay:
1111 return CGF.EmitArrayToPointerDecay(CE->getSubExpr(), BaseInfo, TBAAInfo);
1112
1113 // Derived-to-base conversions.
1114 case CK_UncheckedDerivedToBase:
1115 case CK_DerivedToBase: {
1116 // TODO: Support accesses to members of base classes in TBAA. For now, we
1117 // conservatively pretend that the complete object is of the base class
1118 // type.
1119 if (TBAAInfo)
1120 *TBAAInfo = CGF.CGM.getTBAAAccessInfo(E->getType());
1122 CE->getSubExpr(), BaseInfo, nullptr,
1123 (KnownNonNull_t)(IsKnownNonNull ||
1124 CE->getCastKind() == CK_UncheckedDerivedToBase));
1125 auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl();
1126 return CGF.GetAddressOfBaseClass(
1127 Addr, Derived, CE->path_begin(), CE->path_end(),
1128 CGF.ShouldNullCheckClassCastValue(CE), CE->getExprLoc());
1129 }
1130
1131 // TODO: Is there any reason to treat base-to-derived conversions
1132 // specially?
1133 default:
1134 break;
1135 }
1136 }
1137
1138 // Unary &.
1139 if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) {
1140 if (UO->getOpcode() == UO_AddrOf) {
1141 LValue LV = CGF.EmitLValue(UO->getSubExpr(), IsKnownNonNull);
1142 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1143 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1144 return LV.getAddress(CGF);
1145 }
1146 }
1147
1148 // std::addressof and variants.
1149 if (auto *Call = dyn_cast<CallExpr>(E)) {
1150 switch (Call->getBuiltinCallee()) {
1151 default:
1152 break;
1153 case Builtin::BIaddressof:
1154 case Builtin::BI__addressof:
1155 case Builtin::BI__builtin_addressof: {
1156 LValue LV = CGF.EmitLValue(Call->getArg(0), IsKnownNonNull);
1157 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1158 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1159 return LV.getAddress(CGF);
1160 }
1161 }
1162 }
1163
1164 // TODO: conditional operators, comma.
1165
1166 // Otherwise, use the alignment of the type.
1167 CharUnits Align =
1168 CGF.CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo);
1169 llvm::Type *ElemTy = CGF.ConvertTypeForMem(E->getType()->getPointeeType());
1170 return Address(CGF.EmitScalarExpr(E), ElemTy, Align, IsKnownNonNull);
1171}
1172
1173/// EmitPointerWithAlignment - Given an expression of pointer type, try to
1174/// derive a more accurate bound on the alignment of the pointer.
1176 const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo,
1177 KnownNonNull_t IsKnownNonNull) {
1178 Address Addr =
1179 ::EmitPointerWithAlignment(E, BaseInfo, TBAAInfo, IsKnownNonNull, *this);
1180 if (IsKnownNonNull && !Addr.isKnownNonNull())
1181 Addr.setKnownNonNull();
1182 return Addr;
1183}
1184
1186 llvm::Value *V = RV.getScalarVal();
1187 if (auto MPT = T->getAs<MemberPointerType>())
1188 return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, V, MPT);
1189 return Builder.CreateICmpNE(V, llvm::Constant::getNullValue(V->getType()));
1190}
1191
1193 if (Ty->isVoidType())
1194 return RValue::get(nullptr);
1195
1196 switch (getEvaluationKind(Ty)) {
1197 case TEK_Complex: {
1198 llvm::Type *EltTy =
1200 llvm::Value *U = llvm::UndefValue::get(EltTy);
1201 return RValue::getComplex(std::make_pair(U, U));
1202 }
1203
1204 // If this is a use of an undefined aggregate type, the aggregate must have an
1205 // identifiable address. Just because the contents of the value are undefined
1206 // doesn't mean that the address can't be taken and compared.
1207 case TEK_Aggregate: {
1208 Address DestPtr = CreateMemTemp(Ty, "undef.agg.tmp");
1209 return RValue::getAggregate(DestPtr);
1210 }
1211
1212 case TEK_Scalar:
1213 return RValue::get(llvm::UndefValue::get(ConvertType(Ty)));
1214 }
1215 llvm_unreachable("bad evaluation kind");
1216}
1217
1219 const char *Name) {
1220 ErrorUnsupported(E, Name);
1221 return GetUndefRValue(E->getType());
1222}
1223
1225 const char *Name) {
1226 ErrorUnsupported(E, Name);
1227 llvm::Type *ElTy = ConvertType(E->getType());
1228 llvm::Type *Ty = llvm::PointerType::getUnqual(ElTy);
1229 return MakeAddrLValue(
1230 Address(llvm::UndefValue::get(Ty), ElTy, CharUnits::One()), E->getType());
1231}
1232
1233bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) {
1234 const Expr *Base = Obj;
1235 while (!isa<CXXThisExpr>(Base)) {
1236 // The result of a dynamic_cast can be null.
1237 if (isa<CXXDynamicCastExpr>(Base))
1238 return false;
1239
1240 if (const auto *CE = dyn_cast<CastExpr>(Base)) {
1241 Base = CE->getSubExpr();
1242 } else if (const auto *PE = dyn_cast<ParenExpr>(Base)) {
1243 Base = PE->getSubExpr();
1244 } else if (const auto *UO = dyn_cast<UnaryOperator>(Base)) {
1245 if (UO->getOpcode() == UO_Extension)
1246 Base = UO->getSubExpr();
1247 else
1248 return false;
1249 } else {
1250 return false;
1251 }
1252 }
1253 return true;
1254}
1255
1256LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
1257 LValue LV;
1258 if (SanOpts.has(SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(E))
1259 LV = EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E), /*Accessed*/true);
1260 else
1261 LV = EmitLValue(E);
1262 if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple()) {
1263 SanitizerSet SkippedChecks;
1264 if (const auto *ME = dyn_cast<MemberExpr>(E)) {
1265 bool IsBaseCXXThis = IsWrappedCXXThis(ME->getBase());
1266 if (IsBaseCXXThis)
1267 SkippedChecks.set(SanitizerKind::Alignment, true);
1268 if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase()))
1269 SkippedChecks.set(SanitizerKind::Null, true);
1270 }
1271 EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(),
1272 LV.getAlignment(), SkippedChecks);
1273 }
1274 return LV;
1275}
1276
1277/// EmitLValue - Emit code to compute a designator that specifies the location
1278/// of the expression.
1279///
1280/// This can return one of two things: a simple address or a bitfield reference.
1281/// In either case, the LLVM Value* in the LValue structure is guaranteed to be
1282/// an LLVM pointer type.
1283///
1284/// If this returns a bitfield reference, nothing about the pointee type of the
1285/// LLVM value is known: For example, it may not be a pointer to an integer.
1286///
1287/// If this returns a normal address, and if the lvalue's C type is fixed size,
1288/// this method guarantees that the returned pointer type will point to an LLVM
1289/// type of the same size of the lvalue's type. If the lvalue has a variable
1290/// length type, this is not possible.
1291///
1293 KnownNonNull_t IsKnownNonNull) {
1294 LValue LV = EmitLValueHelper(E, IsKnownNonNull);
1295 if (IsKnownNonNull && !LV.isKnownNonNull())
1296 LV.setKnownNonNull();
1297 return LV;
1298}
1299
1300LValue CodeGenFunction::EmitLValueHelper(const Expr *E,
1301 KnownNonNull_t IsKnownNonNull) {
1302 ApplyDebugLocation DL(*this, E);
1303 switch (E->getStmtClass()) {
1304 default: return EmitUnsupportedLValue(E, "l-value expression");
1305
1306 case Expr::ObjCPropertyRefExprClass:
1307 llvm_unreachable("cannot emit a property reference directly");
1308
1309 case Expr::ObjCSelectorExprClass:
1310 return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E));
1311 case Expr::ObjCIsaExprClass:
1312 return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E));
1313 case Expr::BinaryOperatorClass:
1314 return EmitBinaryOperatorLValue(cast<BinaryOperator>(E));
1315 case Expr::CompoundAssignOperatorClass: {
1316 QualType Ty = E->getType();
1317 if (const AtomicType *AT = Ty->getAs<AtomicType>())
1318 Ty = AT->getValueType();
1319 if (!Ty->isAnyComplexType())
1320 return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
1321 return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
1322 }
1323 case Expr::CallExprClass:
1324 case Expr::CXXMemberCallExprClass:
1325 case Expr::CXXOperatorCallExprClass:
1326 case Expr::UserDefinedLiteralClass:
1327 return EmitCallExprLValue(cast<CallExpr>(E));
1328 case Expr::CXXRewrittenBinaryOperatorClass:
1329 return EmitLValue(cast<CXXRewrittenBinaryOperator>(E)->getSemanticForm(),
1330 IsKnownNonNull);
1331 case Expr::VAArgExprClass:
1332 return EmitVAArgExprLValue(cast<VAArgExpr>(E));
1333 case Expr::DeclRefExprClass:
1334 return EmitDeclRefLValue(cast<DeclRefExpr>(E));
1335 case Expr::ConstantExprClass: {
1336 const ConstantExpr *CE = cast<ConstantExpr>(E);
1337 if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE)) {
1338 QualType RetType = cast<CallExpr>(CE->getSubExpr()->IgnoreImplicit())
1339 ->getCallReturnType(getContext())
1340 ->getPointeeType();
1341 return MakeNaturalAlignAddrLValue(Result, RetType);
1342 }
1343 return EmitLValue(cast<ConstantExpr>(E)->getSubExpr(), IsKnownNonNull);
1344 }
1345 case Expr::ParenExprClass:
1346 return EmitLValue(cast<ParenExpr>(E)->getSubExpr(), IsKnownNonNull);
1347 case Expr::GenericSelectionExprClass:
1348 return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr(),
1349 IsKnownNonNull);
1350 case Expr::PredefinedExprClass:
1351 return EmitPredefinedLValue(cast<PredefinedExpr>(E));
1352 case Expr::StringLiteralClass:
1353 return EmitStringLiteralLValue(cast<StringLiteral>(E));
1354 case Expr::ObjCEncodeExprClass:
1355 return EmitObjCEncodeExprLValue(cast<ObjCEncodeExpr>(E));
1356 case Expr::PseudoObjectExprClass:
1357 return EmitPseudoObjectLValue(cast<PseudoObjectExpr>(E));
1358 case Expr::InitListExprClass:
1359 return EmitInitListLValue(cast<InitListExpr>(E));
1360 case Expr::CXXTemporaryObjectExprClass:
1361 case Expr::CXXConstructExprClass:
1362 return EmitCXXConstructLValue(cast<CXXConstructExpr>(E));
1363 case Expr::CXXBindTemporaryExprClass:
1364 return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E));
1365 case Expr::CXXUuidofExprClass:
1366 return EmitCXXUuidofLValue(cast<CXXUuidofExpr>(E));
1367 case Expr::LambdaExprClass:
1368 return EmitAggExprToLValue(E);
1369
1370 case Expr::ExprWithCleanupsClass: {
1371 const auto *cleanups = cast<ExprWithCleanups>(E);
1372 RunCleanupsScope Scope(*this);
1373 LValue LV = EmitLValue(cleanups->getSubExpr(), IsKnownNonNull);
1374 if (LV.isSimple()) {
1375 // Defend against branches out of gnu statement expressions surrounded by
1376 // cleanups.
1377 Address Addr = LV.getAddress(*this);
1378 llvm::Value *V = Addr.getPointer();
1379 Scope.ForceCleanup({&V});
1380 return LValue::MakeAddr(Addr.withPointer(V, Addr.isKnownNonNull()),
1381 LV.getType(), getContext(), LV.getBaseInfo(),
1382 LV.getTBAAInfo());
1383 }
1384 // FIXME: Is it possible to create an ExprWithCleanups that produces a
1385 // bitfield lvalue or some other non-simple lvalue?
1386 return LV;
1387 }
1388
1389 case Expr::CXXDefaultArgExprClass: {
1390 auto *DAE = cast<CXXDefaultArgExpr>(E);
1391 CXXDefaultArgExprScope Scope(*this, DAE);
1392 return EmitLValue(DAE->getExpr(), IsKnownNonNull);
1393 }
1394 case Expr::CXXDefaultInitExprClass: {
1395 auto *DIE = cast<CXXDefaultInitExpr>(E);
1396 CXXDefaultInitExprScope Scope(*this, DIE);
1397 return EmitLValue(DIE->getExpr(), IsKnownNonNull);
1398 }
1399 case Expr::CXXTypeidExprClass:
1400 return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E));
1401
1402 case Expr::ObjCMessageExprClass:
1403 return EmitObjCMessageExprLValue(cast<ObjCMessageExpr>(E));
1404 case Expr::ObjCIvarRefExprClass:
1405 return EmitObjCIvarRefLValue(cast<ObjCIvarRefExpr>(E));
1406 case Expr::StmtExprClass:
1407 return EmitStmtExprLValue(cast<StmtExpr>(E));
1408 case Expr::UnaryOperatorClass:
1409 return EmitUnaryOpLValue(cast<UnaryOperator>(E));
1410 case Expr::ArraySubscriptExprClass:
1411 return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E));
1412 case Expr::MatrixSubscriptExprClass:
1413 return EmitMatrixSubscriptExpr(cast<MatrixSubscriptExpr>(E));
1414 case Expr::OMPArraySectionExprClass:
1415 return EmitOMPArraySectionExpr(cast<OMPArraySectionExpr>(E));
1416 case Expr::ExtVectorElementExprClass:
1417 return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E));
1418 case Expr::CXXThisExprClass:
1420 case Expr::MemberExprClass:
1421 return EmitMemberExpr(cast<MemberExpr>(E));
1422 case Expr::CompoundLiteralExprClass:
1423 return EmitCompoundLiteralLValue(cast<CompoundLiteralExpr>(E));
1424 case Expr::ConditionalOperatorClass:
1425 return EmitConditionalOperatorLValue(cast<ConditionalOperator>(E));
1426 case Expr::BinaryConditionalOperatorClass:
1427 return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E));
1428 case Expr::ChooseExprClass:
1429 return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr(), IsKnownNonNull);
1430 case Expr::OpaqueValueExprClass:
1431 return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E));
1432 case Expr::SubstNonTypeTemplateParmExprClass:
1433 return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement(),
1434 IsKnownNonNull);
1435 case Expr::ImplicitCastExprClass:
1436 case Expr::CStyleCastExprClass:
1437 case Expr::CXXFunctionalCastExprClass:
1438 case Expr::CXXStaticCastExprClass:
1439 case Expr::CXXDynamicCastExprClass:
1440 case Expr::CXXReinterpretCastExprClass:
1441 case Expr::CXXConstCastExprClass:
1442 case Expr::CXXAddrspaceCastExprClass:
1443 case Expr::ObjCBridgedCastExprClass:
1444 return EmitCastLValue(cast<CastExpr>(E));
1445
1446 case Expr::MaterializeTemporaryExprClass:
1447 return EmitMaterializeTemporaryExpr(cast<MaterializeTemporaryExpr>(E));
1448
1449 case Expr::CoawaitExprClass:
1450 return EmitCoawaitLValue(cast<CoawaitExpr>(E));
1451 case Expr::CoyieldExprClass:
1452 return EmitCoyieldLValue(cast<CoyieldExpr>(E));
1453 }
1454}
1455
1456/// Given an object of the given canonical type, can we safely copy a
1457/// value out of it based on its initializer?
1459 assert(type.isCanonical());
1460 assert(!type->isReferenceType());
1461
1462 // Must be const-qualified but non-volatile.
1463 Qualifiers qs = type.getLocalQualifiers();
1464 if (!qs.hasConst() || qs.hasVolatile()) return false;
1465
1466 // Otherwise, all object types satisfy this except C++ classes with
1467 // mutable subobjects or non-trivial copy/destroy behavior.
1468 if (const auto *RT = dyn_cast<RecordType>(type))
1469 if (const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1470 if (RD->hasMutableFields() || !RD->isTrivial())
1471 return false;
1472
1473 return true;
1474}
1475
1476/// Can we constant-emit a load of a reference to a variable of the
1477/// given type? This is different from predicates like
1478/// Decl::mightBeUsableInConstantExpressions because we do want it to apply
1479/// in situations that don't necessarily satisfy the language's rules
1480/// for this (e.g. C++'s ODR-use rules). For example, we want to able
1481/// to do this with const float variables even if those variables
1482/// aren't marked 'constexpr'.
1490 type = type.getCanonicalType();
1491 if (const auto *ref = dyn_cast<ReferenceType>(type)) {
1492 if (isConstantEmittableObjectType(ref->getPointeeType()))
1494 return CEK_AsReferenceOnly;
1495 }
1497 return CEK_AsValueOnly;
1498 return CEK_None;
1499}
1500
1501/// Try to emit a reference to the given value without producing it as
1502/// an l-value. This is just an optimization, but it avoids us needing
1503/// to emit global copies of variables if they're named without triggering
1504/// a formal use in a context where we can't emit a direct reference to them,
1505/// for instance if a block or lambda or a member of a local class uses a
1506/// const int variable or constexpr variable from an enclosing function.
1507CodeGenFunction::ConstantEmission
1509 ValueDecl *value = refExpr->getDecl();
1510
1511 // The value needs to be an enum constant or a constant variable.
1513 if (isa<ParmVarDecl>(value)) {
1514 CEK = CEK_None;
1515 } else if (auto *var = dyn_cast<VarDecl>(value)) {
1516 CEK = checkVarTypeForConstantEmission(var->getType());
1517 } else if (isa<EnumConstantDecl>(value)) {
1518 CEK = CEK_AsValueOnly;
1519 } else {
1520 CEK = CEK_None;
1521 }
1522 if (CEK == CEK_None) return ConstantEmission();
1523
1524 Expr::EvalResult result;
1525 bool resultIsReference;
1526 QualType resultType;
1527
1528 // It's best to evaluate all the way as an r-value if that's permitted.
1529 if (CEK != CEK_AsReferenceOnly &&
1530 refExpr->EvaluateAsRValue(result, getContext())) {
1531 resultIsReference = false;
1532 resultType = refExpr->getType();
1533
1534 // Otherwise, try to evaluate as an l-value.
1535 } else if (CEK != CEK_AsValueOnly &&
1536 refExpr->EvaluateAsLValue(result, getContext())) {
1537 resultIsReference = true;
1538 resultType = value->getType();
1539
1540 // Failure.
1541 } else {
1542 return ConstantEmission();
1543 }
1544
1545 // In any case, if the initializer has side-effects, abandon ship.
1546 if (result.HasSideEffects)
1547 return ConstantEmission();
1548
1549 // In CUDA/HIP device compilation, a lambda may capture a reference variable
1550 // referencing a global host variable by copy. In this case the lambda should
1551 // make a copy of the value of the global host variable. The DRE of the
1552 // captured reference variable cannot be emitted as load from the host
1553 // global variable as compile time constant, since the host variable is not
1554 // accessible on device. The DRE of the captured reference variable has to be
1555 // loaded from captures.
1556 if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
1558 auto *MD = dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl);
1559 if (MD && MD->getParent()->isLambda() &&
1560 MD->getOverloadedOperator() == OO_Call) {
1561 const APValue::LValueBase &base = result.Val.getLValueBase();
1562 if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {
1563 if (const VarDecl *VD = dyn_cast<const VarDecl>(D)) {
1564 if (!VD->hasAttr<CUDADeviceAttr>()) {
1565 return ConstantEmission();
1566 }
1567 }
1568 }
1569 }
1570 }
1571
1572 // Emit as a constant.
1573 auto C = ConstantEmitter(*this).emitAbstract(refExpr->getLocation(),
1574 result.Val, resultType);
1575
1576 // Make sure we emit a debug reference to the global variable.
1577 // This should probably fire even for
1578 if (isa<VarDecl>(value)) {
1579 if (!getContext().DeclMustBeEmitted(cast<VarDecl>(value)))
1580 EmitDeclRefExprDbgValue(refExpr, result.Val);
1581 } else {
1582 assert(isa<EnumConstantDecl>(value));
1583 EmitDeclRefExprDbgValue(refExpr, result.Val);
1584 }
1585
1586 // If we emitted a reference constant, we need to dereference that.
1587 if (resultIsReference)
1589
1591}
1592
1594 const MemberExpr *ME) {
1595 if (auto *VD = dyn_cast<VarDecl>(ME->getMemberDecl())) {
1596 // Try to emit static variable member expressions as DREs.
1597 return DeclRefExpr::Create(
1599 /*RefersToEnclosingVariableOrCapture=*/false, ME->getExprLoc(),
1600 ME->getType(), ME->getValueKind(), nullptr, nullptr, ME->isNonOdrUse());
1601 }
1602 return nullptr;
1603}
1604
1605CodeGenFunction::ConstantEmission
1608 return tryEmitAsConstant(DRE);
1609 return ConstantEmission();
1610}
1611
1613 const CodeGenFunction::ConstantEmission &Constant, Expr *E) {
1614 assert(Constant && "not a constant");
1615 if (Constant.isReference())
1616 return EmitLoadOfLValue(Constant.getReferenceLValue(*this, E),
1617 E->getExprLoc())
1618 .getScalarVal();
1619 return Constant.getValue();
1620}
1621
1622llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
1623 SourceLocation Loc) {
1624 return EmitLoadOfScalar(lvalue.getAddress(*this), lvalue.isVolatile(),
1625 lvalue.getType(), Loc, lvalue.getBaseInfo(),
1626 lvalue.getTBAAInfo(), lvalue.isNontemporal());
1627}
1628
1630 if (Ty->isBooleanType())
1631 return true;
1632
1633 if (const EnumType *ET = Ty->getAs<EnumType>())
1634 return ET->getDecl()->getIntegerType()->isBooleanType();
1635
1636 if (const AtomicType *AT = Ty->getAs<AtomicType>())
1637 return hasBooleanRepresentation(AT->getValueType());
1638
1639 return false;
1640}
1641
1643 llvm::APInt &Min, llvm::APInt &End,
1644 bool StrictEnums, bool IsBool) {
1645 const EnumType *ET = Ty->getAs<EnumType>();
1646 bool IsRegularCPlusPlusEnum = CGF.getLangOpts().CPlusPlus && StrictEnums &&
1647 ET && !ET->getDecl()->isFixed();
1648 if (!IsBool && !IsRegularCPlusPlusEnum)
1649 return false;
1650
1651 if (IsBool) {
1652 Min = llvm::APInt(CGF.getContext().getTypeSize(Ty), 0);
1653 End = llvm::APInt(CGF.getContext().getTypeSize(Ty), 2);
1654 } else {
1655 const EnumDecl *ED = ET->getDecl();
1656 ED->getValueRange(End, Min);
1657 }
1658 return true;
1659}
1660
1661llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
1662 llvm::APInt Min, End;
1663 if (!getRangeForType(*this, Ty, Min, End, CGM.getCodeGenOpts().StrictEnums,
1665 return nullptr;
1666
1667 llvm::MDBuilder MDHelper(getLLVMContext());
1668 return MDHelper.createRange(Min, End);
1669}
1670
1672 SourceLocation Loc) {
1673 bool HasBoolCheck = SanOpts.has(SanitizerKind::Bool);
1674 bool HasEnumCheck = SanOpts.has(SanitizerKind::Enum);
1675 if (!HasBoolCheck && !HasEnumCheck)
1676 return false;
1677
1678 bool IsBool = hasBooleanRepresentation(Ty) ||
1680 bool NeedsBoolCheck = HasBoolCheck && IsBool;
1681 bool NeedsEnumCheck = HasEnumCheck && Ty->getAs<EnumType>();
1682 if (!NeedsBoolCheck && !NeedsEnumCheck)
1683 return false;
1684
1685 // Single-bit booleans don't need to be checked. Special-case this to avoid
1686 // a bit width mismatch when handling bitfield values. This is handled by
1687 // EmitFromMemory for the non-bitfield case.
1688 if (IsBool &&
1689 cast<llvm::IntegerType>(Value->getType())->getBitWidth() == 1)
1690 return false;
1691
1692 llvm::APInt Min, End;
1693 if (!getRangeForType(*this, Ty, Min, End, /*StrictEnums=*/true, IsBool))
1694 return true;
1695
1696 auto &Ctx = getLLVMContext();
1697 SanitizerScope SanScope(this);
1698 llvm::Value *Check;
1699 --End;
1700 if (!Min) {
1701 Check = Builder.CreateICmpULE(Value, llvm::ConstantInt::get(Ctx, End));
1702 } else {
1703 llvm::Value *Upper =
1704 Builder.CreateICmpSLE(Value, llvm::ConstantInt::get(Ctx, End));
1705 llvm::Value *Lower =
1706 Builder.CreateICmpSGE(Value, llvm::ConstantInt::get(Ctx, Min));
1707 Check = Builder.CreateAnd(Upper, Lower);
1708 }
1709 llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc),
1712 NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool;
1713 EmitCheck(std::make_pair(Check, Kind), SanitizerHandler::LoadInvalidValue,
1714 StaticArgs, EmitCheckValue(Value));
1715 return true;
1716}
1717
1718llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
1719 QualType Ty,
1720 SourceLocation Loc,
1721 LValueBaseInfo BaseInfo,
1722 TBAAAccessInfo TBAAInfo,
1723 bool isNontemporal) {
1724 if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer()))
1725 if (GV->isThreadLocal())
1726 Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV),
1728
1729 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
1730 // Boolean vectors use `iN` as storage type.
1731 if (ClangVecTy->isExtVectorBoolType()) {
1732 llvm::Type *ValTy = ConvertType(Ty);
1733 unsigned ValNumElems =
1734 cast<llvm::FixedVectorType>(ValTy)->getNumElements();
1735 // Load the `iP` storage object (P is the padded vector size).
1736 auto *RawIntV = Builder.CreateLoad(Addr, Volatile, "load_bits");
1737 const auto *RawIntTy = RawIntV->getType();
1738 assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors");
1739 // Bitcast iP --> <P x i1>.
1740 auto *PaddedVecTy = llvm::FixedVectorType::get(
1741 Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
1742 llvm::Value *V = Builder.CreateBitCast(RawIntV, PaddedVecTy);
1743 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
1744 V = emitBoolVecConversion(V, ValNumElems, "extractvec");
1745
1746 return EmitFromMemory(V, Ty);
1747 }
1748
1749 // Handle vectors of size 3 like size 4 for better performance.
1750 const llvm::Type *EltTy = Addr.getElementType();
1751 const auto *VTy = cast<llvm::FixedVectorType>(EltTy);
1752
1753 if (!CGM.getCodeGenOpts().PreserveVec3Type && VTy->getNumElements() == 3) {
1754
1755 // Bitcast to vec4 type.
1756 llvm::VectorType *vec4Ty =
1757 llvm::FixedVectorType::get(VTy->getElementType(), 4);
1758 Address Cast = Builder.CreateElementBitCast(Addr, vec4Ty, "castToVec4");
1759 // Now load value.
1760 llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVec4");
1761
1762 // Shuffle vector to get vec3.
1763 V = Builder.CreateShuffleVector(V, ArrayRef<int>{0, 1, 2}, "extractVec");
1764 return EmitFromMemory(V, Ty);
1765 }
1766 }
1767
1768 // Atomic operations have to be done on integral types.
1769 LValue AtomicLValue =
1770 LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo);
1771 if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(AtomicLValue)) {
1772 return EmitAtomicLoad(AtomicLValue, Loc).getScalarVal();
1773 }
1774
1775 llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile);
1776 if (isNontemporal) {
1777 llvm::MDNode *Node = llvm::MDNode::get(
1778 Load->getContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
1779 Load->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
1780 }
1781
1782 CGM.DecorateInstructionWithTBAA(Load, TBAAInfo);
1783
1784 if (EmitScalarRangeCheck(Load, Ty, Loc)) {
1785 // In order to prevent the optimizer from throwing away the check, don't
1786 // attach range metadata to the load.
1787 } else if (CGM.getCodeGenOpts().OptimizationLevel > 0)
1788 if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {
1789 Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo);
1790 Load->setMetadata(llvm::LLVMContext::MD_noundef,
1791 llvm::MDNode::get(getLLVMContext(), std::nullopt));
1792 }
1793
1794 return EmitFromMemory(Load, Ty);
1795}
1796
1797llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
1798 // Bool has a different representation in memory than in registers.
1799 if (hasBooleanRepresentation(Ty)) {
1800 // This should really always be an i1, but sometimes it's already
1801 // an i8, and it's awkward to track those cases down.
1802 if (Value->getType()->isIntegerTy(1))
1803 return Builder.CreateZExt(Value, ConvertTypeForMem(Ty), "frombool");
1804 assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
1805 "wrong value rep of bool");
1806 }
1807
1808 return Value;
1809}
1810
1811llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
1812 // Bool has a different representation in memory than in registers.
1813 if (hasBooleanRepresentation(Ty)) {
1814 assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
1815 "wrong value rep of bool");
1816 return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool");
1817 }
1818 if (Ty->isExtVectorBoolType()) {
1819 const auto *RawIntTy = Value->getType();
1820 // Bitcast iP --> <P x i1>.
1821 auto *PaddedVecTy = llvm::FixedVectorType::get(
1822 Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
1823 auto *V = Builder.CreateBitCast(Value, PaddedVecTy);
1824 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
1825 llvm::Type *ValTy = ConvertType(Ty);
1826 unsigned ValNumElems = cast<llvm::FixedVectorType>(ValTy)->getNumElements();
1827 return emitBoolVecConversion(V, ValNumElems, "extractvec");
1828 }
1829
1830 return Value;
1831}
1832
1833// Convert the pointer of \p Addr to a pointer to a vector (the value type of
1834// MatrixType), if it points to a array (the memory type of MatrixType).
1836 bool IsVector = true) {
1837 auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.getElementType());
1838 if (ArrayTy && IsVector) {
1839 auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
1840 ArrayTy->getNumElements());
1841
1842 return Address(CGF.Builder.CreateElementBitCast(Addr, VectorTy));
1843 }
1844 auto *VectorTy = dyn_cast<llvm::VectorType>(Addr.getElementType());
1845 if (VectorTy && !IsVector) {
1846 auto *ArrayTy = llvm::ArrayType::get(
1847 VectorTy->getElementType(),
1848 cast<llvm::FixedVectorType>(VectorTy)->getNumElements());
1849
1850 return Address(CGF.Builder.CreateElementBitCast(Addr, ArrayTy));
1851 }
1852
1853 return Addr;
1854}
1855
1856// Emit a store of a matrix LValue. This may require casting the original
1857// pointer to memory address (ArrayType) to a pointer to the value type
1858// (VectorType).
1859static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue,
1860 bool isInit, CodeGenFunction &CGF) {
1861 Address Addr = MaybeConvertMatrixAddress(lvalue.getAddress(CGF), CGF,
1862 value->getType()->isVectorTy());
1863 CGF.EmitStoreOfScalar(value, Addr, lvalue.isVolatile(), lvalue.getType(),
1864 lvalue.getBaseInfo(), lvalue.getTBAAInfo(), isInit,
1865 lvalue.isNontemporal());
1866}
1867
1868void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
1869 bool Volatile, QualType Ty,
1870 LValueBaseInfo BaseInfo,
1871 TBAAAccessInfo TBAAInfo,
1872 bool isInit, bool isNontemporal) {
1873 if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer()))
1874 if (GV->isThreadLocal())
1875 Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV),
1877
1878 llvm::Type *SrcTy = Value->getType();
1879 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
1880 auto *VecTy = dyn_cast<llvm::FixedVectorType>(SrcTy);
1881 if (VecTy && ClangVecTy->isExtVectorBoolType()) {
1882 auto *MemIntTy = cast<llvm::IntegerType>(Addr.getElementType());
1883 // Expand to the memory bit width.
1884 unsigned MemNumElems = MemIntTy->getPrimitiveSizeInBits();
1885 // <N x i1> --> <P x i1>.
1886 Value = emitBoolVecConversion(Value, MemNumElems, "insertvec");
1887 // <P x i1> --> iP.
1888 Value = Builder.CreateBitCast(Value, MemIntTy);
1889 } else if (!CGM.getCodeGenOpts().PreserveVec3Type) {
1890 // Handle vec3 special.
1891 if (VecTy && cast<llvm::FixedVectorType>(VecTy)->getNumElements() == 3) {
1892 // Our source is a vec3, do a shuffle vector to make it a vec4.
1893 Value = Builder.CreateShuffleVector(Value, ArrayRef<int>{0, 1, 2, -1},
1894 "extractVec");
1895 SrcTy = llvm::FixedVectorType::get(VecTy->getElementType(), 4);
1896 }
1897 if (Addr.getElementType() != SrcTy) {
1898 Addr = Builder.CreateElementBitCast(Addr, SrcTy, "storetmp");
1899 }
1900 }
1901 }
1902
1903 Value = EmitToMemory(Value, Ty);
1904
1905 LValue AtomicLValue =
1906 LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo);
1907 if (Ty->isAtomicType() ||
1908 (!isInit && LValueIsSuitableForInlineAtomic(AtomicLValue))) {
1909 EmitAtomicStore(RValue::get(Value), AtomicLValue, isInit);
1910 return;
1911 }
1912
1913 llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
1914 if (isNontemporal) {
1915 llvm::MDNode *Node =
1916 llvm::MDNode::get(Store->getContext(),
1917 llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
1918 Store->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
1919 }
1920
1921 CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
1922}
1923
1924void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
1925 bool isInit) {
1926 if (lvalue.getType()->isConstantMatrixType()) {
1927 EmitStoreOfMatrixScalar(value, lvalue, isInit, *this);
1928 return;
1929 }
1930
1931 EmitStoreOfScalar(value, lvalue.getAddress(*this), lvalue.isVolatile(),
1932 lvalue.getType(), lvalue.getBaseInfo(),
1933 lvalue.getTBAAInfo(), isInit, lvalue.isNontemporal());
1934}
1935
1936// Emit a load of a LValue of matrix type. This may require casting the pointer
1937// to memory address (ArrayType) to a pointer to the value type (VectorType).
1939 CodeGenFunction &CGF) {
1940 assert(LV.getType()->isConstantMatrixType());
1941 Address Addr = MaybeConvertMatrixAddress(LV.getAddress(CGF), CGF);
1942 LV.setAddress(Addr);
1943 return RValue::get(CGF.EmitLoadOfScalar(LV, Loc));
1944}
1945
1946/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
1947/// method emits the address of the lvalue, then loads the result as an rvalue,
1948/// returning the rvalue.
1950 if (LV.isObjCWeak()) {
1951 // load of a __weak object.
1952 Address AddrWeakObj = LV.getAddress(*this);
1954 AddrWeakObj));
1955 }
1957 // In MRC mode, we do a load+autorelease.
1958 if (!getLangOpts().ObjCAutoRefCount) {
1959 return RValue::get(EmitARCLoadWeak(LV.getAddress(*this)));
1960 }
1961
1962 // In ARC mode, we load retained and then consume the value.
1963 llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress(*this));
1964 Object = EmitObjCConsumeObject(LV.getType(), Object);
1965 return RValue::get(Object);
1966 }
1967
1968 if (LV.isSimple()) {
1969 assert(!LV.getType()->isFunctionType());
1970
1971 if (LV.getType()->isConstantMatrixType())
1972 return EmitLoadOfMatrixLValue(LV, Loc, *this);
1973
1974 // Everything needs a load.
1975 return RValue::get(EmitLoadOfScalar(LV, Loc));
1976 }
1977
1978 if (LV.isVectorElt()) {
1979 llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddress(),
1980 LV.isVolatileQualified());
1981 return RValue::get(Builder.CreateExtractElement(Load, LV.getVectorIdx(),
1982 "vecext"));
1983 }
1984
1985 // If this is a reference to a subset of the elements of a vector, either
1986 // shuffle the input or extract/insert them as appropriate.
1987 if (LV.isExtVectorElt()) {
1989 }
1990
1991 // Global Register variables always invoke intrinsics
1992 if (LV.isGlobalReg())
1993 return EmitLoadOfGlobalRegLValue(LV);
1994
1995 if (LV.isMatrixElt()) {
1996 llvm::Value *Idx = LV.getMatrixIdx();
1997 if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
1998 const auto *const MatTy = LV.getType()->castAs<ConstantMatrixType>();
1999 llvm::MatrixBuilder MB(Builder);
2000 MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
2001 }
2002 llvm::LoadInst *Load =
2004 return RValue::get(Builder.CreateExtractElement(Load, Idx, "matrixext"));
2005 }
2006
2007 assert(LV.isBitField() && "Unknown LValue type!");
2008 return EmitLoadOfBitfieldLValue(LV, Loc);
2009}
2010
2012 SourceLocation Loc) {
2013 const CGBitFieldInfo &Info = LV.getBitFieldInfo();
2014
2015 // Get the output type.
2016 llvm::Type *ResLTy = ConvertType(LV.getType());
2017
2018 Address Ptr = LV.getBitFieldAddress();
2019 llvm::Value *Val =
2020 Builder.CreateLoad(Ptr, LV.isVolatileQualified(), "bf.load");
2021
2022 bool UseVolatile = LV.isVolatileQualified() &&
2023 Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget());
2024 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2025 const unsigned StorageSize =
2026 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2027 if (Info.IsSigned) {
2028 assert(static_cast<unsigned>(Offset + Info.Size) <= StorageSize);
2029 unsigned HighBits = StorageSize - Offset - Info.Size;
2030 if (HighBits)
2031 Val = Builder.CreateShl(Val, HighBits, "bf.shl");
2032 if (Offset + HighBits)
2033 Val = Builder.CreateAShr(Val, Offset + HighBits, "bf.ashr");
2034 } else {
2035 if (Offset)
2036 Val = Builder.CreateLShr(Val, Offset, "bf.lshr");
2037 if (static_cast<unsigned>(Offset) + Info.Size < StorageSize)
2038 Val = Builder.CreateAnd(
2039 Val, llvm::APInt::getLowBitsSet(StorageSize, Info.Size), "bf.clear");
2040 }
2041 Val = Builder.CreateIntCast(Val, ResLTy, Info.IsSigned, "bf.cast");
2042 EmitScalarRangeCheck(Val, LV.getType(), Loc);
2043 return RValue::get(Val);
2044}
2045
2046// If this is a reference to a subset of the elements of a vector, create an
2047// appropriate shufflevector.
2049 llvm::Value *Vec = Builder.CreateLoad(LV.getExtVectorAddress(),
2050 LV.isVolatileQualified());
2051
2052 const llvm::Constant *Elts = LV.getExtVectorElts();
2053
2054 // If the result of the expression is a non-vector type, we must be extracting
2055 // a single element. Just codegen as an extractelement.
2056 const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
2057 if (!ExprVT) {
2058 unsigned InIdx = getAccessedFieldNo(0, Elts);
2059 llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);
2060 return RValue::get(Builder.CreateExtractElement(Vec, Elt));
2061 }
2062
2063 // Always use shuffle vector to try to retain the original program structure
2064 unsigned NumResultElts = ExprVT->getNumElements();
2065
2067 for (unsigned i = 0; i != NumResultElts; ++i)
2068 Mask.push_back(getAccessedFieldNo(i, Elts));
2069
2070 Vec = Builder.CreateShuffleVector(Vec, Mask);
2071 return RValue::get(Vec);
2072}
2073
2074/// Generates lvalue for partial ext_vector access.
2076 Address VectorAddress = LV.getExtVectorAddress();
2077 QualType EQT = LV.getType()->castAs<VectorType>()->getElementType();
2078 llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(EQT);
2079
2080 Address CastToPointerElement =
2081 Builder.CreateElementBitCast(VectorAddress, VectorElementTy,
2082 "conv.ptr.element");
2083
2084 const llvm::Constant *Elts = LV.getExtVectorElts();
2085 unsigned ix = getAccessedFieldNo(0, Elts);
2086
2087 Address VectorBasePtrPlusIx =
2088 Builder.CreateConstInBoundsGEP(CastToPointerElement, ix,
2089 "vector.elt");
2090
2091 return VectorBasePtrPlusIx;
2092}
2093
2094/// Load of global gamed gegisters are always calls to intrinsics.
2096 assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) &&
2097 "Bad type for register variable");
2098 llvm::MDNode *RegName = cast<llvm::MDNode>(
2099 cast<llvm::MetadataAsValue>(LV.getGlobalReg())->getMetadata());
2100
2101 // We accept integer and pointer types only
2102 llvm::Type *OrigTy = CGM.getTypes().ConvertType(LV.getType());
2103 llvm::Type *Ty = OrigTy;
2104 if (OrigTy->isPointerTy())
2105 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2106 llvm::Type *Types[] = { Ty };
2107
2108 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
2109 llvm::Value *Call = Builder.CreateCall(
2110 F, llvm::MetadataAsValue::get(Ty->getContext(), RegName));
2111 if (OrigTy->isPointerTy())
2112 Call = Builder.CreateIntToPtr(Call, OrigTy);
2113 return RValue::get(Call);
2114}
2115
2116/// EmitStoreThroughLValue - Store the specified rvalue into the specified
2117/// lvalue, where both are guaranteed to the have the same type, and that type
2118/// is 'Ty'.
2120 bool isInit) {
2121 if (!Dst.isSimple()) {
2122 if (Dst.isVectorElt()) {
2123 // Read/modify/write the vector, inserting the new element.
2124 llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddress(),
2125 Dst.isVolatileQualified());
2126 auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Vec->getType());
2127 if (IRStoreTy) {
2128 auto *IRVecTy = llvm::FixedVectorType::get(
2129 Builder.getInt1Ty(), IRStoreTy->getPrimitiveSizeInBits());
2130 Vec = Builder.CreateBitCast(Vec, IRVecTy);
2131 // iN --> <N x i1>.
2132 }
2133 Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(),
2134 Dst.getVectorIdx(), "vecins");
2135 if (IRStoreTy) {
2136 // <N x i1> --> <iN>.
2137 Vec = Builder.CreateBitCast(Vec, IRStoreTy);
2138 }
2140 Dst.isVolatileQualified());
2141 return;
2142 }
2143
2144 // If this is an update of extended vector elements, insert them as
2145 // appropriate.
2146 if (Dst.isExtVectorElt())
2148
2149 if (Dst.isGlobalReg())
2150 return EmitStoreThroughGlobalRegLValue(Src, Dst);
2151
2152 if (Dst.isMatrixElt()) {
2153 llvm::Value *Idx = Dst.getMatrixIdx();
2154 if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
2155 const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>();
2156 llvm::MatrixBuilder MB(Builder);
2157 MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
2158 }
2159 llvm::Instruction *Load = Builder.CreateLoad(Dst.getMatrixAddress());
2160 llvm::Value *Vec =
2161 Builder.CreateInsertElement(Load, Src.getScalarVal(), Idx, "matins");
2163 Dst.isVolatileQualified());
2164 return;
2165 }
2166
2167 assert(Dst.isBitField() && "Unknown LValue type");
2168 return EmitStoreThroughBitfieldLValue(Src, Dst);
2169 }
2170
2171 // There's special magic for assigning into an ARC-qualified l-value.
2172 if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
2173 switch (Lifetime) {
2175 llvm_unreachable("present but none");
2176
2178 // nothing special
2179 break;
2180
2182 if (isInit) {
2183 Src = RValue::get(EmitARCRetain(Dst.getType(), Src.getScalarVal()));
2184 break;
2185 }
2186 EmitARCStoreStrong(Dst, Src.getScalarVal(), /*ignore*/ true);
2187 return;
2188
2190 if (isInit)
2191 // Initialize and then skip the primitive store.
2192 EmitARCInitWeak(Dst.getAddress(*this), Src.getScalarVal());
2193 else
2194 EmitARCStoreWeak(Dst.getAddress(*this), Src.getScalarVal(),
2195 /*ignore*/ true);
2196 return;
2197
2200 Src.getScalarVal()));
2201 // fall into the normal path
2202 break;
2203 }
2204 }
2205
2206 if (Dst.isObjCWeak() && !Dst.isNonGC()) {
2207 // load of a __weak object.
2208 Address LvalueDst = Dst.getAddress(*this);
2209 llvm::Value *src = Src.getScalarVal();
2210 CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst);
2211 return;
2212 }
2213
2214 if (Dst.isObjCStrong() && !Dst.isNonGC()) {
2215 // load of a __strong object.
2216 Address LvalueDst = Dst.getAddress(*this);
2217 llvm::Value *src = Src.getScalarVal();
2218 if (Dst.isObjCIvar()) {
2219 assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
2220 llvm::Type *ResultType = IntPtrTy;
2222 llvm::Value *RHS = dst.getPointer();
2223 RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");
2224 llvm::Value *LHS =
2225 Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType,
2226 "sub.ptr.lhs.cast");
2227 llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");
2228 CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst,
2229 BytesBetween);
2230 } else if (Dst.isGlobalObjCRef()) {
2231 CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst,
2232 Dst.isThreadLocalRef());
2233 }
2234 else
2235 CGM.getObjCRuntime().EmitObjCStrongCastAssign(*this, src, LvalueDst);
2236 return;
2237 }
2238
2239 assert(Src.isScalar() && "Can't emit an agg store with this method");
2240 EmitStoreOfScalar(Src.getScalarVal(), Dst, isInit);
2241}
2242
2244 llvm::Value **Result) {
2245 const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
2246 llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType());
2247 Address Ptr = Dst.getBitFieldAddress();
2248
2249 // Get the source value, truncated to the width of the bit-field.
2250 llvm::Value *SrcVal = Src.getScalarVal();
2251
2252 // Cast the source to the storage type and shift it into place.
2253 SrcVal = Builder.CreateIntCast(SrcVal, Ptr.getElementType(),
2254 /*isSigned=*/false);
2255 llvm::Value *MaskedVal = SrcVal;
2256
2257 const bool UseVolatile =
2258 CGM.getCodeGenOpts().AAPCSBitfieldWidth && Dst.isVolatileQualified() &&
2259 Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget());
2260 const unsigned StorageSize =
2261 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2262 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2263 // See if there are other bits in the bitfield's storage we'll need to load
2264 // and mask together with source before storing.
2265 if (StorageSize != Info.Size) {
2266 assert(StorageSize > Info.Size && "Invalid bitfield size.");
2267 llvm::Value *Val =
2268 Builder.CreateLoad(Ptr, Dst.isVolatileQualified(), "bf.load");
2269
2270 // Mask the source value as needed.
2272 SrcVal = Builder.CreateAnd(
2273 SrcVal, llvm::APInt::getLowBitsSet(StorageSize, Info.Size),
2274 "bf.value");
2275 MaskedVal = SrcVal;
2276 if (Offset)
2277 SrcVal = Builder.CreateShl(SrcVal, Offset, "bf.shl");
2278
2279 // Mask out the original value.
2280 Val = Builder.CreateAnd(
2281 Val, ~llvm::APInt::getBitsSet(StorageSize, Offset, Offset + Info.Size),
2282 "bf.clear");
2283
2284 // Or together the unchanged values and the source value.
2285 SrcVal = Builder.CreateOr(Val, SrcVal, "bf.set");
2286 } else {
2287 assert(Offset == 0);
2288 // According to the AACPS:
2289 // When a volatile bit-field is written, and its container does not overlap
2290 // with any non-bit-field member, its container must be read exactly once
2291 // and written exactly once using the access width appropriate to the type
2292 // of the container. The two accesses are not atomic.
2293 if (Dst.isVolatileQualified() && isAAPCS(CGM.getTarget()) &&
2294 CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad)
2295 Builder.CreateLoad(Ptr, true, "bf.load");
2296 }
2297
2298 // Write the new value back out.
2299 Builder.CreateStore(SrcVal, Ptr, Dst.isVolatileQualified());
2300
2301 // Return the new value of the bit-field, if requested.
2302 if (Result) {
2303 llvm::Value *ResultVal = MaskedVal;
2304
2305 // Sign extend the value if needed.
2306 if (Info.IsSigned) {
2307 assert(Info.Size <= StorageSize);
2308 unsigned HighBits = StorageSize - Info.Size;
2309 if (HighBits) {
2310 ResultVal = Builder.CreateShl(ResultVal, HighBits, "bf.result.shl");
2311 ResultVal = Builder.CreateAShr(ResultVal, HighBits, "bf.result.ashr");
2312 }
2313 }
2314
2315 ResultVal = Builder.CreateIntCast(ResultVal, ResLTy, Info.IsSigned,
2316 "bf.result.cast");
2317 *Result = EmitFromMemory(ResultVal, Dst.getType());
2318 }
2319}
2320
2322 LValue Dst) {
2323 // This access turns into a read/modify/write of the vector. Load the input
2324 // value now.
2325 llvm::Value *Vec = Builder.CreateLoad(Dst.getExtVectorAddress(),
2326 Dst.isVolatileQualified());
2327 const llvm::Constant *Elts = Dst.getExtVectorElts();
2328
2329 llvm::Value *SrcVal = Src.getScalarVal();
2330
2331 if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
2332 unsigned NumSrcElts = VTy->getNumElements();
2333 unsigned NumDstElts =
2334 cast<llvm::FixedVectorType>(Vec->getType())->getNumElements();
2335 if (NumDstElts == NumSrcElts) {
2336 // Use shuffle vector is the src and destination are the same number of
2337 // elements and restore the vector mask since it is on the side it will be
2338 // stored.
2339 SmallVector<int, 4> Mask(NumDstElts);
2340 for (unsigned i = 0; i != NumSrcElts; ++i)
2341 Mask[getAccessedFieldNo(i, Elts)] = i;
2342
2343 Vec = Builder.CreateShuffleVector(SrcVal, Mask);
2344 } else if (NumDstElts > NumSrcElts) {
2345 // Extended the source vector to the same length and then shuffle it
2346 // into the destination.
2347 // FIXME: since we're shuffling with undef, can we just use the indices
2348 // into that? This could be simpler.
2349 SmallVector<int, 4> ExtMask;
2350 for (unsigned i = 0; i != NumSrcElts; ++i)
2351 ExtMask.push_back(i);
2352 ExtMask.resize(NumDstElts, -1);
2353 llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(SrcVal, ExtMask);
2354 // build identity
2356 for (unsigned i = 0; i != NumDstElts; ++i)
2357 Mask.push_back(i);
2358
2359 // When the vector size is odd and .odd or .hi is used, the last element
2360 // of the Elts constant array will be one past the size of the vector.
2361 // Ignore the last element here, if it is greater than the mask size.
2362 if (getAccessedFieldNo(NumSrcElts - 1, Elts) == Mask.size())
2363 NumSrcElts--;
2364
2365 // modify when what gets shuffled in
2366 for (unsigned i = 0; i != NumSrcElts; ++i)
2367 Mask[getAccessedFieldNo(i, Elts)] = i + NumDstElts;
2368 Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, Mask);
2369 } else {
2370 // We should never shorten the vector
2371 llvm_unreachable("unexpected shorten vector length");
2372 }
2373 } else {
2374 // If the Src is a scalar (not a vector) it must be updating one element.
2375 unsigned InIdx = getAccessedFieldNo(0, Elts);
2376 llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);
2377 Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt);
2378 }
2379
2381 Dst.isVolatileQualified());
2382}
2383
2384/// Store of global named registers are always calls to intrinsics.
2386 assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) &&
2387 "Bad type for register variable");
2388 llvm::MDNode *RegName = cast<llvm::MDNode>(
2389 cast<llvm::MetadataAsValue>(Dst.getGlobalReg())->getMetadata());
2390 assert(RegName && "Register LValue is not metadata");
2391
2392 // We accept integer and pointer types only
2393 llvm::Type *OrigTy = CGM.getTypes().ConvertType(Dst.getType());
2394 llvm::Type *Ty = OrigTy;
2395 if (OrigTy->isPointerTy())
2396 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2397 llvm::Type *Types[] = { Ty };
2398
2399 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
2400 llvm::Value *Value = Src.getScalarVal();
2401 if (OrigTy->isPointerTy())
2402 Value = Builder.CreatePtrToInt(Value, Ty);
2403 Builder.CreateCall(
2404 F, {llvm::MetadataAsValue::get(Ty->getContext(), RegName), Value});
2405}
2406
2407// setObjCGCLValueClass - sets class of the lvalue for the purpose of
2408// generating write-barries API. It is currently a global, ivar,
2409// or neither.
2410static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
2411 LValue &LV,
2412 bool IsMemberAccess=false) {
2413 if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
2414 return;
2415
2416 if (isa<ObjCIvarRefExpr>(E)) {
2417 QualType ExpTy = E->getType();
2418 if (IsMemberAccess && ExpTy->isPointerType()) {
2419 // If ivar is a structure pointer, assigning to field of
2420 // this struct follows gcc's behavior and makes it a non-ivar
2421 // writer-barrier conservatively.
2422 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
2423 if (ExpTy->isRecordType()) {
2424 LV.setObjCIvar(false);
2425 return;
2426 }
2427 }
2428 LV.setObjCIvar(true);
2429 auto *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr *>(E));
2430 LV.setBaseIvarExp(Exp->getBase());
2431 LV.setObjCArray(E->getType()->isArrayType());
2432 return;
2433 }
2434
2435 if (const auto *Exp = dyn_cast<DeclRefExpr>(E)) {
2436 if (const auto *VD = dyn_cast<VarDecl>(Exp->getDecl())) {
2437 if (VD->hasGlobalStorage()) {
2438 LV.setGlobalObjCRef(true);
2439 LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None);
2440 }
2441 }
2442 LV.setObjCArray(E->getType()->isArrayType());
2443 return;
2444 }
2445
2446 if (const auto *Exp = dyn_cast<UnaryOperator>(E)) {
2447 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2448 return;
2449 }
2450
2451 if (const auto *Exp = dyn_cast<ParenExpr>(E)) {
2452 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2453 if (LV.isObjCIvar()) {
2454 // If cast is to a structure pointer, follow gcc's behavior and make it
2455 // a non-ivar write-barrier.
2456 QualType ExpTy = E->getType();
2457 if (ExpTy->isPointerType())
2458 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
2459 if (ExpTy->isRecordType())
2460 LV.setObjCIvar(false);
2461 }
2462 return;
2463 }
2464
2465 if (const auto *Exp = dyn_cast<GenericSelectionExpr>(E)) {
2466 setObjCGCLValueClass(Ctx, Exp->getResultExpr(), LV);
2467 return;
2468 }
2469
2470 if (const auto *Exp = dyn_cast<ImplicitCastExpr>(E)) {
2471 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2472 return;
2473 }
2474
2475 if (const auto *Exp = dyn_cast<CStyleCastExpr>(E)) {
2476 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2477 return;
2478 }
2479
2480 if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) {
2481 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2482 return;
2483 }
2484
2485 if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(E)) {
2486 setObjCGCLValueClass(Ctx, Exp->getBase(), LV);
2487 if (LV.isObjCIvar() && !LV.isObjCArray())
2488 // Using array syntax to assigning to what an ivar points to is not
2489 // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
2490 LV.setObjCIvar(false);
2491 else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
2492 // Using array syntax to assigning to what global points to is not
2493 // same as assigning to the global itself. {id *G;} G[i] = 0;
2494 LV.setGlobalObjCRef(false);
2495 return;
2496 }
2497
2498 if (const auto *Exp = dyn_cast<MemberExpr>(E)) {
2499 setObjCGCLValueClass(Ctx, Exp->getBase(), LV, true);
2500 // We don't know if member is an 'ivar', but this flag is looked at
2501 // only in the context of LV.isObjCIvar().
2502 LV.setObjCArray(E->getType()->isArrayType());
2503 return;
2504 }
2505}
2506
2507static llvm::Value *
2509 llvm::Value *V, llvm::Type *IRType,
2510 StringRef Name = StringRef()) {
2511 unsigned AS = cast<llvm::PointerType>(V->getType())->getAddressSpace();
2512 return CGF.Builder.CreateBitCast(V, IRType->getPointerTo(AS), Name);
2513}
2514
2516 CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,
2517 llvm::Type *RealVarTy, SourceLocation Loc) {
2518 if (CGF.CGM.getLangOpts().OpenMPIRBuilder)
2520 CGF, VD, Addr, Loc);
2521 else
2522 Addr =
2523 CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc);
2524
2525 Addr = CGF.Builder.CreateElementBitCast(Addr, RealVarTy);
2526 return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2527}
2528
2530 const VarDecl *VD, QualType T) {
2531 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2532 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2533 // Return an invalid address if variable is MT_To (or MT_Enter starting with
2534 // OpenMP 5.2) and unified memory is not enabled. For all other cases: MT_Link
2535 // and MT_To (or MT_Enter) with unified memory, return a valid address.
2536 if (!Res || ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2537 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2539 return Address::invalid();
2540 assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
2541 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2542 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2544 "Expected link clause OR to clause with unified memory enabled.");
2545 QualType PtrTy = CGF.getContext().getPointerType(VD->getType());
2547 return CGF.EmitLoadOfPointer(Addr, PtrTy->castAs<PointerType>());
2548}
2549
2550Address
2552 LValueBaseInfo *PointeeBaseInfo,
2553 TBAAAccessInfo *PointeeTBAAInfo) {
2554 llvm::LoadInst *Load =
2555 Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile());
2557
2558 QualType PointeeType = RefLVal.getType()->getPointeeType();
2560 PointeeType, PointeeBaseInfo, PointeeTBAAInfo,
2561 /* forPointeeType= */ true);
2562 return Address(Load, ConvertTypeForMem(PointeeType), Align);
2563}
2564
2566 LValueBaseInfo PointeeBaseInfo;
2567 TBAAAccessInfo PointeeTBAAInfo;
2568 Address PointeeAddr = EmitLoadOfReference(RefLVal, &PointeeBaseInfo,
2569 &PointeeTBAAInfo);
2570 return MakeAddrLValue(PointeeAddr, RefLVal.getType()->getPointeeType(),
2571 PointeeBaseInfo, PointeeTBAAInfo);
2572}
2573
2575 const PointerType *PtrTy,
2576 LValueBaseInfo *BaseInfo,
2577 TBAAAccessInfo *TBAAInfo) {
2578 llvm::Value *Addr = Builder.CreateLoad(Ptr);
2579 return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()),
2580 CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo,
2581 TBAAInfo,
2582 /*forPointeeType=*/true));
2583}
2584
2586 const PointerType *PtrTy) {
2587 LValueBaseInfo BaseInfo;
2588 TBAAAccessInfo TBAAInfo;
2589 Address Addr = EmitLoadOfPointer(PtrAddr, PtrTy, &BaseInfo, &TBAAInfo);
2590 return MakeAddrLValue(Addr, PtrTy->getPointeeType(), BaseInfo, TBAAInfo);
2591}
2592
2594 const Expr *E, const VarDecl *VD) {
2595 QualType T = E->getType();
2596
2597 // If it's thread_local, emit a call to its wrapper function instead.
2598 if (VD->getTLSKind() == VarDecl::TLS_Dynamic &&
2600 return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, T);
2601 // Check if the variable is marked as declare target with link clause in
2602 // device codegen.
2603 if (CGF.getLangOpts().OpenMPIsDevice) {
2604 Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T);
2605 if (Addr.isValid())
2606 return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2607 }
2608
2609 llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD);
2610
2611 if (VD->getTLSKind() != VarDecl::TLS_None)
2612 V = CGF.Builder.CreateThreadLocalAddress(V);
2613
2614 llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType());
2615 V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy);
2616 CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
2617 Address Addr(V, RealVarTy, Alignment);
2618 // Emit reference to the private copy of the variable if it is an OpenMP
2619 // threadprivate variable.
2620 if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
2621 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
2622 return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
2623 E->getExprLoc());
2624 }
2625 LValue LV = VD->getType()->isReferenceType() ?
2626 CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
2629 setObjCGCLValueClass(CGF.getContext(), E, LV);
2630 return LV;
2631}
2632
2633static llvm::Constant *EmitFunctionDeclPointer(CodeGenModule &CGM,
2634 GlobalDecl GD) {
2635 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
2636 if (FD->hasAttr<WeakRefAttr>()) {
2637 ConstantAddress aliasee = CGM.GetWeakRefReference(FD);
2638 return aliasee.getPointer();
2639 }
2640
2641 llvm::Constant *V = CGM.GetAddrOfFunction(GD);
2642 if (!FD->hasPrototype()) {
2643 if (const FunctionProtoType *Proto =
2644 FD->getType()->getAs<FunctionProtoType>()) {
2645 // Ugly case: for a K&R-style definition, the type of the definition
2646 // isn't the same as the type of a use. Correct for this with a
2647 // bitcast.
2648 QualType NoProtoType =
2649 CGM.getContext().getFunctionNoProtoType(Proto->getReturnType());
2650 NoProtoType = CGM.getContext().getPointerType(NoProtoType);
2651 V = llvm::ConstantExpr::getBitCast(V,
2652 CGM.getTypes().ConvertType(NoProtoType));
2653 }
2654 }
2655 return V;
2656}
2657
2659 GlobalDecl GD) {
2660 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
2661 llvm::Value *V = EmitFunctionDeclPointer(CGF.CGM, GD);
2662 CharUnits Alignment = CGF.getContext().getDeclAlign(FD);
2663 return CGF.MakeAddrLValue(V, E->getType(), Alignment,
2665}
2666
2668 llvm::Value *ThisValue) {
2670 LValue LV = CGF.MakeNaturalAlignAddrLValue(ThisValue, TagType);
2671 return CGF.EmitLValueForField(LV, FD);
2672}
2673
2674/// Named Registers are named metadata pointing to the register name
2675/// which will be read from/written to as an argument to the intrinsic
2676/// @llvm.read/write_register.
2677/// So far, only the name is being passed down, but other options such as
2678/// register type, allocation type or even optimization options could be
2679/// passed down via the metadata node.
2681 SmallString<64> Name("llvm.named.register.");
2682 AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>();
2683 assert(Asm->getLabel().size() < 64-Name.size() &&
2684 "Register name too big");
2685 Name.append(Asm->getLabel());
2686 llvm::NamedMDNode *M =
2687 CGM.getModule().getOrInsertNamedMetadata(Name);
2688 if (M->getNumOperands() == 0) {
2689 llvm::MDString *Str = llvm::MDString::get(CGM.getLLVMContext(),
2690 Asm->getLabel());
2691 llvm::Metadata *Ops[] = {Str};
2692 M->addOperand(llvm::MDNode::get(CGM.getLLVMContext(), Ops));
2693 }
2694
2695 CharUnits Alignment = CGM.getContext().getDeclAlign(VD);
2696
2697 llvm::Value *Ptr =
2698 llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0));
2699 return LValue::MakeGlobalReg(Ptr, Alignment, VD->getType());
2700}
2701
2702/// Determine whether we can emit a reference to \p VD from the current
2703/// context, despite not necessarily having seen an odr-use of the variable in
2704/// this context.
2706 const DeclRefExpr *E,
2707 const VarDecl *VD,
2708 bool IsConstant) {
2709 // For a variable declared in an enclosing scope, do not emit a spurious
2710 // reference even if we have a capture, as that will emit an unwarranted
2711 // reference to our capture state, and will likely generate worse code than
2712 // emitting a local copy.
2714 return false;
2715
2716 // For a local declaration declared in this function, we can always reference
2717 // it even if we don't have an odr-use.
2718 if (VD->hasLocalStorage()) {
2719 return VD->getDeclContext() ==
2720 dyn_cast_or_null<DeclContext>(CGF.CurCodeDecl);
2721 }
2722
2723 // For a global declaration, we can emit a reference to it if we know
2724 // for sure that we are able to emit a definition of it.
2725 VD = VD->getDefinition(CGF.getContext());
2726 if (!VD)
2727 return false;
2728
2729 // Don't emit a spurious reference if it might be to a variable that only
2730 // exists on a different device / target.
2731 // FIXME: This is unnecessarily broad. Check whether this would actually be a
2732 // cross-target reference.
2733 if (CGF.getLangOpts().OpenMP || CGF.getLangOpts().CUDA ||
2734 CGF.getLangOpts().OpenCL) {
2735 return false;
2736 }
2737
2738 // We can emit a spurious reference only if the linkage implies that we'll
2739 // be emitting a non-interposable symbol that will be retained until link
2740 // time.
2741 switch (CGF.CGM.getLLVMLinkageVarDefinition(VD, IsConstant)) {
2742 case llvm::GlobalValue::ExternalLinkage:
2743 case llvm::GlobalValue::LinkOnceODRLinkage:
2744 case llvm::GlobalValue::WeakODRLinkage:
2745 case llvm::GlobalValue::InternalLinkage:
2746 case llvm::GlobalValue::PrivateLinkage:
2747 return true;
2748 default:
2749 return false;
2750 }
2751}
2752
2754 const NamedDecl *ND = E->getDecl();
2755 QualType T = E->getType();
2756
2757 assert(E->isNonOdrUse() != NOUR_Unevaluated &&
2758 "should not emit an unevaluated operand");
2759
2760 if (const auto *VD = dyn_cast<VarDecl>(ND)) {
2761 // Global Named registers access via intrinsics only
2762 if (VD->getStorageClass() == SC_Register &&
2763 VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
2764 return EmitGlobalNamedRegister(VD, CGM);
2765
2766 // If this DeclRefExpr does not constitute an odr-use of the variable,
2767 // we're not permitted to emit a reference to it in general, and it might
2768 // not be captured if capture would be necessary for a use. Emit the
2769 // constant value directly instead.
2770 if (E->isNonOdrUse() == NOUR_Constant &&
2771 (VD->getType()->isReferenceType() ||
2772 !canEmitSpuriousReferenceToVariable(*this, E, VD, true))) {
2773 VD->getAnyInitializer(VD);
2774 llvm::Constant *Val = ConstantEmitter(*this).emitAbstract(
2775 E->getLocation(), *VD->evaluateValue(), VD->getType());
2776 assert(Val && "failed to emit constant expression");
2777
2778 Address Addr = Address::invalid();
2779 if (!VD->getType()->isReferenceType()) {
2780 // Spill the constant value to a global.
2781 Addr = CGM.createUnnamedGlobalFrom(*VD, Val,
2782 getContext().getDeclAlign(VD));
2783 llvm::Type *VarTy = getTypes().ConvertTypeForMem(VD->getType());
2784 auto *PTy = llvm::PointerType::get(
2785 VarTy, getTypes().getTargetAddressSpace(VD->getType()));
2786 Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PTy, VarTy);
2787 } else {
2788 // Should we be using the alignment of the constant pointer we emitted?
2789 CharUnits Alignment =
2791 /* BaseInfo= */ nullptr,
2792 /* TBAAInfo= */ nullptr,
2793 /* forPointeeType= */ true);
2794 Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment);
2795 }
2796 return MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2797 }
2798
2799 // FIXME: Handle other kinds of non-odr-use DeclRefExprs.
2800
2801 // Check for captured variables.
2803 VD = VD->getCanonicalDecl();
2804 if (auto *FD = LambdaCaptureFields.lookup(VD))
2805 return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);
2806 if (CapturedStmtInfo) {
2807 auto I = LocalDeclMap.find(VD);
2808 if (I != LocalDeclMap.end()) {
2809 LValue CapLVal;
2810 if (VD->getType()->isReferenceType())
2811 CapLVal = EmitLoadOfReferenceLValue(I->second, VD->getType(),
2813 else
2814 CapLVal = MakeAddrLValue(I->second, T);
2815 // Mark lvalue as nontemporal if the variable is marked as nontemporal
2816 // in simd context.
2817 if (getLangOpts().OpenMP &&
2819 CapLVal.setNontemporal(/*Value=*/true);
2820 return CapLVal;
2821 }
2822 LValue CapLVal =
2825 Address LValueAddress = CapLVal.getAddress(*this);
2826 CapLVal = MakeAddrLValue(
2827 Address(LValueAddress.getPointer(), LValueAddress.getElementType(),
2828 getContext().getDeclAlign(VD)),
2830 CapLVal.getTBAAInfo());
2831 // Mark lvalue as nontemporal if the variable is marked as nontemporal
2832 // in simd context.
2833 if (getLangOpts().OpenMP &&
2835 CapLVal.setNontemporal(/*Value=*/true);
2836 return CapLVal;
2837 }
2838
2839 assert(isa<BlockDecl>(CurCodeDecl));
2840 Address addr = GetAddrOfBlockDecl(VD);
2841 return MakeAddrLValue(addr, T, AlignmentSource::Decl);
2842 }
2843 }
2844
2845 // FIXME: We should be able to assert this for FunctionDecls as well!
2846 // FIXME: We should be able to assert this for all DeclRefExprs, not just
2847 // those with a valid source location.
2848 assert((ND->isUsed(false) || !isa<VarDecl>(ND) || E->isNonOdrUse() ||
2849 !E->getLocation().isValid()) &&
2850 "Should not use decl without marking it used!");
2851
2852 if (ND->hasAttr<WeakRefAttr>()) {
2853 const auto *VD = cast<ValueDecl>(ND);
2855 return MakeAddrLValue(Aliasee, T, AlignmentSource::Decl);
2856 }
2857
2858 if (const auto *VD = dyn_cast<VarDecl>(ND)) {
2859 // Check if this is a global variable.
2860 if (VD->hasLinkage() || VD->isStaticDataMember())
2861 return EmitGlobalVarDeclLValue(*this, E, VD);
2862
2863 Address addr = Address::invalid();
2864
2865 // The variable should generally be present in the local decl map.
2866 auto iter = LocalDeclMap.find(VD);
2867 if (iter != LocalDeclMap.end()) {
2868 addr = iter->second;
2869
2870 // Otherwise, it might be static local we haven't emitted yet for
2871 // some reason; most likely, because it's in an outer function.
2872 } else if (VD->isStaticLocal()) {
2873 llvm::Constant *var = CGM.getOrCreateStaticVarDecl(
2874 *VD, CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false));
2875 addr = Address(
2876 var, ConvertTypeForMem(VD->getType()), getContext().getDeclAlign(VD));
2877
2878 // No other cases for now.
2879 } else {
2880 llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
2881 }
2882
2883 // Handle threadlocal function locals.
2884 if (VD->getTLSKind() != VarDecl::TLS_None)
2885 addr = addr.withPointer(
2886 Builder.CreateThreadLocalAddress(addr.getPointer()), NotKnownNonNull);
2887
2888 // Check for OpenMP threadprivate variables.
2889 if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
2890 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
2892 *this, VD, T, addr, getTypes().ConvertTypeForMem(VD->getType()),
2893 E->getExprLoc());
2894 }
2895
2896 // Drill into block byref variables.
2897 bool isBlockByref = VD->isEscapingByref();
2898 if (isBlockByref) {
2899 addr = emitBlockByrefAddress(addr, VD);
2900 }
2901
2902 // Drill into reference types.
2903 LValue LV = VD->getType()->isReferenceType() ?
2904 EmitLoadOfReferenceLValue(addr, VD->getType(), AlignmentSource::Decl) :
2906
2907 bool isLocalStorage = VD->hasLocalStorage();
2908
2909 bool NonGCable = isLocalStorage &&
2910 !VD->getType()->isReferenceType() &&
2911 !isBlockByref;
2912 if (NonGCable) {
2914 LV.setNonGC(true);
2915 }
2916
2917 bool isImpreciseLifetime =
2918 (isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());
2919 if (isImpreciseLifetime)
2922 return LV;
2923 }
2924
2925 if (const auto *FD = dyn_cast<FunctionDecl>(ND)) {
2926 LValue LV = EmitFunctionDeclLValue(*this, E, FD);
2927
2928 // Emit debuginfo for the function declaration if the target wants to.
2929 if (getContext().getTargetInfo().allowDebugInfoForExternalRef()) {
2930 if (CGDebugInfo *DI = CGM.getModuleDebugInfo()) {
2931 auto *Fn =
2932 cast<llvm::Function>(LV.getPointer(*this)->stripPointerCasts());
2933 if (!Fn->getSubprogram())
2934 DI->EmitFunctionDecl(FD, FD->getLocation(), T, Fn);
2935 }
2936 }
2937
2938 return LV;
2939 }
2940
2941 // FIXME: While we're emitting a binding from an enclosing scope, all other
2942 // DeclRefExprs we see should be implicitly treated as if they also refer to
2943 // an enclosing scope.
2944 if (const auto *BD = dyn_cast<BindingDecl>(ND)) {
2946 auto *FD = LambdaCaptureFields.lookup(BD);
2947 return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);
2948 }
2949 return EmitLValue(BD->getBinding());
2950 }
2951
2952 // We can form DeclRefExprs naming GUID declarations when reconstituting
2953 // non-type template parameters into expressions.
2954 if (const auto *GD = dyn_cast<MSGuidDecl>(ND))
2957
2958 if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND))
2961
2962 llvm_unreachable("Unhandled DeclRefExpr");
2963}
2964
2966 // __extension__ doesn't affect lvalue-ness.
2967 if (E->getOpcode() == UO_Extension)
2968 return EmitLValue(E->getSubExpr());
2969
2971 switch (E->getOpcode()) {
2972 default: llvm_unreachable("Unknown unary operator lvalue!");
2973 case UO_Deref: {
2975 assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
2976
2977 LValueBaseInfo BaseInfo;
2978 TBAAAccessInfo TBAAInfo;
2979 Address Addr = EmitPointerWithAlignment(E->getSubExpr(), &BaseInfo,
2980 &TBAAInfo);
2981 LValue LV = MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
2983
2984 // We should not generate __weak write barrier on indirect reference
2985 // of a pointer to object; as in void foo (__weak id *param); *param = 0;
2986 // But, we continue to generate __strong write barrier on indirect write
2987 // into a pointer to object.
2988 if (getLangOpts().ObjC &&
2989 getLangOpts().getGC() != LangOptions::NonGC &&
2990 LV.isObjCWeak())
2992 return LV;
2993 }
2994 case UO_Real:
2995 case UO_Imag: {
2996 LValue LV = EmitLValue(E->getSubExpr());
2997 assert(LV.isSimple() && "real/imag on non-ordinary l-value");
2998
2999 // __real is valid on scalars. This is a faster way of testing that.
3000 // __imag can only produce an rvalue on scalars.
3001 if (E->getOpcode() == UO_Real &&
3002 !LV.getAddress(*this).getElementType()->isStructTy()) {
3003 assert(E->getSubExpr()->getType()->isArithmeticType());
3004 return LV;
3005 }
3006
3007 QualType T = ExprTy->castAs<ComplexType>()->getElementType();
3008
3009 Address Component =
3010 (E->getOpcode() == UO_Real
3011 ? emitAddrOfRealComponent(LV.getAddress(*this), LV.getType())
3012 : emitAddrOfImagComponent(LV.getAddress(*this), LV.getType()));
3013 LValue ElemLV = MakeAddrLValue(Component, T, LV.getBaseInfo(),
3015 ElemLV.getQuals().addQualifiers(LV.getQuals());
3016 return ElemLV;
3017 }
3018 case UO_PreInc:
3019 case UO_PreDec: {
3020 LValue LV = EmitLValue(E->getSubExpr());
3021 bool isInc = E->getOpcode() == UO_PreInc;
3022
3023 if (E->getType()->isAnyComplexType())
3024 EmitComplexPrePostIncDec(E, LV, isInc, true/*isPre*/);
3025 else
3026 EmitScalarPrePostIncDec(E, LV, isInc, true/*isPre*/);
3027 return LV;
3028 }
3029 }
3030}
3031
3035}
3036
3040}
3041
3043 auto SL = E->getFunctionName();
3044 assert(SL != nullptr && "No StringLiteral name in PredefinedExpr");
3045 StringRef FnName = CurFn->getName();
3046 if (FnName.startswith("\01"))
3047 FnName = FnName.substr(1);
3048 StringRef NameItems[] = {
3050 std::string GVName = llvm::join(NameItems, NameItems + 2, ".");
3051 if (auto *BD = dyn_cast_or_null<BlockDecl>(CurCodeDecl)) {
3052 std::string Name = std::string(SL->getString());
3053 if (!Name.empty()) {
3054 unsigned Discriminator =
3056 if (Discriminator)
3057 Name += "_" + Twine(Discriminator + 1).str();
3058 auto C = CGM.GetAddrOfConstantCString(Name, GVName.c_str());
3060 } else {
3061 auto C =
3062 CGM.GetAddrOfConstantCString(std::string(FnName), GVName.c_str());
3064 }
3065 }
3066 auto C = CGM.GetAddrOfConstantStringFromLiteral(SL, GVName);
3068}
3069
3070/// Emit a type description suitable for use by a runtime sanitizer library. The
3071/// format of a type descriptor is
3072///
3073/// \code
3074/// { i16 TypeKind, i16 TypeInfo }
3075/// \endcode
3076///
3077/// followed by an array of i8 containing the type name. TypeKind is 0 for an
3078/// integer, 1 for a floating point value, and -1 for anything else.
3080 // Only emit each type's descriptor once.
3081 if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(T))
3082 return C;
3083
3084 uint16_t TypeKind = -1;
3085 uint16_t TypeInfo = 0;
3086
3087 if (T->isIntegerType()) {
3088 TypeKind = 0;
3089 TypeInfo = (llvm::Log2_32(getContext().getTypeSize(T)) << 1) |
3090 (T->isSignedIntegerType() ? 1 : 0);
3091 } else if (T->isFloatingType()) {
3092 TypeKind = 1;
3094 }
3095
3096 // Format the type name as if for a diagnostic, including quotes and
3097 // optionally an 'aka'.
3098 SmallString<32> Buffer;
3101 StringRef(), std::nullopt, Buffer, std::nullopt);
3102
3103 llvm::Constant *Components[] = {
3104 Builder.getInt16(TypeKind), Builder.getInt16(TypeInfo),
3105 llvm::ConstantDataArray::getString(getLLVMContext(), Buffer)
3106 };
3107 llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(Components);
3108
3109 auto *GV = new llvm::GlobalVariable(
3110 CGM.getModule(), Descriptor->getType(),
3111 /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, Descriptor);
3112 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3114
3115 // Remember the descriptor for this type.
3117
3118 return GV;
3119}
3120
3121llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
3122 llvm::Type *TargetTy = IntPtrTy;
3123
3124 if (V->getType() == TargetTy)
3125 return V;
3126
3127 // Floating-point types which fit into intptr_t are bitcast to integers
3128 // and then passed directly (after zero-extension, if necessary).
3129 if (V->getType()->isFloatingPointTy()) {
3130 unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedValue();
3131 if (Bits <= TargetTy->getIntegerBitWidth())
3132 V = Builder.CreateBitCast(V, llvm::Type::getIntNTy(getLLVMContext(),
3133 Bits));
3134 }
3135
3136 // Integers which fit in intptr_t are zero-extended and passed directly.
3137 if (V->getType()->isIntegerTy() &&
3138 V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
3139 return Builder.CreateZExt(V, TargetTy);
3140
3141 // Pointers are passed directly, everything else is passed by address.
3142 if (!V->getType()->isPointerTy()) {
3143 Address Ptr = CreateDefaultAlignTempAlloca(V->getType());
3144 Builder.CreateStore(V, Ptr);
3145 V = Ptr.getPointer();
3146 }
3147 return Builder.CreatePtrToInt(V, TargetTy);
3148}
3149
3150/// Emit a representation of a SourceLocation for passing to a handler
3151/// in a sanitizer runtime library. The format for this data is:
3152/// \code
3153/// struct SourceLocation {
3154/// const char *Filename;
3155/// int32_t Line, Column;
3156/// };
3157/// \endcode
3158/// For an invalid SourceLocation, the Filename pointer is null.
3160 llvm::Constant *Filename;
3161 int Line, Column;
3162
3164 if (PLoc.isValid()) {
3165 StringRef FilenameString = PLoc.getFilename();
3166
3167 int PathComponentsToStrip =
3168 CGM.getCodeGenOpts().EmitCheckPathComponentsToStrip;
3169 if (PathComponentsToStrip < 0) {
3170 assert(PathComponentsToStrip != INT_MIN);
3171 int PathComponentsToKeep = -PathComponentsToStrip;
3172 auto I = llvm::sys::path::rbegin(FilenameString);
3173 auto E = llvm::sys::path::rend(FilenameString);
3174 while (I != E && --PathComponentsToKeep)
3175 ++I;
3176
3177 FilenameString = FilenameString.substr(I - E);
3178 } else if (PathComponentsToStrip > 0) {
3179 auto I = llvm::sys::path::begin(FilenameString);
3180 auto E = llvm::sys::path::end(FilenameString);
3181 while (I != E && PathComponentsToStrip--)
3182 ++I;
3183
3184 if (I != E)
3185 FilenameString =
3186 FilenameString.substr(I - llvm::sys::path::begin(FilenameString));
3187 else
3188 FilenameString = llvm::sys::path::filename(FilenameString);
3189 }
3190
3191 auto FilenameGV =
3192 CGM.GetAddrOfConstantCString(std::string(FilenameString), ".src");
3194 cast<llvm::GlobalVariable>(
3195 FilenameGV.getPointer()->stripPointerCasts()));
3196 Filename = FilenameGV.getPointer();
3197 Line = PLoc.getLine();
3198 Column = PLoc.getColumn();
3199 } else {
3200 Filename = llvm::Constant::getNullValue(Int8PtrTy);
3201 Line = Column = 0;
3202 }
3203
3204 llvm::Constant *Data[] = {Filename, Builder.getInt32(Line),
3205 Builder.getInt32(Column)};
3206
3207 return llvm::ConstantStruct::getAnon(Data);
3208}
3209
3210namespace {
3211/// Specify under what conditions this check can be recovered
3212enum class CheckRecoverableKind {
3213 /// Always terminate program execution if this check fails.
3215 /// Check supports recovering, runtime has both fatal (noreturn) and
3216 /// non-fatal handlers for this check.
3217 Recoverable,
3218 /// Runtime conditionally aborts, always need to support recovery.
3220};
3221}
3222
3223static CheckRecoverableKind getRecoverableKind(SanitizerMask Kind) {
3224 assert(Kind.countPopulation() == 1);
3225 if (Kind == SanitizerKind::Function || Kind == SanitizerKind::Vptr)
3226 return CheckRecoverableKind::AlwaysRecoverable;
3227 else if (Kind == SanitizerKind::Return || Kind == SanitizerKind::Unreachable)
3228 return CheckRecoverableKind::Unrecoverable;
3229 else
3230 return CheckRecoverableKind::Recoverable;
3231}
3232
3233namespace {
3234struct SanitizerHandlerInfo {
3235 char const *const Name;
3236 unsigned Version;
3237};
3238}
3239
3240const SanitizerHandlerInfo SanitizerHandlers[] = {
3241#define SANITIZER_CHECK(Enum, Name, Version) {#Name, Version},
3243#undef SANITIZER_CHECK
3244};
3245
3247 llvm::FunctionType *FnType,
3249 SanitizerHandler CheckHandler,
3250 CheckRecoverableKind RecoverKind, bool IsFatal,
3251 llvm::BasicBlock *ContBB) {
3252 assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
3253 std::optional<ApplyDebugLocation> DL;
3254 if (!CGF.Builder.getCurrentDebugLocation()) {
3255 // Ensure that the call has at least an artificial debug location.
3256 DL.emplace(CGF, SourceLocation());
3257 }
3258 bool NeedsAbortSuffix =
3259 IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;
3260 bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime;
3261 const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler];
3262 const StringRef CheckName = CheckInfo.Name;
3263 std::string FnName = "__ubsan_handle_" + CheckName.str();
3264 if (CheckInfo.Version && !MinimalRuntime)
3265 FnName += "_v" + llvm::utostr(CheckInfo.Version);
3266 if (MinimalRuntime)
3267 FnName += "_minimal";
3268 if (NeedsAbortSuffix)
3269 FnName += "_abort";
3270 bool MayReturn =
3271 !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;
3272
3273 llvm::AttrBuilder B(CGF.getLLVMContext());
3274 if (!MayReturn) {
3275 B.addAttribute(llvm::Attribute::NoReturn)
3276 .addAttribute(llvm::Attribute::NoUnwind);
3277 }
3278 B.addUWTableAttr(llvm::UWTableKind::Default);
3279
3280 llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
3281 FnType, FnName,
3282 llvm::AttributeList::get(CGF.getLLVMContext(),
3283 llvm::AttributeList::FunctionIndex, B),
3284 /*Local=*/true);
3285 llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(Fn, FnArgs);
3286 if (!MayReturn) {
3287 HandlerCall->setDoesNotReturn();
3288 CGF.Builder.CreateUnreachable();
3289 } else {
3290 CGF.Builder.CreateBr(ContBB);
3291 }
3292}
3293
3295 ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked,
3296 SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs,
3297 ArrayRef<llvm::Value *> DynamicArgs) {
3298 assert(IsSanitizerScope);
3299 assert(Checked.size() > 0);
3300 assert(CheckHandler >= 0 &&
3301 size_t(CheckHandler) < std::size(SanitizerHandlers));
3302 const StringRef CheckName = SanitizerHandlers[CheckHandler].Name;
3303
3304 llvm::Value *FatalCond = nullptr;
3305 llvm::Value *RecoverableCond = nullptr;
3306 llvm::Value *TrapCond = nullptr;
3307 for (int i = 0, n = Checked.size(); i < n; ++i) {
3308 llvm::Value *Check = Checked[i].first;
3309 // -fsanitize-trap= overrides -fsanitize-recover=.
3310 llvm::Value *&Cond =
3311 CGM.getCodeGenOpts().SanitizeTrap.has(Checked[i].second)
3312 ? TrapCond
3313 : CGM.getCodeGenOpts().SanitizeRecover.has(Checked[i].second)
3314 ? RecoverableCond
3315 : FatalCond;
3316 Cond = Cond ? Builder.CreateAnd(Cond, Check) : Check;
3317 }
3318
3319 if (TrapCond)
3320 EmitTrapCheck(TrapCond, CheckHandler);
3321 if (!FatalCond && !RecoverableCond)
3322 return;
3323
3324 llvm::Value *JointCond;
3325 if (FatalCond && RecoverableCond)
3326 JointCond = Builder.CreateAnd(FatalCond, RecoverableCond);
3327 else
3328 JointCond = FatalCond ? FatalCond : RecoverableCond;
3329 assert(JointCond);
3330
3331 CheckRecoverableKind RecoverKind = getRecoverableKind(Checked[0].second);
3332 assert(SanOpts.has(Checked[0].second));
3333#ifndef NDEBUG
3334 for (int i = 1, n = Checked.size(); i < n; ++i) {
3335 assert(RecoverKind == getRecoverableKind(Checked[i].second) &&
3336 "All recoverable kinds in a single check must be same!");
3337 assert(SanOpts.has(Checked[i].second));
3338 }
3339#endif
3340
3341 llvm::BasicBlock *Cont = createBasicBlock("cont");
3342 llvm::BasicBlock *Handlers = createBasicBlock("handler." + CheckName);
3343 llvm::Instruction *Branch = Builder.CreateCondBr(JointCond, Cont, Handlers);
3344 // Give hint that we very much don't expect to execute the handler
3345 // Value chosen to match UR_NONTAKEN_WEIGHT, see BranchProbabilityInfo.cpp
3346 llvm::MDBuilder MDHelper(getLLVMContext());
3347 llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
3348 Branch->setMetadata(llvm::LLVMContext::MD_prof, Node);
3349 EmitBlock(Handlers);
3350
3351 // Handler functions take an i8* pointing to the (handler-specific) static
3352 // information block, followed by a sequence of intptr_t arguments
3353 // representing operand values.
3356 if (!CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
3357 Args.reserve(DynamicArgs.size() + 1);
3358 ArgTypes.reserve(DynamicArgs.size() + 1);
3359
3360 // Emit handler arguments and create handler function type.
3361 if (!StaticArgs.empty()) {
3362 llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3363 auto *InfoPtr = new llvm::GlobalVariable(
3364 CGM.getModule(), Info->getType(), false,
3365 llvm::GlobalVariable::PrivateLinkage, Info, "", nullptr,
3366 llvm::GlobalVariable::NotThreadLocal,
3367 CGM.getDataLayout().getDefaultGlobalsAddressSpace());
3368 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3370 Args.push_back(EmitCastToVoidPtr(InfoPtr));
3371 ArgTypes.push_back(Args.back()->getType());
3372 }
3373
3374 for (size_t i = 0, n = DynamicArgs.size(); i != n; ++i) {
3375 Args.push_back(EmitCheckValue(DynamicArgs[i]));
3376 ArgTypes.push_back(IntPtrTy);
3377 }
3378 }
3379
3380 llvm::FunctionType *FnType =
3381 llvm::FunctionType::get(CGM.VoidTy, ArgTypes, false);
3382
3383 if (!FatalCond || !RecoverableCond) {
3384 // Simple case: we need to generate a single handler call, either
3385 // fatal, or non-fatal.
3386 emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind,
3387 (FatalCond != nullptr), Cont);
3388 } else {
3389 // Emit two handler calls: first one for set of unrecoverable checks,
3390 // another one for recoverable.
3391 llvm::BasicBlock *NonFatalHandlerBB =
3392 createBasicBlock("non_fatal." + CheckName);
3393 llvm::BasicBlock *FatalHandlerBB = createBasicBlock("fatal." + CheckName);
3394 Builder.CreateCondBr(FatalCond, NonFatalHandlerBB, FatalHandlerBB);
3395 EmitBlock(FatalHandlerBB);
3396 emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, true,
3397 NonFatalHandlerBB);
3398 EmitBlock(NonFatalHandlerBB);
3399 emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, false,
3400 Cont);
3401 }
3402
3403 EmitBlock(Cont);
3404}
3405
3407 SanitizerMask Kind, llvm::Value *Cond, llvm::ConstantInt *TypeId,
3408 llvm::Value *Ptr, ArrayRef<llvm::Constant *> StaticArgs) {
3409 llvm::BasicBlock *Cont = createBasicBlock("cfi.cont");
3410
3411 llvm::BasicBlock *CheckBB = createBasicBlock("cfi.slowpath");
3412 llvm::BranchInst *BI = Builder.CreateCondBr(Cond, Cont, CheckBB);
3413
3414 llvm::MDBuilder MDHelper(getLLVMContext());
3415 llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
3416 BI->setMetadata(llvm::LLVMContext::MD_prof, Node);
3417
3418 EmitBlock(CheckBB);
3419
3420 bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(Kind);
3421
3422 llvm::CallInst *CheckCall;
3423 llvm::FunctionCallee SlowPathFn;
3424 if (WithDiag) {
3425 llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3426 auto *InfoPtr =
3427 new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
3428 llvm::GlobalVariable::PrivateLinkage, Info);
3429 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3431
3432 SlowPathFn = CGM.getModule().getOrInsertFunction(
3433 "__cfi_slowpath_diag",
3434 llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy},
3435 false));
3436 CheckCall = Builder.CreateCall(
3437 SlowPathFn, {TypeId, Ptr, Builder.CreateBitCast(InfoPtr, Int8PtrTy)});
3438 } else {
3439 SlowPathFn = CGM.getModule().getOrInsertFunction(
3440 "__cfi_slowpath",
3441 llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy}, false));
3442 CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr});
3443 }
3444
3446 cast<llvm::GlobalValue>(SlowPathFn.getCallee()->stripPointerCasts()));
3447 CheckCall->setDoesNotThrow();
3448
3449 EmitBlock(Cont);
3450}
3451
3452// Emit a stub for __cfi_check function so that the linker knows about this
3453// symbol in LTO mode.
3455 llvm::Module *M = &CGM.getModule();
3456 auto &Ctx = M->getContext();
3457 llvm::Function *F = llvm::Function::Create(
3458 llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy}, false),
3459 llvm::GlobalValue::WeakAnyLinkage, "__cfi_check", M);
3460 CGM.setDSOLocal(F);
3461 llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx, "entry", F);
3462 // FIXME: consider emitting an intrinsic call like
3463 // call void @llvm.cfi_check(i64 %0, i8* %1, i8* %2)
3464 // which can be lowered in CrossDSOCFI pass to the actual contents of
3465 // __cfi_check. This would allow inlining of __cfi_check calls.
3466 llvm::CallInst::Create(
3467 llvm::Intrinsic::getDeclaration(M, llvm::Intrinsic::trap), "", BB);
3468 llvm::ReturnInst::Create(Ctx, nullptr, BB);
3469}
3470
3471// This function is basically a switch over the CFI failure kind, which is
3472// extracted from CFICheckFailData (1st function argument). Each case is either
3473// llvm.trap or a call to one of the two runtime handlers, based on
3474// -fsanitize-trap and -fsanitize-recover settings. Default case (invalid
3475// failure kind) traps, but this should really never happen. CFICheckFailData
3476// can be nullptr if the calling module has -fsanitize-trap behavior for this
3477// check kind; in this case __cfi_check_fail traps as well.
3479 SanitizerScope SanScope(this);
3480 FunctionArgList Args;
3485 Args.push_back(&ArgData);
3486 Args.push_back(&ArgAddr);
3487
3488 const CGFunctionInfo &FI =
3490
3491 llvm::Function *F = llvm::Function::Create(
3492 llvm::FunctionType::get(VoidTy, {VoidPtrTy, VoidPtrTy}, false),
3493 llvm::GlobalValue::WeakODRLinkage, "__cfi_check_fail", &CGM.getModule());
3494
3495 CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F, /*IsThunk=*/false);
3497 F->setVisibility(llvm::GlobalValue::HiddenVisibility);
3498
3499 StartFunction(GlobalDecl(), CGM.getContext().VoidTy, F, FI, Args,
3500 SourceLocation());
3501
3502 // This function is not affected by NoSanitizeList. This function does
3503 // not have a source location, but "src:*" would still apply. Revert any
3504 // changes to SanOpts made in StartFunction.
3506
3507 llvm::Value *Data =
3508 EmitLoadOfScalar(GetAddrOfLocalVar(&ArgData), /*Volatile=*/false,
3509 CGM.getContext().VoidPtrTy, ArgData.getLocation());
3510 llvm::Value *Addr =
3511 EmitLoadOfScalar(GetAddrOfLocalVar(&ArgAddr), /*Volatile=*/false,
3512 CGM.getContext().VoidPtrTy, ArgAddr.getLocation());
3513
3514 // Data == nullptr means the calling module has trap behaviour for this check.
3515 llvm::Value *DataIsNotNullPtr =
3516 Builder.CreateICmpNE(Data, llvm::ConstantPointerNull::get(Int8PtrTy));
3517 EmitTrapCheck(DataIsNotNullPtr, SanitizerHandler::CFICheckFail);
3518
3519 llvm::StructType *SourceLocationTy =
3520 llvm::StructType::get(VoidPtrTy, Int32Ty, Int32Ty);
3521 llvm::StructType *CfiCheckFailDataTy =
3522 llvm::StructType::get(Int8Ty, SourceLocationTy, VoidPtrTy);
3523
3524 llvm::Value *V = Builder.CreateConstGEP2_32(
3525 CfiCheckFailDataTy,
3526 Builder.CreatePointerCast(Data, CfiCheckFailDataTy->getPointerTo(0)), 0,
3527 0);
3528
3529 Address CheckKindAddr(V, Int8Ty, getIntAlign());
3530 llvm::Value *CheckKind = Builder.CreateLoad(CheckKindAddr);
3531
3532 llvm::Value *AllVtables = llvm::MetadataAsValue::get(
3534 llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
3535 llvm::Value *ValidVtable = Builder.CreateZExt(
3536 Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
3537 {Addr, AllVtables}),
3538 IntPtrTy);
3539
3540 const std::pair<int, SanitizerMask> CheckKinds[] = {
3541 {CFITCK_VCall, SanitizerKind::CFIVCall},
3542 {CFITCK_NVCall, SanitizerKind::CFINVCall},
3543 {CFITCK_DerivedCast, SanitizerKind::CFIDerivedCast},
3544 {CFITCK_UnrelatedCast, SanitizerKind::CFIUnrelatedCast},
3545 {CFITCK_ICall, SanitizerKind::CFIICall}};
3546
3548 for (auto CheckKindMaskPair : CheckKinds) {
3549 int Kind = CheckKindMaskPair.first;
3550 SanitizerMask Mask = CheckKindMaskPair.second;
3551 llvm::Value *Cond =
3552 Builder.CreateICmpNE(CheckKind, llvm::ConstantInt::get(Int8Ty, Kind));
3553 if (CGM.getLangOpts().Sanitize.has(Mask))
3554 EmitCheck(std::make_pair(Cond, Mask), SanitizerHandler::CFICheckFail, {},
3555 {Data, Addr, ValidVtable});
3556 else
3557 EmitTrapCheck(Cond, SanitizerHandler::CFICheckFail);
3558 }
3559
3561 // The only reference to this function will be created during LTO link.
3562 // Make sure it survives until then.
3563 CGM.addUsedGlobal(F);
3564}
3565
3567 if (SanOpts.has(SanitizerKind::Unreachable)) {
3568 SanitizerScope SanScope(this);
3569 EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()),
3570 SanitizerKind::Unreachable),
3571 SanitizerHandler::BuiltinUnreachable,
3572 EmitCheckSourceLocation(Loc), std::nullopt);
3573 }
3574 Builder.CreateUnreachable();
3575}
3576
3577void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked,
3578 SanitizerHandler CheckHandlerID) {
3579 llvm::BasicBlock *Cont = createBasicBlock("cont");
3580
3581 // If we're optimizing, collapse all calls to trap down to just one per
3582 // check-type per function to save on code size.
3583 if (TrapBBs.size() <= CheckHandlerID)
3584 TrapBBs.resize(CheckHandlerID + 1);
3585 llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
3586
3587 if (!CGM.getCodeGenOpts().OptimizationLevel || !TrapBB ||
3588 (CurCodeDecl && CurCodeDecl->hasAttr<OptimizeNoneAttr>())) {
3589 TrapBB = createBasicBlock("trap");
3590 Builder.CreateCondBr(Checked, Cont, TrapBB);
3591 EmitBlock(TrapBB);
3592
3593 llvm::CallInst *TrapCall =
3594 Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::ubsantrap),
3595 llvm::ConstantInt::get(CGM.Int8Ty, CheckHandlerID));
3596
3597 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
3598 auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",
3600 TrapCall->addFnAttr(A);
3601 }
3602 TrapCall->setDoesNotReturn();
3603 TrapCall->setDoesNotThrow();
3604 Builder.CreateUnreachable();
3605 } else {
3606 auto Call = TrapBB->begin();
3607 assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB");
3608
3609 Call->applyMergedLocation(Call->getDebugLoc(),
3610 Builder.getCurrentDebugLocation());
3611 Builder.CreateCondBr(Checked, Cont, TrapBB);
3612 }
3613
3614 EmitBlock(Cont);
3615}
3616
3617llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) {
3618 llvm::CallInst *TrapCall =
3619 Builder.CreateCall(CGM.getIntrinsic(IntrID));
3620
3621 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
3622 auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",
3624 TrapCall->addFnAttr(A);
3625 }
3626
3627 return TrapCall;
3628}
3629
3631 LValueBaseInfo *BaseInfo,
3632 TBAAAccessInfo *TBAAInfo) {
3633 assert(E->getType()->isArrayType() &&
3634 "Array to pointer decay must have array source type!");
3635
3636 // Expressions of array type can't be bitfields or vector elements.
3637 LValue LV = EmitLValue(E);
3638 Address Addr = LV.getAddress(*this);
3639
3640 // If the array type was an incomplete type, we need to make sure
3641 // the decay ends up being the right type.
3642 llvm::Type *NewTy = ConvertType(E->getType());
3643 Addr = Builder.CreateElementBitCast(Addr, NewTy);
3644
3645 // Note that VLA pointers are always decayed, so we don't need to do
3646 // anything here.
3647 if (!E->getType()->isVariableArrayType()) {
3648 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
3649 "Expected pointer to array");
3650 Addr = Builder.CreateConstArrayGEP(Addr, 0, "arraydecay");
3651 }
3652
3653 // The result of this decay conversion points to an array element within the
3654 // base lvalue. However, since TBAA currently does not support representing
3655 // accesses to elements of member arrays, we conservatively represent accesses
3656 // to the pointee object as if it had no any base lvalue specified.
3657 // TODO: Support TBAA for member arrays.
3659 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
3660 if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(EltType);
3661
3662 return Builder.CreateElementBitCast(Addr, ConvertTypeForMem(EltType));
3663}
3664
3665/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
3666/// array to pointer, return the array subexpression.
3667static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
3668 // If this isn't just an array->pointer decay, bail out.
3669 const auto *CE = dyn_cast<CastExpr>(E);
3670 if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)
3671 return nullptr;
3672
3673 // If this is a decay from variable width array, bail out.
3674 const Expr *SubExpr = CE->getSubExpr();
3675 if (SubExpr->getType()->isVariableArrayType())
3676 return nullptr;
3677
3678 return SubExpr;
3679}
3680
3682 llvm::Type *elemType,
3683 llvm::Value *ptr,
3684 ArrayRef<llvm::Value*> indices,
3685 bool inbounds,
3686 bool signedIndices,
3687 SourceLocation loc,
3688 const llvm::Twine &name = "arrayidx") {
3689 if (inbounds) {
3690 return CGF.EmitCheckedInBoundsGEP(elemType, ptr, indices, signedIndices,
3692 name);
3693 } else {
3694 return CGF.Builder.CreateGEP(elemType, ptr, indices, name);
3695 }
3696}
3697
3699 llvm::Value *idx,
3700 CharUnits eltSize) {
3701 // If we have a constant index, we can use the exact offset of the
3702 // element we're accessing.
3703 if (auto constantIdx = dyn_cast<llvm::ConstantInt>(idx)) {
3704 CharUnits offset = constantIdx->getZExtValue() * eltSize;
3705 return arrayAlign.alignmentAtOffset(offset);
3706
3707 // Otherwise, use the worst-case alignment for any element.
3708 } else {
3709 return arrayAlign.alignmentOfArrayElement(eltSize);
3710 }
3711}
3712
3714 const VariableArrayType *vla) {
3715 QualType eltType;
3716 do {
3717 eltType = vla->getElementType();
3718 } while ((vla = ctx.getAsVariableArrayType(eltType)));
3719 return eltType;
3720}
3721
3722/// Given an array base, check whether its member access belongs to a record
3723/// with preserve_access_index attribute or not.
3724static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) {
3725 if (!ArrayBase || !CGF.getDebugInfo())
3726 return false;
3727
3728 // Only support base as either a MemberExpr or DeclRefExpr.
3729 // DeclRefExpr to cover cases like:
3730 // struct s { int a; int b[10]; };
3731 // struct s *p;
3732 // p[1].a
3733 // p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr.
3734 // p->b[5] is a MemberExpr example.
3735 const Expr *E = ArrayBase->IgnoreImpCasts();
3736 if (const auto *ME = dyn_cast<MemberExpr>(E))
3737 return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
3738
3739 if (const auto *DRE = dyn_cast<DeclRefExpr>(E)) {
3740 const auto *VarDef = dyn_cast<VarDecl>(DRE->getDecl());
3741 if (!VarDef)
3742 return false;
3743
3744 const auto *PtrT = VarDef->getType()->getAs<PointerType>();
3745 if (!PtrT)
3746 return false;
3747
3748 const auto *PointeeT = PtrT->getPointeeType()
3750 if (const auto *RecT = dyn_cast<RecordType>(PointeeT))
3751 return RecT->getDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
3752 return false;
3753 }
3754
3755 return false;
3756}
3757
3760 QualType eltType, bool inbounds,
3761 bool signedIndices, SourceLocation loc,
3762 QualType *arrayType = nullptr,
3763 const Expr *Base = nullptr,
3764 const llvm::Twine &name = "arrayidx") {
3765 // All the indices except that last must be zero.
3766#ifndef NDEBUG
3767 for (auto *idx : indices.drop_back())
3768 assert(isa<llvm::ConstantInt>(idx) &&
3769 cast<llvm::ConstantInt>(idx)->isZero());
3770#endif
3771
3772 // Determine the element size of the statically-sized base. This is
3773 // the thing that the indices are expressed in terms of.
3774 if (auto vla = CGF.getContext().getAsVariableArrayType(eltType)) {
3775 eltType = getFixedSizeElementType(CGF.getContext(), vla);
3776 }
3777
3778 // We can use that to compute the best alignment of the element.
3779 CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType);
3780 CharUnits eltAlign =
3781 getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize);
3782
3783 llvm::Value *eltPtr;
3784 auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back());
3785 if (!LastIndex ||
3787 eltPtr = emitArraySubscriptGEP(
3788 CGF, addr.getElementType(), addr.getPointer(), indices, inbounds,
3789 signedIndices, loc, name);
3790 } else {
3791 // Remember the original array subscript for bpf target
3792 unsigned idx = LastIndex->getZExtValue();
3793 llvm::DIType *DbgInfo = nullptr;
3794 if (arrayType)
3795 DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc);
3796 eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getElementType(),
3797 addr.getPointer(),
3798 indices.size() - 1,
3799 idx, DbgInfo);
3800 }
3801
3802 return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign);
3803}
3804
3806 bool Accessed) {
3807 // The index must always be an integer, which is not an aggregate. Emit it
3808 // in lexical order (this complexity is, sadly, required by C++17).
3809 llvm::Value *IdxPre =
3810 (E->getLHS() == E->getIdx()) ? EmitScalarExpr(E->getIdx()) : nullptr;
3811 bool SignedIndices = false;
3812 auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * {
3813 auto *Idx = IdxPre;
3814 if (E->getLHS() != E->getIdx()) {
3815 assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS");
3816 Idx = EmitScalarExpr(E->getIdx());
3817 }
3818
3819 QualType IdxTy = E->getIdx()->getType();
3820 bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
3821 SignedIndices |= IdxSigned;
3822
3823 if (SanOpts.has(SanitizerKind::ArrayBounds))
3824 EmitBoundsCheck(E, E->getBase(), Idx, IdxTy, Accessed);
3825
3826 // Extend or truncate the index type to 32 or 64-bits.
3827 if (Promote && Idx->getType() != IntPtrTy)
3828 Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom");
3829
3830 return Idx;
3831 };
3832 IdxPre = nullptr;
3833
3834 // If the base is a vector type, then we are forming a vector element lvalue
3835 // with this subscript.
3836 if (E->getBase()->getType()->isVectorType() &&
3837 !isa<ExtVectorElementExpr>(E->getBase())) {
3838 // Emit the vector as an lvalue to get its address.
3839 LValue LHS = EmitLValue(E->getBase());
3840 auto *Idx = EmitIdxAfterBase(/*Promote*/false);
3841 assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
3842 return LValue::MakeVectorElt(LHS.getAddress(*this), Idx,
3843 E->getBase()->getType(), LHS.getBaseInfo(),
3844 TBAAAccessInfo());
3845 }
3846
3847 // All the other cases basically behave like simple offsetting.
3848
3849 // Handle the extvector case we ignored above.
3850 if (isa<ExtVectorElementExpr>(E->getBase())) {
3851 LValue LV = EmitLValue(E->getBase());
3852 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3854
3855 QualType EltType = LV.getType()->castAs<VectorType>()->getElementType();
3856 Addr = emitArraySubscriptGEP(*this, Addr, Idx, EltType, /*inbounds*/ true,
3857 SignedIndices, E->getExprLoc());
3858 return MakeAddrLValue(Addr, EltType, LV.getBaseInfo(),
3859 CGM.getTBAAInfoForSubobject(LV, EltType));
3860 }
3861
3862 LValueBaseInfo EltBaseInfo;
3863 TBAAAccessInfo EltTBAAInfo;
3864 Address Addr = Address::invalid();
3865 if (const VariableArrayType *vla =
3866 getContext().getAsVariableArrayType(E->getType())) {
3867 // The base must be a pointer, which is not an aggregate. Emit
3868 // it. It needs to be emitted first in case it's what captures
3869 // the VLA bounds.
3870 Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3871 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3872
3873 // The element count here is the total number of non-VLA elements.
3874 llvm::Value *numElements = getVLASize(vla).NumElts;
3875
3876 // Effectively, the multiply by the VLA size is part of the GEP.
3877 // GEP indexes are signed, and scaling an index isn't permitted to
3878 // signed-overflow, so we use the same semantics for our explicit
3879 // multiply. We suppress this if overflow is not undefined behavior.
3880 if (getLangOpts().isSignedOverflowDefined()) {
3881 Idx = Builder.CreateMul(Idx, numElements);
3882 } else {
3883 Idx = Builder.CreateNSWMul(Idx, numElements);
3884 }
3885
3886 Addr = emitArraySubscriptGEP(*this, Addr, Idx, vla->getElementType(),
3887 !getLangOpts().isSignedOverflowDefined(),
3888 SignedIndices, E->getExprLoc());
3889
3890 } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
3891 // Indexing over an interface, as in "NSString *P; P[4];"
3892
3893 // Emit the base pointer.
3894 Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3895 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3896
3897 CharUnits InterfaceSize = getContext().getTypeSizeInChars(OIT);
3898 llvm::Value *InterfaceSizeVal =
3899 llvm::ConstantInt::get(Idx->getType(), InterfaceSize.getQuantity());
3900
3901 llvm::Value *ScaledIdx = Builder.CreateMul(Idx, InterfaceSizeVal);
3902
3903 // We don't necessarily build correct LLVM struct types for ObjC
3904 // interfaces, so we can't rely on GEP to do this scaling
3905 // correctly, so we need to cast to i8*. FIXME: is this actually
3906 // true? A lot of other things in the fragile ABI would break...
3907 llvm::Type *OrigBaseElemTy = Addr.getElementType();
3908 Addr = Builder.CreateElementBitCast(Addr, Int8Ty);
3909
3910 // Do the GEP.
3911 CharUnits EltAlign =
3912 getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize);
3913 llvm::Value *EltPtr =
3914 emitArraySubscriptGEP(*this, Addr.getElementType(), Addr.getPointer(),
3915 ScaledIdx, false, SignedIndices, E->getExprLoc());
3916 Addr = Address(EltPtr, Addr.getElementType(), EltAlign);
3917
3918 // Cast back.
3919 Addr = Builder.CreateElementBitCast(Addr, OrigBaseElemTy);
3920 } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
3921 // If this is A[i] where A is an array, the frontend will have decayed the
3922 // base to be a ArrayToPointerDecay implicit cast. While correct, it is
3923 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
3924 // "gep x, i" here. Emit one "gep A, 0, i".
3925 assert(Array->getType()->isArrayType() &&
3926 "Array to pointer decay must have array source type!");
3927 LValue ArrayLV;
3928 // For simple multidimensional array indexing, set the 'accessed' flag for
3929 // better bounds-checking of the base expression.
3930 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
3931 ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);
3932 else
3933 ArrayLV = EmitLValue(Array);
3934 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3935
3936 // Propagate the alignment from the array itself to the result.
3937 QualType arrayType = Array->getType();
3938 Addr = emitArraySubscriptGEP(
3939 *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx},
3940 E->getType(), !getLangOpts().isSignedOverflowDefined(), SignedIndices,
3941 E->getExprLoc(), &arrayType, E->getBase());
3942 EltBaseInfo = ArrayLV.getBaseInfo();
3943 EltTBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, E->getType());
3944 } else {
3945 // The base must be a pointer; emit it with an estimate of its alignment.
3946 Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3947 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3948 QualType ptrType = E->getBase()->getType();
3949 Addr = emitArraySubscriptGEP(*this, Addr, Idx, E->getType(),
3950 !getLangOpts().isSignedOverflowDefined(),
3951 SignedIndices, E->getExprLoc(), &ptrType,
3952 E->getBase());
3953 }
3954
3955 LValue LV = MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo);
3956
3957 if (getLangOpts().ObjC &&
3958 getLangOpts().getGC() != LangOptions::NonGC) {
3961 }
3962 return LV;
3963}
3964
3966 assert(
3967 !E->isIncomplete() &&
3968 "incomplete matrix subscript expressions should be rejected during Sema");
3969 LValue Base = EmitLValue(E->getBase());
3970 llvm::Value *RowIdx = EmitScalarExpr(E->getRowIdx());
3971 llvm::Value *ColIdx = EmitScalarExpr(E->getColumnIdx());
3972 llvm::Value *NumRows = Builder.getIntN(
3973 RowIdx->getType()->getScalarSizeInBits(),
3975 llvm::Value *FinalIdx =
3976 Builder.CreateAdd(Builder.CreateMul(ColIdx, NumRows), RowIdx);
3977 return LValue::MakeMatrixElt(
3978 MaybeConvertMatrixAddress(Base.getAddress(*this), *this), FinalIdx,
3979 E->getBase()->getType(), Base.getBaseInfo(), TBAAAccessInfo());
3980}
3981
3983 LValueBaseInfo &BaseInfo,
3984 TBAAAccessInfo &TBAAInfo,
3985 QualType BaseTy, QualType ElTy,
3986 bool IsLowerBound) {
3987 LValue BaseLVal;
3988 if (auto *ASE = dyn_cast<OMPArraySectionExpr>(Base->IgnoreParenImpCasts())) {
3989 BaseLVal = CGF.EmitOMPArraySectionExpr(ASE, IsLowerBound);
3990 if (BaseTy->isArrayType()) {
3991 Address Addr = BaseLVal.getAddress(CGF);
3992 BaseInfo = BaseLVal.getBaseInfo();
3993
3994 // If the array type was an incomplete type, we need to make sure
3995 // the decay ends up being the right type.
3996 llvm::Type *NewTy = CGF.ConvertType(BaseTy);
3997 Addr = CGF.Builder.CreateElementBitCast(Addr, NewTy);
3998
3999 // Note that VLA pointers are always decayed, so we don't need to do
4000 // anything here.
4001 if (!BaseTy->isVariableArrayType()) {
4002 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
4003 "Expected pointer to array");
4004 Addr = CGF.Builder.CreateConstArrayGEP(Addr, 0, "arraydecay");
4005 }
4006
4007 return CGF.Builder.CreateElementBitCast(Addr,
4008 CGF.ConvertTypeForMem(ElTy));
4009 }
4010 LValueBaseInfo TypeBaseInfo;
4011 TBAAAccessInfo TypeTBAAInfo;
4012 CharUnits Align =
4013 CGF.CGM.getNaturalTypeAlignment(ElTy, &TypeBaseInfo, &TypeTBAAInfo);
4014 BaseInfo.mergeForCast(TypeBaseInfo);
4015 TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(TBAAInfo, TypeTBAAInfo);
4016 return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress(CGF)),
4017 CGF.ConvertTypeForMem(ElTy), Align);
4018 }
4019 return CGF.EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo);
4020}
4021
4023 bool IsLowerBound) {
4025 QualType ResultExprTy;
4026 if (auto *AT = getContext().getAsArrayType(BaseTy))
4027 ResultExprTy = AT->getElementType();
4028 else
4029 ResultExprTy = BaseTy->getPointeeType();
4030 llvm::Value *Idx = nullptr;
4031 if (IsLowerBound || E->getColonLocFirst().isInvalid()) {
4032 // Requesting lower bound or upper bound, but without provided length and
4033 // without ':' symbol for the default length -> length = 1.
4034 // Idx = LowerBound ?: 0;
4035 if (auto *LowerBound = E->getLowerBound()) {
4036 Idx = Builder.CreateIntCast(
4037 EmitScalarExpr(LowerBound), IntPtrTy,
4038 LowerBound->getType()->hasSignedIntegerRepresentation());
4039 } else
4040 Idx = llvm::ConstantInt::getNullValue(IntPtrTy);
4041 } else {
4042 // Try to emit length or lower bound as constant. If this is possible, 1
4043 // is subtracted from constant length or lower bound. Otherwise, emit LLVM
4044 // IR (LB + Len) - 1.
4045 auto &C = CGM.getContext();
4046 auto *Length = E->getLength();
4047 llvm::APSInt ConstLength;
4048 if (Length) {
4049 // Idx = LowerBound + Length - 1;
4050 if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(C)) {
4051 ConstLength = CL->zextOrTrunc(PointerWidthInBits);
4052 Length = nullptr;
4053 }
4054 auto *LowerBound = E->getLowerBound();
4055 llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);
4056 if (LowerBound) {
4057 if (std::optional<llvm::APSInt> LB =
4058 LowerBound->getIntegerConstantExpr(C)) {
4059 ConstLowerBound = LB->zextOrTrunc(PointerWidthInBits);
4060 LowerBound = nullptr;
4061 }
4062 }
4063 if (!Length)
4064 --ConstLength;
4065 else if (!LowerBound)
4066 --ConstLowerBound;
4067
4068 if (Length || LowerBound) {
4069 auto *LowerBoundVal =
4070 LowerBound
4071 ? Builder.CreateIntCast(
4072 EmitScalarExpr(LowerBound), IntPtrTy,
4073 LowerBound->getType()->hasSignedIntegerRepresentation())
4074 : llvm::ConstantInt::get(IntPtrTy, ConstLowerBound);
4075 auto *LengthVal =
4076 Length
4077 ? Builder.CreateIntCast(
4078 EmitScalarExpr(Length), IntPtrTy,
4079 Length->getType()->hasSignedIntegerRepresentation())
4080 : llvm::ConstantInt::get(IntPtrTy, ConstLength);
4081 Idx = Builder.CreateAdd(LowerBoundVal, LengthVal, "lb_add_len",
4082 /*HasNUW=*/false,
4083 !getLangOpts().isSignedOverflowDefined());
4084 if (Length && LowerBound) {
4085 Idx = Builder.CreateSub(
4086 Idx, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "idx_sub_1",
4087 /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
4088 }
4089 } else
4090 Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength + ConstLowerBound);
4091 } else {
4092 // Idx = ArraySize - 1;
4093 QualType ArrayTy = BaseTy->isPointerType()
4095 : BaseTy;
4096 if (auto *VAT = C.getAsVariableArrayType(ArrayTy)) {
4097 Length = VAT->getSizeExpr();
4098 if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(C)) {
4099 ConstLength = *L;
4100 Length = nullptr;
4101 }
4102 } else {
4103 auto *CAT = C.getAsConstantArrayType(ArrayTy);
4104 ConstLength = CAT->getSize();
4105 }
4106 if (Length) {
4107 auto *LengthVal = Builder.CreateIntCast(
4108 EmitScalarExpr(Length), IntPtrTy,
4109 Length->getType()->hasSignedIntegerRepresentation());
4110 Idx = Builder.CreateSub(
4111 LengthVal, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "len_sub_1",
4112 /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
4113 } else {
4114 ConstLength = ConstLength.zextOrTrunc(PointerWidthInBits);
4115 --ConstLength;
4116 Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength);
4117 }
4118 }
4119 }
4120 assert(Idx);
4121
4122 Address EltPtr = Address::invalid();
4123 LValueBaseInfo BaseInfo;
4124 TBAAAccessInfo TBAAInfo;
4125 if (auto *VLA = getContext().getAsVariableArrayType(ResultExprTy)) {
4126 // The base must be a pointer, which is not an aggregate. Emit
4127 // it. It needs to be emitted first in case it's what captures
4128 // the VLA bounds.
4129 Address Base =
4130 emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo,
4131 BaseTy, VLA->getElementType(), IsLowerBound);
4132 // The element count here is the total number of non-VLA elements.
4133 llvm::Value *NumElements = getVLASize(VLA).NumElts;
4134
4135 // Effectively, the multiply by the VLA size is part of the GEP.
4136 // GEP indexes are signed, and scaling an index isn't permitted to
4137 // signed-overflow, so we use the same semantics for our explicit
4138 // multiply. We suppress this if overflow is not undefined behavior.
4139 if (getLangOpts().isSignedOverflowDefined())
4140 Idx = Builder.CreateMul(Idx, NumElements);
4141 else
4142 Idx = Builder.CreateNSWMul(Idx, NumElements);
4143 EltPtr = emitArraySubscriptGEP(*this, Base, Idx, VLA->getElementType(),
4144 !getLangOpts().isSignedOverflowDefined(),
4145 /*signedIndices=*/false, E->getExprLoc());
4146 } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
4147 // If this is A[i] where A is an array, the frontend will have decayed the
4148 // base to be a ArrayToPointerDecay implicit cast. While correct, it is
4149 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
4150 // "gep x, i" here. Emit one "gep A, 0, i".
4151 assert(Array->getType()->isArrayType() &&
4152 "Array to pointer decay must have array source type!");
4153 LValue ArrayLV;
4154 // For simple multidimensional array indexing, set the 'accessed' flag for
4155 // better bounds-checking of the base expression.
4156 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
4157 ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);
4158 else
4159 ArrayLV = EmitLValue(Array);
4160
4161 // Propagate the alignment from the array itself to the result.
4162 EltPtr = emitArraySubscriptGEP(
4163 *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx},
4164 ResultExprTy, !getLangOpts().isSignedOverflowDefined(),
4165 /*signedIndices=*/false, E->getExprLoc());
4166 BaseInfo = ArrayLV.getBaseInfo();
4167 TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy);
4168 } else {
4169 Address Base = emitOMPArraySectionBase(*this, E->getBase(), BaseInfo,
4170 TBAAInfo, BaseTy, ResultExprTy,
4171 IsLowerBound);
4172 EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy,
4173 !getLangOpts().isSignedOverflowDefined(),
4174 /*signedIndices=*/false, E->getExprLoc());
4175 }
4176
4177 return MakeAddrLValue(EltPtr, ResultExprTy, BaseInfo, TBAAInfo);
4178}
4179
4182 // Emit the base vector as an l-value.
4183 LValue Base;
4184
4185 // ExtVectorElementExpr's base can either be a vector or pointer to vector.
4186 if (E->isArrow()) {
4187 // If it is a pointer to a vector, emit the address and form an lvalue with
4188 // it.
4189 LValueBaseInfo BaseInfo;
4190 TBAAAccessInfo TBAAInfo;
4191 Address Ptr = EmitPointerWithAlignment(E->getBase(), &BaseInfo, &TBAAInfo);
4192 const auto *PT = E->getBase()->getType()->castAs<PointerType>();
4193 Base = MakeAddrLValue(Ptr, PT->getPointeeType(), BaseInfo, TBAAInfo);
4194 Base.getQuals().removeObjCGCAttr();
4195 } else if (E->getBase()->isGLValue()) {
4196 // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
4197 // emit the base as an lvalue.
4198 assert(E->getBase()->getType()->isVectorType());
4199 Base = EmitLValue(E->getBase());
4200 } else {
4201 // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
4202 assert(E->getBase()->getType()->isVectorType() &&
4203 "Result must be a vector");
4204 llvm::Value *Vec = EmitScalarExpr(E->getBase());
4205
4206 // Store the vector to memory (because LValue wants an address).
4207 Address VecMem = CreateMemTemp(E->getBase()->getType());
4208 Builder.CreateStore(Vec, VecMem);
4209 Base = MakeAddrLValue(VecMem, E->getBase()->getType(),
4211 }
4212
4213 QualType type =
4214 E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
4215
4216 // Encode the element access list into a vector of unsigned indices.
4218 E->getEncodedElementAccess(Indices);
4219
4220 if (Base.isSimple()) {
4221 llvm::Constant *CV =
4222 llvm::ConstantDataVector::get(getLLVMContext(), Indices);
4223 return LValue::MakeExtVectorElt(Base.getAddress(*this), CV, type,
4224 Base.getBaseInfo(), TBAAAccessInfo());
4225 }
4226 assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
4227
4228 llvm::Constant *BaseElts = Base.getExtVectorElts();
4230
4231 for (unsigned i = 0, e = Indices.size(); i != e; ++i)
4232 CElts.push_back(BaseElts->getAggregateElement(Indices[i]));
4233 llvm::Constant *CV = llvm::ConstantVector::get(CElts);
4234 return LValue::MakeExtVectorElt(Base.getExtVectorAddress(), CV, type,
4235 Base.getBaseInfo(), TBAAAccessInfo());
4236}
4237
4239 if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, E)) {
4241 return EmitDeclRefLValue(DRE);
4242 }
4243
4244 Expr *BaseExpr = E->getBase();
4245 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
4246 LValue BaseLV;
4247 if (E->isArrow()) {
4248 LValueBaseInfo BaseInfo;
4249 TBAAAccessInfo TBAAInfo;
4250 Address Addr = EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo);
4251 QualType PtrTy = BaseExpr->getType()->getPointeeType();
4252 SanitizerSet SkippedChecks;
4253 bool IsBaseCXXThis = IsWrappedCXXThis(BaseExpr);
4254 if (IsBaseCXXThis)
4255 SkippedChecks.set(SanitizerKind::Alignment, true);
4256 if (IsBaseCXXThis || isa<DeclRefExpr>(BaseExpr))
4257 SkippedChecks.set(SanitizerKind::Null, true);
4259 /*Alignment=*/CharUnits::Zero(), SkippedChecks);
4260 BaseLV = MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo);
4261 } else
4262 BaseLV = EmitCheckedLValue(BaseExpr, TCK_MemberAccess);
4263
4264 NamedDecl *ND = E->getMemberDecl();
4265 if (auto *Field = dyn_cast<FieldDecl>(ND)) {
4266 LValue LV = EmitLValueForField(BaseLV, Field);
4268 if (getLangOpts().OpenMP) {
4269 // If the member was explicitly marked as nontemporal, mark it as
4270 // nontemporal. If the base lvalue is marked as nontemporal, mark access
4271 // to children as nontemporal too.
4272 if ((IsWrappedCXXThis(BaseExpr) &&
4274 BaseLV.isNontemporal())
4275 LV.setNontemporal(/*Value=*/true);
4276 }
4277 return LV;
4278 }
4279
4280 if (const auto *FD = dyn_cast<FunctionDecl>(ND))
4281 return EmitFunctionDeclLValue(*this, E, FD);
4282
4283 llvm_unreachable("Unhandled member declaration!");
4284}
4285
4286/// Given that we are currently emitting a lambda, emit an l-value for
4287/// one of its members.
4289 if (CurCodeDecl) {
4290 assert(cast<CXXMethodDecl>(CurCodeDecl)->getParent()->isLambda());
4291 assert(cast<CXXMethodDecl>(CurCodeDecl)->getParent() == Field->getParent());
4292 }
4293 QualType LambdaTagType =
4294 getContext().getTagDeclType(Field->getParent());
4295 LValue LambdaLV = MakeNaturalAlignAddrLValue(CXXABIThisValue, LambdaTagType);
4296 return EmitLValueForField(LambdaLV, Field);
4297}
4298
4299/// Get the field index in the debug info. The debug info structure/union
4300/// will ignore the unnamed bitfields.
4302 unsigned FieldIndex) {
4303 unsigned I = 0, Skipped = 0;
4304
4305 for (auto *F : Rec->getDefinition()->fields()) {
4306 if (I == FieldIndex)
4307 break;
4308 if (F->isUnnamedBitfield())
4309 Skipped++;
4310 I++;
4311 }
4312
4313 return FieldIndex - Skipped;
4314}
4315
4316/// Get the address of a zero-sized field within a record. The resulting
4317/// address doesn't necessarily have the right type.
4319 const FieldDecl *Field) {
4321 CGF.getContext().getFieldOffset(Field));
4322 if (Offset.isZero())
4323 return Base;
4326}
4327
4328/// Drill down to the storage of a field without walking into
4329/// reference types.
4330///
4331/// The resulting address doesn't necessarily have the right type.
4333 const FieldDecl *field) {
4334 if (field->isZeroSize(CGF.getContext()))
4335 return emitAddrOfZeroSizeField(CGF, base, field);
4336
4337 const RecordDecl *rec = field->getParent();
4338
4339 unsigned idx =
4340 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
4341
4342 return CGF.Builder.CreateStructGEP(base, idx, field->getName());
4343}
4344
4346 Address addr, const FieldDecl *field) {
4347 const RecordDecl *rec = field->getParent();
4348 llvm::DIType *DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(
4349 base.getType(), rec->getLocation());
4350
4351 unsigned idx =
4352 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
4353
4355 addr, idx, CGF.getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo);
4356}
4357
4358static bool hasAnyVptr(const QualType Type, const ASTContext &Context) {
4359 const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl();
4360 if (!RD)
4361 return false;
4362
4363 if (RD->isDynamicClass())
4364 return true;
4365
4366 for (const auto &Base : RD->bases())
4367 if (hasAnyVptr(Base.getType(), Context))
4368 return true;
4369
4370 for (const FieldDecl *Field : RD->fields())
4371 if (hasAnyVptr(Field->getType(), Context))
4372 return true;
4373
4374 return false;
4375}
4376
4378 const FieldDecl *field) {
4379 LValueBaseInfo BaseInfo = base.getBaseInfo();
4380
4381 if (field->isBitField()) {
4382 const CGRecordLayout &RL =
4384 const CGBitFieldInfo &Info = RL.getBitFieldInfo(field);
4385 const bool UseVolatile = isAAPCS(CGM.getTarget()) &&
4386 CGM.getCodeGenOpts().AAPCSBitfieldWidth &&
4387 Info.VolatileStorageSize != 0 &&
4388 field->getType()
4391 Address Addr = base.getAddress(*this);
4392 unsigned Idx = RL.getLLVMFieldNo(field);
4393 const RecordDecl *rec = field->getParent();
4394 if (!UseVolatile) {
4395 if (!IsInPreservedAIRegion &&
4396 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
4397 if (Idx != 0)
4398 // For structs, we GEP to the field that the record layout suggests.
4399 Addr = Builder.CreateStructGEP(Addr, Idx, field->getName());
4400 } else {
4401 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType(
4402 getContext().getRecordType(rec), rec->getLocation());
4404 Addr, Idx, getDebugInfoFIndex(rec, field->getFieldIndex()),
4405 DbgInfo);
4406 }
4407 }
4408 const unsigned SS =
4409 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
4410 // Get the access type.
4411 llvm::Type *FieldIntTy = llvm::Type::getIntNTy(getLLVMContext(), SS);
4412 if (Addr.getElementType() != FieldIntTy)
4413 Addr = Builder.CreateElementBitCast(Addr, FieldIntTy);
4414 if (UseVolatile) {
4415 const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity();
4416 if (VolatileOffset)
4417 Addr = Builder.CreateConstInBoundsGEP(Addr, VolatileOffset);
4418 }
4419
4420 QualType fieldType =
4421 field->getType().withCVRQualifiers(base.getVRQualifiers());
4422 // TODO: Support TBAA for bit fields.
4423 LValueBaseInfo FieldBaseInfo(BaseInfo.getAlignmentSource());
4424 return LValue::MakeBitfield(Addr, Info, fieldType, FieldBaseInfo,
4425 TBAAAccessInfo());
4426 }
4427
4428 // Fields of may-alias structures are may-alias themselves.
4429 // FIXME: this should get propagated down through anonymous structs
4430 // and unions.
4431 QualType FieldType = field->getType();
4432 const RecordDecl *rec = field->getParent();
4433 AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();
4434 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(BaseAlignSource));
4435 TBAAAccessInfo FieldTBAAInfo;
4436 if (base.getTBAAInfo().isMayAlias() ||
4437 rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {
4438 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
4439 } else if (rec->isUnion()) {
4440 // TODO: Support TBAA for unions.
4441 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
4442 } else {
4443 // If no base type been assigned for the base access, then try to generate
4444 // one for this base lvalue.
4445 FieldTBAAInfo = base.getTBAAInfo();
4446 if (!FieldTBAAInfo.BaseType) {
4447 FieldTBAAInfo.BaseType = CGM.getTBAABaseTypeInfo(base.getType());
4448 assert(!FieldTBAAInfo.Offset &&
4449 "Nonzero offset for an access with no base type!");
4450 }
4451
4452 // Adjust offset to be relative to the base type.
4453 const ASTRecordLayout &Layout =
4455 unsigned CharWidth = getContext().getCharWidth();
4456 if (FieldTBAAInfo.BaseType)
4457 FieldTBAAInfo.Offset +=
4458 Layout.getFieldOffset(field->getFieldIndex()) / CharWidth;
4459
4460 // Update the final access type and size.
4461 FieldTBAAInfo.AccessType = CGM.getTBAATypeInfo(FieldType);
4462 FieldTBAAInfo.Size =
4464 }
4465
4466 Address addr = base.getAddress(*this);
4467 if (auto *ClassDef = dyn_cast<CXXRecordDecl>(rec)) {
4468 if (CGM.getCodeGenOpts().StrictVTablePointers &&
4469 ClassDef->isDynamicClass()) {
4470 // Getting to any field of dynamic object requires stripping dynamic
4471 // information provided by invariant.group. This is because accessing
4472 // fields may leak the real address of dynamic object, which could result
4473 // in miscompilation when leaked pointer would be compared.
4474 auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer());
4475 addr = Address(stripped, addr.getElementType(), addr.getAlignment());
4476 }
4477 }
4478
4479 unsigned RecordCVR = base.getVRQualifiers();
4480 if (rec->isUnion()) {
4481 // For unions, there is no pointer adjustment.
4482 if (CGM.getCodeGenOpts().StrictVTablePointers &&
4483 hasAnyVptr(FieldType, getContext()))
4484 // Because unions can easily skip invariant.barriers, we need to add
4485 // a barrier every time CXXRecord field with vptr is referenced.
4487
4489 (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
4490 // Remember the original union field index
4491 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(base.getType(),
4492 rec->getLocation());
4493 addr = Address(
4494 Builder.CreatePreserveUnionAccessIndex(
4495 addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo),
4496 addr.getElementType(), addr.getAlignment());
4497 }
4498
4499 if (FieldType->isReferenceType())
4501 addr, CGM.getTypes().ConvertTypeForMem(FieldType), field->getName());
4502 } else {
4503 if (!IsInPreservedAIRegion &&
4504 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>()))
4505 // For structs, we GEP to the field that the record layout suggests.
4506 addr = emitAddrOfFieldStorage(*this, addr, field);
4507 else
4508 // Remember the original struct field index
4509 addr = emitPreserveStructAccess(*this, base, addr, field);
4510 }
4511
4512 // If this is a reference field, load the reference right now.
4513 if (FieldType->isReferenceType()) {
4514 LValue RefLVal =
4515 MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);
4516 if (RecordCVR & Qualifiers::Volatile)
4517 RefLVal.getQuals().addVolatile();
4518 addr = EmitLoadOfReference(RefLVal, &FieldBaseInfo, &FieldTBAAInfo);
4519
4520 // Qualifiers on the struct don't apply to the referencee.
4521 RecordCVR = 0;
4522 FieldType = FieldType->getPointeeType();
4523 }